kccachedb.h   kccachedb.h 
skipping to change at line 32 skipping to change at line 32
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h> #include <kcregex.h>
#include <kcdb.h> #include <kcdb.h>
#include <kcplantdb.h> #include <kcplantdb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation.
*/
namespace {
const int32_t CDBSLOTNUM = 16; ///< number of slot tables
const size_t CDBDEFBNUM = 1048583LL; ///< default bucket number
const size_t CDBZMAPBNUM = 32768; ///< mininum number of buckets to
use mmap
const uint32_t CDBKSIZMAX = 0xfffff; ///< maximum size of each key
const size_t CDBRECBUFSIZ = 48; ///< size of the record buffer
const size_t CDBOPAQUESIZ = 16; ///< size of the opaque buffer
const uint32_t CDBLOCKBUSYLOOP = 8192; ///< threshold of busy loop and sl
eep for locking
}
/**
* On-memory hash database with LRU deletion. * On-memory hash database with LRU deletion.
* @note This class is a concrete class to operate a hash database on memor y. This class can be * @note This class is a concrete class to operate a hash database on memor y. This class can be
* inherited but overwriting methods is forbidden. Before every database o peration, it is * inherited but overwriting methods is forbidden. Before every database o peration, it is
* necessary to call the CacheDB::open method in order to open a database f ile and connect the * necessary to call the CacheDB::open method in order to open a database f ile and connect the
* database object to it. To avoid data missing or corruption, it is impor tant to close every * database object to it. To avoid data missing or corruption, it is impor tant to close every
* database file by the CacheDB::close method when the database is no longe r in use. It is * database file by the CacheDB::close method when the database is no longe r in use. It is
* forbidden for multible database objects in a process to open the same da tabase at the same * forbidden for multible database objects in a process to open the same da tabase at the same
* time. It is forbidden to share a database object with child processes. * time. It is forbidden to share a database object with child processes.
*/ */
class CacheDB : public BasicDB { class CacheDB : public BasicDB {
friend class PlantDB<CacheDB, BasicDB::TYPEGRASS>; friend class PlantDB<CacheDB, BasicDB::TYPEGRASS>;
public: public:
class Cursor; class Cursor;
private: private:
struct Record; struct Record;
struct TranLog; struct TranLog;
struct Slot; struct Slot;
class Repeater; class Repeater;
class Setter; class Setter;
class Remover; class Remover;
class ScopedVisitor;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
/** An alias of list of transaction logs. */ /** An alias of list of transaction logs. */
typedef std::list<TranLog> TranLogList; typedef std::list<TranLog> TranLogList;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public BasicDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class CacheDB; friend class CacheDB;
skipping to change at line 119 skipping to change at line 107
return false; return false;
} }
if (writable && !(db_->omode_ & OWRITER)) { if (writable && !(db_->omode_ & OWRITER)) {
db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
if (sidx_ < 0 || !rec_) { if (sidx_ < 0 || !rec_) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
uint32_t rksiz = rec_->ksiz & CDBKSIZMAX; uint32_t rksiz = rec_->ksiz & KSIZMAX;
char* dbuf = (char*)rec_ + sizeof(*rec_); char* dbuf = (char*)rec_ + sizeof(*rec_);
const char* rvbuf = dbuf + rksiz; const char* rvbuf = dbuf + rksiz;
size_t rvsiz = rec_->vsiz; size_t rvsiz = rec_->vsiz;
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (db_->comp_) { if (db_->comp_) {
zbuf = db_->comp_->decompress(rvbuf, rvsiz, &zsiz); zbuf = db_->comp_->decompress(rvbuf, rvsiz, &zsiz);
if (zbuf) { if (zbuf) {
rvbuf = zbuf; rvbuf = zbuf;
rvsiz = zsiz; rvsiz = zsiz;
} }
} }
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vs iz); const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vs iz);
delete[] zbuf; delete[] zbuf;
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
uint64_t hash = db_->hash_record(dbuf, rksiz) / CDBSLOTNUM; uint64_t hash = db_->hash_record(dbuf, rksiz) / SLOTNUM;
Slot* slot = db_->slots_ + sidx_; Slot* slot = db_->slots_ + sidx_;
Repeater repeater(Visitor::REMOVE, 0); Repeater repeater(Visitor::REMOVE, 0);
db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, tr ue); db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, tr ue);
} else if (vbuf == Visitor::NOP) { } else if (vbuf == Visitor::NOP) {
if (step) step_impl(); if (step) step_impl();
} else { } else {
uint64_t hash = db_->hash_record(dbuf, rksiz) / CDBSLOTNUM; uint64_t hash = db_->hash_record(dbuf, rksiz) / SLOTNUM;
Slot* slot = db_->slots_ + sidx_; Slot* slot = db_->slots_ + sidx_;
Repeater repeater(vbuf, vsiz); Repeater repeater(vbuf, vsiz);
db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, tr ue); db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, tr ue);
if (step) step_impl(); if (step) step_impl();
} }
return true; return true;
} }
/** /**
* Jump the cursor to the first record for forward scan. * Jump the cursor to the first record for forward scan.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump() { bool jump() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
Slot* slot = db_->slots_ + i; Slot* slot = db_->slots_ + i;
if (slot->first) { if (slot->first) {
sidx_ = i; sidx_ = i;
rec_ = slot->first; rec_ = slot->first;
return true; return true;
} }
} }
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
sidx_ = -1; sidx_ = -1;
rec_ = NULL; rec_ = NULL;
skipping to change at line 188 skipping to change at line 176
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump(const char* kbuf, size_t ksiz) { bool jump(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (ksiz > CDBKSIZMAX) ksiz = CDBKSIZMAX; if (ksiz > KSIZMAX) ksiz = KSIZMAX;
uint64_t hash = db_->hash_record(kbuf, ksiz); uint64_t hash = db_->hash_record(kbuf, ksiz);
int32_t sidx = hash % CDBSLOTNUM; int32_t sidx = hash % SLOTNUM;
hash /= CDBSLOTNUM; hash /= SLOTNUM;
Slot* slot = db_->slots_ + sidx; Slot* slot = db_->slots_ + sidx;
size_t bidx = hash % slot->bnum; size_t bidx = hash % slot->bnum;
Record* rec = slot->buckets[bidx]; Record* rec = slot->buckets[bidx];
Record** entp = slot->buckets + bidx; Record** entp = slot->buckets + bidx;
uint32_t fhash = db_->fold_hash(hash) & ~CDBKSIZMAX; uint32_t fhash = db_->fold_hash(hash) & ~KSIZMAX;
while (rec) { while (rec) {
uint32_t rhash = rec->ksiz & ~CDBKSIZMAX; uint32_t rhash = rec->ksiz & ~KSIZMAX;
uint32_t rksiz = rec->ksiz & CDBKSIZMAX; uint32_t rksiz = rec->ksiz & KSIZMAX;
if (fhash > rhash) { if (fhash > rhash) {
entp = &rec->left; entp = &rec->left;
rec = rec->left; rec = rec->left;
} else if (fhash < rhash) { } else if (fhash < rhash) {
entp = &rec->right; entp = &rec->right;
rec = rec->right; rec = rec->right;
} else { } else {
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
int32_t kcmp = db_->compare_keys(kbuf, ksiz, dbuf, rksiz); int32_t kcmp = db_->compare_keys(kbuf, ksiz, dbuf, rksiz);
if (kcmp < 0) { if (kcmp < 0) {
skipping to change at line 327 skipping to change at line 315
} }
private: private:
/** /**
* Step the cursor to the next record. * Step the cursor to the next record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool step_impl() { bool step_impl() {
_assert_(true); _assert_(true);
rec_ = rec_->next; rec_ = rec_->next;
if (!rec_) { if (!rec_) {
for (int32_t i = sidx_ + 1; i < CDBSLOTNUM; i++) { for (int32_t i = sidx_ + 1; i < SLOTNUM; i++) {
Slot* slot = db_->slots_ + i; Slot* slot = db_->slots_ + i;
if (slot->first) { if (slot->first) {
sidx_ = i; sidx_ = i;
rec_ = slot->first; rec_ = slot->first;
return true; return true;
} }
} }
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
sidx_ = -1; sidx_ = -1;
rec_ = NULL; rec_ = NULL;
skipping to change at line 374 skipping to change at line 362
enum Flag { enum Flag {
FOPEN = 1 << 0, ///< dummy for compatibility FOPEN = 1 << 0, ///< dummy for compatibility
FFATAL = 1 << 1 ///< dummy for compatibility FFATAL = 1 << 1 ///< dummy for compatibility
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit CacheDB() : explicit CacheDB() :
mlock_(), flock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NU LL), mlock_(), flock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NU LL),
omode_(0), curs_(), path_(""), type_(TYPECACHE), omode_(0), curs_(), path_(""), type_(TYPECACHE),
opts_(0), bnum_(CDBDEFBNUM), capcnt_(-1), capsiz_(-1), opts_(0), bnum_(DEFBNUM), capcnt_(-1), capsiz_(-1),
opaque_(), embcomp_(ZLIBRAWCOMP), comp_(NULL), slots_(), tran_(false) { opaque_(), embcomp_(ZLIBRAWCOMP), comp_(NULL), slots_(), tran_(false) {
_assert_(true); _assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~CacheDB() { virtual ~CacheDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
if (!curs_.empty()) { if (!curs_.empty()) {
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->db_ = NULL; cur->db_ = NULL;
cit++; ++cit;
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 417 skipping to change at line 405
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
if (ksiz > CDBKSIZMAX) ksiz = CDBKSIZMAX; if (ksiz > KSIZMAX) ksiz = KSIZMAX;
uint64_t hash = hash_record(kbuf, ksiz); uint64_t hash = hash_record(kbuf, ksiz);
int32_t sidx = hash % CDBSLOTNUM; int32_t sidx = hash % SLOTNUM;
hash /= CDBSLOTNUM; hash /= SLOTNUM;
Slot* slot = slots_ + sidx; Slot* slot = slots_ + sidx;
slot->lock.lock(); slot->lock.lock();
accept_impl(slot, hash, kbuf, ksiz, visitor, comp_, false); accept_impl(slot, hash, kbuf, ksiz, visitor, comp_, false);
slot->lock.unlock(); slot->lock.unlock();
return true; return true;
} }
/** /**
* Accept a visitor to multiple records at once. * Accept a visitor to multiple records at once.
* @param keys specifies a string vector of the keys. * @param keys specifies a string vector of the keys.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The operations for specified records are performed atomically an d other threads * @note The operations for specified records are performed atomically an d other threads
* accessing the same records are blocked. To avoid deadlock, any explic it database operation * accessing the same records are blocked. To avoid deadlock, any explic it database operation
* must not be performed in this function. * must not be performed in this function.
*/ */
bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor,
bool writable = true) { bool writable = true) {
_assert_(visitor); _assert_(visitor);
size_t knum = keys.size();
if (knum < 1) return true;
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
size_t knum = keys.size();
if (knum < 1) return true;
struct RecordKey { struct RecordKey {
const char* kbuf; const char* kbuf;
size_t ksiz; size_t ksiz;
uint64_t hash; uint64_t hash;
int32_t sidx; int32_t sidx;
}; };
RecordKey* rkeys = new RecordKey[knum]; RecordKey* rkeys = new RecordKey[knum];
std::set<int32_t> sidxs; std::set<int32_t> sidxs;
for (size_t i = 0; i < knum; i++) { for (size_t i = 0; i < knum; i++) {
const std::string& key = keys[i]; const std::string& key = keys[i];
RecordKey* rkey = rkeys + i; RecordKey* rkey = rkeys + i;
rkey->kbuf = key.data(); rkey->kbuf = key.data();
rkey->ksiz = key.size(); rkey->ksiz = key.size();
if (rkey->ksiz > CDBKSIZMAX) rkey->ksiz = CDBKSIZMAX; if (rkey->ksiz > KSIZMAX) rkey->ksiz = KSIZMAX;
rkey->hash = hash_record(rkey->kbuf, rkey->ksiz); rkey->hash = hash_record(rkey->kbuf, rkey->ksiz);
rkey->sidx = rkey->hash % CDBSLOTNUM; rkey->sidx = rkey->hash % SLOTNUM;
sidxs.insert(rkey->sidx); sidxs.insert(rkey->sidx);
rkey->hash /= CDBSLOTNUM; rkey->hash /= SLOTNUM;
} }
std::set<int32_t>::iterator sit = sidxs.begin(); std::set<int32_t>::iterator sit = sidxs.begin();
std::set<int32_t>::iterator sitend = sidxs.end(); std::set<int32_t>::iterator sitend = sidxs.end();
while (sit != sitend) { while (sit != sitend) {
Slot* slot = slots_ + *sit; Slot* slot = slots_ + *sit;
slot->lock.lock(); slot->lock.lock();
sit++; ++sit;
} }
for (size_t i = 0; i < knum; i++) { for (size_t i = 0; i < knum; i++) {
RecordKey* rkey = rkeys + i; RecordKey* rkey = rkeys + i;
Slot* slot = slots_ + rkey->sidx; Slot* slot = slots_ + rkey->sidx;
accept_impl(slot, rkey->hash, rkey->kbuf, rkey->ksiz, visitor, comp_, false); accept_impl(slot, rkey->hash, rkey->kbuf, rkey->ksiz, visitor, comp_, false);
} }
sit = sidxs.begin(); sit = sidxs.begin();
sitend = sidxs.end(); sitend = sidxs.end();
while (sit != sitend) { while (sit != sitend) {
Slot* slot = slots_ + *sit; Slot* slot = slots_ + *sit;
slot->lock.unlock(); slot->lock.unlock();
sit++; ++sit;
} }
delete[] rkeys; delete[] rkeys;
return true; return true;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed. * @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 512 skipping to change at line 501
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
int64_t allcnt = count_impl(); int64_t allcnt = count_impl();
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
int64_t curcnt = 0; int64_t curcnt = 0;
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
Record* rec = slot->first; Record* rec = slot->first;
while (rec) { while (rec) {
Record* next = rec->next; Record* next = rec->next;
uint32_t rksiz = rec->ksiz & CDBKSIZMAX; uint32_t rksiz = rec->ksiz & KSIZMAX;
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
const char* rvbuf = dbuf + rksiz; const char* rvbuf = dbuf + rksiz;
size_t rvsiz = rec->vsiz; size_t rvsiz = rec->vsiz;
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (comp_) { if (comp_) {
zbuf = comp_->decompress(rvbuf, rvsiz, &zsiz); zbuf = comp_->decompress(rvbuf, rvsiz, &zsiz);
if (zbuf) { if (zbuf) {
rvbuf = zbuf; rvbuf = zbuf;
rvsiz = zsiz; rvsiz = zsiz;
} }
} }
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, & vsiz); const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, & vsiz);
delete[] zbuf; delete[] zbuf;
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
uint64_t hash = hash_record(dbuf, rksiz) / CDBSLOTNUM; uint64_t hash = hash_record(dbuf, rksiz) / SLOTNUM;
Repeater repeater(Visitor::REMOVE, 0); Repeater repeater(Visitor::REMOVE, 0);
accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, true); accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, true);
} else if (vbuf != Visitor::NOP) { } else if (vbuf != Visitor::NOP) {
uint64_t hash = hash_record(dbuf, rksiz) / CDBSLOTNUM; uint64_t hash = hash_record(dbuf, rksiz) / SLOTNUM;
Repeater repeater(vbuf, vsiz); Repeater repeater(vbuf, vsiz);
accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, true); accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, true);
} }
rec = next; rec = next;
curcnt++; curcnt++;
if (checker && !checker->check("iterate", "processing", curcnt, all cnt)) { if (checker && !checker->check("iterate", "processing", curcnt, all cnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
} }
skipping to change at line 619 skipping to change at line 609
bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", p ath.c_str()); report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", p ath.c_str());
omode_ = mode; omode_ = mode;
path_.append(path); path_.append(path);
size_t bnum = nearbyprime(bnum_ / CDBSLOTNUM); size_t bnum = nearbyprime(bnum_ / SLOTNUM);
size_t capcnt = capcnt_ > 0 ? capcnt_ / CDBSLOTNUM + 1 : (1ULL << (size size_t capcnt = capcnt_ > 0 ? capcnt_ / SLOTNUM + 1 : (1ULL << (sizeof(
of(capcnt) * 8 - 1)); capcnt) * 8 - 1));
size_t capsiz = capsiz_ > 0 ? capsiz_ / CDBSLOTNUM + 1 : (1ULL << (size size_t capsiz = capsiz_ > 0 ? capsiz_ / SLOTNUM + 1 : (1ULL << (sizeof(
of(capsiz) * 8 - 1)); capsiz) * 8 - 1));
if (capsiz > sizeof(*this) / CDBSLOTNUM) capsiz -= sizeof(*this) / CDBS if (capsiz > sizeof(*this) / SLOTNUM) capsiz -= sizeof(*this) / SLOTNUM
LOTNUM; ;
if (capsiz > bnum * sizeof(Record*)) capsiz -= bnum * sizeof(Record*); if (capsiz > bnum * sizeof(Record*)) capsiz -= bnum * sizeof(Record*);
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
initialize_slot(slots_ + i, bnum, capcnt, capsiz); initialize_slot(slots_ + i, bnum, capcnt, capsiz);
} }
comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL; comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL;
std::memset(opaque_, 0, sizeof(opaque_)); std::memset(opaque_, 0, sizeof(opaque_));
trigger_meta(MetaTrigger::OPEN, "open"); trigger_meta(MetaTrigger::OPEN, "open");
return true; return true;
} }
/** /**
* Close the database file. * Close the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool close() { bool close() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", p ath_.c_str()); report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", p ath_.c_str());
tran_ = false; tran_ = false;
for (int32_t i = CDBSLOTNUM - 1; i >= 0; i--) { for (int32_t i = SLOTNUM - 1; i >= 0; i--) {
destroy_slot(slots_ + i); destroy_slot(slots_ + i);
} }
path_.clear(); path_.clear();
omode_ = 0; omode_ = 0;
trigger_meta(MetaTrigger::CLOSE, "close"); trigger_meta(MetaTrigger::CLOSE, "close");
return true; return true;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
skipping to change at line 711 skipping to change at line 701
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wcnt >= CDBLOCKBUSYLOOP) { if (wcnt >= LOCKBUSYLOOP) {
Thread::chill(); Thread::chill();
} else { } else {
Thread::yield(); Thread::yield();
wcnt++; wcnt++;
} }
} }
tran_ = true; tran_ = true;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction");
mlock_.unlock(); mlock_.unlock();
return true; return true;
skipping to change at line 769 skipping to change at line 759
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!tran_) { if (!tran_) {
set_error(_KCCODELINE_, Error::INVALID, "not in transaction"); set_error(_KCCODELINE_, Error::INVALID, "not in transaction");
return false; return false;
} }
if (!commit) disable_cursors(); if (!commit) disable_cursors();
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
if (!commit) apply_slot_trlogs(slots_ + i); if (!commit) apply_slot_trlogs(slots_ + i);
slots_[i].trlogs.clear(); slots_[i].trlogs.clear();
adjust_slot_capacity(slots_ + i); adjust_slot_capacity(slots_ + i);
} }
tran_ = false; tran_ = false;
trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction"); trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction");
return true; return true;
} }
/** /**
* Remove all records. * Remove all records.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clear() { bool clear() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
disable_cursors(); disable_cursors();
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
clear_slot(slot); clear_slot(slot);
} }
std::memset(opaque_, 0, sizeof(opaque_)); std::memset(opaque_, 0, sizeof(opaque_));
trigger_meta(MetaTrigger::CLEAR, "clear"); trigger_meta(MetaTrigger::CLEAR, "clear");
return true; return true;
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
skipping to change at line 866 skipping to change at line 856
(*strmap)["opts"] = strprintf("%u", opts_); (*strmap)["opts"] = strprintf("%u", opts_);
(*strmap)["bnum"] = strprintf("%lld", (long long)bnum_); (*strmap)["bnum"] = strprintf("%lld", (long long)bnum_);
(*strmap)["capcnt"] = strprintf("%lld", (long long)capcnt_); (*strmap)["capcnt"] = strprintf("%lld", (long long)capcnt_);
(*strmap)["capsiz"] = strprintf("%lld", (long long)capsiz_); (*strmap)["capsiz"] = strprintf("%lld", (long long)capsiz_);
(*strmap)["recovered"] = strprintf("%d", false); (*strmap)["recovered"] = strprintf("%d", false);
(*strmap)["reorganized"] = strprintf("%d", false); (*strmap)["reorganized"] = strprintf("%d", false);
if (strmap->count("opaque") > 0) if (strmap->count("opaque") > 0)
(*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_)); (*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_));
if (strmap->count("bnum_used") > 0) { if (strmap->count("bnum_used") > 0) {
int64_t cnt = 0; int64_t cnt = 0;
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
Record** buckets = slot->buckets; Record** buckets = slot->buckets;
size_t bnum = slot->bnum; size_t bnum = slot->bnum;
for (size_t j = 0; j < bnum; j++) { for (size_t j = 0; j < bnum; j++) {
if (buckets[j]) cnt++; if (buckets[j]) cnt++;
} }
} }
(*strmap)["bnum_used"] = strprintf("%lld", (long long)cnt); (*strmap)["bnum_used"] = strprintf("%lld", (long long)cnt);
} }
(*strmap)["count"] = strprintf("%lld", (long long)count_impl()); (*strmap)["count"] = strprintf("%lld", (long long)count_impl());
skipping to change at line 951 skipping to change at line 941
* @param bnum the number of buckets of the hash table. * @param bnum the number of buckets of the hash table.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_buckets(int64_t bnum) { bool tune_buckets(int64_t bnum) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
bnum_ = bnum >= 0 ? bnum : CDBDEFBNUM; bnum_ = bnum >= 0 ? bnum : DEFBNUM;
return true; return true;
} }
/** /**
* Set the data compressor. * Set the data compressor.
* @param comp the data compressor object. * @param comp the data compressor object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_compressor(Compressor* comp) { bool tune_compressor(Compressor* comp) {
_assert_(comp); _assert_(comp);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
skipping to change at line 1236 skipping to change at line 1226
bool reorganized() { bool reorganized() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return false; return false;
} }
private: private:
/** The number of slot tables. */
static const int32_t SLOTNUM = 16;
/** The default bucket number. */
static const size_t DEFBNUM = 1048583LL;
/** The mininum number of buckets to use mmap. */
static const size_t ZMAPBNUM = 32768;
/** The maximum size of each key. */
static const uint32_t KSIZMAX = 0xfffff;
/** The size of the record buffer. */
static const size_t RECBUFSIZ = 48;
/** The size of the opaque buffer. */
static const size_t OPAQUESIZ = 16;
/** The threshold of busy loop and sleep for locking. */
static const uint32_t LOCKBUSYLOOP = 8192;
/** /**
* Set the power of the alignment of record size. * Set the power of the alignment of record size.
* @note This is a dummy implementation for compatibility. * @note This is a dummy implementation for compatibility.
*/ */
bool tune_alignment(int8_t apow) { bool tune_alignment(int8_t apow) {
return true; return true;
} }
/** /**
* Set the power of the capacity of the free block pool. * Set the power of the capacity of the free block pool.
* @note This is a dummy implementation for compatibility. * @note This is a dummy implementation for compatibility.
skipping to change at line 1405 skipping to change at line 1409
return vbuf_; return vbuf_;
} }
const char* vbuf_; ///< region of the value const char* vbuf_; ///< region of the value
size_t vsiz_; ///< size of the value size_t vsiz_; ///< size of the value
}; };
/** /**
* Removing visitor. * Removing visitor.
*/ */
class Remover : public Visitor { class Remover : public Visitor {
private: private:
/** constructor */ /** visit a record */
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp ); _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp );
return REMOVE; return REMOVE;
} }
}; };
/** /**
* Scoped visiotor.
*/
class ScopedVisitor {
public:
/** constructor */
ScopedVisitor(Visitor* visitor) : visitor_(visitor) {
_assert_(visitor);
visitor_->visit_before();
}
/** destructor */
~ScopedVisitor() {
_assert_(true);
visitor_->visit_after();
}
private:
Visitor* visitor_; ///< visitor
};
/**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param slot the slot of the record. * @param slot the slot of the record.
* @param hash the hash value of the key. * @param hash the hash value of the key.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param comp the data compressor. * @param comp the data compressor.
* @param isiter true for iterator use, or false for direct use. * @param isiter true for iterator use, or false for direct use.
*/ */
void accept_impl(Slot* slot, uint64_t hash, const char* kbuf, size_t ksiz , Visitor* visitor, void accept_impl(Slot* slot, uint64_t hash, const char* kbuf, size_t ksiz , Visitor* visitor,
Compressor* comp, bool isiter) { Compressor* comp, bool isiter) {
_assert_(slot && kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(slot && kbuf && ksiz <= MEMMAXSIZ && visitor);
size_t bidx = hash % slot->bnum; size_t bidx = hash % slot->bnum;
Record* rec = slot->buckets[bidx]; Record* rec = slot->buckets[bidx];
Record** entp = slot->buckets + bidx; Record** entp = slot->buckets + bidx;
uint32_t fhash = fold_hash(hash) & ~CDBKSIZMAX; uint32_t fhash = fold_hash(hash) & ~KSIZMAX;
while (rec) { while (rec) {
uint32_t rhash = rec->ksiz & ~CDBKSIZMAX; uint32_t rhash = rec->ksiz & ~KSIZMAX;
uint32_t rksiz = rec->ksiz & CDBKSIZMAX; uint32_t rksiz = rec->ksiz & KSIZMAX;
if (fhash > rhash) { if (fhash > rhash) {
entp = &rec->left; entp = &rec->left;
rec = rec->left; rec = rec->left;
} else if (fhash < rhash) { } else if (fhash < rhash) {
entp = &rec->right; entp = &rec->right;
rec = rec->right; rec = rec->right;
} else { } else {
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
int32_t kcmp = compare_keys(kbuf, ksiz, dbuf, rksiz); int32_t kcmp = compare_keys(kbuf, ksiz, dbuf, rksiz);
if (kcmp < 0) { if (kcmp < 0) {
skipping to change at line 1595 skipping to change at line 1617
delete[] zbuf; delete[] zbuf;
} }
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
*/ */
int64_t count_impl() { int64_t count_impl() {
_assert_(true); _assert_(true);
int64_t sum = 0; int64_t sum = 0;
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
ScopedSpinLock lock(&slot->lock); ScopedSpinLock lock(&slot->lock);
sum += slot->count; sum += slot->count;
} }
return sum; return sum;
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes. * @return the size of the database file in bytes.
*/ */
int64_t size_impl() { int64_t size_impl() {
_assert_(true); _assert_(true);
int64_t sum = sizeof(*this); int64_t sum = sizeof(*this);
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
ScopedSpinLock lock(&slot->lock); ScopedSpinLock lock(&slot->lock);
sum += slot->bnum * sizeof(Record*); sum += slot->bnum * sizeof(Record*);
sum += slot->size; sum += slot->size;
} }
return sum; return sum;
} }
/** /**
* Initialize a slot table. * Initialize a slot table.
* @param slot the slot table. * @param slot the slot table.
* @param bnum the number of buckets. * @param bnum the number of buckets.
* @param capcnt the capacity of record number. * @param capcnt the capacity of record number.
* @param capsiz the capacity of memory usage. * @param capsiz the capacity of memory usage.
*/ */
void initialize_slot(Slot* slot, size_t bnum, size_t capcnt, size_t capsi z) { void initialize_slot(Slot* slot, size_t bnum, size_t capcnt, size_t capsi z) {
_assert_(slot); _assert_(slot);
Record** buckets; Record** buckets;
if (bnum >= CDBZMAPBNUM) { if (bnum >= ZMAPBNUM) {
buckets = (Record**)mapalloc(sizeof(*buckets) * bnum); buckets = (Record**)mapalloc(sizeof(*buckets) * bnum);
} else { } else {
buckets = new Record*[bnum]; buckets = new Record*[bnum];
for (size_t i = 0; i < bnum; i++) { for (size_t i = 0; i < bnum; i++) {
buckets[i] = NULL; buckets[i] = NULL;
} }
} }
slot->buckets = buckets; slot->buckets = buckets;
slot->bnum = bnum; slot->bnum = bnum;
slot->capcnt = capcnt; slot->capcnt = capcnt;
skipping to change at line 1657 skipping to change at line 1679
*/ */
void destroy_slot(Slot* slot) { void destroy_slot(Slot* slot) {
_assert_(slot); _assert_(slot);
slot->trlogs.clear(); slot->trlogs.clear();
Record* rec = slot->last; Record* rec = slot->last;
while (rec) { while (rec) {
Record* prev = rec->prev; Record* prev = rec->prev;
xfree(rec); xfree(rec);
rec = prev; rec = prev;
} }
if (slot->bnum >= CDBZMAPBNUM) { if (slot->bnum >= ZMAPBNUM) {
mapfree(slot->buckets); mapfree(slot->buckets);
} else { } else {
delete[] slot->buckets; delete[] slot->buckets;
} }
} }
/** /**
* Clear a slot table. * Clear a slot table.
* @param slot the slot table. * @param slot the slot table.
*/ */
void clear_slot(Slot* slot) { void clear_slot(Slot* slot) {
_assert_(slot); _assert_(slot);
Record* rec = slot->last; Record* rec = slot->last;
while (rec) { while (rec) {
if (tran_) { if (tran_) {
uint32_t rksiz = rec->ksiz & CDBKSIZMAX; uint32_t rksiz = rec->ksiz & KSIZMAX;
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
TranLog log(dbuf, rksiz, dbuf + rksiz, rec->vsiz); TranLog log(dbuf, rksiz, dbuf + rksiz, rec->vsiz);
slot->trlogs.push_back(log); slot->trlogs.push_back(log);
} }
Record* prev = rec->prev; Record* prev = rec->prev;
xfree(rec); xfree(rec);
rec = prev; rec = prev;
} }
Record** buckets = slot->buckets; Record** buckets = slot->buckets;
size_t bnum = slot->bnum; size_t bnum = slot->bnum;
skipping to change at line 1701 skipping to change at line 1723
/** /**
* Apply transaction logs of a slot table. * Apply transaction logs of a slot table.
* @param slot the slot table. * @param slot the slot table.
*/ */
void apply_slot_trlogs(Slot* slot) { void apply_slot_trlogs(Slot* slot) {
_assert_(slot); _assert_(slot);
const TranLogList& logs = slot->trlogs; const TranLogList& logs = slot->trlogs;
TranLogList::const_iterator it = logs.end(); TranLogList::const_iterator it = logs.end();
TranLogList::const_iterator itbeg = logs.begin(); TranLogList::const_iterator itbeg = logs.begin();
while (it != itbeg) { while (it != itbeg) {
it--; --it;
const char* kbuf = it->key.c_str(); const char* kbuf = it->key.c_str();
size_t ksiz = it->key.size(); size_t ksiz = it->key.size();
const char* vbuf = it->value.c_str(); const char* vbuf = it->value.c_str();
size_t vsiz = it->value.size(); size_t vsiz = it->value.size();
uint64_t hash = hash_record(kbuf, ksiz) / CDBSLOTNUM; uint64_t hash = hash_record(kbuf, ksiz) / SLOTNUM;
if (it->full) { if (it->full) {
Setter setter(vbuf, vsiz); Setter setter(vbuf, vsiz);
accept_impl(slot, hash, kbuf, ksiz, &setter, NULL, true); accept_impl(slot, hash, kbuf, ksiz, &setter, NULL, true);
} else { } else {
Remover remover; Remover remover;
accept_impl(slot, hash, kbuf, ksiz, &remover, NULL, true); accept_impl(slot, hash, kbuf, ksiz, &remover, NULL, true);
} }
} }
} }
/** /**
* Addjust a slot table to the capacity. * Addjust a slot table to the capacity.
* @param slot the slot table. * @param slot the slot table.
*/ */
void adjust_slot_capacity(Slot* slot) { void adjust_slot_capacity(Slot* slot) {
_assert_(slot); _assert_(slot);
if ((slot->count > slot->capcnt || slot->size > slot->capsiz) && slot-> first) { if ((slot->count > slot->capcnt || slot->size > slot->capsiz) && slot-> first) {
Record* rec = slot->first; Record* rec = slot->first;
uint32_t rksiz = rec->ksiz & CDBKSIZMAX; uint32_t rksiz = rec->ksiz & KSIZMAX;
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
char stack[CDBRECBUFSIZ]; char stack[RECBUFSIZ];
char* kbuf = rksiz > sizeof(stack) ? new char[rksiz] : stack; char* kbuf = rksiz > sizeof(stack) ? new char[rksiz] : stack;
std::memcpy(kbuf, dbuf, rksiz); std::memcpy(kbuf, dbuf, rksiz);
uint64_t hash = hash_record(kbuf, rksiz) / CDBSLOTNUM; uint64_t hash = hash_record(kbuf, rksiz) / SLOTNUM;
Remover remover; Remover remover;
accept_impl(slot, hash, dbuf, rksiz, &remover, NULL, true); accept_impl(slot, hash, dbuf, rksiz, &remover, NULL, true);
if (kbuf != stack) delete[] kbuf; if (kbuf != stack) delete[] kbuf;
} }
} }
/** /**
* Get the hash value of a record. * Get the hash value of a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return the hash value. * @return the hash value.
skipping to change at line 1782 skipping to change at line 1804
*/ */
void escape_cursors(Record* rec) { void escape_cursors(Record* rec) {
_assert_(rec); _assert_(rec);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
if (curs_.empty()) return; if (curs_.empty()) return;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->rec_ == rec) cur->step_impl(); if (cur->rec_ == rec) cur->step_impl();
cit++; ++cit;
} }
} }
/** /**
* Adjust cursors on re-allocated records. * Adjust cursors on re-allocated records.
* @param orec the old address. * @param orec the old address.
* @param nrec the new address. * @param nrec the new address.
*/ */
void adjust_cursors(Record* orec, Record* nrec) { void adjust_cursors(Record* orec, Record* nrec) {
_assert_(orec && nrec); _assert_(orec && nrec);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
if (curs_.empty()) return; if (curs_.empty()) return;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->rec_ == orec) cur->rec_ = nrec; if (cur->rec_ == orec) cur->rec_ = nrec;
cit++; ++cit;
} }
} }
/** /**
* Disable all cursors. * Disable all cursors.
*/ */
void disable_cursors() { void disable_cursors() {
_assert_(true); _assert_(true);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->sidx_ = -1; cur->sidx_ = -1;
cur->rec_ = NULL; cur->rec_ = NULL;
cit++; ++cit;
} }
} }
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
CacheDB(const CacheDB&); CacheDB(const CacheDB&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
CacheDB& operator =(const CacheDB&); CacheDB& operator =(const CacheDB&);
/** The method lock. */ /** The method lock. */
SpinRWLock mlock_; SpinRWLock mlock_;
/** The file lock. */ /** The file lock. */
SpinLock flock_; SpinLock flock_;
skipping to change at line 1850 skipping to change at line 1872
uint8_t type_; uint8_t type_;
/** The options. */ /** The options. */
uint8_t opts_; uint8_t opts_;
/** The bucket number. */ /** The bucket number. */
int64_t bnum_; int64_t bnum_;
/** The capacity of record number. */ /** The capacity of record number. */
int64_t capcnt_; int64_t capcnt_;
/** The capacity of memory usage. */ /** The capacity of memory usage. */
int64_t capsiz_; int64_t capsiz_;
/** The opaque data. */ /** The opaque data. */
char opaque_[CDBOPAQUESIZ]; char opaque_[OPAQUESIZ];
/** The embedded data compressor. */ /** The embedded data compressor. */
Compressor* embcomp_; Compressor* embcomp_;
/** The data compressor. */ /** The data compressor. */
Compressor* comp_; Compressor* comp_;
/** The slot tables. */ /** The slot tables. */
Slot slots_[CDBSLOTNUM]; Slot slots_[SLOTNUM];
/** The flag whether in transaction. */ /** The flag whether in transaction. */
bool tran_; bool tran_;
}; };
/** An alias of the cache tree database. */ /** An alias of the cache tree database. */
typedef PlantDB<CacheDB, BasicDB::TYPEGRASS> GrassDB; typedef PlantDB<CacheDB, BasicDB::TYPEGRASS> GrassDB;
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
 End of changes. 55 change blocks. 
75 lines changed or deleted 95 lines changed or added


 kcdb.h   kcdb.h 
skipping to change at line 26 skipping to change at line 26
#define _KCDB_H #define _KCDB_H
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
namespace kyotocabinet { // common namespace #define KCDBSSMAGICDATA "KCSS\n" ///< The magic data of the snapsho t file
/** namespace kyotocabinet { // common namespace
* Constants for implementation.
*/
namespace {
const char DBSSMAGICDATA[] = "KCSS\n"; ///< magic data of the snapshot fi
le
const size_t DBIOBUFSIZ = 8192; ///< size of the IO buffer
}
/** /**
* Interface of database abstraction. * Interface of database abstraction.
* @note This class is an abstract class to prescribe the interface of reco rd access. * @note This class is an abstract class to prescribe the interface of reco rd access.
*/ */
class DB { class DB {
public: public:
/** /**
* Interface to access a record. * Interface to access a record.
*/ */
skipping to change at line 86 skipping to change at line 80
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param sp the pointer to the variable into which the size of the reg ion of the return * @param sp the pointer to the variable into which the size of the reg ion of the return
* value is assigned. * value is assigned.
* @return If it is the pointer to a region, the value is replaced by t he content. If it * @return If it is the pointer to a region, the value is replaced by t he content. If it
* is Visitor::NOP or Visitor::REMOVE, nothing is modified. * is Visitor::NOP or Visitor::REMOVE, nothing is modified.
*/ */
virtual const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { virtual const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && sp); _assert_(kbuf && ksiz <= MEMMAXSIZ && sp);
return NOP; return NOP;
} }
/**
* Preprocess the main operations.
*/
virtual void visit_before() {
_assert_(true);
}
/**
* Postprocess the main operations.
*/
virtual void visit_after() {
_assert_(true);
}
}; };
/** /**
* Interface of cursor to indicate a record. * Interface of cursor to indicate a record.
*/ */
class Cursor { class Cursor {
public: public:
/** /**
* Destructor. * Destructor.
*/ */
virtual ~Cursor() { virtual ~Cursor() {
skipping to change at line 1196 skipping to change at line 1202
std::ios_base::out | std::ios_base::binary | std::ios_base ::trunc); std::ios_base::out | std::ios_base::binary | std::ios_base ::trunc);
if (!ofs) return false; if (!ofs) return false;
bool err = false; bool err = false;
std::ifstream ifs; std::ifstream ifs;
ifs.open(path.c_str(), std::ios_base::in | std::ios_base::binary); ifs.open(path.c_str(), std::ios_base::in | std::ios_base::binary);
if (checker_ && !checker_->check("copy", "beginning", 0, size)) { if (checker_ && !checker_->check("copy", "beginning", 0, size)) {
db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true; err = true;
} }
if (ifs) { if (ifs) {
char buf[DBIOBUFSIZ]; char buf[IOBUFSIZ];
int64_t curcnt = 0; int64_t curcnt = 0;
while (!err && !ifs.eof()) { while (!err && !ifs.eof()) {
size_t n = ifs.read(buf, sizeof(buf)).gcount(); size_t n = ifs.read(buf, sizeof(buf)).gcount();
if (n > 0) { if (n > 0) {
ofs.write(buf, n); ofs.write(buf, n);
if (!ofs) { if (!ofs) {
err = true; err = true;
break; break;
} }
} }
skipping to change at line 1844 skipping to change at line 1850
*/ */
int64_t set_bulk(const std::map<std::string, std::string>& recs, bool ato mic = true) { int64_t set_bulk(const std::map<std::string, std::string>& recs, bool ato mic = true) {
_assert_(true); _assert_(true);
if (atomic) { if (atomic) {
std::vector<std::string> keys; std::vector<std::string> keys;
keys.reserve(recs.size()); keys.reserve(recs.size());
std::map<std::string, std::string>::const_iterator rit = recs.begin() ; std::map<std::string, std::string>::const_iterator rit = recs.begin() ;
std::map<std::string, std::string>::const_iterator ritend = recs.end( ); std::map<std::string, std::string>::const_iterator ritend = recs.end( );
while (rit != ritend) { while (rit != ritend) {
keys.push_back(rit->first); keys.push_back(rit->first);
rit++; ++rit;
} }
class VisitorImpl : public Visitor { class VisitorImpl : public Visitor {
public: public:
explicit VisitorImpl(const std::map<std::string, std::string>& recs ) : recs_(recs) {} explicit VisitorImpl(const std::map<std::string, std::string>& recs ) : recs_(recs) {}
private: private:
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
std::map<std::string, std::string>::const_iterator rit = std::map<std::string, std::string>::const_iterator rit =
recs_.find(std::string(kbuf, ksiz)); recs_.find(std::string(kbuf, ksiz));
if (rit == recs_.end()) return NOP; if (rit == recs_.end()) return NOP;
skipping to change at line 1876 skipping to change at line 1882
}; };
VisitorImpl visitor(recs); VisitorImpl visitor(recs);
if (!accept_bulk(keys, &visitor, true)) return -1; if (!accept_bulk(keys, &visitor, true)) return -1;
return keys.size(); return keys.size();
} }
std::map<std::string, std::string>::const_iterator rit = recs.begin(); std::map<std::string, std::string>::const_iterator rit = recs.begin();
std::map<std::string, std::string>::const_iterator ritend = recs.end(); std::map<std::string, std::string>::const_iterator ritend = recs.end();
while (rit != ritend) { while (rit != ritend) {
if (!set(rit->first.data(), rit->first.size(), rit->second.data(), ri t->second.size())) if (!set(rit->first.data(), rit->first.size(), rit->second.data(), ri t->second.size()))
return -1; return -1;
rit++; ++rit;
} }
return recs.size(); return recs.size();
} }
/** /**
* Remove records at once. * Remove records at once.
* @param keys the keys of the records to remove. * @param keys the keys of the records to remove.
* @param atomic true to perform all operations atomically, or false for non-atomic operations. * @param atomic true to perform all operations atomically, or false for non-atomic operations.
* @return the number of removed records, or -1 on failure. * @return the number of removed records, or -1 on failure.
*/ */
int64_t remove_bulk(const std::vector<std::string>& keys, bool atomic = t rue) { int64_t remove_bulk(const std::vector<std::string>& keys, bool atomic = t rue) {
skipping to change at line 1916 skipping to change at line 1922
} }
int64_t cnt = 0; int64_t cnt = 0;
std::vector<std::string>::const_iterator kit = keys.begin(); std::vector<std::string>::const_iterator kit = keys.begin();
std::vector<std::string>::const_iterator kitend = keys.end(); std::vector<std::string>::const_iterator kitend = keys.end();
while (kit != kitend) { while (kit != kitend) {
if (remove(kit->data(), kit->size())) { if (remove(kit->data(), kit->size())) {
cnt++; cnt++;
} else if (error() != Error::NOREC) { } else if (error() != Error::NOREC) {
return -1; return -1;
} }
kit++; ++kit;
} }
return cnt; return cnt;
} }
/** /**
* Retrieve records at once. * Retrieve records at once.
* @param keys the keys of the records to retrieve. * @param keys the keys of the records to retrieve.
* @param recs a string map to contain the retrieved records. * @param recs a string map to contain the retrieved records.
* @param atomic true to perform all operations atomically, or false for non-atomic operations. * @param atomic true to perform all operations atomically, or false for non-atomic operations.
* @return the number of retrieved records, or -1 on failure. * @return the number of retrieved records, or -1 on failure.
*/ */
skipping to change at line 1957 skipping to change at line 1963
std::vector<std::string>::const_iterator kitend = keys.end(); std::vector<std::string>::const_iterator kitend = keys.end();
while (kit != kitend) { while (kit != kitend) {
size_t vsiz; size_t vsiz;
const char* vbuf = get(kit->data(), kit->size(), &vsiz); const char* vbuf = get(kit->data(), kit->size(), &vsiz);
if (vbuf) { if (vbuf) {
(*recs)[*kit] = std::string(vbuf, vsiz); (*recs)[*kit] = std::string(vbuf, vsiz);
delete[] vbuf; delete[] vbuf;
} else if (error() != Error::NOREC) { } else if (error() != Error::NOREC) {
return -1; return -1;
} }
kit++; ++kit;
} }
return recs->size(); return recs->size();
} }
/** /**
* Dump records into a data stream. * Dump records into a data stream.
* @param dest the destination stream. * @param dest the destination stream.
* @param checker a progress checker object. If it is NULL, no checking is performed. * @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_snapshot(std::ostream* dest, ProgressChecker* checker = NULL) { bool dump_snapshot(std::ostream* dest, ProgressChecker* checker = NULL) {
skipping to change at line 1993 skipping to change at line 1999
dest_->write(stack_, wp - stack_); dest_->write(stack_, wp - stack_);
dest_->write(kbuf, ksiz); dest_->write(kbuf, ksiz);
dest_->write(vbuf, vsiz); dest_->write(vbuf, vsiz);
return NOP; return NOP;
} }
std::ostream* dest_; std::ostream* dest_;
char stack_[NUMBUFSIZ*2]; char stack_[NUMBUFSIZ*2];
}; };
VisitorImpl visitor(dest); VisitorImpl visitor(dest);
bool err = false; bool err = false;
dest->write(DBSSMAGICDATA, sizeof(DBSSMAGICDATA)); dest->write(KCDBSSMAGICDATA, sizeof(KCDBSSMAGICDATA));
if (iterate(&visitor, false, checker)) { if (iterate(&visitor, false, checker)) {
unsigned char c = 0xff; unsigned char c = 0xff;
dest->write((char*)&c, 1); dest->write((char*)&c, 1);
if (dest->fail()) { if (dest->fail()) {
set_error(_KCCODELINE_, Error::SYSTEM, "stream output error"); set_error(_KCCODELINE_, Error::SYSTEM, "stream output error");
err = true; err = true;
} }
} else { } else {
err = true; err = true;
} }
skipping to change at line 2041 skipping to change at line 2047
* @param src the source stream. * @param src the source stream.
* @param checker a progress checker object. If it is NULL, no checking is performed. * @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_snapshot(std::istream* src, ProgressChecker* checker = NULL) { bool load_snapshot(std::istream* src, ProgressChecker* checker = NULL) {
_assert_(src); _assert_(src);
if (src->fail()) { if (src->fail()) {
set_error(_KCCODELINE_, Error::INVALID, "invalid stream"); set_error(_KCCODELINE_, Error::INVALID, "invalid stream");
return false; return false;
} }
char buf[DBIOBUFSIZ]; char buf[IOBUFSIZ];
src->read(buf, sizeof(DBSSMAGICDATA)); src->read(buf, sizeof(KCDBSSMAGICDATA));
if (src->fail()) { if (src->fail()) {
set_error(_KCCODELINE_, Error::SYSTEM, "stream input error"); set_error(_KCCODELINE_, Error::SYSTEM, "stream input error");
return false; return false;
} }
if (std::memcmp(buf, DBSSMAGICDATA, sizeof(DBSSMAGICDATA))) { if (std::memcmp(buf, KCDBSSMAGICDATA, sizeof(KCDBSSMAGICDATA))) {
set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of input stream"); set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of input stream");
return false; return false;
} }
bool err = false; bool err = false;
if (checker && !checker->check("load_snapshot", "beginning", 0, -1)) { if (checker && !checker->check("load_snapshot", "beginning", 0, -1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true; err = true;
} }
int64_t curcnt = 0; int64_t curcnt = 0;
while (!err) { while (!err) {
skipping to change at line 2198 skipping to change at line 2204
case TYPECACHE: return "cache hash database"; case TYPECACHE: return "cache hash database";
case TYPEGRASS: return "cache tree database"; case TYPEGRASS: return "cache tree database";
case TYPEHASH: return "file hash database"; case TYPEHASH: return "file hash database";
case TYPETREE: return "file tree database"; case TYPETREE: return "file tree database";
case TYPEDIR: return "directory hash database"; case TYPEDIR: return "directory hash database";
case TYPEFOREST: return "directory tree database"; case TYPEFOREST: return "directory tree database";
case TYPEMISC: return "miscellaneous database"; case TYPEMISC: return "miscellaneous database";
} }
return "unknown"; return "unknown";
} }
private:
/** The size of the IO buffer. */
static const size_t IOBUFSIZ = 8192;
}; };
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 12 change blocks. 
18 lines changed or deleted 26 lines changed or added


 kcdbext.h   kcdbext.h 
skipping to change at line 38 skipping to change at line 38
#include <kcprotodb.h> #include <kcprotodb.h>
#include <kcstashdb.h> #include <kcstashdb.h>
#include <kccachedb.h> #include <kccachedb.h>
#include <kchashdb.h> #include <kchashdb.h>
#include <kcdirdb.h> #include <kcdirdb.h>
#include <kcpolydb.h> #include <kcpolydb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation.
*/
namespace {
const size_t MRDEFDBNUM = 8; ///< default number of temporary d
atabases
const size_t MRMAXDBNUM = 256; ///< maxinum number of temporary d
atabases
const int64_t MRDEFCLIM = 512LL << 20; ///< default cache limit
const int64_t MRDEFCBNUM = 1048583LL; ///< default cache bucket numer
const int64_t MRDBBNUM = 512LL << 10; ///< bucket number of temprary dat
abases
const int32_t MRDBPSIZ = 32768; ///< page size of temprary databas
es
const int64_t MRDBMSIZ = 516LL * 4096; ///< mapped size of temprary datab
ases
const int64_t MRDBPCCAP = 16LL << 20; ///< page cache capacity of tempra
ry databases
}
/**
* MapReduce framework. * MapReduce framework.
* @note Although this framework is not distributed or concurrent, it is us eful for aggregate * @note Although this framework is not distributed or concurrent, it is us eful for aggregate
* calculation with less CPU loading and less memory usage. * calculation with less CPU loading and less memory usage.
*/ */
class MapReduce { class MapReduce {
public: public:
class MapEmitter; class MapEmitter;
class ValueIterator; class ValueIterator;
private: private:
class MapVisitor; class MapVisitor;
skipping to change at line 427 skipping to change at line 413
*/ */
void tune_storage(int32_t dbnum, int64_t clim, int64_t cbnum) { void tune_storage(int32_t dbnum, int64_t clim, int64_t cbnum) {
_assert_(true); _assert_(true);
dbnum_ = dbnum > 0 ? dbnum : MRDEFDBNUM; dbnum_ = dbnum > 0 ? dbnum : MRDEFDBNUM;
if (dbnum_ > MRMAXDBNUM) dbnum_ = MRMAXDBNUM; if (dbnum_ > MRMAXDBNUM) dbnum_ = MRMAXDBNUM;
clim_ = clim > 0 ? clim : MRDEFCLIM; clim_ = clim > 0 ? clim : MRDEFCLIM;
cbnum_ = cbnum > 0 ? cbnum : MRDEFCBNUM; cbnum_ = cbnum > 0 ? cbnum : MRDEFCBNUM;
if (cbnum_ > INT16MAX) cbnum_ = nearbyprime(cbnum_); if (cbnum_ > INT16MAX) cbnum_ = nearbyprime(cbnum_);
} }
private: private:
/** The default number of temporary databases. */
static const size_t MRDEFDBNUM = 8;
/** The maxinum number of temporary databases. */
static const size_t MRMAXDBNUM = 256;
/** The default cache limit. */
static const int64_t MRDEFCLIM = 512LL << 20;
/** The default cache bucket numer. */
static const int64_t MRDEFCBNUM = 1048583LL;
/** The bucket number of temprary databases. */
static const int64_t MRDBBNUM = 512LL << 10;
/** The page size of temprary databases. */
static const int32_t MRDBPSIZ = 32768;
/** The mapped size of temprary databases. */
static const int64_t MRDBMSIZ = 516LL * 4096;
/** The page cache capacity of temprary databases. */
static const int64_t MRDBPCCAP = 16LL << 20;
/** /**
* Checker for the map process. * Checker for the map process.
*/ */
class MapChecker : public BasicDB::ProgressChecker { class MapChecker : public BasicDB::ProgressChecker {
public: public:
/** constructor */ /** constructor */
explicit MapChecker() : stop_(false) {} explicit MapChecker() : stop_(false) {}
/** stop the process */ /** stop the process */
void stop() { void stop() {
stop_ = true; stop_ = true;
 End of changes. 2 change blocks. 
20 lines changed or deleted 16 lines changed or added


 kcdirdb.h   kcdirdb.h 
skipping to change at line 29 skipping to change at line 29
#include <kcutil.h> #include <kcutil.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h> #include <kcregex.h>
#include <kcdb.h> #include <kcdb.h>
#include <kcplantdb.h> #include <kcplantdb.h>
namespace kyotocabinet { // common namespace #define KCDDBMAGICFILE "__KCDIR__" ///< magic file of the directory
#define KCDDBMETAFILE "__meta__" ///< meta data file of the directo
ry
#define KCDDBOPAQUEFILE "__opq__" ///< opaque file of the directory
#define KCDDBATRANPREFIX "_x" ///< prefix of files for auto tran
saction
#define KCDDBCHKSUMSEED "__kyotocabinet__" ///< seed of the module checks
um
#define KCDDBMAGICEOF "_EOF_" ///< magic data for the end of fil
e
#define KCDDBWALPATHEXT "wal" ///< extension of the WAL director
y
#define KCDDBTMPPATHEXT "tmp" ///< extension of the temporary di
rectory
/** namespace kyotocabinet { // common namespace
* Constants for implementation.
*/
namespace {
const char* DDBMAGICFILE = "__KCDIR__"; ///< magic file of the directory
const char* DDBMETAFILE = "__meta__"; ///< meta data file of the directo
ry
const char* DDBOPAQUEFILE = "__opq__"; ///< opaque file of the directory
const char* DDBATRANPREFIX = "_x"; ///< prefix of files for auto tran
saction
const char DDBCHKSUMSEED[] = "__kyotocabinet__"; ///< seed of the module c
hecksum
const char DDBMAGICEOF[] = "_EOF_"; ///< magic data for the end of fil
e
const int64_t DDBMETABUFSIZ = 128; ///< size of the meta data buffer
const uint8_t DDBRECMAGIC = 0xcc; ///< magic data for record
const int32_t DDBRLOCKSLOT = 8192; ///< number of slots of the record
lock
const int32_t DDBRECUNITSIZ = 32; ///< unit size of a record
const size_t DDBOPAQUESIZ = 16; ///< size of the opaque buffer
const uint32_t DDBLOCKBUSYLOOP = 8192; ///< threshold of busy loop and sl
eep for locking
const char* DDBWALPATHEXT = "wal"; ///< extension of the WAL director
y
const char* DDBTMPPATHEXT = "tmp"; ///< extension of the temporary di
rectory
}
/** /**
* Directory hash database. * Directory hash database.
* @note This class is a concrete class to operate a hash database in a dir ectory. This class * @note This class is a concrete class to operate a hash database in a dir ectory. This class
* can be inherited but overwriting methods is forbidden. Before every dat abase operation, it is * can be inherited but overwriting methods is forbidden. Before every dat abase operation, it is
* necessary to call the TreeDB::open method in order to open a database fi le and connect the * necessary to call the TreeDB::open method in order to open a database fi le and connect the
* database object to it. To avoid data missing or corruption, it is impor tant to close every * database object to it. To avoid data missing or corruption, it is impor tant to close every
* database file by the TreeDB::close method when the database is no longer in use. It is * database file by the TreeDB::close method when the database is no longer in use. It is
* forbidden for multible database objects in a process to open the same da tabase at the same * forbidden for multible database objects in a process to open the same da tabase at the same
* time. It is forbidden to share a database object with child processes. * time. It is forbidden to share a database object with child processes.
*/ */
class DirDB : public BasicDB { class DirDB : public BasicDB {
friend class PlantDB<DirDB, BasicDB::TYPEFOREST>; friend class PlantDB<DirDB, BasicDB::TYPEFOREST>;
public: public:
class Cursor; class Cursor;
private: private:
struct Record; struct Record;
class ScopedVisitor;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
/** An alias of vector of strings. */ /** An alias of vector of strings. */
typedef std::vector<std::string> StringVector; typedef std::vector<std::string> StringVector;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public BasicDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class DirDB; friend class DirDB;
skipping to change at line 135 skipping to change at line 125
if (db_->read_record(rpath, &rec)) { if (db_->read_record(rpath, &rec)) {
if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec.rsiz, if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec.rsiz,
visitor, rpath, name_.c_str())) err = t rue; visitor, rpath, name_.c_str())) err = t rue;
delete[] rec.rbuf; delete[] rec.rbuf;
if (alive_ && step && db_->count_ == cnt) { if (alive_ && step && db_->count_ == cnt) {
do { do {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
if (!disable()) err = true; if (!disable()) err = true;
break; break;
} }
} while (*name_.c_str() == *DDBMAGICFILE); } while (*name_.c_str() == *KCDDBMAGICFILE);
} }
} else { } else {
while (true) { while (true) {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
break; break;
} }
if (*name_.c_str() == *DDBMAGICFILE) continue; if (*name_.c_str() == *KCDDBMAGICFILE) continue;
const std::string& npath = db_->path_ + File::PATHCHR + name_; const std::string& npath = db_->path_ + File::PATHCHR + name_;
if (!File::status(npath)) continue; if (!File::status(npath)) continue;
if (db_->read_record(npath, &rec)) { if (db_->read_record(npath, &rec)) {
if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.v siz, rec.rsiz, if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.v siz, rec.rsiz,
visitor, npath, name_.c_str())) err = true; visitor, npath, name_.c_str())) err = true;
delete[] rec.rbuf; delete[] rec.rbuf;
if (alive_ && step && db_->count_ == cnt) { if (alive_ && step && db_->count_ == cnt) {
do { do {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
if (!disable()) err = true; if (!disable()) err = true;
break; break;
} }
} while (*name_.c_str() == *DDBMAGICFILE); } while (*name_.c_str() == *KCDDBMAGICFILE);
} }
} else { } else {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
err = true; err = true;
} }
break; break;
} }
} }
return !err; return !err;
} }
skipping to change at line 191 skipping to change at line 181
db_->set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory fa iled"); db_->set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory fa iled");
return false; return false;
} }
alive_ = true; alive_ = true;
do { do {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
return false; return false;
} }
} while (*name_.c_str() == *DDBMAGICFILE); } while (*name_.c_str() == *KCDDBMAGICFILE);
return true; return true;
} }
/** /**
* Jump the cursor to a record for forward scan. * Jump the cursor to a record for forward scan.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump(const char* kbuf, size_t ksiz) { bool jump(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
skipping to change at line 215 skipping to change at line 205
db_->set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory fa iled"); db_->set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory fa iled");
return false; return false;
} }
alive_ = true; alive_ = true;
while (true) { while (true) {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
return false; return false;
} }
if (*name_.c_str() == *DDBMAGICFILE) continue; if (*name_.c_str() == *KCDDBMAGICFILE) continue;
const std::string& rpath = db_->path_ + File::PATHCHR + name_; const std::string& rpath = db_->path_ + File::PATHCHR + name_;
Record rec; Record rec;
if (db_->read_record(rpath, &rec)) { if (db_->read_record(rpath, &rec)) {
if (rec.ksiz == ksiz && !std::memcmp(rec.kbuf, kbuf, ksiz)) { if (rec.ksiz == ksiz && !std::memcmp(rec.kbuf, kbuf, ksiz)) {
delete[] rec.rbuf; delete[] rec.rbuf;
break; break;
} }
delete[] rec.rbuf; delete[] rec.rbuf;
} else { } else {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
skipping to change at line 303 skipping to change at line 293
if (!alive_) { if (!alive_) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
do { do {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
return false; return false;
} }
} while (*name_.c_str() == *DDBMAGICFILE); } while (*name_.c_str() == *KCDDBMAGICFILE);
return true; return true;
} }
/** /**
* Step the cursor to the previous record. * Step the cursor to the previous record.
* @note This is a dummy implementation for compatibility. * @note This is a dummy implementation for compatibility.
*/ */
bool step_back() { bool step_back() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
skipping to change at line 374 skipping to change at line 364
* Status flags. * Status flags.
*/ */
enum Flag { enum Flag {
FOPEN = 1 << 0, ///< dummy for compatibility FOPEN = 1 << 0, ///< dummy for compatibility
FFATAL = 1 << 1 ///< dummy for compatibility FFATAL = 1 << 1 ///< dummy for compatibility
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit DirDB() : explicit DirDB() :
mlock_(), rlock_(DDBRLOCKSLOT), error_(), logger_(NULL), logkinds_(0), mtrigger_(NULL), mlock_(), rlock_(RLOCKSLOT), error_(), logger_(NULL), logkinds_(0), mtr igger_(NULL),
omode_(0), writer_(false), autotran_(false), autosync_(false), recov_(f alse), reorg_(false), omode_(0), writer_(false), autotran_(false), autosync_(false), recov_(f alse), reorg_(false),
file_(), curs_(), path_(""), file_(), curs_(), path_(""),
libver_(LIBVER), librev_(LIBREV), fmtver_(FMTVER), chksum_(0), type_(TY PEDIR), libver_(LIBVER), librev_(LIBREV), fmtver_(FMTVER), chksum_(0), type_(TY PEDIR),
flags_(0), opts_(0), count_(0), size_(0), opaque_(), embcomp_(ZLIBRAWCO MP), comp_(NULL), flags_(0), opts_(0), count_(0), size_(0), opaque_(), embcomp_(ZLIBRAWCO MP), comp_(NULL),
tran_(false), trhard_(false), trcount_(0), trsize_(0), walpath_(""), tm ppath_("") { tran_(false), trhard_(false), trcount_(0), trsize_(0), walpath_(""), tm ppath_("") {
_assert_(true); _assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~DirDB() { virtual ~DirDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
if (!curs_.empty()) { if (!curs_.empty()) {
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->db_ = NULL; cur->db_ = NULL;
cit++; ++cit;
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 423 skipping to change at line 413
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
bool err = false; bool err = false;
char name[NUMBUFSIZ]; char name[NUMBUFSIZ];
size_t lidx = hashpath(kbuf, ksiz, name) % DDBRLOCKSLOT; size_t lidx = hashpath(kbuf, ksiz, name) % RLOCKSLOT;
if (writable) { if (writable) {
rlock_.lock_writer(lidx); rlock_.lock_writer(lidx);
} else { } else {
rlock_.lock_reader(lidx); rlock_.lock_reader(lidx);
} }
if (!accept_impl(kbuf, ksiz, visitor, name)) err = true; if (!accept_impl(kbuf, ksiz, visitor, name)) err = true;
rlock_.unlock(lidx); rlock_.unlock(lidx);
return !err; return !err;
} }
/** /**
skipping to change at line 446 skipping to change at line 436
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The operations for specified records are performed atomically an d other threads * @note The operations for specified records are performed atomically an d other threads
* accessing the same records are blocked. To avoid deadlock, any explic it database operation * accessing the same records are blocked. To avoid deadlock, any explic it database operation
* must not be performed in this function. * must not be performed in this function.
*/ */
bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor,
bool writable = true) { bool writable = true) {
_assert_(visitor); _assert_(visitor);
size_t knum = keys.size();
if (knum < 1) return true;
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
size_t knum = keys.size();
if (knum < 1) return true;
bool err = false; bool err = false;
struct RecordKey { struct RecordKey {
const char* kbuf; const char* kbuf;
size_t ksiz; size_t ksiz;
char name[NUMBUFSIZ]; char name[NUMBUFSIZ];
}; };
RecordKey* rkeys = new RecordKey[knum]; RecordKey* rkeys = new RecordKey[knum];
std::set<size_t> lidxs; std::set<size_t> lidxs;
for (size_t i = 0; i < knum; i++) { for (size_t i = 0; i < knum; i++) {
const std::string& key = keys[i]; const std::string& key = keys[i];
RecordKey* rkey = rkeys + i; RecordKey* rkey = rkeys + i;
rkey->kbuf = key.data(); rkey->kbuf = key.data();
rkey->ksiz = key.size(); rkey->ksiz = key.size();
lidxs.insert(hashpath(rkey->kbuf, rkey->ksiz, rkey->name) % DDBRLOCKS LOT); lidxs.insert(hashpath(rkey->kbuf, rkey->ksiz, rkey->name) % RLOCKSLOT );
} }
std::set<size_t>::iterator lit = lidxs.begin(); std::set<size_t>::iterator lit = lidxs.begin();
std::set<size_t>::iterator litend = lidxs.end(); std::set<size_t>::iterator litend = lidxs.end();
while (lit != litend) { while (lit != litend) {
if (writable) { if (writable) {
rlock_.lock_writer(*lit); rlock_.lock_writer(*lit);
} else { } else {
rlock_.lock_reader(*lit); rlock_.lock_reader(*lit);
} }
lit++; ++lit;
} }
for (size_t i = 0; i < knum; i++) { for (size_t i = 0; i < knum; i++) {
RecordKey* rkey = rkeys + i; RecordKey* rkey = rkeys + i;
if (!accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->name)) { if (!accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->name)) {
err = true; err = true;
break; break;
} }
} }
lit = lidxs.begin(); lit = lidxs.begin();
litend = lidxs.end(); litend = lidxs.end();
while (lit != litend) { while (lit != litend) {
rlock_.unlock(*lit); rlock_.unlock(*lit);
lit++; ++lit;
} }
delete[] rkeys; delete[] rkeys;
return !err; return !err;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed. * @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 518 skipping to change at line 509
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
bool err = false; bool err = false;
if (!iterate_impl(visitor, checker)) err = true; if (!iterate_impl(visitor, checker)) err = true;
trigger_meta(MetaTrigger::ITERATE, "iterate"); trigger_meta(MetaTrigger::ITERATE, "iterate");
return !err; return !err;
} }
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
Error error() const { Error error() const {
skipping to change at line 599 skipping to change at line 591
if (mode & OAUTOTRAN) autotran_ = true; if (mode & OAUTOTRAN) autotran_ = true;
if (mode & OAUTOSYNC) autosync_ = true; if (mode & OAUTOSYNC) autosync_ = true;
} }
if (mode & ONOLOCK) fmode |= File::ONOLOCK; if (mode & ONOLOCK) fmode |= File::ONOLOCK;
if (mode & OTRYLOCK) fmode |= File::OTRYLOCK; if (mode & OTRYLOCK) fmode |= File::OTRYLOCK;
size_t psiz = path.size(); size_t psiz = path.size();
while (psiz > 0 && path[psiz-1] == File::PATHCHR) { while (psiz > 0 && path[psiz-1] == File::PATHCHR) {
psiz--; psiz--;
} }
const std::string& cpath = path.substr(0, psiz); const std::string& cpath = path.substr(0, psiz);
const std::string& magicpath = cpath + File::PATHCHR + DDBMAGICFILE; const std::string& magicpath = cpath + File::PATHCHR + KCDDBMAGICFILE;
const std::string& metapath = cpath + File::PATHCHR + DDBMETAFILE; const std::string& metapath = cpath + File::PATHCHR + KCDDBMETAFILE;
const std::string& opqpath = cpath + File::PATHCHR + DDBOPAQUEFILE; const std::string& opqpath = cpath + File::PATHCHR + KCDDBOPAQUEFILE;
const std::string& walpath = cpath + File::EXTCHR + DDBWALPATHEXT; const std::string& walpath = cpath + File::EXTCHR + KCDDBWALPATHEXT;
const std::string& tmppath = cpath + File::EXTCHR + DDBTMPPATHEXT; const std::string& tmppath = cpath + File::EXTCHR + KCDDBTMPPATHEXT;
bool hot = false; bool hot = false;
if (writer_ && (mode & OTRUNCATE) && File::status(magicpath)) { if (writer_ && (mode & OTRUNCATE) && File::status(magicpath)) {
if (!file_.open(magicpath, fmode)) { if (!file_.open(magicpath, fmode)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
if (!remove_files(cpath)) { if (!remove_files(cpath)) {
file_.close(); file_.close();
return false; return false;
} }
skipping to change at line 842 skipping to change at line 834
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wcnt >= DDBLOCKBUSYLOOP) { if (wcnt >= LOCKBUSYLOOP) {
Thread::chill(); Thread::chill();
} else { } else {
Thread::yield(); Thread::yield();
wcnt++; wcnt++;
} }
} }
trhard_ = hard; trhard_ = hard;
if (!begin_transaction_impl()) { if (!begin_transaction_impl()) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
skipping to change at line 941 skipping to change at line 933
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
bool err = false; bool err = false;
if (!disable_cursors()) err = true; if (!disable_cursors()) err = true;
if (tran_) { if (tran_) {
DirStream dir; DirStream dir;
if (dir.open(path_)) { if (dir.open(path_)) {
std::string name; std::string name;
while (dir.read(&name)) { while (dir.read(&name)) {
if (*name.c_str() == *DDBMAGICFILE) continue; if (*name.c_str() == *KCDDBMAGICFILE) continue;
const std::string& rpath = path_ + File::PATHCHR + name; const std::string& rpath = path_ + File::PATHCHR + name;
const std::string& walpath = walpath_ + File::PATHCHR + name; const std::string& walpath = walpath_ + File::PATHCHR + name;
if (File::status(walpath)) { if (File::status(walpath)) {
if (!File::remove(rpath)) { if (!File::remove(rpath)) {
set_error(_KCCODELINE_, Error::SYSTEM, "removing a file faile d"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a file faile d");
err = true; err = true;
} }
} else if (!File::rename(rpath, walpath)) { } else if (!File::rename(rpath, walpath)) {
set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed" ); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed" );
err = true; err = true;
skipping to change at line 1368 skipping to change at line 1360
bool reorganized() { bool reorganized() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return reorg_; return reorg_;
} }
private: private:
/** The size of the meta data buffer. */
static const int64_t METABUFSIZ = 128;
/** The magic data for record. */
static const uint8_t RECMAGIC = 0xcc;
/** The number of slots of the record lock. */
static const int32_t RLOCKSLOT = 2048;
/** The unit size of a record. */
static const int32_t RECUNITSIZ = 32;
/** The size of the opaque buffer. */
static const size_t OPAQUESIZ = 16;
/** The threshold of busy loop and sleep for locking. */
static const uint32_t LOCKBUSYLOOP = 8192;
/** /**
* Set the power of the alignment of record size. * Set the power of the alignment of record size.
* @note This is a dummy implementation for compatibility. * @note This is a dummy implementation for compatibility.
*/ */
bool tune_alignment(int8_t apow) { bool tune_alignment(int8_t apow) {
return true; return true;
} }
/** /**
* Set the power of the capacity of the free block pool. * Set the power of the capacity of the free block pool.
* @note This is a dummy implementation for compatibility. * @note This is a dummy implementation for compatibility.
skipping to change at line 1458 skipping to change at line 1462
*/ */
struct Record { struct Record {
char* rbuf; ///< record buffer char* rbuf; ///< record buffer
size_t rsiz; ///< record size size_t rsiz; ///< record size
const char* kbuf; ///< key buffer const char* kbuf; ///< key buffer
size_t ksiz; ///< key size size_t ksiz; ///< key size
const char* vbuf; ///< value buffer const char* vbuf; ///< value buffer
size_t vsiz; ///< value size size_t vsiz; ///< value size
}; };
/** /**
* Scoped visiotor.
*/
class ScopedVisitor {
public:
/** constructor */
ScopedVisitor(Visitor* visitor) : visitor_(visitor) {
_assert_(visitor);
visitor_->visit_before();
}
/** destructor */
~ScopedVisitor() {
_assert_(true);
visitor_->visit_after();
}
private:
Visitor* visitor_; ///< visitor
};
/**
* Dump the magic data into the file. * Dump the magic data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_magic() { bool dump_magic() {
_assert_(true); _assert_(true);
const std::string& buf = format_magic(count_, size_); const std::string& buf = format_magic(count_, size_);
if (!file_.write(0, buf.c_str(), buf.size())) { if (!file_.write(0, buf.c_str(), buf.size())) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
return true; return true;
} }
/** /**
* Format the magic data. * Format the magic data.
* @return the result string. * @return the result string.
*/ */
std::string format_magic(int64_t count, int64_t size) { std::string format_magic(int64_t count, int64_t size) {
return strprintf("%lld\n%lld\n%s\n", (long long)count, (long long)size, DDBMAGICEOF); return strprintf("%lld\n%lld\n%s\n", (long long)count, (long long)size, KCDDBMAGICEOF);
} }
/** /**
* Load the magic data from the file. * Load the magic data from the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_magic() { bool load_magic() {
_assert_(true); _assert_(true);
char buf[NUMBUFSIZ*3]; char buf[NUMBUFSIZ*3];
size_t len = file_.size(); size_t len = file_.size();
if (len > sizeof(buf) - 1) len = sizeof(buf) - 1; if (len > sizeof(buf) - 1) len = sizeof(buf) - 1;
skipping to change at line 1497 skipping to change at line 1519
buf[len] = '\0'; buf[len] = '\0';
char* rp = buf; char* rp = buf;
int64_t count = atoi(rp); int64_t count = atoi(rp);
char* pv = std::strchr(rp, '\n'); char* pv = std::strchr(rp, '\n');
if (!pv) return false; if (!pv) return false;
rp = pv + 1; rp = pv + 1;
int64_t size = atoi(rp); int64_t size = atoi(rp);
pv = std::strchr(rp, '\n'); pv = std::strchr(rp, '\n');
if (!pv) return false; if (!pv) return false;
rp = pv + 1; rp = pv + 1;
if (std::strlen(rp) < sizeof(DDBMAGICEOF) - 1 || if (std::strlen(rp) < sizeof(KCDDBMAGICEOF) - 1 ||
std::memcmp(rp, DDBMAGICEOF, sizeof(DDBMAGICEOF) - 1)) return false std::memcmp(rp, KCDDBMAGICEOF, sizeof(KCDDBMAGICEOF) - 1)) return f
; alse;
flags_ = 0; flags_ = 0;
count_ = count; count_ = count;
size_ = size; size_ = size;
return true; return true;
} }
/** /**
* Calculate magic data. * Calculate magic data.
* @param cpath the path of the database file. * @param cpath the path of the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
skipping to change at line 1521 skipping to change at line 1543
count_ = 0; count_ = 0;
size_ = 0; size_ = 0;
DirStream dir; DirStream dir;
if (!dir.open(cpath)) { if (!dir.open(cpath)) {
set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed");
return false; return false;
} }
bool err = false; bool err = false;
std::string name; std::string name;
while (dir.read(&name)) { while (dir.read(&name)) {
if (*name.c_str() == *DDBMAGICFILE) continue; if (*name.c_str() == *KCDDBMAGICFILE) continue;
const std::string& rpath = cpath + File::PATHCHR + name; const std::string& rpath = cpath + File::PATHCHR + name;
File::Status sbuf; File::Status sbuf;
if (File::status(rpath, &sbuf)) { if (File::status(rpath, &sbuf)) {
if (sbuf.size >= 4) { if (sbuf.size >= 4) {
count_ += 1; count_ += 1;
size_ += sbuf.size - 4; size_ += sbuf.size - 4;
} else { } else {
File::remove(rpath); File::remove(rpath);
} }
} else { } else {
skipping to change at line 1548 skipping to change at line 1570
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Calculate the module checksum. * Calculate the module checksum.
* @return the module checksum. * @return the module checksum.
*/ */
uint8_t calc_checksum() { uint8_t calc_checksum() {
_assert_(true); _assert_(true);
const char* kbuf = DDBCHKSUMSEED; const char* kbuf = KCDDBCHKSUMSEED;
size_t ksiz = sizeof(DDBCHKSUMSEED) - 1; size_t ksiz = sizeof(KCDDBCHKSUMSEED) - 1;
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (comp_) { if (comp_) {
zbuf = comp_->compress(kbuf, ksiz, &zsiz); zbuf = comp_->compress(kbuf, ksiz, &zsiz);
if (!zbuf) return 0; if (!zbuf) return 0;
kbuf = zbuf; kbuf = zbuf;
ksiz = zsiz; ksiz = zsiz;
} }
char name[NUMBUFSIZ]; char name[NUMBUFSIZ];
uint32_t hash = hashpath(kbuf, ksiz, name); uint32_t hash = hashpath(kbuf, ksiz, name);
skipping to change at line 1572 skipping to change at line 1594
return hash; return hash;
} }
/** /**
* Dump the meta data into the file. * Dump the meta data into the file.
* @param metapath the path of the meta data file. * @param metapath the path of the meta data file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_meta(const std::string& metapath) { bool dump_meta(const std::string& metapath) {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
char buf[DDBMETABUFSIZ]; char buf[METABUFSIZ];
char* wp = buf; char* wp = buf;
wp += std::sprintf(wp, "%u\n", libver_); wp += std::sprintf(wp, "%u\n", libver_);
wp += std::sprintf(wp, "%u\n", librev_); wp += std::sprintf(wp, "%u\n", librev_);
wp += std::sprintf(wp, "%u\n", fmtver_); wp += std::sprintf(wp, "%u\n", fmtver_);
wp += std::sprintf(wp, "%u\n", chksum_); wp += std::sprintf(wp, "%u\n", chksum_);
wp += std::sprintf(wp, "%u\n", type_); wp += std::sprintf(wp, "%u\n", type_);
wp += std::sprintf(wp, "%u\n", opts_); wp += std::sprintf(wp, "%u\n", opts_);
wp += std::sprintf(wp, "%s\n", DDBMAGICEOF); wp += std::sprintf(wp, "%s\n", KCDDBMAGICEOF);
if (!File::write_file(metapath, buf, wp - buf)) { if (!File::write_file(metapath, buf, wp - buf)) {
set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Load the meta data from the file. * Load the meta data from the file.
* @param metapath the path of the meta data file. * @param metapath the path of the meta data file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_meta(const std::string& metapath) { bool load_meta(const std::string& metapath) {
_assert_(true); _assert_(true);
int64_t size; int64_t size;
char* buf = File::read_file(metapath, &size, DDBMETABUFSIZ); char* buf = File::read_file(metapath, &size, METABUFSIZ);
if (!buf) { if (!buf) {
set_error(_KCCODELINE_, Error::SYSTEM, "reading a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "reading a file failed");
return false; return false;
} }
std::string str(buf, size); std::string str(buf, size);
delete[] buf; delete[] buf;
std::vector<std::string> elems; std::vector<std::string> elems;
if (strsplit(str, '\n', &elems) < 7 || elems[6] != DDBMAGICEOF) { if (strsplit(str, '\n', &elems) < 7 || elems[6] != KCDDBMAGICEOF) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data file"); set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data file");
return false; return false;
} }
libver_ = atoi(elems[0].c_str()); libver_ = atoi(elems[0].c_str());
librev_ = atoi(elems[1].c_str()); librev_ = atoi(elems[1].c_str());
fmtver_ = atoi(elems[2].c_str()); fmtver_ = atoi(elems[2].c_str());
chksum_ = atoi(elems[3].c_str()); chksum_ = atoi(elems[3].c_str());
type_ = atoi(elems[4].c_str()); type_ = atoi(elems[4].c_str());
opts_ = atoi(elems[5].c_str()); opts_ = atoi(elems[5].c_str());
return true; return true;
} }
/** /**
* Dump the opaque data into the file. * Dump the opaque data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_opaque() { bool dump_opaque() {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
const std::string& opath = path_ + File::PATHCHR + DDBOPAQUEFILE; const std::string& opath = path_ + File::PATHCHR + KCDDBOPAQUEFILE;
if (!File::write_file(opath, opaque_, sizeof(opaque_))) { if (!File::write_file(opath, opaque_, sizeof(opaque_))) {
set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Load the opaque data from the file. * Load the opaque data from the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
void load_opaque() { void load_opaque() {
_assert_(true); _assert_(true);
std::memset(opaque_, 0, sizeof(opaque_)); std::memset(opaque_, 0, sizeof(opaque_));
const std::string& opath = path_ + File::PATHCHR + DDBOPAQUEFILE; const std::string& opath = path_ + File::PATHCHR + KCDDBOPAQUEFILE;
int64_t size; int64_t size;
char* buf = File::read_file(opath, &size, sizeof(opaque_)); char* buf = File::read_file(opath, &size, sizeof(opaque_));
if (buf) { if (buf) {
std::memcpy(opaque_, buf, size); std::memcpy(opaque_, buf, size);
delete[] buf; delete[] buf;
} }
} }
/** /**
* Remove inner files. * Remove inner files.
* @param cpath the path of the database file. * @param cpath the path of the database file.
skipping to change at line 1659 skipping to change at line 1681
bool remove_files(const std::string& cpath) { bool remove_files(const std::string& cpath) {
_assert_(true); _assert_(true);
DirStream dir; DirStream dir;
if (!dir.open(cpath)) { if (!dir.open(cpath)) {
set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed"); set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed");
return false; return false;
} }
bool err = false; bool err = false;
std::string name; std::string name;
while (dir.read(&name)) { while (dir.read(&name)) {
if (*name.c_str() == *DDBMAGICFILE) continue; if (*name.c_str() == *KCDDBMAGICFILE) continue;
const std::string& rpath = cpath + File::PATHCHR + name; const std::string& rpath = cpath + File::PATHCHR + name;
if (!File::remove(rpath)) { if (!File::remove(rpath)) {
set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed");
err = true; err = true;
} }
} }
if (!dir.close()) { if (!dir.close()) {
set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed"); set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed");
err = true; err = true;
} }
skipping to change at line 1697 skipping to change at line 1719
if (!zbuf) { if (!zbuf) {
set_error(_KCCODELINE_, Error::SYSTEM, "data decompression failed") ; set_error(_KCCODELINE_, Error::SYSTEM, "data decompression failed") ;
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
delete[] rbuf; delete[] rbuf;
rbuf = zbuf; rbuf = zbuf;
rsiz = zsiz; rsiz = zsiz;
} }
const char* rp = rbuf; const char* rp = rbuf;
if (rsiz < 4 || *(const unsigned char*)rp != DDBRECMAGIC) { if (rsiz < 4 || *(const unsigned char*)rp != RECMAGIC) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a recor d"); set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a recor d");
report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str()); report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str());
report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
rp++; rp++;
uint64_t num; uint64_t num;
size_t step = readvarnum(rp, rsiz, &num); size_t step = readvarnum(rp, rsiz, &num);
rp += step; rp += step;
skipping to change at line 1720 skipping to change at line 1742
if (rsiz < 2) { if (rsiz < 2) {
report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str()); report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str());
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
step = readvarnum(rp, rsiz, &num); step = readvarnum(rp, rsiz, &num);
rp += step; rp += step;
rsiz -= step; rsiz -= step;
size_t vsiz = num; size_t vsiz = num;
if (rsiz < 1 + (int64_t)ksiz + (int64_t)vsiz || if (rsiz < 1 + (int64_t)ksiz + (int64_t)vsiz ||
((const unsigned char*)rp)[ksiz+vsiz] != DDBRECMAGIC) { ((const unsigned char*)rp)[ksiz+vsiz] != RECMAGIC) {
set_error(_KCCODELINE_, Error::BROKEN, "too short record"); set_error(_KCCODELINE_, Error::BROKEN, "too short record");
report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str()); report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str());
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
rec->rbuf = rbuf; rec->rbuf = rbuf;
rec->kbuf = rp; rec->kbuf = rp;
rec->ksiz = ksiz; rec->ksiz = ksiz;
rec->vbuf = rp + ksiz; rec->vbuf = rp + ksiz;
rec->vsiz = vsiz; rec->vsiz = vsiz;
skipping to change at line 1751 skipping to change at line 1773
* @param wsp the pointer to the variable into which the size of the writ ten record is * @param wsp the pointer to the variable into which the size of the writ ten record is
* assigned. * assigned.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool write_record(const std::string& rpath, const char* name, const char* kbuf, size_t ksiz, bool write_record(const std::string& rpath, const char* name, const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* wsp) { const char* vbuf, size_t vsiz, size_t* wsp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && wsp) ; _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && wsp) ;
bool err = false; bool err = false;
char* rbuf = new char[NUMBUFSIZ*2+ksiz+vsiz]; char* rbuf = new char[NUMBUFSIZ*2+ksiz+vsiz];
char* wp = rbuf; char* wp = rbuf;
*(wp++) = DDBRECMAGIC; *(wp++) = RECMAGIC;
wp += writevarnum(wp, ksiz); wp += writevarnum(wp, ksiz);
wp += writevarnum(wp, vsiz); wp += writevarnum(wp, vsiz);
std::memcpy(wp, kbuf, ksiz); std::memcpy(wp, kbuf, ksiz);
wp += ksiz; wp += ksiz;
std::memcpy(wp, vbuf, vsiz); std::memcpy(wp, vbuf, vsiz);
wp += vsiz; wp += vsiz;
*(wp++) = DDBRECMAGIC; *(wp++) = RECMAGIC;
size_t rsiz = wp - rbuf; size_t rsiz = wp - rbuf;
if (comp_) { if (comp_) {
size_t zsiz; size_t zsiz;
char* zbuf = comp_->compress(rbuf, rsiz, &zsiz); char* zbuf = comp_->compress(rbuf, rsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed"); set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed");
delete[] rbuf; delete[] rbuf;
*wsp = 0; *wsp = 0;
return false; return false;
} }
delete[] rbuf; delete[] rbuf;
rbuf = zbuf; rbuf = zbuf;
rsiz = zsiz; rsiz = zsiz;
} }
if (autotran_ && !tran_) { if (autotran_ && !tran_) {
const std::string& tpath = path_ + File::PATHCHR + DDBATRANPREFIX + n ame; const std::string& tpath = path_ + File::PATHCHR + KCDDBATRANPREFIX + name;
if (!File::write_file(tpath, rbuf, rsiz)) { if (!File::write_file(tpath, rbuf, rsiz)) {
set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed");
err = true; err = true;
} }
if (!File::rename(tpath, rpath)) { if (!File::rename(tpath, rpath)) {
set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed");
err = true; err = true;
File::remove(tpath); File::remove(tpath);
} }
} else { } else {
skipping to change at line 1807 skipping to change at line 1829
*/ */
bool disable_cursors() { bool disable_cursors() {
_assert_(true); _assert_(true);
if (curs_.empty()) return true; if (curs_.empty()) return true;
bool err = false; bool err = false;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->alive_ && !cur->disable()) err = true; if (cur->alive_ && !cur->disable()) err = true;
cit++; ++cit;
} }
return !err; return !err;
} }
/** /**
* Escape cursors on a free block. * Escape cursors on a free block.
* @param rpath the file path of the record. * @param rpath the file path of the record.
* @param name the file name of the record. * @param name the file name of the record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool escape_cursors(const std::string& rpath, const char* name) { bool escape_cursors(const std::string& rpath, const char* name) {
skipping to change at line 1830 skipping to change at line 1852
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->alive_ && cur->name_ == name) { if (cur->alive_ && cur->name_ == name) {
do { do {
if (!cur->dir_.read(&cur->name_)) { if (!cur->dir_.read(&cur->name_)) {
if (!cur->disable()) err = true; if (!cur->disable()) err = true;
break; break;
} }
} while (*cur->name_.c_str() == *DDBMAGICFILE); } while (*cur->name_.c_str() == *KCDDBMAGICFILE);
} }
cit++; ++cit;
} }
return !err; return !err;
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param name the encoded key. * @param name the encoded key.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 1981 skipping to change at line 2003
} }
int64_t allcnt = count_; int64_t allcnt = count_;
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
bool err = false; bool err = false;
std::string name; std::string name;
int64_t curcnt = 0; int64_t curcnt = 0;
while (dir.read(&name)) { while (dir.read(&name)) {
if (*name.c_str() == *DDBMAGICFILE) continue; if (*name.c_str() == *KCDDBMAGICFILE) continue;
const std::string& rpath = path_ + File::PATHCHR + name; const std::string& rpath = path_ + File::PATHCHR + name;
Record rec; Record rec;
if (read_record(rpath, &rec)) { if (read_record(rpath, &rec)) {
if (!accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec. rsiz, if (!accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec. rsiz,
visitor, rpath, name.c_str())) err = true; visitor, rpath, name.c_str())) err = true;
delete[] rec.rbuf; delete[] rec.rbuf;
} else { } else {
set_error(_KCCODELINE_, Error::BROKEN, "missing record"); set_error(_KCCODELINE_, Error::BROKEN, "missing record");
err = true; err = true;
} }
skipping to change at line 2153 skipping to change at line 2175
set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes. * @return the size of the database file in bytes.
*/ */
int64_t size_impl() { int64_t size_impl() {
return size_ + count_ * DDBRECUNITSIZ; return size_ + count_ * RECUNITSIZ;
} }
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
DirDB(const DirDB&); DirDB(const DirDB&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
DirDB& operator =(const DirDB&); DirDB& operator =(const DirDB&);
/** The method lock. */ /** The method lock. */
SpinRWLock mlock_; SpinRWLock mlock_;
/** The record locks. */ /** The record locks. */
SlottedRWLock rlock_; SlottedRWLock rlock_;
/** The last happened error. */ /** The last happened error. */
skipping to change at line 2208 skipping to change at line 2230
uint8_t type_; uint8_t type_;
/** The status flags. */ /** The status flags. */
uint8_t flags_; uint8_t flags_;
/** The options. */ /** The options. */
uint8_t opts_; uint8_t opts_;
/** The record number. */ /** The record number. */
AtomicInt64 count_; AtomicInt64 count_;
/** The total size of records. */ /** The total size of records. */
AtomicInt64 size_; AtomicInt64 size_;
/** The opaque data. */ /** The opaque data. */
char opaque_[DDBOPAQUESIZ]; char opaque_[OPAQUESIZ];
/** The embedded data compressor. */ /** The embedded data compressor. */
Compressor* embcomp_; Compressor* embcomp_;
/** The data compressor. */ /** The data compressor. */
Compressor* comp_; Compressor* comp_;
/** The compression checksum. */ /** The compression checksum. */
bool tran_; bool tran_;
/** The flag whether hard transaction. */ /** The flag whether hard transaction. */
bool trhard_; bool trhard_;
/** The old count before transaction. */ /** The old count before transaction. */
int64_t trcount_; int64_t trcount_;
 End of changes. 45 change blocks. 
74 lines changed or deleted 94 lines changed or added


 kchashdb.h   kchashdb.h 
skipping to change at line 29 skipping to change at line 29
#include <kcutil.h> #include <kcutil.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h> #include <kcregex.h>
#include <kcdb.h> #include <kcdb.h>
#include <kcplantdb.h> #include <kcplantdb.h>
namespace kyotocabinet { // common namespace #define KCHDBMAGICDATA "KC\n" ///< magic data of the file
#define KCHDBCHKSUMSEED "__kyotocabinet__" ///< seed of the module checks
um
#define KCHDBTMPPATHEXT "tmpkch" ///< extension of the temporary fi
le
/** namespace kyotocabinet { // common namespace
* Constants for implementation.
*/
namespace {
const char HDBMAGICDATA[] = "KC\n"; ///< magic data of the file
const char HDBCHKSUMSEED[] = "__kyotocabinet__"; ///< seed of the module c
hecksum
const int64_t HDBMOFFLIBVER = 4; ///< offset of the library version
const int64_t HDBMOFFLIBREV = 5; ///< offset of the library revisio
n
const int64_t HDBMOFFFMTVER = 6; ///< offset of the format revision
const int64_t HDBMOFFCHKSUM = 7; ///< offset of the module checksum
const int64_t HDBMOFFTYPE = 8; ///< offset of the database type
const int64_t HDBMOFFAPOW = 9; ///< offset of the alignment power
const int64_t HDBMOFFFPOW = 10; ///< offset of the free block pool
power
const int64_t HDBMOFFOPTS = 11; ///< offset of the options
const int64_t HDBMOFFBNUM = 16; ///< offset of the bucket number
const int64_t HDBMOFFFLAGS = 24; ///< offset of the status flags
const int64_t HDBMOFFCOUNT = 32; ///< offset of the record number
const int64_t HDBMOFFSIZE = 40; ///< offset of the file size
const int64_t HDBMOFFOPAQUE = 48; ///< offset of the opaque data
const int64_t HDBHEADSIZ = 64; ///< size of the header
const int32_t HDBFBPWIDTH = 6; ///< width of the free block
const int32_t HDBWIDTHLARGE = 6; ///< large width of the record add
ress
const int32_t HDBWIDTHSMALL = 4; ///< small width of the record add
ress
const size_t HDBRECBUFSIZ = 48; ///< size of the record buffer
const size_t HDBIOBUFSIZ = 1024; ///< size of the IO buffer
const int32_t HDBRLOCKSLOT = 4096; ///< number of slots of the record
lock
const uint8_t HDBDEFAPOW = 3; ///< default alignment power
const uint8_t HDBMAXAPOW = 15; ///< maximum alignment power
const uint8_t HDBDEFFPOW = 10; ///< default free block pool power
const uint8_t HDBMAXFPOW = 20; ///< maximum free block pool power
const int64_t HDBDEFBNUM = 1048583LL; ///< default bucket number
const int64_t HDBDEFMSIZ = 64LL << 20; ///< default size of the memory-ma
pped region
const uint8_t HDBRECMAGIC = 0xcc; ///< magic data for record
const uint8_t HDBPADMAGIC = 0xee; ///< magic data for padding
const uint8_t HDBFBMAGIC = 0xdd; ///< magic data for free block
const int32_t HDBDFRGMAX = 512; ///< maximum unit of auto defragme
ntation
const int32_t HDBDFRGCEF = 2; ///< coefficient of auto defragmen
tation
const int64_t HDBSLVGWIDTH = 1LL << 20; ///< checking width for record sal
vage
const uint32_t HDBLOCKBUSYLOOP = 8192; ///< threshold of busy loop and sl
eep for locking
const char* HDBTMPPATHEXT = "tmpkch"; ///< extension of the temporary fi
le
}
/** /**
* File hash database. * File hash database.
* @note This class is a concrete class to operate a hash database on a fil e. This class can be * @note This class is a concrete class to operate a hash database on a fil e. This class can be
* inherited but overwriting methods is forbidden. Before every database o peration, it is * inherited but overwriting methods is forbidden. Before every database o peration, it is
* necessary to call the HashDB::open method in order to open a database fi le and connect the * necessary to call the HashDB::open method in order to open a database fi le and connect the
* database object to it. To avoid data missing or corruption, it is impor tant to close every * database object to it. To avoid data missing or corruption, it is impor tant to close every
* database file by the HashDB::close method when the database is no longer in use. It is * database file by the HashDB::close method when the database is no longer in use. It is
* forbidden for multible database objects in a process to open the same da tabase at the same * forbidden for multible database objects in a process to open the same da tabase at the same
* time. It is forbidden to share a database object with child processes. * time. It is forbidden to share a database object with child processes.
*/ */
class HashDB : public BasicDB { class HashDB : public BasicDB {
friend class PlantDB<HashDB, BasicDB::TYPETREE>; friend class PlantDB<HashDB, BasicDB::TYPETREE>;
public: public:
class Cursor; class Cursor;
private: private:
struct Record; struct Record;
struct FreeBlock; struct FreeBlock;
struct FreeBlockComparator; struct FreeBlockComparator;
class Repeater; class Repeater;
class ScopedVisitor;
/** An alias of set of free blocks. */ /** An alias of set of free blocks. */
typedef std::set<FreeBlock> FBP; typedef std::set<FreeBlock> FBP;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public BasicDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class HashDB; friend class HashDB;
skipping to change at line 153 skipping to change at line 116
if (!(db_->flags_ & FOPEN) && !db_->autotran_ && !db_->tran_ && if (!(db_->flags_ & FOPEN) && !db_->autotran_ && !db_->tran_ &&
!db_->set_flag(FOPEN, true)) { !db_->set_flag(FOPEN, true)) {
return false; return false;
} }
} }
if (off_ < 1) { if (off_ < 1) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[RECBUFSIZ];
if (!step_impl(&rec, rbuf, 0)) return false; if (!step_impl(&rec, rbuf, 0)) return false;
if (!rec.vbuf && !db_->read_record_body(&rec)) { if (!rec.vbuf && !db_->read_record_body(&rec)) {
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
const char* vbuf = rec.vbuf; const char* vbuf = rec.vbuf;
size_t vsiz = rec.vsiz; size_t vsiz = rec.vsiz;
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (db_->comp_) { if (db_->comp_) {
skipping to change at line 241 skipping to change at line 204
if (!db_->accept_impl(rec.kbuf, rec.ksiz, &repeater, bidx, pivot, true)) { if (!db_->accept_impl(rec.kbuf, rec.ksiz, &repeater, bidx, pivot, true)) {
delete[] zbuf; delete[] zbuf;
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
delete[] zbuf; delete[] zbuf;
delete[] rec.bbuf; delete[] rec.bbuf;
} }
} }
if (db_->dfunit_ > 0 && db_->frgcnt_ >= db_->dfunit_) { if (db_->dfunit_ > 0 && db_->frgcnt_ >= db_->dfunit_) {
if (!db_->defrag_impl(db_->dfunit_ * HDBDFRGCEF)) return false; if (!db_->defrag_impl(db_->dfunit_ * DFRGCEF)) return false;
db_->frgcnt_ -= db_->dfunit_; db_->frgcnt_ -= db_->dfunit_;
} }
return true; return true;
} }
/** /**
* Jump the cursor to the first record for forward scan. * Jump the cursor to the first record for forward scan.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump() { bool jump() {
_assert_(true); _assert_(true);
skipping to change at line 286 skipping to change at line 249
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
off_ = 0; off_ = 0;
uint64_t hash = db_->hash_record(kbuf, ksiz); uint64_t hash = db_->hash_record(kbuf, ksiz);
uint32_t pivot = db_->fold_hash(hash); uint32_t pivot = db_->fold_hash(hash);
int64_t bidx = hash % db_->bnum_; int64_t bidx = hash % db_->bnum_;
int64_t off = db_->get_bucket(bidx); int64_t off = db_->get_bucket(bidx);
if (off < 0) return false; if (off < 0) return false;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[RECBUFSIZ];
while (off > 0) { while (off > 0) {
rec.off = off; rec.off = off;
if (!db_->read_record(&rec, rbuf)) return false; if (!db_->read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16MAX) { if (rec.psiz == UINT16MAX) {
db_->set_error(_KCCODELINE_, Error::BROKEN, "free block in the ch ain"); db_->set_error(_KCCODELINE_, Error::BROKEN, "free block in the ch ain");
db_->report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz= %lld", db_->report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz= %lld",
(long long)db_->psiz_, (long long)rec.off, (long long )db_->file_.size()); (long long)db_->psiz_, (long long)rec.off, (long long )db_->file_.size());
return false; return false;
} }
uint32_t tpivot = db_->linear_ ? pivot : uint32_t tpivot = db_->linear_ ? pivot :
skipping to change at line 391 skipping to change at line 354
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (off_ < 1) { if (off_ < 1) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
bool err = false; bool err = false;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[RECBUFSIZ];
if (step_impl(&rec, rbuf, 1)) { if (step_impl(&rec, rbuf, 1)) {
delete[] rec.bbuf; delete[] rec.bbuf;
} else { } else {
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Step the cursor to the previous record. * Step the cursor to the previous record.
* @note This is a dummy implementation for compatibility. * @note This is a dummy implementation for compatibility.
skipping to change at line 483 skipping to change at line 446
* Status flags. * Status flags.
*/ */
enum Flag { enum Flag {
FOPEN = 1 << 0, ///< whether opened FOPEN = 1 << 0, ///< whether opened
FFATAL = 1 << 1 ///< whether with fatal error FFATAL = 1 << 1 ///< whether with fatal error
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit HashDB() : explicit HashDB() :
mlock_(), rlock_(HDBRLOCKSLOT), flock_(), atlock_(), error_(), mlock_(), rlock_(RLOCKSLOT), flock_(), atlock_(), error_(),
logger_(NULL), logkinds_(0), mtrigger_(NULL), logger_(NULL), logkinds_(0), mtrigger_(NULL),
omode_(0), writer_(false), autotran_(false), autosync_(false), reorg_(f alse), trim_(false), omode_(0), writer_(false), autotran_(false), autosync_(false), reorg_(f alse), trim_(false),
file_(), fbp_(), curs_(), path_(""), file_(), fbp_(), curs_(), path_(""),
libver_(0), librev_(0), fmtver_(0), chksum_(0), type_(TYPEHASH), libver_(0), librev_(0), fmtver_(0), chksum_(0), type_(TYPEHASH),
apow_(HDBDEFAPOW), fpow_(HDBDEFFPOW), opts_(0), bnum_(HDBDEFBNUM), apow_(DEFAPOW), fpow_(DEFFPOW), opts_(0), bnum_(DEFBNUM),
flags_(0), flagopen_(false), count_(0), lsiz_(0), psiz_(0), opaque_(), flags_(0), flagopen_(false), count_(0), lsiz_(0), psiz_(0), opaque_(),
msiz_(HDBDEFMSIZ), dfunit_(0), embcomp_(ZLIBRAWCOMP), msiz_(DEFMSIZ), dfunit_(0), embcomp_(ZLIBRAWCOMP),
align_(0), fbpnum_(0), width_(0), linear_(false), align_(0), fbpnum_(0), width_(0), linear_(false),
comp_(NULL), rhsiz_(0), boff_(0), roff_(0), dfcur_(0), frgcnt_(0), comp_(NULL), rhsiz_(0), boff_(0), roff_(0), dfcur_(0), frgcnt_(0),
tran_(false), trhard_(false), trfbp_(), trcount_(0), trsize_(0) { tran_(false), trhard_(false), trfbp_(), trcount_(0), trsize_(0) {
_assert_(true); _assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~HashDB() { virtual ~HashDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
if (!curs_.empty()) { if (!curs_.empty()) {
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->db_ = NULL; cur->db_ = NULL;
cit++; ++cit;
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 547 skipping to change at line 510
} }
if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, tru e)) { if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, tru e)) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
} }
bool err = false; bool err = false;
uint64_t hash = hash_record(kbuf, ksiz); uint64_t hash = hash_record(kbuf, ksiz);
uint32_t pivot = fold_hash(hash); uint32_t pivot = fold_hash(hash);
int64_t bidx = hash % bnum_; int64_t bidx = hash % bnum_;
size_t lidx = bidx % HDBRLOCKSLOT; size_t lidx = bidx % RLOCKSLOT;
if (writable) { if (writable) {
rlock_.lock_writer(lidx); rlock_.lock_writer(lidx);
} else { } else {
rlock_.lock_reader(lidx); rlock_.lock_reader(lidx);
} }
if (!accept_impl(kbuf, ksiz, visitor, bidx, pivot, false)) err = true; if (!accept_impl(kbuf, ksiz, visitor, bidx, pivot, false)) err = true;
rlock_.unlock(lidx); rlock_.unlock(lidx);
if (!err && dfunit_ > 0 && frgcnt_ >= dfunit_ && mlock_.promote()) { if (!err && dfunit_ > 0 && frgcnt_ >= dfunit_ && mlock_.promote()) {
int64_t unit = frgcnt_; int64_t unit = frgcnt_;
if (unit >= dfunit_) { if (unit >= dfunit_) {
if (unit > HDBDFRGMAX) unit = HDBDFRGMAX; if (unit > DFRGMAX) unit = DFRGMAX;
if (!defrag_impl(unit * HDBDFRGCEF)) err = true; if (!defrag_impl(unit * DFRGCEF)) err = true;
frgcnt_ -= unit; frgcnt_ -= unit;
} }
} }
mlock_.unlock(); mlock_.unlock();
return !err; return !err;
} }
/** /**
* Accept a visitor to multiple records at once. * Accept a visitor to multiple records at once.
* @param keys specifies a string vector of the keys. * @param keys specifies a string vector of the keys.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The operations for specified records are performed atomically an d other threads * @note The operations for specified records are performed atomically an d other threads
* accessing the same records are blocked. To avoid deadlock, any explic it database operation * accessing the same records are blocked. To avoid deadlock, any explic it database operation
* must not be performed in this function. * must not be performed in this function.
*/ */
bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor,
bool writable = true) { bool writable = true) {
_assert_(visitor); _assert_(visitor);
size_t knum = keys.size();
if (knum < 1) return true;
mlock_.lock_reader(); mlock_.lock_reader();
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (writable) { if (writable) {
if (!writer_) { if (!writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, tru e)) { if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, tru e)) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
} }
visitor->visit_before();
size_t knum = keys.size();
if (knum < 1) {
visitor->visit_after();
mlock_.unlock();
return true;
}
bool err = false; bool err = false;
struct RecordKey { struct RecordKey {
const char* kbuf; const char* kbuf;
size_t ksiz; size_t ksiz;
uint32_t pivot; uint32_t pivot;
uint64_t bidx; uint64_t bidx;
}; };
RecordKey* rkeys = new RecordKey[knum]; RecordKey* rkeys = new RecordKey[knum];
std::set<size_t> lidxs; std::set<size_t> lidxs;
for (size_t i = 0; i < knum; i++) { for (size_t i = 0; i < knum; i++) {
const std::string& key = keys[i]; const std::string& key = keys[i];
RecordKey* rkey = rkeys + i; RecordKey* rkey = rkeys + i;
rkey->kbuf = key.data(); rkey->kbuf = key.data();
rkey->ksiz = key.size(); rkey->ksiz = key.size();
uint64_t hash = hash_record(rkey->kbuf, rkey->ksiz); uint64_t hash = hash_record(rkey->kbuf, rkey->ksiz);
rkey->pivot = fold_hash(hash); rkey->pivot = fold_hash(hash);
rkey->bidx = hash % bnum_; rkey->bidx = hash % bnum_;
lidxs.insert(rkey->bidx % HDBRLOCKSLOT); lidxs.insert(rkey->bidx % RLOCKSLOT);
} }
std::set<size_t>::iterator lit = lidxs.begin(); std::set<size_t>::iterator lit = lidxs.begin();
std::set<size_t>::iterator litend = lidxs.end(); std::set<size_t>::iterator litend = lidxs.end();
while (lit != litend) { while (lit != litend) {
if (writable) { if (writable) {
rlock_.lock_writer(*lit); rlock_.lock_writer(*lit);
} else { } else {
rlock_.lock_reader(*lit); rlock_.lock_reader(*lit);
} }
lit++; ++lit;
} }
for (size_t i = 0; i < knum; i++) { for (size_t i = 0; i < knum; i++) {
RecordKey* rkey = rkeys + i; RecordKey* rkey = rkeys + i;
if (!accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->bidx, rkey->p ivot, false)) { if (!accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->bidx, rkey->p ivot, false)) {
err = true; err = true;
break; break;
} }
} }
lit = lidxs.begin(); lit = lidxs.begin();
litend = lidxs.end(); litend = lidxs.end();
while (lit != litend) { while (lit != litend) {
rlock_.unlock(*lit); rlock_.unlock(*lit);
lit++; ++lit;
} }
delete[] rkeys; delete[] rkeys;
if (!err && dfunit_ > 0 && frgcnt_ >= dfunit_ && mlock_.promote()) { if (!err && dfunit_ > 0 && frgcnt_ >= dfunit_ && mlock_.promote()) {
int64_t unit = frgcnt_; int64_t unit = frgcnt_;
if (unit >= dfunit_) { if (unit >= dfunit_) {
if (unit > HDBDFRGMAX) unit = HDBDFRGMAX; if (unit > DFRGMAX) unit = DFRGMAX;
if (!defrag_impl(unit * HDBDFRGCEF)) err = true; if (!defrag_impl(unit * DFRGCEF)) err = true;
frgcnt_ -= unit; frgcnt_ -= unit;
} }
} }
visitor->visit_after();
mlock_.unlock(); mlock_.unlock();
return !err; return !err;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed. * @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The whole iteration is performed atomically and other threads ar e blocked. To avoid * @note The whole iteration is performed atomically and other threads ar e blocked. To avoid
skipping to change at line 678 skipping to change at line 647
if (writable) { if (writable) {
if (!writer_) { if (!writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, tru e)) { if (!(flags_ & FOPEN) && !autotran_ && !tran_ && !set_flag(FOPEN, tru e)) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
} }
ScopedVisitor svis(visitor);
bool err = false; bool err = false;
if (!iterate_impl(visitor, checker)) err = true; if (!iterate_impl(visitor, checker)) err = true;
trigger_meta(MetaTrigger::ITERATE, "iterate"); trigger_meta(MetaTrigger::ITERATE, "iterate");
return !err; return !err;
} }
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
Error error() const { Error error() const {
skipping to change at line 821 skipping to change at line 791
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
if (!load_meta()) { if (!load_meta()) {
file_.close(); file_.close();
return false; return false;
} }
calc_meta(); calc_meta();
reorg_ = true; reorg_ = true;
} }
if (type_ == 0 || apow_ > HDBMAXAPOW || fpow_ > HDBMAXFPOW || if (type_ == 0 || apow_ > MAXAPOW || fpow_ > MAXFPOW ||
bnum_ < 1 || count_ < 0 || lsiz_ < roff_) { bnum_ < 1 || count_ < 0 || lsiz_ < roff_) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data"); set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data");
report(_KCCODELINE_, Logger::WARN, "type=0x%02X apow=%d fpow=%d bnum= %lld count=%lld" report(_KCCODELINE_, Logger::WARN, "type=0x%02X apow=%d fpow=%d bnum= %lld count=%lld"
" lsiz=%lld fsiz=%lld", (unsigned)type_, (int)apow_, (int)fpow _, (long long)bnum_, " lsiz=%lld fsiz=%lld", (unsigned)type_, (int)apow_, (int)fpow _, (long long)bnum_,
(long long)count_, (long long)lsiz_, (long long)file_.size()); (long long)count_, (long long)lsiz_, (long long)file_.size());
file_.close(); file_.close();
return false; return false;
} }
if (file_.size() < lsiz_) { if (file_.size() < lsiz_) {
set_error(_KCCODELINE_, Error::BROKEN, "inconsistent file size"); set_error(_KCCODELINE_, Error::BROKEN, "inconsistent file size");
skipping to change at line 935 skipping to change at line 905
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wcnt >= HDBLOCKBUSYLOOP) { if (wcnt >= LOCKBUSYLOOP) {
Thread::chill(); Thread::chill();
} else { } else {
Thread::yield(); Thread::yield();
wcnt++; wcnt++;
} }
} }
trhard_ = hard; trhard_ = hard;
if (!begin_transaction_impl()) { if (!begin_transaction_impl()) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
skipping to change at line 1028 skipping to change at line 998
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
disable_cursors(); disable_cursors();
if (!file_.truncate(HDBHEADSIZ)) { if (!file_.truncate(HEADSIZ)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
fbp_.clear(); fbp_.clear();
bool err = false; bool err = false;
reorg_ = false; reorg_ = false;
trim_ = false; trim_ = false;
flags_ = 0; flags_ = 0;
flagopen_ = false; flagopen_ = false;
count_ = 0; count_ = 0;
skipping to change at line 1200 skipping to change at line 1170
* @param apow the power of the alignment of record size. * @param apow the power of the alignment of record size.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_alignment(int8_t apow) { bool tune_alignment(int8_t apow) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
apow_ = apow >= 0 ? apow : HDBDEFAPOW; apow_ = apow >= 0 ? apow : DEFAPOW;
if (apow_ > HDBMAXAPOW) apow_ = HDBMAXAPOW; if (apow_ > MAXAPOW) apow_ = MAXAPOW;
return true; return true;
} }
/** /**
* Set the power of the capacity of the free block pool. * Set the power of the capacity of the free block pool.
* @param fpow the power of the capacity of the free block pool. * @param fpow the power of the capacity of the free block pool.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_fbp(int8_t fpow) { bool tune_fbp(int8_t fpow) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
fpow_ = fpow >= 0 ? fpow : HDBDEFFPOW; fpow_ = fpow >= 0 ? fpow : DEFFPOW;
if (fpow_ > HDBMAXFPOW) fpow_ = HDBMAXFPOW; if (fpow_ > MAXFPOW) fpow_ = MAXFPOW;
return true; return true;
} }
/** /**
* Set the optional features. * Set the optional features.
* @param opts the optional features by bitwise-or: HashDB::TSMALL to use 32-bit addressing, * @param opts the optional features by bitwise-or: HashDB::TSMALL to use 32-bit addressing,
* HashDB::TLINEAR to use linear collision chaining, HashDB::TCOMPRESS to compress each record. * HashDB::TLINEAR to use linear collision chaining, HashDB::TCOMPRESS to compress each record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_options(int8_t opts) { bool tune_options(int8_t opts) {
_assert_(true); _assert_(true);
skipping to change at line 1248 skipping to change at line 1218
* @param bnum the number of buckets of the hash table. * @param bnum the number of buckets of the hash table.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_buckets(int64_t bnum) { bool tune_buckets(int64_t bnum) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
bnum_ = bnum > 0 ? bnum : HDBDEFBNUM; bnum_ = bnum > 0 ? bnum : DEFBNUM;
if (bnum_ > INT16MAX) bnum_ = nearbyprime(bnum_); if (bnum_ > INT16MAX) bnum_ = nearbyprime(bnum_);
return true; return true;
} }
/** /**
* Set the size of the internal memory-mapped region. * Set the size of the internal memory-mapped region.
* @param msiz the size of the internal memory-mapped region. * @param msiz the size of the internal memory-mapped region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_map(int64_t msiz) { bool tune_map(int64_t msiz) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
msiz_ = msiz >= 0 ? msiz : HDBDEFMSIZ; msiz_ = msiz >= 0 ? msiz : DEFMSIZ;
return true; return true;
} }
/** /**
* Set the unit step number of auto defragmentation. * Set the unit step number of auto defragmentation.
* @param dfunit the unit step number of auto defragmentation. * @param dfunit the unit step number of auto defragmentation.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_defrag(int64_t dfunit) { bool tune_defrag(int64_t dfunit) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
skipping to change at line 1640 skipping to change at line 1610
bool reorganized() { bool reorganized() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return reorg_; return reorg_;
} }
private: private:
/** The offset of the library version. */
static const int64_t MOFFLIBVER = 4;
/** The offset of the library revision. */
static const int64_t MOFFLIBREV = 5;
/** The offset of the format revision. */
static const int64_t MOFFFMTVER = 6;
/** The offset of the module checksum. */
static const int64_t MOFFCHKSUM = 7;
/** The offset of the database type. */
static const int64_t MOFFTYPE = 8;
/** The offset of the alignment power. */
static const int64_t MOFFAPOW = 9;
/** The offset of the free block pool power. */
static const int64_t MOFFFPOW = 10;
/** The offset of the options. */
static const int64_t MOFFOPTS = 11;
/** The offset of the bucket number. */
static const int64_t MOFFBNUM = 16;
/** The offset of the status flags. */
static const int64_t MOFFFLAGS = 24;
/** The offset of the record number. */
static const int64_t MOFFCOUNT = 32;
/** The offset of the file size. */
static const int64_t MOFFSIZE = 40;
/** The offset of the opaque data. */
static const int64_t MOFFOPAQUE = 48;
/** The size of the header. */
static const int64_t HEADSIZ = 64;
/** The width of the free block. */
static const int32_t FBPWIDTH = 6;
/** The large width of the record address. */
static const int32_t WIDTHLARGE = 6;
/** The small width of the record address. */
static const int32_t WIDTHSMALL = 4;
/** The size of the record buffer. */
static const size_t RECBUFSIZ = 48;
/** The size of the IO buffer. */
static const size_t IOBUFSIZ = 1024;
/** The number of slots of the record lock. */
static const int32_t RLOCKSLOT = 1024;
/** The default alignment power. */
static const uint8_t DEFAPOW = 3;
/** The maximum alignment power. */
static const uint8_t MAXAPOW = 15;
/** The default free block pool power. */
static const uint8_t DEFFPOW = 10;
/** The maximum free block pool power. */
static const uint8_t MAXFPOW = 20;
/** The default bucket number. */
static const int64_t DEFBNUM = 1048583LL;
/** The default size of the memory-mapped region. */
static const int64_t DEFMSIZ = 64LL << 20;
/** The magic data for record. */
static const uint8_t RECMAGIC = 0xcc;
/** The magic data for padding. */
static const uint8_t PADMAGIC = 0xee;
/** The magic data for free block. */
static const uint8_t FBMAGIC = 0xdd;
/** The maximum unit of auto defragmentation. */
static const int32_t DFRGMAX = 512;
/** The coefficient of auto defragmentation. */
static const int32_t DFRGCEF = 2;
/** The checking width for record salvage. */
static const int64_t SLVGWIDTH = 1LL << 20;
/** The threshold of busy loop and sleep for locking. */
static const uint32_t LOCKBUSYLOOP = 8192;
/** /**
* Record data. * Record data.
*/ */
struct Record { struct Record {
int64_t off; ///< offset int64_t off; ///< offset
size_t rsiz; ///< whole size size_t rsiz; ///< whole size
size_t psiz; ///< size of the padding size_t psiz; ///< size of the padding
size_t ksiz; ///< size of the key size_t ksiz; ///< size of the key
size_t vsiz; ///< size of the value size_t vsiz; ///< size of the value
int64_t left; ///< address of the left child rec ord int64_t left; ///< address of the left child rec ord
skipping to change at line 1685 skipping to change at line 1721
bool operator ()(const FreeBlock& a, const FreeBlock& b) const { bool operator ()(const FreeBlock& a, const FreeBlock& b) const {
_assert_(true); _assert_(true);
return a.off < b.off; return a.off < b.off;
} }
}; };
/** /**
* Repeating visitor. * Repeating visitor.
*/ */
class Repeater : public Visitor { class Repeater : public Visitor {
public: public:
/** constructor */
explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(v siz) { explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(v siz) {
_assert_(vbuf); _assert_(vbuf);
} }
private: private:
/** visit a record */
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp ); _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp );
*sp = vsiz_; *sp = vsiz_;
return vbuf_; return vbuf_;
} }
const char* vbuf_; const char* vbuf_;
size_t vsiz_; size_t vsiz_;
}; };
/** /**
* Scoped visiotor.
*/
class ScopedVisitor {
public:
/** constructor */
ScopedVisitor(Visitor* visitor) : visitor_(visitor) {
_assert_(visitor);
visitor_->visit_before();
}
/** destructor */
~ScopedVisitor() {
_assert_(true);
visitor_->visit_after();
}
private:
Visitor* visitor_; ///< visitor
};
/**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param bidx the bucket index. * @param bidx the bucket index.
* @param pivot the second hash value. * @param pivot the second hash value.
@ @param isiter true for iterator use, or false for direct use. @ @param isiter true for iterator use, or false for direct use.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor,
int64_t bidx, uint32_t pivot, bool isiter) { int64_t bidx, uint32_t pivot, bool isiter) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor && bidx >= 0); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor && bidx >= 0);
int64_t top = get_bucket(bidx); int64_t top = get_bucket(bidx);
int64_t off = top; int64_t off = top;
if (off < 0) return false; if (off < 0) return false;
enum { DIREMPTY, DIRLEFT, DIRRIGHT, DIRMIXED } entdir = DIREMPTY; enum { DIREMPTY, DIRLEFT, DIRRIGHT, DIRMIXED } entdir = DIREMPTY;
int64_t entoff = 0; int64_t entoff = 0;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[RECBUFSIZ];
while (off > 0) { while (off > 0) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16MAX) { if (rec.psiz == UINT16MAX) {
set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain"); set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain");
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long long)psiz_, (long long)rec.off, (long long)file_.size( )); (long long)psiz_, (long long)rec.off, (long long)file_.size( ));
return false; return false;
} }
uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, r ec.ksiz)); uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, r ec.ksiz));
skipping to change at line 2023 skipping to change at line 2079
bool iterate_impl(Visitor* visitor, ProgressChecker* checker) { bool iterate_impl(Visitor* visitor, ProgressChecker* checker) {
_assert_(visitor); _assert_(visitor);
int64_t allcnt = count_; int64_t allcnt = count_;
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
int64_t off = roff_; int64_t off = roff_;
int64_t end = lsiz_; int64_t end = lsiz_;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[RECBUFSIZ];
int64_t curcnt = 0; int64_t curcnt = 0;
while (off > 0 && off < end) { while (off > 0 && off < end) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16MAX) { if (rec.psiz == UINT16MAX) {
off += rec.rsiz; off += rec.rsiz;
} else { } else {
if (!rec.vbuf && !read_record_body(&rec)) { if (!rec.vbuf && !read_record_body(&rec)) {
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
skipping to change at line 2184 skipping to change at line 2240
} }
/** /**
* Perform defragmentation. * Perform defragmentation.
* @param step the number of steps. * @param step the number of steps.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool defrag_impl(int64_t step) { bool defrag_impl(int64_t step) {
_assert_(step >= 0); _assert_(step >= 0);
int64_t end = lsiz_; int64_t end = lsiz_;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[RECBUFSIZ];
while (true) { while (true) {
if (dfcur_ >= end) { if (dfcur_ >= end) {
dfcur_ = roff_; dfcur_ = roff_;
return true; return true;
} }
if (step-- < 1) return true; if (step-- < 1) return true;
rec.off = dfcur_; rec.off = dfcur_;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16MAX) break; if (rec.psiz == UINT16MAX) break;
delete[] rec.bbuf; delete[] rec.bbuf;
skipping to change at line 2267 skipping to change at line 2323
} }
return true; return true;
} }
/** /**
* Calculate meta data with saved ones. * Calculate meta data with saved ones.
*/ */
void calc_meta() { void calc_meta() {
_assert_(true); _assert_(true);
align_ = 1 << apow_; align_ = 1 << apow_;
fbpnum_ = fpow_ > 0 ? 1 << fpow_ : 0; fbpnum_ = fpow_ > 0 ? 1 << fpow_ : 0;
width_ = (opts_ & TSMALL) ? HDBWIDTHSMALL : HDBWIDTHLARGE; //width_ = (opts_ & TSMALL) ? WIDTHSMALL : WIDTHLARGE;
width_ = (opts_ & TSMALL) ? sizeof(uint32_t) : sizeof(uint32_t) + 2;
linear_ = (opts_ & TLINEAR) ? true : false; linear_ = (opts_ & TLINEAR) ? true : false;
comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL; comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL;
rhsiz_ = sizeof(uint16_t) + sizeof(uint8_t) * 2; rhsiz_ = sizeof(uint16_t) + sizeof(uint8_t) * 2;
rhsiz_ += linear_ ? width_ : width_ * 2; rhsiz_ += linear_ ? width_ : width_ * 2;
boff_ = HDBHEADSIZ + HDBFBPWIDTH * fbpnum_; boff_ = HEADSIZ + FBPWIDTH * fbpnum_;
if (fbpnum_ > 0) boff_ += width_ * 2 + sizeof(uint8_t) * 2; if (fbpnum_ > 0) boff_ += width_ * 2 + sizeof(uint8_t) * 2;
roff_ = boff_ + width_ * bnum_; roff_ = boff_ + width_ * bnum_;
int64_t rem = roff_ % align_; int64_t rem = roff_ % align_;
if (rem > 0) roff_ += align_ - rem; if (rem > 0) roff_ += align_ - rem;
dfcur_ = roff_; dfcur_ = roff_;
frgcnt_ = 0; frgcnt_ = 0;
tran_ = false; tran_ = false;
} }
/** /**
* Calculate the module checksum. * Calculate the module checksum.
* @return the module checksum. * @return the module checksum.
*/ */
uint8_t calc_checksum() { uint8_t calc_checksum() {
_assert_(true); _assert_(true);
const char* kbuf = HDBCHKSUMSEED; const char* kbuf = KCHDBCHKSUMSEED;
size_t ksiz = sizeof(HDBCHKSUMSEED) - 1; size_t ksiz = sizeof(KCHDBCHKSUMSEED) - 1;
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (comp_) { if (comp_) {
zbuf = comp_->compress(kbuf, ksiz, &zsiz); zbuf = comp_->compress(kbuf, ksiz, &zsiz);
if (!zbuf) return 0; if (!zbuf) return 0;
kbuf = zbuf; kbuf = zbuf;
ksiz = zsiz; ksiz = zsiz;
} }
uint32_t hash = fold_hash(hash_record(kbuf, ksiz)); uint32_t hash = fold_hash(hash_record(kbuf, ksiz));
delete[] zbuf; delete[] zbuf;
return (hash >> 24) ^ (hash >> 16) ^ (hash >> 8) ^ (hash >> 0); return (hash >> 24) ^ (hash >> 16) ^ (hash >> 8) ^ (hash >> 0);
} }
/** /**
* Dump the meta data into the file. * Dump the meta data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_meta() { bool dump_meta() {
_assert_(true); _assert_(true);
char head[HDBHEADSIZ]; char head[HEADSIZ];
std::memset(head, 0, sizeof(head)); std::memset(head, 0, sizeof(head));
std::memcpy(head, HDBMAGICDATA, sizeof(HDBMAGICDATA)); std::memcpy(head, KCHDBMAGICDATA, sizeof(KCHDBMAGICDATA));
std::memcpy(head + HDBMOFFLIBVER, &libver_, sizeof(libver_)); std::memcpy(head + MOFFLIBVER, &libver_, sizeof(libver_));
std::memcpy(head + HDBMOFFLIBREV, &librev_, sizeof(librev_)); std::memcpy(head + MOFFLIBREV, &librev_, sizeof(librev_));
std::memcpy(head + HDBMOFFFMTVER, &fmtver_, sizeof(fmtver_)); std::memcpy(head + MOFFFMTVER, &fmtver_, sizeof(fmtver_));
std::memcpy(head + HDBMOFFCHKSUM, &chksum_, sizeof(chksum_)); std::memcpy(head + MOFFCHKSUM, &chksum_, sizeof(chksum_));
std::memcpy(head + HDBMOFFTYPE, &type_, sizeof(type_)); std::memcpy(head + MOFFTYPE, &type_, sizeof(type_));
std::memcpy(head + HDBMOFFAPOW, &apow_, sizeof(apow_)); std::memcpy(head + MOFFAPOW, &apow_, sizeof(apow_));
std::memcpy(head + HDBMOFFFPOW, &fpow_, sizeof(fpow_)); std::memcpy(head + MOFFFPOW, &fpow_, sizeof(fpow_));
std::memcpy(head + HDBMOFFOPTS, &opts_, sizeof(opts_)); std::memcpy(head + MOFFOPTS, &opts_, sizeof(opts_));
uint64_t num = hton64(bnum_); uint64_t num = hton64(bnum_);
std::memcpy(head + HDBMOFFBNUM, &num, sizeof(num)); std::memcpy(head + MOFFBNUM, &num, sizeof(num));
if (!flagopen_) flags_ &= ~FOPEN; if (!flagopen_) flags_ &= ~FOPEN;
std::memcpy(head + HDBMOFFFLAGS, &flags_, sizeof(flags_)); std::memcpy(head + MOFFFLAGS, &flags_, sizeof(flags_));
num = hton64(count_); num = hton64(count_);
std::memcpy(head + HDBMOFFCOUNT, &num, sizeof(num)); std::memcpy(head + MOFFCOUNT, &num, sizeof(num));
num = hton64(lsiz_); num = hton64(lsiz_);
std::memcpy(head + HDBMOFFSIZE, &num, sizeof(num)); std::memcpy(head + MOFFSIZE, &num, sizeof(num));
std::memcpy(head + HDBMOFFOPAQUE, opaque_, sizeof(opaque_)); std::memcpy(head + MOFFOPAQUE, opaque_, sizeof(opaque_));
if (!file_.write(0, head, sizeof(head))) { if (!file_.write(0, head, sizeof(head))) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
trcount_ = count_; trcount_ = count_;
trsize_ = lsiz_; trsize_ = lsiz_;
return true; return true;
} }
/** /**
* Dump the meta data into the file. * Dump the meta data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_auto_meta() { bool dump_auto_meta() {
_assert_(true); _assert_(true);
const int64_t hsiz = HDBMOFFOPAQUE - HDBMOFFCOUNT; const int64_t hsiz = MOFFOPAQUE - MOFFCOUNT;
char head[hsiz]; char head[hsiz];
std::memset(head, 0, hsiz); std::memset(head, 0, hsiz);
uint64_t num = hton64(count_); uint64_t num = hton64(count_);
std::memcpy(head, &num, sizeof(num)); std::memcpy(head, &num, sizeof(num));
num = hton64(lsiz_); num = hton64(lsiz_);
std::memcpy(head + HDBMOFFSIZE - HDBMOFFCOUNT, &num, sizeof(num)); std::memcpy(head + MOFFSIZE - MOFFCOUNT, &num, sizeof(num));
if (!file_.write_fast(HDBMOFFCOUNT, head, sizeof(head))) { if (!file_.write_fast(MOFFCOUNT, head, sizeof(head))) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
trcount_ = count_; trcount_ = count_;
trsize_ = lsiz_; trsize_ = lsiz_;
return true; return true;
} }
/** /**
* Dump the opaque data into the file. * Dump the opaque data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_opaque() { bool dump_opaque() {
_assert_(true); _assert_(true);
if (!file_.write_fast(HDBMOFFOPAQUE, opaque_, sizeof(opaque_))) { if (!file_.write_fast(MOFFOPAQUE, opaque_, sizeof(opaque_))) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
return true; return true;
} }
/** /**
* Load the meta data from the file. * Load the meta data from the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_meta() { bool load_meta() {
_assert_(true); _assert_(true);
char head[HDBHEADSIZ]; char head[HEADSIZ];
if (file_.size() < (int64_t)sizeof(head)) { if (file_.size() < (int64_t)sizeof(head)) {
set_error(_KCCODELINE_, Error::INVALID, "missing magic data of the fi le"); set_error(_KCCODELINE_, Error::INVALID, "missing magic data of the fi le");
return false; return false;
} }
if (!file_.read(0, head, sizeof(head))) { if (!file_.read(0, head, sizeof(head))) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long long)psiz_, (long long)0, (long long)file_.size()); (long long)psiz_, (long long)0, (long long)file_.size());
return false; return false;
} }
if (std::memcmp(head, HDBMAGICDATA, sizeof(HDBMAGICDATA))) { if (std::memcmp(head, KCHDBMAGICDATA, sizeof(KCHDBMAGICDATA))) {
set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of the fi le"); set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of the fi le");
return false; return false;
} }
std::memcpy(&libver_, head + HDBMOFFLIBVER, sizeof(libver_)); std::memcpy(&libver_, head + MOFFLIBVER, sizeof(libver_));
std::memcpy(&librev_, head + HDBMOFFLIBREV, sizeof(librev_)); std::memcpy(&librev_, head + MOFFLIBREV, sizeof(librev_));
std::memcpy(&fmtver_, head + HDBMOFFFMTVER, sizeof(fmtver_)); std::memcpy(&fmtver_, head + MOFFFMTVER, sizeof(fmtver_));
std::memcpy(&chksum_, head + HDBMOFFCHKSUM, sizeof(chksum_)); std::memcpy(&chksum_, head + MOFFCHKSUM, sizeof(chksum_));
std::memcpy(&type_, head + HDBMOFFTYPE, sizeof(type_)); std::memcpy(&type_, head + MOFFTYPE, sizeof(type_));
std::memcpy(&apow_, head + HDBMOFFAPOW, sizeof(apow_)); std::memcpy(&apow_, head + MOFFAPOW, sizeof(apow_));
std::memcpy(&fpow_, head + HDBMOFFFPOW, sizeof(fpow_)); std::memcpy(&fpow_, head + MOFFFPOW, sizeof(fpow_));
std::memcpy(&opts_, head + HDBMOFFOPTS, sizeof(opts_)); std::memcpy(&opts_, head + MOFFOPTS, sizeof(opts_));
uint64_t num; uint64_t num;
std::memcpy(&num, head + HDBMOFFBNUM, sizeof(num)); std::memcpy(&num, head + MOFFBNUM, sizeof(num));
bnum_ = ntoh64(num); bnum_ = ntoh64(num);
std::memcpy(&flags_, head + HDBMOFFFLAGS, sizeof(flags_)); std::memcpy(&flags_, head + MOFFFLAGS, sizeof(flags_));
flagopen_ = flags_ & FOPEN; flagopen_ = flags_ & FOPEN;
std::memcpy(&num, head + HDBMOFFCOUNT, sizeof(num)); std::memcpy(&num, head + MOFFCOUNT, sizeof(num));
count_ = ntoh64(num); count_ = ntoh64(num);
std::memcpy(&num, head + HDBMOFFSIZE, sizeof(num)); std::memcpy(&num, head + MOFFSIZE, sizeof(num));
lsiz_ = ntoh64(num); lsiz_ = ntoh64(num);
psiz_ = lsiz_; psiz_ = lsiz_;
std::memcpy(opaque_, head + HDBMOFFOPAQUE, sizeof(opaque_)); std::memcpy(opaque_, head + MOFFOPAQUE, sizeof(opaque_));
trcount_ = count_; trcount_ = count_;
trsize_ = lsiz_; trsize_ = lsiz_;
return true; return true;
} }
/** /**
* Set a status flag. * Set a status flag.
* @param flag the flag kind. * @param flag the flag kind.
* @param sign whether to set or unset. * @param sign whether to set or unset.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool set_flag(uint8_t flag, bool sign) { bool set_flag(uint8_t flag, bool sign) {
_assert_(true); _assert_(true);
uint8_t flags; uint8_t flags;
if (!file_.read(HDBMOFFFLAGS, &flags, sizeof(flags))) { if (!file_.read(MOFFFLAGS, &flags, sizeof(flags))) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long long)psiz_, (long long)HDBMOFFFLAGS, (long long)file_.si ze()); (long long)psiz_, (long long)MOFFFLAGS, (long long)file_.size( ));
return false; return false;
} }
if (sign) { if (sign) {
flags |= flag; flags |= flag;
} else { } else {
flags &= ~flag; flags &= ~flag;
} }
if (!file_.write(HDBMOFFFLAGS, &flags, sizeof(flags))) { if (!file_.write(MOFFFLAGS, &flags, sizeof(flags))) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
flags_ = flags; flags_ = flags;
return true; return true;
} }
/** /**
* Reorganize the whole file. * Reorganize the whole file.
* @param path the path of the database file. * @param path the path of the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 2455 skipping to change at line 2513
_assert_(true); _assert_(true);
bool err = false; bool err = false;
HashDB db; HashDB db;
db.tune_type(type_); db.tune_type(type_);
db.tune_alignment(apow_); db.tune_alignment(apow_);
db.tune_fbp(fpow_); db.tune_fbp(fpow_);
db.tune_options(opts_); db.tune_options(opts_);
db.tune_buckets(bnum_); db.tune_buckets(bnum_);
db.tune_map(msiz_); db.tune_map(msiz_);
if (embcomp_) db.tune_compressor(embcomp_); if (embcomp_) db.tune_compressor(embcomp_);
const std::string& npath = path + File::EXTCHR + HDBTMPPATHEXT; const std::string& npath = path + File::EXTCHR + KCHDBTMPPATHEXT;
if (db.open(npath, OWRITER | OCREATE | OTRUNCATE)) { if (db.open(npath, OWRITER | OCREATE | OTRUNCATE)) {
report(_KCCODELINE_, Logger::WARN, "reorganizing the database"); report(_KCCODELINE_, Logger::WARN, "reorganizing the database");
lsiz_ = file_.size(); lsiz_ = file_.size();
psiz_ = lsiz_; psiz_ = lsiz_;
if (copy_records(&db)) { if (copy_records(&db)) {
if (db.close()) { if (db.close()) {
if (!File::rename(npath, path)) { if (!File::rename(npath, path)) {
set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destinatio n failed"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destinatio n failed");
err = true; err = true;
} }
skipping to change at line 2493 skipping to change at line 2551
* @param dest the destination database. * @param dest the destination database.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool copy_records(HashDB* dest) { bool copy_records(HashDB* dest) {
_assert_(dest); _assert_(dest);
Logger* logger = logger_; Logger* logger = logger_;
logger_ = NULL; logger_ = NULL;
int64_t off = roff_; int64_t off = roff_;
int64_t end = psiz_; int64_t end = psiz_;
Record rec, nrec; Record rec, nrec;
char rbuf[HDBRECBUFSIZ], nbuf[HDBRECBUFSIZ]; char rbuf[RECBUFSIZ], nbuf[RECBUFSIZ];
while (off > 0 && off < end) { while (off > 0 && off < end) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) { if (!read_record(&rec, rbuf)) {
int64_t checkend = off + HDBSLVGWIDTH; int64_t checkend = off + SLVGWIDTH;
if (checkend > end - (int64_t)rhsiz_) checkend = end - rhsiz_; if (checkend > end - (int64_t)rhsiz_) checkend = end - rhsiz_;
bool hit = false; bool hit = false;
for (off += rhsiz_; off < checkend; off++) { for (off += rhsiz_; off < checkend; off++) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) continue; if (!read_record(&rec, rbuf)) continue;
if ((int64_t)rec.rsiz > HDBSLVGWIDTH || rec.off + (int64_t)rec.rs iz >= checkend) { if ((int64_t)rec.rsiz > SLVGWIDTH || rec.off + (int64_t)rec.rsiz >= checkend) {
delete[] rec.bbuf; delete[] rec.bbuf;
continue; continue;
} }
if (rec.psiz != UINT16MAX && !rec.vbuf && !read_record_body(&rec) ) { if (rec.psiz != UINT16MAX && !rec.vbuf && !read_record_body(&rec) ) {
delete[] rec.bbuf; delete[] rec.bbuf;
continue; continue;
} }
delete[] rec.bbuf; delete[] rec.bbuf;
nrec.off = off + rec.rsiz; nrec.off = off + rec.rsiz;
if (!read_record(&nrec, nbuf)) continue; if (!read_record(&nrec, nbuf)) continue;
if ((int64_t)nrec.rsiz > HDBSLVGWIDTH || nrec.off + (int64_t)nrec .rsiz >= checkend) { if ((int64_t)nrec.rsiz > SLVGWIDTH || nrec.off + (int64_t)nrec.rs iz >= checkend) {
delete[] nrec.bbuf; delete[] nrec.bbuf;
continue; continue;
} }
if (nrec.psiz != UINT16MAX && !nrec.vbuf && !read_record_body(&nr ec)) { if (nrec.psiz != UINT16MAX && !nrec.vbuf && !read_record_body(&nr ec)) {
delete[] nrec.bbuf; delete[] nrec.bbuf;
continue; continue;
} }
delete[] nrec.bbuf; delete[] nrec.bbuf;
hit = true; hit = true;
break; break;
skipping to change at line 2711 skipping to change at line 2769
*/ */
bool read_record(Record* rec, char* rbuf) { bool read_record(Record* rec, char* rbuf) {
_assert_(rec && rbuf); _assert_(rec && rbuf);
if (rec->off < roff_) { if (rec->off < roff_) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid record offset"); set_error(_KCCODELINE_, Error::BROKEN, "invalid record offset");
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long long)psiz_, (long long)rec->off, (long long)file_.size() ); (long long)psiz_, (long long)rec->off, (long long)file_.size() );
return false; return false;
} }
size_t rsiz = psiz_ - rec->off; size_t rsiz = psiz_ - rec->off;
if (rsiz > HDBRECBUFSIZ) { if (rsiz > RECBUFSIZ) {
rsiz = HDBRECBUFSIZ; rsiz = RECBUFSIZ;
} else { } else {
if (rsiz < rhsiz_) { if (rsiz < rhsiz_) {
set_error(_KCCODELINE_, Error::BROKEN, "too short record region"); set_error(_KCCODELINE_, Error::BROKEN, "too short record region");
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs iz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs iz=%lld",
(long long)psiz_, (long long)rec->off, (long long)rsiz, (lon g long)file_.size()); (long long)psiz_, (long long)rec->off, (long long)rsiz, (lon g long)file_.size());
return false; return false;
} }
rsiz = rhsiz_; rsiz = rhsiz_;
} }
if (!file_.read_fast(rec->off, rbuf, rsiz)) { if (!file_.read_fast(rec->off, rbuf, rsiz)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz =%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz =%lld",
(long long)psiz_, (long long)rec->off, (long long)rsiz, (long long)file_.size()); (long long)psiz_, (long long)rec->off, (long long)rsiz, (long long)file_.size());
return false; return false;
} }
const char* rp = rbuf; const char* rp = rbuf;
uint16_t snum; uint16_t snum;
if (*(uint8_t*)rp == HDBRECMAGIC) { if (*(uint8_t*)rp == RECMAGIC) {
((uint8_t*)&snum)[0] = 0; ((uint8_t*)&snum)[0] = 0;
((uint8_t*)&snum)[1] = *(uint8_t*)(rp + 1); ((uint8_t*)&snum)[1] = *(uint8_t*)(rp + 1);
} else if (*(uint8_t*)rp >= 0x80) { } else if (*(uint8_t*)rp >= 0x80) {
if (*(uint8_t*)(rp++) != HDBFBMAGIC || *(uint8_t*)(rp++) != HDBFBMAGI C) { if (*(uint8_t*)(rp++) != FBMAGIC || *(uint8_t*)(rp++) != FBMAGIC) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a fre e block"); set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a fre e block");
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs iz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs iz=%lld",
(long long)psiz_, (long long)rec->off, (long long)rsiz, (lon g long)file_.size()); (long long)psiz_, (long long)rec->off, (long long)rsiz, (lon g long)file_.size());
report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
rec->rsiz = readfixnum(rp, width_) << apow_; rec->rsiz = readfixnum(rp, width_) << apow_;
rp += width_; rp += width_;
if (*(uint8_t*)(rp++) != HDBPADMAGIC || *(uint8_t*)(rp++) != HDBPADMA GIC) { if (*(uint8_t*)(rp++) != PADMAGIC || *(uint8_t*)(rp++) != PADMAGIC) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a fre e block"); set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a fre e block");
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs iz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs iz=%lld",
(long long)psiz_, (long long)rec->off, (long long)rsiz, (lon g long)file_.size()); (long long)psiz_, (long long)rec->off, (long long)rsiz, (lon g long)file_.size());
report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
if (rec->rsiz < rhsiz_) { if (rec->rsiz < rhsiz_) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid size of a free bloc k"); set_error(_KCCODELINE_, Error::BROKEN, "invalid size of a free bloc k");
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs iz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs iz=%lld",
(long long)psiz_, (long long)rec->off, (long long)rsiz, (lon g long)file_.size()); (long long)psiz_, (long long)rec->off, (long long)rsiz, (lon g long)file_.size());
skipping to change at line 2829 skipping to change at line 2887
rec->bbuf = NULL; rec->bbuf = NULL;
if (rsiz >= rec->ksiz) { if (rsiz >= rec->ksiz) {
rec->kbuf = rp; rec->kbuf = rp;
rp += rec->ksiz; rp += rec->ksiz;
rsiz -= rec->ksiz; rsiz -= rec->ksiz;
if (rsiz >= rec->vsiz) { if (rsiz >= rec->vsiz) {
rec->vbuf = rp; rec->vbuf = rp;
if (rec->psiz > 0) { if (rec->psiz > 0) {
rp += rec->vsiz; rp += rec->vsiz;
rsiz -= rec->vsiz; rsiz -= rec->vsiz;
if (rsiz > 0 && *(uint8_t*)rp != HDBPADMAGIC) { if (rsiz > 0 && *(uint8_t*)rp != PADMAGIC) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a record"); set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a record");
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%ll d fsiz=%lld" report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%ll d fsiz=%lld"
" snum=%04X", (long long)psiz_, (long long)rec->off, (lo ng long)rsiz, " snum=%04X", (long long)psiz_, (long long)rec->off, (lo ng long)rsiz,
(long long)file_.size(), snum); (long long)file_.size(), snum);
report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
} }
} }
} else { } else {
skipping to change at line 2868 skipping to change at line 2926
size_t bsiz = rec->ksiz + rec->vsiz; size_t bsiz = rec->ksiz + rec->vsiz;
if (rec->psiz > 0) bsiz++; if (rec->psiz > 0) bsiz++;
char* bbuf = new char[bsiz]; char* bbuf = new char[bsiz];
if (!file_.read_fast(rec->boff, bbuf, bsiz)) { if (!file_.read_fast(rec->boff, bbuf, bsiz)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long long)psiz_, (long long)rec->boff, (long long)file_.size( )); (long long)psiz_, (long long)rec->boff, (long long)file_.size( ));
delete[] bbuf; delete[] bbuf;
return false; return false;
} }
if (rec->psiz > 0 && ((uint8_t*)bbuf)[bsiz-1] != HDBPADMAGIC) { if (rec->psiz > 0 && ((uint8_t*)bbuf)[bsiz-1] != PADMAGIC) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a recor d"); set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a recor d");
report_binary(_KCCODELINE_, Logger::WARN, "bbuf", bbuf, bsiz); report_binary(_KCCODELINE_, Logger::WARN, "bbuf", bbuf, bsiz);
delete[] bbuf; delete[] bbuf;
return false; return false;
} }
rec->bbuf = bbuf; rec->bbuf = bbuf;
rec->kbuf = rec->bbuf; rec->kbuf = rec->bbuf;
rec->vbuf = rec->bbuf + rec->ksiz; rec->vbuf = rec->bbuf + rec->ksiz;
return true; return true;
} }
/** /**
* Write a record into the file. * Write a record into the file.
* @param rec the record structure. * @param rec the record structure.
* @param over true for overwriting, or false for new record. * @param over true for overwriting, or false for new record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool write_record(Record* rec, bool over) { bool write_record(Record* rec, bool over) {
_assert_(rec); _assert_(rec);
char stack[HDBIOBUFSIZ]; char stack[IOBUFSIZ];
char* rbuf = rec->rsiz > sizeof(stack) ? new char[rec->rsiz] : stack; char* rbuf = rec->rsiz > sizeof(stack) ? new char[rec->rsiz] : stack;
char* wp = rbuf; char* wp = rbuf;
uint16_t snum = hton16(rec->psiz); uint16_t snum = hton16(rec->psiz);
std::memcpy(wp, &snum, sizeof(snum)); std::memcpy(wp, &snum, sizeof(snum));
if (rec->psiz < 0x100) *wp = HDBRECMAGIC; if (rec->psiz < 0x100) *wp = RECMAGIC;
wp += sizeof(snum); wp += sizeof(snum);
writefixnum(wp, rec->left >> apow_, width_); writefixnum(wp, rec->left >> apow_, width_);
wp += width_; wp += width_;
if (!linear_) { if (!linear_) {
writefixnum(wp, rec->right >> apow_, width_); writefixnum(wp, rec->right >> apow_, width_);
wp += width_; wp += width_;
} }
wp += writevarnum(wp, rec->ksiz); wp += writevarnum(wp, rec->ksiz);
wp += writevarnum(wp, rec->vsiz); wp += writevarnum(wp, rec->vsiz);
std::memcpy(wp, rec->kbuf, rec->ksiz); std::memcpy(wp, rec->kbuf, rec->ksiz);
wp += rec->ksiz; wp += rec->ksiz;
std::memcpy(wp, rec->vbuf, rec->vsiz); std::memcpy(wp, rec->vbuf, rec->vsiz);
wp += rec->vsiz; wp += rec->vsiz;
if (rec->psiz > 0) { if (rec->psiz > 0) {
std::memset(wp, 0, rec->psiz); std::memset(wp, 0, rec->psiz);
*wp = HDBPADMAGIC; *wp = PADMAGIC;
wp += rec->psiz; wp += rec->psiz;
} }
bool err = false; bool err = false;
if (over) { if (over) {
if (!file_.write_fast(rec->off, rbuf, rec->rsiz)) { if (!file_.write_fast(rec->off, rbuf, rec->rsiz)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
} else { } else {
if (!file_.write(rec->off, rbuf, rec->rsiz)) { if (!file_.write(rec->off, rbuf, rec->rsiz)) {
skipping to change at line 2939 skipping to change at line 2997
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool adjust_record(Record* rec) { bool adjust_record(Record* rec) {
_assert_(rec); _assert_(rec);
if (rec->psiz > (size_t)INT16MAX || rec->psiz > rec->rsiz / 2) { if (rec->psiz > (size_t)INT16MAX || rec->psiz > rec->rsiz / 2) {
size_t nsiz = (rec->psiz >> apow_) << apow_; size_t nsiz = (rec->psiz >> apow_) << apow_;
if (nsiz < rhsiz_) return true; if (nsiz < rhsiz_) return true;
rec->rsiz -= nsiz; rec->rsiz -= nsiz;
rec->psiz -= nsiz; rec->psiz -= nsiz;
int64_t noff = rec->off + rec->rsiz; int64_t noff = rec->off + rec->rsiz;
char nbuf[HDBRECBUFSIZ]; char nbuf[RECBUFSIZ];
if (!write_free_block(noff, nsiz, nbuf)) return false; if (!write_free_block(noff, nsiz, nbuf)) return false;
insert_free_block(noff, nsiz); insert_free_block(noff, nsiz);
} }
return true; return true;
} }
/** /**
* Calculate the size of a record. * Calculate the size of a record.
* @param ksiz the size of the key. * @param ksiz the size of the key.
* @param vsiz the size of the value. * @param vsiz the size of the value.
* @return the size of the record. * @return the size of the record.
skipping to change at line 3012 skipping to change at line 3070
int64_t off = get_bucket(bidx); int64_t off = get_bucket(bidx);
if (off < 0) return false; if (off < 0) return false;
if (off == orec->off) { if (off == orec->off) {
orec->off = dest; orec->off = dest;
if (!write_record(orec, true)) return false; if (!write_record(orec, true)) return false;
if (!set_bucket(bidx, dest)) return false; if (!set_bucket(bidx, dest)) return false;
return true; return true;
} }
int64_t entoff = 0; int64_t entoff = 0;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[RECBUFSIZ];
while (off > 0) { while (off > 0) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16MAX) { if (rec.psiz == UINT16MAX) {
set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain"); set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain");
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long long)psiz_, (long long)rec.off, (long long)file_.size( )); (long long)psiz_, (long long)rec.off, (long long)file_.size( ));
return false; return false;
} }
uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, r ec.ksiz)); uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, r ec.ksiz));
skipping to change at line 3070 skipping to change at line 3128
/** /**
* Write a free block into the file. * Write a free block into the file.
* @param off the offset of the free block. * @param off the offset of the free block.
* @param rsiz the size of the free block. * @param rsiz the size of the free block.
* @param rbuf the working buffer. * @param rbuf the working buffer.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool write_free_block(int64_t off, size_t rsiz, char* rbuf) { bool write_free_block(int64_t off, size_t rsiz, char* rbuf) {
_assert_(off >= 0 && rbuf); _assert_(off >= 0 && rbuf);
char* wp = rbuf; char* wp = rbuf;
*(wp++) = HDBFBMAGIC; *(wp++) = FBMAGIC;
*(wp++) = HDBFBMAGIC; *(wp++) = FBMAGIC;
writefixnum(wp, rsiz >> apow_, width_); writefixnum(wp, rsiz >> apow_, width_);
wp += width_; wp += width_;
*(wp++) = HDBPADMAGIC; *(wp++) = PADMAGIC;
*(wp++) = HDBPADMAGIC; *(wp++) = PADMAGIC;
if (!file_.write_fast(off, rbuf, wp - rbuf)) { if (!file_.write_fast(off, rbuf, wp - rbuf)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
return true; return true;
} }
/** /**
* Insert a free block to the free block pool. * Insert a free block to the free block pool.
* @param off the offset of the free block. * @param off the offset of the free block.
* @param rsiz the size of the free block. * @param rsiz the size of the free block.
skipping to change at line 3132 skipping to change at line 3190
* @param end the end offset. * @param end the end offset.
*/ */
void trim_free_blocks(int64_t begin, int64_t end) { void trim_free_blocks(int64_t begin, int64_t end) {
_assert_(begin >= 0 && end >= 0); _assert_(begin >= 0 && end >= 0);
FBP::const_iterator it = fbp_.begin(); FBP::const_iterator it = fbp_.begin();
FBP::const_iterator itend = fbp_.end(); FBP::const_iterator itend = fbp_.end();
while (it != itend) { while (it != itend) {
if (it->off >= begin && it->off < end) { if (it->off >= begin && it->off < end) {
fbp_.erase(it++); fbp_.erase(it++);
} else { } else {
it++; ++it;
} }
} }
} }
/** /**
* Dump all free blocks into the file. * Dump all free blocks into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_free_blocks() { bool dump_free_blocks() {
_assert_(true); _assert_(true);
if (fbpnum_ < 1) return true; if (fbpnum_ < 1) return true;
size_t size = boff_ - HDBHEADSIZ; size_t size = boff_ - HEADSIZ;
char* rbuf = new char[size]; char* rbuf = new char[size];
char* wp = rbuf; char* wp = rbuf;
char* end = rbuf + size - width_ * 2 - sizeof(uint8_t) * 2; char* end = rbuf + size - width_ * 2 - sizeof(uint8_t) * 2;
size_t num = fbp_.size(); size_t num = fbp_.size();
if (num > 0) { if (num > 0) {
FreeBlock* blocks = new FreeBlock[num]; FreeBlock* blocks = new FreeBlock[num];
size_t cnt = 0; size_t cnt = 0;
FBP::const_iterator it = fbp_.begin(); FBP::const_iterator it = fbp_.begin();
FBP::const_iterator itend = fbp_.end(); FBP::const_iterator itend = fbp_.end();
while (it != itend) { while (it != itend) {
blocks[cnt++] = *it; blocks[cnt++] = *it;
it++; ++it;
} }
std::sort(blocks, blocks + num, FreeBlockComparator()); std::sort(blocks, blocks + num, FreeBlockComparator());
for (size_t i = num - 1; i > 0; i--) { for (size_t i = num - 1; i > 0; i--) {
blocks[i].off -= blocks[i-1].off; blocks[i].off -= blocks[i-1].off;
} }
for (size_t i = 0; wp < end && i < num; i++) { for (size_t i = 0; wp < end && i < num; i++) {
wp += writevarnum(wp, blocks[i].off >> apow_); wp += writevarnum(wp, blocks[i].off >> apow_);
wp += writevarnum(wp, blocks[i].rsiz >> apow_); wp += writevarnum(wp, blocks[i].rsiz >> apow_);
} }
delete[] blocks; delete[] blocks;
} }
*(wp++) = 0; *(wp++) = 0;
*(wp++) = 0; *(wp++) = 0;
bool err = false; bool err = false;
if (!file_.write(HDBHEADSIZ, rbuf, wp - rbuf)) { if (!file_.write(HEADSIZ, rbuf, wp - rbuf)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
delete[] rbuf; delete[] rbuf;
return !err; return !err;
} }
/** /**
* Dump an empty set of free blocks into the file. * Dump an empty set of free blocks into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_empty_free_blocks() { bool dump_empty_free_blocks() {
_assert_(true); _assert_(true);
if (fbpnum_ < 1) return true; if (fbpnum_ < 1) return true;
char rbuf[2]; char rbuf[2];
char* wp = rbuf; char* wp = rbuf;
*(wp++) = 0; *(wp++) = 0;
*(wp++) = 0; *(wp++) = 0;
bool err = false; bool err = false;
if (!file_.write(HDBHEADSIZ, rbuf, wp - rbuf)) { if (!file_.write(HEADSIZ, rbuf, wp - rbuf)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Load all free blocks from from the file. * Load all free blocks from from the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_free_blocks() { bool load_free_blocks() {
_assert_(true); _assert_(true);
if (fbpnum_ < 1) return true; if (fbpnum_ < 1) return true;
size_t size = boff_ - HDBHEADSIZ; size_t size = boff_ - HEADSIZ;
char* rbuf = new char[size]; char* rbuf = new char[size];
if (!file_.read(HDBHEADSIZ, rbuf, size)) { if (!file_.read(HEADSIZ, rbuf, size)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long long)psiz_, (long long)HDBHEADSIZ, (long long)file_.size ()); (long long)psiz_, (long long)HEADSIZ, (long long)file_.size()) ;
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
const char* rp = rbuf; const char* rp = rbuf;
FreeBlock* blocks = new FreeBlock[fbpnum_]; FreeBlock* blocks = new FreeBlock[fbpnum_];
int32_t num = 0; int32_t num = 0;
while (num < fbpnum_ && size > 1 && *rp != '\0') { while (num < fbpnum_ && size > 1 && *rp != '\0') {
uint64_t off; uint64_t off;
size_t step = readvarnum(rp, size, &off); size_t step = readvarnum(rp, size, &off);
if (step < 1 || off < 1) { if (step < 1 || off < 1) {
skipping to change at line 3265 skipping to change at line 3323
* Disable all cursors. * Disable all cursors.
*/ */
void disable_cursors() { void disable_cursors() {
_assert_(true); _assert_(true);
if (curs_.empty()) return; if (curs_.empty()) return;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->off_ = 0; cur->off_ = 0;
cit++; ++cit;
} }
} }
/** /**
* Escape cursors on a free block. * Escape cursors on a free block.
* @param off the offset of the free block. * @param off the offset of the free block.
* @param dest the destination offset. * @param dest the destination offset.
*/ */
void escape_cursors(int64_t off, int64_t dest) { void escape_cursors(int64_t off, int64_t dest) {
_assert_(off >= 0 && dest >= 0); _assert_(off >= 0 && dest >= 0);
if (curs_.empty()) return; if (curs_.empty()) return;
skipping to change at line 3288 skipping to change at line 3346
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->end_ == off) { if (cur->end_ == off) {
cur->end_ = dest; cur->end_ = dest;
if (cur->off_ >= cur->end_) cur->off_ = 0; if (cur->off_ >= cur->end_) cur->off_ = 0;
} }
if (cur->off_ == off) { if (cur->off_ == off) {
cur->off_ = dest; cur->off_ = dest;
if (cur->off_ >= cur->end_) cur->off_ = 0; if (cur->off_ >= cur->end_) cur->off_ = 0;
} }
cit++; ++cit;
} }
} }
/** /**
* Trim invalid cursors. * Trim invalid cursors.
*/ */
void trim_cursors() { void trim_cursors() {
_assert_(true); _assert_(true);
if (curs_.empty()) return; if (curs_.empty()) return;
int64_t end = lsiz_; int64_t end = lsiz_;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->off_ >= end) { if (cur->off_ >= end) {
cur->off_ = 0; cur->off_ = 0;
} else if (cur->end_ > end) { } else if (cur->end_ > end) {
cur->end_ = end; cur->end_ = end;
} }
cit++; ++cit;
} }
} }
/** /**
* Remove a record from a bucket chain. * Remove a record from a bucket chain.
* @param rec the record structure. * @param rec the record structure.
* @param rbuf the working buffer. * @param rbuf the working buffer.
* @param bidx the bucket index. * @param bidx the bucket index.
* @param entoff the offset of the entry pointer. * @param entoff the offset of the entry pointer.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
skipping to change at line 3384 skipping to change at line 3442
* Begin transaction. * Begin transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_impl() { bool begin_transaction_impl() {
_assert_(true); _assert_(true);
if ((count_ != trcount_ || lsiz_ != trsize_) && !dump_meta()) return fa lse; if ((count_ != trcount_ || lsiz_ != trsize_) && !dump_meta()) return fa lse;
if (!file_.begin_transaction(trhard_, boff_)) { if (!file_.begin_transaction(trhard_, boff_)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
if (!file_.write_transaction(HDBMOFFBNUM, HDBHEADSIZ - HDBMOFFBNUM)) { if (!file_.write_transaction(MOFFBNUM, HEADSIZ - MOFFBNUM)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
file_.end_transaction(false); file_.end_transaction(false);
return false; return false;
} }
if (fbpnum_ > 0) { if (fbpnum_ > 0) {
FBP::const_iterator it = fbp_.end(); FBP::const_iterator it = fbp_.end();
FBP::const_iterator itbeg = fbp_.begin(); FBP::const_iterator itbeg = fbp_.begin();
for (int32_t cnt = fpow_ * 2 + 1; cnt > 0; cnt--) { for (int32_t cnt = fpow_ * 2 + 1; cnt > 0; cnt--) {
if (it == itbeg) break; if (it == itbeg) break;
it--; --it;
trfbp_.insert(*it); trfbp_.insert(*it);
} }
} }
return true; return true;
} }
/** /**
* Begin auto transaction. * Begin auto transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_auto_transaction() { bool begin_auto_transaction() {
_assert_(true); _assert_(true);
atlock_.lock(); atlock_.lock();
if (!file_.begin_transaction(autosync_, boff_)) { if (!file_.begin_transaction(autosync_, boff_)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
atlock_.unlock(); atlock_.unlock();
return false; return false;
} }
if (!file_.write_transaction(HDBMOFFCOUNT, HDBMOFFOPAQUE - HDBMOFFCOUNT )) { if (!file_.write_transaction(MOFFCOUNT, MOFFOPAQUE - MOFFCOUNT)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
file_.end_transaction(false); file_.end_transaction(false);
atlock_.unlock(); atlock_.unlock();
return false; return false;
} }
return true; return true;
} }
/** /**
* Commit transaction. * Commit transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 3557 skipping to change at line 3615
uint8_t flags_; uint8_t flags_;
/** The flag for open. */ /** The flag for open. */
bool flagopen_; bool flagopen_;
/** The record number. */ /** The record number. */
AtomicInt64 count_; AtomicInt64 count_;
/** The logical size of the file. */ /** The logical size of the file. */
AtomicInt64 lsiz_; AtomicInt64 lsiz_;
/** The physical size of the file. */ /** The physical size of the file. */
AtomicInt64 psiz_; AtomicInt64 psiz_;
/** The opaque data. */ /** The opaque data. */
char opaque_[HDBHEADSIZ-HDBMOFFOPAQUE]; char opaque_[HEADSIZ-MOFFOPAQUE];
/** The size of the internal memory-mapped region. */ /** The size of the internal memory-mapped region. */
int64_t msiz_; int64_t msiz_;
/** The unit step number of auto defragmentation. */ /** The unit step number of auto defragmentation. */
int64_t dfunit_; int64_t dfunit_;
/** The embedded data compressor. */ /** The embedded data compressor. */
Compressor* embcomp_; Compressor* embcomp_;
/** The alignment of records. */ /** The alignment of records. */
size_t align_; size_t align_;
/** The number of elements of the free block pool. */ /** The number of elements of the free block pool. */
int32_t fbpnum_; int32_t fbpnum_;
 End of changes. 91 change blocks. 
161 lines changed or deleted 209 lines changed or added


 kclangc.h   kclangc.h 
skipping to change at line 22 skipping to change at line 22
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
************************************************************************** ***********************/ ************************************************************************** ***********************/
#ifndef _KCLANGC_H /* duplication check */ #ifndef _KCLANGC_H /* duplication check */
#define _KCLANGC_H #define _KCLANGC_H
#if defined(__cplusplus) #if defined(__cplusplus)
extern "C" { extern "C" {
#endif #endif
#if !defined(__STDC_LIMIT_MACROS)
#define __STDC_LIMIT_MACROS 1
#endif
#include <assert.h> #include <assert.h>
#include <ctype.h> #include <ctype.h>
#include <errno.h> #include <errno.h>
#include <float.h> #include <float.h>
#include <limits.h> #include <limits.h>
#include <locale.h> #include <locale.h>
#include <math.h> #include <math.h>
#include <setjmp.h> #include <setjmp.h>
#include <stdarg.h> #include <stdarg.h>
#include <stddef.h> #include <stddef.h>
 End of changes. 1 change blocks. 
0 lines changed or deleted 4 lines changed or added


 kcmap.h   kcmap.h 
skipping to change at line 24 skipping to change at line 24
#ifndef _KCMAP_H // duplication check #ifndef _KCMAP_H // duplication check
#define _KCMAP_H #define _KCMAP_H
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation.
*/
namespace {
const size_t MAPDEFBNUM = 31; ///< default bucket number of hash
table
const size_t MAPZMAPBNUM = 32768; ///< mininum number of buckets to
use mmap
}
/**
* Memory-saving hash map. * Memory-saving hash map.
*/ */
class TinyHashMap { class TinyHashMap {
public: public:
class Iterator; class Iterator;
private: private:
struct Record; struct Record;
struct RecordComparator; struct RecordComparator;
public: public:
/** /**
skipping to change at line 148 skipping to change at line 140
} }
/** /**
* Release recources of the current records. * Release recources of the current records.
*/ */
void free_records() { void free_records() {
std::vector<char*>::iterator it = recs_.begin(); std::vector<char*>::iterator it = recs_.begin();
std::vector<char*>::iterator itend = recs_.end(); std::vector<char*>::iterator itend = recs_.end();
while (it != itend) { while (it != itend) {
char* rbuf = *it; char* rbuf = *it;
delete[] rbuf; delete[] rbuf;
it++; ++it;
} }
recs_.clear(); recs_.clear();
} }
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
Iterator(const Iterator&); Iterator(const Iterator&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
Iterator& operator =(const Iterator&); Iterator& operator =(const Iterator&);
/** The container. */ /** The container. */
TinyHashMap* map_; TinyHashMap* map_;
/** The current bucket index. */ /** The current bucket index. */
skipping to change at line 487 skipping to change at line 479
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records. * @return the number of records.
*/ */
size_t count() { size_t count() {
_assert_(true); _assert_(true);
return count_; return count_;
} }
private: private:
/** The default bucket number of hash table. */
static const size_t MAPDEFBNUM = 31;
/** The mininum number of buckets to use mmap. */
static const size_t MAPZMAPBNUM = 32768;
/** /**
* Record data. * Record data.
*/ */
struct Record { struct Record {
/** constructor */ /** constructor */
Record(char* child, const char* kbuf, uint64_t ksiz, Record(char* child, const char* kbuf, uint64_t ksiz,
const char* vbuf, uint64_t vsiz, uint64_t psiz) : const char* vbuf, uint64_t vsiz, uint64_t psiz) :
child_(child), kbuf_(kbuf), ksiz_(ksiz), vbuf_(vbuf), vsiz_(vsiz), ps iz_(psiz) { child_(child), kbuf_(kbuf), ksiz_(ksiz), vbuf_(vbuf), vsiz_(vsiz), ps iz_(psiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && ps iz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && ps iz <= MEMMAXSIZ);
} }
skipping to change at line 1161 skipping to change at line 1157
} }
/** /**
* Get the reference of the value of the last record. * Get the reference of the value of the last record.
* @return the reference of the value of the last record. * @return the reference of the value of the last record.
*/ */
VALUE& last_value() { VALUE& last_value() {
_assert_(true); _assert_(true);
return last_->value; return last_->value;
} }
private: private:
/** The default bucket number of hash table. */
static const size_t MAPDEFBNUM = 31;
/** The mininum number of buckets to use mmap. */
static const size_t MAPZMAPBNUM = 32768;
/** /**
* Record data. * Record data.
*/ */
struct Record { struct Record {
KEY key; ///< key KEY key; ///< key
VALUE value; ///< value VALUE value; ///< value
Record* child; ///< child record Record* child; ///< child record
Record* prev; ///< previous record Record* prev; ///< previous record
Record* next; ///< next record Record* next; ///< next record
/** constructor */ /** constructor */
 End of changes. 4 change blocks. 
11 lines changed or deleted 9 lines changed or added


 kcplantdb.h   kcplantdb.h 
skipping to change at line 28 skipping to change at line 28
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h> #include <kcregex.h>
#include <kcdb.h> #include <kcdb.h>
namespace kyotocabinet { // common namespace #define KCPDBMETAKEY "@" ///< key of the record for meta da
ta
#define KCPDBTMPPATHEXT "tmpkct" ///< extension of the temporary fi
le
/** namespace kyotocabinet { // common namespace
* Constants for implementation.
*/
namespace {
const int32_t PLDBSLOTNUM = 16; ///< number of cache slots
const uint8_t PLDBDEFAPOW = 8; ///< default alignment power
const uint8_t PLDBDEFFPOW = 10; ///< default free block pool power
const int64_t PLDBDEFBNUM = 64LL << 10; ///< default bucket number
const int32_t PLDBDEFPSIZ = 8192; ///< default page size
const int64_t PLDBDEFPCCAP = 64LL << 20; ///< default capacity size of the
page cache
const char PLDBMETAKEY[] = "@"; ///< key of the record for meta da
ta
const int64_t PLDBHEADSIZ = 80; ///< size of the header
const int64_t PLDBMOFFNUMS = 8; ///< offset of the numbers
const char PLDBLNPREFIX = 'L'; ///< prefix of leaf nodes
const char PLDBINPREFIX = 'I'; ///< prefix of inner nodes
const size_t PLDBAVGWAY = 16; ///< average number of ways of eac
h node
const size_t PLDBWARMRATIO = 4; ///< ratio of the warm cache
const size_t PLDBINFLRATIO = 32; ///< ratio of flushing inner nodes
const size_t PLDBDEFLINUM = 64; ///< default number of items in ea
ch leaf node
const size_t PLDBDEFIINUM = 128; ///< default number of items in ea
ch inner node
const size_t PLDBRECBUFSIZ = 64; ///< size of the record buffer
const int64_t PLDBINIDBASE = 1LL << 48; ///< base ID number for inner node
s
const size_t PLDBINLINKMIN = 8; ///< minimum number of links in ea
ch inner node
const int32_t PLDBLEVELMAX = 16; ///< maximum level of B+ tree
const int32_t PLDBATRANCNUM = 256; ///< number of cached nodes for au
to transaction
const uint32_t PLDBLOCKBUSYLOOP = 8192; ///< threshold of busy loop and sl
eep for locking
const char* PLDBTMPPATHEXT = "tmpkct"; ///< extension of the temporary fi
le
}
/** /**
* Plant database. * Plant database.
* @param BASEDB a class compatible with the file hash database class. * @param BASEDB a class compatible with the file hash database class.
* @param DBTYPE the database type number of the class. * @param DBTYPE the database type number of the class.
* @note This class template is a template for concrete classes to operate tree databases. * @note This class template is a template for concrete classes to operate tree databases.
* Template instance classes can be inherited but overwriting methods is fo rbidden. The class * Template instance classes can be inherited but overwriting methods is fo rbidden. The class
* TreeDB is the instance of the file tree database. The class ForestDB is the instance of the * TreeDB is the instance of the file tree database. The class ForestDB is the instance of the
* directory tree database. Before every database operation, it is necessa ry to call the * directory tree database. Before every database operation, it is necessa ry to call the
* BasicDB::open method in order to open a database file and connect the da tabase object to it. * BasicDB::open method in order to open a database file and connect the da tabase object to it.
skipping to change at line 86 skipping to change at line 60
class Cursor; class Cursor;
private: private:
struct Record; struct Record;
struct RecordComparator; struct RecordComparator;
struct LeafNode; struct LeafNode;
struct Link; struct Link;
struct LinkComparator; struct LinkComparator;
struct InnerNode; struct InnerNode;
struct LeafSlot; struct LeafSlot;
struct InnerSlot; struct InnerSlot;
class ScopedVisitor;
/** An alias of array of records. */ /** An alias of array of records. */
typedef std::vector<Record*> RecordArray; typedef std::vector<Record*> RecordArray;
/** An alias of array of records. */ /** An alias of array of records. */
typedef std::vector<Link*> LinkArray; typedef std::vector<Link*> LinkArray;
/** An alias of leaf node cache. */ /** An alias of leaf node cache. */
typedef LinkedHashMap<int64_t, LeafNode*> LeafCache; typedef LinkedHashMap<int64_t, LeafNode*> LeafCache;
/** An alias of inner node cache. */ /** An alias of inner node cache. */
typedef LinkedHashMap<int64_t, InnerNode*> InnerCache; typedef LinkedHashMap<int64_t, InnerNode*> InnerCache;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
skipping to change at line 437 skipping to change at line 412
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only operation. * @param writable true for writable operation, or false for read-only operation.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @param hitp the pointer to the variable for the hit flag. * @param hitp the pointer to the variable for the hit flag.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool accept_spec(Visitor* visitor, bool writable, bool step, bool* hitp ) { bool accept_spec(Visitor* visitor, bool writable, bool step, bool* hitp ) {
_assert_(visitor && hitp); _assert_(visitor && hitp);
bool err = false; bool err = false;
bool hit = false; bool hit = false;
char rstack[PLDBRECBUFSIZ]; char rstack[PlantDB::RECBUFSIZ];
size_t rsiz = sizeof(Record) + ksiz_; size_t rsiz = sizeof(Record) + ksiz_;
char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack;
Record* rec = (Record*)rbuf; Record* rec = (Record*)rbuf;
rec->ksiz = ksiz_; rec->ksiz = ksiz_;
rec->vsiz = 0; rec->vsiz = 0;
std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_);
LeafNode* node = db_->load_leaf_node(lid_, false); LeafNode* node = db_->load_leaf_node(lid_, false);
if (node) { if (node) {
char lstack[PLDBRECBUFSIZ]; char lstack[PlantDB::RECBUFSIZ];
char* lbuf = NULL; char* lbuf = NULL;
size_t lsiz = 0; size_t lsiz = 0;
Link* link = NULL; Link* link = NULL;
int64_t hist[PLDBLEVELMAX]; int64_t hist[LEVELMAX];
int32_t hnum = 0; int32_t hnum = 0;
if (writable) { if (writable) {
node->lock.lock_writer(); node->lock.lock_writer();
} else { } else {
node->lock.lock_reader(); node->lock.lock_reader();
} }
RecordArray& recs = node->recs; RecordArray& recs = node->recs;
if (!recs.empty()) { if (!recs.empty()) {
Record* frec = recs.front(); Record* frec = recs.front();
Record* lrec = recs.back(); Record* lrec = recs.back();
skipping to change at line 528 skipping to change at line 503
if (node->size > db_->psiz_ && recs.size() > 1) { if (node->size > db_->psiz_ && recs.size() > 1) {
lsiz = sizeof(Link) + ksiz; lsiz = sizeof(Link) + ksiz;
lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack;
link = (Link*)lbuf; link = (Link*)lbuf;
link->child = 0; link->child = 0;
link->ksiz = ksiz; link->ksiz = ksiz;
std::memcpy(lbuf + sizeof(*link), kbuf, ksiz); std::memcpy(lbuf + sizeof(*link), kbuf, ksiz);
} }
} }
if (step) { if (step) {
rit++; ++rit;
if (rit != ritend) { if (rit != ritend) {
clear_position(); clear_position();
set_position(*rit, node->id); set_position(*rit, node->id);
step = false; step = false;
} }
} }
} }
} }
} }
bool atran = db_->autotran_ && !db_->tran_ && node->dirty; bool atran = db_->autotran_ && !db_->tran_ && node->dirty;
skipping to change at line 565 skipping to change at line 540
if (link) { if (link) {
node = db_->search_tree(link, true, hist, &hnum); node = db_->search_tree(link, true, hist, &hnum);
if (node) { if (node) {
if (!db_->reorganize_tree(node, hist, hnum)) err = true; if (!db_->reorganize_tree(node, hist, hnum)) err = true;
if (atran && !db_->tran_ && !db_->fix_auto_transaction_tree ()) err = true; if (atran && !db_->tran_ && !db_->fix_auto_transaction_tree ()) err = true;
} else { } else {
db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed" ); db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed" );
err = true; err = true;
} }
} else if (flush) { } else if (flush) {
int32_t idx = id % PLDBSLOTNUM; int32_t idx = id % SLOTNUM;
LeafSlot* lslot = db_->lslots_ + idx; LeafSlot* lslot = db_->lslots_ + idx;
if (!db_->flush_leaf_cache_part(lslot)) err = true; if (!db_->flush_leaf_cache_part(lslot)) err = true;
InnerSlot* islot = db_->islots_ + idx; InnerSlot* islot = db_->islots_ + idx;
if (islot->warm->count() > lslot->warm->count() + lslot->hot- >count() + 1 && if (islot->warm->count() > lslot->warm->count() + lslot->hot- >count() + 1 &&
!db_->flush_inner_cache_part(islot)) err = true; !db_->flush_inner_cache_part(islot)) err = true;
} }
if (async && !db_->fix_auto_synchronization()) err = true; if (async && !db_->fix_auto_synchronization()) err = true;
} }
} }
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
skipping to change at line 593 skipping to change at line 568
* @param visitor a visitor object. * @param visitor a visitor object.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @param retryp the pointer to the variable for the retry flag. * @param retryp the pointer to the variable for the retry flag.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool accept_atom(Visitor* visitor, bool step, bool *retryp) { bool accept_atom(Visitor* visitor, bool step, bool *retryp) {
_assert_(visitor && retryp); _assert_(visitor && retryp);
bool err = false; bool err = false;
bool reorg = false; bool reorg = false;
*retryp = false; *retryp = false;
char lstack[PLDBRECBUFSIZ]; char lstack[PlantDB::RECBUFSIZ];
size_t lsiz = sizeof(Link) + ksiz_; size_t lsiz = sizeof(Link) + ksiz_;
char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack;
Link* link = (Link*)lbuf; Link* link = (Link*)lbuf;
link->child = 0; link->child = 0;
link->ksiz = ksiz_; link->ksiz = ksiz_;
std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_); std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_);
int64_t hist[PLDBLEVELMAX]; int64_t hist[LEVELMAX];
int32_t hnum = 0; int32_t hnum = 0;
LeafNode* node = db_->search_tree(link, true, hist, &hnum); LeafNode* node = db_->search_tree(link, true, hist, &hnum);
if (!node) { if (!node) {
db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed"); db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed");
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
return false; return false;
} }
if (node->recs.empty()) { if (node->recs.empty()) {
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
clear_position(); clear_position();
skipping to change at line 630 skipping to change at line 605
link->child = 0; link->child = 0;
link->ksiz = ksiz_; link->ksiz = ksiz_;
std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_); std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_);
node = db_->search_tree(link, true, hist, &hnum); node = db_->search_tree(link, true, hist, &hnum);
if (node->id != lid_) { if (node->id != lid_) {
db_->set_error(_KCCODELINE_, Error::BROKEN, "invalid tree"); db_->set_error(_KCCODELINE_, Error::BROKEN, "invalid tree");
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
return false; return false;
} }
} }
char rstack[PLDBRECBUFSIZ]; char rstack[PlantDB::RECBUFSIZ];
size_t rsiz = sizeof(Record) + ksiz_; size_t rsiz = sizeof(Record) + ksiz_;
char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack;
Record* rec = (Record*)rbuf; Record* rec = (Record*)rbuf;
rec->ksiz = ksiz_; rec->ksiz = ksiz_;
rec->vsiz = 0; rec->vsiz = 0;
std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_);
RecordArray& recs = node->recs; RecordArray& recs = node->recs;
typename RecordArray::iterator ritend = recs.end(); typename RecordArray::iterator ritend = recs.end();
typename RecordArray::iterator rit = std::lower_bound(recs.begin(), r itend, typename RecordArray::iterator rit = std::lower_bound(recs.begin(), r itend,
rec, db_->recco mp_); rec, db_->recco mp_);
skipping to change at line 689 skipping to change at line 664
if (vsiz > rec->vsiz) { if (vsiz > rec->vsiz) {
*rit = (Record*)xrealloc(rec, sizeof(*rec) + rec->ksiz + vsiz); *rit = (Record*)xrealloc(rec, sizeof(*rec) + rec->ksiz + vsiz);
rec = *rit; rec = *rit;
kbuf = (char*)rec + sizeof(*rec); kbuf = (char*)rec + sizeof(*rec);
} }
std::memcpy(kbuf + rec->ksiz, vbuf, vsiz); std::memcpy(kbuf + rec->ksiz, vbuf, vsiz);
rec->vsiz = vsiz; rec->vsiz = vsiz;
if (node->size > db_->psiz_ && recs.size() > 1) reorg = true; if (node->size > db_->psiz_ && recs.size() > 1) reorg = true;
} }
if (step) { if (step) {
rit++; ++rit;
if (rit != ritend) { if (rit != ritend) {
clear_position(); clear_position();
set_position(*rit, node->id); set_position(*rit, node->id);
} else { } else {
clear_position(); clear_position();
set_position(node->next); set_position(node->next);
} }
} }
bool atran = db_->autotran_ && !db_->tran_ && node->dirty; bool atran = db_->autotran_ && !db_->tran_ && node->dirty;
bool async = db_->autosync_ && !db_->autotran_ && !db_->tran_ && no de->dirty; bool async = db_->autosync_ && !db_->autotran_ && !db_->tran_ && no de->dirty;
if (atran && !reorg && !db_->fix_auto_transaction_leaf(node)) err = true; if (atran && !reorg && !db_->fix_auto_transaction_leaf(node)) err = true;
if (reorg) { if (reorg) {
if (!db_->reorganize_tree(node, hist, hnum)) err = true; if (!db_->reorganize_tree(node, hist, hnum)) err = true;
if (atran && !db_->fix_auto_transaction_tree()) err = true; if (atran && !db_->fix_auto_transaction_tree()) err = true;
} else if (db_->cusage_ > db_->pccap_) { } else if (db_->cusage_ > db_->pccap_) {
int32_t idx = node->id % PLDBSLOTNUM; int32_t idx = node->id % SLOTNUM;
LeafSlot* lslot = db_->lslots_ + idx; LeafSlot* lslot = db_->lslots_ + idx;
if (!db_->flush_leaf_cache_part(lslot)) err = true; if (!db_->flush_leaf_cache_part(lslot)) err = true;
InnerSlot* islot = db_->islots_ + idx; InnerSlot* islot = db_->islots_ + idx;
if (islot->warm->count() > lslot->warm->count() + lslot->hot->cou nt() + 1 && if (islot->warm->count() > lslot->warm->count() + lslot->hot->cou nt() + 1 &&
!db_->flush_inner_cache_part(islot)) err = true; !db_->flush_inner_cache_part(islot)) err = true;
} }
if (async && !db_->fix_auto_synchronization()) err = true; if (async && !db_->fix_auto_synchronization()) err = true;
} else { } else {
int64_t lid = lid_; int64_t lid = lid_;
clear_position(); clear_position();
skipping to change at line 738 skipping to change at line 713
if (rbuf != rstack) delete[] rbuf; if (rbuf != rstack) delete[] rbuf;
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
return !err; return !err;
} }
/** /**
* Adjust the position to an existing record. * Adjust the position to an existing record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool adjust_position() { bool adjust_position() {
_assert_(true); _assert_(true);
char lstack[PLDBRECBUFSIZ]; char lstack[PlantDB::RECBUFSIZ];
size_t lsiz = sizeof(Link) + ksiz_; size_t lsiz = sizeof(Link) + ksiz_;
char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack;
Link* link = (Link*)lbuf; Link* link = (Link*)lbuf;
link->child = 0; link->child = 0;
link->ksiz = ksiz_; link->ksiz = ksiz_;
std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_); std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_);
int64_t hist[PLDBLEVELMAX]; int64_t hist[LEVELMAX];
int32_t hnum = 0; int32_t hnum = 0;
LeafNode* node = db_->search_tree(link, true, hist, &hnum); LeafNode* node = db_->search_tree(link, true, hist, &hnum);
if (!node) { if (!node) {
db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed"); db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed");
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
return false; return false;
} }
char rstack[PLDBRECBUFSIZ]; char rstack[PlantDB::RECBUFSIZ];
size_t rsiz = sizeof(Record) + ksiz_; size_t rsiz = sizeof(Record) + ksiz_;
char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack;
Record* rec = (Record*)rbuf; Record* rec = (Record*)rbuf;
rec->ksiz = ksiz_; rec->ksiz = ksiz_;
rec->vsiz = 0; rec->vsiz = 0;
std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_);
bool err = false; bool err = false;
node->lock.lock_reader(); node->lock.lock_reader();
const RecordArray& recs = node->recs; const RecordArray& recs = node->recs;
typename RecordArray::const_iterator ritend = node->recs.end(); typename RecordArray::const_iterator ritend = node->recs.end();
skipping to change at line 787 skipping to change at line 762
} }
/** /**
* Back the position to the previous record speculatively. * Back the position to the previous record speculatively.
* @param hitp the pointer to the variable for the hit flag. * @param hitp the pointer to the variable for the hit flag.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool back_position_spec(bool* hitp) { bool back_position_spec(bool* hitp) {
_assert_(hitp); _assert_(hitp);
bool err = false; bool err = false;
bool hit = false; bool hit = false;
char rstack[PLDBRECBUFSIZ]; char rstack[PlantDB::RECBUFSIZ];
size_t rsiz = sizeof(Record) + ksiz_; size_t rsiz = sizeof(Record) + ksiz_;
char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack;
Record* rec = (Record*)rbuf; Record* rec = (Record*)rbuf;
rec->ksiz = ksiz_; rec->ksiz = ksiz_;
rec->vsiz = 0; rec->vsiz = 0;
std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_);
LeafNode* node = db_->load_leaf_node(lid_, false); LeafNode* node = db_->load_leaf_node(lid_, false);
if (node) { if (node) {
node->lock.lock_reader(); node->lock.lock_reader();
RecordArray& recs = node->recs; RecordArray& recs = node->recs;
skipping to change at line 821 skipping to change at line 796
hit = true; hit = true;
typename RecordArray::iterator ritbeg = recs.begin(); typename RecordArray::iterator ritbeg = recs.begin();
typename RecordArray::iterator ritend = recs.end(); typename RecordArray::iterator ritend = recs.end();
typename RecordArray::iterator rit = std::lower_bound(recs.begi n(), ritend, typename RecordArray::iterator rit = std::lower_bound(recs.begi n(), ritend,
rec, db_- >reccomp_); rec, db_- >reccomp_);
clear_position(); clear_position();
if (rit == ritbeg) { if (rit == ritbeg) {
node->lock.unlock(); node->lock.unlock();
if (!set_position_back(node->prev)) err = true; if (!set_position_back(node->prev)) err = true;
} else { } else {
rit--; --rit;
set_position(*rit, node->id); set_position(*rit, node->id);
node->lock.unlock(); node->lock.unlock();
} }
} }
} }
} }
if (rbuf != rstack) delete[] rbuf; if (rbuf != rstack) delete[] rbuf;
*hitp = hit; *hitp = hit;
return !err; return !err;
} }
/** /**
* Back the position to the previous record atomically. * Back the position to the previous record atomically.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool back_position_atom() { bool back_position_atom() {
_assert_(true); _assert_(true);
char lstack[PLDBRECBUFSIZ]; char lstack[PlantDB::RECBUFSIZ];
size_t lsiz = sizeof(Link) + ksiz_; size_t lsiz = sizeof(Link) + ksiz_;
char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack;
Link* link = (Link*)lbuf; Link* link = (Link*)lbuf;
link->child = 0; link->child = 0;
link->ksiz = ksiz_; link->ksiz = ksiz_;
std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_); std::memcpy(lbuf + sizeof(*link), kbuf_, ksiz_);
int64_t hist[PLDBLEVELMAX]; int64_t hist[LEVELMAX];
int32_t hnum = 0; int32_t hnum = 0;
LeafNode* node = db_->search_tree(link, true, hist, &hnum); LeafNode* node = db_->search_tree(link, true, hist, &hnum);
if (!node) { if (!node) {
db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed"); db_->set_error(_KCCODELINE_, Error::BROKEN, "search failed");
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
return false; return false;
} }
char rstack[PLDBRECBUFSIZ]; char rstack[PlantDB::RECBUFSIZ];
size_t rsiz = sizeof(Record) + ksiz_; size_t rsiz = sizeof(Record) + ksiz_;
char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack;
Record* rec = (Record*)rbuf; Record* rec = (Record*)rbuf;
rec->ksiz = ksiz_; rec->ksiz = ksiz_;
rec->vsiz = 0; rec->vsiz = 0;
std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_); std::memcpy(rbuf + sizeof(*rec), kbuf_, ksiz_);
bool err = false; bool err = false;
node->lock.lock_reader(); node->lock.lock_reader();
const RecordArray& recs = node->recs; const RecordArray& recs = node->recs;
typename RecordArray::const_iterator ritbeg = node->recs.begin(); typename RecordArray::const_iterator ritbeg = node->recs.begin();
skipping to change at line 876 skipping to change at line 851
rec, db_- >reccomp_); rec, db_- >reccomp_);
clear_position(); clear_position();
if (rit == ritbeg) { if (rit == ritbeg) {
node->lock.unlock(); node->lock.unlock();
if (!set_position_back(node->prev)) err = true; if (!set_position_back(node->prev)) err = true;
} else if (rit == ritend) { } else if (rit == ritend) {
ritend--; ritend--;
set_position(*ritend, node->id); set_position(*ritend, node->id);
node->lock.unlock(); node->lock.unlock();
} else { } else {
rit--; --rit;
set_position(*rit, node->id); set_position(*rit, node->id);
node->lock.unlock(); node->lock.unlock();
} }
if (rbuf != rstack) delete[] rbuf; if (rbuf != rstack) delete[] rbuf;
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
return !err; return !err;
} }
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
Cursor(const Cursor&); Cursor(const Cursor&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
Cursor& operator =(const Cursor&); Cursor& operator =(const Cursor&);
/** The inner database. */ /** The inner database. */
PlantDB* db_; PlantDB* db_;
/** The stack buffer for the key. */ /** The stack buffer for the key. */
char stack_[PLDBRECBUFSIZ]; char stack_[PlantDB::RECBUFSIZ];
/** The pointer to the key region. */ /** The pointer to the key region. */
char* kbuf_; char* kbuf_;
/** The size of the key region. */ /** The size of the key region. */
size_t ksiz_; size_t ksiz_;
/** The last visited leaf. */ /** The last visited leaf. */
int64_t lid_; int64_t lid_;
}; };
/** /**
* Tuning options. * Tuning options.
*/ */
skipping to change at line 919 skipping to change at line 894
*/ */
enum Flag { enum Flag {
FOPEN = BASEDB::FOPEN, ///< whether opened FOPEN = BASEDB::FOPEN, ///< whether opened
FFATAL = BASEDB::FFATAL ///< whether with fatal error FFATAL = BASEDB::FFATAL ///< whether with fatal error
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit PlantDB() : explicit PlantDB() :
mlock_(), mtrigger_(NULL), omode_(0), writer_(false), autotran_(false), autosync_(false), mlock_(), mtrigger_(NULL), omode_(0), writer_(false), autotran_(false), autosync_(false),
db_(), curs_(), apow_(PLDBDEFAPOW), fpow_(PLDBDEFFPOW), opts_(0), bnum_ db_(), curs_(), apow_(DEFAPOW), fpow_(DEFFPOW), opts_(0), bnum_(DEFBNUM
(PLDBDEFBNUM), ),
psiz_(PLDBDEFPSIZ), pccap_(PLDBDEFPCCAP), psiz_(DEFPSIZ), pccap_(DEFPCCAP),
root_(0), first_(0), last_(0), lcnt_(0), icnt_(0), count_(0), cusage_(0 ), root_(0), first_(0), last_(0), lcnt_(0), icnt_(0), count_(0), cusage_(0 ),
lslots_(), islots_(), reccomp_(), linkcomp_(), lslots_(), islots_(), reccomp_(), linkcomp_(),
tran_(false), trclock_(0), trlcnt_(0), trcount_(0) { tran_(false), trclock_(0), trlcnt_(0), trcount_(0) {
_assert_(true); _assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~PlantDB() { virtual ~PlantDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->db_ = NULL; cur->db_ = NULL;
cit++; ++cit;
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 967 skipping to change at line 942
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
char lstack[PLDBRECBUFSIZ]; char lstack[RECBUFSIZ];
size_t lsiz = sizeof(Link) + ksiz; size_t lsiz = sizeof(Link) + ksiz;
char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack;
Link* link = (Link*)lbuf; Link* link = (Link*)lbuf;
link->child = 0; link->child = 0;
link->ksiz = ksiz; link->ksiz = ksiz;
std::memcpy(lbuf + sizeof(*link), kbuf, ksiz); std::memcpy(lbuf + sizeof(*link), kbuf, ksiz);
int64_t hist[PLDBLEVELMAX]; int64_t hist[LEVELMAX];
int32_t hnum = 0; int32_t hnum = 0;
LeafNode* node = search_tree(link, true, hist, &hnum); LeafNode* node = search_tree(link, true, hist, &hnum);
if (!node) { if (!node) {
set_error(_KCCODELINE_, Error::BROKEN, "search failed"); set_error(_KCCODELINE_, Error::BROKEN, "search failed");
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
char rstack[PLDBRECBUFSIZ]; char rstack[RECBUFSIZ];
size_t rsiz = sizeof(Record) + ksiz; size_t rsiz = sizeof(Record) + ksiz;
char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack;
Record* rec = (Record*)rbuf; Record* rec = (Record*)rbuf;
rec->ksiz = ksiz; rec->ksiz = ksiz;
rec->vsiz = 0; rec->vsiz = 0;
std::memcpy(rbuf + sizeof(*rec), kbuf, ksiz); std::memcpy(rbuf + sizeof(*rec), kbuf, ksiz);
if (writable) { if (writable) {
node->lock.lock_writer(); node->lock.lock_writer();
} else { } else {
node->lock.lock_reader(); node->lock.lock_reader();
skipping to change at line 1008 skipping to change at line 983
node->lock.unlock(); node->lock.unlock();
bool flush = false; bool flush = false;
bool err = false; bool err = false;
int64_t id = node->id; int64_t id = node->id;
if (atran && !reorg && !fix_auto_transaction_leaf(node)) err = true; if (atran && !reorg && !fix_auto_transaction_leaf(node)) err = true;
if (reorg && mlock_.promote()) { if (reorg && mlock_.promote()) {
if (!reorganize_tree(node, hist, hnum)) err = true; if (!reorganize_tree(node, hist, hnum)) err = true;
if (atran && !fix_auto_transaction_tree()) err = true; if (atran && !fix_auto_transaction_tree()) err = true;
reorg = false; reorg = false;
} else if (cusage_ > pccap_) { } else if (cusage_ > pccap_) {
int32_t idx = id % PLDBSLOTNUM; int32_t idx = id % SLOTNUM;
LeafSlot* lslot = lslots_ + idx; LeafSlot* lslot = lslots_ + idx;
if (!clean_leaf_cache_part(lslot)) err = true; if (!clean_leaf_cache_part(lslot)) err = true;
if (mlock_.promote()) { if (mlock_.promote()) {
if (!flush_leaf_cache_part(lslot)) err = true; if (!flush_leaf_cache_part(lslot)) err = true;
InnerSlot* islot = islots_ + idx; InnerSlot* islot = islots_ + idx;
if (islot->warm->count() > lslot->warm->count() + lslot->hot->count () + 1 && if (islot->warm->count() > lslot->warm->count() + lslot->hot->count () + 1 &&
!flush_inner_cache_part(islot)) err = true; !flush_inner_cache_part(islot)) err = true;
} else { } else {
flush = true; flush = true;
} }
skipping to change at line 1030 skipping to change at line 1005
mlock_.unlock(); mlock_.unlock();
if (reorg) { if (reorg) {
mlock_.lock_writer(); mlock_.lock_writer();
node = search_tree(link, false, hist, &hnum); node = search_tree(link, false, hist, &hnum);
if (node) { if (node) {
if (!reorganize_tree(node, hist, hnum)) err = true; if (!reorganize_tree(node, hist, hnum)) err = true;
if (atran && !tran_ && !fix_auto_transaction_tree()) err = true; if (atran && !tran_ && !fix_auto_transaction_tree()) err = true;
} }
mlock_.unlock(); mlock_.unlock();
} else if (flush) { } else if (flush) {
int32_t idx = id % PLDBSLOTNUM; int32_t idx = id % SLOTNUM;
LeafSlot* lslot = lslots_ + idx; LeafSlot* lslot = lslots_ + idx;
mlock_.lock_writer(); mlock_.lock_writer();
if (!flush_leaf_cache_part(lslot)) err = true; if (!flush_leaf_cache_part(lslot)) err = true;
InnerSlot* islot = islots_ + idx; InnerSlot* islot = islots_ + idx;
if (islot->warm->count() > lslot->warm->count() + lslot->hot->count() + 1 && if (islot->warm->count() > lslot->warm->count() + lslot->hot->count() + 1 &&
!flush_inner_cache_part(islot)) err = true; !flush_inner_cache_part(islot)) err = true;
mlock_.unlock(); mlock_.unlock();
} }
if (rbuf != rstack) delete[] rbuf; if (rbuf != rstack) delete[] rbuf;
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
skipping to change at line 1070 skipping to change at line 1045
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
if (keys.empty()) return true;
bool err = false; bool err = false;
std::vector<std::string>::const_iterator kit = keys.begin(); std::vector<std::string>::const_iterator kit = keys.begin();
std::vector<std::string>::const_iterator kitend = keys.end(); std::vector<std::string>::const_iterator kitend = keys.end();
while (!err && kit != kitend) { while (!err && kit != kitend) {
const char* kbuf = kit->data(); const char* kbuf = kit->data();
size_t ksiz = kit->size(); size_t ksiz = kit->size();
char lstack[PLDBRECBUFSIZ]; char lstack[RECBUFSIZ];
size_t lsiz = sizeof(Link) + ksiz; size_t lsiz = sizeof(Link) + ksiz;
char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack;
Link* link = (Link*)lbuf; Link* link = (Link*)lbuf;
link->child = 0; link->child = 0;
link->ksiz = ksiz; link->ksiz = ksiz;
std::memcpy(lbuf + sizeof(*link), kbuf, ksiz); std::memcpy(lbuf + sizeof(*link), kbuf, ksiz);
int64_t hist[PLDBLEVELMAX]; int64_t hist[LEVELMAX];
int32_t hnum = 0; int32_t hnum = 0;
LeafNode* node = search_tree(link, true, hist, &hnum); LeafNode* node = search_tree(link, true, hist, &hnum);
if (!node) { if (!node) {
set_error(_KCCODELINE_, Error::BROKEN, "search failed"); set_error(_KCCODELINE_, Error::BROKEN, "search failed");
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
err = true; err = true;
break; break;
} }
char rstack[PLDBRECBUFSIZ]; char rstack[RECBUFSIZ];
size_t rsiz = sizeof(Record) + ksiz; size_t rsiz = sizeof(Record) + ksiz;
char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack; char* rbuf = rsiz > sizeof(rstack) ? new char[rsiz] : rstack;
Record* rec = (Record*)rbuf; Record* rec = (Record*)rbuf;
rec->ksiz = ksiz; rec->ksiz = ksiz;
rec->vsiz = 0; rec->vsiz = 0;
std::memcpy(rbuf + sizeof(*rec), kbuf, ksiz); std::memcpy(rbuf + sizeof(*rec), kbuf, ksiz);
bool reorg = accept_impl(node, rec, visitor); bool reorg = accept_impl(node, rec, visitor);
bool atran = autotran_ && !tran_ && node->dirty; bool atran = autotran_ && !tran_ && node->dirty;
bool async = autosync_ && !autotran_ && !tran_ && node->dirty; bool async = autosync_ && !autotran_ && !tran_ && node->dirty;
if (atran && !reorg && !fix_auto_transaction_leaf(node)) err = true; if (atran && !reorg && !fix_auto_transaction_leaf(node)) err = true;
if (reorg) { if (reorg) {
if (!reorganize_tree(node, hist, hnum)) err = true; if (!reorganize_tree(node, hist, hnum)) err = true;
if (atran && !fix_auto_transaction_tree()) err = true; if (atran && !fix_auto_transaction_tree()) err = true;
} else if (cusage_ > pccap_) { } else if (cusage_ > pccap_) {
int32_t idx = node->id % PLDBSLOTNUM; int32_t idx = node->id % SLOTNUM;
LeafSlot* lslot = lslots_ + idx; LeafSlot* lslot = lslots_ + idx;
if (!clean_leaf_cache_part(lslot)) err = true; if (!clean_leaf_cache_part(lslot)) err = true;
if (!flush_leaf_cache_part(lslot)) err = true; if (!flush_leaf_cache_part(lslot)) err = true;
InnerSlot* islot = islots_ + idx; InnerSlot* islot = islots_ + idx;
if (islot->warm->count() > lslot->warm->count() + lslot->hot->count () + 1 && if (islot->warm->count() > lslot->warm->count() + lslot->hot->count () + 1 &&
!flush_inner_cache_part(islot)) err = true; !flush_inner_cache_part(islot)) err = true;
} }
if (rbuf != rstack) delete[] rbuf; if (rbuf != rstack) delete[] rbuf;
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
if (async && !fix_auto_synchronization()) err = true; if (async && !fix_auto_synchronization()) err = true;
kit++; ++kit;
} }
return !err; return !err;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed. * @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The whole iteration is performed atomically and other threads ar e blocked. To avoid * @note The whole iteration is performed atomically and other threads ar e blocked. To avoid
skipping to change at line 1142 skipping to change at line 1119
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
int64_t allcnt = count_; int64_t allcnt = count_;
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
bool err = false; bool err = false;
bool atran = false; bool atran = false;
if (autotran_ && writable && !tran_) { if (autotran_ && writable && !tran_) {
if (begin_transaction_impl(autosync_)) { if (begin_transaction_impl(autosync_)) {
atran = true; atran = true;
skipping to change at line 1182 skipping to change at line 1160
while (rit != ritend) { while (rit != ritend) {
Record* rec = *rit; Record* rec = *rit;
size_t rsiz = sizeof(*rec) + rec->ksiz; size_t rsiz = sizeof(*rec) + rec->ksiz;
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
Record* key = (Record*)xmalloc(rsiz); Record* key = (Record*)xmalloc(rsiz);
key->ksiz = rec->ksiz; key->ksiz = rec->ksiz;
key->vsiz = 0; key->vsiz = 0;
char* kbuf = (char*)key + sizeof(*key); char* kbuf = (char*)key + sizeof(*key);
std::memcpy(kbuf, dbuf, rec->ksiz); std::memcpy(kbuf, dbuf, rec->ksiz);
keys.push_back(key); keys.push_back(key);
rit++; ++rit;
} }
typename RecordArray::const_iterator kit = keys.begin(); typename RecordArray::const_iterator kit = keys.begin();
typename RecordArray::const_iterator kitend = keys.end(); typename RecordArray::const_iterator kitend = keys.end();
bool reorg = false; bool reorg = false;
while (kit != kitend) { while (kit != kitend) {
Record* rec = *kit; Record* rec = *kit;
if (accept_impl(node, rec, visitor)) reorg = true; if (accept_impl(node, rec, visitor)) reorg = true;
curcnt++; curcnt++;
if (checker && !checker->check("iterate", "processing", curcnt, all cnt)) { if (checker && !checker->check("iterate", "processing", curcnt, all cnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true; err = true;
break; break;
} }
kit++; ++kit;
} }
if (reorg) { if (reorg) {
Record* rec = keys.front(); Record* rec = keys.front();
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
char lstack[PLDBRECBUFSIZ]; char lstack[RECBUFSIZ];
size_t lsiz = sizeof(Link) + rec->ksiz; size_t lsiz = sizeof(Link) + rec->ksiz;
char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack; char* lbuf = lsiz > sizeof(lstack) ? new char[lsiz] : lstack;
Link* link = (Link*)lbuf; Link* link = (Link*)lbuf;
link->child = 0; link->child = 0;
link->ksiz = rec->ksiz; link->ksiz = rec->ksiz;
std::memcpy(lbuf + sizeof(*link), dbuf, rec->ksiz); std::memcpy(lbuf + sizeof(*link), dbuf, rec->ksiz);
int64_t hist[PLDBLEVELMAX]; int64_t hist[LEVELMAX];
int32_t hnum = 0; int32_t hnum = 0;
node = search_tree(link, false, hist, &hnum); node = search_tree(link, false, hist, &hnum);
if (node) { if (node) {
if (!reorganize_tree(node, hist, hnum)) err = true; if (!reorganize_tree(node, hist, hnum)) err = true;
} else { } else {
set_error(_KCCODELINE_, Error::BROKEN, "search failed"); set_error(_KCCODELINE_, Error::BROKEN, "search failed");
err = true; err = true;
} }
if (lbuf != lstack) delete[] lbuf; if (lbuf != lstack) delete[] lbuf;
} }
if (cusage_ > pccap_) { if (cusage_ > pccap_) {
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
LeafSlot* lslot = lslots_ + i; LeafSlot* lslot = lslots_ + i;
if (!flush_leaf_cache_part(lslot)) err = true; if (!flush_leaf_cache_part(lslot)) err = true;
} }
InnerSlot* islot = islots_ + (flcnt++) % PLDBSLOTNUM; InnerSlot* islot = islots_ + (flcnt++) % SLOTNUM;
if (islot->warm->count() > 2 && !flush_inner_cache_part(islot)) err = true; if (islot->warm->count() > 2 && !flush_inner_cache_part(islot)) err = true;
} }
kit = keys.begin(); kit = keys.begin();
while (kit != kitend) { while (kit != kitend) {
xfree(*kit); xfree(*kit);
kit++; ++kit;
} }
} }
if (checker && !checker->check("iterate", "ending", -1, allcnt)) { if (checker && !checker->check("iterate", "ending", -1, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true; err = true;
} }
if (atran && !commit_transaction()) err = true; if (atran && !commit_transaction()) err = true;
if (autosync_ && !autotran_ && writable && !fix_auto_synchronization()) err = true; if (autosync_ && !autotran_ && writable && !fix_auto_synchronization()) err = true;
trigger_meta(MetaTrigger::ITERATE, "iterate"); trigger_meta(MetaTrigger::ITERATE, "iterate");
return !err; return !err;
skipping to change at line 1516 skipping to change at line 1494
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wcnt >= PLDBLOCKBUSYLOOP) { if (wcnt >= LOCKBUSYLOOP) {
Thread::chill(); Thread::chill();
} else { } else {
Thread::yield(); Thread::yield();
wcnt++; wcnt++;
} }
} }
if (!begin_transaction_impl(hard)) { if (!begin_transaction_impl(hard)) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
skipping to change at line 1711 skipping to change at line 1689
(*strmap)["cusage_lcnt"] = strprintf("%lld", (long long)calc_leaf_cac he_count()); (*strmap)["cusage_lcnt"] = strprintf("%lld", (long long)calc_leaf_cac he_count());
if (strmap->count("cusage_lsiz") > 0) if (strmap->count("cusage_lsiz") > 0)
(*strmap)["cusage_lsiz"] = strprintf("%lld", (long long)calc_leaf_cac he_size()); (*strmap)["cusage_lsiz"] = strprintf("%lld", (long long)calc_leaf_cac he_size());
if (strmap->count("cusage_icnt") > 0) if (strmap->count("cusage_icnt") > 0)
(*strmap)["cusage_icnt"] = strprintf("%lld", (long long)calc_inner_ca che_count()); (*strmap)["cusage_icnt"] = strprintf("%lld", (long long)calc_inner_ca che_count());
if (strmap->count("cusage_isiz") > 0) if (strmap->count("cusage_isiz") > 0)
(*strmap)["cusage_isiz"] = strprintf("%lld", (long long)calc_inner_ca che_size()); (*strmap)["cusage_isiz"] = strprintf("%lld", (long long)calc_inner_ca che_size());
if (strmap->count("tree_level") > 0) { if (strmap->count("tree_level") > 0) {
Link link; Link link;
link.ksiz = 0; link.ksiz = 0;
int64_t hist[PLDBLEVELMAX]; int64_t hist[LEVELMAX];
int32_t hnum = 0; int32_t hnum = 0;
search_tree(&link, false, hist, &hnum); search_tree(&link, false, hist, &hnum);
(*strmap)["tree_level"] = strprintf("%d", hnum + 1); (*strmap)["tree_level"] = strprintf("%d", hnum + 1);
} }
return true; return true;
} }
/** /**
* Create a cursor object. * Create a cursor object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note Because the object of the return value is allocated by the const ructor, it should be * @note Because the object of the return value is allocated by the const ructor, it should be
skipping to change at line 1772 skipping to change at line 1750
* @param apow the power of the alignment of record size. * @param apow the power of the alignment of record size.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_alignment(int8_t apow) { bool tune_alignment(int8_t apow) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
apow_ = apow >= 0 ? apow : PLDBDEFAPOW; apow_ = apow >= 0 ? apow : DEFAPOW;
return true; return true;
} }
/** /**
* Set the power of the capacity of the free block pool. * Set the power of the capacity of the free block pool.
* @param fpow the power of the capacity of the free block pool. * @param fpow the power of the capacity of the free block pool.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_fbp(int8_t fpow) { bool tune_fbp(int8_t fpow) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
fpow_ = fpow >= 0 ? fpow : PLDBDEFFPOW; fpow_ = fpow >= 0 ? fpow : DEFFPOW;
return true; return true;
} }
/** /**
* Set the optional features. * Set the optional features.
* @param opts the optional features by bitwise-or: BasicDB::TSMALL to us e 32-bit addressing, * @param opts the optional features by bitwise-or: BasicDB::TSMALL to us e 32-bit addressing,
* BasicDB::TLINEAR to use linear collision chaining, BasicDB::TCOMPRESS to compress each * BasicDB::TLINEAR to use linear collision chaining, BasicDB::TCOMPRESS to compress each
* record. * record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_options(int8_t opts) { bool tune_options(int8_t opts) {
skipping to change at line 1819 skipping to change at line 1797
* @param bnum the number of buckets of the hash table. * @param bnum the number of buckets of the hash table.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_buckets(int64_t bnum) { bool tune_buckets(int64_t bnum) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
bnum_ = bnum > 0 ? bnum : PLDBDEFBNUM; bnum_ = bnum > 0 ? bnum : DEFBNUM;
return true; return true;
} }
/** /**
* Set the size of each page. * Set the size of each page.
* @param psiz the size of each page. * @param psiz the size of each page.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_page(int32_t psiz) { bool tune_page(int32_t psiz) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
psiz_ = psiz > 0 ? psiz : PLDBDEFPSIZ; psiz_ = psiz > 0 ? psiz : DEFPSIZ;
return true; return true;
} }
/** /**
* Set the size of the internal memory-mapped region. * Set the size of the internal memory-mapped region.
* @param msiz the size of the internal memory-mapped region. * @param msiz the size of the internal memory-mapped region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_map(int64_t msiz) { bool tune_map(int64_t msiz) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
skipping to change at line 1877 skipping to change at line 1855
* @param pccap the capacity size of the page cache. * @param pccap the capacity size of the page cache.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_page_cache(int64_t pccap) { bool tune_page_cache(int64_t pccap) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
pccap_ = pccap > 0 ? pccap : PLDBDEFPCCAP; pccap_ = pccap > 0 ? pccap : DEFPCCAP;
return true; return true;
} }
/** /**
* Set the data compressor. * Set the data compressor.
* @param comp the data compressor object. * @param comp the data compressor object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_compressor(Compressor* comp) { bool tune_compressor(Compressor* comp) {
_assert_(comp); _assert_(comp);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
skipping to change at line 2042 skipping to change at line 2020
* MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f or beginning * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f or beginning
* transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT rigger::ABORTTRAN * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT rigger::ABORTTRAN
* for aborting transaction, and MetaTrigger::MISC for miscellaneous oper ations. * for aborting transaction, and MetaTrigger::MISC for miscellaneous oper ations.
* @param message the supplement message. * @param message the supplement message.
*/ */
void trigger_meta(MetaTrigger::Kind kind, const char* message) { void trigger_meta(MetaTrigger::Kind kind, const char* message) {
_assert_(message); _assert_(message);
if (mtrigger_) mtrigger_->trigger(kind, message); if (mtrigger_) mtrigger_->trigger(kind, message);
} }
private: private:
/** The number of cache slots. */
static const int32_t SLOTNUM = 16;
/** The default alignment power. */
static const uint8_t DEFAPOW = 8;
/** The default free block pool power. */
static const uint8_t DEFFPOW = 10;
/** The default bucket number. */
static const int64_t DEFBNUM = 64LL << 10;
/** The default page size. */
static const int32_t DEFPSIZ = 8192;
/** The default capacity size of the page cache. */
static const int64_t DEFPCCAP = 64LL << 20;
/** The size of the header. */
static const int64_t HEADSIZ = 80;
/** The offset of the numbers. */
static const int64_t MOFFNUMS = 8;
/** The prefix of leaf nodes. */
static const char LNPREFIX = 'L';
/** The prefix of inner nodes. */
static const char INPREFIX = 'I';
/** The average number of ways of each node. */
static const size_t AVGWAY = 16;
/** The ratio of the warm cache. */
static const size_t WARMRATIO = 4;
/** The ratio of flushing inner nodes. */
static const size_t INFLRATIO = 32;
/** The default number of items in each leaf node. */
static const size_t DEFLINUM = 64;
/** The default number of items in each inner node. */
static const size_t DEFIINUM = 128;
/** The size of the record buffer. */
static const size_t RECBUFSIZ = 64;
/** The base ID number for inner nodes. */
static const int64_t INIDBASE = 1LL << 48;
/** The minimum number of links in each inner node. */
static const size_t INLINKMIN = 8;
/** The maximum level of B+ tree. */
static const int32_t LEVELMAX = 16;
/** The number of cached nodes for auto transaction. */
static const int32_t ATRANCNUM = 256;
/** The threshold of busy loop and sleep for locking. */
static const uint32_t LOCKBUSYLOOP = 8192;
/** /**
* Record data. * Record data.
*/ */
struct Record { struct Record {
uint32_t ksiz; ///< size of the key uint32_t ksiz; ///< size of the key
uint32_t vsiz; ///< size of the value uint32_t vsiz; ///< size of the value
}; };
/** /**
* Comparator for records. * Comparator for records.
*/ */
skipping to change at line 2130 skipping to change at line 2150
LeafCache* warm; ///< warm cache LeafCache* warm; ///< warm cache
}; };
/** /**
* Slot cache of inner nodes. * Slot cache of inner nodes.
*/ */
struct InnerSlot { struct InnerSlot {
SpinLock lock; ///< lock SpinLock lock; ///< lock
InnerCache* warm; ///< warm cache InnerCache* warm; ///< warm cache
}; };
/** /**
* Scoped visiotor.
*/
class ScopedVisitor {
public:
/** constructor */
ScopedVisitor(Visitor* visitor) : visitor_(visitor) {
_assert_(visitor);
visitor_->visit_before();
}
/** destructor */
~ScopedVisitor() {
_assert_(true);
visitor_->visit_after();
}
private:
Visitor* visitor_; ///< visitor
};
/**
* Open the leaf cache. * Open the leaf cache.
*/ */
void create_leaf_cache() { void create_leaf_cache() {
_assert_(true); _assert_(true);
int64_t bnum = bnum_ / PLDBSLOTNUM + 1; int64_t bnum = bnum_ / SLOTNUM + 1;
if (bnum < INT8MAX) bnum = INT8MAX; if (bnum < INT8MAX) bnum = INT8MAX;
bnum = nearbyprime(bnum); bnum = nearbyprime(bnum);
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
lslots_[i].hot = new LeafCache(bnum); lslots_[i].hot = new LeafCache(bnum);
lslots_[i].warm = new LeafCache(bnum); lslots_[i].warm = new LeafCache(bnum);
} }
} }
/** /**
* Close the leaf cache. * Close the leaf cache.
*/ */
void delete_leaf_cache() { void delete_leaf_cache() {
_assert_(true); _assert_(true);
for (int32_t i = PLDBSLOTNUM - 1; i >= 0; i--) { for (int32_t i = SLOTNUM - 1; i >= 0; i--) {
LeafSlot* slot = lslots_ + i; LeafSlot* slot = lslots_ + i;
delete slot->warm; delete slot->warm;
delete slot->hot; delete slot->hot;
} }
} }
/** /**
* Remove all leaf nodes from the leaf cache. * Remove all leaf nodes from the leaf cache.
* @param save whether to save dirty nodes. * @param save whether to save dirty nodes.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool flush_leaf_cache(bool save) { bool flush_leaf_cache(bool save) {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
for (int32_t i = PLDBSLOTNUM - 1; i >= 0; i--) { for (int32_t i = SLOTNUM - 1; i >= 0; i--) {
LeafSlot* slot = lslots_ + i; LeafSlot* slot = lslots_ + i;
typename LeafCache::Iterator it = slot->warm->begin(); typename LeafCache::Iterator it = slot->warm->begin();
typename LeafCache::Iterator itend = slot->warm->end(); typename LeafCache::Iterator itend = slot->warm->end();
while (it != itend) { while (it != itend) {
LeafNode* node = it.value(); LeafNode* node = it.value();
it++; ++it;
if (!flush_leaf_node(node, save)) err = true; if (!flush_leaf_node(node, save)) err = true;
} }
it = slot->hot->begin(); it = slot->hot->begin();
itend = slot->hot->end(); itend = slot->hot->end();
while (it != itend) { while (it != itend) {
LeafNode* node = it.value(); LeafNode* node = it.value();
it++; ++it;
if (!flush_leaf_node(node, save)) err = true; if (!flush_leaf_node(node, save)) err = true;
} }
} }
return !err; return !err;
} }
/** /**
* Flush a part of the leaf cache. * Flush a part of the leaf cache.
* @param slot a slot of leaf nodes. * @param slot a slot of leaf nodes.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
skipping to change at line 2204 skipping to change at line 2242
} }
return !err; return !err;
} }
/** /**
* Clean all of the leaf cache. * Clean all of the leaf cache.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clean_leaf_cache() { bool clean_leaf_cache() {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
LeafSlot* slot = lslots_ + i; LeafSlot* slot = lslots_ + i;
ScopedMutex lock(&slot->lock); ScopedMutex lock(&slot->lock);
typename LeafCache::Iterator it = slot->warm->begin(); typename LeafCache::Iterator it = slot->warm->begin();
typename LeafCache::Iterator itend = slot->warm->end(); typename LeafCache::Iterator itend = slot->warm->end();
while (it != itend) { while (it != itend) {
LeafNode* node = it.value(); LeafNode* node = it.value();
if (!save_leaf_node(node)) err = true; if (!save_leaf_node(node)) err = true;
it++; ++it;
} }
it = slot->hot->begin(); it = slot->hot->begin();
itend = slot->hot->end(); itend = slot->hot->end();
while (it != itend) { while (it != itend) {
LeafNode* node = it.value(); LeafNode* node = it.value();
if (!save_leaf_node(node)) err = true; if (!save_leaf_node(node)) err = true;
it++; ++it;
} }
} }
return !err; return !err;
} }
/** /**
* Clean a part of the leaf cache. * Clean a part of the leaf cache.
* @param slot a slot of leaf nodes. * @param slot a slot of leaf nodes.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clean_leaf_cache_part(LeafSlot* slot) { bool clean_leaf_cache_part(LeafSlot* slot) {
skipping to change at line 2253 skipping to change at line 2291
* Create a new leaf node. * Create a new leaf node.
* @param prev the ID of the previous node. * @param prev the ID of the previous node.
* @param next the ID of the next node. * @param next the ID of the next node.
* @return the created leaf node. * @return the created leaf node.
*/ */
LeafNode* create_leaf_node(int64_t prev, int64_t next) { LeafNode* create_leaf_node(int64_t prev, int64_t next) {
_assert_(true); _assert_(true);
LeafNode* node = new LeafNode; LeafNode* node = new LeafNode;
node->id = ++lcnt_; node->id = ++lcnt_;
node->size = sizeof(int32_t) * 2; node->size = sizeof(int32_t) * 2;
node->recs.reserve(PLDBDEFLINUM); node->recs.reserve(DEFLINUM);
node->prev = prev; node->prev = prev;
node->next = next; node->next = next;
node->hot = false; node->hot = false;
node->dirty = true; node->dirty = true;
node->dead = false; node->dead = false;
int32_t sidx = node->id % PLDBSLOTNUM; int32_t sidx = node->id % SLOTNUM;
LeafSlot* slot = lslots_ + sidx; LeafSlot* slot = lslots_ + sidx;
slot->warm->set(node->id, node, LeafCache::MLAST); slot->warm->set(node->id, node, LeafCache::MLAST);
cusage_ += node->size; cusage_ += node->size;
return node; return node;
} }
/** /**
* Remove a leaf node from the cache. * Remove a leaf node from the cache.
* @param node the leaf node. * @param node the leaf node.
* @param save whether to save dirty node. * @param save whether to save dirty node.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool flush_leaf_node(LeafNode* node, bool save) { bool flush_leaf_node(LeafNode* node, bool save) {
_assert_(node); _assert_(node);
bool err = false; bool err = false;
if (save && !save_leaf_node(node)) err = true; if (save && !save_leaf_node(node)) err = true;
typename RecordArray::const_iterator rit = node->recs.begin(); typename RecordArray::const_iterator rit = node->recs.begin();
typename RecordArray::const_iterator ritend = node->recs.end(); typename RecordArray::const_iterator ritend = node->recs.end();
while (rit != ritend) { while (rit != ritend) {
Record* rec = *rit; Record* rec = *rit;
xfree(rec); xfree(rec);
rit++; ++rit;
} }
int32_t sidx = node->id % PLDBSLOTNUM; int32_t sidx = node->id % SLOTNUM;
LeafSlot* slot = lslots_ + sidx; LeafSlot* slot = lslots_ + sidx;
if (node->hot) { if (node->hot) {
slot->hot->remove(node->id); slot->hot->remove(node->id);
} else { } else {
slot->warm->remove(node->id); slot->warm->remove(node->id);
} }
cusage_ -= node->size; cusage_ -= node->size;
delete node; delete node;
return !err; return !err;
} }
skipping to change at line 2304 skipping to change at line 2342
* Save a leaf node. * Save a leaf node.
* @param node the leaf node. * @param node the leaf node.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool save_leaf_node(LeafNode* node) { bool save_leaf_node(LeafNode* node) {
_assert_(node); _assert_(node);
ScopedSpinRWLock lock(&node->lock, false); ScopedSpinRWLock lock(&node->lock, false);
if (!node->dirty) return true; if (!node->dirty) return true;
bool err = false; bool err = false;
char hbuf[NUMBUFSIZ]; char hbuf[NUMBUFSIZ];
size_t hsiz = std::sprintf(hbuf, "%c%llX", PLDBLNPREFIX, (long long)nod e->id); size_t hsiz = std::sprintf(hbuf, "%c%llX", LNPREFIX, (long long)node->i d);
if (node->dead) { if (node->dead) {
if (!db_.remove(hbuf, hsiz) && db_.error().code() != Error::NOREC) er r = true; if (!db_.remove(hbuf, hsiz) && db_.error().code() != Error::NOREC) er r = true;
} else { } else {
char* rbuf = new char[node->size]; char* rbuf = new char[node->size];
char* wp = rbuf; char* wp = rbuf;
wp += writevarnum(wp, node->prev); wp += writevarnum(wp, node->prev);
wp += writevarnum(wp, node->next); wp += writevarnum(wp, node->next);
typename RecordArray::const_iterator rit = node->recs.begin(); typename RecordArray::const_iterator rit = node->recs.begin();
typename RecordArray::const_iterator ritend = node->recs.end(); typename RecordArray::const_iterator ritend = node->recs.end();
while (rit != ritend) { while (rit != ritend) {
Record* rec = *rit; Record* rec = *rit;
wp += writevarnum(wp, rec->ksiz); wp += writevarnum(wp, rec->ksiz);
wp += writevarnum(wp, rec->vsiz); wp += writevarnum(wp, rec->vsiz);
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
std::memcpy(wp, dbuf, rec->ksiz); std::memcpy(wp, dbuf, rec->ksiz);
wp += rec->ksiz; wp += rec->ksiz;
std::memcpy(wp, dbuf + rec->ksiz, rec->vsiz); std::memcpy(wp, dbuf + rec->ksiz, rec->vsiz);
wp += rec->vsiz; wp += rec->vsiz;
rit++; ++rit;
} }
if (!db_.set(hbuf, hsiz, rbuf, wp - rbuf)) err = true; if (!db_.set(hbuf, hsiz, rbuf, wp - rbuf)) err = true;
delete[] rbuf; delete[] rbuf;
} }
node->dirty = false; node->dirty = false;
return !err; return !err;
} }
/** /**
* Load a leaf node. * Load a leaf node.
* @param id the ID number of the leaf node. * @param id the ID number of the leaf node.
* @param prom whether to promote the warm cache. * @param prom whether to promote the warm cache.
* @return the loaded leaf node. * @return the loaded leaf node.
*/ */
LeafNode* load_leaf_node(int64_t id, bool prom) { LeafNode* load_leaf_node(int64_t id, bool prom) {
_assert_(id > 0); _assert_(id > 0);
int32_t sidx = id % PLDBSLOTNUM; int32_t sidx = id % SLOTNUM;
LeafSlot* slot = lslots_ + sidx; LeafSlot* slot = lslots_ + sidx;
ScopedMutex lock(&slot->lock); ScopedMutex lock(&slot->lock);
LeafNode** np = slot->hot->get(id, LeafCache::MLAST); LeafNode** np = slot->hot->get(id, LeafCache::MLAST);
if (np) return *np; if (np) return *np;
if (prom) { if (prom) {
if (slot->hot->count() * PLDBWARMRATIO > slot->warm->count() + PLDBWA RMRATIO) { if (slot->hot->count() * WARMRATIO > slot->warm->count() + WARMRATIO) {
slot->hot->first_value()->hot = false; slot->hot->first_value()->hot = false;
slot->hot->migrate(slot->hot->first_key(), slot->warm, LeafCache::M LAST); slot->hot->migrate(slot->hot->first_key(), slot->warm, LeafCache::M LAST);
} }
np = slot->warm->migrate(id, slot->hot, LeafCache::MLAST); np = slot->warm->migrate(id, slot->hot, LeafCache::MLAST);
if (np) { if (np) {
(*np)->hot = true; (*np)->hot = true;
return *np; return *np;
} }
} else { } else {
LeafNode** np = slot->warm->get(id, LeafCache::MLAST); LeafNode** np = slot->warm->get(id, LeafCache::MLAST);
if (np) return *np; if (np) return *np;
} }
char hbuf[NUMBUFSIZ]; char hbuf[NUMBUFSIZ];
size_t hsiz = std::sprintf(hbuf, "%c%llX", PLDBLNPREFIX, (long long)id) ; size_t hsiz = std::sprintf(hbuf, "%c%llX", LNPREFIX, (long long)id);
class VisitorImpl : public DB::Visitor { class VisitorImpl : public DB::Visitor {
public: public:
explicit VisitorImpl() : node_(NULL) {} explicit VisitorImpl() : node_(NULL) {}
LeafNode* pop() { LeafNode* pop() {
return node_; return node_;
} }
private: private:
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
uint64_t prev; uint64_t prev;
skipping to change at line 2415 skipping to change at line 2453
vsiz -= rvsiz; vsiz -= rvsiz;
node->recs.push_back(rec); node->recs.push_back(rec);
node->size += rsiz; node->size += rsiz;
} }
if (vsiz != 0) { if (vsiz != 0) {
typename RecordArray::const_iterator rit = node->recs.begin(); typename RecordArray::const_iterator rit = node->recs.begin();
typename RecordArray::const_iterator ritend = node->recs.end(); typename RecordArray::const_iterator ritend = node->recs.end();
while (rit != ritend) { while (rit != ritend) {
Record* rec = *rit; Record* rec = *rit;
xfree(rec); xfree(rec);
rit++; ++rit;
} }
delete node; delete node;
return NOP; return NOP;
} }
node_ = node; node_ = node;
return NOP; return NOP;
} }
LeafNode* node_; LeafNode* node_;
} visitor; } visitor;
if (!db_.accept(hbuf, hsiz, &visitor, false)) return NULL; if (!db_.accept(hbuf, hsiz, &visitor, false)) return NULL;
skipping to change at line 2546 skipping to change at line 2584
typename RecordArray::iterator mid = recs.begin() + recs.size() / 2; typename RecordArray::iterator mid = recs.begin() + recs.size() / 2;
typename RecordArray::iterator rit = mid; typename RecordArray::iterator rit = mid;
typename RecordArray::iterator ritend = recs.end(); typename RecordArray::iterator ritend = recs.end();
RecordArray& newrecs = newnode->recs; RecordArray& newrecs = newnode->recs;
while (rit != ritend) { while (rit != ritend) {
Record* rec = *rit; Record* rec = *rit;
newrecs.push_back(rec); newrecs.push_back(rec);
size_t rsiz = sizeof(*rec) + rec->ksiz + rec->vsiz; size_t rsiz = sizeof(*rec) + rec->ksiz + rec->vsiz;
node->size -= rsiz; node->size -= rsiz;
newnode->size += rsiz; newnode->size += rsiz;
rit++; ++rit;
} }
escape_cursors(node->id, node->next, *mid); escape_cursors(node->id, node->next, *mid);
recs.erase(mid, ritend); recs.erase(mid, ritend);
return newnode; return newnode;
} }
/** /**
* Open the inner cache. * Open the inner cache.
*/ */
void create_inner_cache() { void create_inner_cache() {
_assert_(true); _assert_(true);
int64_t bnum = (bnum_ / PLDBAVGWAY) / PLDBSLOTNUM + 1; int64_t bnum = (bnum_ / AVGWAY) / SLOTNUM + 1;
if (bnum < INT8MAX) bnum = INT8MAX; if (bnum < INT8MAX) bnum = INT8MAX;
bnum = nearbyprime(bnum); bnum = nearbyprime(bnum);
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
islots_[i].warm = new InnerCache(bnum); islots_[i].warm = new InnerCache(bnum);
} }
} }
/** /**
* Close the inner cache. * Close the inner cache.
*/ */
void delete_inner_cache() { void delete_inner_cache() {
_assert_(true); _assert_(true);
for (int32_t i = PLDBSLOTNUM - 1; i >= 0; i--) { for (int32_t i = SLOTNUM - 1; i >= 0; i--) {
InnerSlot* slot = islots_ + i; InnerSlot* slot = islots_ + i;
delete slot->warm; delete slot->warm;
} }
} }
/** /**
* Remove all inner nodes from the inner cache. * Remove all inner nodes from the inner cache.
* @param save whether to save dirty nodes. * @param save whether to save dirty nodes.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool flush_inner_cache(bool save) { bool flush_inner_cache(bool save) {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
for (int32_t i = PLDBSLOTNUM - 1; i >= 0; i--) { for (int32_t i = SLOTNUM - 1; i >= 0; i--) {
InnerSlot* slot = islots_ + i; InnerSlot* slot = islots_ + i;
typename InnerCache::Iterator it = slot->warm->begin(); typename InnerCache::Iterator it = slot->warm->begin();
typename InnerCache::Iterator itend = slot->warm->end(); typename InnerCache::Iterator itend = slot->warm->end();
while (it != itend) { while (it != itend) {
InnerNode* node = it.value(); InnerNode* node = it.value();
it++; ++it;
if (!flush_inner_node(node, save)) err = true; if (!flush_inner_node(node, save)) err = true;
} }
} }
return !err; return !err;
} }
/** /**
* Flush a part of the inner cache. * Flush a part of the inner cache.
* @param slot a slot of inner nodes. * @param slot a slot of inner nodes.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
skipping to change at line 2615 skipping to change at line 2653
} }
return !err; return !err;
} }
/** /**
* Clean all of the inner cache. * Clean all of the inner cache.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clean_inner_cache() { bool clean_inner_cache() {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
InnerSlot* slot = islots_ + i; InnerSlot* slot = islots_ + i;
ScopedSpinLock lock(&slot->lock); ScopedSpinLock lock(&slot->lock);
typename InnerCache::Iterator it = slot->warm->begin(); typename InnerCache::Iterator it = slot->warm->begin();
typename InnerCache::Iterator itend = slot->warm->end(); typename InnerCache::Iterator itend = slot->warm->end();
while (it != itend) { while (it != itend) {
InnerNode* node = it.value(); InnerNode* node = it.value();
if (!save_inner_node(node)) err = true; if (!save_inner_node(node)) err = true;
it++; ++it;
} }
} }
return !err; return !err;
} }
/** /**
* Create a new inner node. * Create a new inner node.
* @param heir the ID of the child before the first link. * @param heir the ID of the child before the first link.
* @return the created inner node. * @return the created inner node.
*/ */
InnerNode* create_inner_node(int64_t heir) { InnerNode* create_inner_node(int64_t heir) {
_assert_(true); _assert_(true);
InnerNode* node = new InnerNode; InnerNode* node = new InnerNode;
node->id = ++icnt_ + PLDBINIDBASE; node->id = ++icnt_ + INIDBASE;
node->heir = heir; node->heir = heir;
node->links.reserve(PLDBDEFIINUM); node->links.reserve(DEFIINUM);
node->size = sizeof(int64_t); node->size = sizeof(int64_t);
node->dirty = true; node->dirty = true;
node->dead = false; node->dead = false;
int32_t sidx = node->id % PLDBSLOTNUM; int32_t sidx = node->id % SLOTNUM;
InnerSlot* slot = islots_ + sidx; InnerSlot* slot = islots_ + sidx;
slot->warm->set(node->id, node, InnerCache::MLAST); slot->warm->set(node->id, node, InnerCache::MLAST);
cusage_ += node->size; cusage_ += node->size;
return node; return node;
} }
/** /**
* Remove an inner node from the cache. * Remove an inner node from the cache.
* @param node the inner node. * @param node the inner node.
* @param save whether to save dirty node. * @param save whether to save dirty node.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool flush_inner_node(InnerNode* node, bool save) { bool flush_inner_node(InnerNode* node, bool save) {
_assert_(node); _assert_(node);
bool err = false; bool err = false;
if (save && !save_inner_node(node)) err = true; if (save && !save_inner_node(node)) err = true;
typename LinkArray::const_iterator lit = node->links.begin(); typename LinkArray::const_iterator lit = node->links.begin();
typename LinkArray::const_iterator litend = node->links.end(); typename LinkArray::const_iterator litend = node->links.end();
while (lit != litend) { while (lit != litend) {
Link* link = *lit; Link* link = *lit;
xfree(link); xfree(link);
lit++; ++lit;
} }
int32_t sidx = node->id % PLDBSLOTNUM; int32_t sidx = node->id % SLOTNUM;
InnerSlot* slot = islots_ + sidx; InnerSlot* slot = islots_ + sidx;
slot->warm->remove(node->id); slot->warm->remove(node->id);
cusage_ -= node->size; cusage_ -= node->size;
delete node; delete node;
return !err; return !err;
} }
/** /**
* Save a inner node. * Save a inner node.
* @param node the inner node. * @param node the inner node.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool save_inner_node(InnerNode* node) { bool save_inner_node(InnerNode* node) {
_assert_(true); _assert_(true);
if (!node->dirty) return true; if (!node->dirty) return true;
bool err = false; bool err = false;
char hbuf[NUMBUFSIZ]; char hbuf[NUMBUFSIZ];
size_t hsiz = std::sprintf(hbuf, "%c%llX", size_t hsiz = std::sprintf(hbuf, "%c%llX",
PLDBINPREFIX, (long long)(node->id - PLDBINI DBASE)); INPREFIX, (long long)(node->id - INIDBASE));
if (node->dead) { if (node->dead) {
if (!db_.remove(hbuf, hsiz) && db_.error().code() != Error::NOREC) er r = true; if (!db_.remove(hbuf, hsiz) && db_.error().code() != Error::NOREC) er r = true;
} else { } else {
char* rbuf = new char[node->size]; char* rbuf = new char[node->size];
char* wp = rbuf; char* wp = rbuf;
wp += writevarnum(wp, node->heir); wp += writevarnum(wp, node->heir);
typename LinkArray::const_iterator lit = node->links.begin(); typename LinkArray::const_iterator lit = node->links.begin();
typename LinkArray::const_iterator litend = node->links.end(); typename LinkArray::const_iterator litend = node->links.end();
while (lit != litend) { while (lit != litend) {
Link* link = *lit; Link* link = *lit;
wp += writevarnum(wp, link->child); wp += writevarnum(wp, link->child);
wp += writevarnum(wp, link->ksiz); wp += writevarnum(wp, link->ksiz);
char* dbuf = (char*)link + sizeof(*link); char* dbuf = (char*)link + sizeof(*link);
std::memcpy(wp, dbuf, link->ksiz); std::memcpy(wp, dbuf, link->ksiz);
wp += link->ksiz; wp += link->ksiz;
lit++; ++lit;
} }
if (!db_.set(hbuf, hsiz, rbuf, wp - rbuf)) err = true; if (!db_.set(hbuf, hsiz, rbuf, wp - rbuf)) err = true;
delete[] rbuf; delete[] rbuf;
} }
node->dirty = false; node->dirty = false;
return !err; return !err;
} }
/** /**
* Load an inner node. * Load an inner node.
* @param id the ID number of the inner node. * @param id the ID number of the inner node.
* @return the loaded inner node. * @return the loaded inner node.
*/ */
InnerNode* load_inner_node(int64_t id) { InnerNode* load_inner_node(int64_t id) {
_assert_(id > 0); _assert_(id > 0);
int32_t sidx = id % PLDBSLOTNUM; int32_t sidx = id % SLOTNUM;
InnerSlot* slot = islots_ + sidx; InnerSlot* slot = islots_ + sidx;
ScopedSpinLock lock(&slot->lock); ScopedSpinLock lock(&slot->lock);
InnerNode** np = slot->warm->get(id, InnerCache::MLAST); InnerNode** np = slot->warm->get(id, InnerCache::MLAST);
if (np) return *np; if (np) return *np;
char hbuf[NUMBUFSIZ]; char hbuf[NUMBUFSIZ];
size_t hsiz = std::sprintf(hbuf, "%c%llX", PLDBINPREFIX, (long long)(id - PLDBINIDBASE)); size_t hsiz = std::sprintf(hbuf, "%c%llX", INPREFIX, (long long)(id - I NIDBASE));
class VisitorImpl : public DB::Visitor { class VisitorImpl : public DB::Visitor {
public: public:
explicit VisitorImpl() : node_(NULL) {} explicit VisitorImpl() : node_(NULL) {}
InnerNode* pop() { InnerNode* pop() {
return node_; return node_;
} }
private: private:
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
uint64_t heir; uint64_t heir;
skipping to change at line 2766 skipping to change at line 2804
vsiz -= rksiz; vsiz -= rksiz;
node->links.push_back(link); node->links.push_back(link);
node->size += sizeof(*link) + rksiz; node->size += sizeof(*link) + rksiz;
} }
if (vsiz != 0) { if (vsiz != 0) {
typename LinkArray::const_iterator lit = node->links.begin(); typename LinkArray::const_iterator lit = node->links.begin();
typename LinkArray::const_iterator litend = node->links.end(); typename LinkArray::const_iterator litend = node->links.end();
while (lit != litend) { while (lit != litend) {
Link* link = *lit; Link* link = *lit;
xfree(link); xfree(link);
lit++; ++lit;
} }
delete node; delete node;
return NOP; return NOP;
} }
node_ = node; node_ = node;
return NOP; return NOP;
} }
InnerNode* node_; InnerNode* node_;
} visitor; } visitor;
if (!db_.accept(hbuf, hsiz, &visitor, false)) return NULL; if (!db_.accept(hbuf, hsiz, &visitor, false)) return NULL;
skipping to change at line 2798 skipping to change at line 2836
* @param link the link containing the key only. * @param link the link containing the key only.
* @param prom whether to promote the warm cache. * @param prom whether to promote the warm cache.
* @param hist the array of visiting history. * @param hist the array of visiting history.
* @param hnp the pointer to the variable into which the number of the hi story is assigned. * @param hnp the pointer to the variable into which the number of the hi story is assigned.
* @return the corresponding leaf node, or NULL on failure. * @return the corresponding leaf node, or NULL on failure.
*/ */
LeafNode* search_tree(Link* link, bool prom, int64_t* hist, int32_t* hnp) { LeafNode* search_tree(Link* link, bool prom, int64_t* hist, int32_t* hnp) {
_assert_(link && hist && hnp); _assert_(link && hist && hnp);
int64_t id = root_; int64_t id = root_;
int32_t hnum = 0; int32_t hnum = 0;
while (id > PLDBINIDBASE) { while (id > INIDBASE) {
InnerNode* node = load_inner_node(id); InnerNode* node = load_inner_node(id);
if (!node) { if (!node) {
set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); set_error(_KCCODELINE_, Error::BROKEN, "missing inner node");
db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)id); db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)id);
return NULL; return NULL;
} }
hist[hnum++] = id; hist[hnum++] = id;
const LinkArray& links = node->links; const LinkArray& links = node->links;
typename LinkArray::const_iterator litbeg = links.begin(); typename LinkArray::const_iterator litbeg = links.begin();
typename LinkArray::const_iterator litend = links.end(); typename LinkArray::const_iterator litend = links.end();
typename LinkArray::const_iterator lit = std::upper_bound(litbeg, lit end, link, linkcomp_); typename LinkArray::const_iterator lit = std::upper_bound(litbeg, lit end, link, linkcomp_);
if (lit == litbeg) { if (lit == litbeg) {
id = node->heir; id = node->heir;
} else { } else {
lit--; --lit;
Link* link = *lit; Link* link = *lit;
id = link->child; id = link->child;
} }
} }
*hnp = hnum; *hnp = hnum;
return load_leaf_node(id, prom); return load_leaf_node(id, prom);
} }
/** /**
* Reorganize the B+ tree. * Reorganize the B+ tree.
* @param node a leaf node. * @param node a leaf node.
skipping to change at line 2860 skipping to change at line 2898
InnerNode* inode = load_inner_node(parent); InnerNode* inode = load_inner_node(parent);
if (!inode) { if (!inode) {
set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); set_error(_KCCODELINE_, Error::BROKEN, "missing inner node");
db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)pare nt); db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)pare nt);
delete[] kbuf; delete[] kbuf;
return false; return false;
} }
add_link_inner_node(inode, child, kbuf, ksiz); add_link_inner_node(inode, child, kbuf, ksiz);
delete[] kbuf; delete[] kbuf;
LinkArray& links = inode->links; LinkArray& links = inode->links;
if (inode->size <= psiz_ || links.size() <= PLDBINLINKMIN) break; if (inode->size <= psiz_ || links.size() <= INLINKMIN) break;
typename LinkArray::iterator litbeg = links.begin(); typename LinkArray::iterator litbeg = links.begin();
typename LinkArray::iterator mid = litbeg + links.size() / 2; typename LinkArray::iterator mid = litbeg + links.size() / 2;
Link* link = *mid; Link* link = *mid;
InnerNode* newinode = create_inner_node(link->child); InnerNode* newinode = create_inner_node(link->child);
heir = inode->id; heir = inode->id;
child = newinode->id; child = newinode->id;
char* dbuf = (char*)link + sizeof(*link); char* dbuf = (char*)link + sizeof(*link);
ksiz = link->ksiz; ksiz = link->ksiz;
kbuf = new char[ksiz]; kbuf = new char[ksiz];
std::memcpy(kbuf, dbuf, ksiz); std::memcpy(kbuf, dbuf, ksiz);
typename LinkArray::iterator lit = mid + 1; typename LinkArray::iterator lit = mid + 1;
typename LinkArray::iterator litend = links.end(); typename LinkArray::iterator litend = links.end();
while (lit != litend) { while (lit != litend) {
link = *lit; link = *lit;
char* dbuf = (char*)link + sizeof(*link); char* dbuf = (char*)link + sizeof(*link);
add_link_inner_node(newinode, link->child, dbuf, link->ksiz); add_link_inner_node(newinode, link->child, dbuf, link->ksiz);
lit++; ++lit;
} }
int32_t num = newinode->links.size(); int32_t num = newinode->links.size();
for (int32_t i = 0; i <= num; i++) { for (int32_t i = 0; i <= num; i++) {
Link* link = links.back(); Link* link = links.back();
size_t rsiz = sizeof(*link) + link->ksiz; size_t rsiz = sizeof(*link) + link->ksiz;
cusage_ -= rsiz; cusage_ -= rsiz;
inode->size -= rsiz; inode->size -= rsiz;
xfree(link); xfree(link);
links.pop_back(); links.pop_back();
} }
skipping to change at line 2982 skipping to change at line 3020
if (!pnode) { if (!pnode) {
set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); set_error(_KCCODELINE_, Error::BROKEN, "missing inner node");
db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)hist [hnum]); db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)hist [hnum]);
return false; return false;
} }
node->dead = true; node->dead = true;
return sub_link_tree(pnode, node->id, hist, hnum); return sub_link_tree(pnode, node->id, hist, hnum);
} }
node->dead = true; node->dead = true;
root_ = child; root_ = child;
while (child > PLDBINIDBASE) { while (child > INIDBASE) {
node = load_inner_node(child); node = load_inner_node(child);
if (!node) { if (!node) {
set_error(_KCCODELINE_, Error::BROKEN, "missing inner node"); set_error(_KCCODELINE_, Error::BROKEN, "missing inner node");
db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)chil d); db_.report(_KCCODELINE_, Logger::WARN, "id=%lld", (long long)chil d);
return false; return false;
} }
if (node->dead) { if (node->dead) {
child = node->heir; child = node->heir;
root_ = child; root_ = child;
} else { } else {
skipping to change at line 3005 skipping to change at line 3043
} }
return false; return false;
} }
while (lit != litend) { while (lit != litend) {
Link* link = *lit; Link* link = *lit;
if (link->child == child) { if (link->child == child) {
xfree(link); xfree(link);
links.erase(lit); links.erase(lit);
return true; return true;
} }
lit++; ++lit;
} }
set_error(_KCCODELINE_, Error::BROKEN, "invalid tree"); set_error(_KCCODELINE_, Error::BROKEN, "invalid tree");
return false; return false;
} }
/** /**
* Dump the meta data into the file. * Dump the meta data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_meta() { bool dump_meta() {
_assert_(true); _assert_(true);
char head[PLDBHEADSIZ]; char head[HEADSIZ];
std::memset(head, 0, sizeof(head)); std::memset(head, 0, sizeof(head));
char* wp = head; char* wp = head;
if (reccomp_.comp == LEXICALCOMP) { if (reccomp_.comp == LEXICALCOMP) {
*(uint8_t*)(wp++) = 0x10; *(uint8_t*)(wp++) = 0x10;
} else if (reccomp_.comp == DECIMALCOMP) { } else if (reccomp_.comp == DECIMALCOMP) {
*(uint8_t*)(wp++) = 0x11; *(uint8_t*)(wp++) = 0x11;
} else if (reccomp_.comp == LEXICALDESCCOMP) { } else if (reccomp_.comp == LEXICALDESCCOMP) {
*(uint8_t*)(wp++) = 0x18; *(uint8_t*)(wp++) = 0x18;
} else if (reccomp_.comp == DECIMALDESCCOMP) { } else if (reccomp_.comp == DECIMALDESCCOMP) {
*(uint8_t*)(wp++) = 0x19; *(uint8_t*)(wp++) = 0x19;
} else { } else {
*(uint8_t*)(wp++) = 0xff; *(uint8_t*)(wp++) = 0xff;
} }
wp = head + PLDBMOFFNUMS; wp = head + MOFFNUMS;
uint64_t num = hton64(psiz_); uint64_t num = hton64(psiz_);
std::memcpy(wp, &num, sizeof(num)); std::memcpy(wp, &num, sizeof(num));
wp += sizeof(num); wp += sizeof(num);
num = hton64(root_); num = hton64(root_);
std::memcpy(wp, &num, sizeof(num)); std::memcpy(wp, &num, sizeof(num));
wp += sizeof(num); wp += sizeof(num);
num = hton64(first_); num = hton64(first_);
std::memcpy(wp, &num, sizeof(num)); std::memcpy(wp, &num, sizeof(num));
wp += sizeof(num); wp += sizeof(num);
num = hton64(last_); num = hton64(last_);
skipping to change at line 3057 skipping to change at line 3095
std::memcpy(wp, &num, sizeof(num)); std::memcpy(wp, &num, sizeof(num));
wp += sizeof(num); wp += sizeof(num);
num = hton64(count_); num = hton64(count_);
std::memcpy(wp, &num, sizeof(num)); std::memcpy(wp, &num, sizeof(num));
wp += sizeof(num); wp += sizeof(num);
num = hton64(bnum_); num = hton64(bnum_);
std::memcpy(wp, &num, sizeof(num)); std::memcpy(wp, &num, sizeof(num));
wp += sizeof(num); wp += sizeof(num);
std::memcpy(wp, "\x0a\x42\x6f\x6f\x66\x79\x21\x0a", sizeof(num)); std::memcpy(wp, "\x0a\x42\x6f\x6f\x66\x79\x21\x0a", sizeof(num));
wp += sizeof(num); wp += sizeof(num);
if (!db_.set(PLDBMETAKEY, sizeof(PLDBMETAKEY) - 1, head, sizeof(head))) return false; if (!db_.set(KCPDBMETAKEY, sizeof(KCPDBMETAKEY) - 1, head, sizeof(head) )) return false;
trlcnt_ = lcnt_; trlcnt_ = lcnt_;
trcount_ = count_; trcount_ = count_;
return true; return true;
} }
/** /**
* Load the meta data from the file. * Load the meta data from the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_meta() { bool load_meta() {
_assert_(true); _assert_(true);
char head[PLDBHEADSIZ]; char head[HEADSIZ];
int32_t hsiz = db_.get(PLDBMETAKEY, sizeof(PLDBMETAKEY) - 1, head, size int32_t hsiz = db_.get(KCPDBMETAKEY, sizeof(KCPDBMETAKEY) - 1, head, si
of(head)); zeof(head));
if (hsiz < 0) return false; if (hsiz < 0) return false;
if (hsiz != sizeof(head)) { if (hsiz != sizeof(head)) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data record"); set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data record");
db_.report(_KCCODELINE_, Logger::WARN, "hsiz=%d", hsiz); db_.report(_KCCODELINE_, Logger::WARN, "hsiz=%d", hsiz);
return false; return false;
} }
const char* rp = head; const char* rp = head;
if (*(uint8_t*)rp == 0x10) { if (*(uint8_t*)rp == 0x10) {
reccomp_.comp = LEXICALCOMP; reccomp_.comp = LEXICALCOMP;
linkcomp_.comp = LEXICALCOMP; linkcomp_.comp = LEXICALCOMP;
skipping to change at line 3099 skipping to change at line 3137
} else if (*(uint8_t*)rp == 0xff) { } else if (*(uint8_t*)rp == 0xff) {
if (!reccomp_.comp) { if (!reccomp_.comp) {
set_error(_KCCODELINE_, Error::INVALID, "the custom comparator is n ot given"); set_error(_KCCODELINE_, Error::INVALID, "the custom comparator is n ot given");
return false; return false;
} }
linkcomp_.comp = reccomp_.comp; linkcomp_.comp = reccomp_.comp;
} else { } else {
set_error(_KCCODELINE_, Error::BROKEN, "comparator is invalid"); set_error(_KCCODELINE_, Error::BROKEN, "comparator is invalid");
return false; return false;
} }
rp = head + PLDBMOFFNUMS; rp = head + MOFFNUMS;
uint64_t num; uint64_t num;
std::memcpy(&num, rp, sizeof(num)); std::memcpy(&num, rp, sizeof(num));
psiz_ = ntoh64(num); psiz_ = ntoh64(num);
rp += sizeof(num); rp += sizeof(num);
std::memcpy(&num, rp, sizeof(num)); std::memcpy(&num, rp, sizeof(num));
root_ = ntoh64(num); root_ = ntoh64(num);
rp += sizeof(num); rp += sizeof(num);
std::memcpy(&num, rp, sizeof(num)); std::memcpy(&num, rp, sizeof(num));
first_ = ntoh64(num); first_ = ntoh64(num);
rp += sizeof(num); rp += sizeof(num);
skipping to change at line 3136 skipping to change at line 3174
trcount_ = count_; trcount_ = count_;
return true; return true;
} }
/** /**
* Caluculate the total number of nodes in the leaf cache. * Caluculate the total number of nodes in the leaf cache.
* @return the total number of nodes in the leaf cache. * @return the total number of nodes in the leaf cache.
*/ */
int64_t calc_leaf_cache_count() { int64_t calc_leaf_cache_count() {
_assert_(true); _assert_(true);
int64_t sum = 0; int64_t sum = 0;
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
LeafSlot* slot = lslots_ + i; LeafSlot* slot = lslots_ + i;
sum += slot->warm->count(); sum += slot->warm->count();
sum += slot->hot->count(); sum += slot->hot->count();
} }
return sum; return sum;
} }
/** /**
* Caluculate the amount of memory usage of the leaf cache. * Caluculate the amount of memory usage of the leaf cache.
* @return the amount of memory usage of the leaf cache. * @return the amount of memory usage of the leaf cache.
*/ */
int64_t calc_leaf_cache_size() { int64_t calc_leaf_cache_size() {
_assert_(true); _assert_(true);
int64_t sum = 0; int64_t sum = 0;
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
LeafSlot* slot = lslots_ + i; LeafSlot* slot = lslots_ + i;
typename LeafCache::Iterator it = slot->warm->begin(); typename LeafCache::Iterator it = slot->warm->begin();
typename LeafCache::Iterator itend = slot->warm->end(); typename LeafCache::Iterator itend = slot->warm->end();
while (it != itend) { while (it != itend) {
LeafNode* node = it.value(); LeafNode* node = it.value();
sum += node->size; sum += node->size;
it++; ++it;
} }
it = slot->hot->begin(); it = slot->hot->begin();
itend = slot->hot->end(); itend = slot->hot->end();
while (it != itend) { while (it != itend) {
LeafNode* node = it.value(); LeafNode* node = it.value();
sum += node->size; sum += node->size;
it++; ++it;
} }
} }
return sum; return sum;
} }
/** /**
* Caluculate the total number of nodes in the inner cache. * Caluculate the total number of nodes in the inner cache.
* @return the total number of nodes in the inner cache. * @return the total number of nodes in the inner cache.
*/ */
int64_t calc_inner_cache_count() { int64_t calc_inner_cache_count() {
_assert_(true); _assert_(true);
int64_t sum = 0; int64_t sum = 0;
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
InnerSlot* slot = islots_ + i; InnerSlot* slot = islots_ + i;
sum += slot->warm->count(); sum += slot->warm->count();
} }
return sum; return sum;
} }
/** /**
* Caluculate the amount of memory usage of the inner cache. * Caluculate the amount of memory usage of the inner cache.
* @return the amount of memory usage of the inner cache. * @return the amount of memory usage of the inner cache.
*/ */
int64_t calc_inner_cache_size() { int64_t calc_inner_cache_size() {
_assert_(true); _assert_(true);
int64_t sum = 0; int64_t sum = 0;
for (int32_t i = 0; i < PLDBSLOTNUM; i++) { for (int32_t i = 0; i < SLOTNUM; i++) {
InnerSlot* slot = islots_ + i; InnerSlot* slot = islots_ + i;
typename InnerCache::Iterator it = slot->warm->begin(); typename InnerCache::Iterator it = slot->warm->begin();
typename InnerCache::Iterator itend = slot->warm->end(); typename InnerCache::Iterator itend = slot->warm->end();
while (it != itend) { while (it != itend) {
InnerNode* node = it.value(); InnerNode* node = it.value();
sum += node->size; sum += node->size;
it++; ++it;
} }
} }
return sum; return sum;
} }
/** /**
* Disable all cursors. * Disable all cursors.
*/ */
void disable_cursors() { void disable_cursors() {
_assert_(true); _assert_(true);
if (curs_.empty()) return; if (curs_.empty()) return;
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->kbuf_) cur->clear_position(); if (cur->kbuf_) cur->clear_position();
cit++; ++cit;
} }
} }
/** /**
* Escape cursors on a divided leaf node. * Escape cursors on a divided leaf node.
* @param src the ID of the source node. * @param src the ID of the source node.
* @param dest the ID of the destination node. * @param dest the ID of the destination node.
* @param rec the pivot record. * @param rec the pivot record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
void escape_cursors(int64_t src, int64_t dest, Record* rec) { void escape_cursors(int64_t src, int64_t dest, Record* rec) {
skipping to change at line 3234 skipping to change at line 3272
if (curs_.empty()) return; if (curs_.empty()) return;
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->lid_ == src) { if (cur->lid_ == src) {
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
if (reccomp_.comp->compare(cur->kbuf_, cur->ksiz_, dbuf, rec->ksiz) >= 0) if (reccomp_.comp->compare(cur->kbuf_, cur->ksiz_, dbuf, rec->ksiz) >= 0)
cur->lid_ = dest; cur->lid_ = dest;
} }
cit++; ++cit;
} }
} }
/** /**
* Escape cursors on a removed leaf node. * Escape cursors on a removed leaf node.
* @param src the ID of the source node. * @param src the ID of the source node.
* @param dest the ID of the destination node. * @param dest the ID of the destination node.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool escape_cursors(int64_t src, int64_t dest) { bool escape_cursors(int64_t src, int64_t dest) {
_assert_(src > 0 && dest >= 0); _assert_(src > 0 && dest >= 0);
if (curs_.empty()) return true; if (curs_.empty()) return true;
bool err = false; bool err = false;
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->lid_ == src) { if (cur->lid_ == src) {
cur->clear_position(); cur->clear_position();
if (!cur->set_position(dest) && db_.error().code() != Error::NOREC) err = true; if (!cur->set_position(dest) && db_.error().code() != Error::NOREC) err = true;
} }
cit++; ++cit;
} }
return !err; return !err;
} }
/** /**
* Recalculate the count data. * Recalculate the count data.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool recalc_count() { bool recalc_count() {
_assert_(true); _assert_(true);
if (!load_meta()) return false; if (!load_meta()) return false;
bool err = false; bool err = false;
class VisitorImpl : public DB::Visitor { class VisitorImpl : public DB::Visitor {
public: public:
explicit VisitorImpl() : count_(0) {} explicit VisitorImpl() : count_(0) {}
int64_t count() { int64_t count() {
return count_; return count_;
} }
private: private:
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
if (ksiz < 2 || kbuf[0] != PLDBLNPREFIX) return NOP; if (ksiz < 2 || kbuf[0] != LNPREFIX) return NOP;
uint64_t prev; uint64_t prev;
size_t step = readvarnum(vbuf, vsiz, &prev); size_t step = readvarnum(vbuf, vsiz, &prev);
if (step < 1) return NOP; if (step < 1) return NOP;
vbuf += step; vbuf += step;
vsiz -= step; vsiz -= step;
uint64_t next; uint64_t next;
step = readvarnum(vbuf, vsiz, &next); step = readvarnum(vbuf, vsiz, &next);
if (step < 1) return NOP; if (step < 1) return NOP;
vbuf += step; vbuf += step;
vsiz -= step; vsiz -= step;
skipping to change at line 3333 skipping to change at line 3371
_assert_(true); _assert_(true);
if (!load_meta()) { if (!load_meta()) {
if (reccomp_.comp) { if (reccomp_.comp) {
linkcomp_.comp = reccomp_.comp; linkcomp_.comp = reccomp_.comp;
} else { } else {
reccomp_.comp = LEXICALCOMP; reccomp_.comp = LEXICALCOMP;
linkcomp_.comp = LEXICALCOMP; linkcomp_.comp = LEXICALCOMP;
} }
} }
const std::string& path = db_.path(); const std::string& path = db_.path();
const std::string& npath = path + File::EXTCHR + PLDBTMPPATHEXT; const std::string& npath = path + File::EXTCHR + KCPDBTMPPATHEXT;
PlantDB tdb; PlantDB tdb;
tdb.tune_comparator(reccomp_.comp); tdb.tune_comparator(reccomp_.comp);
if (!tdb.open(npath, OWRITER | OCREATE | OTRUNCATE)) { if (!tdb.open(npath, OWRITER | OCREATE | OTRUNCATE)) {
set_error(_KCCODELINE_, tdb.error().code(), "opening the destination failed"); set_error(_KCCODELINE_, tdb.error().code(), "opening the destination failed");
return false; return false;
} }
db_.report(_KCCODELINE_, Logger::WARN, "reorganizing the database"); db_.report(_KCCODELINE_, Logger::WARN, "reorganizing the database");
bool err = false; bool err = false;
create_leaf_cache(); create_leaf_cache();
create_inner_cache(); create_inner_cache();
DB::Cursor* cur = db_.cursor(); DB::Cursor* cur = db_.cursor();
cur->jump(); cur->jump();
char* kbuf; char* kbuf;
size_t ksiz; size_t ksiz;
while (!err && (kbuf = cur->get_key(&ksiz)) != NULL) { while (!err && (kbuf = cur->get_key(&ksiz)) != NULL) {
if (*kbuf == PLDBLNPREFIX) { if (*kbuf == LNPREFIX) {
int64_t id = std::strtol(kbuf + 1, NULL, 16); int64_t id = std::strtol(kbuf + 1, NULL, 16);
if (id > 0 && id < PLDBINIDBASE) { if (id > 0 && id < INIDBASE) {
LeafNode* node = load_leaf_node(id, false); LeafNode* node = load_leaf_node(id, false);
if (node) { if (node) {
const RecordArray& recs = node->recs; const RecordArray& recs = node->recs;
typename RecordArray::const_iterator rit = recs.begin(); typename RecordArray::const_iterator rit = recs.begin();
typename RecordArray::const_iterator ritend = recs.end(); typename RecordArray::const_iterator ritend = recs.end();
while (rit != ritend) { while (rit != ritend) {
Record* rec = *rit; Record* rec = *rit;
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
if (!tdb.set(dbuf, rec->ksiz, dbuf + rec->ksiz, rec->vsiz)) { if (!tdb.set(dbuf, rec->ksiz, dbuf + rec->ksiz, rec->vsiz)) {
set_error(_KCCODELINE_, tdb.error().code(), set_error(_KCCODELINE_, tdb.error().code(),
"opening the destination failed"); "opening the destination failed");
err = true; err = true;
} }
rit++; ++rit;
} }
flush_leaf_node(node, false); flush_leaf_node(node, false);
} }
} }
} }
delete[] kbuf; delete[] kbuf;
cur->step(); cur->step();
} }
delete cur; delete cur;
delete_inner_cache(); delete_inner_cache();
skipping to change at line 3391 skipping to change at line 3429
if (DBTYPE == TYPETREE) { if (DBTYPE == TYPETREE) {
if (File::rename(npath, path)) { if (File::rename(npath, path)) {
if (!db_.close()) err = true; if (!db_.close()) err = true;
if (!db_.open(path, mode)) err = true; if (!db_.open(path, mode)) err = true;
} else { } else {
set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destination fa iled"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destination fa iled");
err = true; err = true;
} }
File::remove(npath); File::remove(npath);
} else if (DBTYPE == TYPEFOREST) { } else if (DBTYPE == TYPEFOREST) {
const std::string& tpath = npath + File::EXTCHR + PLDBTMPPATHEXT; const std::string& tpath = npath + File::EXTCHR + KCPDBTMPPATHEXT;
File::remove_recursively(tpath); File::remove_recursively(tpath);
if (File::rename(path, tpath)) { if (File::rename(path, tpath)) {
if (File::rename(npath, path)) { if (File::rename(npath, path)) {
if (!db_.close()) err = true; if (!db_.close()) err = true;
if (!db_.open(path, mode)) err = true; if (!db_.open(path, mode)) err = true;
} else { } else {
set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destination failed"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destination failed");
File::rename(tpath, path); File::rename(tpath, path);
err = true; err = true;
} }
skipping to change at line 3445 skipping to change at line 3483
/** /**
* Begin transaction. * Begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_impl(bool hard) { bool begin_transaction_impl(bool hard) {
_assert_(true); _assert_(true);
if (!clean_leaf_cache()) return false; if (!clean_leaf_cache()) return false;
if (!clean_inner_cache()) return false; if (!clean_inner_cache()) return false;
int32_t idx = trclock_++ % PLDBSLOTNUM; int32_t idx = trclock_++ % SLOTNUM;
LeafSlot* lslot = lslots_ + idx; LeafSlot* lslot = lslots_ + idx;
if (lslot->warm->count() + lslot->hot->count() > 1) flush_leaf_cache_pa rt(lslot); if (lslot->warm->count() + lslot->hot->count() > 1) flush_leaf_cache_pa rt(lslot);
InnerSlot* islot = islots_ + idx; InnerSlot* islot = islots_ + idx;
if (islot->warm->count() > 1) flush_inner_cache_part(islot); if (islot->warm->count() > 1) flush_inner_cache_part(islot);
if ((trlcnt_ != lcnt_ || count_ != trcount_) && !dump_meta()) return fa lse; if ((trlcnt_ != lcnt_ || count_ != trcount_) && !dump_meta()) return fa lse;
if (!db_.begin_transaction(hard)) return false; if (!db_.begin_transaction(hard)) return false;
return true; return true;
} }
/** /**
* Commit transaction. * Commit transaction.
skipping to change at line 3491 skipping to change at line 3529
/** /**
* Fix auto transaction for the B+ tree. * Fix auto transaction for the B+ tree.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool fix_auto_transaction_tree() { bool fix_auto_transaction_tree() {
_assert_(true); _assert_(true);
if (!db_.begin_transaction(autosync_)) return false; if (!db_.begin_transaction(autosync_)) return false;
bool err = false; bool err = false;
if (!clean_leaf_cache()) err = true; if (!clean_leaf_cache()) err = true;
if (!clean_inner_cache()) err = true; if (!clean_inner_cache()) err = true;
size_t cnum = PLDBATRANCNUM / PLDBSLOTNUM; size_t cnum = ATRANCNUM / SLOTNUM;
int32_t idx = trclock_++ % PLDBSLOTNUM; int32_t idx = trclock_++ % SLOTNUM;
LeafSlot* lslot = lslots_ + idx; LeafSlot* lslot = lslots_ + idx;
if (lslot->warm->count() + lslot->hot->count() > cnum) flush_leaf_cache _part(lslot); if (lslot->warm->count() + lslot->hot->count() > cnum) flush_leaf_cache _part(lslot);
InnerSlot* islot = islots_ + idx; InnerSlot* islot = islots_ + idx;
if (islot->warm->count() > cnum) flush_inner_cache_part(islot); if (islot->warm->count() > cnum) flush_inner_cache_part(islot);
if (!dump_meta()) err = true; if (!dump_meta()) err = true;
if (!db_.end_transaction(true)) err = true; if (!db_.end_transaction(true)) err = true;
return !err; return !err;
} }
/** /**
* Fix auto transaction for a leaf. * Fix auto transaction for a leaf.
skipping to change at line 3571 skipping to change at line 3609
int64_t last_; int64_t last_;
/** The count of leaf nodes. */ /** The count of leaf nodes. */
int64_t lcnt_; int64_t lcnt_;
/** The count of inner nodes. */ /** The count of inner nodes. */
int64_t icnt_; int64_t icnt_;
/** The record number. */ /** The record number. */
AtomicInt64 count_; AtomicInt64 count_;
/** The cache memory usage. */ /** The cache memory usage. */
AtomicInt64 cusage_; AtomicInt64 cusage_;
/** The Slots of leaf nodes. */ /** The Slots of leaf nodes. */
LeafSlot lslots_[PLDBSLOTNUM]; LeafSlot lslots_[SLOTNUM];
/** The Slots of inner nodes. */ /** The Slots of inner nodes. */
InnerSlot islots_[PLDBSLOTNUM]; InnerSlot islots_[SLOTNUM];
/** The record comparator. */ /** The record comparator. */
RecordComparator reccomp_; RecordComparator reccomp_;
/** The link comparator. */ /** The link comparator. */
LinkComparator linkcomp_; LinkComparator linkcomp_;
/** The flag whether in transaction. */ /** The flag whether in transaction. */
bool tran_; bool tran_;
/** The logical clock for transaction. */ /** The logical clock for transaction. */
int64_t trclock_; int64_t trclock_;
/** The leaf count history for transaction. */ /** The leaf count history for transaction. */
int64_t trlcnt_; int64_t trlcnt_;
 End of changes. 121 change blocks. 
158 lines changed or deleted 188 lines changed or added


 kcpolydb.h   kcpolydb.h 
skipping to change at line 453 skipping to change at line 453
int64_t dfunit = -1; int64_t dfunit = -1;
std::string zcompname = ""; std::string zcompname = "";
int64_t psiz = -1; int64_t psiz = -1;
Comparator* rcomp = NULL; Comparator* rcomp = NULL;
int64_t pccap = 0; int64_t pccap = 0;
std::string zkey = ""; std::string zkey = "";
std::vector<std::string>::iterator it = elems.begin(); std::vector<std::string>::iterator it = elems.begin();
std::vector<std::string>::iterator itend = elems.end(); std::vector<std::string>::iterator itend = elems.end();
if (it != itend) { if (it != itend) {
fpath = *it; fpath = *it;
it++; ++it;
} }
const char* fstr = fpath.c_str(); const char* fstr = fpath.c_str();
const char* pv = std::strrchr(fstr, File::PATHCHR); const char* pv = std::strrchr(fstr, File::PATHCHR);
if (pv) fstr = pv + 1; if (pv) fstr = pv + 1;
if (!std::strcmp(fstr, "-")) { if (!std::strcmp(fstr, "-")) {
type = TYPEPHASH; type = TYPEPHASH;
} else if (!std::strcmp(fstr, "+")) { } else if (!std::strcmp(fstr, "+")) {
type = TYPEPTREE; type = TYPEPTREE;
} else if (!std::strcmp(fstr, ":")) { } else if (!std::strcmp(fstr, ":")) {
type = TYPESTASH; type = TYPESTASH;
skipping to change at line 589 skipping to change at line 589
} else if (!std::strcmp(value, "lexdesc") || !std::strcmp(value, "lexicaldesc")) { } else if (!std::strcmp(value, "lexdesc") || !std::strcmp(value, "lexicaldesc")) {
rcomp = LEXICALDESCCOMP; rcomp = LEXICALDESCCOMP;
} else if (!std::strcmp(value, "decdesc") || !std::strcmp(value, "decimaldesc")) { } else if (!std::strcmp(value, "decdesc") || !std::strcmp(value, "decimaldesc")) {
rcomp = DECIMALDESCCOMP; rcomp = DECIMALDESCCOMP;
} }
} else if (!std::strcmp(key, "zkey") || !std::strcmp(key, "pass") | | } else if (!std::strcmp(key, "zkey") || !std::strcmp(key, "pass") | |
!std::strcmp(key, "password")) { !std::strcmp(key, "password")) {
zkey = value; zkey = value;
} }
} }
it++; ++it;
} }
delete stdlogger_; delete stdlogger_;
delete stdlogstrm_; delete stdlogstrm_;
stdlogstrm_ = NULL; stdlogstrm_ = NULL;
stdlogger_ = NULL; stdlogger_ = NULL;
if (!logname.empty()) { if (!logname.empty()) {
std::ostream* stdlogstrm = NULL; std::ostream* stdlogstrm = NULL;
if (logname == "-" || logname == "[stdout]" || logname == "[cout]") { if (logname == "-" || logname == "[stdout]" || logname == "[cout]") {
stdlogstrm = &std::cout; stdlogstrm = &std::cout;
} else if (logname == "+" || logname == "[stderr]" || logname == "[ce rr]") { } else if (logname == "+" || logname == "[stderr]" || logname == "[ce rr]") {
 End of changes. 2 change blocks. 
2 lines changed or deleted 2 lines changed or added


 kcprotodb.h   kcprotodb.h 
skipping to change at line 32 skipping to change at line 32
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h> #include <kcregex.h>
#include <kcdb.h> #include <kcdb.h>
#include <kcplantdb.h> #include <kcplantdb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation.
*/
namespace {
const size_t PRDBHASHBNUM = 1048583LL; ///< bucket number of hash table
const size_t PRDBOPAQUESIZ = 16; ///< size of the opaque buffer
const uint32_t PRDBLOCKBUSYLOOP = 8192; ///< threshold of busy loop and sl
eep for locking
}
/**
* Helper functions.
*/
namespace {
template <class STRMAP>
typename STRMAP::iterator map_find(STRMAP* map, const std::string& key) {
return map->find(key);
}
template <>
StringTreeMap::iterator map_find(StringTreeMap* map, const std::string& key
) {
StringTreeMap::iterator it = map->find(key);
if (it != map->end()) return it;
return map->upper_bound(key);
}
template <class STRMAP>
void map_tune(STRMAP* map) {}
template <>
void map_tune(StringHashMap* map) {
map->rehash(PRDBHASHBNUM);
map->max_load_factor(FLT_MAX);
}
template <class ITER>
bool iter_back(ITER* itp) {
return false;
}
template <>
bool iter_back(StringTreeMap::iterator* itp) {
--(*itp);
return true;
}
}
/**
* Prototype implementation of database with STL. * Prototype implementation of database with STL.
* @param STRMAP a class compatible with the map class of STL. * @param STRMAP a class compatible with the map class of STL.
* @param DBTYPE the database type number of the class. * @param DBTYPE the database type number of the class.
* @note This class template is a template for concrete classes which wrap data structures * @note This class template is a template for concrete classes which wrap data structures
* compatible with std::map. Template instance classes can be inherited bu t overwriting methods * compatible with std::map. Template instance classes can be inherited bu t overwriting methods
* is forbidden. The class ProtoHashDB is the instance using std::unordere d_map. The class * is forbidden. The class ProtoHashDB is the instance using std::unordere d_map. The class
* ProtoTreeDB is the instance using std::map. Before every database opera tion, it is necessary * ProtoTreeDB is the instance using std::map. Before every database opera tion, it is necessary
* to call the BasicDB::open method in order to open a database file and co nnect the database * to call the BasicDB::open method in order to open a database file and co nnect the database
* object to it. To avoid data missing or corruption, it is important to c lose every database * object to it. To avoid data missing or corruption, it is important to c lose every database
* file by the BasicDB::close method when the database is no longer in use. It is forbidden for * file by the BasicDB::close method when the database is no longer in use. It is forbidden for
* multible database objects in a process to open the same database at the same time. It is * multible database objects in a process to open the same database at the same time. It is
* forbidden to share a database object with child processes. * forbidden to share a database object with child processes.
*/ */
template <class STRMAP, uint8_t DBTYPE> template <class STRMAP, uint8_t DBTYPE>
class ProtoDB : public BasicDB { class ProtoDB : public BasicDB {
public: public:
class Cursor; class Cursor;
private: private:
struct TranLog; struct TranLog;
class ScopedVisitor;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
/** An alias of list of transaction logs. */ /** An alias of list of transaction logs. */
typedef std::list<TranLog> TranLogList; typedef std::list<TranLog> TranLogList;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public BasicDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class ProtoDB; friend class ProtoDB;
skipping to change at line 163 skipping to change at line 123
TranLog log(key, value); TranLog log(key, value);
db_->trlogs_.push_back(log); db_->trlogs_.push_back(log);
} }
db_->size_ -= key.size() + value.size(); db_->size_ -= key.size() + value.size();
if (db_->curs_.size() > 1) { if (db_->curs_.size() > 1) {
typename CursorList::const_iterator cit = db_->curs_.begin(); typename CursorList::const_iterator cit = db_->curs_.begin();
typename CursorList::const_iterator citend = db_->curs_.end(); typename CursorList::const_iterator citend = db_->curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur != this && cur->it_ == it_) cur->it_++; if (cur != this && cur->it_ == it_) cur->it_++;
cit++; ++cit;
} }
} }
db_->recs_.erase(it_++); db_->recs_.erase(it_++);
} else if (vbuf == Visitor::NOP) { } else if (vbuf == Visitor::NOP) {
if (step) it_++; if (step) it_++;
} else { } else {
if (db_->tran_) { if (db_->tran_) {
TranLog log(key, value); TranLog log(key, value);
db_->trlogs_.push_back(log); db_->trlogs_.push_back(log);
} }
skipping to change at line 213 skipping to change at line 173
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump(const char* kbuf, size_t ksiz) { bool jump(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
std::string key(kbuf, ksiz); std::string key(kbuf, ksiz);
it_ = map_find(&db_->recs_, key); search(key);
if (it_ == db_->recs_.end()) { if (it_ == db_->recs_.end()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
return true; return true;
} }
/** /**
* Jump the cursor to a record for forward scan. * Jump the cursor to a record for forward scan.
* @note Equal to the original Cursor::jump method except that the para meter is std::string. * @note Equal to the original Cursor::jump method except that the para meter is std::string.
*/ */
skipping to change at line 244 skipping to change at line 204
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
it_ = db_->recs_.end(); it_ = db_->recs_.end();
if (it_ == db_->recs_.begin()) { if (it_ == db_->recs_.begin()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
if (!iter_back(&it_)) { if (!iter_back()) {
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false; return false;
} }
return true; return true;
} }
/** /**
* Jump the cursor to a record for backward scan. * Jump the cursor to a record for backward scan.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump_back(const char* kbuf, size_t ksiz) { bool jump_back(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
std::string key(kbuf, ksiz); std::string key(kbuf, ksiz);
it_ = map_find(&db_->recs_, key); search(key);
if (it_ == db_->recs_.end()) { if (it_ == db_->recs_.end()) {
if (it_ == db_->recs_.begin()) { if (it_ == db_->recs_.begin()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
if (!iter_back(&it_)) { if (!iter_back()) {
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false; return false;
} }
} else { } else {
std::string key(kbuf, ksiz); std::string key(kbuf, ksiz);
if (key < it_->first) { if (key < it_->first) {
if (it_ == db_->recs_.begin()) { if (it_ == db_->recs_.begin()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
it_ = db_->recs_.end(); it_ = db_->recs_.end();
return false; return false;
} }
if (!iter_back(&it_)) { if (!iter_back()) {
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
it_ = db_->recs_.end(); it_ = db_->recs_.end();
return false; return false;
} }
} }
} }
return true; return true;
} }
/** /**
* Jump the cursor to a record for backward scan. * Jump the cursor to a record for backward scan.
skipping to change at line 338 skipping to change at line 298
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (it_ == db_->recs_.begin()) { if (it_ == db_->recs_.begin()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
it_ = db_->recs_.end(); it_ = db_->recs_.end();
return false; return false;
} }
if (!iter_back(&it_)) { if (!iter_back()) {
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented"); db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
it_ = db_->recs_.end();
return false; return false;
} }
return true; return true;
} }
/** /**
* Get the database object. * Get the database object.
* @return the database object. * @return the database object.
*/ */
ProtoDB* db() { ProtoDB* db() {
_assert_(true); _assert_(true);
return db_; return db_;
} }
private: private:
/**
* Search for a record.
*/
void search(const std::string& key);
/**
* Place back the inner iterator.
* @return true on success, or false on failure.
*/
bool iter_back();
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
Cursor(const Cursor&); Cursor(const Cursor&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
Cursor& operator =(const Cursor&); Cursor& operator =(const Cursor&);
/** The inner database. */ /** The inner database. */
ProtoDB* db_; ProtoDB* db_;
/** The inner iterator. */ /** The inner iterator. */
typename STRMAP::iterator it_; typename STRMAP::iterator it_;
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit ProtoDB() : explicit ProtoDB() :
mlock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NULL), mlock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NULL),
omode_(0), recs_(), curs_(), path_(""), size_(0), opaque_(), omode_(0), recs_(), curs_(), path_(""), size_(0), opaque_(),
tran_(false), trlogs_(), trsize_(0) { tran_(false), trlogs_(), trsize_(0) {
_assert_(true); _assert_(true);
map_tune(&recs_); map_tune();
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~ProtoDB() { virtual ~ProtoDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->db_ = NULL; cur->db_ = NULL;
cit++; ++cit;
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 441 skipping to change at line 411
TranLog log(key, value); TranLog log(key, value);
trlogs_.push_back(log); trlogs_.push_back(log);
} }
size_ -= ksiz + value.size(); size_ -= ksiz + value.size();
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->it_ == it) cur->it_++; if (cur->it_ == it) cur->it_++;
cit++; ++cit;
} }
} }
recs_.erase(it); recs_.erase(it);
} else if (vbuf != Visitor::NOP) { } else if (vbuf != Visitor::NOP) {
if (tran_) { if (tran_) {
TranLog log(key, value); TranLog log(key, value);
trlogs_.push_back(log); trlogs_.push_back(log);
} }
size_ -= value.size(); size_ -= value.size();
size_ += vsiz; size_ += vsiz;
skipping to change at line 505 skipping to change at line 475
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
if (keys.empty()) return true;
std::vector<std::string>::const_iterator kit = keys.begin(); std::vector<std::string>::const_iterator kit = keys.begin();
std::vector<std::string>::const_iterator kitend = keys.end(); std::vector<std::string>::const_iterator kitend = keys.end();
while (kit != kitend) { while (kit != kitend) {
const std::string& key = *kit; const std::string& key = *kit;
typename STRMAP::iterator it = recs_.find(key); typename STRMAP::iterator it = recs_.find(key);
if (it == recs_.end()) { if (it == recs_.end()) {
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_empty(key.data(), key.size(), &vs iz); const char* vbuf = visitor->visit_empty(key.data(), key.size(), &vs iz);
if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) {
if (tran_) { if (tran_) {
skipping to change at line 538 skipping to change at line 510
TranLog log(key, value); TranLog log(key, value);
trlogs_.push_back(log); trlogs_.push_back(log);
} }
size_ -= key.size() + value.size(); size_ -= key.size() + value.size();
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->it_ == it) cur->it_++; if (cur->it_ == it) cur->it_++;
cit++; ++cit;
} }
} }
recs_.erase(it); recs_.erase(it);
} else if (vbuf != Visitor::NOP) { } else if (vbuf != Visitor::NOP) {
if (tran_) { if (tran_) {
TranLog log(key, value); TranLog log(key, value);
trlogs_.push_back(log); trlogs_.push_back(log);
} }
size_ -= value.size(); size_ -= value.size();
size_ += vsiz; size_ += vsiz;
it->second = std::string(vbuf, vsiz); it->second = std::string(vbuf, vsiz);
} }
} }
kit++; ++kit;
} }
return true; return true;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed. * @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The whole iteration is performed atomically and other threads ar e blocked. To avoid * @note The whole iteration is performed atomically and other threads ar e blocked. To avoid
skipping to change at line 576 skipping to change at line 548
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
int64_t allcnt = recs_.size(); int64_t allcnt = recs_.size();
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
typename STRMAP::iterator it = recs_.begin(); typename STRMAP::iterator it = recs_.begin();
typename STRMAP::iterator itend = recs_.end(); typename STRMAP::iterator itend = recs_.end();
int64_t curcnt = 0; int64_t curcnt = 0;
while (it != itend) { while (it != itend) {
const std::string& key = it->first; const std::string& key = it->first;
const std::string& value = it->second; const std::string& value = it->second;
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(key.data(), key.size(), const char* vbuf = visitor->visit_full(key.data(), key.size(),
value.data(), value.size(), &v siz); value.data(), value.size(), &v siz);
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
size_ -= key.size() + value.size(); size_ -= key.size() + value.size();
recs_.erase(it++); recs_.erase(it++);
} else if (vbuf == Visitor::NOP) { } else if (vbuf == Visitor::NOP) {
it++; ++it;
} else { } else {
size_ -= value.size(); size_ -= value.size();
size_ += vsiz; size_ += vsiz;
it->second = std::string(vbuf, vsiz); it->second = std::string(vbuf, vsiz);
it++; ++it;
} }
curcnt++; curcnt++;
if (checker && !checker->check("iterate", "processing", curcnt, allcn t)) { if (checker && !checker->check("iterate", "processing", curcnt, allcn t)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
} }
if (checker && !checker->check("iterate", "ending", -1, allcnt)) { if (checker && !checker->check("iterate", "ending", -1, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
skipping to change at line 695 skipping to change at line 668
report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", p ath_.c_str()); report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", p ath_.c_str());
tran_ = false; tran_ = false;
trlogs_.clear(); trlogs_.clear();
recs_.clear(); recs_.clear();
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->it_ = recs_.end(); cur->it_ = recs_.end();
cit++; ++cit;
} }
} }
path_.clear(); path_.clear();
omode_ = 0; omode_ = 0;
trigger_meta(MetaTrigger::CLOSE, "close"); trigger_meta(MetaTrigger::CLOSE, "close");
return true; return true;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
skipping to change at line 761 skipping to change at line 734
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wcnt >= PRDBLOCKBUSYLOOP) { if (wcnt >= LOCKBUSYLOOP) {
Thread::chill(); Thread::chill();
} else { } else {
Thread::yield(); Thread::yield();
wcnt++; wcnt++;
} }
} }
tran_ = true; tran_ = true;
trsize_ = size_; trsize_ = size_;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction");
mlock_.unlock(); mlock_.unlock();
skipping to change at line 827 skipping to change at line 800
set_error(_KCCODELINE_, Error::INVALID, "not in transaction"); set_error(_KCCODELINE_, Error::INVALID, "not in transaction");
return false; return false;
} }
if (!commit) { if (!commit) {
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->it_ = recs_.end(); cur->it_ = recs_.end();
cit++; ++cit;
} }
} }
const TranLogList& logs = trlogs_; const TranLogList& logs = trlogs_;
typename TranLogList::const_iterator lit = logs.end(); typename TranLogList::const_iterator lit = logs.end();
typename TranLogList::const_iterator litbeg = logs.begin(); typename TranLogList::const_iterator litbeg = logs.begin();
while (lit != litbeg) { while (lit != litbeg) {
lit--; --lit;
if (lit->full) { if (lit->full) {
recs_[lit->key] = lit->value; recs_[lit->key] = lit->value;
} else { } else {
recs_.erase(lit->key); recs_.erase(lit->key);
} }
} }
size_ = trsize_; size_ = trsize_;
} }
trlogs_.clear(); trlogs_.clear();
tran_ = false; tran_ = false;
skipping to change at line 866 skipping to change at line 839
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
recs_.clear(); recs_.clear();
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->it_ = recs_.end(); cur->it_ = recs_.end();
cit++; ++cit;
} }
} }
std::memset(opaque_, 0, sizeof(opaque_)); std::memset(opaque_, 0, sizeof(opaque_));
trigger_meta(MetaTrigger::CLEAR, "clear"); trigger_meta(MetaTrigger::CLEAR, "clear");
return true; return true;
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
*/ */
skipping to change at line 1082 skipping to change at line 1055
* MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f or beginning * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f or beginning
* transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT rigger::ABORTTRAN * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT rigger::ABORTTRAN
* for aborting transaction, and MetaTrigger::MISC for miscellaneous oper ations. * for aborting transaction, and MetaTrigger::MISC for miscellaneous oper ations.
* @param message the supplement message. * @param message the supplement message.
*/ */
void trigger_meta(MetaTrigger::Kind kind, const char* message) { void trigger_meta(MetaTrigger::Kind kind, const char* message) {
_assert_(message); _assert_(message);
if (mtrigger_) mtrigger_->trigger(kind, message); if (mtrigger_) mtrigger_->trigger(kind, message);
} }
private: private:
/** The size of the opaque buffer. */
static const size_t OPAQUESIZ = 16;
/** The threshold of busy loop and sleep for locking. */
static const uint32_t LOCKBUSYLOOP = 8192;
/** /**
* Transaction log. * Transaction log.
*/ */
struct TranLog { struct TranLog {
bool full; ///< flag whether full bool full; ///< flag whether full
std::string key; ///< old key std::string key; ///< old key
std::string value; ///< old value std::string value; ///< old value
/** constructor for a full record */ /** constructor for a full record */
explicit TranLog(const std::string& pkey, const std::string& pvalue) : explicit TranLog(const std::string& pkey, const std::string& pvalue) :
full(true), key(pkey), value(pvalue) { full(true), key(pkey), value(pvalue) {
_assert_(true); _assert_(true);
} }
/** constructor for an empty record */ /** constructor for an empty record */
explicit TranLog(const std::string& pkey) : full(false), key(pkey) { explicit TranLog(const std::string& pkey) : full(false), key(pkey) {
_assert_(true); _assert_(true);
} }
}; };
/**
* Scoped visiotor.
*/
class ScopedVisitor {
public:
/** constructor */
ScopedVisitor(Visitor* visitor) : visitor_(visitor) {
_assert_(visitor);
visitor_->visit_before();
}
/** destructor */
~ScopedVisitor() {
_assert_(true);
visitor_->visit_after();
}
private:
Visitor* visitor_; ///< visitor
};
/**
* Tune the internal map object.
*/
void map_tune();
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
ProtoDB(const ProtoDB&); ProtoDB(const ProtoDB&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
ProtoDB& operator =(const ProtoDB&); ProtoDB& operator =(const ProtoDB&);
/** The method lock. */ /** The method lock. */
SpinRWLock mlock_; SpinRWLock mlock_;
/** The last happened error. */ /** The last happened error. */
TSD<Error> error_; TSD<Error> error_;
/** The internal logger. */ /** The internal logger. */
Logger* logger_; Logger* logger_;
skipping to change at line 1124 skipping to change at line 1123
uint32_t omode_; uint32_t omode_;
/** The map of records. */ /** The map of records. */
STRMAP recs_; STRMAP recs_;
/** The cursor objects. */ /** The cursor objects. */
CursorList curs_; CursorList curs_;
/** The path of the database file. */ /** The path of the database file. */
std::string path_; std::string path_;
/** The total size of records. */ /** The total size of records. */
int64_t size_; int64_t size_;
/** The opaque data. */ /** The opaque data. */
char opaque_[PRDBOPAQUESIZ]; char opaque_[OPAQUESIZ];
/** The flag whether in transaction. */ /** The flag whether in transaction. */
bool tran_; bool tran_;
/** The transaction logs. */ /** The transaction logs. */
TranLogList trlogs_; TranLogList trlogs_;
/** The old size before transaction. */ /** The old size before transaction. */
size_t trsize_; size_t trsize_;
}; };
/**
* Search for a record.
*/
template <class STRMAP, uint8_t DBTYPE>
inline void ProtoDB<STRMAP, DBTYPE>::Cursor::search(const std::string& key)
{
_assert_(true);
it_ = db_->recs_.find(key);
}
/**
* Search for a record.
*/
template <>
inline void ProtoDB<StringTreeMap, BasicDB::TYPEPTREE>::Cursor::search(cons
t std::string& key) {
_assert_(true);
it_ = db_->recs_.lower_bound(key);
}
/**
* Place back the inner iterator.
*/
template <class STRMAP, uint8_t DBTYPE>
inline bool ProtoDB<STRMAP, DBTYPE>::Cursor::iter_back() {
_assert_(true);
return false;
}
/**
* Place back the inner iterator.
*/
template <>
inline bool ProtoDB<StringTreeMap, BasicDB::TYPEPTREE>::Cursor::iter_back()
{
_assert_(true);
--it_;
return true;
}
/**
* Tune the internal map object.
*/
template <class STRMAP, uint8_t DBTYPE>
inline void ProtoDB<STRMAP, DBTYPE>::map_tune() {
_assert_(true);
}
/**
* Tune the internal map object.
*/
template <>
inline void ProtoDB<StringHashMap, BasicDB::TYPEPHASH>::map_tune() {
_assert_(true);
recs_.rehash(1048583LL);
recs_.max_load_factor(FLT_MAX);
}
/** An alias of the prototype hash database. */ /** An alias of the prototype hash database. */
typedef ProtoDB<StringHashMap, BasicDB::TYPEPHASH> ProtoHashDB; typedef ProtoDB<StringHashMap, BasicDB::TYPEPHASH> ProtoHashDB;
/** An alias of the prototype tree database. */ /** An alias of the prototype tree database. */
typedef ProtoDB<StringTreeMap, BasicDB::TYPEPTREE> ProtoTreeDB; typedef ProtoDB<StringTreeMap, BasicDB::TYPEPTREE> ProtoTreeDB;
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
 End of changes. 29 change blocks. 
63 lines changed or deleted 118 lines changed or added


 kcstashdb.h   kcstashdb.h 
skipping to change at line 32 skipping to change at line 32
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h> #include <kcregex.h>
#include <kcdb.h> #include <kcdb.h>
#include <kcplantdb.h> #include <kcplantdb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation.
*/
namespace {
const int32_t SDBRLOCKSLOT = 1024; ///< number of slots of the record
lock
const size_t SDBDEFBNUM = 1048583LL; ///< default bucket number
const size_t SDBOPAQUESIZ = 16; ///< size of the opaque buffer
const uint32_t SDBLOCKBUSYLOOP = 8192; ///< threshold of busy loop and sl
eep for locking
}
/**
* Economical on-memory hash database. * Economical on-memory hash database.
* @note This class is a concrete class to operate a hash database on memor y. This class can be * @note This class is a concrete class to operate a hash database on memor y. This class can be
* inherited but overwriting methods is forbidden. Before every database o peration, it is * inherited but overwriting methods is forbidden. Before every database o peration, it is
* necessary to call the StashDB::open method in order to open a database f ile and connect the * necessary to call the StashDB::open method in order to open a database f ile and connect the
* database object to it. To avoid data missing or corruption, it is impor tant to close every * database object to it. To avoid data missing or corruption, it is impor tant to close every
* database file by the StashDB::close method when the database is no longe r in use. It is * database file by the StashDB::close method when the database is no longe r in use. It is
* forbidden for multible database objects in a process to open the same da tabase at the same * forbidden for multible database objects in a process to open the same da tabase at the same
* time. It is forbidden to share a database object with child processes. * time. It is forbidden to share a database object with child processes.
*/ */
class StashDB : public BasicDB { class StashDB : public BasicDB {
public: public:
class Cursor; class Cursor;
private: private:
struct Record; struct Record;
struct TranLog; struct TranLog;
class Repeater; class Repeater;
class Setter; class Setter;
class Remover;
class ScopedVisitor;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
/** An alias of list of transaction logs. */ /** An alias of list of transaction logs. */
typedef std::list<TranLog> TranLogList; typedef std::list<TranLog> TranLogList;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public BasicDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class StashDB; friend class StashDB;
skipping to change at line 309 skipping to change at line 301
StashDB* db_; StashDB* db_;
/** The index of the current bucket. */ /** The index of the current bucket. */
int64_t bidx_; int64_t bidx_;
/** The buffer of the current record. */ /** The buffer of the current record. */
char* rbuf_; char* rbuf_;
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit StashDB() : explicit StashDB() :
mlock_(), rlock_(SDBRLOCKSLOT), flock_(), error_(), mlock_(), rlock_(RLOCKSLOT), flock_(), error_(),
logger_(NULL), logkinds_(0), mtrigger_(NULL), logger_(NULL), logkinds_(0), mtrigger_(NULL),
omode_(0), curs_(), path_(""), bnum_(SDBDEFBNUM), opaque_(), omode_(0), curs_(), path_(""), bnum_(DEFBNUM), opaque_(),
count_(0), size_(0), buckets_(NULL), count_(0), size_(0), buckets_(NULL),
tran_(false), trlogs_(), trcount_(0), trsize_(0) { tran_(false), trlogs_(), trcount_(0), trsize_(0) {
_assert_(true); _assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
~StashDB() { ~StashDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
if (!curs_.empty()) { if (!curs_.empty()) {
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->db_ = NULL; cur->db_ = NULL;
cit++; ++cit;
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 356 skipping to change at line 348
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
size_t bidx = hash_record(kbuf, ksiz) % bnum_; size_t bidx = hash_record(kbuf, ksiz) % bnum_;
size_t lidx = bidx % SDBRLOCKSLOT; size_t lidx = bidx % RLOCKSLOT;
if (writable) { if (writable) {
rlock_.lock_writer(lidx); rlock_.lock_writer(lidx);
} else { } else {
rlock_.lock_reader(lidx); rlock_.lock_reader(lidx);
} }
accept_impl(kbuf, ksiz, visitor, bidx); accept_impl(kbuf, ksiz, visitor, bidx);
rlock_.unlock(lidx); rlock_.unlock(lidx);
return true; return true;
} }
/** /**
skipping to change at line 379 skipping to change at line 371
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The operations for specified records are performed atomically an d other threads * @note The operations for specified records are performed atomically an d other threads
* accessing the same records are blocked. To avoid deadlock, any explic it database operation * accessing the same records are blocked. To avoid deadlock, any explic it database operation
* must not be performed in this function. * must not be performed in this function.
*/ */
bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor, bool accept_bulk(const std::vector<std::string>& keys, Visitor* visitor,
bool writable = true) { bool writable = true) {
_assert_(visitor); _assert_(visitor);
size_t knum = keys.size();
if (knum < 1) return true;
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
size_t knum = keys.size();
if (knum < 1) return true;
struct RecordKey { struct RecordKey {
const char* kbuf; const char* kbuf;
size_t ksiz; size_t ksiz;
size_t bidx; size_t bidx;
}; };
RecordKey* rkeys = new RecordKey[knum]; RecordKey* rkeys = new RecordKey[knum];
std::set<size_t> lidxs; std::set<size_t> lidxs;
for (size_t i = 0; i < knum; i++) { for (size_t i = 0; i < knum; i++) {
const std::string& key = keys[i]; const std::string& key = keys[i];
RecordKey* rkey = rkeys + i; RecordKey* rkey = rkeys + i;
rkey->kbuf = key.data(); rkey->kbuf = key.data();
rkey->ksiz = key.size(); rkey->ksiz = key.size();
rkey->bidx = hash_record(rkey->kbuf, rkey->ksiz) % bnum_; rkey->bidx = hash_record(rkey->kbuf, rkey->ksiz) % bnum_;
lidxs.insert(rkey->bidx % SDBRLOCKSLOT); lidxs.insert(rkey->bidx % RLOCKSLOT);
} }
std::set<size_t>::iterator lit = lidxs.begin(); std::set<size_t>::iterator lit = lidxs.begin();
std::set<size_t>::iterator litend = lidxs.end(); std::set<size_t>::iterator litend = lidxs.end();
while (lit != litend) { while (lit != litend) {
if (writable) { if (writable) {
rlock_.lock_writer(*lit); rlock_.lock_writer(*lit);
} else { } else {
rlock_.lock_reader(*lit); rlock_.lock_reader(*lit);
} }
lit++; ++lit;
} }
for (size_t i = 0; i < knum; i++) { for (size_t i = 0; i < knum; i++) {
RecordKey* rkey = rkeys + i; RecordKey* rkey = rkeys + i;
accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->bidx); accept_impl(rkey->kbuf, rkey->ksiz, visitor, rkey->bidx);
} }
lit = lidxs.begin(); lit = lidxs.begin();
litend = lidxs.end(); litend = lidxs.end();
while (lit != litend) { while (lit != litend) {
rlock_.unlock(*lit); rlock_.unlock(*lit);
lit++; ++lit;
} }
delete[] rkeys; delete[] rkeys;
return true; return true;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed. * @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 448 skipping to change at line 441
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
ScopedVisitor svis(visitor);
int64_t allcnt = count_; int64_t allcnt = count_;
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) { if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
int64_t curcnt = 0; int64_t curcnt = 0;
for (size_t i = 0; i < bnum_; i++) { for (size_t i = 0; i < bnum_; i++) {
char* rbuf = buckets_[i]; char* rbuf = buckets_[i];
while (rbuf) { while (rbuf) {
curcnt++; curcnt++;
skipping to change at line 642 skipping to change at line 636
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wcnt >= SDBLOCKBUSYLOOP) { if (wcnt >= LOCKBUSYLOOP) {
Thread::chill(); Thread::chill();
} else { } else {
Thread::yield(); Thread::yield();
wcnt++; wcnt++;
} }
} }
tran_ = true; tran_ = true;
trcount_ = count_; trcount_ = count_;
trsize_ = size_; trsize_ = size_;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction"); trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction");
skipping to change at line 867 skipping to change at line 861
* @param bnum the number of buckets of the hash table. * @param bnum the number of buckets of the hash table.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_buckets(int64_t bnum) { bool tune_buckets(int64_t bnum) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
bnum_ = bnum >= 0 ? bnum : SDBDEFBNUM; bnum_ = bnum >= 0 ? bnum : DEFBNUM;
if (bnum_ > (size_t)INT16MAX) bnum_ = nearbyprime(bnum_); if (bnum_ > (size_t)INT16MAX) bnum_ = nearbyprime(bnum_);
return true; return true;
} }
/** /**
* Get the opaque data. * Get the opaque data.
* @return the pointer to the opaque data region, whose size is 16 bytes. * @return the pointer to the opaque data region, whose size is 16 bytes.
*/ */
char* opaque() { char* opaque() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
skipping to change at line 976 skipping to change at line 970
* MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f or beginning * MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f or beginning
* transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT rigger::ABORTTRAN * transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT rigger::ABORTTRAN
* for aborting transaction, and MetaTrigger::MISC for miscellaneous oper ations. * for aborting transaction, and MetaTrigger::MISC for miscellaneous oper ations.
* @param message the supplement message. * @param message the supplement message.
*/ */
void trigger_meta(MetaTrigger::Kind kind, const char* message) { void trigger_meta(MetaTrigger::Kind kind, const char* message) {
_assert_(message); _assert_(message);
if (mtrigger_) mtrigger_->trigger(kind, message); if (mtrigger_) mtrigger_->trigger(kind, message);
} }
private: private:
/** The number of slots of the record lock. */
static const int32_t RLOCKSLOT = 1024;
/** The default bucket number. */
static const size_t DEFBNUM = 1048583LL;
/** The size of the opaque buffer. */
static const size_t OPAQUESIZ = 16;
/** The threshold of busy loop and sleep for locking. */
static const uint32_t LOCKBUSYLOOP = 8192;
/** The mininum number of buckets to use mmap. */
static const size_t MAPZMAPBNUM = 32768;
/** /**
* Record data. * Record data.
*/ */
struct Record { struct Record {
/** constructor */ /** constructor */
Record(char* child, const char* kbuf, uint64_t ksiz, const char* vbuf, uint64_t vsiz) : Record(char* child, const char* kbuf, uint64_t ksiz, const char* vbuf, uint64_t vsiz) :
child_(child), kbuf_(kbuf), ksiz_(ksiz), vbuf_(vbuf), vsiz_(vsiz) { child_(child), kbuf_(kbuf), ksiz_(ksiz), vbuf_(vbuf), vsiz_(vsiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ);
} }
/** constructor */ /** constructor */
skipping to change at line 1103 skipping to change at line 1107
return vbuf_; return vbuf_;
} }
const char* vbuf_; ///< region of the value const char* vbuf_; ///< region of the value
size_t vsiz_; ///< size of the value size_t vsiz_; ///< size of the value
}; };
/** /**
* Removing visitor. * Removing visitor.
*/ */
class Remover : public Visitor { class Remover : public Visitor {
private: private:
/** constructor */ /** visit a record */
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp ); _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp );
return REMOVE; return REMOVE;
} }
}; };
/** /**
* Scoped visiotor.
*/
class ScopedVisitor {
public:
/** constructor */
ScopedVisitor(Visitor* visitor) : visitor_(visitor) {
_assert_(visitor);
visitor_->visit_before();
}
/** destructor */
~ScopedVisitor() {
_assert_(true);
visitor_->visit_after();
}
private:
Visitor* visitor_; ///< visitor
};
/**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param bidx the bucket index. * @param bidx the bucket index.
*/ */
void accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, size_t bidx) { void accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, size_t bidx) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
char* rbuf = buckets_[bidx]; char* rbuf = buckets_[bidx];
char** entp = buckets_ + bidx; char** entp = buckets_ + bidx;
skipping to change at line 1207 skipping to change at line 1229
*/ */
void escape_cursors(char* rbuf) { void escape_cursors(char* rbuf) {
_assert_(rbuf); _assert_(rbuf);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
if (curs_.empty()) return; if (curs_.empty()) return;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->rbuf_ == rbuf) cur->step_impl(); if (cur->rbuf_ == rbuf) cur->step_impl();
cit++; ++cit;
} }
} }
/** /**
* Adjust cursors on re-allocated records. * Adjust cursors on re-allocated records.
* @param obuf the old address. * @param obuf the old address.
* @param nbuf the new address. * @param nbuf the new address.
*/ */
void adjust_cursors(char* obuf, char* nbuf) { void adjust_cursors(char* obuf, char* nbuf) {
_assert_(obuf && nbuf); _assert_(obuf && nbuf);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
if (curs_.empty()) return; if (curs_.empty()) return;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->rbuf_ == obuf) cur->rbuf_ = nbuf; if (cur->rbuf_ == obuf) cur->rbuf_ = nbuf;
cit++; ++cit;
} }
} }
/** /**
* Disable all cursors. * Disable all cursors.
*/ */
void disable_cursors() { void disable_cursors() {
_assert_(true); _assert_(true);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->bidx_ = -1; cur->bidx_ = -1;
cur->rbuf_ = NULL; cur->rbuf_ = NULL;
cit++; ++cit;
} }
} }
/** /**
* Apply transaction logs. * Apply transaction logs.
*/ */
void apply_trlogs() { void apply_trlogs() {
_assert_(true); _assert_(true);
TranLogList::const_iterator it = trlogs_.end(); TranLogList::const_iterator it = trlogs_.end();
TranLogList::const_iterator itbeg = trlogs_.begin(); TranLogList::const_iterator itbeg = trlogs_.begin();
while (it != itbeg) { while (it != itbeg) {
it--; --it;
const char* kbuf = it->key.c_str(); const char* kbuf = it->key.c_str();
size_t ksiz = it->key.size(); size_t ksiz = it->key.size();
const char* vbuf = it->value.c_str(); const char* vbuf = it->value.c_str();
size_t vsiz = it->value.size(); size_t vsiz = it->value.size();
size_t bidx = hash_record(kbuf, ksiz) % bnum_; size_t bidx = hash_record(kbuf, ksiz) % bnum_;
if (it->full) { if (it->full) {
Setter setter(vbuf, vsiz); Setter setter(vbuf, vsiz);
accept_impl(kbuf, ksiz, &setter, bidx); accept_impl(kbuf, ksiz, &setter, bidx);
} else { } else {
Remover remover; Remover remover;
skipping to change at line 1292 skipping to change at line 1314
MetaTrigger* mtrigger_; MetaTrigger* mtrigger_;
/** The open mode. */ /** The open mode. */
uint32_t omode_; uint32_t omode_;
/** The cursor objects. */ /** The cursor objects. */
CursorList curs_; CursorList curs_;
/** The path of the database file. */ /** The path of the database file. */
std::string path_; std::string path_;
/** The number of buckets. */ /** The number of buckets. */
size_t bnum_; size_t bnum_;
/** The opaque data. */ /** The opaque data. */
char opaque_[SDBOPAQUESIZ]; char opaque_[OPAQUESIZ];
/** The record number. */ /** The record number. */
AtomicInt64 count_; AtomicInt64 count_;
/** The total size of records. */ /** The total size of records. */
AtomicInt64 size_; AtomicInt64 size_;
/** The bucket array. */ /** The bucket array. */
char** buckets_; char** buckets_;
/** The flag whether in transaction. */ /** The flag whether in transaction. */
bool tran_; bool tran_;
/** The list of transaction logs. */ /** The list of transaction logs. */
TranLogList trlogs_; TranLogList trlogs_;
 End of changes. 22 change blocks. 
29 lines changed or deleted 49 lines changed or added


 kcthread.h   kcthread.h 
skipping to change at line 946 skipping to change at line 946
* @note This function blocks until all tasks in the queue are popped. * @note This function blocks until all tasks in the queue are popped.
*/ */
void finish() { void finish() {
_assert_(true); _assert_(true);
mutex_.lock(); mutex_.lock();
TaskList::iterator it = tasks_.begin(); TaskList::iterator it = tasks_.begin();
TaskList::iterator itend = tasks_.end(); TaskList::iterator itend = tasks_.end();
while (it != itend) { while (it != itend) {
Task* task = *it; Task* task = *it;
task->aborted_ = true; task->aborted_ = true;
it++; ++it;
} }
cond_.broadcast(); cond_.broadcast();
mutex_.unlock(); mutex_.unlock();
Thread::yield(); Thread::yield();
for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) { for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) {
mutex_.lock(); mutex_.lock();
if (tasks_.empty()) { if (tasks_.empty()) {
mutex_.unlock(); mutex_.unlock();
break; break;
} }
 End of changes. 1 change blocks. 
1 lines changed or deleted 1 lines changed or added


 kcutil.h   kcutil.h 
skipping to change at line 41 skipping to change at line 41
/** The library version. */ /** The library version. */
extern const int32_t LIBVER; extern const int32_t LIBVER;
/** The library revision. */ /** The library revision. */
extern const int32_t LIBREV; extern const int32_t LIBREV;
/** The database format version. */ /** The database format version. */
extern const int32_t FMTVER; extern const int32_t FMTVER;
/** The system name. */ /** The system name. */
extern const char* SYSNAME; extern const char* const SYSNAME;
/** The flag for big endian environments. */ /** The flag for big endian environments. */
extern const bool BIGEND; extern const bool BIGEND;
/** The clock tick of interruption. */ /** The clock tick of interruption. */
extern const int32_t CLOCKTICK; extern const int32_t CLOCKTICK;
/** The size of a page. */ /** The size of a page. */
extern const int32_t PAGESIZE; extern const int32_t PAGESIZE;
skipping to change at line 1351 skipping to change at line 1351
_assert_(elems); _assert_(elems);
elems->clear(); elems->clear();
std::string::const_iterator it = str.begin(); std::string::const_iterator it = str.begin();
std::string::const_iterator pv = it; std::string::const_iterator pv = it;
while (it != str.end()) { while (it != str.end()) {
if (*it == delim) { if (*it == delim) {
std::string col(pv, it); std::string col(pv, it);
elems->push_back(col); elems->push_back(col);
pv = it + 1; pv = it + 1;
} }
it++; ++it;
} }
std::string col(pv, it); std::string col(pv, it);
elems->push_back(col); elems->push_back(col);
return elems->size(); return elems->size();
} }
/** /**
* Split a string with delimiters. * Split a string with delimiters.
*/ */
inline size_t strsplit(const std::string& str, const std::string& delims, inline size_t strsplit(const std::string& str, const std::string& delims,
skipping to change at line 1374 skipping to change at line 1374
elems->clear(); elems->clear();
std::string::const_iterator it = str.begin(); std::string::const_iterator it = str.begin();
std::string::const_iterator pv = it; std::string::const_iterator pv = it;
while (it != str.end()) { while (it != str.end()) {
while (delims.find(*it, 0) != std::string::npos) { while (delims.find(*it, 0) != std::string::npos) {
std::string col(pv, it); std::string col(pv, it);
elems->push_back(col); elems->push_back(col);
pv = it + 1; pv = it + 1;
break; break;
} }
it++; ++it;
} }
std::string col(pv, it); std::string col(pv, it);
elems->push_back(col); elems->push_back(col);
return elems->size(); return elems->size();
} }
/** /**
* Serialize a string vector object into a string object. * Serialize a string vector object into a string object.
*/ */
inline void strvecdump(const std::vector<std::string>& src, std::string* de st) { inline void strvecdump(const std::vector<std::string>& src, std::string* de st) {
_assert_(dest); _assert_(dest);
std::vector<std::string>::const_iterator it = src.begin(); std::vector<std::string>::const_iterator it = src.begin();
std::vector<std::string>::const_iterator itend = src.end(); std::vector<std::string>::const_iterator itend = src.end();
size_t dsiz = 1; size_t dsiz = 1;
while (it != itend) { while (it != itend) {
dsiz += 2 + it->size(); dsiz += 2 + it->size();
it++; ++it;
} }
dest->reserve(dest->size() + dsiz); dest->reserve(dest->size() + dsiz);
it = src.begin(); it = src.begin();
while (it != itend) { while (it != itend) {
char nbuf[NUMBUFSIZ]; char nbuf[NUMBUFSIZ];
size_t nsiz = writevarnum(nbuf, it->size()); size_t nsiz = writevarnum(nbuf, it->size());
dest->append(nbuf, nsiz); dest->append(nbuf, nsiz);
dest->append(it->data(), it->size()); dest->append(it->data(), it->size());
it++; ++it;
} }
} }
/** /**
* Deserialize a string object into a string vector object. * Deserialize a string object into a string vector object.
*/ */
inline void strvecload(const std::string& src, std::vector<std::string>* de st) { inline void strvecload(const std::string& src, std::vector<std::string>* de st) {
_assert_(dest); _assert_(dest);
const char* rp = src.data(); const char* rp = src.data();
size_t size = src.size(); size_t size = src.size();
skipping to change at line 1433 skipping to change at line 1433
/** /**
* Serialize a string vector object into a string object. * Serialize a string vector object into a string object.
*/ */
inline void strmapdump(const std::map<std::string, std::string>& src, std:: string* dest) { inline void strmapdump(const std::map<std::string, std::string>& src, std:: string* dest) {
_assert_(dest); _assert_(dest);
std::map<std::string, std::string>::const_iterator it = src.begin(); std::map<std::string, std::string>::const_iterator it = src.begin();
std::map<std::string, std::string>::const_iterator itend = src.end(); std::map<std::string, std::string>::const_iterator itend = src.end();
size_t dsiz = 1; size_t dsiz = 1;
while (it != itend) { while (it != itend) {
dsiz += 4 + it->first.size() + it->second.size(); dsiz += 4 + it->first.size() + it->second.size();
it++; ++it;
} }
dest->reserve(dest->size() + dsiz); dest->reserve(dest->size() + dsiz);
it = src.begin(); it = src.begin();
while (it != itend) { while (it != itend) {
char nbuf[NUMBUFSIZ*2]; char nbuf[NUMBUFSIZ*2];
size_t nsiz = writevarnum(nbuf, it->first.size()); size_t nsiz = writevarnum(nbuf, it->first.size());
nsiz += writevarnum(nbuf + nsiz, it->second.size()); nsiz += writevarnum(nbuf + nsiz, it->second.size());
dest->append(nbuf, nsiz); dest->append(nbuf, nsiz);
dest->append(it->first.data(), it->first.size()); dest->append(it->first.data(), it->first.size());
dest->append(it->second.data(), it->second.size()); dest->append(it->second.data(), it->second.size());
it++; ++it;
} }
} }
/** /**
* Deserialize a string object into a string map object. * Deserialize a string object into a string map object.
*/ */
inline void strmapload(const std::string& src, std::map<std::string, std::s tring>* dest) { inline void strmapload(const std::string& src, std::map<std::string, std::s tring>* dest) {
_assert_(dest); _assert_(dest);
const char* rp = src.data(); const char* rp = src.data();
int64_t size = src.size(); int64_t size = src.size();
 End of changes. 7 change blocks. 
7 lines changed or deleted 7 lines changed or added

This html diff was produced by rfcdiff 1.41. The latest version is available from http://tools.ietf.org/tools/rfcdiff/