kccachedb.h   kccachedb.h 
/************************************************************************** *********************** /************************************************************************** ***********************
* Cache database * Cache hash database
* Copyright (C) 2009-2010 FAL Labs * Copyright (C) 2009-2010 FAL Labs
* This file is part of Kyoto Cabinet. * This file is part of Kyoto Cabinet.
* This program is free software: you can redistribute it and/or modify it under the terms of * This program is free software: you can redistribute it and/or modify it under the terms of
* the GNU General Public License as published by the Free Software Foundat ion, either version * the GNU General Public License as published by the Free Software Foundat ion, either version
* 3 of the License, or any later version. * 3 of the License, or any later version.
* This program is distributed in the hope that it will be useful, but WITH OUT ANY WARRANTY; * This program is distributed in the hope that it will be useful, but WITH OUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PA RTICULAR PURPOSE. * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PA RTICULAR PURPOSE.
* See the GNU General Public License for more details. * See the GNU General Public License for more details.
* You should have received a copy of the GNU General Public License along with this program. * You should have received a copy of the GNU General Public License along with this program.
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
************************************************************************** ***********************/ ************************************************************************** ***********************/
#ifndef _KCCACHEDB_H // duplication check #ifndef _KCCACHEDB_H // duplication check
#define _KCCACHEDB_H #define _KCCACHEDB_H
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
#include <kcdb.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h>
#include <kcdb.h>
#include <kcplantdb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation. * Constants for implementation.
*/ */
namespace { namespace {
const int32_t CDBSLOTNUM = 16; ///< number of slot tables const int32_t CDBSLOTNUM = 16; ///< number of slot tables
const size_t CDBDEFBNUM = 1048583LL; ///< default bucket number const size_t CDBDEFBNUM = 1048583LL; ///< default bucket number
const size_t CDBZMAPBNUM = 32768; ///< mininum number of buckets to use mmap const size_t CDBZMAPBNUM = 32768; ///< mininum number of buckets to use mmap
const uint32_t CDBKSIZMAX = 0xfffff; ///< maximum size of each key const uint32_t CDBKSIZMAX = 0xfffff; ///< maximum size of each key
const size_t CDBRECBUFSIZ = 48; ///< size of the record buffer const size_t CDBRECBUFSIZ = 48; ///< size of the record buffer
const size_t CDBOPAQUESIZ = 16; ///< size of the opaque buffer
} }
/** /**
* On-memory hash database with LRU deletion. * On-memory hash database with LRU deletion.
* @note This class is a concrete class to operate a hash database on memor y. This class can be * @note This class is a concrete class to operate a hash database on memor y. This class can be
* inherited but overwriting methods is forbidden. Before every database o peration, it is * inherited but overwriting methods is forbidden. Before every database o peration, it is
* necessary to call the CacheDB::open method in order to open a database f ile and connect the * necessary to call the CacheDB::open method in order to open a database f ile and connect the
* database object to it. To avoid data missing or corruption, it is impor tant to close every * database object to it. To avoid data missing or corruption, it is impor tant to close every
* database file by the CacheDB::close method when the database is no longe r in use. It is * database file by the CacheDB::close method when the database is no longe r in use. It is
* forbidden for multible database objects in a process to open the same da tabase at the same * forbidden for multible database objects in a process to open the same da tabase at the same
* time. * time. It is forbidden to share a database object with child processes.
*/ */
class CacheDB : public FileDB { class CacheDB : public BasicDB {
friend class PlantDB<CacheDB, BasicDB::TYPEGRASS>;
public: public:
class Cursor; class Cursor;
private: private:
struct Record; struct Record;
struct TranLog; struct TranLog;
struct Slot; struct Slot;
class Repeater; class Repeater;
class Setter; class Setter;
class Remover; class Remover;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
/** An alias of list of transaction logs. */ /** An alias of list of transaction logs. */
typedef std::list<TranLog> TranLogList; typedef std::list<TranLog> TranLogList;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public FileDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class CacheDB; friend class CacheDB;
public: public:
/** /**
* Constructor. * Constructor.
* @param db the container database object. * @param db the container database object.
*/ */
explicit Cursor(CacheDB* db) : db_(db), sidx_(-1), rec_(NULL) { explicit Cursor(CacheDB* db) : db_(db), sidx_(-1), rec_(NULL) {
_assert_(db); _assert_(db);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
db_->curs_.push_back(this); db_->curs_.push_back(this);
skipping to change at line 95 skipping to change at line 99
if (!db_) return; if (!db_) return;
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
db_->curs_.remove(this); db_->curs_.remove(this);
} }
/** /**
* Accept a visitor to the current record. * Accept a visitor to the current record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only operation. * @param writable true for writable operation, or false for read-only operation.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and othe * @note The operation for each record is performed atomically and othe
r threads accessing r threads accessing
* the same record are blocked. * the same record are blocked. To avoid deadlock, any database operat
ion must not be
* performed in this function.
*/ */
bool accept(Visitor* visitor, bool writable = true, bool step = false) { bool accept(Visitor* visitor, bool writable = true, bool step = false) {
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(db_->omode_ & OWRITER)) { if (writable && !(db_->omode_ & OWRITER)) {
db_->set_error(Error::NOPERM, "permission denied"); db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
if (sidx_ < 0 || !rec_) { if (sidx_ < 0 || !rec_) {
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
uint32_t rksiz = rec_->ksiz & CDBKSIZMAX; uint32_t rksiz = rec_->ksiz & CDBKSIZMAX;
char* dbuf = (char*)rec_ + sizeof(*rec_); char* dbuf = (char*)rec_ + sizeof(*rec_);
const char* rvbuf = dbuf + rksiz;
size_t rvsiz = rec_->vsiz;
char* zbuf = NULL;
size_t zsiz = 0;
if (db_->comp_) {
zbuf = db_->comp_->decompress(rvbuf, rvsiz, &zsiz);
if (zbuf) {
rvbuf = zbuf;
rvsiz = zsiz;
}
}
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(dbuf, rksiz, dbuf + rksiz, rec const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &vs
_->vsiz, &vsiz); iz);
delete[] zbuf;
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
uint64_t hash = db_->hash_record(dbuf, rksiz) / CDBSLOTNUM; uint64_t hash = db_->hash_record(dbuf, rksiz) / CDBSLOTNUM;
Slot* slot = db_->slots_ + sidx_; Slot* slot = db_->slots_ + sidx_;
Repeater repeater(Visitor::REMOVE, 0); Repeater repeater(Visitor::REMOVE, 0);
db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, true); db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, tr ue);
} else if (vbuf == Visitor::NOP) { } else if (vbuf == Visitor::NOP) {
if (step) step_impl(); if (step) step_impl();
} else { } else {
uint64_t hash = db_->hash_record(dbuf, rksiz) / CDBSLOTNUM; uint64_t hash = db_->hash_record(dbuf, rksiz) / CDBSLOTNUM;
Slot* slot = db_->slots_ + sidx_; Slot* slot = db_->slots_ + sidx_;
Repeater repeater(vbuf, vsiz); Repeater repeater(vbuf, vsiz);
db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, true); db_->accept_impl(slot, hash, dbuf, rksiz, &repeater, db_->comp_, tr ue);
if (step) step_impl(); if (step) step_impl();
} }
return true; return true;
} }
/** /**
* Jump the cursor to the first record. * Jump the cursor to the first record for forward scan.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump() { bool jump() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < CDBSLOTNUM; i++) {
Slot* slot = db_->slots_ + i; Slot* slot = db_->slots_ + i;
if (slot->first) { if (slot->first) {
sidx_ = i; sidx_ = i;
rec_ = slot->first; rec_ = slot->first;
return true; return true;
} }
} }
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
sidx_ = -1; sidx_ = -1;
rec_ = NULL; rec_ = NULL;
return false; return false;
} }
/** /**
* Jump the cursor onto a record. * Jump the cursor to a record for forward scan.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump(const char* kbuf, size_t ksiz) { bool jump(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (ksiz > CDBKSIZMAX) ksiz = CDBKSIZMAX; if (ksiz > CDBKSIZMAX) ksiz = CDBKSIZMAX;
uint64_t hash = db_->hash_record(kbuf, ksiz); uint64_t hash = db_->hash_record(kbuf, ksiz);
int32_t sidx = hash % CDBSLOTNUM; int32_t sidx = hash % CDBSLOTNUM;
hash /= CDBSLOTNUM; hash /= CDBSLOTNUM;
Slot* slot = db_->slots_ + sidx; Slot* slot = db_->slots_ + sidx;
size_t bidx = hash % slot->bnum; size_t bidx = hash % slot->bnum;
Record* rec = slot->buckets[bidx]; Record* rec = slot->buckets[bidx];
Record** entp = slot->buckets + bidx; Record** entp = slot->buckets + bidx;
skipping to change at line 204 skipping to change at line 221
} else if (kcmp > 0) { } else if (kcmp > 0) {
entp = &rec->right; entp = &rec->right;
rec = rec->right; rec = rec->right;
} else { } else {
sidx_ = sidx; sidx_ = sidx;
rec_ = rec; rec_ = rec;
return true; return true;
} }
} }
} }
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
sidx_ = -1; sidx_ = -1;
rec_ = NULL; rec_ = NULL;
return false; return false;
} }
/** /**
* Jump the cursor to a record. * Jump the cursor to a record for forward scan.
* @note Equal to the original Cursor::jump method except that the para meter is std::string. * @note Equal to the original Cursor::jump method except that the para meter is std::string.
*/ */
bool jump(const std::string& key) { bool jump(const std::string& key) {
_assert_(true); _assert_(true);
return jump(key.c_str(), key.size()); return jump(key.c_str(), key.size());
} }
/** /**
* Jump the cursor to the last record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back() {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Jump the cursor to a record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Jump the cursor to a record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back(const std::string& key) {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Step the cursor to the next record. * Step the cursor to the next record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool step() { bool step() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (sidx_ < 0 || !rec_) { if (sidx_ < 0 || !rec_) {
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
bool err = false; bool err = false;
if (!step_impl()) err = true; if (!step_impl()) err = true;
return !err; return !err;
} }
/** /**
* Step the cursor to the previous record.
* @note This is a dummy implementation for compatibility.
*/
bool step_back() {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Get the database object. * Get the database object.
* @return the database object. * @return the database object.
*/ */
CacheDB* db() { CacheDB* db() {
_assert_(true); _assert_(true);
return db_; return db_;
} }
private: private:
/** Dummy constructor to forbid the use. */
Cursor(const Cursor&);
/** Dummy Operator to forbid the use. */
Cursor& operator =(const Cursor&);
/** /**
* Step the cursor to the next record. * Step the cursor to the next record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool step_impl() { bool step_impl() {
_assert_(true);
rec_ = rec_->next; rec_ = rec_->next;
if (!rec_) { if (!rec_) {
for (int32_t i = sidx_ + 1; i < CDBSLOTNUM; i++) { for (int32_t i = sidx_ + 1; i < CDBSLOTNUM; i++) {
Slot* slot = db_->slots_ + i; Slot* slot = db_->slots_ + i;
if (slot->first) { if (slot->first) {
sidx_ = i; sidx_ = i;
rec_ = slot->first; rec_ = slot->first;
return true; return true;
} }
} }
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
sidx_ = -1; sidx_ = -1;
rec_ = NULL; rec_ = NULL;
return false; return false;
} }
return true; return true;
} }
/** Dummy constructor to forbid the use. */
Cursor(const Cursor&);
/** Dummy Operator to forbid the use. */
Cursor& operator =(const Cursor&);
/** The inner database. */ /** The inner database. */
CacheDB* db_; CacheDB* db_;
/** The index of the current slot. */ /** The index of the current slot. */
int32_t sidx_; int32_t sidx_;
/** The current record. */ /** The current record. */
Record* rec_; Record* rec_;
}; };
/** /**
* Tuning options.
*/
enum Option {
TSMALL = 1 << 0, ///< dummy for compatibility
TLINEAR = 1 << 1, ///< dummy for compatibility
TCOMPRESS = 1 << 2 ///< compress each record
};
/**
* Status flags.
*/
enum Flag {
FOPEN = 1 << 0, ///< dummy for compatibility
FFATAL = 1 << 1 ///< dummy for compatibility
};
/**
* Default constructor. * Default constructor.
*/ */
explicit CacheDB() : explicit CacheDB() :
mlock_(), flock_(), error_(), omode_(0), curs_(), path_(""), mlock_(), flock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NU
bnum_(CDBDEFBNUM), capcnt_(-1), capsiz_(-1), slots_(), tran_(false) { LL),
omode_(0), curs_(), path_(""), type_(TYPECACHE),
opts_(0), bnum_(CDBDEFBNUM), capcnt_(-1), capsiz_(-1),
opaque_(), embcomp_(&ZLIBRAWCOMP), comp_(NULL), slots_(), tran_(false)
{
_assert_(true); _assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~CacheDB() { virtual ~CacheDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
if (!curs_.empty()) { if (!curs_.empty()) {
skipping to change at line 310 skipping to change at line 401
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and other * @note The operation for each record is performed atomically and other
threads accessing the threads accessing the
* same record are blocked. * same record are blocked. To avoid deadlock, any database operation mu
st not be performed in
* this function.
*/ */
bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) { bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
if (ksiz > CDBKSIZMAX) ksiz = CDBKSIZMAX; if (ksiz > CDBKSIZMAX) ksiz = CDBKSIZMAX;
uint64_t hash = hash_record(kbuf, ksiz); uint64_t hash = hash_record(kbuf, ksiz);
int32_t sidx = hash % CDBSLOTNUM; int32_t sidx = hash % CDBSLOTNUM;
hash /= CDBSLOTNUM; hash /= CDBSLOTNUM;
Slot* slot = slots_ + sidx; Slot* slot = slots_ + sidx;
slot->lock.lock(); slot->lock.lock();
accept_impl(slot, hash, kbuf, ksiz, visitor, false); accept_impl(slot, hash, kbuf, ksiz, visitor, comp_, false);
slot->lock.unlock(); slot->lock.unlock();
return true; return true;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the whole iteration is performed atomically and other threads ar * @note The whole iteration is performed atomically and other threads ar
e blocked. e blocked. To avoid
* deadlock, any database operation must not be performed in this functio
n.
*/ */
bool iterate(Visitor *visitor, bool writable = true) { bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* che cker = NULL) {
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
int64_t allcnt = count_impl();
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
int64_t curcnt = 0;
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < CDBSLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
Record* rec = slot->first; Record* rec = slot->first;
while (rec) { while (rec) {
Record* next = rec->next; Record* next = rec->next;
uint32_t rksiz = rec->ksiz & CDBKSIZMAX; uint32_t rksiz = rec->ksiz & CDBKSIZMAX;
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
const char* rvbuf = dbuf + rksiz;
size_t rvsiz = rec->vsiz;
char* zbuf = NULL;
size_t zsiz = 0;
if (comp_) {
zbuf = comp_->decompress(rvbuf, rvsiz, &zsiz);
if (zbuf) {
rvbuf = zbuf;
rvsiz = zsiz;
}
}
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(dbuf, rksiz, dbuf + rksiz, r const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz, &
ec->vsiz, &vsiz); vsiz);
delete[] zbuf;
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
uint64_t hash = hash_record(dbuf, rksiz) / CDBSLOTNUM; uint64_t hash = hash_record(dbuf, rksiz) / CDBSLOTNUM;
Repeater repeater(Visitor::REMOVE, 0); Repeater repeater(Visitor::REMOVE, 0);
accept_impl(slot, hash, dbuf, rksiz, &repeater, true); accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, true);
} else if (vbuf != Visitor::NOP) { } else if (vbuf != Visitor::NOP) {
uint64_t hash = hash_record(dbuf, rksiz) / CDBSLOTNUM; uint64_t hash = hash_record(dbuf, rksiz) / CDBSLOTNUM;
Repeater repeater(vbuf, vsiz); Repeater repeater(vbuf, vsiz);
accept_impl(slot, hash, dbuf, rksiz, &repeater, true); accept_impl(slot, hash, dbuf, rksiz, &repeater, comp_, true);
} }
rec = next; rec = next;
curcnt++;
if (checker && !checker->check("iterate", "processing", curcnt, all
cnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
} }
} }
if (checker && !checker->check("iterate", "ending", -1, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
trigger_meta(MetaTrigger::ITERATE, "iterate");
return true; return true;
} }
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
Error error() const { Error error() const {
_assert_(true); _assert_(true);
return error_; return error_;
} }
/** /**
* Set the error information. * Set the error information.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param code an error code. * @param code an error code.
* @param message a supplement message. * @param message a supplement message.
*/ */
void set_error(Error::Code code, const char* message) { void set_error(const char* file, int32_t line, const char* func,
_assert_(message); Error::Code code, const char* message) {
_assert_(file && line > 0 && func && message);
error_->set(code, message); error_->set(code, message);
if (logger_) {
Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ?
Logger::ERROR : Logger::INFO;
if (kind & logkinds_)
report(file, line, func, kind, "%d: %s: %s", code, Error::codename(
code), message);
}
} }
/** /**
* Open a database file. * Open a database file.
* @param path the path of a database file. * @param path the path of a database file.
* @param mode the connection mode. CacheDB::OWRITER as a writer, CacheD B::OREADER as a * @param mode the connection mode. CacheDB::OWRITER as a writer, CacheD B::OREADER as a
* reader. The following may be added to the writer mode by bitwise-or: CacheDB::OCREATE, * reader. The following may be added to the writer mode by bitwise-or: CacheDB::OCREATE,
* which means it creates a new database if the file does not exist, Cach eDB::OTRUNCATE, which * which means it creates a new database if the file does not exist, Cach eDB::OTRUNCATE, which
* means it creates a new database regardless if the file exists, CacheDB ::OAUTOTRAN, which * means it creates a new database regardless if the file exists, CacheDB ::OAUTOTRAN, which
* means each updating operation is performed in implicit transaction, Ca cheDB::OAUTOSYNC, * means each updating operation is performed in implicit transaction, Ca cheDB::OAUTOSYNC,
* which means each updating operation is followed by implicit synchroniz ation with the file * which means each updating operation is followed by implicit synchroniz ation with the file
skipping to change at line 415 skipping to change at line 547
* detected. * detected.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note Every opened database must be closed by the CacheDB::close metho d when it is no * @note Every opened database must be closed by the CacheDB::close metho d when it is no
* longer in use. It is not allowed for two or more database objects in the same process to * longer in use. It is not allowed for two or more database objects in the same process to
* keep their connections to the same database file at the same time. * keep their connections to the same database file at the same time.
*/ */
bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", p ath.c_str());
omode_ = mode; omode_ = mode;
path_.append(path); path_.append(path);
size_t bnum = nearbyprime(bnum_ / CDBSLOTNUM); size_t bnum = nearbyprime(bnum_ / CDBSLOTNUM);
size_t capcnt = capcnt_ > 0 ? capcnt_ / CDBSLOTNUM + 1 : (1ULL << (size of(capcnt) * 8 - 1)); size_t capcnt = capcnt_ > 0 ? capcnt_ / CDBSLOTNUM + 1 : (1ULL << (size of(capcnt) * 8 - 1));
size_t capsiz = capsiz_ > 0 ? capsiz_ / CDBSLOTNUM + 1 : (1ULL << (size of(capsiz) * 8 - 1)); size_t capsiz = capsiz_ > 0 ? capsiz_ / CDBSLOTNUM + 1 : (1ULL << (size of(capsiz) * 8 - 1));
if (capsiz > sizeof(*this) / CDBSLOTNUM) capsiz -= sizeof(*this) / CDBS LOTNUM; if (capsiz > sizeof(*this) / CDBSLOTNUM) capsiz -= sizeof(*this) / CDBS LOTNUM;
if (capsiz > bnum * sizeof(Record*)) capsiz -= bnum * sizeof(Record*); if (capsiz > bnum * sizeof(Record*)) capsiz -= bnum * sizeof(Record*);
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < CDBSLOTNUM; i++) {
initialize_slot(slots_ + i, bnum, capcnt, capsiz); initialize_slot(slots_ + i, bnum, capcnt, capsiz);
} }
comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL;
std::memset(opaque_, 0, sizeof(opaque_));
trigger_meta(MetaTrigger::OPEN, "open");
return true; return true;
} }
/** /**
* Close the database file. * Close the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool close() { bool close() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", p ath_.c_str());
tran_ = false; tran_ = false;
for (int32_t i = CDBSLOTNUM - 1; i >= 0; i--) { for (int32_t i = CDBSLOTNUM - 1; i >= 0; i--) {
destroy_slot(slots_ + i); destroy_slot(slots_ + i);
} }
path_.clear(); path_.clear();
omode_ = 0; omode_ = 0;
trigger_meta(MetaTrigger::CLOSE, "close");
return true; return true;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @param proc a postprocessor object. If it is NULL, no postprocessing is performed. * @param proc a postprocessor object. If it is NULL, no postprocessing is performed.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize(bool hard = false, FileProcessor* proc = NULL) { bool synchronize(bool hard = false, FileProcessor* proc = NULL,
ProgressChecker* checker = NULL) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!(omode_ & OWRITER)) { bool err = false;
set_error(Error::NOPERM, "permission denied"); if ((omode_ & OWRITER) && checker &&
!checker->check("synchronize", "nothing to be synchronized", -1, -1
)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
bool err = false; if (proc) {
if (proc && !proc->process(path_, count_impl(), size_impl())) { if (checker && !checker->check("synchronize", "running the post proce
set_error(Error::LOGIC, "postprocessing failed"); ssor", -1, -1)) {
err = true; set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
if (!proc->process(path_, count_impl(), size_impl())) {
set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed");
err = true;
}
} }
trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize");
return !err; return !err;
} }
/** /**
* Begin transaction. * Begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction(bool hard = false) { bool begin_transaction(bool hard = false) {
_assert_(true); _assert_(true);
for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) { for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) {
mlock_.lock_writer(); mlock_.lock_writer();
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wsec > 1.0) wsec = 1.0; if (wsec > 1.0) wsec = 1.0;
Thread::sleep(wsec); Thread::sleep(wsec);
} }
tran_ = true; tran_ = true;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction");
mlock_.unlock(); mlock_.unlock();
return true; return true;
} }
/** /**
* Try to begin transaction. * Try to begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_try(bool hard = false) { bool begin_transaction_try(bool hard = false) {
_assert_(true); _assert_(true);
mlock_.lock_writer(); mlock_.lock_writer();
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (tran_) { if (tran_) {
set_error(Error::LOGIC, "competition avoided"); set_error(_KCCODELINE_, Error::LOGIC, "competition avoided");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
tran_ = true; tran_ = true;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try");
mlock_.unlock(); mlock_.unlock();
return true; return true;
} }
/** /**
* End transaction. * End transaction.
* @param commit true to commit the transaction, or false to abort the tr ansaction. * @param commit true to commit the transaction, or false to abort the tr ansaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool end_transaction(bool commit = true) { bool end_transaction(bool commit = true) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!tran_) { if (!tran_) {
set_error(Error::INVALID, "not in transaction"); set_error(_KCCODELINE_, Error::INVALID, "not in transaction");
return false; return false;
} }
if (!commit) disable_cursors(); if (!commit) disable_cursors();
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < CDBSLOTNUM; i++) {
if (!commit) apply_slot_trlogs(slots_ + i); if (!commit) apply_slot_trlogs(slots_ + i);
slots_[i].trlogs.clear(); slots_[i].trlogs.clear();
adjust_slot_capacity(slots_ + i); adjust_slot_capacity(slots_ + i);
} }
tran_ = false; tran_ = false;
trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction");
return true; return true;
} }
/** /**
* Remove all records. * Remove all records.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clear() { bool clear() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
disable_cursors(); disable_cursors();
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < CDBSLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
clear_slot(slot); clear_slot(slot);
} }
std::memset(opaque_, 0, sizeof(opaque_));
trigger_meta(MetaTrigger::CLEAR, "clear");
return true; return true;
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
*/ */
int64_t count() { int64_t count() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return count_impl(); return count_impl();
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes, or -1 on failure. * @return the size of the database file in bytes, or -1 on failure.
*/ */
int64_t size() { int64_t size() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return size_impl(); return size_impl();
} }
/** /**
* Get the path of the database file. * Get the path of the database file.
* @return the path of the database file, or an empty string on failure. * @return the path of the database file, or an empty string on failure.
*/ */
std::string path() { std::string path() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return ""; return "";
} }
return path_; return path_;
} }
/** /**
* Get the miscellaneous status information. * Get the miscellaneous status information.
* @param strmap a string map to contain the result. * @param strmap a string map to contain the result.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool status(std::map<std::string, std::string>* strmap) { bool status(std::map<std::string, std::string>* strmap) {
_assert_(strmap); _assert_(strmap);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
(*strmap)["type"] = "CacheDB"; (*strmap)["type"] = strprintf("%u", (unsigned)TYPECACHE);
(*strmap)["realtype"] = strprintf("%u", (unsigned)TYPECACHE); (*strmap)["realtype"] = strprintf("%u", (unsigned)type_);
(*strmap)["path"] = path_; (*strmap)["path"] = path_;
(*strmap)["libver"] = strprintf("%u", LIBVER);
(*strmap)["librev"] = strprintf("%u", LIBREV);
(*strmap)["fmtver"] = strprintf("%u", FMTVER);
(*strmap)["chksum"] = strprintf("%u", 0xff);
(*strmap)["opts"] = strprintf("%u", opts_);
(*strmap)["bnum"] = strprintf("%lld", (long long)bnum_);
(*strmap)["capcnt"] = strprintf("%lld", (long long)capcnt_);
(*strmap)["capsiz"] = strprintf("%lld", (long long)capsiz_);
(*strmap)["recovered"] = strprintf("%d", false);
(*strmap)["reorganized"] = strprintf("%d", false);
if (strmap->count("opaque") > 0)
(*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_));
if (strmap->count("bnum_used") > 0) {
int64_t cnt = 0;
for (int32_t i = 0; i < CDBSLOTNUM; i++) {
Slot* slot = slots_ + i;
Record** buckets = slot->buckets;
size_t bnum = slot->bnum;
for (size_t j = 0; j < bnum; j++) {
if (buckets[j]) cnt++;
}
}
(*strmap)["bnum_used"] = strprintf("%lld", (long long)cnt);
}
(*strmap)["count"] = strprintf("%lld", (long long)count_impl()); (*strmap)["count"] = strprintf("%lld", (long long)count_impl());
(*strmap)["size"] = strprintf("%lld", (long long)size_impl()); (*strmap)["size"] = strprintf("%lld", (long long)size_impl());
return true; return true;
} }
/** /**
* Create a cursor object. * Create a cursor object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note Because the object of the return value is allocated by the const ructor, it should be * @note Because the object of the return value is allocated by the const ructor, it should be
* released with the delete operator when it is no longer in use. * released with the delete operator when it is no longer in use.
*/ */
Cursor* cursor() { Cursor* cursor() {
_assert_(true); _assert_(true);
return new Cursor(this); return new Cursor(this);
} }
/** /**
* Set the internal logger.
* @param logger the logger object.
* @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for
debugging,
* Logger::INFO for normal information, Logger::WARN for warning, and Log
ger::ERROR for fatal
* error.
* @return true on success, or false on failure.
*/
bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::
ERROR) {
_assert_(logger);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
logger_ = logger;
logkinds_ = kinds;
return true;
}
/**
* Set the internal meta operation trigger.
* @param trigger the trigger object.
* @return true on success, or false on failure.
*/
bool tune_meta_trigger(MetaTrigger* trigger) {
_assert_(trigger);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
mtrigger_ = trigger;
return true;
}
/**
* Set the optional features.
* @param opts the optional features by bitwise-or: DirDB::TCOMPRESS to c
ompress each record.
* @return true on success, or false on failure.
*/
bool tune_options(int8_t opts) {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
opts_ = opts;
return true;
}
/**
* Set the number of buckets of the hash table. * Set the number of buckets of the hash table.
* @param bnum the number of buckets of the hash table. * @param bnum the number of buckets of the hash table.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_buckets(int64_t bnum) { bool tune_buckets(int64_t bnum) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
bnum_ = bnum >= 0 ? bnum : CDBDEFBNUM; bnum_ = bnum >= 0 ? bnum : CDBDEFBNUM;
return true; return true;
} }
/** /**
* Set the data compressor.
* @param comp the data compressor object.
* @return true on success, or false on failure.
*/
bool tune_compressor(Compressor* comp) {
_assert_(comp);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
embcomp_ = comp;
return true;
}
/**
* Set the capacity by record number. * Set the capacity by record number.
* @param count the maximum number of records. * @param count the maximum number of records.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool cap_count(int64_t count) { bool cap_count(int64_t count) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
capcnt_ = count; capcnt_ = count;
return true; return true;
} }
/** /**
* Set the capacity by memory usage. * Set the capacity by memory usage.
* @param size the maximum size of memory usage. * @param size the maximum size of memory usage.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool cap_size(int64_t size) { bool cap_size(int64_t size) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
capsiz_ = size; capsiz_ = size;
return true; return true;
} }
/**
* Get the opaque data.
* @return the pointer to the opaque data region, whose size is 16 bytes.
*/
char* opaque() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return NULL;
}
return opaque_;
}
/**
* Synchronize the opaque data.
* @return true on success, or false on failure.
*/
bool synchronize_opaque() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
if (!(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false;
}
return true;
}
protected:
/**
* Report a message for debugging.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param format the printf-like format string.
* @param ... used according to the format string.
*/
void report(const char* file, int32_t line, const char* func, Logger::Kin
d kind,
const char* format, ...) {
_assert_(file && line > 0 && func && format);
if (!logger_ || !(kind & logkinds_)) return;
std::string message;
strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str());
va_list ap;
va_start(ap, format);
vstrprintf(&message, format, ap);
va_end(ap);
logger_->log(file, line, func, kind, message.c_str());
}
/**
* Report a message for debugging with variable number of arguments.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param format the printf-like format string.
* @param ap used according to the format string.
*/
void report_valist(const char* file, int32_t line, const char* func, Logg
er::Kind kind,
const char* format, va_list ap) {
_assert_(file && line > 0 && func && format);
if (!logger_ || !(kind & logkinds_)) return;
std::string message;
strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str());
vstrprintf(&message, format, ap);
logger_->log(file, line, func, kind, message.c_str());
}
/**
* Report the content of a binary buffer for debugging.
* @param file the file name of the epicenter.
* @param line the line number of the epicenter.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param name the name of the information.
* @param buf the binary buffer.
* @param size the size of the binary buffer
*/
void report_binary(const char* file, int32_t line, const char* func, Logg
er::Kind kind,
const char* name, const char* buf, size_t size) {
_assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ);
if (!logger_) return;
char* hex = hexencode(buf, size);
report(file, line, func, kind, "%s=%s", name, hex);
delete[] hex;
}
/**
* Trigger a meta database operation.
* @param kind the kind of the event. MetaTrigger::OPEN for opening, Met
aTrigger::CLOSE for
* closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for ite
ration,
* MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f
or beginning
* transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT
rigger::ABORTTRAN
* for aborting transaction, and MetaTrigger::MISC for miscellaneous oper
ations.
* @param message the supplement message.
*/
void trigger_meta(MetaTrigger::Kind kind, const char* message) {
_assert_(message);
if (mtrigger_) mtrigger_->trigger(kind, message);
}
/**
* Set the database type.
* @param type the database type.
* @return true on success, or false on failure.
*/
bool tune_type(int8_t type) {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
type_ = type;
return true;
}
/**
* Get the library version.
* @return the library version, or 0 on failure.
*/
uint8_t libver() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return LIBVER;
}
/**
* Get the library revision.
* @return the library revision, or 0 on failure.
*/
uint8_t librev() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return LIBREV;
}
/**
* Get the format version.
* @return the format version, or 0 on failure.
*/
uint8_t fmtver() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return FMTVER;
}
/**
* Get the module checksum.
* @return the module checksum, or 0 on failure.
*/
uint8_t chksum() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return 0xff;
}
/**
* Get the database type.
* @return the database type, or 0 on failure.
*/
uint8_t type() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return type_;
}
/**
* Get the options.
* @return the options, or 0 on failure.
*/
uint8_t opts() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return opts_;
}
/**
* Get the data compressor.
* @return the data compressor, or NULL on failure.
*/
Compressor* comp() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return NULL;
}
return comp_;
}
/**
* Check whether the database was recovered or not.
* @return true if recovered, or false if not.
*/
bool recovered() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
return false;
}
/**
* Check whether the database was reorganized or not.
* @return true if recovered, or false if not.
*/
bool reorganized() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
return false;
}
private:
/**
* Set the power of the alignment of record size.
* @note This is a dummy implementation for compatibility.
*/
bool tune_alignment(int8_t apow) {
return true;
}
/**
* Set the power of the capacity of the free block pool.
* @note This is a dummy implementation for compatibility.
*/
bool tune_fbp(int8_t fpow) {
return true;
}
/**
* Set the size of the internal memory-mapped region.
* @note This is a dummy implementation for compatibility.
*/
bool tune_map(int64_t msiz) {
return true;
}
/**
* Set the unit step number of auto defragmentation.
* @note This is a dummy implementation for compatibility.
*/
bool tune_defrag(int64_t dfunit) {
return true;
}
/**
* Perform defragmentation of the file.
* @note This is a dummy implementation for compatibility.
*/
bool defrag(int64_t step = 0) {
return true;
}
/**
* Get the status flags.
* @note This is a dummy implementation for compatibility.
*/
uint8_t flags() {
return 0;
}
/**
* Get the alignment power.
* @note This is a dummy implementation for compatibility.
*/
uint8_t apow() {
return 0;
}
/**
* Get the free block pool power.
* @note This is a dummy implementation for compatibility.
*/
uint8_t fpow() {
return 0;
}
/**
* Get the bucket number.
* @note This is a dummy implementation for compatibility.
*/
int64_t bnum() {
return 1;
}
/**
* Get the size of the internal memory-mapped region.
* @note This is a dummy implementation for compatibility.
*/
int64_t msiz() {
return 0;
}
/**
* Get the unit step number of auto defragmentation.
* @note This is a dummy implementation for compatibility.
*/
int64_t dfunit() {
return 0;
}
private: private:
/** /**
* Record data. * Record data.
*/ */
struct Record { struct Record {
uint32_t ksiz; ///< size of the key uint32_t ksiz; ///< size of the key
uint32_t vsiz; ///< size of the value uint32_t vsiz; ///< size of the value
Record* left; ///< left child record Record* left; ///< left child record
Record* right; ///< right child record Record* right; ///< right child record
Record* prev; ///< privious record Record* prev; ///< privious record
skipping to change at line 708 skipping to change at line 1263
}; };
/** /**
* Transaction log. * Transaction log.
*/ */
struct TranLog { struct TranLog {
bool full; ///< flag whether full bool full; ///< flag whether full
std::string key; ///< old key std::string key; ///< old key
std::string value; ///< old value std::string value; ///< old value
/** constructor for a full record */ /** constructor for a full record */
explicit TranLog(const char* kbuf, size_t ksiz, const char* vbuf, size_ t vsiz) : explicit TranLog(const char* kbuf, size_t ksiz, const char* vbuf, size_ t vsiz) :
full(true), key(kbuf, ksiz), value(vbuf, vsiz) {} full(true), key(kbuf, ksiz), value(vbuf, vsiz) {
_assert_(true);
}
/** constructor for an empty record */ /** constructor for an empty record */
explicit TranLog(const char* kbuf, size_t ksiz) : full(false), key(kbuf explicit TranLog(const char* kbuf, size_t ksiz) : full(false), key(kbuf
, ksiz) {} , ksiz) {
_assert_(true);
}
}; };
/** /**
* Slot table. * Slot table.
*/ */
struct Slot { struct Slot {
SpinLock lock; ///< lock SpinLock lock; ///< lock
Record** buckets; ///< bucket array Record** buckets; ///< bucket array
size_t bnum; ///< number of buckets size_t bnum; ///< number of buckets
size_t capcnt; ///< cap of record number size_t capcnt; ///< cap of record number
size_t capsiz; ///< cap of memory usage size_t capsiz; ///< cap of memory usage
skipping to change at line 733 skipping to change at line 1292
size_t count; ///< number of records size_t count; ///< number of records
size_t size; ///< total size of records size_t size; ///< total size of records
TranLogList trlogs; ///< transaction logs TranLogList trlogs; ///< transaction logs
size_t trsize; ///< size before transaction size_t trsize; ///< size before transaction
}; };
/** /**
* Repeating visitor. * Repeating visitor.
*/ */
class Repeater : public Visitor { class Repeater : public Visitor {
public: public:
/** constructor */
explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(v siz) {} explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(v siz) {}
private: private:
/** process a full record */
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp );
*sp = vsiz_; *sp = vsiz_;
return vbuf_; return vbuf_;
} }
const char* vbuf_; const char* vbuf_; ///< region of the value
size_t vsiz_; size_t vsiz_; ///< size of the value
}; };
/** /**
* Setting visitor. * Setting visitor.
*/ */
class Setter : public Visitor { class Setter : public Visitor {
public: public:
/** constructor */
explicit Setter(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(vsi z) {} explicit Setter(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(vsi z) {}
private: private:
/** process a full record */
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp );
*sp = vsiz_; *sp = vsiz_;
return vbuf_; return vbuf_;
} }
/** process an empty record */
const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) { const char* visit_empty(const char* kbuf, size_t ksiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && sp);
*sp = vsiz_; *sp = vsiz_;
return vbuf_; return vbuf_;
} }
const char* vbuf_; const char* vbuf_; ///< region of the value
size_t vsiz_; size_t vsiz_; ///< size of the value
}; };
/** /**
* Removing visitor. * Removing visitor.
*/ */
class Remover : public Visitor { class Remover : public Visitor {
private: private:
/** constructor */
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp );
return REMOVE; return REMOVE;
} }
}; };
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param slot the slot of the record. * @param slot the slot of the record.
* @param hash the hash value of the key. * @param hash the hash value of the key.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param comp the data compressor.
* @param isiter true for iterator use, or false for direct use. * @param isiter true for iterator use, or false for direct use.
*/ */
void accept_impl(Slot* slot, uint64_t hash, const char* kbuf, size_t ksiz , Visitor* visitor, void accept_impl(Slot* slot, uint64_t hash, const char* kbuf, size_t ksiz , Visitor* visitor,
bool isiter) { Compressor* comp, bool isiter) {
_assert_(slot && kbuf && ksiz <= MEMMAXSIZ && visitor);
size_t bidx = hash % slot->bnum; size_t bidx = hash % slot->bnum;
Record* rec = slot->buckets[bidx]; Record* rec = slot->buckets[bidx];
Record** entp = slot->buckets + bidx; Record** entp = slot->buckets + bidx;
uint32_t fhash = fold_hash(hash) & ~CDBKSIZMAX; uint32_t fhash = fold_hash(hash) & ~CDBKSIZMAX;
while (rec) { while (rec) {
uint32_t rhash = rec->ksiz & ~CDBKSIZMAX; uint32_t rhash = rec->ksiz & ~CDBKSIZMAX;
uint32_t rksiz = rec->ksiz & CDBKSIZMAX; uint32_t rksiz = rec->ksiz & CDBKSIZMAX;
if (fhash > rhash) { if (fhash > rhash) {
entp = &rec->left; entp = &rec->left;
rec = rec->left; rec = rec->left;
skipping to change at line 806 skipping to change at line 1377
} else { } else {
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
int32_t kcmp = compare_keys(kbuf, ksiz, dbuf, rksiz); int32_t kcmp = compare_keys(kbuf, ksiz, dbuf, rksiz);
if (kcmp < 0) { if (kcmp < 0) {
entp = &rec->left; entp = &rec->left;
rec = rec->left; rec = rec->left;
} else if (kcmp > 0) { } else if (kcmp > 0) {
entp = &rec->right; entp = &rec->right;
rec = rec->right; rec = rec->right;
} else { } else {
const char* rvbuf = dbuf + rksiz;
size_t rvsiz = rec->vsiz;
char* zbuf = NULL;
size_t zsiz = 0;
if (comp) {
zbuf = comp->decompress(rvbuf, rvsiz, &zsiz);
if (zbuf) {
rvbuf = zbuf;
rvsiz = zsiz;
}
}
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(dbuf, rksiz, dbuf + rksiz, const char* vbuf = visitor->visit_full(dbuf, rksiz, rvbuf, rvsiz,
rec->vsiz, &vsiz); &vsiz);
delete[] zbuf;
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
if (tran_) { if (tran_) {
TranLog log(kbuf, ksiz, dbuf + rksiz, rec->vsiz); TranLog log(kbuf, ksiz, dbuf + rksiz, rec->vsiz);
slot->trlogs.push_back(log); slot->trlogs.push_back(log);
} }
if (!curs_.empty()) escape_cursors(rec); if (!curs_.empty()) escape_cursors(rec);
if (rec == slot->first) slot->first = rec->next; if (rec == slot->first) slot->first = rec->next;
if (rec == slot->last) slot->last = rec->prev; if (rec == slot->last) slot->last = rec->prev;
if (rec->prev) rec->prev->next = rec->next; if (rec->prev) rec->prev->next = rec->next;
if (rec->next) rec->next->prev = rec->prev; if (rec->next) rec->next->prev = rec->prev;
skipping to change at line 848 skipping to change at line 1431
*entp = pivot; *entp = pivot;
pivot->right = rec->right; pivot->right = rec->right;
} }
} }
slot->count--; slot->count--;
slot->size -= sizeof(Record) + rksiz + rec->vsiz; slot->size -= sizeof(Record) + rksiz + rec->vsiz;
xfree(rec); xfree(rec);
} else { } else {
bool adj = false; bool adj = false;
if (vbuf != Visitor::NOP) { if (vbuf != Visitor::NOP) {
char* zbuf = NULL;
size_t zsiz = 0;
if (comp) {
zbuf = comp->compress(vbuf, vsiz, &zsiz);
if (zbuf) {
vbuf = zbuf;
vsiz = zsiz;
}
}
if (tran_) { if (tran_) {
TranLog log(kbuf, ksiz, dbuf + rksiz, rec->vsiz); TranLog log(kbuf, ksiz, dbuf + rksiz, rec->vsiz);
slot->trlogs.push_back(log); slot->trlogs.push_back(log);
} else { } else {
adj = vsiz > rec->vsiz; adj = vsiz > rec->vsiz;
} }
slot->size -= rec->vsiz; slot->size -= rec->vsiz;
slot->size += vsiz; slot->size += vsiz;
if (vsiz > rec->vsiz) { if (vsiz > rec->vsiz) {
Record* old = rec; Record* old = rec;
skipping to change at line 871 skipping to change at line 1463
if (slot->first == old) slot->first = rec; if (slot->first == old) slot->first = rec;
if (slot->last == old) slot->last = rec; if (slot->last == old) slot->last = rec;
*entp = rec; *entp = rec;
if (rec->prev) rec->prev->next = rec; if (rec->prev) rec->prev->next = rec;
if (rec->next) rec->next->prev = rec; if (rec->next) rec->next->prev = rec;
dbuf = (char*)rec + sizeof(*rec); dbuf = (char*)rec + sizeof(*rec);
} }
} }
std::memcpy(dbuf + ksiz, vbuf, vsiz); std::memcpy(dbuf + ksiz, vbuf, vsiz);
rec->vsiz = vsiz; rec->vsiz = vsiz;
delete[] zbuf;
} }
if (!isiter && slot->last != rec) { if (!isiter && slot->last != rec) {
if (!curs_.empty()) escape_cursors(rec); if (!curs_.empty()) escape_cursors(rec);
if (slot->first == rec) slot->first = rec->next; if (slot->first == rec) slot->first = rec->next;
if (rec->prev) rec->prev->next = rec->next; if (rec->prev) rec->prev->next = rec->next;
if (rec->next) rec->next->prev = rec->prev; if (rec->next) rec->next->prev = rec->prev;
rec->prev = slot->last; rec->prev = slot->last;
rec->next = NULL; rec->next = NULL;
slot->last->next = rec; slot->last->next = rec;
slot->last = rec; slot->last = rec;
} }
if (adj) adjust_slot_capacity(slot); if (adj) adjust_slot_capacity(slot);
} }
return; return;
} }
} }
} }
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz);
if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) {
char* zbuf = NULL;
size_t zsiz = 0;
if (comp) {
zbuf = comp->compress(vbuf, vsiz, &zsiz);
if (zbuf) {
vbuf = zbuf;
vsiz = zsiz;
}
}
if (tran_) { if (tran_) {
TranLog log(kbuf, ksiz); TranLog log(kbuf, ksiz);
slot->trlogs.push_back(log); slot->trlogs.push_back(log);
} }
slot->size += sizeof(Record) + ksiz + vsiz; slot->size += sizeof(Record) + ksiz + vsiz;
rec = (Record*)xmalloc(sizeof(*rec) + ksiz + vsiz); rec = (Record*)xmalloc(sizeof(*rec) + ksiz + vsiz);
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
std::memcpy(dbuf, kbuf, ksiz); std::memcpy(dbuf, kbuf, ksiz);
rec->ksiz = ksiz | fhash; rec->ksiz = ksiz | fhash;
std::memcpy(dbuf + ksiz, vbuf, vsiz); std::memcpy(dbuf + ksiz, vbuf, vsiz);
skipping to change at line 912 skipping to change at line 1514
rec->left = NULL; rec->left = NULL;
rec->right = NULL; rec->right = NULL;
rec->prev = slot->last; rec->prev = slot->last;
rec->next = NULL; rec->next = NULL;
*entp = rec; *entp = rec;
if (!slot->first) slot->first = rec; if (!slot->first) slot->first = rec;
if (slot->last) slot->last->next = rec; if (slot->last) slot->last->next = rec;
slot->last = rec; slot->last = rec;
slot->count++; slot->count++;
if (!tran_) adjust_slot_capacity(slot); if (!tran_) adjust_slot_capacity(slot);
delete[] zbuf;
} }
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
*/ */
int64_t count_impl() { int64_t count_impl() {
_assert_(true);
int64_t sum = 0; int64_t sum = 0;
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < CDBSLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
ScopedSpinLock lock(&slot->lock); ScopedSpinLock lock(&slot->lock);
sum += slot->count; sum += slot->count;
} }
return sum; return sum;
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes. * @return the size of the database file in bytes.
*/ */
int64_t size_impl() { int64_t size_impl() {
_assert_(true);
int64_t sum = sizeof(*this); int64_t sum = sizeof(*this);
for (int32_t i = 0; i < CDBSLOTNUM; i++) { for (int32_t i = 0; i < CDBSLOTNUM; i++) {
Slot* slot = slots_ + i; Slot* slot = slots_ + i;
ScopedSpinLock lock(&slot->lock); ScopedSpinLock lock(&slot->lock);
sum += slot->bnum * sizeof(Record*); sum += slot->bnum * sizeof(Record*);
sum += slot->size; sum += slot->size;
} }
return sum; return sum;
} }
/** /**
* Initialize a slot table. * Initialize a slot table.
* @param slot the slot table. * @param slot the slot table.
* @param bnum the number of buckets. * @param bnum the number of buckets.
* @param capcnt the capacity of record number. * @param capcnt the capacity of record number.
* @param capsiz the capacity of memory usage. * @param capsiz the capacity of memory usage.
*/ */
void initialize_slot(Slot* slot, size_t bnum, size_t capcnt, size_t capsi z) { void initialize_slot(Slot* slot, size_t bnum, size_t capcnt, size_t capsi z) {
_assert_(slot);
Record** buckets; Record** buckets;
if (bnum >= CDBZMAPBNUM) { if (bnum >= CDBZMAPBNUM) {
buckets = (Record**)mapalloc(sizeof(*buckets) * bnum); buckets = (Record**)mapalloc(sizeof(*buckets) * bnum);
} else { } else {
buckets = new Record*[bnum]; buckets = new Record*[bnum];
for (size_t i = 0; i < bnum; i++) { for (size_t i = 0; i < bnum; i++) {
buckets[i] = NULL; buckets[i] = NULL;
} }
} }
slot->buckets = buckets; slot->buckets = buckets;
skipping to change at line 972 skipping to change at line 1578
slot->first = NULL; slot->first = NULL;
slot->last = NULL; slot->last = NULL;
slot->count = 0; slot->count = 0;
slot->size = 0; slot->size = 0;
} }
/** /**
* Destroy a slot table. * Destroy a slot table.
* @param slot the slot table. * @param slot the slot table.
*/ */
void destroy_slot(Slot* slot) { void destroy_slot(Slot* slot) {
_assert_(slot);
slot->trlogs.clear(); slot->trlogs.clear();
Record* rec = slot->last; Record* rec = slot->last;
while (rec) { while (rec) {
Record* prev = rec->prev; Record* prev = rec->prev;
xfree(rec); xfree(rec);
rec = prev; rec = prev;
} }
if (slot->bnum >= CDBZMAPBNUM) { if (slot->bnum >= CDBZMAPBNUM) {
mapfree(slot->buckets); mapfree(slot->buckets);
} else { } else {
delete[] slot->buckets; delete[] slot->buckets;
} }
} }
/** /**
* Clear a slot table. * Clear a slot table.
* @param slot the slot table. * @param slot the slot table.
*/ */
void clear_slot(Slot* slot) { void clear_slot(Slot* slot) {
_assert_(slot);
Record* rec = slot->last; Record* rec = slot->last;
while (rec) { while (rec) {
if (tran_) {
uint32_t rksiz = rec->ksiz & CDBKSIZMAX;
char* dbuf = (char*)rec + sizeof(*rec);
TranLog log(dbuf, rksiz, dbuf + rksiz, rec->vsiz);
slot->trlogs.push_back(log);
}
Record* prev = rec->prev; Record* prev = rec->prev;
xfree(rec); xfree(rec);
rec = prev; rec = prev;
} }
Record** buckets = slot->buckets; Record** buckets = slot->buckets;
size_t bnum = slot->bnum; size_t bnum = slot->bnum;
for (size_t i = 0; i < bnum; i++) { for (size_t i = 0; i < bnum; i++) {
buckets[i] = NULL; buckets[i] = NULL;
} }
slot->first = NULL; slot->first = NULL;
slot->last = NULL; slot->last = NULL;
slot->count = 0; slot->count = 0;
slot->size = 0; slot->size = 0;
} }
/** /**
* Apply transaction logs of a slot table. * Apply transaction logs of a slot table.
* @param slot the slot table. * @param slot the slot table.
*/ */
void apply_slot_trlogs(Slot* slot) { void apply_slot_trlogs(Slot* slot) {
_assert_(slot);
const TranLogList& logs = slot->trlogs; const TranLogList& logs = slot->trlogs;
TranLogList::const_iterator it = logs.end(); TranLogList::const_iterator it = logs.end();
TranLogList::const_iterator itbeg = logs.begin(); TranLogList::const_iterator itbeg = logs.begin();
while (it != itbeg) { while (it != itbeg) {
it--; it--;
const char* kbuf = it->key.c_str(); const char* kbuf = it->key.c_str();
size_t ksiz = it->key.size(); size_t ksiz = it->key.size();
const char* vbuf = it->value.c_str(); const char* vbuf = it->value.c_str();
size_t vsiz = it->value.size(); size_t vsiz = it->value.size();
uint64_t hash = hash_record(kbuf, ksiz) / CDBSLOTNUM; uint64_t hash = hash_record(kbuf, ksiz) / CDBSLOTNUM;
if (it->full) { if (it->full) {
Setter setter(vbuf, vsiz); Setter setter(vbuf, vsiz);
accept_impl(slot, hash, kbuf, ksiz, &setter, true); accept_impl(slot, hash, kbuf, ksiz, &setter, NULL, true);
} else { } else {
Remover remover; Remover remover;
accept_impl(slot, hash, kbuf, ksiz, &remover, true); accept_impl(slot, hash, kbuf, ksiz, &remover, NULL, true);
} }
} }
} }
/** /**
* Addjust a slot table to the capacity. * Addjust a slot table to the capacity.
* @param slot the slot table. * @param slot the slot table.
*/ */
void adjust_slot_capacity(Slot* slot) { void adjust_slot_capacity(Slot* slot) {
_assert_(slot);
if ((slot->count > slot->capcnt || slot->size > slot->capsiz) && slot-> first) { if ((slot->count > slot->capcnt || slot->size > slot->capsiz) && slot-> first) {
Record* rec = slot->first; Record* rec = slot->first;
uint32_t rksiz = rec->ksiz & CDBKSIZMAX; uint32_t rksiz = rec->ksiz & CDBKSIZMAX;
char* dbuf = (char*)rec + sizeof(*rec); char* dbuf = (char*)rec + sizeof(*rec);
char stack[CDBRECBUFSIZ]; char stack[CDBRECBUFSIZ];
char* kbuf = rksiz > sizeof(stack) ? new char[rksiz] : stack; char* kbuf = rksiz > sizeof(stack) ? new char[rksiz] : stack;
std::memcpy(kbuf, dbuf, rksiz); std::memcpy(kbuf, dbuf, rksiz);
uint64_t hash = hash_record(kbuf, rksiz) / CDBSLOTNUM; uint64_t hash = hash_record(kbuf, rksiz) / CDBSLOTNUM;
Remover remover; Remover remover;
accept_impl(slot, hash, dbuf, rksiz, &remover, true); accept_impl(slot, hash, dbuf, rksiz, &remover, NULL, true);
if (kbuf != stack) delete[] kbuf; if (kbuf != stack) delete[] kbuf;
} }
} }
/** /**
* Get the hash value of a record. * Get the hash value of a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return the hash value. * @return the hash value.
*/ */
uint64_t hash_record(const char* kbuf, size_t ksiz) { uint64_t hash_record(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ);
return hashmurmur(kbuf, ksiz); return hashmurmur(kbuf, ksiz);
} }
/** /**
* Fold a hash value into a small number. * Fold a hash value into a small number.
* @param hash the hash number. * @param hash the hash number.
* @return the result number. * @return the result number.
*/ */
uint32_t fold_hash(uint64_t hash) { uint32_t fold_hash(uint64_t hash) {
_assert_(true);
return ((hash & 0xffffffff00000000ULL) >> 32) ^ ((hash & 0x0000ffffffff 0000ULL) >> 16) ^ return ((hash & 0xffffffff00000000ULL) >> 32) ^ ((hash & 0x0000ffffffff 0000ULL) >> 16) ^
((hash & 0x000000000000ffffULL) << 16) ^ ((hash & 0x00000000ffff0000U LL) >> 0); ((hash & 0x000000000000ffffULL) << 16) ^ ((hash & 0x00000000ffff0000U LL) >> 0);
} }
/** /**
* Compare two keys in lexical order. * Compare two keys in lexical order.
* @param abuf one key. * @param abuf one key.
* @param asiz the size of the one key. * @param asiz the size of the one key.
* @param bbuf the other key. * @param bbuf the other key.
* @param bsiz the size of the other key. * @param bsiz the size of the other key.
* @return positive if the former is big, or negative if the latter is bi g, or 0 if both are * @return positive if the former is big, or negative if the latter is bi g, or 0 if both are
* equivalent. * equivalent.
*/ */
int32_t compare_keys(const char* abuf, size_t asiz, const char* bbuf, siz e_t bsiz) { int32_t compare_keys(const char* abuf, size_t asiz, const char* bbuf, siz e_t bsiz) {
_assert_(abuf && asiz <= MEMMAXSIZ && bbuf && bsiz <= MEMMAXSIZ);
if (asiz != bsiz) return (int32_t)asiz - (int32_t)bsiz; if (asiz != bsiz) return (int32_t)asiz - (int32_t)bsiz;
return std::memcmp(abuf, bbuf, asiz); return std::memcmp(abuf, bbuf, asiz);
} }
/** /**
* Escape cursors on a shifted or removed records. * Escape cursors on a shifted or removed records.
* @param rec the record. * @param rec the record.
*/ */
void escape_cursors(Record* rec) { void escape_cursors(Record* rec) {
_assert_(rec);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
if (curs_.empty()) return; if (curs_.empty()) return;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->rec_ == rec) cur->step_impl(); if (cur->rec_ == rec) cur->step_impl();
cit++; cit++;
} }
} }
/** /**
* Adjust cursors on re-allocated records. * Adjust cursors on re-allocated records.
* @param orec the old address. * @param orec the old address.
* @param nrec the new address. * @param nrec the new address.
*/ */
void adjust_cursors(Record* orec, Record* nrec) { void adjust_cursors(Record* orec, Record* nrec) {
_assert_(orec && nrec);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
if (curs_.empty()) return; if (curs_.empty()) return;
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
if (cur->rec_ == orec) cur->rec_ = nrec; if (cur->rec_ == orec) cur->rec_ = nrec;
cit++; cit++;
} }
} }
/** /**
* Disable all cursors. * Disable all cursors.
*/ */
void disable_cursors() { void disable_cursors() {
_assert_(true);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
CursorList::const_iterator cit = curs_.begin(); CursorList::const_iterator cit = curs_.begin();
CursorList::const_iterator citend = curs_.end(); CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->sidx_ = -1; cur->sidx_ = -1;
cur->rec_ = NULL; cur->rec_ = NULL;
cit++; cit++;
} }
} }
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
CacheDB(const CacheDB&); CacheDB(const CacheDB&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
CacheDB& operator =(const CacheDB&); CacheDB& operator =(const CacheDB&);
/** The method lock. */ /** The method lock. */
SpinRWLock mlock_; SpinRWLock mlock_;
/** The file lock. */ /** The file lock. */
SpinLock flock_; SpinLock flock_;
/** The last happened error. */ /** The last happened error. */
TSD<Error> error_; TSD<Error> error_;
/** The internal logger. */
Logger* logger_;
/** The kinds of logged messages. */
uint32_t logkinds_;
/** The internal meta operation trigger. */
MetaTrigger* mtrigger_;
/** The open mode. */ /** The open mode. */
uint32_t omode_; uint32_t omode_;
/** The cursor objects. */ /** The cursor objects. */
CursorList curs_; CursorList curs_;
/** The path of the database file. */ /** The path of the database file. */
std::string path_; std::string path_;
/** The database type. */
uint8_t type_;
/** The options. */
uint8_t opts_;
/** The bucket number. */ /** The bucket number. */
int64_t bnum_; int64_t bnum_;
/** The capacity of record number. */ /** The capacity of record number. */
int64_t capcnt_; int64_t capcnt_;
/** The capacity of memory usage. */ /** The capacity of memory usage. */
int64_t capsiz_; int64_t capsiz_;
/** The opaque data. */
char opaque_[CDBOPAQUESIZ];
/** The embedded data compressor. */
Compressor* embcomp_;
/** The data compressor. */
Compressor* comp_;
/** The slot tables. */ /** The slot tables. */
Slot slots_[CDBSLOTNUM]; Slot slots_[CDBSLOTNUM];
/** The flag whether in transaction. */ /** The flag whether in transaction. */
bool tran_; bool tran_;
}; };
/** An alias of the cache tree database. */
typedef PlantDB<CacheDB, BasicDB::TYPEGRASS> GrassDB;
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 130 change blocks. 
88 lines changed or deleted 756 lines changed or added


 kccommon.h   kccommon.h 
skipping to change at line 60 skipping to change at line 60
#include <memory> #include <memory>
#include <iterator> #include <iterator>
#include <algorithm> #include <algorithm>
#include <locale> #include <locale>
#include <string> #include <string>
#include <vector> #include <vector>
#include <list> #include <list>
#include <map> #include <map>
#include <set> #include <set>
#include <queue>
#include <ios> #include <ios>
#include <iostream> #include <iostream>
#include <streambuf> #include <streambuf>
#include <fstream> #include <fstream>
#include <sstream> #include <sstream>
#if defined(_MSC_VER) #if defined(_MSC_VER)
#define snprintf _snprintf #define snprintf _snprintf
#endif #endif
#if defined(__CYGWIN__)
inline long double modfl(long double val, long double* iptr) {
double integ;
double fract = std::modf(val, &integ);
*iptr = integ;
return fract;
}
#endif
namespace std { namespace std {
using ::modfl; using ::modfl;
using ::snprintf; using ::snprintf;
} }
#if __cplusplus > 199711L || defined(__GXX_EXPERIMENTAL_CXX0X__) || defined (_MSC_VER) #if __cplusplus > 199711L || defined(__GXX_EXPERIMENTAL_CXX0X__) || defined (_MSC_VER)
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
skipping to change at line 110 skipping to change at line 120
#undef PAGESIZE #undef PAGESIZE
#if defined(_KCUYIELD) #if defined(_KCUYIELD)
#if defined(_MSC_VER) #if defined(_MSC_VER)
#include <windows.h> #include <windows.h>
#define _yield_() ::Sleep(0) #define _yield_() ::Sleep(0)
#else #else
#include <sched.h> #include <sched.h>
#define _yield_() ::sched_yield() #define _yield_() ::sched_yield()
#endif #endif
#define _testyield_() \ #define _testyield_() \
do { \ do { \
static uint32_t _KC_seed = 725; \ static uint32_t _KC_seed = 725; \
_KC_seed = _KC_seed * 123456761 + 211; \ _KC_seed = _KC_seed * 123456761 + 211; \
if (_KC_seed % 0x100 == 0) _yield_(); \ if (_KC_seed % 0x100 == 0) _yield_(); \
} while(false) } while(false)
#define _assert_(KC_a) \ #define _assert_(KC_a) \
do { \ do { \
_testyield_(); \ _testyield_(); \
assert(KC_a); \ assert(KC_a); \
} while(false) } while(false)
#elif defined(_KCDEBUG) #elif defined(_KCDEBUG)
#define _yield_() #define _yield_()
#define _testyield_() #define _testyield_()
#define _assert_(KC_a) assert(KC_a) #define _assert_(KC_a) assert(KC_a)
#else #else
#define _yield_() ///< for debugging #define _yield_() ///< for debugging
#define _testyield_() ///< for debugging #define _testyield_() ///< for debugging
#define _assert_(KC_a) ///< for debugging #define _assert_(KC_a) ///< for debugging
#endif #endif
#if defined(__GNUC__)
#define __KCFUNC__ __func__ ///< for debugging
#elif defined(_MSC_VER)
#define __KCFUNC__ __FUNCTION__ ///< for debugging
#else
#define __KCFUNC__ "-" ///< for debugging
#endif
#define _KCCODELINE_ __FILE__, __LINE__, __KCFUNC__ ///< for debugging
/** /**
* All symbols of Kyoto Cabinet. * All symbols of Kyoto Cabinet.
*/ */
namespace kyotocabinet {} namespace kyotocabinet {}
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 5 change blocks. 
9 lines changed or deleted 28 lines changed or added


 kccompress.h   kccompress.h 
skipping to change at line 60 skipping to change at line 60
* @return the pointer to the result data, or NULL on failure. * @return the pointer to the result data, or NULL on failure.
* @note Because an additional zero code is appended at the end of the re gion of the return * @note Because an additional zero code is appended at the end of the re gion of the return
* value, the return value can be treated as a C-style string. Because t he region of the * value, the return value can be treated as a C-style string. Because t he region of the
* return value is allocated with the the new[] operator, it should be re leased with the * return value is allocated with the the new[] operator, it should be re leased with the
* delete[] operator when it is no longer in use. * delete[] operator when it is no longer in use.
*/ */
virtual char* decompress(const void* buf, size_t size, size_t* sp) = 0; virtual char* decompress(const void* buf, size_t size, size_t* sp) = 0;
}; };
/** /**
* Zlib compressor. * ZLIB compressor.
*/ */
class Zlib { class ZLIB {
public: public:
/** /**
* Compression modes. * Compression modes.
*/ */
enum Mode { enum Mode {
RAW, ///< without any checksum RAW, ///< without any checksum
DEFLATE, ///< with Adler32 checksum DEFLATE, ///< with Adler32 checksum
GZIP ///< with CRC32 checksum and vario us meta data GZIP ///< with CRC32 checksum and vario us meta data
}; };
/** /**
skipping to change at line 109 skipping to change at line 109
* Calculate the CRC32 checksum of a serial data. * Calculate the CRC32 checksum of a serial data.
* @param buf the input buffer. * @param buf the input buffer.
* @param size the size of the input buffer. * @param size the size of the input buffer.
* @param seed the cyclic seed value. * @param seed the cyclic seed value.
* @return the CRC32 checksum. * @return the CRC32 checksum.
*/ */
static uint32_t calculate_crc(const void* buf, size_t size, uint32_t seed = 0); static uint32_t calculate_crc(const void* buf, size_t size, uint32_t seed = 0);
}; };
/** /**
* Compressor with the Zlib. * LZO compressor.
*/ */
template <Zlib::Mode MODE> class LZO {
class ZlibCompressor : public Compressor { public:
/**
* Compression modes.
*/
enum Mode {
RAW, ///< without any checksum
CRC ///< with CRC32 checksum
};
/**
* Compress a serial data.
* @param buf the input buffer.
* @param size the size of the input buffer.
* @param sp the pointer to the variable into which the size of the regio
n of the return
* value is assigned.
* @param mode the compression mode.
* @return the pointer to the result data, or NULL on failure.
* @note Because the region of the return value is allocated with the the
new[] operator, it
* should be released with the delete[] operator when it is no longer in
use.
*/
static char* compress(const void* buf, size_t size, size_t* sp, Mode mode
= RAW);
/**
* Decompress a serial data.
* @param buf the input buffer.
* @param size the size of the input buffer.
* @param sp the pointer to the variable into which the size of the regio
n of the return
* value is assigned.
* @param mode the compression mode.
* @return the pointer to the result data, or NULL on failure.
* @note Because an additional zero code is appended at the end of the re
gion of the return
* value, the return value can be treated as a C-style string. Because t
he region of the
* return value is allocated with the the new[] operator, it should be re
leased with the
* delete[] operator when it is no longer in use.
*/
static char* decompress(const void* buf, size_t size, size_t* sp, Mode mo
de = RAW);
/**
* Calculate the CRC32 checksum of a serial data.
* @param buf the input buffer.
* @param size the size of the input buffer.
* @param seed the cyclic seed value.
* @return the CRC32 checksum.
*/
static uint32_t calculate_crc(const void* buf, size_t size, uint32_t seed
= 0);
};
/**
* LZMA compressor.
*/
class LZMA {
public:
/**
* Compression modes.
*/
enum Mode {
RAW, ///< without any checksum
CRC, ///< with CRC32 checksum
SHA ///< with SHA256 checksum
};
/**
* Compress a serial data.
* @param buf the input buffer.
* @param size the size of the input buffer.
* @param sp the pointer to the variable into which the size of the regio
n of the return
* value is assigned.
* @param mode the compression mode.
* @return the pointer to the result data, or NULL on failure.
* @note Because the region of the return value is allocated with the the
new[] operator, it
* should be released with the delete[] operator when it is no longer in
use.
*/
static char* compress(const void* buf, size_t size, size_t* sp, Mode mode
= RAW);
/**
* Decompress a serial data.
* @param buf the input buffer.
* @param size the size of the input buffer.
* @param sp the pointer to the variable into which the size of the regio
n of the return
* value is assigned.
* @param mode the compression mode.
* @return the pointer to the result data, or NULL on failure.
* @note Because an additional zero code is appended at the end of the re
gion of the return
* value, the return value can be treated as a C-style string. Because t
he region of the
* return value is allocated with the the new[] operator, it should be re
leased with the
* delete[] operator when it is no longer in use.
*/
static char* decompress(const void* buf, size_t size, size_t* sp, Mode mo
de = RAW);
/**
* Calculate the CRC32 checksum of a serial data.
* @param buf the input buffer.
* @param size the size of the input buffer.
* @param seed the cyclic seed value.
* @return the CRC32 checksum.
*/
static uint32_t calculate_crc(const void* buf, size_t size, uint32_t seed
= 0);
};
/**
* Compressor with ZLIB.
*/
template <ZLIB::Mode MODE>
class ZLIBCompressor : public Compressor {
private: private:
/** /**
* Compress a serial data. * Compress a serial data.
*/ */
char* compress(const void* buf, size_t size, size_t* sp) { char* compress(const void* buf, size_t size, size_t* sp) {
_assert_(buf && size <= MEMMAXSIZ && sp); _assert_(buf && size <= MEMMAXSIZ && sp);
return Zlib::compress(buf, size, sp, MODE); return ZLIB::compress(buf, size, sp, MODE);
} }
/** /**
* Decompress a serial data. * Decompress a serial data.
*/ */
char* decompress(const void* buf, size_t size, size_t* sp) { char* decompress(const void* buf, size_t size, size_t* sp) {
_assert_(buf && size <= MEMMAXSIZ && sp); _assert_(buf && size <= MEMMAXSIZ && sp);
return Zlib::decompress(buf, size, sp, MODE); return ZLIB::decompress(buf, size, sp, MODE);
}
};
/**
* Compressor with LZO.
*/
template <LZO::Mode MODE>
class LZOCompressor : public Compressor {
private:
/**
* Compress a serial data.
*/
char* compress(const void* buf, size_t size, size_t* sp) {
_assert_(buf && size <= MEMMAXSIZ && sp);
return LZO::compress(buf, size, sp, MODE);
}
/**
* Decompress a serial data.
*/
char* decompress(const void* buf, size_t size, size_t* sp) {
_assert_(buf && size <= MEMMAXSIZ && sp);
return LZO::decompress(buf, size, sp, MODE);
}
};
/**
* Compressor with LZMA.
*/
template <LZMA::Mode MODE>
class LZMACompressor : public Compressor {
private:
/**
* Compress a serial data.
*/
char* compress(const void* buf, size_t size, size_t* sp) {
_assert_(buf && size <= MEMMAXSIZ && sp);
return LZMA::compress(buf, size, sp, MODE);
}
/**
* Decompress a serial data.
*/
char* decompress(const void* buf, size_t size, size_t* sp) {
_assert_(buf && size <= MEMMAXSIZ && sp);
return LZMA::decompress(buf, size, sp, MODE);
} }
}; };
/** /**
* Compressor with the Arcfour cipher. * Compressor with the Arcfour cipher.
*/ */
class ArcfourCompressor : public Compressor { class ArcfourCompressor : public Compressor {
public: public:
/** /**
* Constructor. * Constructor.
*/ */
ArcfourCompressor() : kbuf_(NULL), ksiz_(0), salt_(0), cycle_(false) { ArcfourCompressor() : kbuf_(NULL), ksiz_(0), comp_(NULL), salt_(0), cycle _(false) {
_assert_(true); _assert_(true);
kbuf_ = new char[0]; kbuf_ = new char[1];
ksiz_ = 0; ksiz_ = 0;
} }
/** /**
* Destructor. * Destructor.
*/ */
~ArcfourCompressor() { ~ArcfourCompressor() {
_assert_(true); _assert_(true);
delete[] kbuf_; delete[] kbuf_;
} }
/** /**
skipping to change at line 164 skipping to change at line 305
*/ */
void set_key(const void* kbuf, size_t ksiz) { void set_key(const void* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
delete[] kbuf_; delete[] kbuf_;
if (ksiz > NUMBUFSIZ) ksiz = NUMBUFSIZ; if (ksiz > NUMBUFSIZ) ksiz = NUMBUFSIZ;
kbuf_ = new char[ksiz]; kbuf_ = new char[ksiz];
std::memcpy(kbuf_, kbuf, ksiz); std::memcpy(kbuf_, kbuf, ksiz);
ksiz_ = ksiz; ksiz_ = ksiz;
} }
/** /**
* Set an additional data compressor.
* @param comp the additional data data compressor.
*/
void set_compressor(Compressor* comp) {
_assert_(comp);
comp_ = comp;
}
/**
* Begin the cycle of ciper salt. * Begin the cycle of ciper salt.
* @param salt the additional cipher salt. * @param salt the additional cipher salt.
*/ */
void begin_cycle(uint64_t salt = 0) { void begin_cycle(uint64_t salt = 0) {
salt_ = salt; salt_ = salt;
cycle_ = true; cycle_ = true;
} }
private: private:
/** /**
* Compress a serial data. * Compress a serial data.
*/ */
char* compress(const void* buf, size_t size, size_t* sp) { char* compress(const void* buf, size_t size, size_t* sp) {
_assert_(buf && size <= MEMMAXSIZ && sp); _assert_(buf && size <= MEMMAXSIZ && sp);
uint64_t salt = cycle_ ? salt_.add(1) : 0; uint64_t salt = cycle_ ? salt_.add(1) : 0;
char kbuf[NUMBUFSIZ*2]; char kbuf[NUMBUFSIZ*2];
writefixnum(kbuf, salt, sizeof(salt)); writefixnum(kbuf, salt, sizeof(salt));
std::memcpy(kbuf + sizeof(salt), kbuf_, ksiz_); std::memcpy(kbuf + sizeof(salt), kbuf_, ksiz_);
char* zbuf = new char[NUMBUFSIZ+size]; char* tbuf = NULL;
char* wp = zbuf; if (comp_) {
writefixnum(wp, salt, sizeof(salt)); tbuf = comp_->compress(buf, size, &size);
wp += sizeof(salt); if (!tbuf) return NULL;
arccipher(buf, size, kbuf, sizeof(salt) + ksiz_, wp); buf = tbuf;
}
size_t zsiz = sizeof(salt) + size;
char* zbuf = new char[zsiz];
writefixnum(zbuf, salt, sizeof(salt));
arccipher(buf, size, kbuf, sizeof(salt) + ksiz_, zbuf + sizeof(salt));
delete[] tbuf;
if (cycle_) { if (cycle_) {
size_t range = size < sizeof(uint64_t) ? size : sizeof(uint64_t); size_t range = zsiz - sizeof(salt);
salt_.add(hashmurmur(wp, range) << 32); if (range > INT8_MAX) range = INT8_MAX;
salt_.add(hashmurmur(zbuf + sizeof(salt), range) << 32);
} }
*sp = sizeof(salt) + size; *sp = zsiz;
return zbuf; return zbuf;
} }
/** /**
* Decompress a serial data. * Decompress a serial data.
*/ */
char* decompress(const void* buf, size_t size, size_t* sp) { char* decompress(const void* buf, size_t size, size_t* sp) {
_assert_(buf && size <= MEMMAXSIZ && sp); _assert_(buf && size <= MEMMAXSIZ && sp);
if (size < sizeof(uint64_t)) return NULL; if (size < sizeof(uint64_t)) return NULL;
char kbuf[NUMBUFSIZ*2]; char kbuf[NUMBUFSIZ*2];
std::memcpy(kbuf, buf, sizeof(uint64_t)); std::memcpy(kbuf, buf, sizeof(uint64_t));
std::memcpy(kbuf + sizeof(uint64_t), kbuf_, ksiz_); std::memcpy(kbuf + sizeof(uint64_t), kbuf_, ksiz_);
buf = (char*)buf + sizeof(uint64_t); buf = (char*)buf + sizeof(uint64_t);
size -= sizeof(uint64_t); size -= sizeof(uint64_t);
char* zbuf = new char[size]; char* zbuf = new char[size];
arccipher(buf, size, kbuf, sizeof(uint64_t) + ksiz_, zbuf); arccipher(buf, size, kbuf, sizeof(uint64_t) + ksiz_, zbuf);
if (comp_) {
char* tbuf = comp_->decompress(zbuf, size, &size);
delete[] zbuf;
if (!tbuf) return NULL;
zbuf = tbuf;
}
*sp = size; *sp = size;
return zbuf; return zbuf;
} }
/** The pointer to the key. */ /** The pointer to the key. */
char* kbuf_; char* kbuf_;
/** The size of the key. */ /** The size of the key. */
size_t ksiz_; size_t ksiz_;
/** The data compressor. */
Compressor* comp_;
/** The cipher salt. */ /** The cipher salt. */
AtomicInt64 salt_; AtomicInt64 salt_;
/** The flag of the salt cycle */ /** The flag of the salt cycle */
bool cycle_; bool cycle_;
}; };
/** /**
* Prepared variable of the compressor with the Zlib raw mode. * Prepared variable of the compressor with ZLIB raw mode.
*/ */
extern ZlibCompressor<Zlib::RAW> ZLIBRAWCOMP; extern ZLIBCompressor<ZLIB::RAW> ZLIBRAWCOMP;
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 16 change blocks. 
19 lines changed or deleted 203 lines changed or added


 kcdb.h   kcdb.h 
skipping to change at line 20 skipping to change at line 20
* See the GNU General Public License for more details. * See the GNU General Public License for more details.
* You should have received a copy of the GNU General Public License along with this program. * You should have received a copy of the GNU General Public License along with this program.
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
************************************************************************** ***********************/ ************************************************************************** ***********************/
#ifndef _KCDB_H // duplication check #ifndef _KCDB_H // duplication check
#define _KCDB_H #define _KCDB_H
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
#include <kcthread.h>
#include <kcfile.h>
#include <kccompress.h>
#include <kccompare.h>
#include <kcmap.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation. * Constants for implementation.
*/ */
namespace { namespace {
const char DBSSMAGICDATA[] = "KCSS\n"; ///< magic data of the file const char DBSSMAGICDATA[] = "KCSS\n"; ///< magic data of the file
const size_t DBIOBUFSIZ = 8192; ///< size of the IO buffer const size_t DBIOBUFSIZ = 8192; ///< size of the IO buffer
} }
skipping to change at line 99 skipping to change at line 104
*/ */
virtual ~Cursor() { virtual ~Cursor() {
_assert_(true); _assert_(true);
} }
/** /**
* Accept a visitor to the current record. * Accept a visitor to the current record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only operation. * @param writable true for writable operation, or false for read-only operation.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and othe
r threads accessing
* the same record are blocked. To avoid deadlock, any database operat
ion must not be
* performed in this function.
*/ */
virtual bool accept(Visitor* visitor, bool writable = true, bool step = false) = 0; virtual bool accept(Visitor* visitor, bool writable = true, bool step = false) = 0;
/** /**
* Set the value of the current record. * Set the value of the current record.
* @param vbuf the pointer to the value region. * @param vbuf the pointer to the value region.
* @param vsiz the size of the value region. * @param vsiz the size of the value region.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
virtual bool set_value(const char* vbuf, size_t vsiz, bool step = false ) = 0; virtual bool set_value(const char* vbuf, size_t vsiz, bool step = false ) = 0;
/** /**
* Set the value of the current record. * Set the value of the current record.
* @note Equal to the original Cursor::set_value method except that the parameter is * @note Equal to the original Cursor::set_value method except that the parameter is
* std::string. * std::string.
*/ */
virtual bool set_value(const std::string& value, bool step = false) = 0 ; virtual bool set_value_str(const std::string& value, bool step = false) = 0;
/** /**
* Remove the current record. * Remove the current record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note If no record corresponds to the key, false is returned. The c ursor is moved to the * @note If no record corresponds to the key, false is returned. The c ursor is moved to the
* next record implicitly. * next record implicitly.
*/ */
virtual bool remove() = 0; virtual bool remove() = 0;
/** /**
* Get the key of the current record. * Get the key of the current record.
* @param sp the pointer to the variable into which the size of the reg ion of the return * @param sp the pointer to the variable into which the size of the reg ion of the return
skipping to change at line 171 skipping to change at line 179
virtual std::string* get_value(bool step = false) = 0; virtual std::string* get_value(bool step = false) = 0;
/** /**
* Get a pair of the key and the value of the current record. * Get a pair of the key and the value of the current record.
* @param ksp the pointer to the variable into which the size of the re gion of the return * @param ksp the pointer to the variable into which the size of the re gion of the return
* value is assigned. * value is assigned.
* @param vbp the pointer to the variable into which the pointer to the value region is * @param vbp the pointer to the variable into which the pointer to the value region is
* assigned. * assigned.
* @param vsp the pointer to the variable into which the size of the va lue region is * @param vsp the pointer to the variable into which the size of the va lue region is
* assigned. * assigned.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return the pointer to the pair of the key region, or NULL on failur e. * @return the pointer to the key region, or NULL on failure.
* @note If the cursor is invalidated, NULL is returned. Because an ad ditional zero code is * @note If the cursor is invalidated, NULL is returned. Because an ad ditional zero code is
* appended at the end of each region of the key and the value, each re gion can be treated * appended at the end of each region of the key and the value, each re gion can be treated
* as a C-style string. The return value should be deleted explicitly by the caller with * as a C-style string. The return value should be deleted explicitly by the caller with
* the detele[] operator. * the detele[] operator.
*/ */
virtual char* get(size_t* ksp, const char** vbp, size_t* vsp, bool step = false) = 0; virtual char* get(size_t* ksp, const char** vbp, size_t* vsp, bool step = false) = 0;
/** /**
* Get a pair of the key and the value of the current record. * Get a pair of the key and the value of the current record.
* @param step true to move the cursor to the next record, or false for no move.
* @return the pointer to the pair of the key and the value, or NULL on failure. * @return the pointer to the pair of the key and the value, or NULL on failure.
* @note Equal to the original Cursor::get method except that the retur
n value is std::pair.
* The return value should be deleted explicitly by the caller.
* @note If the cursor is invalidated, NULL is returned. The return va lue should be deleted * @note If the cursor is invalidated, NULL is returned. The return va lue should be deleted
* explicitly by the caller. * explicitly by the caller.
*/ */
virtual std::pair<std::string, std::string>* get_pair(bool step = false ) = 0; virtual std::pair<std::string, std::string>* get_pair(bool step = false ) = 0;
/** /**
* Jump the cursor to the first record. * Jump the cursor to the first record for forward scan.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
virtual bool jump() = 0; virtual bool jump() = 0;
/** /**
* Jump the cursor to a record. * Jump the cursor to a record for forward scan.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
virtual bool jump(const char* kbuf, size_t ksiz) = 0; virtual bool jump(const char* kbuf, size_t ksiz) = 0;
/** /**
* Jump the cursor to a record. * Jump the cursor to a record for forward scan.
* @note Equal to the original Cursor::set method except that the param * @note Equal to the original Cursor::jump method except that the para
eter is std::string. meter is std::string.
*/ */
virtual bool jump(const std::string& key) = 0; virtual bool jump(const std::string& key) = 0;
/** /**
* Jump the cursor to the last record for backward scan.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database typ
es, especially hash
* databases, will provide a dummy implementation.
*/
virtual bool jump_back() = 0;
/**
* Jump the cursor to a record for backward scan.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database typ
es, especially hash
* databases, will provide a dummy implementation.
*/
virtual bool jump_back(const char* kbuf, size_t ksiz) = 0;
/**
* Jump the cursor to a record for backward scan.
* @note Equal to the original Cursor::jump_back method except that the
parameter is
* std::string.
*/
virtual bool jump_back(const std::string& key) = 0;
/**
* Step the cursor to the next record. * Step the cursor to the next record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
virtual bool step() = 0; virtual bool step() = 0;
/** /**
* Step the cursor to the previous record.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database typ
es, especially hash
* databases, will provide a dummy implementation.
*/
virtual bool step_back() = 0;
/**
* Get the database object. * Get the database object.
* @return the database object. * @return the database object.
*/ */
virtual DB* db() = 0; virtual DB* db() = 0;
}; };
/** /**
* Destructor. * Destructor.
*/ */
virtual ~DB() { virtual ~DB() {
_assert_(true); _assert_(true);
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The operation for each record is performed atomically and other threads accessing the * @note The operation for each record is performed atomically and other threads accessing the
* same record are blocked. * same record are blocked. To avoid deadlock, any database operation mu
st not be performed in
* this function.
*/ */
virtual bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) = 0; virtual bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writable = true) = 0;
/** /**
* Iterate to accept a visitor for each record.
* @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op
eration.
* @return true on success, or false on failure.
* @note The whole iteration is performed atomically and other threads ar
e blocked.
*/
virtual bool iterate(Visitor *visitor, bool writable = true) = 0;
/**
* Set the value of a record. * Set the value of a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param vbuf the pointer to the value region. * @param vbuf the pointer to the value region.
* @param vsiz the size of the value region. * @param vsiz the size of the value region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note If no record corresponds to the key, a new record is created. I f the corresponding * @note If no record corresponds to the key, a new record is created. I f the corresponding
* record exists, the value is overwritten. * record exists, the value is overwritten.
*/ */
virtual bool set(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0; virtual bool set(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0;
skipping to change at line 273 skipping to change at line 302
* @note If no record corresponds to the key, a new record is created. I f the corresponding * @note If no record corresponds to the key, a new record is created. I f the corresponding
* record exists, the record is not modified and false is returned. * record exists, the record is not modified and false is returned.
*/ */
virtual bool add(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0; virtual bool add(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) = 0;
/** /**
* Set the value of a record. * Set the value of a record.
* @note Equal to the original DB::add method except that the parameters are std::string. * @note Equal to the original DB::add method except that the parameters are std::string.
*/ */
virtual bool add(const std::string& key, const std::string& value) = 0; virtual bool add(const std::string& key, const std::string& value) = 0;
/** /**
* Replace the value of a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @param vbuf the pointer to the value region.
* @param vsiz the size of the value region.
* @return true on success, or false on failure.
* @note If no record corresponds to the key, no new record is created an
d false is returned.
* If the corresponding record exists, the value is modified.
*/
virtual bool replace(const char* kbuf, size_t ksiz, const char* vbuf, siz
e_t vsiz) = 0;
/**
* Replace the value of a record.
* @note Equal to the original DB::replace method except that the paramet
ers are std::string.
*/
virtual bool replace(const std::string& key, const std::string& value) =
0;
/**
* Append the value of a record. * Append the value of a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param vbuf the pointer to the value region. * @param vbuf the pointer to the value region.
* @param vsiz the size of the value region. * @param vsiz the size of the value region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note If no record corresponds to the key, a new record is created. I f the corresponding * @note If no record corresponds to the key, a new record is created. I f the corresponding
* record exists, the given value is appended at the end of the existing value. * record exists, the given value is appended at the end of the existing value.
*/ */
virtual bool append(const char* kbuf, size_t ksiz, const char* vbuf, size _t vsiz) = 0; virtual bool append(const char* kbuf, size_t ksiz, const char* vbuf, size _t vsiz) = 0;
/** /**
* Set the value of a record. * Set the value of a record.
* @note Equal to the original DB::append method except that the paramete rs are std::string. * @note Equal to the original DB::append method except that the paramete rs are std::string.
*/ */
virtual bool append(const std::string& key, const std::string& value) = 0 ; virtual bool append(const std::string& key, const std::string& value) = 0 ;
/** /**
* Add a number to the numeric value of a record. * Add a number to the numeric integer value of a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param num the additional number. * @param num the additional number.
* @return the result value, or INT64_MIN on failure. * @return the result value, or INT64_MIN on failure.
*/ */
virtual int64_t increment(const char* kbuf, size_t ksiz, int64_t num) = 0 ; virtual int64_t increment(const char* kbuf, size_t ksiz, int64_t num) = 0 ;
/** /**
* Add a number to the numeric value of a record. * Add a number to the numeric integer value of a record.
* @note Equal to the original DB::increment method except that the param eter is std::string. * @note Equal to the original DB::increment method except that the param eter is std::string.
*/ */
virtual int64_t increment(const std::string& key, int64_t num) = 0; virtual int64_t increment(const std::string& key, int64_t num) = 0;
/** /**
* Add a number to the numeric value of a record. * Add a number to the numeric double value of a record.
* @note Equal to the original DB::increment method except that the param * @param kbuf the pointer to the key region.
eter and the return * @param ksiz the size of the key region.
* value are double. Not-a-number is returned on failure. * @param num the additional number.
* @return the result value, or Not-a-number on failure.
*/ */
virtual double increment(const char* kbuf, size_t ksiz, double num) = 0; virtual double increment_double(const char* kbuf, size_t ksiz, double num ) = 0;
/** /**
* Add a number to the numeric value of a record. * Add a number to the numeric double value of a record.
* @note Equal to the original DB::increment method except that the param * @note Equal to the original DB::increment_double method except that th
eter is std::string e parameter is
* and the return value is double. Not-a-number is returned on failure. * std::string.
*/ */
virtual double increment(const std::string& key, double num) = 0; virtual double increment_double(const std::string& key, double num) = 0;
/** /**
* Perform compare-and-swap. * Perform compare-and-swap.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param ovbuf the pointer to the old value region. NULL means that no record corresponds. * @param ovbuf the pointer to the old value region. NULL means that no record corresponds.
* @param ovsiz the size of the old value region. * @param ovsiz the size of the old value region.
* @param nvbuf the pointer to the new value region. NULL means that the record is removed. * @param nvbuf the pointer to the new value region. NULL means that the record is removed.
* @param nvsiz the size of new old value region. * @param nvsiz the size of new old value region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
skipping to change at line 394 skipping to change at line 441
/** /**
* Create a cursor object. * Create a cursor object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note Because the object of the return value is allocated by the const ructor, it should be * @note Because the object of the return value is allocated by the const ructor, it should be
* released with the delete operator when it is no longer in use. * released with the delete operator when it is no longer in use.
*/ */
virtual Cursor* cursor() = 0; virtual Cursor* cursor() = 0;
}; };
/** /**
* Basic implementation for file database. * Basic implementation of database.
* @note This class is an abstract class to prescribe the interface of file operations and * @note This class is an abstract class to prescribe the interface of file operations and
* provide mix-in methods. This class can be inherited but overwriting met hods is forbidden. * provide mix-in methods. This class can be inherited but overwriting met hods is forbidden.
* Before every database operation, it is necessary to call the FileDB::ope n method in order to * Before every database operation, it is necessary to call the BasicDB::op en method in order to
* open a database file and connect the database object to it. To avoid da ta missing or * open a database file and connect the database object to it. To avoid da ta missing or
* corruption, it is important to close every database file by the FileDB:: close method when the * corruption, it is important to close every database file by the BasicDB: :close method when the
* database is no longer in use. It is forbidden for multible database obj ects in a process to * database is no longer in use. It is forbidden for multible database obj ects in a process to
* open the same database at the same time. * open the same database at the same time. It is forbidden to share a dat
abase object with
* child processes.
*/ */
class FileDB : public DB { class BasicDB : public DB {
public: public:
class Error;
class Cursor; class Cursor;
class Error;
class ProgressChecker;
class FileProcessor;
class Logger;
class MetaTrigger;
public: public:
/** /**
* Database types. * Database types.
*/ */
enum Type { enum Type {
TYPEVOID = 0x00, ///< void database TYPEVOID = 0x00, ///< void database
TYPEPHASH = 0x01, ///< prototype hash database TYPEPHASH = 0x10, ///< prototype hash database
TYPEPTREE = 0x02, ///< prototype tree database TYPEPTREE = 0x11, ///< prototype tree database
TYPEPMISC = 0x08, ///< miscellaneous prototype datab TYPESTASH = 0x18, ///< stash database
ase TYPECACHE = 0x20, ///< cache hash database
TYPECACHE = 0x09, ///< cache database TYPEGRASS = 0x21, ///< cache tree database
TYPEHASH = 0x11, ///< file hash database TYPEHASH = 0x30, ///< file hash database
TYPETREE = 0x12, ///< file tree database TYPETREE = 0x31, ///< file tree database
TYPEDIR = 0x18, ///< directory database TYPEDIR = 0x40, ///< directory hash database
TYPEMISC = 0x20 ///< miscellaneous database TYPEFOREST = 0x41, ///< directory tree database
TYPEMISC = 0x80 ///< miscellaneous database
}; };
/** /**
* Interface of cursor to indicate a record. * Interface of cursor to indicate a record.
*/ */
class Cursor : public DB::Cursor { class Cursor : public DB::Cursor {
public: public:
/** /**
* Destructor. * Destructor.
*/ */
virtual ~Cursor() { virtual ~Cursor() {
skipping to change at line 470 skipping to change at line 524
VisitorImpl visitor(vbuf, vsiz); VisitorImpl visitor(vbuf, vsiz);
if (!accept(&visitor, true, step)) return false; if (!accept(&visitor, true, step)) return false;
if (!visitor.ok()) return false; if (!visitor.ok()) return false;
return true; return true;
} }
/** /**
* Set the value of the current record. * Set the value of the current record.
* @note Equal to the original Cursor::set_value method except that the parameter is * @note Equal to the original Cursor::set_value method except that the parameter is
* std::string. * std::string.
*/ */
bool set_value(const std::string& value, bool step = false) { bool set_value_str(const std::string& value, bool step = false) {
_assert_(true); _assert_(true);
return set_value(value.c_str(), value.size(), step); return set_value(value.c_str(), value.size(), step);
} }
/** /**
* Remove the current record. * Remove the current record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note If no record corresponds to the key, false is returned. The c ursor is moved to the * @note If no record corresponds to the key, false is returned. The c ursor is moved to the
* next record implicitly. * next record implicitly.
*/ */
bool remove() { bool remove() {
skipping to change at line 640 skipping to change at line 694
} }
/** /**
* Get a pair of the key and the value of the current record. * Get a pair of the key and the value of the current record.
* @param ksp the pointer to the variable into which the size of the re gion of the return * @param ksp the pointer to the variable into which the size of the re gion of the return
* value is assigned. * value is assigned.
* @param vbp the pointer to the variable into which the pointer to the value region is * @param vbp the pointer to the variable into which the pointer to the value region is
* assigned. * assigned.
* @param vsp the pointer to the variable into which the size of the va lue region is * @param vsp the pointer to the variable into which the size of the va lue region is
* assigned. * assigned.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return the pointer to the pair of the key region, or NULL on failur e. * @return the pointer to the key region, or NULL on failure.
* @note If the cursor is invalidated, NULL is returned. Because an ad ditional zero code is * @note If the cursor is invalidated, NULL is returned. Because an ad ditional zero code is
* appended at the end of each region of the key and the value, each re gion can be treated * appended at the end of each region of the key and the value, each re gion can be treated
* as a C-style string. The return value should be deleted explicitly by the caller with * as a C-style string. The return value should be deleted explicitly by the caller with
* the detele[] operator. * the detele[] operator.
*/ */
char* get(size_t* ksp, const char** vbp, size_t* vsp, bool step = false ) { char* get(size_t* ksp, const char** vbp, size_t* vsp, bool step = false ) {
_assert_(ksp && vbp && vsp); _assert_(ksp && vbp && vsp);
class VisitorImpl : public Visitor { class VisitorImpl : public Visitor {
public: public:
explicit VisitorImpl() : kbuf_(NULL), ksiz_(0), vbuf_(NULL), vsiz_( 0) {} explicit VisitorImpl() : kbuf_(NULL), ksiz_(0), vbuf_(NULL), vsiz_( 0) {}
skipping to change at line 722 skipping to change at line 776
Record* rec_; Record* rec_;
}; };
VisitorImpl visitor; VisitorImpl visitor;
if (!accept(&visitor, false, step)) return NULL; if (!accept(&visitor, false, step)) return NULL;
return visitor.pop(); return visitor.pop();
} }
/** /**
* Get the database object. * Get the database object.
* @return the database object. * @return the database object.
*/ */
virtual FileDB* db() = 0; virtual BasicDB* db() = 0;
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
Error error() { Error error() {
_assert_(true); _assert_(true);
return db()->error(); return db()->error();
} }
}; };
/** /**
skipping to change at line 744 skipping to change at line 798
*/ */
class Error { class Error {
public: public:
/** /**
* Error codes. * Error codes.
*/ */
enum Code { enum Code {
SUCCESS, ///< success SUCCESS, ///< success
NOIMPL, ///< not implemented NOIMPL, ///< not implemented
INVALID, ///< invalid operation INVALID, ///< invalid operation
NOFILE, ///< file not found NOREPOS, ///< no repository
NOPERM, ///< no permission NOPERM, ///< no permission
BROKEN, ///< broken file BROKEN, ///< broken file
DUPREC, ///< record duplication DUPREC, ///< record duplication
NOREC, ///< no record NOREC, ///< no record
LOGIC, ///< logical inconsistency LOGIC, ///< logical inconsistency
SYSTEM, ///< system error SYSTEM, ///< system error
MISC = 15 ///< miscellaneous error MISC = 15 ///< miscellaneous error
}; };
/** /**
* Default constructor. * Default constructor.
skipping to change at line 786 skipping to change at line 840
*/ */
~Error() { ~Error() {
_assert_(true); _assert_(true);
} }
/** /**
* Set the error information. * Set the error information.
* @param code an error code. * @param code an error code.
* @param message a supplement message. * @param message a supplement message.
*/ */
void set(Code code, const char* message) { void set(Code code, const char* message) {
_assert_(message);
code_ = code; code_ = code;
message_ = message; message_ = message;
} }
/** /**
* Get the error code. * Get the error code.
* @return the error code. * @return the error code.
*/ */
Code code() const { Code code() const {
_assert_(true);
return code_; return code_;
} }
/** /**
* Get the readable string of the code. * Get the readable string of the code.
* @return the readable string of the code. * @return the readable string of the code.
*/ */
const char* name() const { const char* name() const {
_assert_(true);
return codename(code_); return codename(code_);
} }
/** /**
* Get the supplement message. * Get the supplement message.
* @return the supplement message. * @return the supplement message.
*/ */
const char* message() const { const char* message() const {
_assert_(true);
return message_; return message_;
} }
/** /**
* Get the readable string of an error code. * Get the readable string of an error code.
* @param code the error code. * @param code the error code.
* @return the readable string of the error code. * @return the readable string of the error code.
*/ */
static const char* codename(Code code) { static const char* codename(Code code) {
_assert_(true);
switch (code) { switch (code) {
case SUCCESS: return "success"; case SUCCESS: return "success";
case NOIMPL: return "not implemented"; case NOIMPL: return "not implemented";
case INVALID: return "invalid operation"; case INVALID: return "invalid operation";
case NOFILE: return "file not found"; case NOREPOS: return "no repository";
case NOPERM: return "no permission"; case NOPERM: return "no permission";
case BROKEN: return "broken file"; case BROKEN: return "broken file";
case DUPREC: return "record duplication"; case DUPREC: return "record duplication";
case NOREC: return "no record"; case NOREC: return "no record";
case LOGIC: return "logical inconsistency"; case LOGIC: return "logical inconsistency";
case SYSTEM: return "system error"; case SYSTEM: return "system error";
default: break; default: break;
} }
return "miscellaneous error"; return "miscellaneous error";
} }
skipping to change at line 847 skipping to change at line 906
_assert_(true); _assert_(true);
if (&right == this) return *this; if (&right == this) return *this;
code_ = right.code_; code_ = right.code_;
message_ = right.message_; message_ = right.message_;
return *this; return *this;
} }
/** /**
* Cast operator to integer. * Cast operator to integer.
* @return the error code. * @return the error code.
*/ */
operator int32_t() { operator int32_t() const {
return code_; return code_;
} }
private: private:
/** The error code. */ /** The error code. */
Code code_; Code code_;
/** The supplement message. */ /** The supplement message. */
const char* message_; const char* message_;
}; };
/** /**
* Interface to check progress status of long process.
*/
class ProgressChecker {
public:
/**
* Destructor.
*/
virtual ~ProgressChecker() {
_assert_(true);
}
/**
* Check the progress status.
* @param name the name of the process.
* @param message a supplement message.
* @param curcnt the count of the current step of the progress, or -1 i
f not applicable.
* @param allcnt the estimation count of all steps of the progress, or
-1 if not applicable.
* @return true to continue the process, or false to stop the process.
*/
virtual bool check(const char* name, const char* message,
int64_t curcnt, int64_t allcnt) = 0;
};
/**
* Interface to process the database file. * Interface to process the database file.
*/ */
class FileProcessor { class FileProcessor {
public: public:
/** /**
* Destructor. * Destructor.
*/ */
virtual ~FileProcessor() { virtual ~FileProcessor() {
_assert_(true); _assert_(true);
} }
/** /**
* Process the database file. * Process the database file.
* @param path the path of the database file. * @param path the path of the database file.
* @param count the number of records. * @param count the number of records.
* @param size the size of the available region. * @param size the size of the available region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
virtual bool process(const std::string& path, int64_t count, int64_t si ze) = 0; virtual bool process(const std::string& path, int64_t count, int64_t si ze) = 0;
}; };
/** /**
* Interface to log internal information and errors.
*/
class Logger {
public:
/**
* Event kinds.
*/
enum Kind {
DEBUG = 1 << 0, ///< debugging
INFO = 1 << 1, ///< normal information
WARN = 1 << 2, ///< warning
ERROR = 1 << 3 ///< error
};
/**
* Destructor.
*/
virtual ~Logger() {
_assert_(true);
}
/**
* Process a log message.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Log
ger::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal e
rror.
* @param message the supplement message.
*/
virtual void log(const char* file, int32_t line, const char* func, Kind
kind,
const char* message) = 0;
};
/**
* Interface to trigger meta database operations.
*/
class MetaTrigger {
public:
/**
* Event kinds.
*/
enum Kind {
OPEN, ///< opening
CLOSE, ///< closing
CLEAR, ///< clearing
ITERATE, ///< iteration
SYNCHRONIZE, ///< synchronization
BEGINTRAN, ///< beginning transaction
COMMITTRAN, ///< committing transaction
ABORTTRAN, ///< aborting transaction
MISC = 15 ///< miscellaneous operation
};
/**
* Destructor.
*/
virtual ~MetaTrigger() {
_assert_(true);
}
/**
* Trigger a meta database operation.
* @param kind the kind of the event. MetaTrigger::OPEN for opening, M
etaTrigger::CLOSE for
* closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for i
teration,
* MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN
for beginning
* transaction, MetaTrigger::COMMITTRAN for committing transaction, Met
aTrigger::ABORTTRAN
* for aborting transaction, and MetaTrigger::MISC for miscellaneous op
erations.
* @param message the supplement message.
*/
virtual void trigger(Kind kind, const char* message) = 0;
};
/**
* Open modes. * Open modes.
*/ */
enum OpenMode { enum OpenMode {
OREADER = 1 << 0, ///< open as a reader OREADER = 1 << 0, ///< open as a reader
OWRITER = 1 << 1, ///< open as a writer OWRITER = 1 << 1, ///< open as a writer
OCREATE = 1 << 2, ///< writer creating OCREATE = 1 << 2, ///< writer creating
OTRUNCATE = 1 << 3, ///< writer truncating OTRUNCATE = 1 << 3, ///< writer truncating
OAUTOTRAN = 1 << 4, ///< auto transaction OAUTOTRAN = 1 << 4, ///< auto transaction
OAUTOSYNC = 1 << 5, ///< auto synchronization OAUTOSYNC = 1 << 5, ///< auto synchronization
ONOLOCK = 1 << 6, ///< open without locking ONOLOCK = 1 << 6, ///< open without locking
OTRYLOCK = 1 << 7, ///< lock without blocking OTRYLOCK = 1 << 7, ///< lock without blocking
ONOREPAIR = 1 << 8 ///< open without auto repair ONOREPAIR = 1 << 8 ///< open without auto repair
}; };
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~FileDB() { virtual ~BasicDB() {
_assert_(true); _assert_(true);
} }
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
virtual Error error() const = 0; virtual Error error() const = 0;
/** /**
* Set the error information. * Set the error information.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param code an error code. * @param code an error code.
* @param message a supplement message. * @param message a supplement message.
*/ */
virtual void set_error(Error::Code code, const char* message) = 0; virtual void set_error(const char* file, int32_t line, const char* func,
Error::Code code, const char* message) = 0;
/** /**
* Open a database file. * Open a database file.
* @param path the path of a database file. * @param path the path of a database file.
* @param mode the connection mode. FileDB::OWRITER as a writer, FileDB: * @param mode the connection mode. BasicDB::OWRITER as a writer, BasicD
:OREADER as a reader. B::OREADER as a
* The following may be added to the writer mode by bitwise-or: FileDB::O * reader. The following may be added to the writer mode by bitwise-or:
CREATE, which means BasicDB::OCREATE,
* it creates a new database if the file does not exist, FileDB::OTRUNCAT * which means it creates a new database if the file does not exist, Basi
E, which means it cDB::OTRUNCATE, which
* creates a new database regardless if the file exists, FileDB::OAUTOTRA * means it creates a new database regardless if the file exists, BasicDB
N, which means each ::OAUTOTRAN, which
* updating operation is performed in implicit transaction, FileDB::OAUTO * means each updating operation is performed in implicit transaction, Ba
SYNC, which means sicDB::OAUTOSYNC,
* each updating operation is followed by implicit synchronization with t * which means each updating operation is followed by implicit synchroniz
he file system. The ation with the file
* following may be added to both of the reader mode and the writer mode * system. The following may be added to both of the reader mode and the
by bitwise-or: writer mode by
* FileDB::ONOLOCK, which means it opens the database file without file l * bitwise-or: BasicDB::ONOLOCK, which means it opens the database file w
ocking, ithout file locking,
* FileDB::OTRYLOCK, which means locking is performed without blocking, F * BasicDB::OTRYLOCK, which means locking is performed without blocking,
ile::ONOREPAIR, which File::ONOREPAIR, which
* means the database file is not repaired implicitly even if file destru ction is detected. * means the database file is not repaired implicitly even if file destru ction is detected.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note Every opened database must be closed by the FileDB::close method when it is no longer * @note Every opened database must be closed by the BasicDB::close metho d when it is no longer
* in use. It is not allowed for two or more database objects in the sam e process to keep * in use. It is not allowed for two or more database objects in the sam e process to keep
* their connections to the same database file at the same time. * their connections to the same database file at the same time.
*/ */
virtual bool open(const std::string& path, uint32_t mode = OWRITER | OCRE ATE) = 0; virtual bool open(const std::string& path, uint32_t mode = OWRITER | OCRE ATE) = 0;
/** /**
* Close the database file. * Close the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
virtual bool close() = 0; virtual bool close() = 0;
/** /**
* Iterate to accept a visitor for each record.
* @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op
eration.
* @param checker a progress checker object. If it is NULL, no checking
is performed.
* @return true on success, or false on failure.
* @note The whole iteration is performed atomically and other threads ar
e blocked. To avoid
* deadlock, any database operation must not be performed in this functio
n.
*/
virtual bool iterate(Visitor *visitor, bool writable = true,
ProgressChecker* checker = NULL) = 0;
/**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @param proc a postprocessor object. If it is NULL, no postprocessing is performed. * @param proc a postprocessor object. If it is NULL, no postprocessing is performed.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
virtual bool synchronize(bool hard = false, FileProcessor* proc = NULL) = virtual bool synchronize(bool hard = false, FileProcessor* proc = NULL,
0; ProgressChecker* checker = NULL) = 0;
/** /**
* Create a copy of the database file. * Create a copy of the database file.
* @param dest the path of the destination file. * @param dest the path of the destination file.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool copy(const std::string& dest) { bool copy(const std::string& dest, ProgressChecker* checker = NULL) {
_assert_(true); _assert_(true);
class FileProcessorImpl : public FileProcessor { class FileProcessorImpl : public FileProcessor {
public: public:
FileProcessorImpl(const std::string& dest) : dest_(dest) {} explicit FileProcessorImpl(const std::string& dest, ProgressChecker*
checker,
BasicDB* db) :
dest_(dest), checker_(checker), db_(db) {}
private: private:
bool process(const std::string& path, int64_t count, int64_t size) { bool process(const std::string& path, int64_t count, int64_t size) {
File::Status sbuf;
if (!File::status(path, &sbuf)) return false;
if (sbuf.isdir) {
if (!File::make_directory(dest_)) return false;
bool err = false;
DirStream dir;
if (dir.open(path)) {
if (checker_ && !checker_->check("copy", "beginning", 0, -1)) {
db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
std::string name;
int64_t curcnt = 0;
while (!err && dir.read(&name)) {
const std::string& spath = path + File::PATHCHR + name;
const std::string& dpath = dest_ + File::PATHCHR + name;
int64_t dsiz;
char* dbuf = File::read_file(spath, &dsiz);
if (dbuf) {
if (!File::write_file(dpath, dbuf, dsiz)) err = true;
delete[] dbuf;
} else {
err = true;
}
curcnt++;
if (checker_ && !checker_->check("copy", "processing", curcnt
, -1)) {
db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed"
);
err = true;
break;
}
}
if (checker_ && !checker_->check("copy", "ending", -1, -1)) {
db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
if (!dir.close()) err = true;
} else {
err = true;
}
return !err;
}
std::ofstream ofs; std::ofstream ofs;
ofs.open(dest_.c_str(), ofs.open(dest_.c_str(),
std::ios_base::out | std::ios_base::binary | std::ios_base ::trunc); std::ios_base::out | std::ios_base::binary | std::ios_base ::trunc);
if (!ofs) return false; if (!ofs) return false;
bool err = false; bool err = false;
std::ifstream ifs; std::ifstream ifs;
ifs.open(path.c_str(), std::ios_base::in | std::ios_base::binary); ifs.open(path.c_str(), std::ios_base::in | std::ios_base::binary);
if (checker_ && !checker_->check("copy", "beginning", 0, size)) {
db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
if (ifs) { if (ifs) {
char buf[DBIOBUFSIZ]; char buf[DBIOBUFSIZ];
while (!ifs.eof()) { int64_t curcnt = 0;
while (!err && !ifs.eof()) {
size_t n = ifs.read(buf, sizeof(buf)).gcount(); size_t n = ifs.read(buf, sizeof(buf)).gcount();
if (n > 0) { if (n > 0) {
ofs.write(buf, n); ofs.write(buf, n);
if (!ofs) { if (!ofs) {
err = true; err = true;
break; break;
} }
} }
curcnt += n;
if (checker_ && !checker_->check("copy", "processing", curcnt,
size)) {
db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
break;
}
} }
ifs.close(); ifs.close();
if (ifs.bad()) err = true; if (ifs.bad()) err = true;
} else { } else {
err = true; err = true;
} }
if (checker_ && !checker_->check("copy", "ending", -1, size)) {
db_->set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
ofs.close(); ofs.close();
if (!ofs) err = true; if (!ofs) err = true;
return !err; return !err;
} }
const std::string& dest_; const std::string& dest_;
ProgressChecker* checker_;
BasicDB* db_;
}; };
FileProcessorImpl proc(dest); FileProcessorImpl proc(dest, checker, this);
return synchronize(false, &proc); return synchronize(false, &proc, checker);
} }
/** /**
* Begin transaction. * Begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
virtual bool begin_transaction(bool hard = false) = 0; virtual bool begin_transaction(bool hard = false) = 0;
/** /**
* Try to begin transaction. * Try to begin transaction.
skipping to change at line 1093 skipping to change at line 1320
*sp = vsiz_; *sp = vsiz_;
return vbuf_; return vbuf_;
} }
const char* vbuf_; const char* vbuf_;
size_t vsiz_; size_t vsiz_;
bool ok_; bool ok_;
}; };
VisitorImpl visitor(vbuf, vsiz); VisitorImpl visitor(vbuf, vsiz);
if (!accept(kbuf, ksiz, &visitor, true)) return false; if (!accept(kbuf, ksiz, &visitor, true)) return false;
if (!visitor.ok()) { if (!visitor.ok()) {
set_error(Error::DUPREC, "record duplication"); set_error(_KCCODELINE_, Error::DUPREC, "record duplication");
return false; return false;
} }
return true; return true;
} }
/** /**
* Set the value of a record. * Set the value of a record.
* @note Equal to the original DB::add method except that the parameters are std::string. * @note Equal to the original DB::add method except that the parameters are std::string.
*/ */
bool add(const std::string& key, const std::string& value) { bool add(const std::string& key, const std::string& value) {
_assert_(true); _assert_(true);
return add(key.c_str(), key.size(), value.c_str(), value.size()); return add(key.c_str(), key.size(), value.c_str(), value.size());
} }
/** /**
* Replace the value of a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @param vbuf the pointer to the value region.
* @param vsiz the size of the value region.
* @return true on success, or false on failure.
* @note If no record corresponds to the key, no new record is created an
d false is returned.
* If the corresponding record exists, the value is modified.
*/
bool replace(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz
) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ);
class VisitorImpl : public Visitor {
public:
explicit VisitorImpl(const char* vbuf, size_t vsiz) :
vbuf_(vbuf), vsiz_(vsiz), ok_(false) {}
bool ok() const {
return ok_;
}
private:
const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) {
ok_ = true;
*sp = vsiz_;
return vbuf_;
}
const char* vbuf_;
size_t vsiz_;
bool ok_;
};
VisitorImpl visitor(vbuf, vsiz);
if (!accept(kbuf, ksiz, &visitor, true)) return false;
if (!visitor.ok()) {
set_error(_KCCODELINE_, Error::NOREC, "no record");
return false;
}
return true;
}
/**
* Replace the value of a record.
* @note Equal to the original DB::replace method except that the paramet
ers are std::string.
*/
bool replace(const std::string& key, const std::string& value) {
_assert_(true);
return replace(key.c_str(), key.size(), value.c_str(), value.size());
}
/**
* Append the value of a record. * Append the value of a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param vbuf the pointer to the value region. * @param vbuf the pointer to the value region.
* @param vsiz the size of the value region. * @param vsiz the size of the value region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note If no record corresponds to the key, a new record is created. I f the corresponding * @note If no record corresponds to the key, a new record is created. I f the corresponding
* record exists, the given value is appended at the end of the existing value. * record exists, the given value is appended at the end of the existing value.
*/ */
bool append(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) { bool append(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) {
skipping to change at line 1201 skipping to change at line 1474
*sp = sizeof(big_); *sp = sizeof(big_);
return (const char*)&big_; return (const char*)&big_;
} }
int64_t num_; int64_t num_;
uint64_t big_; uint64_t big_;
}; };
VisitorImpl visitor(num); VisitorImpl visitor(num);
if (!accept(kbuf, ksiz, &visitor, true)) return INT64_MIN; if (!accept(kbuf, ksiz, &visitor, true)) return INT64_MIN;
num = visitor.num(); num = visitor.num();
if (num == INT64_MIN) { if (num == INT64_MIN) {
set_error(Error::LOGIC, "logical inconsistency"); set_error(_KCCODELINE_, Error::LOGIC, "logical inconsistency");
return num; return num;
} }
return num; return num;
} }
/** /**
* Add a number to the numeric value of a record. * Add a number to the numeric value of a record.
* @note Equal to the original DB::increment method except that the param eter is std::string. * @note Equal to the original DB::increment method except that the param eter is std::string.
*/ */
int64_t increment(const std::string& key, int64_t num) { int64_t increment(const std::string& key, int64_t num) {
_assert_(true); _assert_(true);
return increment(key.c_str(), key.size(), num); return increment(key.c_str(), key.size(), num);
} }
/** /**
* Add a number to the numeric value of a record. * Add a number to the numeric double value of a record.
* @note Equal to the original DB::increment method except that the param * @param kbuf the pointer to the key region.
eter and the return * @param ksiz the size of the key region.
* value are double. * @param num the additional number.
* @return the result value, or Not-a-number on failure.
*/ */
double increment(const char* kbuf, size_t ksiz, double num) { double increment_double(const char* kbuf, size_t ksiz, double num) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
class VisitorImpl : public Visitor { class VisitorImpl : public Visitor {
public: public:
explicit VisitorImpl(double num) : DECUNIT(1000000000000000LL), num_( num), buf_() {} explicit VisitorImpl(double num) : DECUNIT(1000000000000000LL), num_( num), buf_() {}
double num() { double num() {
return num_; return num_;
} }
private: private:
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
skipping to change at line 1308 skipping to change at line 1583
return buf_; return buf_;
} }
const int64_t DECUNIT; const int64_t DECUNIT;
double num_; double num_;
char buf_[sizeof(int64_t)*2]; char buf_[sizeof(int64_t)*2];
}; };
VisitorImpl visitor(num); VisitorImpl visitor(num);
if (!accept(kbuf, ksiz, &visitor, true)) return nan(); if (!accept(kbuf, ksiz, &visitor, true)) return nan();
num = visitor.num(); num = visitor.num();
if (chknan(num)) { if (chknan(num)) {
set_error(Error::LOGIC, "logical inconsistency"); set_error(_KCCODELINE_, Error::LOGIC, "logical inconsistency");
return nan(); return nan();
} }
return num; return num;
} }
/** /**
* Add a number to the numeric value of a record. * Add a number to the numeric double value of a record.
* @note Equal to the original DB::increment method except that the param * @note Equal to the original DB::increment_double method except that th
eter is std::string e parameter is
* and the return value is double. * std::string.
*/ */
double increment(const std::string& key, double num) { double increment_double(const std::string& key, double num) {
_assert_(true); _assert_(true);
return increment(key.c_str(), key.size(), num); return increment_double(key.c_str(), key.size(), num);
} }
/** /**
* Perform compare-and-swap. * Perform compare-and-swap.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param ovbuf the pointer to the old value region. NULL means that no record corresponds. * @param ovbuf the pointer to the old value region. NULL means that no record corresponds.
* @param ovsiz the size of the old value region. * @param ovsiz the size of the old value region.
* @param nvbuf the pointer to the new value region. NULL means that the record is removed. * @param nvbuf the pointer to the new value region. NULL means that the record is removed.
* @param nvsiz the size of new old value region. * @param nvsiz the size of new old value region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 1367 skipping to change at line 1642
} }
const char* ovbuf_; const char* ovbuf_;
size_t ovsiz_; size_t ovsiz_;
const char* nvbuf_; const char* nvbuf_;
size_t nvsiz_; size_t nvsiz_;
bool ok_; bool ok_;
}; };
VisitorImpl visitor(ovbuf, ovsiz, nvbuf, nvsiz); VisitorImpl visitor(ovbuf, ovsiz, nvbuf, nvsiz);
if (!accept(kbuf, ksiz, &visitor, true)) return false; if (!accept(kbuf, ksiz, &visitor, true)) return false;
if (!visitor.ok()) { if (!visitor.ok()) {
set_error(Error::LOGIC, "status conflict"); set_error(_KCCODELINE_, Error::LOGIC, "status conflict");
return false; return false;
} }
return true; return true;
} }
/** /**
* Perform compare-and-swap. * Perform compare-and-swap.
* @note Equal to the original DB::cas method except that the parameters are std::string. * @note Equal to the original DB::cas method except that the parameters are std::string.
*/ */
bool cas(const std::string& key, bool cas(const std::string& key,
const std::string& ovalue, const std::string& nvalue) { const std::string& ovalue, const std::string& nvalue) {
skipping to change at line 1408 skipping to change at line 1683
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
ok_ = true; ok_ = true;
return REMOVE; return REMOVE;
} }
bool ok_; bool ok_;
}; };
VisitorImpl visitor; VisitorImpl visitor;
if (!accept(kbuf, ksiz, &visitor, true)) return false; if (!accept(kbuf, ksiz, &visitor, true)) return false;
if (!visitor.ok()) { if (!visitor.ok()) {
set_error(Error::NOREC, "no record"); set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
return true; return true;
} }
/** /**
* Remove a record. * Remove a record.
* @note Equal to the original DB::remove method except that the paramete r is std::string. * @note Equal to the original DB::remove method except that the paramete r is std::string.
*/ */
bool remove(const std::string& key) { bool remove(const std::string& key) {
_assert_(true); _assert_(true);
skipping to change at line 1463 skipping to change at line 1738
size_t vsiz_; size_t vsiz_;
}; };
VisitorImpl visitor; VisitorImpl visitor;
if (!accept(kbuf, ksiz, &visitor, false)) { if (!accept(kbuf, ksiz, &visitor, false)) {
*sp = 0; *sp = 0;
return NULL; return NULL;
} }
size_t vsiz; size_t vsiz;
char* vbuf = visitor.pop(&vsiz); char* vbuf = visitor.pop(&vsiz);
if (!vbuf) { if (!vbuf) {
set_error(Error::NOREC, "no record"); set_error(_KCCODELINE_, Error::NOREC, "no record");
*sp = 0; *sp = 0;
return NULL; return NULL;
} }
*sp = vsiz; *sp = vsiz;
return vbuf; return vbuf;
} }
/** /**
* Retrieve the value of a record. * Retrieve the value of a record.
* @note Equal to the original DB::get method except that the parameter a nd the return value * @note Equal to the original DB::get method except that the parameter a nd the return value
* are std::string. The return value should be deleted explicitly by the caller. * are std::string. The return value should be deleted explicitly by the caller.
skipping to change at line 1517 skipping to change at line 1792
return NOP; return NOP;
} }
char* vbuf_; char* vbuf_;
size_t max_; size_t max_;
int32_t vsiz_; int32_t vsiz_;
}; };
VisitorImpl visitor(vbuf, max); VisitorImpl visitor(vbuf, max);
if (!accept(kbuf, ksiz, &visitor, false)) return -1; if (!accept(kbuf, ksiz, &visitor, false)) return -1;
int32_t vsiz = visitor.vsiz(); int32_t vsiz = visitor.vsiz();
if (vsiz < 0) { if (vsiz < 0) {
set_error(Error::NOREC, "no record"); set_error(_KCCODELINE_, Error::NOREC, "no record");
return -1; return -1;
} }
return vsiz; return vsiz;
} }
/** /**
* Dump records into a data stream. * Dump records into a data stream.
* @param dest the destination stream. * @param dest the destination stream.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_snapshot(std::ostream* dest) { bool dump_snapshot(std::ostream* dest, ProgressChecker* checker = NULL) {
_assert_(dest); _assert_(dest);
if (dest->fail()) { if (dest->fail()) {
set_error(Error::INVALID, "invalid stream"); set_error(_KCCODELINE_, Error::INVALID, "invalid stream");
return false; return false;
} }
class VisitorImpl : public Visitor { class VisitorImpl : public Visitor {
public: public:
explicit VisitorImpl(std::ostream* dest) : dest_(dest), stack_() {} explicit VisitorImpl(std::ostream* dest) : dest_(dest), stack_() {}
private: private:
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
char* wp = stack_; char* wp = stack_;
*(wp++) = 0x00; *(wp++) = 0x00;
skipping to change at line 1554 skipping to change at line 1830
dest_->write(kbuf, ksiz); dest_->write(kbuf, ksiz);
dest_->write(vbuf, vsiz); dest_->write(vbuf, vsiz);
return NOP; return NOP;
} }
std::ostream* dest_; std::ostream* dest_;
char stack_[NUMBUFSIZ*2]; char stack_[NUMBUFSIZ*2];
}; };
VisitorImpl visitor(dest); VisitorImpl visitor(dest);
bool err = false; bool err = false;
dest->write(DBSSMAGICDATA, sizeof(DBSSMAGICDATA)); dest->write(DBSSMAGICDATA, sizeof(DBSSMAGICDATA));
if (iterate(&visitor, false)) { if (iterate(&visitor, false, checker)) {
unsigned char c = 0xff; unsigned char c = 0xff;
dest->write((char*)&c, 1); dest->write((char*)&c, 1);
if (dest->fail()) { if (dest->fail()) {
set_error(Error::SYSTEM, "stream output error"); set_error(_KCCODELINE_, Error::SYSTEM, "stream output error");
err = true; err = true;
} }
} else { } else {
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Dump records into a file. * Dump records into a file.
* @param dest the path of the destination file. * @param dest the path of the destination file.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_snapshot(const std::string& dest) { bool dump_snapshot(const std::string& dest, ProgressChecker* checker = NU LL) {
_assert_(true); _assert_(true);
std::ofstream ofs; std::ofstream ofs;
ofs.open(dest.c_str(), std::ios_base::out | std::ios_base::binary | std ::ios_base::trunc); ofs.open(dest.c_str(), std::ios_base::out | std::ios_base::binary | std ::ios_base::trunc);
if (!ofs) { if (!ofs) {
set_error(Error::NOFILE, "open failed"); set_error(_KCCODELINE_, Error::NOREPOS, "open failed");
return false; return false;
} }
bool err = false; bool err = false;
if (!dump_snapshot(&ofs)) err = true; if (!dump_snapshot(&ofs, checker)) err = true;
ofs.close(); ofs.close();
if (!ofs) { if (!ofs) {
set_error(Error::SYSTEM, "close failed"); set_error(_KCCODELINE_, Error::SYSTEM, "close failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Load records from a data stream. * Load records from a data stream.
* @param src the source stream. * @param src the source stream.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_snapshot(std::istream* src) { bool load_snapshot(std::istream* src, ProgressChecker* checker = NULL) {
_assert_(src); _assert_(src);
if (src->fail()) { if (src->fail()) {
set_error(Error::INVALID, "invalid stream"); set_error(_KCCODELINE_, Error::INVALID, "invalid stream");
return false; return false;
} }
char buf[DBIOBUFSIZ]; char buf[DBIOBUFSIZ];
src->read(buf, sizeof(DBSSMAGICDATA)); src->read(buf, sizeof(DBSSMAGICDATA));
if (src->fail()) { if (src->fail()) {
set_error(Error::SYSTEM, "stream input error"); set_error(_KCCODELINE_, Error::SYSTEM, "stream input error");
return false; return false;
} }
if (std::memcmp(buf, DBSSMAGICDATA, sizeof(DBSSMAGICDATA))) { if (std::memcmp(buf, DBSSMAGICDATA, sizeof(DBSSMAGICDATA))) {
set_error(Error::INVALID, "invalid magic data of input stream"); set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of input stream");
return false; return false;
} }
bool err = false; bool err = false;
while (true) { if (checker && !checker->check("load_snapshot", "beginning", 0, -1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
int64_t curcnt = 0;
while (!err) {
int32_t c = src->get(); int32_t c = src->get();
if (src->fail()) { if (src->fail()) {
set_error(Error::SYSTEM, "stream input error"); set_error(_KCCODELINE_, Error::SYSTEM, "stream input error");
err = true; err = true;
break; break;
} }
if (c == 0xff) break; if (c == 0xff) break;
if (c == 0x00) { if (c == 0x00) {
size_t ksiz = 0; size_t ksiz = 0;
do { do {
c = src->get(); c = src->get();
ksiz = (ksiz << 7) + (c & 0x7f); ksiz = (ksiz << 7) + (c & 0x7f);
} while (c >= 0x80); } while (c >= 0x80);
size_t vsiz = 0; size_t vsiz = 0;
do { do {
c = src->get(); c = src->get();
vsiz = (vsiz << 7) + (c & 0x7f); vsiz = (vsiz << 7) + (c & 0x7f);
} while (c >= 0x80); } while (c >= 0x80);
size_t rsiz = ksiz + vsiz; size_t rsiz = ksiz + vsiz;
char* rbuf = rsiz > sizeof(buf) ? new char[rsiz] : buf; char* rbuf = rsiz > sizeof(buf) ? new char[rsiz] : buf;
src->read(rbuf, ksiz + vsiz); src->read(rbuf, ksiz + vsiz);
if (src->fail()) { if (src->fail()) {
set_error(Error::SYSTEM, "stream input error"); set_error(_KCCODELINE_, Error::SYSTEM, "stream input error");
err = true; err = true;
if (rbuf != buf) delete[] rbuf; if (rbuf != buf) delete[] rbuf;
break; break;
} }
if (!set(rbuf, ksiz, rbuf + ksiz, vsiz)) { if (!set(rbuf, ksiz, rbuf + ksiz, vsiz)) {
err = true; err = true;
if (rbuf != buf) delete[] rbuf; if (rbuf != buf) delete[] rbuf;
break; break;
} }
if (rbuf != buf) delete[] rbuf; if (rbuf != buf) delete[] rbuf;
} else { } else {
set_error(Error::INVALID, "invalid magic data of input stream"); set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of inpu
t stream");
err = true;
break;
}
curcnt++;
if (checker && !checker->check("load_snapshot", "processing", curcnt,
-1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true; err = true;
break; break;
} }
} }
if (checker && !checker->check("load_snapshot", "ending", -1, -1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
return !err; return !err;
} }
/** /**
* Load records from a file. * Load records from a file.
* @param src the path of the source file. * @param src the path of the source file.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_snapshot(const std::string& src) { bool load_snapshot(const std::string& src, ProgressChecker* checker = NUL L) {
_assert_(true); _assert_(true);
std::ifstream ifs; std::ifstream ifs;
ifs.open(src.c_str(), std::ios_base::in | std::ios_base::binary); ifs.open(src.c_str(), std::ios_base::in | std::ios_base::binary);
if (!ifs) { if (!ifs) {
set_error(Error::NOFILE, "open failed"); set_error(_KCCODELINE_, Error::NOREPOS, "open failed");
return false; return false;
} }
bool err = false; bool err = false;
if (!load_snapshot(&ifs)) err = true; if (!load_snapshot(&ifs, checker)) err = true;
ifs.close(); ifs.close();
if (ifs.bad()) { if (ifs.bad()) {
set_error(Error::SYSTEM, "close failed"); set_error(_KCCODELINE_, Error::SYSTEM, "close failed");
return false; return false;
} }
return !err; return !err;
} }
/** /**
* Create a cursor object. * Create a cursor object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note Because the object of the return value is allocated by the const ructor, it should be * @note Because the object of the return value is allocated by the const ructor, it should be
* released with the delete operator when it is no longer in use. * released with the delete operator when it is no longer in use.
*/ */
virtual Cursor* cursor() = 0; virtual Cursor* cursor() = 0;
/** /**
* Get the string of a database type. * Set the internal logger.
* @param logger the logger object.
* @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for
debugging,
* Logger::INFO for normal information, Logger::WARN for warning, and Log
ger::ERROR for fatal
* error.
* @return true on success, or false on failure.
*/
virtual bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN |
Logger::ERROR) = 0;
/**
* Set the internal meta operation trigger.
* @param trigger the trigger object.
* @return true on success, or false on failure.
*/
virtual bool tune_meta_trigger(MetaTrigger* trigger) = 0;
/**
* Get the class name of a database type.
* @param type the database type.
* @return the string of the type name.
*/
static const char* typecname(uint32_t type) {
_assert_(true);
switch (type) {
case TYPEVOID: return "void";
case TYPEPHASH: return "ProtoHashDB";
case TYPEPTREE: return "ProtoTreeDB";
case TYPESTASH: return "StashDB";
case TYPECACHE: return "CacheDB";
case TYPEGRASS: return "GrassDB";
case TYPEHASH: return "HashDB";
case TYPETREE: return "TreeDB";
case TYPEDIR: return "DirDB";
case TYPEFOREST: return "ForestDB";
case TYPEMISC: return "misc";
}
return "unknown";
}
/**
* Get the description string of a database type.
* @param type the database type. * @param type the database type.
* @return the string of the type name. * @return the string of the type name.
*/ */
static const char* typestring(uint32_t type) { static const char* typestring(uint32_t type) {
_assert_(true); _assert_(true);
switch (type) { switch (type) {
case TYPEVOID: return "void"; case TYPEVOID: return "void";
case TYPEPHASH: return "prototype hash database"; case TYPEPHASH: return "prototype hash database";
case TYPEPTREE: return "prototype tree database"; case TYPEPTREE: return "prototype tree database";
case TYPEPMISC: return "miscellaneous prototype database"; case TYPESTASH: return "stash database";
case TYPECACHE: return "cache database"; case TYPECACHE: return "cache hash database";
case TYPEGRASS: return "cache tree database";
case TYPEHASH: return "file hash database"; case TYPEHASH: return "file hash database";
case TYPETREE: return "file tree database"; case TYPETREE: return "file tree database";
case TYPEDIR: return "directory database"; case TYPEDIR: return "directory hash database";
case TYPEFOREST: return "directory tree database";
case TYPEMISC: return "miscellaneous database"; case TYPEMISC: return "miscellaneous database";
} }
return "unknown"; return "unknown";
} }
}; };
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
 End of changes. 100 change blocks. 
124 lines changed or deleted 488 lines changed or added


 kcdirdb.h   kcdirdb.h 
/************************************************************************** *********************** /************************************************************************** ***********************
* Directory database * Directory hash database
* Copyright (C) 2009-2010 FAL Labs * Copyright (C) 2009-2010 FAL Labs
* This file is part of Kyoto Cabinet. * This file is part of Kyoto Cabinet.
* This program is free software: you can redistribute it and/or modify it under the terms of * This program is free software: you can redistribute it and/or modify it under the terms of
* the GNU General Public License as published by the Free Software Foundat ion, either version * the GNU General Public License as published by the Free Software Foundat ion, either version
* 3 of the License, or any later version. * 3 of the License, or any later version.
* This program is distributed in the hope that it will be useful, but WITH OUT ANY WARRANTY; * This program is distributed in the hope that it will be useful, but WITH OUT ANY WARRANTY;
* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PA RTICULAR PURPOSE. * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PA RTICULAR PURPOSE.
* See the GNU General Public License for more details. * See the GNU General Public License for more details.
* You should have received a copy of the GNU General Public License along with this program. * You should have received a copy of the GNU General Public License along with this program.
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
************************************************************************** ***********************/ ************************************************************************** ***********************/
#ifndef _KCDIRDB_H // duplication check #ifndef _KCDIRDB_H // duplication check
#define _KCDIRDB_H #define _KCDIRDB_H
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
#include <kcdb.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h>
#include <kcdb.h>
#include <kcplantdb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation. * Constants for implementation.
*/ */
namespace { namespace {
const char* DDBMAGICFILE = "__KCDIR__"; ///< magic file of the directory const char* DDBMAGICFILE = "__KCDIR__"; ///< magic file of the directory
const char* DDBCOMPFILE = "__comp__"; ///< compression file of the direc const char* DDBMETAFILE = "__meta__"; ///< meta data file of the directo
tory ry
const char* DDBOPAQUEFILE = "__opq__"; ///< opaque file of the directory
const char* DDBATRANPREFIX = "_x"; ///< prefix of files for auto tran saction const char* DDBATRANPREFIX = "_x"; ///< prefix of files for auto tran saction
const char DDBCHKSUMSEED[] = "__kyotocabinet__"; ///< seed of the module c hecksum const char DDBCHKSUMSEED[] = "__kyotocabinet__"; ///< seed of the module c hecksum
const char DDBMAGICEOF[] = "_EOF_"; ///< magic data for the end of fil e const char DDBMAGICEOF[] = "_EOF_"; ///< magic data for the end of fil e
const int64_t DDBMETABUFSIZ = 128; ///< size of the meta data buffer
const uint8_t DDBRECMAGIC = 0xcc; ///< magic data for record const uint8_t DDBRECMAGIC = 0xcc; ///< magic data for record
const int32_t DDBRLOCKSLOT = 64; ///< number of slots of the record lock const int32_t DDBRLOCKSLOT = 64; ///< number of slots of the record lock
const int32_t DDBRECUNITSIZ = 32; ///< unit size of a record const int32_t DDBRECUNITSIZ = 32; ///< unit size of a record
const size_t DDBOPAQUESIZ = 16; ///< size of the opaque buffer
const char* DDBWALPATHEXT = "wal"; ///< extension of the WAL director y const char* DDBWALPATHEXT = "wal"; ///< extension of the WAL director y
const char* DDBTMPPATHEXT = "tmp"; ///< extension of the temporary di rectory const char* DDBTMPPATHEXT = "tmp"; ///< extension of the temporary di rectory
} }
/** /**
* Directory database. * Directory hash database.
* @note This class is a concrete class to operate a hash database in a dir ectory. This class * @note This class is a concrete class to operate a hash database in a dir ectory. This class
* can be inherited but overwriting methods is forbidden. Before every dat abase operation, it is * can be inherited but overwriting methods is forbidden. Before every dat abase operation, it is
* necessary to call the TreeDB::open method in order to open a database fi le and connect the * necessary to call the TreeDB::open method in order to open a database fi le and connect the
* database object to it. To avoid data missing or corruption, it is impor tant to close every * database object to it. To avoid data missing or corruption, it is impor tant to close every
* database file by the TreeDB::close method when the database is no longer in use. It is * database file by the TreeDB::close method when the database is no longer in use. It is
* forbidden for multible database objects in a process to open the same da tabase at the same * forbidden for multible database objects in a process to open the same da tabase at the same
* time. * time. It is forbidden to share a database object with child processes.
*/ */
class DirDB : public FileDB { class DirDB : public BasicDB {
friend class PlantDB<DirDB, BasicDB::TYPEFOREST>;
public: public:
class Cursor; class Cursor;
private: private:
struct Record; struct Record;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
/** An alias of vector of strings. */ /** An alias of vector of strings. */
typedef std::vector<std::string> StringVector; typedef std::vector<std::string> StringVector;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public FileDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class DirDB; friend class DirDB;
public: public:
/** /**
* Constructor. * Constructor.
* @param db the container database object. * @param db the container database object.
*/ */
explicit Cursor(DirDB* db) : db_(db), dir_(), alive_(false), name_("") { explicit Cursor(DirDB* db) : db_(db), dir_(), alive_(false), name_("") {
_assert_(db); _assert_(db);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
db_->curs_.push_back(this); db_->curs_.push_back(this);
skipping to change at line 95 skipping to change at line 101
if (!db_) return; if (!db_) return;
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
db_->curs_.remove(this); db_->curs_.remove(this);
} }
/** /**
* Accept a visitor to the current record. * Accept a visitor to the current record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only operation. * @param writable true for writable operation, or false for read-only operation.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and othe * @note The operation for each record is performed atomically and othe
r threads accessing r threads accessing
* the same record are blocked. * the same record are blocked. To avoid deadlock, any database operat
ion must not be
* performed in this function.
*/ */
bool accept(Visitor* visitor, bool writable = true, bool step = false) { bool accept(Visitor* visitor, bool writable = true, bool step = false) {
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(db_->writer_)) { if (writable && !(db_->writer_)) {
db_->set_error(__FILE__, __LINE__, Error::NOPERM, "permission denie d"); db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
if (!alive_) { if (!alive_) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
bool err = false; bool err = false;
const std::string& rpath = db_->path_ + File::PATHCHR + name_; const std::string& rpath = db_->path_ + File::PATHCHR + name_;
int64_t cnt = db_->count_; int64_t cnt = db_->count_;
Record rec; Record rec;
if (db_->read_record(rpath, &rec)) { if (db_->read_record(rpath, &rec)) {
if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec.rsiz, if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec.rsiz,
visitor, rpath, name_.c_str())) err = t rue; visitor, rpath, name_.c_str())) err = t rue;
delete[] rec.rbuf; delete[] rec.rbuf;
skipping to change at line 132 skipping to change at line 139
do { do {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
if (!disable()) err = true; if (!disable()) err = true;
break; break;
} }
} while (*name_.c_str() == *DDBMAGICFILE); } while (*name_.c_str() == *DDBMAGICFILE);
} }
} else { } else {
while (true) { while (true) {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
break; break;
} }
if (*name_.c_str() == *DDBMAGICFILE) continue; if (*name_.c_str() == *DDBMAGICFILE) continue;
const std::string& npath = db_->path_ + File::PATHCHR + name_; const std::string& npath = db_->path_ + File::PATHCHR + name_;
if (!File::status(npath)) continue; if (!File::status(npath)) continue;
if (db_->read_record(npath, &rec)) { if (db_->read_record(npath, &rec)) {
if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.v siz, rec.rsiz, if (!db_->accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.v siz, rec.rsiz,
visitor, npath, name_.c_str())) err = true; visitor, npath, name_.c_str())) err = true;
delete[] rec.rbuf; delete[] rec.rbuf;
if (alive_ && step && db_->count_ == cnt) { if (alive_ && step && db_->count_ == cnt) {
do { do {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
if (!disable()) err = true; if (!disable()) err = true;
break; break;
} }
} while (*name_.c_str() == *DDBMAGICFILE); } while (*name_.c_str() == *DDBMAGICFILE);
} }
} else { } else {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
err = true; err = true;
} }
break; break;
} }
} }
return !err; return !err;
} }
/** /**
* Jump the cursor to the first record. * Jump the cursor to the first record for forward scan.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump() { bool jump() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (alive_ && !disable()) return false; if (alive_ && !disable()) return false;
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!dir_.open(db_->path_)) { if (!dir_.open(db_->path_)) {
db_->set_error(__FILE__, __LINE__, Error::SYSTEM, "opening a direct ory failed"); db_->set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory fa iled");
return false; return false;
} }
alive_ = true; alive_ = true;
do { do {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
return false; return false;
} }
} while (*name_.c_str() == *DDBMAGICFILE); } while (*name_.c_str() == *DDBMAGICFILE);
return true; return true;
} }
/** /**
* Jump the cursor onto a record. * Jump the cursor to a record for forward scan.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump(const char* kbuf, size_t ksiz) { bool jump(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (alive_ && !disable()) return false; if (alive_ && !disable()) return false;
if (!dir_.open(db_->path_)) { if (!dir_.open(db_->path_)) {
db_->set_error(__FILE__, __LINE__, Error::SYSTEM, "opening a direct ory failed"); db_->set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory fa iled");
return false; return false;
} }
alive_ = true; alive_ = true;
while (true) { while (true) {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
return false; return false;
} }
if (*name_.c_str() == *DDBMAGICFILE) continue; if (*name_.c_str() == *DDBMAGICFILE) continue;
const std::string& rpath = db_->path_ + File::PATHCHR + name_; const std::string& rpath = db_->path_ + File::PATHCHR + name_;
Record rec; Record rec;
if (db_->read_record(rpath, &rec)) { if (db_->read_record(rpath, &rec)) {
if (rec.ksiz == ksiz && !std::memcmp(rec.kbuf, kbuf, ksiz)) { if (rec.ksiz == ksiz && !std::memcmp(rec.kbuf, kbuf, ksiz)) {
delete[] rec.rbuf; delete[] rec.rbuf;
break; break;
} }
delete[] rec.rbuf; delete[] rec.rbuf;
} else { } else {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
return false; return false;
} }
} }
return true; return true;
} }
/** /**
* Jump the cursor to a record. * Jump the cursor to a record for forward scan.
* @note Equal to the original Cursor::jump method except that the para meter is std::string. * @note Equal to the original Cursor::jump method except that the para meter is std::string.
*/ */
bool jump(const std::string& key) { bool jump(const std::string& key) {
_assert_(true); _assert_(true);
return jump(key.c_str(), key.size()); return jump(key.c_str(), key.size());
} }
/** /**
* Jump the cursor to the last record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back() {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Jump the cursor to a record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Jump the cursor to a record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back(const std::string& key) {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Step the cursor to the next record. * Step the cursor to the next record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool step() { bool step() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!alive_) { if (!alive_) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
do { do {
if (!dir_.read(&name_)) { if (!dir_.read(&name_)) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
disable(); disable();
return false; return false;
} }
} while (*name_.c_str() == *DDBMAGICFILE); } while (*name_.c_str() == *DDBMAGICFILE);
return true; return true;
} }
/** /**
* Step the cursor to the previous record.
* @note This is a dummy implementation for compatibility.
*/
bool step_back() {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Get the database object. * Get the database object.
* @return the database object. * @return the database object.
*/ */
DirDB* db() { DirDB* db() {
_assert_(true); _assert_(true);
return db_; return db_;
} }
private: private:
/** /**
* Disable the cursor. * Disable the cursor.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool disable() { bool disable() {
bool err = false; bool err = false;
if (!dir_.close()) { if (!dir_.close()) {
db_->set_error(__FILE__, __LINE__, Error::SYSTEM, "closing a direct ory failed"); db_->set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory fa iled");
err = true; err = true;
} }
alive_ = false; alive_ = false;
return !err; return !err;
} }
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
Cursor(const Cursor&); Cursor(const Cursor&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
Cursor& operator =(const Cursor&); Cursor& operator =(const Cursor&);
/** The inner database. */ /** The inner database. */
DirDB* db_; DirDB* db_;
/** The inner directory stream. */ /** The inner directory stream. */
DirStream dir_; DirStream dir_;
/** The flag if alive. */ /** The flag if alive. */
bool alive_; bool alive_;
/** The current name. */ /** The current name. */
std::string name_; std::string name_;
}; };
/** /**
* Tuning Options. * Tuning options.
*/ */
enum Option { enum Option {
TCOMPRESS = 1 << 0 ///< compress each record TSMALL = 1 << 0, ///< dummy for compatibility
TLINEAR = 1 << 1, ///< dummy for compatibility
TCOMPRESS = 1 << 2 ///< compress each record
};
/**
* Status flags.
*/
enum Flag {
FOPEN = 1 << 0, ///< dummy for compatibility
FFATAL = 1 << 1 ///< dummy for compatibility
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit DirDB() : explicit DirDB() :
mlock_(), rlock_(), error_(), erstrm_(NULL), ervbs_(false), mlock_(), rlock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NU
omode_(0), writer_(false), autotran_(false), autosync_(false), file_(), LL),
curs_(), path_(""), omode_(0), writer_(false), autotran_(false), autosync_(false), recov_(f
opts_(0), count_(0), size_(0), embcomp_(&ZLIBRAWCOMP), comp_(NULL), com alse), reorg_(false),
pchk_(0), file_(), curs_(), path_(""),
libver_(LIBVER), librev_(LIBREV), fmtver_(FMTVER), chksum_(0), type_(TY
PEDIR),
flags_(0), opts_(0), count_(0), size_(0), opaque_(), embcomp_(&ZLIBRAWC
OMP), comp_(NULL),
tran_(false), trhard_(false), trcount_(0), trsize_(0), walpath_(""), tm ppath_("") { tran_(false), trhard_(false), trcount_(0), trsize_(0), walpath_(""), tm ppath_("") {
_assert_(true); _assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~DirDB() { virtual ~DirDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
skipping to change at line 331 skipping to change at line 405
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and other * @note The operation for each record is performed atomically and other
threads accessing the threads accessing the
* same record are blocked. * same record are blocked. To avoid deadlock, any database operation mu
st not be performed in
* this function.
*/ */
bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) { bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
char name[NUMBUFSIZ]; char name[NUMBUFSIZ];
size_t lidx = hashpath(kbuf, ksiz, name) % DDBRLOCKSLOT; size_t lidx = hashpath(kbuf, ksiz, name) % DDBRLOCKSLOT;
if (writable) { if (writable) {
rlock_.lock_writer(lidx); rlock_.lock_writer(lidx);
} else { } else {
rlock_.lock_reader(lidx); rlock_.lock_reader(lidx);
} }
bool err = false; bool err = false;
if (!accept_impl(kbuf, ksiz, visitor, name)) err = true; if (!accept_impl(kbuf, ksiz, visitor, name)) err = true;
rlock_.unlock(lidx); rlock_.unlock(lidx);
return !err; return !err;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the whole iteration is performed atomically and other threads ar * @note The whole iteration is performed atomically and other threads ar
e blocked. e blocked. To avoid
* deadlock, any database operation must not be performed in this functio
n.
*/ */
bool iterate(Visitor *visitor, bool writable = true) { bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* che cker = NULL) {
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
bool err = false; bool err = false;
if (!iterate_impl(visitor)) err = true; if (!iterate_impl(visitor, checker)) err = true;
trigger_meta(MetaTrigger::ITERATE, "iterate");
return !err; return !err;
} }
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
Error error() const { Error error() const {
_assert_(true); _assert_(true);
return error_; return error_;
} }
/** /**
* Set the error information. * Set the error information.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param code an error code. * @param code an error code.
* @param message a supplement message. * @param message a supplement message.
*/ */
void set_error(Error::Code code, const char* message) { void set_error(const char* file, int32_t line, const char* func,
_assert_(message); Error::Code code, const char* message) {
_assert_(file && line > 0 && func && message);
error_->set(code, message); error_->set(code, message);
if (code == Error::BROKEN || code == Error::SYSTEM) flags_ |= FFATAL;
if (logger_) {
Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ?
Logger::ERROR : Logger::INFO;
if (kind & logkinds_)
report(file, line, func, kind, "%d: %s: %s", code, Error::codename(
code), message);
}
} }
/** /**
* Open a database file. * Open a database file.
* @param path the path of a database file. * @param path the path of a database file.
* @param mode the connection mode. DirDB::OWRITER as a writer, DirDB::O READER as a * @param mode the connection mode. DirDB::OWRITER as a writer, DirDB::O READER as a
* reader. The following may be added to the writer mode by bitwise-or: DirDB::OCREATE, * reader. The following may be added to the writer mode by bitwise-or: DirDB::OCREATE,
* which means it creates a new database if the file does not exist, DirD B::OTRUNCATE, which * which means it creates a new database if the file does not exist, DirD B::OTRUNCATE, which
* means it creates a new database regardless if the file exists, DirDB:: OAUTOTRAN, which * means it creates a new database regardless if the file exists, DirDB:: OAUTOTRAN, which
* means each updating operation is performed in implicit transaction, Di rDB::OAUTOSYNC, * means each updating operation is performed in implicit transaction, Di rDB::OAUTOSYNC,
* which means each updating operation is followed by implicit synchroniz ation with the file * which means each updating operation is followed by implicit synchroniz ation with the file
skipping to change at line 421 skipping to change at line 510
* detected. * detected.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note Every opened database must be closed by the DirDB::close method when it is no * @note Every opened database must be closed by the DirDB::close method when it is no
* longer in use. It is not allowed for two or more database objects in the same process to * longer in use. It is not allowed for two or more database objects in the same process to
* keep their connections to the same database file at the same time. * keep their connections to the same database file at the same time.
*/ */
bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", p ath.c_str());
writer_ = false; writer_ = false;
autotran_ = false; autotran_ = false;
autosync_ = false; autosync_ = false;
recov_ = false;
reorg_ = false;
uint32_t fmode = File::OREADER; uint32_t fmode = File::OREADER;
if (mode & OWRITER) { if (mode & OWRITER) {
writer_ = true; writer_ = true;
fmode = File::OWRITER; fmode = File::OWRITER;
if (mode & OCREATE) fmode |= File::OCREATE; if (mode & OCREATE) fmode |= File::OCREATE;
if (mode & OTRUNCATE) fmode |= File::OTRUNCATE; if (mode & OTRUNCATE) fmode |= File::OTRUNCATE;
if (mode & OAUTOTRAN) autotran_ = true; if (mode & OAUTOTRAN) autotran_ = true;
if (mode & OAUTOSYNC) autosync_ = true; if (mode & OAUTOSYNC) autosync_ = true;
} }
if (mode & ONOLOCK) fmode |= File::ONOLOCK; if (mode & ONOLOCK) fmode |= File::ONOLOCK;
if (mode & OTRYLOCK) fmode |= File::OTRYLOCK; if (mode & OTRYLOCK) fmode |= File::OTRYLOCK;
size_t psiz = path.size(); size_t psiz = path.size();
while (psiz > 0 && path[psiz-1] == File::PATHCHR) { while (psiz > 0 && path[psiz-1] == File::PATHCHR) {
psiz--; psiz--;
} }
const std::string& cpath = path.substr(0, psiz); const std::string& cpath = path.substr(0, psiz);
const std::string& metapath = cpath + File::PATHCHR + DDBMAGICFILE; const std::string& magicpath = cpath + File::PATHCHR + DDBMAGICFILE;
const std::string& comppath = cpath + File::PATHCHR + DDBCOMPFILE; const std::string& metapath = cpath + File::PATHCHR + DDBMETAFILE;
const std::string& opqpath = cpath + File::PATHCHR + DDBOPAQUEFILE;
const std::string& walpath = cpath + File::EXTCHR + DDBWALPATHEXT; const std::string& walpath = cpath + File::EXTCHR + DDBWALPATHEXT;
const std::string& tmppath = cpath + File::EXTCHR + DDBTMPPATHEXT; const std::string& tmppath = cpath + File::EXTCHR + DDBTMPPATHEXT;
bool hot = false; bool hot = false;
if (writer_ && (mode & OTRUNCATE) && File::status(metapath)) { if (writer_ && (mode & OTRUNCATE) && File::status(magicpath)) {
if (!file_.open(metapath, fmode)) { if (!file_.open(magicpath, fmode)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
if (!remove_files(cpath)) { if (!remove_files(cpath)) {
file_.close(); file_.close();
return false; return false;
} }
if (File::status(walpath)) { if (File::status(walpath)) {
remove_files(walpath); remove_files(walpath);
File::remove_directory(walpath); File::remove_directory(walpath);
} }
if (!file_.close()) { if (!file_.close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
const std::string& buf = format_meta(0, 0); const std::string& buf = format_magic(0, 0);
if (!File::write_file(metapath, buf.c_str(), buf.size())) { if (!File::write_file(magicpath, buf.c_str(), buf.size())) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "writing a file failed set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed");
");
return false; return false;
} }
if (File::status(comppath) && !File::remove(comppath)) { if (File::status(metapath) && !File::remove(metapath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a file faile set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed");
d"); return false;
}
if (File::status(opqpath) && !File::remove(opqpath)) {
set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed");
return false; return false;
} }
hot = true; hot = true;
} }
File::Status sbuf; File::Status sbuf;
if (File::status(cpath, &sbuf)) { if (File::status(cpath, &sbuf)) {
if (!sbuf.isdir) { if (!sbuf.isdir) {
set_error(__FILE__, __LINE__, Error::NOPERM, "invalid path (not dir ectory)"); set_error(_KCCODELINE_, Error::NOPERM, "invalid path (not directory )");
return false; return false;
} }
if (!File::status(metapath)) { if (!File::status(magicpath)) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid meta data"); set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data");
return false; return false;
} }
if (!file_.open(metapath, fmode)) { if (!file_.open(magicpath, fmode)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
} else if (writer_ && (mode & OCREATE)) { } else if (writer_ && (mode & OCREATE)) {
hot = true; hot = true;
if (!File::make_directory(cpath)) { if (!File::make_directory(cpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "making a directory fa iled"); set_error(_KCCODELINE_, Error::SYSTEM, "making a directory failed") ;
return false; return false;
} }
if (!file_.open(metapath, fmode)) { if (!file_.open(magicpath, fmode)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
} else { } else {
set_error(__FILE__, __LINE__, Error::NOFILE, "open failed (file not f ound)"); set_error(_KCCODELINE_, Error::NOREPOS, "open failed (file not found) ");
return false; return false;
} }
if (hot) { if (hot) {
count_ = 0; count_ = 0;
size_ = 0; size_ = 0;
if (opts_ & TCOMPRESS) { comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL;
comp_ = embcomp_; libver_ = LIBVER;
compchk_ = calc_comp_checksum(); librev_ = LIBREV;
const std::string& buf = strprintf("%u\n", (unsigned)compchk_); fmtver_ = FMTVER;
if (!File::write_file(comppath, buf.c_str(), buf.size())) { chksum_ = calc_checksum();
set_error(__FILE__, __LINE__, Error::SYSTEM, "writing a file fail if (!dump_meta(metapath)) {
ed"); file_.close();
file_.close(); return false;
return false;
}
} else {
comp_ = NULL;
compchk_ = 0;
} }
std::memset(opaque_, 0, sizeof(opaque_));
if (autosync_ && !File::synchronize_whole()) { if (autosync_ && !File::synchronize_whole()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the fil e system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file syst em failed");
file_.close(); file_.close();
return false; return false;
} }
} else { } else {
if (File::status(walpath, &sbuf)) { if (File::status(walpath, &sbuf)) {
if (writer_) { if (writer_) {
file_.truncate(0); file_.truncate(0);
} else { } else {
File::write_file(metapath, "", 0); File::write_file(magicpath, "", 0);
file_.refresh(); file_.refresh();
} }
DirStream dir; DirStream dir;
if (dir.open(walpath)) { if (dir.open(walpath)) {
std::string name; std::string name;
while (dir.read(&name)) { while (dir.read(&name)) {
const std::string& srcpath = walpath + File::PATHCHR + name; const std::string& srcpath = walpath + File::PATHCHR + name;
const std::string& destpath = cpath + File::PATHCHR + name; const std::string& destpath = cpath + File::PATHCHR + name;
File::Status sbuf; File::Status sbuf;
if (File::status(srcpath, &sbuf)) { if (File::status(srcpath, &sbuf)) {
if (sbuf.size > 1) { if (sbuf.size > 1) {
File::rename(srcpath, destpath); File::rename(srcpath, destpath);
} else { } else {
if (File::remove(destpath) || !File::status(destpath)) File ::remove(srcpath); if (File::remove(destpath) || !File::status(destpath)) File ::remove(srcpath);
} }
} }
} }
dir.close(); dir.close();
File::remove_directory(walpath); File::remove_directory(walpath);
report(__FILE__, __LINE__, "info", "recovered by the WAL director recov_ = true;
y"); report(_KCCODELINE_, Logger::WARN, "recovered by the WAL director
y");
} }
} }
if (File::status(comppath)) { if (!load_meta(metapath)) {
opts_ |= TCOMPRESS; file_.close();
comp_ = embcomp_; return false;
compchk_ = calc_comp_checksum();
int64_t nsiz;
char* nbuf = File::read_file(comppath, &nsiz, NUMBUFSIZ);
if (nbuf) {
uint32_t chk = atoi(nbuf);
delete[] nbuf;
if (chk != compchk_) {
set_error(__FILE__, __LINE__, Error::INVALID, "invalid compress
ion checksum");
file_.close();
return false;
}
} else {
set_error(__FILE__, __LINE__, Error::SYSTEM, "reading a file fail
ed");
file_.close();
return false;
}
} }
if (!load_meta()) { comp_ = (opts_ & TCOMPRESS) ? embcomp_ : NULL;
if (!calc_meta(cpath)) { uint8_t chksum = calc_checksum();
if (chksum != chksum_) {
set_error(_KCCODELINE_, Error::INVALID, "invalid module checksum");
report(_KCCODELINE_, Logger::WARN, "saved=%02X calculated=%02X",
(unsigned)chksum_, (unsigned)chksum);
file_.close();
return false;
}
if (!load_magic()) {
if (!calc_magic(cpath)) {
file_.close(); file_.close();
return false; return false;
} }
reorg_ = true;
if (!writer_ && !(mode & ONOLOCK)) { if (!writer_ && !(mode & ONOLOCK)) {
const std::string& buf = format_meta(count_, size_); const std::string& buf = format_magic(count_, size_);
if (!File::write_file(metapath, buf.c_str(), buf.size())) { if (!File::write_file(magicpath, buf.c_str(), buf.size())) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "writing a file fa set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed")
iled"); ;
file_.close(); file_.close();
return false; return false;
} }
if (!file_.refresh()) { if (!file_.refresh()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
file_.close(); file_.close();
return false; return false;
} }
} }
report(__FILE__, __LINE__, "info", "re-calculated meta data"); report(_KCCODELINE_, Logger::WARN, "re-calculated magic data");
} }
} }
if (writer_ && !file_.truncate(0)) { if (writer_ && !file_.truncate(0)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
file_.close(); file_.close();
return false; return false;
} }
if (File::status(walpath)) { if (File::status(walpath)) {
remove_files(walpath); remove_files(walpath);
File::remove_directory(walpath); File::remove_directory(walpath);
} }
if (File::status(tmppath)) { if (File::status(tmppath)) {
remove_files(tmppath); remove_files(tmppath);
File::remove_directory(tmppath); File::remove_directory(tmppath);
} }
omode_ = mode; omode_ = mode;
path_ = cpath; path_ = cpath;
tran_ = false; tran_ = false;
walpath_ = walpath; walpath_ = walpath;
tmppath_ = tmppath; tmppath_ = tmppath;
load_opaque();
trigger_meta(MetaTrigger::OPEN, "open");
return true; return true;
} }
/** /**
* Close the database file. * Close the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool close() { bool close() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", p ath_.c_str());
bool err = false; bool err = false;
if (tran_ && !abort_transaction()) err = true; if (tran_ && !abort_transaction()) err = true;
if (!disable_cursors()) err = true; if (!disable_cursors()) err = true;
if (writer_ && !dump_meta()) err = true; if (writer_) {
if (!dump_magic()) err = true;
if (!dump_opaque()) err = true;
}
if (!file_.close()) { if (!file_.close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
omode_ = 0; omode_ = 0;
trigger_meta(MetaTrigger::CLOSE, "close");
return !err; return !err;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @param proc a postprocessor object. If it is NULL, no postprocessing is performed. * @param proc a postprocessor object. If it is NULL, no postprocessing is performed.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize(bool hard = false, FileProcessor* proc = NULL) { bool synchronize(bool hard = false, FileProcessor* proc = NULL,
ProgressChecker* checker = NULL) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied");
return false; return false;
} }
rlock_.lock_reader_all(); rlock_.lock_reader_all();
bool err = false; bool err = false;
if (!synchronize_impl(hard, proc)) err = true; if (!synchronize_impl(hard, proc, checker)) err = true;
trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize");
rlock_.unlock_all(); rlock_.unlock_all();
return !err; return !err;
} }
/** /**
* Begin transaction. * Begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction(bool hard = false) { bool begin_transaction(bool hard = false) {
_assert_(true); _assert_(true);
for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) { for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) {
mlock_.lock_writer(); mlock_.lock_writer();
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wsec > 1.0) wsec = 1.0; if (wsec > 1.0) wsec = 1.0;
Thread::sleep(wsec); Thread::sleep(wsec);
} }
trhard_ = hard; trhard_ = hard;
if (!begin_transaction_impl()) { if (!begin_transaction_impl()) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
tran_ = true; tran_ = true;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction");
mlock_.unlock(); mlock_.unlock();
return true; return true;
} }
/** /**
* Try to begin transaction. * Try to begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_try(bool hard = false) { bool begin_transaction_try(bool hard = false) {
_assert_(true); _assert_(true);
mlock_.lock_writer(); mlock_.lock_writer();
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (tran_) { if (tran_) {
set_error(__FILE__, __LINE__, Error::LOGIC, "competition avoided"); set_error(_KCCODELINE_, Error::LOGIC, "competition avoided");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
trhard_ = hard; trhard_ = hard;
if (!begin_transaction_impl()) { if (!begin_transaction_impl()) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
tran_ = true; tran_ = true;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try");
mlock_.unlock(); mlock_.unlock();
return true; return true;
} }
/** /**
* End transaction. * End transaction.
* @param commit true to commit the transaction, or false to abort the tr ansaction. * @param commit true to commit the transaction, or false to abort the tr ansaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool end_transaction(bool commit = true) { bool end_transaction(bool commit = true) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!tran_) { if (!tran_) {
set_error(__FILE__, __LINE__, Error::INVALID, "not in transaction"); set_error(_KCCODELINE_, Error::INVALID, "not in transaction");
return false; return false;
} }
bool err = false; bool err = false;
if (commit) { if (commit) {
if (!commit_transaction()) err = true; if (!commit_transaction()) err = true;
} else { } else {
if (!abort_transaction()) err = true; if (!abort_transaction()) err = true;
} }
tran_ = false; tran_ = false;
trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction");
return !err; return !err;
} }
/** /**
* Remove all records. * Remove all records.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clear() { bool clear() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
bool err = false; bool err = false;
if (!disable_cursors()) err = true; if (!disable_cursors()) err = true;
if (tran_) { if (tran_) {
DirStream dir; DirStream dir;
if (dir.open(path_)) { if (dir.open(path_)) {
std::string name; std::string name;
while (dir.read(&name)) { while (dir.read(&name)) {
if (*name.c_str() == *DDBMAGICFILE) continue; if (*name.c_str() == *DDBMAGICFILE) continue;
const std::string& rpath = path_ + File::PATHCHR + name; const std::string& rpath = path_ + File::PATHCHR + name;
const std::string& walpath = walpath_ + File::PATHCHR + name; const std::string& walpath = walpath_ + File::PATHCHR + name;
if (File::status(walpath)) { if (File::status(walpath)) {
if (!File::remove(rpath)) { if (!File::remove(rpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a file faile d");
err = true; err = true;
} }
} else if (!File::rename(rpath, walpath)) { } else if (!File::rename(rpath, walpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "renaming a file f ailed"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed" );
err = true; err = true;
} }
} }
if (!dir.close()) { if (!dir.close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "closing a directory failed"); set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory faile d");
err = true; err = true;
} }
} else { } else {
set_error(__FILE__, __LINE__, Error::SYSTEM, "opening a directory f ailed"); set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed" );
err = true; err = true;
} }
} else { } else {
if (!remove_files(path_)) err = true; if (!remove_files(path_)) err = true;
} }
recov_ = false;
reorg_ = false;
flags_ = 0;
std::memset(opaque_, 0, sizeof(opaque_));
count_ = 0; count_ = 0;
size_ = 0; size_ = 0;
trigger_meta(MetaTrigger::CLEAR, "clear");
return !err; return !err;
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
*/ */
int64_t count() { int64_t count() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return count_; return count_;
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes, or -1 on failure. * @return the size of the database file in bytes, or -1 on failure.
*/ */
int64_t size() { int64_t size() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return size_impl(); return size_impl();
} }
/** /**
* Get the path of the database file. * Get the path of the database file.
* @return the path of the database file, or an empty string on failure. * @return the path of the database file, or an empty string on failure.
*/ */
std::string path() { std::string path() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return ""; return "";
} }
return path_; return path_;
} }
/** /**
* Get the miscellaneous status information. * Get the miscellaneous status information.
* @param strmap a string map to contain the result. * @param strmap a string map to contain the result.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool status(std::map<std::string, std::string>* strmap) { bool status(std::map<std::string, std::string>* strmap) {
_assert_(strmap); _assert_(strmap);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
(*strmap)["type"] = "DirDB"; (*strmap)["type"] = strprintf("%u", (unsigned)TYPEDIR);
(*strmap)["realtype"] = strprintf("%u", (unsigned)TYPEDIR); (*strmap)["realtype"] = strprintf("%u", (unsigned)type_);
(*strmap)["path"] = path_; (*strmap)["path"] = path_;
(*strmap)["libver"] = strprintf("%u", libver_);
(*strmap)["librev"] = strprintf("%u", librev_);
(*strmap)["fmtver"] = strprintf("%u", fmtver_);
(*strmap)["chksum"] = strprintf("%u", chksum_);
(*strmap)["flags"] = strprintf("%u", flags_);
(*strmap)["opts"] = strprintf("%u", opts_); (*strmap)["opts"] = strprintf("%u", opts_);
(*strmap)["compchk"] = strprintf("%u", (unsigned)compchk_); (*strmap)["recovered"] = strprintf("%d", recov_);
(*strmap)["reorganized"] = strprintf("%d", reorg_);
if (strmap->count("opaque") > 0)
(*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_));
(*strmap)["count"] = strprintf("%lld", (long long)count_); (*strmap)["count"] = strprintf("%lld", (long long)count_);
(*strmap)["size"] = strprintf("%lld", (long long)size_impl()); (*strmap)["size"] = strprintf("%lld", (long long)size_impl());
return true; return true;
} }
/** /**
* Create a cursor object. * Create a cursor object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note Because the object of the return value is allocated by the const ructor, it should be * @note Because the object of the return value is allocated by the const ructor, it should be
* released with the delete operator when it is no longer in use. * released with the delete operator when it is no longer in use.
*/ */
Cursor* cursor() { Cursor* cursor() {
_assert_(true); _assert_(true);
return new Cursor(this); return new Cursor(this);
} }
/** /**
* Set the internal error reporter. * Set the internal logger.
* @param erstrm a stream object into which internal error messages are s * @param logger the logger object.
tored. * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for
* @param ervbs true to report all errors, or false to report fatal error debugging,
s only. * Logger::INFO for normal information, Logger::WARN for warning, and Log
ger::ERROR for fatal
* error.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_error_reporter(std::ostream* erstrm, bool ervbs) { bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::
_assert_(erstrm); ERROR) {
_assert_(logger);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
erstrm_ = erstrm; logger_ = logger;
ervbs_ = ervbs; logkinds_ = kinds;
return true;
}
/**
* Set the internal meta operation trigger.
* @param trigger the trigger object.
* @return true on success, or false on failure.
*/
bool tune_meta_trigger(MetaTrigger* trigger) {
_assert_(trigger);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
mtrigger_ = trigger;
return true; return true;
} }
/** /**
* Set the optional features. * Set the optional features.
* @param opts the optional features by bitwise-or: DirDB::TCOMPRESS to c ompress each record. * @param opts the optional features by bitwise-or: DirDB::TCOMPRESS to c ompress each record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_options(int8_t opts) { bool tune_options(int8_t opts) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
opts_ = opts; opts_ = opts;
return true; return true;
} }
/** /**
* Set the data compressor. * Set the data compressor.
* @param comp the data compressor object. * @param comp the data compressor object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_compressor(Compressor* comp) { bool tune_compressor(Compressor* comp) {
_assert_(comp); _assert_(comp);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
embcomp_ = comp; embcomp_ = comp;
return true; return true;
} }
protected:
/** /**
* Set the error information. * Get the opaque data.
* @param file the file name of the epicenter. * @return the pointer to the opaque data region, whose size is 16 bytes.
* @param line the line number of the epicenter.
* @param code an error code.
* @param message a supplement message.
*/ */
void set_error(const char* file, int32_t line, char* opaque() {
Error::Code code, const char* message) { _assert_(true);
_assert_(file && message); ScopedSpinRWLock lock(&mlock_, false);
set_error(code, message); if (omode_ == 0) {
if (ervbs_ || code == Error::BROKEN || code == Error::SYSTEM) set_error(_KCCODELINE_, Error::INVALID, "not opened");
report(file, line, "error", "%d: %s: %s", code, Error::codename(code) return NULL;
, message); }
return opaque_;
}
/**
* Synchronize the opaque data.
* @return true on success, or false on failure.
*/
bool synchronize_opaque() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
if (!writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false;
}
bool err = false;
if (!dump_opaque()) err = true;
return !err;
}
/**
* Get the status flags.
* @note This is a dummy implementation for compatibility.
*/
uint8_t flags() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return 0;
} }
protected:
/** /**
* Report a message for debugging. * Report a message for debugging.
* @param file the file name of the epicenter. * @param file the file name of the program source code.
* @param line the line number of the epicenter. * @param line the line number of the program source code.
* @param type the type string. * @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param format the printf-like format string. * @param format the printf-like format string.
* @param ... used according to the format string. * @param ... used according to the format string.
*/ */
void report(const char* file, int32_t line, const char* type, void report(const char* file, int32_t line, const char* func, Logger::Kin d kind,
const char* format, ...) { const char* format, ...) {
_assert_(file && line > 0 && type && format); _assert_(file && line > 0 && func && format);
if (!erstrm_) return; if (!logger_ || !(kind & logkinds_)) return;
const std::string& path = path_.empty() ? "-" : path_;
std::string message; std::string message;
strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str());
va_list ap; va_list ap;
va_start(ap, format); va_start(ap, format);
strprintf(&message, format, ap); vstrprintf(&message, format, ap);
va_end(ap); va_end(ap);
*erstrm_ << "[" << type << "]: " << path << ": " << file << ": " << lin logger_->log(file, line, func, kind, message.c_str());
e; }
*erstrm_ << ": " << message << std::endl; /**
* Report a message for debugging with variable number of arguments.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param format the printf-like format string.
* @param ap used according to the format string.
*/
void report_valist(const char* file, int32_t line, const char* func, Logg
er::Kind kind,
const char* format, va_list ap) {
_assert_(file && line > 0 && func && format);
if (!logger_ || !(kind & logkinds_)) return;
std::string message;
strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str());
vstrprintf(&message, format, ap);
logger_->log(file, line, func, kind, message.c_str());
} }
/** /**
* Report the content of a binary buffer for debugging. * Report the content of a binary buffer for debugging.
* @param file the file name of the epicenter. * @param file the file name of the epicenter.
* @param line the line number of the epicenter. * @param line the line number of the epicenter.
* @param type the type string. * @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param name the name of the information. * @param name the name of the information.
* @param buf the binary buffer. * @param buf the binary buffer.
* @param size the size of the binary buffer * @param size the size of the binary buffer
*/ */
void report_binary(const char* file, int32_t line, const char* type, void report_binary(const char* file, int32_t line, const char* func, Logg er::Kind kind,
const char* name, const char* buf, size_t size) { const char* name, const char* buf, size_t size) {
_assert_(file && line > 0 && type && name && buf && size <= MEMMAXSIZ); _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ);
if (!erstrm_) return; if (!logger_) return;
char* hex = hexencode(buf, size); char* hex = hexencode(buf, size);
report(file, line, type, "%s=%s", name, hex); report(file, line, func, kind, "%s=%s", name, hex);
delete[] hex; delete[] hex;
} }
/**
* Trigger a meta database operation.
* @param kind the kind of the event. MetaTrigger::OPEN for opening, Met
aTrigger::CLOSE for
* closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for ite
ration,
* MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f
or beginning
* transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT
rigger::ABORTTRAN
* for aborting transaction, and MetaTrigger::MISC for miscellaneous oper
ations.
* @param message the supplement message.
*/
void trigger_meta(MetaTrigger::Kind kind, const char* message) {
_assert_(message);
if (mtrigger_) mtrigger_->trigger(kind, message);
}
/**
* Set the database type.
* @param type the database type.
* @return true on success, or false on failure.
*/
bool tune_type(int8_t type) {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
type_ = type;
return true;
}
/**
* Get the library version.
* @return the library version, or 0 on failure.
*/
uint8_t libver() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return libver_;
}
/**
* Get the library revision.
* @return the library revision, or 0 on failure.
*/
uint8_t librev() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return librev_;
}
/**
* Get the format version.
* @return the format version, or 0 on failure.
*/
uint8_t fmtver() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return fmtver_;
}
/**
* Get the module checksum.
* @return the module checksum, or 0 on failure.
*/
uint8_t chksum() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return chksum_;
}
/**
* Get the database type.
* @return the database type, or 0 on failure.
*/
uint8_t type() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return type_;
}
/**
* Get the options.
* @return the options, or 0 on failure.
*/
uint8_t opts() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0;
}
return opts_;
}
/**
* Get the data compressor.
* @return the data compressor, or NULL on failure.
*/
Compressor* comp() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return NULL;
}
return comp_;
}
/**
* Check whether the database was recovered or not.
* @return true if recovered, or false if not.
*/
bool recovered() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
return recov_;
}
/**
* Check whether the database was reorganized or not.
* @return true if recovered, or false if not.
*/
bool reorganized() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
return reorg_;
}
private:
/**
* Set the power of the alignment of record size.
* @note This is a dummy implementation for compatibility.
*/
bool tune_alignment(int8_t apow) {
return true;
}
/**
* Set the power of the capacity of the free block pool.
* @note This is a dummy implementation for compatibility.
*/
bool tune_fbp(int8_t fpow) {
return true;
}
/**
* Set the number of buckets of the hash table.
* @note This is a dummy implementation for compatibility.
*/
bool tune_buckets(int64_t bnum) {
return true;
}
/**
* Set the size of the internal memory-mapped region.
* @note This is a dummy implementation for compatibility.
*/
bool tune_map(int64_t msiz) {
return true;
}
/**
* Set the unit step number of auto defragmentation.
* @note This is a dummy implementation for compatibility.
*/
bool tune_defrag(int64_t dfunit) {
return true;
}
/**
* Perform defragmentation of the file.
* @note This is a dummy implementation for compatibility.
*/
bool defrag(int64_t step = 0) {
return true;
}
/**
* Get the alignment power.
* @note This is a dummy implementation for compatibility.
*/
uint8_t apow() {
return 0;
}
/**
* Get the free block pool power.
* @note This is a dummy implementation for compatibility.
*/
uint8_t fpow() {
return 0;
}
/**
* Get the bucket number.
* @note This is a dummy implementation for compatibility.
*/
int64_t bnum() {
return 1;
}
/**
* Get the size of the internal memory-mapped region.
* @note This is a dummy implementation for compatibility.
*/
int64_t msiz() {
return 0;
}
/**
* Get the unit step number of auto defragmentation.
* @note This is a dummy implementation for compatibility.
*/
int64_t dfunit() {
return 0;
}
private: private:
/** /**
* Record data. * Record data.
*/ */
struct Record { struct Record {
char* rbuf; ///< record buffer char* rbuf; ///< record buffer
size_t rsiz; ///< record size size_t rsiz; ///< record size
const char* kbuf; ///< key buffer const char* kbuf; ///< key buffer
size_t ksiz; ///< key size size_t ksiz; ///< key size
const char* vbuf; ///< value buffer const char* vbuf; ///< value buffer
size_t vsiz; ///< value size size_t vsiz; ///< value size
}; };
/** /**
* Dump the meta data into the file. * Dump the magic data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_meta() { bool dump_magic() {
_assert_(true); _assert_(true);
const std::string& buf = format_meta(count_, size_); const std::string& buf = format_magic(count_, size_);
if (!file_.write(0, buf.c_str(), buf.size())) { if (!file_.write(0, buf.c_str(), buf.size())) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
return true; return true;
} }
/** /**
* Format the meta data. * Format the magic data.
* @return the result string. * @return the result string.
*/ */
std::string format_meta(int64_t count, int64_t size) { std::string format_magic(int64_t count, int64_t size) {
return strprintf("%lld\n%lld\n%s\n", (long long)count, (long long)size, DDBMAGICEOF); return strprintf("%lld\n%lld\n%s\n", (long long)count, (long long)size, DDBMAGICEOF);
} }
/** /**
* Load the meta data from the file. * Load the magic data from the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_meta() { bool load_magic() {
_assert_(true); _assert_(true);
char buf[NUMBUFSIZ*3]; char buf[NUMBUFSIZ*3];
size_t len = file_.size(); size_t len = file_.size();
if (len > sizeof(buf) - 1) len = sizeof(buf) - 1; if (len > sizeof(buf) - 1) len = sizeof(buf) - 1;
if (!file_.read(0, buf, len)) return false; if (!file_.read(0, buf, len)) return false;
buf[len] = '\0'; buf[len] = '\0';
char* rp = buf; char* rp = buf;
int64_t count = atoi(rp); int64_t count = atoi(rp);
char* pv = std::strchr(rp, '\n'); char* pv = std::strchr(rp, '\n');
if (!pv) return false; if (!pv) return false;
rp = pv + 1; rp = pv + 1;
int64_t size = atoi(rp); int64_t size = atoi(rp);
pv = std::strchr(rp, '\n'); pv = std::strchr(rp, '\n');
if (!pv) return false; if (!pv) return false;
rp = pv + 1; rp = pv + 1;
if (std::strlen(rp) < sizeof(DDBMAGICEOF) - 1 || if (std::strlen(rp) < sizeof(DDBMAGICEOF) - 1 ||
std::memcmp(rp, DDBMAGICEOF, sizeof(DDBMAGICEOF) - 1)) return false ; std::memcmp(rp, DDBMAGICEOF, sizeof(DDBMAGICEOF) - 1)) return false ;
flags_ = 0;
count_ = count; count_ = count;
size_ = size; size_ = size;
return true; return true;
} }
/** /**
* Calculate meta data. * Calculate magic data.
* @param cpath the path of the database file. * @param cpath the path of the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool calc_meta(const std::string& cpath) { bool calc_magic(const std::string& cpath) {
_assert_(true); _assert_(true);
count_ = 0; count_ = 0;
size_ = 0; size_ = 0;
DirStream dir; DirStream dir;
if (!dir.open(cpath)) { if (!dir.open(cpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "opening a directory fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed");
return false; return false;
} }
bool err = false; bool err = false;
std::string name; std::string name;
while (dir.read(&name)) { while (dir.read(&name)) {
if (*name.c_str() == *DDBMAGICFILE) continue; if (*name.c_str() == *DDBMAGICFILE) continue;
const std::string& rpath = cpath + File::PATHCHR + name; const std::string& rpath = cpath + File::PATHCHR + name;
File::Status sbuf; File::Status sbuf;
if (File::status(rpath, &sbuf)) { if (File::status(rpath, &sbuf)) {
count_ += 1; if (sbuf.size >= 4) {
size_ += sbuf.size - 4; count_ += 1;
size_ += sbuf.size - 4;
} else {
File::remove(rpath);
}
} else { } else {
set_error(__FILE__, __LINE__, Error::SYSTEM, "checking the status o f a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "checking the status of a fi le failed");
err = true; err = true;
} }
} }
if (!dir.close()) { if (!dir.close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "closing a directory fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Calculate the compression checksum. * Calculate the module checksum.
* @return the compression checksum. * @return the module checksum.
*/ */
uint32_t calc_comp_checksum() { uint8_t calc_checksum() {
_assert_(true); _assert_(true);
size_t zsiz; const char* kbuf = DDBCHKSUMSEED;
char* zbuf = comp_->compress(DDBCHKSUMSEED, sizeof(DDBCHKSUMSEED) - 1, size_t ksiz = sizeof(DDBCHKSUMSEED) - 1;
&zsiz); char* zbuf = NULL;
if (!zbuf) return 0; size_t zsiz = 0;
if (comp_) {
zbuf = comp_->compress(kbuf, ksiz, &zsiz);
if (!zbuf) return 0;
kbuf = zbuf;
ksiz = zsiz;
}
char name[NUMBUFSIZ]; char name[NUMBUFSIZ];
uint32_t hash = hashpath(zbuf, zsiz, name); uint32_t hash = hashpath(kbuf, ksiz, name);
hash += hashmurmur(name, std::strlen(name));
delete[] zbuf; delete[] zbuf;
return hash; return hash;
} }
/** /**
* Dump the meta data into the file.
* @param metapath the path of the meta data file.
* @return true on success, or false on failure.
*/
bool dump_meta(const std::string& metapath) {
_assert_(true);
bool err = false;
char buf[DDBMETABUFSIZ];
char* wp = buf;
wp += std::sprintf(wp, "%u\n", libver_);
wp += std::sprintf(wp, "%u\n", librev_);
wp += std::sprintf(wp, "%u\n", fmtver_);
wp += std::sprintf(wp, "%u\n", chksum_);
wp += std::sprintf(wp, "%u\n", type_);
wp += std::sprintf(wp, "%u\n", opts_);
wp += std::sprintf(wp, "%s\n", DDBMAGICEOF);
if (!File::write_file(metapath, buf, wp - buf)) {
set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed");
err = true;
}
return !err;
}
/**
* Load the meta data from the file.
* @param metapath the path of the meta data file.
* @return true on success, or false on failure.
*/
bool load_meta(const std::string& metapath) {
_assert_(true);
int64_t size;
char* buf = File::read_file(metapath, &size, DDBMETABUFSIZ);
if (!buf) {
set_error(_KCCODELINE_, Error::SYSTEM, "reading a file failed");
return false;
}
std::string str(buf, size);
delete[] buf;
std::vector<std::string> elems;
if (strsplit(str, '\n', &elems) < 7 || elems[6] != DDBMAGICEOF) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data file");
return false;
}
libver_ = atoi(elems[0].c_str());
librev_ = atoi(elems[1].c_str());
fmtver_ = atoi(elems[2].c_str());
chksum_ = atoi(elems[3].c_str());
type_ = atoi(elems[4].c_str());
opts_ = atoi(elems[5].c_str());
return true;
}
/**
* Dump the opaque data into the file.
* @return true on success, or false on failure.
*/
bool dump_opaque() {
_assert_(true);
bool err = false;
const std::string& opath = path_ + File::PATHCHR + DDBOPAQUEFILE;
if (!File::write_file(opath, opaque_, sizeof(opaque_))) {
set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed");
err = true;
}
return !err;
}
/**
* Load the opaque data from the file.
* @return true on success, or false on failure.
*/
void load_opaque() {
_assert_(true);
std::memset(opaque_, 0, sizeof(opaque_));
const std::string& opath = path_ + File::PATHCHR + DDBOPAQUEFILE;
int64_t size;
char* buf = File::read_file(opath, &size, sizeof(opaque_));
if (buf) {
std::memcpy(opaque_, buf, size);
delete[] buf;
}
}
/**
* Remove inner files. * Remove inner files.
* @param cpath the path of the database file. * @param cpath the path of the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool remove_files(const std::string& cpath) { bool remove_files(const std::string& cpath) {
_assert_(true); _assert_(true);
DirStream dir; DirStream dir;
if (!dir.open(cpath)) { if (!dir.open(cpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "opening a directory fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed");
return false; return false;
} }
bool err = false; bool err = false;
std::string name; std::string name;
while (dir.read(&name)) { while (dir.read(&name)) {
if (*name.c_str() == *DDBMAGICFILE) continue; if (*name.c_str() == *DDBMAGICFILE) continue;
const std::string& rpath = cpath + File::PATHCHR + name; const std::string& rpath = cpath + File::PATHCHR + name;
if (!File::remove(rpath)) { if (!File::remove(rpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a file faile d"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed");
err = true; err = true;
} }
} }
if (!dir.close()) { if (!dir.close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "closing a directory fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Read a record. * Read a record.
* @param rpath the path of the record. * @param rpath the path of the record.
* @param rec the record structure. * @param rec the record structure.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool read_record(const std::string& rpath, Record* rec) { bool read_record(const std::string& rpath, Record* rec) {
_assert_(rec); _assert_(rec);
int64_t rsiz; int64_t rsiz;
char* rbuf = File::read_file(rpath, &rsiz); char* rbuf = File::read_file(rpath, &rsiz);
if (!rbuf) return false; if (!rbuf) return false;
rec->rsiz = rsiz; rec->rsiz = rsiz;
if (comp_) { if (comp_) {
size_t zsiz; size_t zsiz;
char* zbuf = comp_->decompress(rbuf, rsiz, &zsiz); char* zbuf = comp_->decompress(rbuf, rsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "data decompression fa iled"); set_error(_KCCODELINE_, Error::SYSTEM, "data decompression failed") ;
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
delete[] rbuf; delete[] rbuf;
rbuf = zbuf; rbuf = zbuf;
rsiz = zsiz; rsiz = zsiz;
} }
const char* rp = rbuf; const char* rp = rbuf;
if (rsiz < 4 || *(const unsigned char*)rp != DDBRECMAGIC) { if (rsiz < 4 || *(const unsigned char*)rp != DDBRECMAGIC) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid magic data of a set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a recor
record"); d");
report(__FILE__, __LINE__, "info", "rpath=%s", rpath.c_str()); report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str());
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
rp++; rp++;
uint64_t num; uint64_t num;
size_t step = readvarnum(rp, rsiz, &num); size_t step = readvarnum(rp, rsiz, &num);
rp += step; rp += step;
rsiz -= step; rsiz -= step;
size_t ksiz = num; size_t ksiz = num;
if (rsiz < 2) { if (rsiz < 2) {
report(__FILE__, __LINE__, "info", "rpath=%s", rpath.c_str()); report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str());
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
step = readvarnum(rp, rsiz, &num); step = readvarnum(rp, rsiz, &num);
rp += step; rp += step;
rsiz -= step; rsiz -= step;
size_t vsiz = num; size_t vsiz = num;
if (rsiz < 1 + (int64_t)ksiz + (int64_t)vsiz || if (rsiz < 1 + (int64_t)ksiz + (int64_t)vsiz ||
((const unsigned char*)rp)[ksiz+vsiz] != DDBRECMAGIC) { ((const unsigned char*)rp)[ksiz+vsiz] != DDBRECMAGIC) {
set_error(__FILE__, __LINE__, Error::BROKEN, "too short record"); set_error(_KCCODELINE_, Error::BROKEN, "too short record");
report(__FILE__, __LINE__, "info", "rpath=%s", rpath.c_str()); report(_KCCODELINE_, Logger::WARN, "rpath=%s", rpath.c_str());
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
rec->rbuf = rbuf; rec->rbuf = rbuf;
rec->kbuf = rp; rec->kbuf = rp;
rec->ksiz = ksiz; rec->ksiz = ksiz;
rec->vbuf = rp + ksiz; rec->vbuf = rp + ksiz;
rec->vsiz = vsiz; rec->vsiz = vsiz;
return true; return true;
} }
skipping to change at line 1198 skipping to change at line 1695
std::memcpy(wp, kbuf, ksiz); std::memcpy(wp, kbuf, ksiz);
wp += ksiz; wp += ksiz;
std::memcpy(wp, vbuf, vsiz); std::memcpy(wp, vbuf, vsiz);
wp += vsiz; wp += vsiz;
*(wp++) = DDBRECMAGIC; *(wp++) = DDBRECMAGIC;
size_t rsiz = wp - rbuf; size_t rsiz = wp - rbuf;
if (comp_) { if (comp_) {
size_t zsiz; size_t zsiz;
char* zbuf = comp_->compress(rbuf, rsiz, &zsiz); char* zbuf = comp_->compress(rbuf, rsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "data compression fail ed"); set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed");
delete[] rbuf; delete[] rbuf;
*wsp = 0; *wsp = 0;
return false; return false;
} }
delete[] rbuf; delete[] rbuf;
rbuf = zbuf; rbuf = zbuf;
rsiz = zsiz; rsiz = zsiz;
} }
if (autotran_ && !tran_) { if (autotran_ && !tran_) {
const std::string& tpath = path_ + File::PATHCHR + DDBATRANPREFIX + n ame; const std::string& tpath = path_ + File::PATHCHR + DDBATRANPREFIX + n ame;
if (!File::write_file(tpath, rbuf, rsiz)) { if (!File::write_file(tpath, rbuf, rsiz)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "writing a file failed "); set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed");
err = true; err = true;
} }
if (!File::rename(tpath, rpath)) { if (!File::rename(tpath, rpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "renaming a file faile d"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed");
err = true; err = true;
File::remove(tpath); File::remove(tpath);
} }
} else { } else {
if (!File::write_file(rpath, rbuf, rsiz)) { if (!File::write_file(rpath, rbuf, rsiz)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "writing a file failed "); set_error(_KCCODELINE_, Error::SYSTEM, "writing a file failed");
err = true; err = true;
} }
} }
delete[] rbuf; delete[] rbuf;
*wsp = rsiz; *wsp = rsiz;
return !err; return !err;
} }
/** /**
* Disable all cursors. * Disable all cursors.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 1288 skipping to change at line 1785
bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, const c har* name) { bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, const c har* name) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor && name); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor && name);
bool err = false; bool err = false;
const std::string& rpath = path_ + File::PATHCHR + name; const std::string& rpath = path_ + File::PATHCHR + name;
Record rec; Record rec;
if (read_record(rpath, &rec)) { if (read_record(rpath, &rec)) {
if (rec.ksiz == ksiz || !std::memcmp(rec.kbuf, kbuf, ksiz)) { if (rec.ksiz == ksiz || !std::memcmp(rec.kbuf, kbuf, ksiz)) {
if (!accept_visit_full(kbuf, ksiz, rec.vbuf, rec.vsiz, rec.rsiz, if (!accept_visit_full(kbuf, ksiz, rec.vbuf, rec.vsiz, rec.rsiz,
visitor, rpath, name)) err = true; visitor, rpath, name)) err = true;
} else { } else {
set_error(__FILE__, __LINE__, Error::LOGIC, "collision of the hash values"); set_error(_KCCODELINE_, Error::LOGIC, "collision of the hash values ");
err = true; err = true;
} }
delete[] rec.rbuf; delete[] rec.rbuf;
} else { } else {
if (!accept_visit_empty(kbuf, ksiz, visitor, rpath, name)) err = true ; if (!accept_visit_empty(kbuf, ksiz, visitor, rpath, name)) err = true ;
} }
return !err; return !err;
} }
/** /**
* Accept the visit_full method. * Accept the visit_full method.
skipping to change at line 1321 skipping to change at line 1818
const char* name) { const char* name) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && visi tor); _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && visi tor);
bool err = false; bool err = false;
size_t rsiz; size_t rsiz;
const char* rbuf = visitor->visit_full(kbuf, ksiz, vbuf, vsiz, &rsiz); const char* rbuf = visitor->visit_full(kbuf, ksiz, vbuf, vsiz, &rsiz);
if (rbuf == Visitor::REMOVE) { if (rbuf == Visitor::REMOVE) {
if (tran_) { if (tran_) {
const std::string& walpath = walpath_ + File::PATHCHR + name; const std::string& walpath = walpath_ + File::PATHCHR + name;
if (File::status(walpath)) { if (File::status(walpath)) {
if (!File::remove(rpath)) { if (!File::remove(rpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a file f ailed"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed" );
err = true; err = true;
} }
} else if (!File::rename(rpath, walpath)) { } else if (!File::rename(rpath, walpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "renaming a file fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed");
err = true; err = true;
} }
} else { } else {
if (!File::remove(rpath)) { if (!File::remove(rpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a file fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a file failed");
err = true; err = true;
} }
} }
if (!escape_cursors(rpath, name)) err = true; if (!escape_cursors(rpath, name)) err = true;
count_ -= 1; count_ -= 1;
size_ -= osiz; size_ -= osiz;
if (autosync_ && !File::synchronize_whole()) { if (autosync_ && !File::synchronize_whole()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the fil e system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file syst em failed");
err = true; err = true;
} }
} else if (rbuf != Visitor::NOP) { } else if (rbuf != Visitor::NOP) {
if (tran_) { if (tran_) {
const std::string& walpath = walpath_ + File::PATHCHR + name; const std::string& walpath = walpath_ + File::PATHCHR + name;
if (!File::status(walpath) && !File::rename(rpath, walpath)) { if (!File::status(walpath) && !File::rename(rpath, walpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "renaming a file fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed");
err = true; err = true;
} }
} }
size_t wsiz; size_t wsiz;
if (!write_record(rpath, name, kbuf, ksiz, rbuf, rsiz, &wsiz)) err = true; if (!write_record(rpath, name, kbuf, ksiz, rbuf, rsiz, &wsiz)) err = true;
size_ += (int64_t)wsiz - (int64_t)osiz; size_ += (int64_t)wsiz - (int64_t)osiz;
if (autosync_ && !File::synchronize_whole()) { if (autosync_ && !File::synchronize_whole()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the fil e system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file syst em failed");
err = true; err = true;
} }
} }
return !err; return !err;
} }
/** /**
* Accept the visit_empty method. * Accept the visit_empty method.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
skipping to change at line 1378 skipping to change at line 1875
bool accept_visit_empty(const char* kbuf, size_t ksiz, bool accept_visit_empty(const char* kbuf, size_t ksiz,
Visitor *visitor, const std::string& rpath, const char* name) { Visitor *visitor, const std::string& rpath, const char* name) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
bool err = false; bool err = false;
size_t rsiz; size_t rsiz;
const char* rbuf = visitor->visit_empty(kbuf, ksiz, &rsiz); const char* rbuf = visitor->visit_empty(kbuf, ksiz, &rsiz);
if (rbuf != Visitor::NOP && rbuf != Visitor::REMOVE) { if (rbuf != Visitor::NOP && rbuf != Visitor::REMOVE) {
if (tran_) { if (tran_) {
const std::string& walpath = walpath_ + File::PATHCHR + name; const std::string& walpath = walpath_ + File::PATHCHR + name;
if (!File::status(walpath) && !File::write_file(walpath, "", 0)) { if (!File::status(walpath) && !File::write_file(walpath, "", 0)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "renaming a file fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file failed");
err = true; err = true;
} }
} }
size_t wsiz; size_t wsiz;
if (!write_record(rpath, name, kbuf, ksiz, rbuf, rsiz, &wsiz)) err = true; if (!write_record(rpath, name, kbuf, ksiz, rbuf, rsiz, &wsiz)) err = true;
count_ += 1; count_ += 1;
size_ += wsiz; size_ += wsiz;
if (autosync_ && !File::synchronize_whole()) { if (autosync_ && !File::synchronize_whole()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the fil e system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file syst em failed");
err = true; err = true;
} }
} }
return !err; return !err;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param checker a progress checker object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool iterate_impl(Visitor* visitor) { bool iterate_impl(Visitor* visitor, ProgressChecker* checker) {
_assert_(visitor); _assert_(visitor);
DirStream dir; DirStream dir;
if (!dir.open(path_)) { if (!dir.open(path_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "opening a directory fai set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed");
led"); return false;
}
int64_t allcnt = count_;
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
bool err = false; bool err = false;
std::string name; std::string name;
int64_t curcnt = 0;
while (dir.read(&name)) { while (dir.read(&name)) {
if (*name.c_str() == *DDBMAGICFILE) continue; if (*name.c_str() == *DDBMAGICFILE) continue;
const std::string& rpath = path_ + File::PATHCHR + name; const std::string& rpath = path_ + File::PATHCHR + name;
Record rec; Record rec;
if (read_record(rpath, &rec)) { if (read_record(rpath, &rec)) {
if (!accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec. rsiz, if (!accept_visit_full(rec.kbuf, rec.ksiz, rec.vbuf, rec.vsiz, rec. rsiz,
visitor, rpath, name.c_str())) err = true; visitor, rpath, name.c_str())) err = true;
delete[] rec.rbuf; delete[] rec.rbuf;
} else { } else {
set_error(__FILE__, __LINE__, Error::BROKEN, "missing record"); set_error(_KCCODELINE_, Error::BROKEN, "missing record");
err = true;
}
curcnt++;
if (checker && !checker->check("iterate", "processing", curcnt, allcn
t)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true; err = true;
break;
} }
} }
if (checker && !checker->check("iterate", "ending", -1, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
if (!dir.close()) { if (!dir.close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "closing a directory fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @param proc a postprocessor object. * @param proc a postprocessor object.
* @param checker a progress checker object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize_impl(bool hard, FileProcessor* proc) { bool synchronize_impl(bool hard, FileProcessor* proc, ProgressChecker* ch ecker) {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
if (!dump_meta()) err = true; if (writer_) {
if (hard && !File::synchronize_whole()) { if (checker && !checker->check("synchronize", "dumping the magic data
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the file ", -1, -1)) {
system failed"); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true; return false;
}
if (!dump_magic()) err = true;
if (checker && !checker->check("synchronize", "synchronizing the dire
ctory", -1, -1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
if (hard && !File::synchronize_whole()) {
set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file syst
em failed");
err = true;
}
} }
if (proc && !proc->process(path_, count_, size_impl())) { if (proc) {
set_error(__FILE__, __LINE__, Error::LOGIC, "postprocessing failed"); if (checker && !checker->check("synchronize", "running the post proce
err = true; ssor", -1, -1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
if (!proc->process(path_, count_, size_impl())) {
set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed");
err = true;
}
} }
if (!file_.truncate(0)) { if (writer_ && !file_.truncate(0)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Begin transaction. * Begin transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_impl() { bool begin_transaction_impl() {
_assert_(true); _assert_(true);
if (!File::make_directory(walpath_)) { if (!File::make_directory(walpath_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "making a directory fail ed"); set_error(_KCCODELINE_, Error::SYSTEM, "making a directory failed");
return false; return false;
} }
if (trhard_ && !File::synchronize_whole()) { if (trhard_ && !File::synchronize_whole()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the file system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed");
return false; return false;
} }
trcount_ = count_; trcount_ = count_;
trsize_ = size_; trsize_ = size_;
return true; return true;
} }
/** /**
* Commit transaction. * Commit transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool commit_transaction() { bool commit_transaction() {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
if (!File::rename(walpath_, tmppath_)) { if (!File::rename(walpath_, tmppath_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "renaming a directory fa iled"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a directory failed") ;
err = true; err = true;
} }
if (!remove_files(tmppath_)) err = true; if (!remove_files(tmppath_)) err = true;
if (!File::remove_directory(tmppath_)) { if (!File::remove_directory(tmppath_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a directory fa iled"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a directory failed") ;
return false; return false;
} }
if (trhard_ && !File::synchronize_whole()) { if (trhard_ && !File::synchronize_whole()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the file system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Abort transaction. * Abort transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool abort_transaction() { bool abort_transaction() {
_assert_(true); _assert_(true);
skipping to change at line 1509 skipping to change at line 2040
DirStream dir; DirStream dir;
if (dir.open(walpath_)) { if (dir.open(walpath_)) {
std::string name; std::string name;
while (dir.read(&name)) { while (dir.read(&name)) {
const std::string& srcpath = walpath_ + File::PATHCHR + name; const std::string& srcpath = walpath_ + File::PATHCHR + name;
const std::string& destpath = path_ + File::PATHCHR + name; const std::string& destpath = path_ + File::PATHCHR + name;
File::Status sbuf; File::Status sbuf;
if (File::status(srcpath, &sbuf)) { if (File::status(srcpath, &sbuf)) {
if (sbuf.size > 1) { if (sbuf.size > 1) {
if (!File::rename(srcpath, destpath)) { if (!File::rename(srcpath, destpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "renaming a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "renaming a file faile d");
err = true; err = true;
} }
} else { } else {
if (File::remove(destpath) || !File::status(destpath)) { if (File::remove(destpath) || !File::status(destpath)) {
if (!File::remove(srcpath)) { if (!File::remove(srcpath)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a fi le failed"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a file fai led");
err = true; err = true;
} }
} else { } else {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a file failed"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a file faile d");
err = true; err = true;
} }
} }
} else { } else {
set_error(__FILE__, __LINE__, Error::SYSTEM, "checking a file fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "checking a file failed");
err = true; err = true;
} }
} }
if (!dir.close()) { if (!dir.close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "closing a directory f ailed"); set_error(_KCCODELINE_, Error::SYSTEM, "closing a directory failed" );
err = true; err = true;
} }
if (!File::remove_directory(walpath_)) { if (!File::remove_directory(walpath_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "removing a directory failed"); set_error(_KCCODELINE_, Error::SYSTEM, "removing a directory failed ");
err = true; err = true;
} }
} else { } else {
set_error(__FILE__, __LINE__, Error::SYSTEM, "opening a directory fai led"); set_error(_KCCODELINE_, Error::SYSTEM, "opening a directory failed");
err = true; err = true;
} }
count_ = trcount_; count_ = trcount_;
size_ = trsize_; size_ = trsize_;
if (trhard_ && !File::synchronize_whole()) { if (trhard_ && !File::synchronize_whole()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the file system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file system failed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes. * @return the size of the database file in bytes.
*/ */
int64_t size_impl() { int64_t size_impl() {
return size_ + count_ * DDBRECUNITSIZ; return size_ + count_ * DDBRECUNITSIZ;
skipping to change at line 1565 skipping to change at line 2096
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
DirDB(const DirDB&); DirDB(const DirDB&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
DirDB& operator =(const DirDB&); DirDB& operator =(const DirDB&);
/** The method lock. */ /** The method lock. */
SpinRWLock mlock_; SpinRWLock mlock_;
/** The record locks. */ /** The record locks. */
SlottedSpinRWLock<DDBRLOCKSLOT> rlock_; SlottedSpinRWLock<DDBRLOCKSLOT> rlock_;
/** The last happened error. */ /** The last happened error. */
TSD<Error> error_; TSD<Error> error_;
/** The internal error reporter. */ /** The internal logger. */
std::ostream* erstrm_; Logger* logger_;
/** The flag to report all errors. */ /** The kinds of logged messages. */
bool ervbs_; uint32_t logkinds_;
/** The internal meta operation trigger. */
MetaTrigger* mtrigger_;
/** The open mode. */ /** The open mode. */
uint32_t omode_; uint32_t omode_;
/** The flag for writer. */ /** The flag for writer. */
bool writer_; bool writer_;
/** The flag for auto transaction. */ /** The flag for auto transaction. */
bool autotran_; bool autotran_;
/** The flag for auto synchronization. */ /** The flag for auto synchronization. */
bool autosync_; bool autosync_;
/** The file for meta data. */ /** The flag for recovered. */
bool recov_;
/** The flag for reorganized. */
bool reorg_;
/** The file for magic data. */
File file_; File file_;
/** The cursor objects. */ /** The cursor objects. */
CursorList curs_; CursorList curs_;
/** The path of the database file. */ /** The path of the database file. */
std::string path_; std::string path_;
/** The library version. */
uint8_t libver_;
/** The library revision. */
uint8_t librev_;
/** The format revision. */
uint8_t fmtver_;
/** The module checksum. */
uint8_t chksum_;
/** The database type. */
uint8_t type_;
/** The status flags. */
uint8_t flags_;
/** The options. */ /** The options. */
uint8_t opts_; uint8_t opts_;
/** The record number. */ /** The record number. */
AtomicInt64 count_; AtomicInt64 count_;
/** The total size of records. */ /** The total size of records. */
AtomicInt64 size_; AtomicInt64 size_;
/** The opaque data. */
char opaque_[DDBOPAQUESIZ];
/** The embedded data compressor. */ /** The embedded data compressor. */
Compressor* embcomp_; Compressor* embcomp_;
/** The data compressor. */ /** The data compressor. */
Compressor* comp_; Compressor* comp_;
/** The compression checksum. */ /** The compression checksum. */
uint32_t compchk_;
/** The flag whether in transaction. */
bool tran_; bool tran_;
/** The flag whether hard transaction. */ /** The flag whether hard transaction. */
bool trhard_; bool trhard_;
/** The old count before transaction. */ /** The old count before transaction. */
int64_t trcount_; int64_t trcount_;
/** The old size before transaction. */ /** The old size before transaction. */
int64_t trsize_; int64_t trsize_;
/** The WAL directory for transaction. */ /** The WAL directory for transaction. */
std::string walpath_; std::string walpath_;
/** The temporary directory. */ /** The temporary directory. */
std::string tmppath_; std::string tmppath_;
}; };
/** An alias of the directory tree database. */
typedef PlantDB<DirDB, BasicDB::TYPEFOREST> ForestDB;
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 200 change blocks. 
273 lines changed or deleted 839 lines changed or added


 kcfile.h   kcfile.h 
skipping to change at line 29 skipping to change at line 29
#include <kcutil.h> #include <kcutil.h>
#include <kcthread.h> #include <kcthread.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Filesystem abstraction. * Filesystem abstraction.
*/ */
class File { class File {
public: public:
struct Status;
public:
/** Path delimiter character. */ /** Path delimiter character. */
static const char PATHCHR; static const char PATHCHR;
/** Path delimiter string. */ /** Path delimiter string. */
static const char* const PATHSTR; static const char* const PATHSTR;
/** Extension delimiter character. */ /** Extension delimiter character. */
static const char EXTCHR; static const char EXTCHR;
/** Extension delimiter string. */ /** Extension delimiter string. */
static const char* const EXTSTR; static const char* const EXTSTR;
/** Current directory string. */ /** Current directory string. */
static const char* const CDIRSTR; static const char* const CDIRSTR;
skipping to change at line 308 skipping to change at line 310
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
static bool make_directory(const std::string& path); static bool make_directory(const std::string& path);
/** /**
* Remove a directory. * Remove a directory.
* @param path the path of a directory. * @param path the path of a directory.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
static bool remove_directory(const std::string& path); static bool remove_directory(const std::string& path);
/** /**
* Remove a file or a directory recursively.
* @param path the path of a file or a directory.
* @return true on success, or false on failure.
*/
static bool remove_recursively(const std::string& path);
/**
* Get the path of the current working directory. * Get the path of the current working directory.
* @return the path of the current working directory, or an empty string on failure. * @return the path of the current working directory, or an empty string on failure.
*/ */
static std::string get_current_directory(); static std::string get_current_directory();
/** /**
* Set the current working directory. * Set the current working directory.
* @param path the path of a directory. * @param path the path of a directory.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
static bool set_current_directory(const std::string& path); static bool set_current_directory(const std::string& path);
 End of changes. 2 change blocks. 
0 lines changed or deleted 8 lines changed or added


 kchashdb.h   kchashdb.h 
skipping to change at line 20 skipping to change at line 20
* See the GNU General Public License for more details. * See the GNU General Public License for more details.
* You should have received a copy of the GNU General Public License along with this program. * You should have received a copy of the GNU General Public License along with this program.
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
************************************************************************** ***********************/ ************************************************************************** ***********************/
#ifndef _KCHASHDB_H // duplication check #ifndef _KCHASHDB_H // duplication check
#define _KCHASHDB_H #define _KCHASHDB_H
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
#include <kcdb.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h>
#include <kcdb.h>
#include <kcplantdb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation. * Constants for implementation.
*/ */
namespace { namespace {
const char HDBMAGICDATA[] = "KC\n"; ///< magic data of the file const char HDBMAGICDATA[] = "KC\n"; ///< magic data of the file
const char HDBCHKSUMSEED[] = "__kyotocabinet__"; ///< seed of the module c hecksum const char HDBCHKSUMSEED[] = "__kyotocabinet__"; ///< seed of the module c hecksum
const int64_t HDBMOFFLIBVER = 4; ///< offset of the library version const int64_t HDBMOFFLIBVER = 4; ///< offset of the library version
skipping to change at line 66 skipping to change at line 68
const uint8_t HDBMAXAPOW = 15; ///< maximum alignment power const uint8_t HDBMAXAPOW = 15; ///< maximum alignment power
const uint8_t HDBDEFFPOW = 10; ///< default free block pool power const uint8_t HDBDEFFPOW = 10; ///< default free block pool power
const uint8_t HDBMAXFPOW = 20; ///< maximum free block pool power const uint8_t HDBMAXFPOW = 20; ///< maximum free block pool power
const int64_t HDBDEFBNUM = 1048583LL; ///< default bucket number const int64_t HDBDEFBNUM = 1048583LL; ///< default bucket number
const int64_t HDBDEFMSIZ = 64LL << 20; ///< default size of the memory-ma pped region const int64_t HDBDEFMSIZ = 64LL << 20; ///< default size of the memory-ma pped region
const uint8_t HDBRECMAGIC = 0xcc; ///< magic data for record const uint8_t HDBRECMAGIC = 0xcc; ///< magic data for record
const uint8_t HDBPADMAGIC = 0xee; ///< magic data for padding const uint8_t HDBPADMAGIC = 0xee; ///< magic data for padding
const uint8_t HDBFBMAGIC = 0xdd; ///< magic data for free block const uint8_t HDBFBMAGIC = 0xdd; ///< magic data for free block
const int32_t HDBDFRGMAX = 512; ///< maximum unit of auto defragme ntation const int32_t HDBDFRGMAX = 512; ///< maximum unit of auto defragme ntation
const int32_t HDBDFRGCEF = 2; ///< coefficient of auto defragmen tation const int32_t HDBDFRGCEF = 2; ///< coefficient of auto defragmen tation
const int64_t HDBSLVGWIDTH = 1LL << 20; ///< checking width for record sal vage
const char* HDBTMPPATHEXT = "tmpkch"; ///< extension of the temporary fi le const char* HDBTMPPATHEXT = "tmpkch"; ///< extension of the temporary fi le
} }
/** /**
* File hash database. * File hash database.
* @note This class is a concrete class to operate a hash database on a fil e. This class can be * @note This class is a concrete class to operate a hash database on a fil e. This class can be
* inherited but overwriting methods is forbidden. Before every database o peration, it is * inherited but overwriting methods is forbidden. Before every database o peration, it is
* necessary to call the HashDB::open method in order to open a database fi le and connect the * necessary to call the HashDB::open method in order to open a database fi le and connect the
* database object to it. To avoid data missing or corruption, it is impor tant to close every * database object to it. To avoid data missing or corruption, it is impor tant to close every
* database file by the HashDB::close method when the database is no longer in use. It is * database file by the HashDB::close method when the database is no longer in use. It is
* forbidden for multible database objects in a process to open the same da tabase at the same * forbidden for multible database objects in a process to open the same da tabase at the same
* time. * time. It is forbidden to share a database object with child processes.
*/ */
class HashDB : public FileDB { class HashDB : public BasicDB {
friend class PlantDB<HashDB, BasicDB::TYPETREE>;
public: public:
class Cursor; class Cursor;
private: private:
struct Record; struct Record;
struct FreeBlock; struct FreeBlock;
struct FreeBlockComparator; struct FreeBlockComparator;
class Repeater; class Repeater;
class Transactor;
friend class TreeDB;
/** An alias of set of free blocks. */ /** An alias of set of free blocks. */
typedef std::set<FreeBlock> FBP; typedef std::set<FreeBlock> FBP;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public FileDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class HashDB; friend class HashDB;
public: public:
/** /**
* Constructor. * Constructor.
* @param db the container database object. * @param db the container database object.
*/ */
explicit Cursor(HashDB* db) : db_(db), off_(0), end_(0) { explicit Cursor(HashDB* db) : db_(db), off_(0), end_(0) {
_assert_(db); _assert_(db);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
db_->curs_.push_back(this); db_->curs_.push_back(this);
skipping to change at line 124 skipping to change at line 126
if (!db_) return; if (!db_) return;
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
db_->curs_.remove(this); db_->curs_.remove(this);
} }
/** /**
* Accept a visitor to the current record. * Accept a visitor to the current record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only operation. * @param writable true for writable operation, or false for read-only operation.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and othe * @note The operation for each record is performed atomically and othe
r threads accessing r threads accessing
* the same record are blocked. * the same record are blocked. To avoid deadlock, any database operat
ion must not be
* performed in this function.
*/ */
bool accept(Visitor* visitor, bool writable = true, bool step = false) { bool accept(Visitor* visitor, bool writable = true, bool step = false) {
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(db_->writer_)) { if (writable && !(db_->writer_)) {
db_->set_error(__FILE__, __LINE__, Error::NOPERM, "permission denie d"); db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
if (off_ < 1) { if (off_ < 1) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[HDBRECBUFSIZ];
if (!step_impl(&rec, rbuf, 0)) return false; if (!step_impl(&rec, rbuf, 0)) return false;
if (!rec.vbuf && !db_->read_record_body(&rec)) { if (!rec.vbuf && !db_->read_record_body(&rec)) {
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
const char* vbuf = rec.vbuf; const char* vbuf = rec.vbuf;
size_t vsiz = rec.vsiz; size_t vsiz = rec.vsiz;
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (db_->comp_) { if (db_->comp_) {
zbuf = db_->comp_->decompress(vbuf, vsiz, &zsiz); zbuf = db_->comp_->decompress(vbuf, vsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
db_->set_error(__FILE__, __LINE__, Error::SYSTEM, "data decompres sion failed"); db_->set_error(_KCCODELINE_, Error::SYSTEM, "data decompression f ailed");
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
vbuf = zbuf; vbuf = zbuf;
vsiz = zsiz; vsiz = zsiz;
} }
vbuf = visitor->visit_full(rec.kbuf, rec.ksiz, vbuf, vsiz, &vsiz); vbuf = visitor->visit_full(rec.kbuf, rec.ksiz, vbuf, vsiz, &vsiz);
delete[] zbuf; delete[] zbuf;
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
uint64_t hash = db_->hash_record(rec.kbuf, rec.ksiz); uint64_t hash = db_->hash_record(rec.kbuf, rec.ksiz);
skipping to change at line 190 skipping to change at line 193
} else if (db_->error().code() != Error::NOREC) { } else if (db_->error().code() != Error::NOREC) {
return false; return false;
} }
} }
} else { } else {
zbuf = NULL; zbuf = NULL;
zsiz = 0; zsiz = 0;
if (db_->comp_) { if (db_->comp_) {
zbuf = db_->comp_->compress(vbuf, vsiz, &zsiz); zbuf = db_->comp_->compress(vbuf, vsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
db_->set_error(__FILE__, __LINE__, Error::SYSTEM, "data compres sion failed"); db_->set_error(_KCCODELINE_, Error::SYSTEM, "data compression f ailed");
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
vbuf = zbuf; vbuf = zbuf;
vsiz = zsiz; vsiz = zsiz;
} }
size_t rsiz = db_->calc_record_size(rec.ksiz, vsiz); size_t rsiz = db_->calc_record_size(rec.ksiz, vsiz);
if (rsiz <= rec.rsiz) { if (rsiz <= rec.rsiz) {
rec.psiz = rec.rsiz - rsiz; rec.psiz = rec.rsiz - rsiz;
rec.vsiz = vsiz; rec.vsiz = vsiz;
skipping to change at line 237 skipping to change at line 240
delete[] rec.bbuf; delete[] rec.bbuf;
} }
} }
if (db_->dfunit_ > 0 && db_->frgcnt_ >= db_->dfunit_) { if (db_->dfunit_ > 0 && db_->frgcnt_ >= db_->dfunit_) {
if (!db_->defrag_impl(db_->dfunit_ * HDBDFRGCEF)) return false; if (!db_->defrag_impl(db_->dfunit_ * HDBDFRGCEF)) return false;
db_->frgcnt_ -= db_->dfunit_; db_->frgcnt_ -= db_->dfunit_;
} }
return true; return true;
} }
/** /**
* Jump the cursor to the first record. * Jump the cursor to the first record for forward scan.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump() { bool jump() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
off_ = 0; off_ = 0;
if (db_->lsiz_ <= db_->roff_) { if (db_->lsiz_ <= db_->roff_) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
off_ = db_->roff_; off_ = db_->roff_;
end_ = db_->lsiz_; end_ = db_->lsiz_;
return true; return true;
} }
/** /**
* Jump the cursor onto a record. * Jump the cursor to a record for forward scan.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump(const char* kbuf, size_t ksiz) { bool jump(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
off_ = 0; off_ = 0;
uint64_t hash = db_->hash_record(kbuf, ksiz); uint64_t hash = db_->hash_record(kbuf, ksiz);
uint32_t pivot = db_->fold_hash(hash); uint32_t pivot = db_->fold_hash(hash);
int64_t bidx = hash % db_->bnum_; int64_t bidx = hash % db_->bnum_;
int64_t off = db_->get_bucket(bidx); int64_t off = db_->get_bucket(bidx);
if (off < 0) return false; if (off < 0) return false;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[HDBRECBUFSIZ];
while (off > 0) { while (off > 0) {
rec.off = off; rec.off = off;
if (!db_->read_record(&rec, rbuf)) return false; if (!db_->read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16_MAX) { if (rec.psiz == UINT16_MAX) {
db_->set_error(__FILE__, __LINE__, Error::BROKEN, "free block in db_->set_error(_KCCODELINE_, Error::BROKEN, "free block in the ch
the chain"); ain");
db_->report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%l db_->report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=
d", %lld",
(long)db_->psiz_, (long)rec.off, (long)db_->file_.siz (long long)db_->psiz_, (long long)rec.off, (long long
e()); )db_->file_.size());
return false; return false;
} }
uint32_t tpivot = db_->linear_ ? pivot : uint32_t tpivot = db_->linear_ ? pivot :
db_->fold_hash(db_->hash_record(rec.kbuf, rec.ksiz)); db_->fold_hash(db_->hash_record(rec.kbuf, rec.ksiz));
if (pivot > tpivot) { if (pivot > tpivot) {
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.left; off = rec.left;
} else if (pivot < tpivot) { } else if (pivot < tpivot) {
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.right; off = rec.right;
skipping to change at line 311 skipping to change at line 314
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.right; off = rec.right;
} else { } else {
delete[] rec.bbuf; delete[] rec.bbuf;
off_ = off; off_ = off;
end_ = db_->lsiz_; end_ = db_->lsiz_;
return true; return true;
} }
} }
} }
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
/** /**
* Jump the cursor to a record. * Jump the cursor to a record for forward scan.
* @note Equal to the original Cursor::jump method except that the para meter is std::string. * @note Equal to the original Cursor::jump method except that the para meter is std::string.
*/ */
bool jump(const std::string& key) { bool jump(const std::string& key) {
_assert_(true); _assert_(true);
return jump(key.c_str(), key.size()); return jump(key.c_str(), key.size());
} }
/** /**
* Jump the cursor to the last record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back() {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Jump the cursor to a record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Jump the cursor to a record for backward scan.
* @note This is a dummy implementation for compatibility.
*/
bool jump_back(const std::string& key) {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Step the cursor to the next record. * Step the cursor to the next record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool step() { bool step() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (off_ < 1) { if (off_ < 1) {
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
bool err = false; bool err = false;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[HDBRECBUFSIZ];
if (step_impl(&rec, rbuf, 1)) { if (step_impl(&rec, rbuf, 1)) {
delete[] rec.bbuf; delete[] rec.bbuf;
} else { } else {
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Step the cursor to the previous record.
* @note This is a dummy implementation for compatibility.
*/
bool step_back() {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
/**
* Get the database object. * Get the database object.
* @return the database object. * @return the database object.
*/ */
HashDB* db() { HashDB* db() {
_assert_(true); _assert_(true);
return db_; return db_;
} }
private: private:
/** /**
* Step the cursor to the next record. * Step the cursor to the next record.
* @param rec the record structure. * @param rec the record structure.
* @param rbuf the working buffer. * @param rbuf the working buffer.
* @param skip the number of skipping blocks. * @param skip the number of skipping blocks.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool step_impl(Record* rec, char* rbuf, int64_t skip) { bool step_impl(Record* rec, char* rbuf, int64_t skip) {
_assert_(rec && rbuf && skip >= 0); _assert_(rec && rbuf && skip >= 0);
if (off_ >= end_) { if (off_ >= end_) {
db_->set_error(__FILE__, __LINE__, Error::BROKEN, "cursor after the db_->set_error(_KCCODELINE_, Error::BROKEN, "cursor after the end")
end"); ;
db_->report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld" db_->report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%l
, ld",
(long)db_->psiz_, (long)rec->off, (long)db_->file_.size (long long)db_->psiz_, (long long)rec->off, (long long)
()); db_->file_.size());
return false; return false;
} }
while (off_ < end_) { while (off_ < end_) {
rec->off = off_; rec->off = off_;
if (!db_->read_record(rec, rbuf)) return false; if (!db_->read_record(rec, rbuf)) return false;
skip--; skip--;
if (rec->psiz == UINT16_MAX) { if (rec->psiz == UINT16_MAX) {
off_ += rec->rsiz; off_ += rec->rsiz;
} else { } else {
if (skip < 0) return true; if (skip < 0) return true;
delete[] rec->bbuf; delete[] rec->bbuf;
off_ += rec->rsiz; off_ += rec->rsiz;
} }
} }
db_->set_error(__FILE__, __LINE__, Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
off_ = 0; off_ = 0;
return false; return false;
} }
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
Cursor(const Cursor&); Cursor(const Cursor&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
Cursor& operator =(const Cursor&); Cursor& operator =(const Cursor&);
/** The inner database. */ /** The inner database. */
HashDB* db_; HashDB* db_;
/** The current offset. */ /** The current offset. */
int64_t off_; int64_t off_;
/** The end offset. */ /** The end offset. */
int64_t end_; int64_t end_;
}; };
/** /**
* Tuning Options. * Tuning options.
*/ */
enum Option { enum Option {
TSMALL = 1 << 0, ///< use 32-bit addressing TSMALL = 1 << 0, ///< use 32-bit addressing
TLINEAR = 1 << 1, ///< use linear collision chaining TLINEAR = 1 << 1, ///< use linear collision chaining
TCOMPRESS = 1 << 2 ///< compress each record TCOMPRESS = 1 << 2 ///< compress each record
}; };
/** /**
* Status flags. * Status flags.
*/ */
enum Flag { enum Flag {
FOPEN = 1 << 0, ///< whether opened FOPEN = 1 << 0, ///< whether opened
FFATAL = 1 << 1 ///< whether with fatal error FFATAL = 1 << 1 ///< whether with fatal error
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit HashDB() : explicit HashDB() :
mlock_(), rlock_(), flock_(), atlock_(), error_(), erstrm_(NULL), ervbs mlock_(), rlock_(), flock_(), atlock_(), error_(),
_(false), logger_(NULL), logkinds_(0), mtrigger_(NULL),
omode_(0), writer_(false), autotran_(false), autosync_(false), reorg_(f alse), trim_(false), omode_(0), writer_(false), autotran_(false), autosync_(false), reorg_(f alse), trim_(false),
file_(), fbp_(), curs_(), path_(""), file_(), fbp_(), curs_(), path_(""),
libver_(LIBVER), librev_(LIBREV), fmtver_(FMTVER), chksum_(0), type_(TY PEHASH), libver_(0), librev_(0), fmtver_(0), chksum_(0), type_(TYPEHASH),
apow_(HDBDEFAPOW), fpow_(HDBDEFFPOW), opts_(0), bnum_(HDBDEFBNUM), apow_(HDBDEFAPOW), fpow_(HDBDEFFPOW), opts_(0), bnum_(HDBDEFBNUM),
flags_(0), flagopen_(false), count_(0), lsiz_(0), psiz_(0), opaque_(), flags_(0), flagopen_(false), count_(0), lsiz_(0), psiz_(0), opaque_(),
msiz_(HDBDEFMSIZ), dfunit_(0), embcomp_(&ZLIBRAWCOMP), msiz_(HDBDEFMSIZ), dfunit_(0), embcomp_(&ZLIBRAWCOMP),
align_(0), fbpnum_(0), width_(0), linear_(false), align_(0), fbpnum_(0), width_(0), linear_(false),
comp_(NULL), rhsiz_(0), boff_(0), roff_(0), dfcur_(0), frgcnt_(0), comp_(NULL), rhsiz_(0), boff_(0), roff_(0), dfcur_(0), frgcnt_(0),
tran_(false), trhard_(false), trfbp_() { tran_(false), trhard_(false), trfbp_(), trcount_(0), trsize_(0) {
_assert_(true); _assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~HashDB() { virtual ~HashDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
if (!curs_.empty()) { if (!curs_.empty()) {
skipping to change at line 453 skipping to change at line 513
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and other * @note The operation for each record is performed atomically and other
threads accessing the threads accessing the
* same record are blocked. * same record are blocked. To avoid deadlock, any database operation mu
st not be performed in
* this function.
*/ */
bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) { bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
mlock_.lock_reader(); mlock_.lock_reader();
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
uint64_t hash = hash_record(kbuf, ksiz); uint64_t hash = hash_record(kbuf, ksiz);
uint32_t pivot = fold_hash(hash); uint32_t pivot = fold_hash(hash);
int64_t bidx = hash % bnum_; int64_t bidx = hash % bnum_;
size_t lidx = bidx % HDBRLOCKSLOT; size_t lidx = bidx % HDBRLOCKSLOT;
if (writable) { if (writable) {
rlock_.lock_writer(lidx); rlock_.lock_writer(lidx);
} else { } else {
skipping to change at line 496 skipping to change at line 557
frgcnt_ -= unit; frgcnt_ -= unit;
} }
} }
mlock_.unlock(); mlock_.unlock();
return !err; return !err;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the whole iteration is performed atomically and other threads ar * @note The whole iteration is performed atomically and other threads ar
e blocked. e blocked. To avoid
* deadlock, any database operation must not be performed in this functio
n.
*/ */
bool iterate(Visitor *visitor, bool writable = true) { bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* che cker = NULL) {
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !writer_) { if (writable && !writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
bool err = false; bool err = false;
if (!iterate_impl(visitor)) err = true; if (!iterate_impl(visitor, checker)) err = true;
trigger_meta(MetaTrigger::ITERATE, "iterate");
return !err; return !err;
} }
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
Error error() const { Error error() const {
_assert_(true); _assert_(true);
return error_; return error_;
} }
/** /**
* Set the error information. * Set the error information.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param code an error code. * @param code an error code.
* @param message a supplement message. * @param message a supplement message.
*/ */
void set_error(Error::Code code, const char* message) { void set_error(const char* file, int32_t line, const char* func,
_assert_(message); Error::Code code, const char* message) {
_assert_(file && line > 0 && func && message);
error_->set(code, message); error_->set(code, message);
if (code == Error::BROKEN || code == Error::SYSTEM) flags_ |= FFATAL; if (code == Error::BROKEN || code == Error::SYSTEM) flags_ |= FFATAL;
if (logger_) {
Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ?
Logger::ERROR : Logger::INFO;
if (kind & logkinds_)
report(file, line, func, kind, "%d: %s: %s", code, Error::codename(
code), message);
}
} }
/** /**
* Open a database file. * Open a database file.
* @param path the path of a database file. * @param path the path of a database file.
* @param mode the connection mode. HashDB::OWRITER as a writer, HashDB: :OREADER as a * @param mode the connection mode. HashDB::OWRITER as a writer, HashDB: :OREADER as a
* reader. The following may be added to the writer mode by bitwise-or: HashDB::OCREATE, * reader. The following may be added to the writer mode by bitwise-or: HashDB::OCREATE,
* which means it creates a new database if the file does not exist, Hash DB::OTRUNCATE, which * which means it creates a new database if the file does not exist, Hash DB::OTRUNCATE, which
* means it creates a new database regardless if the file exists, HashDB: :OAUTOTRAN, which * means it creates a new database regardless if the file exists, HashDB: :OAUTOTRAN, which
* means each updating operation is performed in implicit transaction, Ha shDB::OAUTOSYNC, * means each updating operation is performed in implicit transaction, Ha shDB::OAUTOSYNC,
* which means each updating operation is followed by implicit synchroniz ation with the file * which means each updating operation is followed by implicit synchroniz ation with the file
skipping to change at line 555 skipping to change at line 629
* detected. * detected.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note Every opened database must be closed by the HashDB::close method when it is no * @note Every opened database must be closed by the HashDB::close method when it is no
* longer in use. It is not allowed for two or more database objects in the same process to * longer in use. It is not allowed for two or more database objects in the same process to
* keep their connections to the same database file at the same time. * keep their connections to the same database file at the same time.
*/ */
bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", p ath.c_str());
writer_ = false; writer_ = false;
autotran_ = false; autotran_ = false;
autosync_ = false; autosync_ = false;
reorg_ = false; reorg_ = false;
trim_ = false; trim_ = false;
uint32_t fmode = File::OREADER; uint32_t fmode = File::OREADER;
if (mode & OWRITER) { if (mode & OWRITER) {
writer_ = true; writer_ = true;
fmode = File::OWRITER; fmode = File::OWRITER;
if (mode & OCREATE) fmode |= File::OCREATE; if (mode & OCREATE) fmode |= File::OCREATE;
skipping to change at line 580 skipping to change at line 655
if (mode & OAUTOSYNC) autosync_ = true; if (mode & OAUTOSYNC) autosync_ = true;
} }
if (mode & ONOLOCK) fmode |= File::ONOLOCK; if (mode & ONOLOCK) fmode |= File::ONOLOCK;
if (mode & OTRYLOCK) fmode |= File::OTRYLOCK; if (mode & OTRYLOCK) fmode |= File::OTRYLOCK;
if (!file_.open(path, fmode, msiz_)) { if (!file_.open(path, fmode, msiz_)) {
const char* emsg = file_.error(); const char* emsg = file_.error();
Error::Code code = Error::SYSTEM; Error::Code code = Error::SYSTEM;
if (std::strstr(emsg, "(permission denied)") || std::strstr(emsg, "(d irectory)")) { if (std::strstr(emsg, "(permission denied)") || std::strstr(emsg, "(d irectory)")) {
code = Error::NOPERM; code = Error::NOPERM;
} else if (std::strstr(emsg, "(file not found)") || std::strstr(emsg, "(invalid path)")) { } else if (std::strstr(emsg, "(file not found)") || std::strstr(emsg, "(invalid path)")) {
code = Error::NOFILE; code = Error::NOREPOS;
} }
set_error(__FILE__, __LINE__, code, emsg); set_error(_KCCODELINE_, code, emsg);
return false; return false;
} }
if (file_.recovered()) report(__FILE__, __LINE__, "info", "recovered by the WAL file"); if (file_.recovered()) report(_KCCODELINE_, Logger::WARN, "recovered by the WAL file");
if ((mode & OWRITER) && file_.size() < 1) { if ((mode & OWRITER) && file_.size() < 1) {
calc_meta(); calc_meta();
libver_ = LIBVER;
librev_ = LIBREV;
fmtver_ = FMTVER;
chksum_ = calc_checksum(); chksum_ = calc_checksum();
lsiz_ = roff_; lsiz_ = roff_;
if (!file_.truncate(lsiz_)) { if (!file_.truncate(lsiz_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
file_.close(); file_.close();
return false; return false;
} }
if (!dump_meta()) { if (!dump_meta()) {
file_.close(); file_.close();
return false; return false;
} }
if (autosync_ && !File::synchronize_whole()) { if (autosync_ && !File::synchronize_whole()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "synchronizing the fil e system failed"); set_error(_KCCODELINE_, Error::SYSTEM, "synchronizing the file syst em failed");
file_.close(); file_.close();
return false; return false;
} }
} }
if (!load_meta()) { if (!load_meta()) {
file_.close(); file_.close();
return false; return false;
} }
calc_meta(); calc_meta();
uint8_t chksum = calc_checksum(); uint8_t chksum = calc_checksum();
if (chksum != chksum_) { if (chksum != chksum_) {
set_error(__FILE__, __LINE__, Error::INVALID, "invalid module checksu set_error(_KCCODELINE_, Error::INVALID, "invalid module checksum");
m"); report(_KCCODELINE_, Logger::WARN, "saved=%02X calculated=%02X",
report(__FILE__, __LINE__, "info", "saved=%02X calculated=%02X",
(unsigned)chksum_, (unsigned)chksum); (unsigned)chksum_, (unsigned)chksum);
file_.close(); file_.close();
return false; return false;
} }
if (((flags_ & FOPEN) || (flags_ & FFATAL)) && !(mode & ONOREPAIR) && ! if (((flags_ & FOPEN) || (flags_ & FFATAL)) && !(mode & ONOREPAIR) && !
(mode & ONOLOCK) && (mode & ONOLOCK)) {
!reorganize_file(path)) { if (!reorganize_file(path)) {
file_.close(); file_.close();
return false; return false;
}
if (!file_.close()) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false;
}
if (!file_.open(path, fmode, msiz_)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false;
}
if (!load_meta()) {
file_.close();
return false;
}
calc_meta();
reorg_ = true;
} }
if (type_ == 0 || apow_ > HDBMAXAPOW || fpow_ > HDBMAXFPOW || if (type_ == 0 || apow_ > HDBMAXAPOW || fpow_ > HDBMAXFPOW ||
bnum_ < 1 || count_ < 0 || lsiz_ < roff_) { bnum_ < 1 || count_ < 0 || lsiz_ < roff_) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid meta data"); set_error(_KCCODELINE_, Error::BROKEN, "invalid meta data");
report(__FILE__, __LINE__, "info", "type=0x%02X apow=%d fpow=%d bnum= report(_KCCODELINE_, Logger::WARN, "type=0x%02X apow=%d fpow=%d bnum=
%ld count=%ld" %lld count=%lld"
" lsiz=%ld fsiz=%ld", (unsigned)type_, (int)apow_, (int)fpow_, " lsiz=%lld fsiz=%lld", (unsigned)type_, (int)apow_, (int)fpow
(long)bnum_, _, (long long)bnum_,
(long)count_, (long)lsiz_, (long)file_.size()); (long long)count_, (long long)lsiz_, (long long)file_.size());
file_.close(); file_.close();
return false; return false;
} }
if (file_.size() < lsiz_) { if (file_.size() < lsiz_) {
set_error(__FILE__, __LINE__, Error::BROKEN, "inconsistent file size" set_error(_KCCODELINE_, Error::BROKEN, "inconsistent file size");
); report(_KCCODELINE_, Logger::WARN, "lsiz=%lld fsiz=%lld",
report(__FILE__, __LINE__, "info", "lsiz=%ld fsiz=%ld", (long)lsiz_, (long long)lsiz_, (long long)file_.size());
(long)file_.size());
file_.close(); file_.close();
return false; return false;
} }
if (file_.size() != lsiz_ && !(mode & ONOREPAIR) && !(mode & ONOLOCK) & & !trim_file(path)) { if (file_.size() != lsiz_ && !(mode & ONOREPAIR) && !(mode & ONOLOCK) & & !trim_file(path)) {
file_.close(); file_.close();
return false; return false;
} }
if (mode & OWRITER) { if (mode & OWRITER) {
if (!(flags_ & FOPEN) && !(flags_ & FFATAL) && !load_free_blocks()) { if (!(flags_ & FOPEN) && !(flags_ & FFATAL) && !load_free_blocks()) {
file_.close(); file_.close();
skipping to change at line 658 skipping to change at line 752
file_.close(); file_.close();
return false; return false;
} }
if (!autotran_ && !set_flag(FOPEN, true)) { if (!autotran_ && !set_flag(FOPEN, true)) {
file_.close(); file_.close();
return false; return false;
} }
} }
path_.append(path); path_.append(path);
omode_ = mode; omode_ = mode;
trigger_meta(MetaTrigger::OPEN, "open");
return true; return true;
} }
/** /**
* Close the database file. * Close the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool close() { bool close() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", p ath_.c_str());
bool err = false; bool err = false;
if (tran_ && !abort_transaction()) err = true; if (tran_ && !abort_transaction()) err = true;
disable_cursors(); disable_cursors();
if (writer_) { if (writer_) {
if (!dump_free_blocks()) err = true; if (!dump_free_blocks()) err = true;
if (!dump_meta()) err = true; if (!dump_meta()) err = true;
} }
if (!file_.close()) { if (!file_.close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
fbp_.clear(); fbp_.clear();
omode_ = 0; omode_ = 0;
path_.clear(); path_.clear();
trigger_meta(MetaTrigger::CLOSE, "close");
return !err; return !err;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @param proc a postprocessor object. If it is NULL, no postprocessing is performed. * @param proc a postprocessor object. If it is NULL, no postprocessing is performed.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize(bool hard = false, FileProcessor* proc = NULL) { bool synchronize(bool hard = false, FileProcessor* proc = NULL,
ProgressChecker* checker = NULL) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied");
return false; return false;
} }
rlock_.lock_reader_all(); rlock_.lock_reader_all();
bool err = false; bool err = false;
if (!synchronize_impl(hard, proc)) err = true; if (!synchronize_impl(hard, proc, checker)) err = true;
trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize");
rlock_.unlock_all(); rlock_.unlock_all();
return !err; return !err;
} }
/** /**
* Begin transaction. * Begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction(bool hard = false) { bool begin_transaction(bool hard = false) {
_assert_(true); _assert_(true);
for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) { for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) {
mlock_.lock_writer(); mlock_.lock_writer();
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wsec > 1.0) wsec = 1.0; if (wsec > 1.0) wsec = 1.0;
Thread::sleep(wsec); Thread::sleep(wsec);
} }
trhard_ = hard; trhard_ = hard;
if (!begin_transaction_impl()) { if (!begin_transaction_impl()) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
tran_ = true; tran_ = true;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction");
mlock_.unlock(); mlock_.unlock();
return true; return true;
} }
/** /**
* Try to begin transaction. * Try to begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_try(bool hard = false) { bool begin_transaction_try(bool hard = false) {
_assert_(true); _assert_(true);
mlock_.lock_writer(); mlock_.lock_writer();
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (tran_) { if (tran_) {
set_error(__FILE__, __LINE__, Error::LOGIC, "competition avoided"); set_error(_KCCODELINE_, Error::LOGIC, "competition avoided");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
trhard_ = hard; trhard_ = hard;
if (!begin_transaction_impl()) { if (!begin_transaction_impl()) {
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
tran_ = true; tran_ = true;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try");
mlock_.unlock(); mlock_.unlock();
return true; return true;
} }
/** /**
* End transaction. * End transaction.
* @param commit true to commit the transaction, or false to abort the tr ansaction. * @param commit true to commit the transaction, or false to abort the tr ansaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool end_transaction(bool commit = true) { bool end_transaction(bool commit = true) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!tran_) { if (!tran_) {
set_error(__FILE__, __LINE__, Error::INVALID, "not in transaction"); set_error(_KCCODELINE_, Error::INVALID, "not in transaction");
return false; return false;
} }
bool err = false; bool err = false;
if (commit) { if (commit) {
if (!commit_transaction()) err = true; if (!commit_transaction()) err = true;
} else { } else {
if (!abort_transaction()) err = true; if (!abort_transaction()) err = true;
} }
tran_ = false; tran_ = false;
trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction");
return !err; return !err;
} }
/** /**
* Remove all records. * Remove all records.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clear() { bool clear() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
disable_cursors(); disable_cursors();
if (!file_.truncate(HDBHEADSIZ)) { if (!file_.truncate(HDBHEADSIZ)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
fbp_.clear(); fbp_.clear();
bool err = false; bool err = false;
reorg_ = false;
trim_ = false;
flags_ = 0; flags_ = 0;
flagopen_ = false; flagopen_ = false;
count_ = 0; count_ = 0;
lsiz_ = roff_; lsiz_ = roff_;
psiz_ = lsiz_; psiz_ = lsiz_;
dfcur_ = roff_; dfcur_ = roff_;
std::memset(opaque_, 0, sizeof(opaque_)); std::memset(opaque_, 0, sizeof(opaque_));
if (!file_.truncate(lsiz_)) { if (!file_.truncate(lsiz_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
if (!dump_meta()) err = true; if (!dump_meta()) err = true;
if (!set_flag(FOPEN, true)) err = true; if (!set_flag(FOPEN, true)) err = true;
trigger_meta(MetaTrigger::CLEAR, "clear");
return true; return true;
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
*/ */
int64_t count() { int64_t count() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return count_; return count_;
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes, or -1 on failure. * @return the size of the database file in bytes, or -1 on failure.
*/ */
int64_t size() { int64_t size() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return lsiz_; return lsiz_;
} }
/** /**
* Get the path of the database file. * Get the path of the database file.
* @return the path of the database file, or an empty string on failure. * @return the path of the database file, or an empty string on failure.
*/ */
std::string path() { std::string path() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return ""; return "";
} }
return path_; return path_;
} }
/** /**
* Get the miscellaneous status information. * Get the miscellaneous status information.
* @param strmap a string map to contain the result. * @param strmap a string map to contain the result.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool status(std::map<std::string, std::string>* strmap) { bool status(std::map<std::string, std::string>* strmap) {
_assert_(strmap); _assert_(strmap);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
(*strmap)["type"] = "HashDB"; (*strmap)["type"] = strprintf("%u", (unsigned)TYPEHASH);
(*strmap)["realtype"] = strprintf("%u", type_); (*strmap)["realtype"] = strprintf("%u", (unsigned)type_);
(*strmap)["path"] = path_; (*strmap)["path"] = path_;
(*strmap)["libver"] = strprintf("%u", libver_); (*strmap)["libver"] = strprintf("%u", libver_);
(*strmap)["librev"] = strprintf("%u", librev_); (*strmap)["librev"] = strprintf("%u", librev_);
(*strmap)["fmtver"] = strprintf("%u", fmtver_); (*strmap)["fmtver"] = strprintf("%u", fmtver_);
(*strmap)["chksum"] = strprintf("%u", chksum_); (*strmap)["chksum"] = strprintf("%u", chksum_);
(*strmap)["flags"] = strprintf("%u", flags_); (*strmap)["flags"] = strprintf("%u", flags_);
(*strmap)["apow"] = strprintf("%u", apow_); (*strmap)["apow"] = strprintf("%u", apow_);
(*strmap)["fpow"] = strprintf("%u", fpow_); (*strmap)["fpow"] = strprintf("%u", fpow_);
(*strmap)["opts"] = strprintf("%u", opts_); (*strmap)["opts"] = strprintf("%u", opts_);
(*strmap)["bnum"] = strprintf("%lld", (long long)bnum_); (*strmap)["bnum"] = strprintf("%lld", (long long)bnum_);
(*strmap)["msiz"] = strprintf("%lld", (long long)msiz_); (*strmap)["msiz"] = strprintf("%lld", (long long)msiz_);
(*strmap)["dfunit"] = strprintf("%lld", (long long)dfunit_); (*strmap)["dfunit"] = strprintf("%lld", (long long)dfunit_);
(*strmap)["frgcnt"] = strprintf("%lld", (long long)(frgcnt_ > 0 ? (int6 4_t)frgcnt_ : 0)); (*strmap)["frgcnt"] = strprintf("%lld", (long long)(frgcnt_ > 0 ? (int6 4_t)frgcnt_ : 0));
(*strmap)["realsize"] = strprintf("%lld", (long long)file_.size()); (*strmap)["realsize"] = strprintf("%lld", (long long)file_.size());
(*strmap)["recovered"] = strprintf("%d", file_.recovered()); (*strmap)["recovered"] = strprintf("%d", file_.recovered());
(*strmap)["reorganized"] = strprintf("%d", reorg_); (*strmap)["reorganized"] = strprintf("%d", reorg_);
(*strmap)["trimmed"] = strprintf("%d", trim_);
if (strmap->count("opaque") > 0)
(*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_));
if (strmap->count("fbpnum_used") > 0) { if (strmap->count("fbpnum_used") > 0) {
if (writer_) { if (writer_) {
(*strmap)["fbpnum_used"] = strprintf("%lld", (long long)fbp_.size() ); (*strmap)["fbpnum_used"] = strprintf("%lld", (long long)fbp_.size() );
} else { } else {
if (!load_free_blocks()) return false; if (!load_free_blocks()) return false;
(*strmap)["fbpnum_used"] = strprintf("%lld", (long long)fbp_.size() ); (*strmap)["fbpnum_used"] = strprintf("%lld", (long long)fbp_.size() );
fbp_.clear(); fbp_.clear();
} }
} }
if (strmap->count("bnum_used") > 0) { if (strmap->count("bnum_used") > 0) {
int64_t cnt = 0; int64_t cnt = 0;
for (int64_t i = 0; i < bnum_; i++) { for (int64_t i = 0; i < bnum_; i++) {
if (get_bucket(i) > 0) cnt++; if (get_bucket(i) > 0) cnt++;
} }
(*strmap)["bnum_used"] = strprintf("%lld", (long long)cnt); (*strmap)["bnum_used"] = strprintf("%lld", (long long)cnt);
} }
if (strmap->count("opaque") > 0)
(*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_));
(*strmap)["count"] = strprintf("%lld", (long long)count_); (*strmap)["count"] = strprintf("%lld", (long long)count_);
(*strmap)["size"] = strprintf("%lld", (long long)lsiz_); (*strmap)["size"] = strprintf("%lld", (long long)lsiz_);
return true; return true;
} }
/** /**
* Create a cursor object. * Create a cursor object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note Because the object of the return value is allocated by the const ructor, it should be * @note Because the object of the return value is allocated by the const ructor, it should be
* released with the delete operator when it is no longer in use. * released with the delete operator when it is no longer in use.
*/ */
Cursor* cursor() { Cursor* cursor() {
_assert_(true); _assert_(true);
return new Cursor(this); return new Cursor(this);
} }
/** /**
* Set the internal error reporter. * Set the internal logger.
* @param erstrm a stream object into which internal error messages are s * @param logger the logger object.
tored. * @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for
* @param ervbs true to report all errors, or false to report fatal error debugging,
s only. * Logger::INFO for normal information, Logger::WARN for warning, and Log
ger::ERROR for fatal
* error.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_error_reporter(std::ostream* erstrm, bool ervbs) { bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::
_assert_(erstrm); ERROR) {
_assert_(logger);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
erstrm_ = erstrm; logger_ = logger;
ervbs_ = ervbs; logkinds_ = kinds;
return true;
}
/**
* Set the internal meta operation trigger.
* @param trigger the trigger object.
* @return true on success, or false on failure.
*/
bool tune_meta_trigger(MetaTrigger* trigger) {
_assert_(trigger);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
mtrigger_ = trigger;
return true; return true;
} }
/** /**
* Set the power of the alignment of record size. * Set the power of the alignment of record size.
* @param apow the power of the alignment of record size. * @param apow the power of the alignment of record size.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_alignment(int8_t apow) { bool tune_alignment(int8_t apow) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
apow_ = apow >= 0 ? apow : HDBDEFAPOW; apow_ = apow >= 0 ? apow : HDBDEFAPOW;
if (apow_ > HDBMAXAPOW) apow_ = HDBMAXAPOW; if (apow_ > HDBMAXAPOW) apow_ = HDBMAXAPOW;
return true; return true;
} }
/** /**
* Set the power of the capacity of the free block pool. * Set the power of the capacity of the free block pool.
* @param fpow the power of the capacity of the free block pool. * @param fpow the power of the capacity of the free block pool.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_fbp(int8_t fpow) { bool tune_fbp(int8_t fpow) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
fpow_ = fpow >= 0 ? fpow : HDBDEFFPOW; fpow_ = fpow >= 0 ? fpow : HDBDEFFPOW;
if (fpow_ > HDBMAXFPOW) fpow_ = HDBMAXFPOW; if (fpow_ > HDBMAXFPOW) fpow_ = HDBMAXFPOW;
return true; return true;
} }
/** /**
* Set the optional features. * Set the optional features.
* @param opts the optional features by bitwise-or: HashDB::TSMALL to use 32-bit addressing, * @param opts the optional features by bitwise-or: HashDB::TSMALL to use 32-bit addressing,
* HashDB::TLINEAR to use linear collision chaining, HashDB::TCOMPRESS to compress each record. * HashDB::TLINEAR to use linear collision chaining, HashDB::TCOMPRESS to compress each record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_options(int8_t opts) { bool tune_options(int8_t opts) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
opts_ = opts; opts_ = opts;
return true; return true;
} }
/** /**
* Set the number of buckets of the hash table. * Set the number of buckets of the hash table.
* @param bnum the number of buckets of the hash table. * @param bnum the number of buckets of the hash table.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_buckets(int64_t bnum) { bool tune_buckets(int64_t bnum) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
bnum_ = bnum > 0 ? bnum : HDBDEFBNUM; bnum_ = bnum > 0 ? bnum : HDBDEFBNUM;
if (bnum_ > INT16_MAX) bnum_ = nearbyprime(bnum_); if (bnum_ > INT16_MAX) bnum_ = nearbyprime(bnum_);
return true; return true;
} }
/** /**
* Set the size of the internal memory-mapped region. * Set the size of the internal memory-mapped region.
* @param msiz the size of the internal memory-mapped region. * @param msiz the size of the internal memory-mapped region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_map(int64_t msiz) { bool tune_map(int64_t msiz) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
msiz_ = msiz >= 0 ? msiz : HDBDEFMSIZ; msiz_ = msiz >= 0 ? msiz : HDBDEFMSIZ;
return true; return true;
} }
/** /**
* Set the unit step number of auto defragmentation. * Set the unit step number of auto defragmentation.
* @param dfunit the unit step number of auto defragmentation. * @param dfunit the unit step number of auto defragmentation.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_defrag(int64_t dfunit) { bool tune_defrag(int64_t dfunit) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
dfunit_ = dfunit > 0 ? dfunit : 0; dfunit_ = dfunit > 0 ? dfunit : 0;
return true; return true;
} }
/** /**
* Set the data compressor. * Set the data compressor.
* @param comp the data compressor object. * @param comp the data compressor object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_compressor(Compressor* comp) { bool tune_compressor(Compressor* comp) {
_assert_(comp); _assert_(comp);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
embcomp_ = comp; embcomp_ = comp;
return true; return true;
} }
/** /**
* Get the opaque data. * Get the opaque data.
* @return the pointer to the opaque data region, whose size is 16 bytes. * @return the pointer to the opaque data region, whose size is 16 bytes.
*/ */
char* opaque() { char* opaque() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return NULL; return NULL;
} }
return opaque_; return opaque_;
} }
/** /**
* Synchronize the opaque data. * Synchronize the opaque data.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize_opaque() { bool synchronize_opaque() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
if (!writer_) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
bool err = false; bool err = false;
if (!dump_opaque()) err = true; if (!dump_opaque()) err = true;
return !err; return !err;
} }
/** /**
* Perform defragmentation of the file. * Perform defragmentation of the file.
* @param step the number of steps. If it is not more than 0, the whole region is defraged. * @param step the number of steps. If it is not more than 0, the whole region is defraged.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool defrag(int64_t step) { bool defrag(int64_t step = 0) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!writer_) { if (!writer_) {
set_error(__FILE__, __LINE__, Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
bool err = false; bool err = false;
if (step > 0) { if (step > 0) {
if (!defrag_impl(step)) err = true; if (!defrag_impl(step)) err = true;
} else { } else {
dfcur_ = roff_; dfcur_ = roff_;
if (!defrag_impl(INT64_MAX)) err = true; if (!defrag_impl(INT64_MAX)) err = true;
} }
frgcnt_ = 0; frgcnt_ = 0;
return !err; return !err;
} }
/** /**
* Get the status flags. * Get the status flags.
* @return the status flags, or 0 on failure. * @return the status flags, or 0 on failure.
*/ */
uint8_t flags() { uint8_t flags() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return flags_; return flags_;
} }
protected: protected:
/** /**
* Set the error information.
* @param file the file name of the epicenter.
* @param line the line number of the epicenter.
* @param code an error code.
* @param message a supplement message.
*/
void set_error(const char* file, int32_t line,
Error::Code code, const char* message) {
_assert_(file && message);
set_error(code, message);
if (ervbs_ || code == Error::BROKEN || code == Error::SYSTEM)
report(file, line, "error", "%d: %s: %s", code, Error::codename(code)
, message);
}
/**
* Report a message for debugging. * Report a message for debugging.
* @param file the file name of the epicenter. * @param file the file name of the program source code.
* @param line the line number of the epicenter. * @param line the line number of the program source code.
* @param type the type string. * @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param format the printf-like format string. * @param format the printf-like format string.
* @param ... used according to the format string. * @param ... used according to the format string.
*/ */
void report(const char* file, int32_t line, const char* type, void report(const char* file, int32_t line, const char* func, Logger::Kin d kind,
const char* format, ...) { const char* format, ...) {
_assert_(file && line > 0 && type && format); _assert_(file && line > 0 && func && format);
if (!erstrm_) return; if (!logger_ || !(kind & logkinds_)) return;
const std::string& path = path_.empty() ? "-" : path_;
std::string message; std::string message;
strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str());
va_list ap; va_list ap;
va_start(ap, format); va_start(ap, format);
strprintf(&message, format, ap); vstrprintf(&message, format, ap);
va_end(ap); va_end(ap);
*erstrm_ << "[" << type << "]: " << path << ": " << file << ": " << lin logger_->log(file, line, func, kind, message.c_str());
e; }
*erstrm_ << ": " << message << std::endl; /**
* Report a message for debugging with variable number of arguments.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param format the printf-like format string.
* @param ap used according to the format string.
*/
void report_valist(const char* file, int32_t line, const char* func, Logg
er::Kind kind,
const char* format, va_list ap) {
_assert_(file && line > 0 && func && format);
if (!logger_ || !(kind & logkinds_)) return;
std::string message;
strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str());
vstrprintf(&message, format, ap);
logger_->log(file, line, func, kind, message.c_str());
} }
/** /**
* Report the content of a binary buffer for debugging. * Report the content of a binary buffer for debugging.
* @param file the file name of the epicenter. * @param file the file name of the epicenter.
* @param line the line number of the epicenter. * @param line the line number of the epicenter.
* @param type the type string. * @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param name the name of the information. * @param name the name of the information.
* @param buf the binary buffer. * @param buf the binary buffer.
* @param size the size of the binary buffer * @param size the size of the binary buffer
*/ */
void report_binary(const char* file, int32_t line, const char* type, void report_binary(const char* file, int32_t line, const char* func, Logg er::Kind kind,
const char* name, const char* buf, size_t size) { const char* name, const char* buf, size_t size) {
_assert_(file && line > 0 && type && name && buf && size <= MEMMAXSIZ); _assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ);
if (!erstrm_) return; if (!logger_) return;
char* hex = hexencode(buf, size); char* hex = hexencode(buf, size);
report(file, line, type, "%s=%s", name, hex); report(file, line, func, kind, "%s=%s", name, hex);
delete[] hex; delete[] hex;
} }
/** /**
* Trigger a meta database operation.
* @param kind the kind of the event. MetaTrigger::OPEN for opening, Met
aTrigger::CLOSE for
* closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for ite
ration,
* MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f
or beginning
* transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT
rigger::ABORTTRAN
* for aborting transaction, and MetaTrigger::MISC for miscellaneous oper
ations.
* @param message the supplement message.
*/
void trigger_meta(MetaTrigger::Kind kind, const char* message) {
_assert_(message);
if (mtrigger_) mtrigger_->trigger(kind, message);
}
/**
* Set the database type. * Set the database type.
* @param type the database type. * @param type the database type.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool tune_type(int8_t type) { bool tune_type(int8_t type) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
type_ = type; type_ = type;
return true; return true;
} }
/** /**
* Get the library version. * Get the library version.
* @return the library version, or 0 on failure. * @return the library version, or 0 on failure.
*/ */
uint8_t libver() { uint8_t libver() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return libver_; return libver_;
} }
/** /**
* Get the library revision. * Get the library revision.
* @return the library revision, or 0 on failure. * @return the library revision, or 0 on failure.
*/ */
uint8_t librev() { uint8_t librev() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return librev_; return librev_;
} }
/** /**
* Get the format version. * Get the format version.
* @return the format version, or 0 on failure. * @return the format version, or 0 on failure.
*/ */
uint8_t fmtver() { uint8_t fmtver() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return fmtver_; return fmtver_;
} }
/** /**
* Get the module checksum. * Get the module checksum.
* @return the module checksum, or 0 on failure. * @return the module checksum, or 0 on failure.
*/ */
uint8_t chksum() { uint8_t chksum() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return chksum_; return chksum_;
} }
/** /**
* Get the database type. * Get the database type.
* @return the database type, or 0 on failure. * @return the database type, or 0 on failure.
*/ */
uint8_t type() { uint8_t type() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return type_; return type_;
} }
/** /**
* Get the alignment power. * Get the alignment power.
* @return the alignment power, or 0 on failure. * @return the alignment power, or 0 on failure.
*/ */
uint8_t apow() { uint8_t apow() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return apow_; return apow_;
} }
/** /**
* Get the free block pool power. * Get the free block pool power.
* @return the free block pool power, or 0 on failure. * @return the free block pool power, or 0 on failure.
*/ */
uint8_t fpow() { uint8_t fpow() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return fpow_; return fpow_;
} }
/** /**
* Get the options. * Get the options.
* @return the options, or 0 on failure. * @return the options, or 0 on failure.
*/ */
uint8_t opts() { uint8_t opts() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return opts_; return opts_;
} }
/** /**
* Get the bucket number. * Get the bucket number.
* @return the bucket number, or 0 on failure. * @return the bucket number, or 0 on failure.
*/ */
int64_t bnum() { int64_t bnum() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return bnum_; return bnum_;
} }
/** /**
* Get the size of the internal memory-mapped region. * Get the size of the internal memory-mapped region.
* @return the size of the internal memory-mapped region, or 0 on failure . * @return the size of the internal memory-mapped region, or 0 on failure .
*/ */
int64_t msiz() { int64_t msiz() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return msiz_; return msiz_;
} }
/** /**
* Get the unit step number of auto defragmentation. * Get the unit step number of auto defragmentation.
* @return the unit step number of auto defragmentation, or 0 on failure. * @return the unit step number of auto defragmentation, or 0 on failure.
*/ */
int64_t dfunit() { int64_t dfunit() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return 0; return 0;
} }
return dfunit_; return dfunit_;
} }
/** /**
* Get the data compressor. * Get the data compressor.
* @return the data compressor, or NULL on failure. * @return the data compressor, or NULL on failure.
*/ */
Compressor *comp() { Compressor* comp() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return NULL; return NULL;
} }
return comp_; return comp_;
} }
/** /**
* Check whether the database was recovered or not. * Check whether the database was recovered or not.
* @return true if recovered, or false if not. * @return true if recovered, or false if not.
*/ */
bool recovered() { bool recovered() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return file_.recovered(); return file_.recovered();
} }
/** /**
* Check whether the database was reorganized or not. * Check whether the database was reorganized or not.
* @return true if recovered, or false if not. * @return true if recovered, or false if not.
*/ */
bool reorganized() { bool reorganized() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(__FILE__, __LINE__, Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return reorg_; return reorg_;
} }
private: private:
/** /**
* Record data. * Record data.
*/ */
struct Record { struct Record {
int64_t off; ///< offset int64_t off; ///< offset
skipping to change at line 1436 skipping to change at line 1581
* Repeating visitor. * Repeating visitor.
*/ */
class Repeater : public Visitor { class Repeater : public Visitor {
public: public:
explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(v siz) { explicit Repeater(const char* vbuf, size_t vsiz) : vbuf_(vbuf), vsiz_(v siz) {
_assert_(vbuf); _assert_(vbuf);
} }
private: private:
const char* visit_full(const char* kbuf, size_t ksiz, const char* visit_full(const char* kbuf, size_t ksiz,
const char* vbuf, size_t vsiz, size_t* sp) { const char* vbuf, size_t vsiz, size_t* sp) {
_assert_(kbuf && vbuf && sp); _assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && sp );
*sp = vsiz_; *sp = vsiz_;
return vbuf_; return vbuf_;
} }
const char* vbuf_; const char* vbuf_;
size_t vsiz_; size_t vsiz_;
}; };
/** Dummy constructor to forbid the use. */
HashDB(const HashDB&);
/** Dummy Operator to forbid the use. */
HashDB& operator =(const HashDB&);
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param bidx the bucket index. * @param bidx the bucket index.
* @param pivot the second hash value. * @param pivot the second hash value.
@ @param isiter true for iterator use, or false for direct use. @ @param isiter true for iterator use, or false for direct use.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor, bool accept_impl(const char* kbuf, size_t ksiz, Visitor* visitor,
int64_t bidx, uint32_t pivot, bool isiter) { int64_t bidx, uint32_t pivot, bool isiter) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor && bidx >= 0); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor && bidx >= 0);
int64_t off = get_bucket(bidx); int64_t top = get_bucket(bidx);
int64_t off = top;
if (off < 0) return false; if (off < 0) return false;
enum { DIREMPTY, DIRLEFT, DIRRIGHT, DIRMIXED } entdir = DIREMPTY;
int64_t entoff = 0; int64_t entoff = 0;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[HDBRECBUFSIZ];
while (off > 0) { while (off > 0) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16_MAX) { if (rec.psiz == UINT16_MAX) {
set_error(__FILE__, __LINE__, Error::BROKEN, "free block in the cha set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain");
in"); report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", (long long)psiz_, (long long)rec.off, (long long)file_.size(
(long)psiz_, (long)rec.off, (long)file_.size()); ));
return false; return false;
} }
uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, r ec.ksiz)); uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, r ec.ksiz));
if (pivot > tpivot) { if (pivot > tpivot) {
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.left; off = rec.left;
switch (entdir) {
case DIREMPTY: entdir = DIRLEFT; break;
case DIRRIGHT: entdir = DIRMIXED; break;
default: break;
}
entoff = rec.off + sizeof(uint16_t); entoff = rec.off + sizeof(uint16_t);
} else if (pivot < tpivot) { } else if (pivot < tpivot) {
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.right; off = rec.right;
switch (entdir) {
case DIREMPTY: entdir = DIRRIGHT; break;
case DIRLEFT: entdir = DIRMIXED; break;
default: break;
}
entoff = rec.off + sizeof(uint16_t) + width_; entoff = rec.off + sizeof(uint16_t) + width_;
} else { } else {
int32_t kcmp = compare_keys(kbuf, ksiz, rec.kbuf, rec.ksiz); int32_t kcmp = compare_keys(kbuf, ksiz, rec.kbuf, rec.ksiz);
if (linear_ && kcmp != 0) kcmp = 1; if (linear_ && kcmp != 0) kcmp = 1;
if (kcmp > 0) { if (kcmp > 0) {
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.left; off = rec.left;
switch (entdir) {
case DIREMPTY: entdir = DIRLEFT; break;
case DIRRIGHT: entdir = DIRMIXED; break;
default: break;
}
entoff = rec.off + sizeof(uint16_t); entoff = rec.off + sizeof(uint16_t);
} else if (kcmp < 0) { } else if (kcmp < 0) {
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.right; off = rec.right;
switch (entdir) {
case DIREMPTY: entdir = DIRRIGHT; break;
case DIRLEFT: entdir = DIRMIXED; break;
default: break;
}
entoff = rec.off + sizeof(uint16_t) + width_; entoff = rec.off + sizeof(uint16_t) + width_;
} else { } else {
if (!rec.vbuf && !read_record_body(&rec)) { if (!rec.vbuf && !read_record_body(&rec)) {
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
const char* vbuf = rec.vbuf; const char* vbuf = rec.vbuf;
size_t vsiz = rec.vsiz; size_t vsiz = rec.vsiz;
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (comp_) { if (comp_) {
zbuf = comp_->decompress(vbuf, vsiz, &zsiz); zbuf = comp_->decompress(vbuf, vsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "data decompress ion failed"); set_error(_KCCODELINE_, Error::SYSTEM, "data decompression fa iled");
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
vbuf = zbuf; vbuf = zbuf;
vsiz = zsiz; vsiz = zsiz;
} }
vbuf = visitor->visit_full(kbuf, ksiz, vbuf, vsiz, &vsiz); vbuf = visitor->visit_full(kbuf, ksiz, vbuf, vsiz, &vsiz);
delete[] zbuf; delete[] zbuf;
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
bool atran = false; bool atran = false;
skipping to change at line 1550 skipping to change at line 1713
if (!synchronize_meta()) return false; if (!synchronize_meta()) return false;
} }
} else if (vbuf == Visitor::NOP) { } else if (vbuf == Visitor::NOP) {
delete[] rec.bbuf; delete[] rec.bbuf;
} else { } else {
zbuf = NULL; zbuf = NULL;
zsiz = 0; zsiz = 0;
if (comp_ && !isiter) { if (comp_ && !isiter) {
zbuf = comp_->compress(vbuf, vsiz, &zsiz); zbuf = comp_->compress(vbuf, vsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "data compress ion failed"); set_error(_KCCODELINE_, Error::SYSTEM, "data compression fa iled");
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
vbuf = zbuf; vbuf = zbuf;
vsiz = zsiz; vsiz = zsiz;
} }
bool atran = false; bool atran = false;
if (autotran_ && !tran_) { if (autotran_ && !tran_) {
if (!begin_auto_transaction()) { if (!begin_auto_transaction()) {
delete[] zbuf; delete[] zbuf;
skipping to change at line 1648 skipping to change at line 1811
} }
} }
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz);
if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) {
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (comp_) { if (comp_) {
zbuf = comp_->compress(vbuf, vsiz, &zsiz); zbuf = comp_->compress(vbuf, vsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "data compression fa iled"); set_error(_KCCODELINE_, Error::SYSTEM, "data compression failed") ;
return false; return false;
} }
vbuf = zbuf; vbuf = zbuf;
vsiz = zsiz; vsiz = zsiz;
} }
bool atran = false; bool atran = false;
if (autotran_ && !tran_) { if (autotran_ && !tran_) {
if (!begin_auto_transaction()) { if (!begin_auto_transaction()) {
delete[] zbuf; delete[] zbuf;
return false; return false;
} }
atran = true; atran = true;
} }
size_t rsiz = calc_record_size(ksiz, vsiz); size_t rsiz = calc_record_size(ksiz, vsiz);
size_t psiz = calc_record_padding(rsiz); size_t psiz = calc_record_padding(rsiz);
rec.rsiz = rsiz + psiz; rec.rsiz = rsiz + psiz;
rec.psiz = psiz; rec.psiz = psiz;
rec.ksiz = ksiz; rec.ksiz = ksiz;
rec.vsiz = vsiz; rec.vsiz = vsiz;
rec.left = 0; switch (entdir) {
rec.right = 0; default: {
rec.left = 0;
rec.right = 0;
break;
}
case DIRLEFT: {
if (linear_) {
rec.left = top;
rec.right = 0;
} else {
rec.left = 0;
rec.right = top;
}
break;
}
case DIRRIGHT: {
rec.left = top;
rec.right = 0;
break;
}
}
rec.kbuf = kbuf; rec.kbuf = kbuf;
rec.vbuf = vbuf; rec.vbuf = vbuf;
bool over = false; bool over = false;
FreeBlock fb; FreeBlock fb;
if (fetch_free_block(rec.rsiz, &fb)) { if (fetch_free_block(rec.rsiz, &fb)) {
rec.off = fb.off; rec.off = fb.off;
rec.rsiz = fb.rsiz; rec.rsiz = fb.rsiz;
rec.psiz = rec.rsiz - rsiz; rec.psiz = rec.rsiz - rsiz;
over = true; over = true;
if (!adjust_record(&rec)) { if (!adjust_record(&rec)) {
skipping to change at line 1694 skipping to change at line 1877
} else { } else {
rec.off = lsiz_.add(rec.rsiz); rec.off = lsiz_.add(rec.rsiz);
} }
if (!write_record(&rec, over)) { if (!write_record(&rec, over)) {
if (atran) abort_auto_transaction(); if (atran) abort_auto_transaction();
delete[] zbuf; delete[] zbuf;
return false; return false;
} }
if (!over) psiz_.secure_least(rec.off + rec.rsiz); if (!over) psiz_.secure_least(rec.off + rec.rsiz);
delete[] zbuf; delete[] zbuf;
if (entoff > 0) { if (entoff < 1 || entdir == DIRLEFT || entdir == DIRRIGHT) {
if (!set_chain(entoff, rec.off)) { if (!set_bucket(bidx, rec.off)) {
if (atran) abort_auto_transaction(); if (atran) abort_auto_transaction();
return false; return false;
} }
} else { } else {
if (!set_bucket(bidx, rec.off)) { if (!set_chain(entoff, rec.off)) {
if (atran) abort_auto_transaction(); if (atran) abort_auto_transaction();
return false; return false;
} }
} }
count_ += 1; count_ += 1;
if (atran) { if (atran) {
if (!commit_auto_transaction()) return false; if (!commit_auto_transaction()) return false;
} else if (autosync_) { } else if (autosync_) {
if (!synchronize_meta()) return false; if (!synchronize_meta()) return false;
} }
} }
return true; return true;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param checker a progress checker object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool iterate_impl(Visitor* visitor) { bool iterate_impl(Visitor* visitor, ProgressChecker* checker) {
_assert_(visitor); _assert_(visitor);
int64_t allcnt = count_;
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
int64_t off = roff_; int64_t off = roff_;
int64_t end = lsiz_; int64_t end = lsiz_;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[HDBRECBUFSIZ];
int64_t curcnt = 0;
while (off > 0 && off < end) { while (off > 0 && off < end) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16_MAX) { if (rec.psiz == UINT16_MAX) {
off += rec.rsiz; off += rec.rsiz;
} else { } else {
if (!rec.vbuf && !read_record_body(&rec)) { if (!rec.vbuf && !read_record_body(&rec)) {
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
const char* vbuf = rec.vbuf; const char* vbuf = rec.vbuf;
size_t vsiz = rec.vsiz; size_t vsiz = rec.vsiz;
char* zbuf = NULL; char* zbuf = NULL;
size_t zsiz = 0; size_t zsiz = 0;
if (comp_) { if (comp_) {
zbuf = comp_->decompress(vbuf, vsiz, &zsiz); zbuf = comp_->decompress(vbuf, vsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "data decompressio n failed"); set_error(_KCCODELINE_, Error::SYSTEM, "data decompression fail ed");
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
vbuf = zbuf; vbuf = zbuf;
vsiz = zsiz; vsiz = zsiz;
} }
vbuf = visitor->visit_full(rec.kbuf, rec.ksiz, vbuf, vsiz, &vsiz); vbuf = visitor->visit_full(rec.kbuf, rec.ksiz, vbuf, vsiz, &vsiz);
delete[] zbuf; delete[] zbuf;
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
uint64_t hash = hash_record(rec.kbuf, rec.ksiz); uint64_t hash = hash_record(rec.kbuf, rec.ksiz);
skipping to change at line 1769 skipping to change at line 1959
} }
delete[] rec.bbuf; delete[] rec.bbuf;
} else if (vbuf == Visitor::NOP) { } else if (vbuf == Visitor::NOP) {
delete[] rec.bbuf; delete[] rec.bbuf;
} else { } else {
zbuf = NULL; zbuf = NULL;
zsiz = 0; zsiz = 0;
if (comp_) { if (comp_) {
zbuf = comp_->compress(vbuf, vsiz, &zsiz); zbuf = comp_->compress(vbuf, vsiz, &zsiz);
if (!zbuf) { if (!zbuf) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "data compressio n failed"); set_error(_KCCODELINE_, Error::SYSTEM, "data compression fail ed");
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
vbuf = zbuf; vbuf = zbuf;
vsiz = zsiz; vsiz = zsiz;
} }
size_t rsiz = calc_record_size(rec.ksiz, vsiz); size_t rsiz = calc_record_size(rec.ksiz, vsiz);
if (rsiz <= rec.rsiz) { if (rsiz <= rec.rsiz) {
rec.psiz = rec.rsiz - rsiz; rec.psiz = rec.rsiz - rsiz;
rec.vsiz = vsiz; rec.vsiz = vsiz;
skipping to change at line 1803 skipping to change at line 1993
if (!accept_impl(rec.kbuf, rec.ksiz, &repeater, bidx, pivot, tr ue)) { if (!accept_impl(rec.kbuf, rec.ksiz, &repeater, bidx, pivot, tr ue)) {
delete[] zbuf; delete[] zbuf;
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
delete[] zbuf; delete[] zbuf;
delete[] rec.bbuf; delete[] rec.bbuf;
} }
} }
off += rec.rsiz; off += rec.rsiz;
curcnt++;
if (checker && !checker->check("iterate", "processing", curcnt, all
cnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
} }
} }
if (checker && !checker->check("iterate", "ending", -1, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
return true; return true;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @param proc a postprocessor object. * @param proc a postprocessor object.
* @param checker a progress checker object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize_impl(bool hard, FileProcessor* proc) { bool synchronize_impl(bool hard, FileProcessor* proc, ProgressChecker* ch ecker) {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
if (hard && !dump_free_blocks()) err = true; if (writer_) {
if (!dump_meta()) err = true; if (checker && !checker->check("synchronize", "dumping the free block
if (!file_.synchronize(hard)) { s", -1, -1)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true; return false;
}
if (hard && !dump_free_blocks()) err = true;
if (checker && !checker->check("synchronize", "dumping the meta data"
, -1, -1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
if (!dump_meta()) err = true;
if (checker && !checker->check("synchronize", "synchronizing the file
", -1, -1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
if (!file_.synchronize(hard)) {
set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true;
}
} }
if (proc && !proc->process(path_, count_, lsiz_)) { if (proc) {
set_error(__FILE__, __LINE__, Error::LOGIC, "postprocessing failed"); if (checker && !checker->check("synchronize", "running the post proce
err = true; ssor", -1, -1)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
if (!proc->process(path_, count_, lsiz_)) {
set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed");
err = true;
}
} }
if (writer_ && !autotran_ && !set_flag(FOPEN, true)) err = true;
return !err; return !err;
} }
/** /**
* Synchronize meta data with the file and the device. * Synchronize meta data with the file and the device.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize_meta() { bool synchronize_meta() {
_assert_(true); _assert_(true);
ScopedSpinLock lock(&flock_); ScopedSpinLock lock(&flock_);
bool err = false; bool err = false;
if (!dump_meta()) err = true; if (!dump_meta()) err = true;
if (!file_.synchronize(true)) { if (!file_.synchronize(true)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Perform defragmentation. * Perform defragmentation.
* @param step the number of steps. * @param step the number of steps.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool defrag_impl(int64_t step) { bool defrag_impl(int64_t step) {
skipping to change at line 1866 skipping to change at line 2087
dfcur_ = roff_; dfcur_ = roff_;
return true; return true;
} }
if (step-- < 1) return true; if (step-- < 1) return true;
rec.off = dfcur_; rec.off = dfcur_;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16_MAX) break; if (rec.psiz == UINT16_MAX) break;
delete[] rec.bbuf; delete[] rec.bbuf;
dfcur_ += rec.rsiz; dfcur_ += rec.rsiz;
} }
bool atran = false;
if (autotran_ && !tran_) {
if (!begin_auto_transaction()) return false;
atran = true;
}
int64_t base = dfcur_; int64_t base = dfcur_;
int64_t dest = base; int64_t dest = base;
dfcur_ += rec.rsiz; dfcur_ += rec.rsiz;
step++; step++;
while (step-- > 0 && dfcur_ < end) { while (step-- > 0 && dfcur_ < end) {
rec.off = dfcur_; rec.off = dfcur_;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) {
if (atran) abort_auto_transaction();
return false;
}
escape_cursors(rec.off, dest); escape_cursors(rec.off, dest);
dfcur_ += rec.rsiz; dfcur_ += rec.rsiz;
if (rec.psiz != UINT16_MAX) { if (rec.psiz != UINT16_MAX) {
if (!rec.vbuf && !read_record_body(&rec)) { if (!rec.vbuf && !read_record_body(&rec)) {
if (atran) abort_auto_transaction();
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
if (rec.psiz >= align_) { if (rec.psiz >= align_) {
size_t diff = rec.psiz - rec.psiz % align_; size_t diff = rec.psiz - rec.psiz % align_;
rec.psiz -= diff; rec.psiz -= diff;
rec.rsiz -= diff; rec.rsiz -= diff;
} }
if (!shift_record(&rec, dest)) { if (!shift_record(&rec, dest)) {
if (atran) abort_auto_transaction();
delete[] rec.bbuf; delete[] rec.bbuf;
return false; return false;
} }
delete[] rec.bbuf; delete[] rec.bbuf;
dest += rec.rsiz; dest += rec.rsiz;
} }
} }
trim_free_blocks(base, dfcur_); trim_free_blocks(base, dfcur_);
if (dfcur_ >= end) { if (dfcur_ >= end) {
lsiz_ = dest; lsiz_ = dest;
psiz_ = lsiz_; psiz_ = lsiz_;
if (!file_.truncate(lsiz_)) return false; if (!file_.truncate(lsiz_)) {
if (atran) abort_auto_transaction();
return false;
}
trim_cursors(); trim_cursors();
dfcur_ = roff_; dfcur_ = roff_;
} else { } else {
size_t rsiz = dfcur_ - dest; size_t rsiz = dfcur_ - dest;
if (!write_free_block(dest, rsiz, rbuf)) return false; if (!write_free_block(dest, rsiz, rbuf)) {
if (atran) abort_auto_transaction();
return false;
}
insert_free_block(dest, rsiz); insert_free_block(dest, rsiz);
dfcur_ = dest; dfcur_ = dest;
} }
if (atran) {
if (!commit_auto_transaction()) return false;
} else if (autosync_) {
if (!synchronize_meta()) return false;
}
return true; return true;
} }
/** /**
* Calculate meta data with saved ones. * Calculate meta data with saved ones.
*/ */
void calc_meta() { void calc_meta() {
_assert_(true); _assert_(true);
align_ = 1 << apow_; align_ = 1 << apow_;
fbpnum_ = fpow_ > 0 ? 1 << fpow_ : 0; fbpnum_ = fpow_ > 0 ? 1 << fpow_ : 0;
width_ = (opts_ & TSMALL) ? HDBWIDTHSMALL : HDBWIDTHLARGE; width_ = (opts_ & TSMALL) ? HDBWIDTHSMALL : HDBWIDTHLARGE;
skipping to change at line 1977 skipping to change at line 2219
std::memcpy(head + HDBMOFFBNUM, &num, sizeof(num)); std::memcpy(head + HDBMOFFBNUM, &num, sizeof(num));
uint8_t flags = flags_; uint8_t flags = flags_;
if (!flagopen_) flags &= ~FOPEN; if (!flagopen_) flags &= ~FOPEN;
std::memcpy(head + HDBMOFFFLAGS, &flags, sizeof(flags)); std::memcpy(head + HDBMOFFFLAGS, &flags, sizeof(flags));
num = hton64(count_); num = hton64(count_);
std::memcpy(head + HDBMOFFCOUNT, &num, sizeof(num)); std::memcpy(head + HDBMOFFCOUNT, &num, sizeof(num));
num = hton64(lsiz_); num = hton64(lsiz_);
std::memcpy(head + HDBMOFFSIZE, &num, sizeof(num)); std::memcpy(head + HDBMOFFSIZE, &num, sizeof(num));
std::memcpy(head + HDBMOFFOPAQUE, opaque_, sizeof(opaque_)); std::memcpy(head + HDBMOFFOPAQUE, opaque_, sizeof(opaque_));
if (!file_.write(0, head, sizeof(head))) { if (!file_.write(0, head, sizeof(head))) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
trcount_ = count_;
trsize_ = lsiz_;
return true; return true;
} }
/** /**
* Dump the meta data into the file. * Dump the meta data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_auto_meta() { bool dump_auto_meta() {
_assert_(true); _assert_(true);
const int64_t hsiz = HDBMOFFOPAQUE - HDBMOFFCOUNT; const int64_t hsiz = HDBMOFFOPAQUE - HDBMOFFCOUNT;
char head[hsiz]; char head[hsiz];
std::memset(head, 0, hsiz); std::memset(head, 0, hsiz);
uint64_t num = hton64(count_); uint64_t num = hton64(count_);
std::memcpy(head, &num, sizeof(num)); std::memcpy(head, &num, sizeof(num));
num = hton64(lsiz_); num = hton64(lsiz_);
std::memcpy(head + HDBMOFFSIZE - HDBMOFFCOUNT, &num, sizeof(num)); std::memcpy(head + HDBMOFFSIZE - HDBMOFFCOUNT, &num, sizeof(num));
if (!file_.write_fast(HDBMOFFCOUNT, head, sizeof(head))) { if (!file_.write_fast(HDBMOFFCOUNT, head, sizeof(head))) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
trcount_ = count_;
trsize_ = lsiz_;
return true; return true;
} }
/** /**
* Dump the opaque data into the file. * Dump the opaque data into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_opaque() { bool dump_opaque() {
_assert_(true); _assert_(true);
if (!file_.write_fast(HDBMOFFOPAQUE, opaque_, sizeof(opaque_))) { if (!file_.write_fast(HDBMOFFOPAQUE, opaque_, sizeof(opaque_))) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
return true; return true;
} }
/** /**
* Load the meta data from the file. * Load the meta data from the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_meta() { bool load_meta() {
_assert_(true); _assert_(true);
char head[HDBHEADSIZ]; char head[HDBHEADSIZ];
if (file_.size() < (int64_t)sizeof(head)) { if (file_.size() < (int64_t)sizeof(head)) {
set_error(__FILE__, __LINE__, Error::INVALID, "missing magic data of the file"); set_error(_KCCODELINE_, Error::INVALID, "missing magic data of the fi le");
return false; return false;
} }
if (!file_.read(0, head, sizeof(head))) { if (!file_.read(0, head, sizeof(head))) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long)psiz_, (long)0, (long)file_.size()); (long long)psiz_, (long long)0, (long long)file_.size());
return false; return false;
} }
if (std::memcmp(head, HDBMAGICDATA, sizeof(HDBMAGICDATA))) { if (std::memcmp(head, HDBMAGICDATA, sizeof(HDBMAGICDATA))) {
set_error(__FILE__, __LINE__, Error::INVALID, "invalid magic data of the file"); set_error(_KCCODELINE_, Error::INVALID, "invalid magic data of the fi le");
return false; return false;
} }
std::memcpy(&libver_, head + HDBMOFFLIBVER, sizeof(libver_)); std::memcpy(&libver_, head + HDBMOFFLIBVER, sizeof(libver_));
std::memcpy(&librev_, head + HDBMOFFLIBREV, sizeof(librev_)); std::memcpy(&librev_, head + HDBMOFFLIBREV, sizeof(librev_));
std::memcpy(&fmtver_, head + HDBMOFFFMTVER, sizeof(fmtver_)); std::memcpy(&fmtver_, head + HDBMOFFFMTVER, sizeof(fmtver_));
std::memcpy(&chksum_, head + HDBMOFFCHKSUM, sizeof(chksum_)); std::memcpy(&chksum_, head + HDBMOFFCHKSUM, sizeof(chksum_));
std::memcpy(&type_, head + HDBMOFFTYPE, sizeof(type_)); std::memcpy(&type_, head + HDBMOFFTYPE, sizeof(type_));
std::memcpy(&apow_, head + HDBMOFFAPOW, sizeof(apow_)); std::memcpy(&apow_, head + HDBMOFFAPOW, sizeof(apow_));
std::memcpy(&fpow_, head + HDBMOFFFPOW, sizeof(fpow_)); std::memcpy(&fpow_, head + HDBMOFFFPOW, sizeof(fpow_));
std::memcpy(&opts_, head + HDBMOFFOPTS, sizeof(opts_)); std::memcpy(&opts_, head + HDBMOFFOPTS, sizeof(opts_));
skipping to change at line 2053 skipping to change at line 2299
std::memcpy(&num, head + HDBMOFFBNUM, sizeof(num)); std::memcpy(&num, head + HDBMOFFBNUM, sizeof(num));
bnum_ = ntoh64(num); bnum_ = ntoh64(num);
std::memcpy(&flags_, head + HDBMOFFFLAGS, sizeof(flags_)); std::memcpy(&flags_, head + HDBMOFFFLAGS, sizeof(flags_));
flagopen_ = flags_ & FOPEN; flagopen_ = flags_ & FOPEN;
std::memcpy(&num, head + HDBMOFFCOUNT, sizeof(num)); std::memcpy(&num, head + HDBMOFFCOUNT, sizeof(num));
count_ = ntoh64(num); count_ = ntoh64(num);
std::memcpy(&num, head + HDBMOFFSIZE, sizeof(num)); std::memcpy(&num, head + HDBMOFFSIZE, sizeof(num));
lsiz_ = ntoh64(num); lsiz_ = ntoh64(num);
psiz_ = lsiz_; psiz_ = lsiz_;
std::memcpy(opaque_, head + HDBMOFFOPAQUE, sizeof(opaque_)); std::memcpy(opaque_, head + HDBMOFFOPAQUE, sizeof(opaque_));
trcount_ = count_;
trsize_ = lsiz_;
return true; return true;
} }
/** /**
* Set a status flag. * Set a status flag.
* @param flag the flag kind. * @param flag the flag kind.
* @param sign whether to set or unset. * @param sign whether to set or unset.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool set_flag(uint8_t flag, bool sign) { bool set_flag(uint8_t flag, bool sign) {
_assert_(true); _assert_(true);
uint8_t flags; uint8_t flags;
if (!file_.read(HDBMOFFFLAGS, &flags, sizeof(flags))) { if (!file_.read(HDBMOFFFLAGS, &flags, sizeof(flags))) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long)psiz_, (long)HDBMOFFFLAGS, (long)file_.size()); (long long)psiz_, (long long)HDBMOFFFLAGS, (long long)file_.si
ze());
return false; return false;
} }
if (sign) { if (sign) {
flags |= flag; flags |= flag;
} else { } else {
flags &= ~flag; flags &= ~flag;
} }
if (!file_.write(HDBMOFFFLAGS, &flags, sizeof(flags))) { if (!file_.write(HDBMOFFFLAGS, &flags, sizeof(flags))) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
flags_ = flags; flags_ = flags;
return true; return true;
} }
/** /**
* Reorganize the whole file. * Reorganize the whole file.
* @param path the path of the database file. * @param path the path of the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
skipping to change at line 2100 skipping to change at line 2348
HashDB db; HashDB db;
db.tune_type(type_); db.tune_type(type_);
db.tune_alignment(apow_); db.tune_alignment(apow_);
db.tune_fbp(fpow_); db.tune_fbp(fpow_);
db.tune_options(opts_); db.tune_options(opts_);
db.tune_buckets(bnum_); db.tune_buckets(bnum_);
db.tune_map(msiz_); db.tune_map(msiz_);
if (embcomp_) db.tune_compressor(embcomp_); if (embcomp_) db.tune_compressor(embcomp_);
const std::string& npath = path + File::EXTCHR + HDBTMPPATHEXT; const std::string& npath = path + File::EXTCHR + HDBTMPPATHEXT;
if (db.open(npath, OWRITER | OCREATE | OTRUNCATE)) { if (db.open(npath, OWRITER | OCREATE | OTRUNCATE)) {
report(__FILE__, __LINE__, "info", "reorganizing the database"); report(_KCCODELINE_, Logger::WARN, "reorganizing the database");
lsiz_ = file_.size(); lsiz_ = file_.size();
psiz_ = lsiz_; psiz_ = lsiz_;
if (copy_records(&db)) { if (copy_records(&db)) {
if (db.close()) { if (db.close()) {
File src; if (!File::rename(npath, path)) {
if (src.open(npath, File::OREADER | File::ONOLOCK, 0)) { set_error(_KCCODELINE_, Error::SYSTEM, "renaming the destinatio
File* dest = writer_ ? &file_ : new File(); n failed");
if (dest == &file_ || dest->open(path, File::OWRITER | File::ON
OLOCK, 0)) {
int64_t off = 0;
int64_t size = src.size();
char buf[HDBIOBUFSIZ*4];
while (off < size) {
int32_t psiz = size - off;
if (psiz > (int32_t)sizeof(buf)) psiz = sizeof(buf);
if (src.read(off, buf, psiz)) {
if (!dest->write(off, buf, psiz)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, dest->erro
r());
err = true;
break;
}
} else {
set_error(__FILE__, __LINE__, Error::SYSTEM, src.error())
;
err = true;
break;
}
off += psiz;
}
if (!dest->truncate(size)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, dest->error())
;
err = true;
}
if (dest != &file_) {
if (!dest->close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, dest->error(
));
err = true;
}
if (!file_.refresh()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error(
));
err = true;
}
}
if (!load_meta()) err = true;
calc_meta();
reorg_ = true;
if (dest != &file_) delete dest;
} else {
set_error(__FILE__, __LINE__, Error::SYSTEM, dest->error());
err = true;
}
src.close();
} else {
set_error(__FILE__, __LINE__, Error::SYSTEM, src.error());
err = true; err = true;
} }
} else { } else {
set_error(__FILE__, __LINE__, db.error().code(), "closing the des tination failed"); set_error(_KCCODELINE_, db.error().code(), "closing the destinati on failed");
err = true; err = true;
} }
} else { } else {
set_error(__FILE__, __LINE__, db.error().code(), "record copying fa iled"); set_error(_KCCODELINE_, db.error().code(), "record copying failed") ;
err = true; err = true;
} }
File::remove(npath); File::remove(npath);
} else { } else {
set_error(__FILE__, __LINE__, db.error().code(), "opening the destina tion failed"); set_error(_KCCODELINE_, db.error().code(), "opening the destination f ailed");
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Copy all records to another database. * Copy all records to another database.
* @param dest the destination database. * @param dest the destination database.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool copy_records(HashDB* dest) { bool copy_records(HashDB* dest) {
_assert_(dest); _assert_(dest);
std::ostream* erstrm = erstrm_; Logger* logger = logger_;
erstrm_ = NULL; logger_ = NULL;
int64_t off = roff_; int64_t off = roff_;
int64_t end = psiz_; int64_t end = psiz_;
Record rec; Record rec, nrec;
char rbuf[HDBRECBUFSIZ]; char rbuf[HDBRECBUFSIZ], nbuf[HDBRECBUFSIZ];
while (off > 0 && off < end) { while (off > 0 && off < end) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) break; if (!read_record(&rec, rbuf)) {
if (rec.psiz == UINT16_MAX) { int64_t checkend = off + HDBSLVGWIDTH;
off += rec.rsiz; if (checkend > end - (int64_t)rhsiz_) checkend = end - rhsiz_;
} else { bool hit = false;
if (!rec.vbuf && !read_record_body(&rec)) { for (off += rhsiz_; off < checkend; off++) {
rec.off = off;
if (!read_record(&rec, rbuf)) continue;
if ((int64_t)rec.rsiz > HDBSLVGWIDTH || rec.off + (int64_t)rec.rs
iz >= checkend) {
delete[] rec.bbuf;
continue;
}
if (rec.psiz != UINT16_MAX && !rec.vbuf && !read_record_body(&rec
)) {
delete[] rec.bbuf;
continue;
}
delete[] rec.bbuf; delete[] rec.bbuf;
nrec.off = off + rec.rsiz;
if (!read_record(&nrec, nbuf)) continue;
if ((int64_t)nrec.rsiz > HDBSLVGWIDTH || nrec.off + (int64_t)nrec
.rsiz >= checkend) {
delete[] nrec.bbuf;
continue;
}
if (nrec.psiz != UINT16_MAX && !nrec.vbuf && !read_record_body(&n
rec)) {
delete[] nrec.bbuf;
continue;
}
delete[] nrec.bbuf;
hit = true;
break; break;
} }
const char* vbuf = rec.vbuf; if (!hit || !read_record(&rec, rbuf)) break;
size_t vsiz = rec.vsiz; }
char* zbuf = NULL; if (rec.psiz == UINT16_MAX) {
size_t zsiz = 0; off += rec.rsiz;
if (comp_) { continue;
zbuf = comp_->decompress(vbuf, vsiz, &zsiz); }
if (!zbuf) { if (!rec.vbuf && !read_record_body(&rec)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, "data decompressio delete[] rec.bbuf;
n failed"); bool hit = false;
delete[] rec.bbuf; if (rec.rsiz <= MEMMAXSIZ && off + (int64_t)rec.rsiz < end) {
break; nrec.off = off + rec.rsiz;
if (read_record(&nrec, nbuf)) {
if (nrec.rsiz > MEMMAXSIZ || nrec.off + (int64_t)nrec.rsiz >= e
nd) {
delete[] nrec.bbuf;
} else if (nrec.psiz != UINT16_MAX && !nrec.vbuf && !read_recor
d_body(&nrec)) {
delete[] nrec.bbuf;
} else {
delete[] nrec.bbuf;
hit = true;
}
} }
vbuf = zbuf;
vsiz = zsiz;
} }
if (!dest->set(rec.kbuf, rec.ksiz, vbuf, vsiz)) { if (hit) {
delete[] zbuf; off += rec.rsiz;
delete[] rec.bbuf; continue;
} else {
break; break;
} }
}
const char* vbuf = rec.vbuf;
size_t vsiz = rec.vsiz;
char* zbuf = NULL;
size_t zsiz = 0;
if (comp_) {
zbuf = comp_->decompress(vbuf, vsiz, &zsiz);
if (!zbuf) {
delete[] rec.bbuf;
off += rec.rsiz;
continue;
}
vbuf = zbuf;
vsiz = zsiz;
}
if (!dest->set(rec.kbuf, rec.ksiz, vbuf, vsiz)) {
delete[] zbuf; delete[] zbuf;
delete[] rec.bbuf; delete[] rec.bbuf;
off += rec.rsiz; break;
} }
delete[] zbuf;
delete[] rec.bbuf;
off += rec.rsiz;
} }
erstrm_ = erstrm; logger_ = logger;
return true; return true;
} }
/** /**
* Trim the file size. * Trim the file size.
* @param path the path of the database file. * @param path the path of the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool trim_file(const std::string& path) { bool trim_file(const std::string& path) {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
report(__FILE__, __LINE__, "info", "trimming the database"); report(_KCCODELINE_, Logger::WARN, "trimming the database");
File* dest = writer_ ? &file_ : new File(); File* dest = writer_ ? &file_ : new File();
if (dest == &file_ || dest->open(path, File::OWRITER | File::ONOLOCK, 0 )) { if (dest == &file_ || dest->open(path, File::OWRITER | File::ONOLOCK, 0 )) {
if (!dest->truncate(lsiz_)) { if (!dest->truncate(lsiz_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, dest->error()); set_error(_KCCODELINE_, Error::SYSTEM, dest->error());
err = true; err = true;
} }
if (dest != &file_) { if (dest != &file_) {
if (!dest->close()) { if (!dest->close()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, dest->error()); set_error(_KCCODELINE_, Error::SYSTEM, dest->error());
err = true; err = true;
} }
if (!file_.refresh()) { if (!file_.refresh()) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
} }
trim_ = true; trim_ = true;
} else { } else {
set_error(__FILE__, __LINE__, Error::SYSTEM, dest->error()); set_error(_KCCODELINE_, Error::SYSTEM, dest->error());
err = true; err = true;
} }
if (dest != &file_) delete dest; if (dest != &file_) delete dest;
return !err; return !err;
} }
/** /**
* Get the hash value of a record. * Get the hash value of a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return the hash value. * @return the hash value.
*/ */
uint64_t hash_record(const char* kbuf, size_t ksiz) { uint64_t hash_record(const char* kbuf, size_t ksiz) {
_assert_(kbuf); _assert_(kbuf && ksiz <= MEMMAXSIZ);
return hashmurmur(kbuf, ksiz); return hashmurmur(kbuf, ksiz);
} }
/** /**
* Fold a hash value into a small number. * Fold a hash value into a small number.
* @param hash the hash number. * @param hash the hash number.
* @return the result number. * @return the result number.
*/ */
uint32_t fold_hash(uint64_t hash) { uint32_t fold_hash(uint64_t hash) {
_assert_(true); _assert_(true);
return (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff000 00000ULL) >> 16)) ^ return (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff000 00000ULL) >> 16)) ^
skipping to change at line 2298 skipping to change at line 2550
* Set an address into a bucket. * Set an address into a bucket.
* @param bidx the index of the bucket. * @param bidx the index of the bucket.
* @param off the address. * @param off the address.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool set_bucket(int64_t bidx, int64_t off) { bool set_bucket(int64_t bidx, int64_t off) {
_assert_(bidx >= 0 && off >= 0); _assert_(bidx >= 0 && off >= 0);
char buf[sizeof(uint64_t)]; char buf[sizeof(uint64_t)];
writefixnum(buf, off >> apow_, width_); writefixnum(buf, off >> apow_, width_);
if (!file_.write_fast(boff_ + bidx * width_, buf, width_)) { if (!file_.write_fast(boff_ + bidx * width_, buf, width_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
return true; return true;
} }
/** /**
* Get an address from a bucket. * Get an address from a bucket.
* @param bidx the index of the bucket. * @param bidx the index of the bucket.
* @return the address, or -1 on failure. * @return the address, or -1 on failure.
*/ */
int64_t get_bucket(int64_t bidx) { int64_t get_bucket(int64_t bidx) {
_assert_(bidx >= 0); _assert_(bidx >= 0);
char buf[sizeof(uint64_t)]; char buf[sizeof(uint64_t)];
if (!file_.read_fast(boff_ + bidx * width_, buf, width_)) { if (!file_.read_fast(boff_ + bidx * width_, buf, width_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long)psiz_, (long)boff_ + bidx * width_, (long)file_.size()); (long long)psiz_, (long long)boff_ + bidx * width_, (long long
)file_.size());
return -1; return -1;
} }
return readfixnum(buf, width_) << apow_; return readfixnum(buf, width_) << apow_;
} }
/** /**
* Set an address into a chain slot. * Set an address into a chain slot.
* @param entoff the address of the chain slot. * @param entoff the address of the chain slot.
* @param off the destination address. * @param off the destination address.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool set_chain(int64_t entoff, int64_t off) { bool set_chain(int64_t entoff, int64_t off) {
_assert_(entoff >= 0 && off >= 0); _assert_(entoff >= 0 && off >= 0);
char buf[sizeof(uint64_t)]; char buf[sizeof(uint64_t)];
writefixnum(buf, off >> apow_, width_); writefixnum(buf, off >> apow_, width_);
if (!file_.write_fast(entoff, buf, width_)) { if (!file_.write_fast(entoff, buf, width_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
return true; return true;
} }
/** /**
* Read a record from the file. * Read a record from the file.
* @param rec the record structure. * @param rec the record structure.
* @param rbuf the working buffer. * @param rbuf the working buffer.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool read_record(Record* rec, char* rbuf) { bool read_record(Record* rec, char* rbuf) {
_assert_(rec && rbuf); _assert_(rec && rbuf);
if (rec->off < roff_) { if (rec->off < roff_) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid record offset") set_error(_KCCODELINE_, Error::BROKEN, "invalid record offset");
; report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", (long long)psiz_, (long long)rec->off, (long long)file_.size()
(long)psiz_, (long)rec->off, (long)file_.size()); );
return false; return false;
} }
size_t rsiz = psiz_ - rec->off; size_t rsiz = psiz_ - rec->off;
if (rsiz > HDBRECBUFSIZ) { if (rsiz > HDBRECBUFSIZ) {
rsiz = HDBRECBUFSIZ; rsiz = HDBRECBUFSIZ;
} else { } else {
if (rsiz < rhsiz_) { if (rsiz < rhsiz_) {
set_error(__FILE__, __LINE__, Error::BROKEN, "too short record regi set_error(_KCCODELINE_, Error::BROKEN, "too short record region");
on"); report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz= iz=%lld",
%ld", (long long)psiz_, (long long)rec->off, (long long)rsiz, (lon
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.size()) g long)file_.size());
;
return false; return false;
} }
rsiz = rhsiz_; rsiz = rhsiz_;
} }
if (!file_.read_fast(rec->off, rbuf, rsiz)) { if (!file_.read_fast(rec->off, rbuf, rsiz)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz=%l report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz
d", =%lld",
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.size()); (long long)psiz_, (long long)rec->off, (long long)rsiz, (long
long)file_.size());
return false; return false;
} }
const char* rp = rbuf; const char* rp = rbuf;
uint16_t snum; uint16_t snum;
if (*(uint8_t*)rp == HDBRECMAGIC) { if (*(uint8_t*)rp == HDBRECMAGIC) {
((uint8_t*)&snum)[0] = 0; ((uint8_t*)&snum)[0] = 0;
((uint8_t*)&snum)[1] = *(uint8_t*)(rp + 1); ((uint8_t*)&snum)[1] = *(uint8_t*)(rp + 1);
} else if (*(uint8_t*)rp >= 0x80) { } else if (*(uint8_t*)rp >= 0x80) {
if (*(uint8_t*)(rp++) != HDBFBMAGIC || *(uint8_t*)(rp++) != HDBFBMAGI C) { if (*(uint8_t*)(rp++) != HDBFBMAGIC || *(uint8_t*)(rp++) != HDBFBMAGI C) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid magic data of set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a fre
a free block"); e block");
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz= report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs
%ld", iz=%lld",
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.size()) (long long)psiz_, (long long)rec->off, (long long)rsiz, (lon
; g long)file_.size());
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
rec->rsiz = readfixnum(rp, width_) << apow_; rec->rsiz = readfixnum(rp, width_) << apow_;
rp += width_; rp += width_;
if (*(uint8_t*)(rp++) != HDBPADMAGIC || *(uint8_t*)(rp++) != HDBPADMA GIC) { if (*(uint8_t*)(rp++) != HDBPADMAGIC || *(uint8_t*)(rp++) != HDBPADMA GIC) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid magic data of set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a fre
a free block"); e block");
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz= report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs
%ld", iz=%lld",
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.size()) (long long)psiz_, (long long)rec->off, (long long)rsiz, (lon
; g long)file_.size());
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
if (rec->rsiz < rhsiz_) { if (rec->rsiz < rhsiz_) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid size of a fre set_error(_KCCODELINE_, Error::BROKEN, "invalid size of a free bloc
e block"); k");
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz= report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs
%ld", iz=%lld",
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.size()) (long long)psiz_, (long long)rec->off, (long long)rsiz, (lon
; g long)file_.size());
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
rec->psiz = UINT16_MAX; rec->psiz = UINT16_MAX;
rec->ksiz = 0; rec->ksiz = 0;
rec->vsiz = 0; rec->vsiz = 0;
rec->left = 0; rec->left = 0;
rec->right = 0; rec->right = 0;
rec->kbuf = NULL; rec->kbuf = NULL;
rec->vbuf = NULL; rec->vbuf = NULL;
rec->boff = 0; rec->boff = 0;
rec->bbuf = NULL; rec->bbuf = NULL;
return true; return true;
} else if (*rp == 0) { } else if (*rp == 0) {
set_error(__FILE__, __LINE__, Error::BROKEN, "nullified region"); set_error(_KCCODELINE_, Error::BROKEN, "nullified region");
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz=%l report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz
d", =%lld",
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.size()); (long long)psiz_, (long long)rec->off, (long long)rsiz, (long
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rsiz); long)file_.size());
report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} else { } else {
std::memcpy(&snum, rp, sizeof(snum)); std::memcpy(&snum, rp, sizeof(snum));
} }
rp += sizeof(snum); rp += sizeof(snum);
rsiz -= sizeof(snum); rsiz -= sizeof(snum);
rec->psiz = ntoh16(snum); rec->psiz = ntoh16(snum);
rec->left = readfixnum(rp, width_) << apow_; rec->left = readfixnum(rp, width_) << apow_;
rp += width_; rp += width_;
rsiz -= width_; rsiz -= width_;
if (linear_) { if (linear_) {
rec->right = 0; rec->right = 0;
} else { } else {
rec->right = readfixnum(rp, width_) << apow_; rec->right = readfixnum(rp, width_) << apow_;
rp += width_; rp += width_;
rsiz -= width_; rsiz -= width_;
} }
uint64_t num; uint64_t num;
size_t step = readvarnum(rp, rsiz, &num); size_t step = readvarnum(rp, rsiz, &num);
if (step < 1) { if (step < 1) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid key length"); set_error(_KCCODELINE_, Error::BROKEN, "invalid key length");
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz=%l report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz
d snum=%04X", =%lld snum=%04X",
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.size(), s (long long)psiz_, (long long)rec->off, (long long)rsiz,
num); (long long)file_.size(), snum);
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
rec->ksiz = num; rec->ksiz = num;
rp += step; rp += step;
rsiz -= step; rsiz -= step;
step = readvarnum(rp, rsiz, &num); step = readvarnum(rp, rsiz, &num);
if (step < 1) { if (step < 1) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid value length"); set_error(_KCCODELINE_, Error::BROKEN, "invalid value length");
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz=%l report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fsiz
d snum=%04X", =%lld snum=%04X",
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.size(), s (long long)psiz_, (long long)rec->off, (long long)rsiz,
num); (long long)file_.size(), snum);
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rsiz); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
rec->vsiz = num; rec->vsiz = num;
rp += step; rp += step;
rsiz -= step; rsiz -= step;
size_t hsiz = rp - rbuf; size_t hsiz = rp - rbuf;
rec->rsiz = hsiz + rec->ksiz + rec->vsiz + rec->psiz; rec->rsiz = hsiz + rec->ksiz + rec->vsiz + rec->psiz;
rec->kbuf = NULL; rec->kbuf = NULL;
rec->vbuf = NULL; rec->vbuf = NULL;
rec->boff = rec->off + hsiz; rec->boff = rec->off + hsiz;
skipping to change at line 2467 skipping to change at line 2721
if (rsiz >= rec->ksiz) { if (rsiz >= rec->ksiz) {
rec->kbuf = rp; rec->kbuf = rp;
rp += rec->ksiz; rp += rec->ksiz;
rsiz -= rec->ksiz; rsiz -= rec->ksiz;
if (rsiz >= rec->vsiz) { if (rsiz >= rec->vsiz) {
rec->vbuf = rp; rec->vbuf = rp;
if (rec->psiz > 0) { if (rec->psiz > 0) {
rp += rec->vsiz; rp += rec->vsiz;
rsiz -= rec->vsiz; rsiz -= rec->vsiz;
if (rsiz > 0 && *(uint8_t*)rp != HDBPADMAGIC) { if (rsiz > 0 && *(uint8_t*)rp != HDBPADMAGIC) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid magic dat set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a
a of a record"); record");
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld f report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%ll
siz=%ld snum=%04X", d fsiz=%lld"
(long)psiz_, (long)rec->off, (long)rsiz, (long)file_.siz " snum=%04X", (long long)psiz_, (long long)rec->off, (lo
e(), snum); ng long)rsiz,
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rsiz); (long long)file_.size(), snum);
report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rsiz);
return false; return false;
} }
} }
} }
} else if (!read_record_body(rec)) { } else {
return false; if (rec->off + (int64_t)rec->rsiz > psiz_) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid length of a record"
);
report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs
iz=%lld"
" snum=%04X", (long long)psiz_, (long long)rec->off, (long l
ong)rec->rsiz,
(long long)file_.size(), snum);
return false;
}
if (!read_record_body(rec)) return false;
} }
return true; return true;
} }
/** /**
* Read the body of a record from the file. * Read the body of a record from the file.
* @param rec the record structure. * @param rec the record structure.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool read_record_body(Record* rec) { bool read_record_body(Record* rec) {
_assert_(rec); _assert_(rec);
size_t bsiz = rec->ksiz + rec->vsiz; size_t bsiz = rec->ksiz + rec->vsiz;
if (rec->psiz > 0) bsiz++;
char* bbuf = new char[bsiz]; char* bbuf = new char[bsiz];
if (!file_.read_fast(rec->boff, bbuf, bsiz)) { if (!file_.read_fast(rec->boff, bbuf, bsiz)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long)psiz_, (long)rec->boff, (long)file_.size()); (long long)psiz_, (long long)rec->boff, (long long)file_.size(
));
delete[] bbuf;
return false;
}
if (rec->psiz > 0 && ((uint8_t*)bbuf)[bsiz-1] != HDBPADMAGIC) {
set_error(_KCCODELINE_, Error::BROKEN, "invalid magic data of a recor
d");
report_binary(_KCCODELINE_, Logger::WARN, "bbuf", bbuf, bsiz);
delete[] bbuf; delete[] bbuf;
return false; return false;
} }
rec->bbuf = bbuf; rec->bbuf = bbuf;
rec->kbuf = rec->bbuf; rec->kbuf = rec->bbuf;
rec->vbuf = rec->bbuf + rec->ksiz; rec->vbuf = rec->bbuf + rec->ksiz;
return true; return true;
} }
/** /**
* Write a record into the file. * Write a record into the file.
skipping to change at line 2536 skipping to change at line 2805
std::memcpy(wp, rec->vbuf, rec->vsiz); std::memcpy(wp, rec->vbuf, rec->vsiz);
wp += rec->vsiz; wp += rec->vsiz;
if (rec->psiz > 0) { if (rec->psiz > 0) {
std::memset(wp, 0, rec->psiz); std::memset(wp, 0, rec->psiz);
*wp = HDBPADMAGIC; *wp = HDBPADMAGIC;
wp += rec->psiz; wp += rec->psiz;
} }
bool err = false; bool err = false;
if (over) { if (over) {
if (!file_.write_fast(rec->off, rbuf, rec->rsiz)) { if (!file_.write_fast(rec->off, rbuf, rec->rsiz)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
} else { } else {
if (!file_.write(rec->off, rbuf, rec->rsiz)) { if (!file_.write(rec->off, rbuf, rec->rsiz)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
} }
if (rbuf != stack) delete[] rbuf; if (rbuf != stack) delete[] rbuf;
return !err; return !err;
} }
/** /**
* Adjust the padding of a record. * Adjust the padding of a record.
* @param rec the record structure. * @param rec the record structure.
* @return true on success, or false on failure. * @return true on success, or false on failure.
skipping to change at line 2639 skipping to change at line 2908
if (!set_bucket(bidx, dest)) return false; if (!set_bucket(bidx, dest)) return false;
return true; return true;
} }
int64_t entoff = 0; int64_t entoff = 0;
Record rec; Record rec;
char rbuf[HDBRECBUFSIZ]; char rbuf[HDBRECBUFSIZ];
while (off > 0) { while (off > 0) {
rec.off = off; rec.off = off;
if (!read_record(&rec, rbuf)) return false; if (!read_record(&rec, rbuf)) return false;
if (rec.psiz == UINT16_MAX) { if (rec.psiz == UINT16_MAX) {
set_error(__FILE__, __LINE__, Error::BROKEN, "free block in the cha set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain");
in"); report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", (long long)psiz_, (long long)rec.off, (long long)file_.size(
(long)psiz_, (long)rec.off, (long)file_.size()); ));
return false; return false;
} }
uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, r ec.ksiz)); uint32_t tpivot = linear_ ? pivot : fold_hash(hash_record(rec.kbuf, r ec.ksiz));
if (pivot > tpivot) { if (pivot > tpivot) {
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.left; off = rec.left;
entoff = rec.off + sizeof(uint16_t); entoff = rec.off + sizeof(uint16_t);
} else if (pivot < tpivot) { } else if (pivot < tpivot) {
delete[] rec.bbuf; delete[] rec.bbuf;
off = rec.right; off = rec.right;
skipping to change at line 2677 skipping to change at line 2946
if (!write_record(orec, true)) return false; if (!write_record(orec, true)) return false;
if (entoff > 0) { if (entoff > 0) {
if (!set_chain(entoff, dest)) return false; if (!set_chain(entoff, dest)) return false;
} else { } else {
if (!set_bucket(bidx, dest)) return false; if (!set_bucket(bidx, dest)) return false;
} }
return true; return true;
} }
} }
} }
set_error(__FILE__, __LINE__, Error::BROKEN, "no record to shift"); set_error(_KCCODELINE_, Error::BROKEN, "no record to shift");
report(__FILE__, __LINE__, "info", "psiz=%ld fsiz=%ld", (long)psiz_, (l report(_KCCODELINE_, Logger::WARN, "psiz=%lld fsiz=%lld",
ong)file_.size()); (long long)psiz_, (long long)file_.size());
return false; return false;
} }
/** /**
* Write a free block into the file. * Write a free block into the file.
* @param off the offset of the free block. * @param off the offset of the free block.
* @param rsiz the size of the free block. * @param rsiz the size of the free block.
* @param rbuf the working buffer. * @param rbuf the working buffer.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool write_free_block(int64_t off, size_t rsiz, char* rbuf) { bool write_free_block(int64_t off, size_t rsiz, char* rbuf) {
_assert_(off >= 0 && rbuf); _assert_(off >= 0 && rbuf);
char* wp = rbuf; char* wp = rbuf;
*(wp++) = HDBFBMAGIC; *(wp++) = HDBFBMAGIC;
*(wp++) = HDBFBMAGIC; *(wp++) = HDBFBMAGIC;
writefixnum(wp, rsiz >> apow_, width_); writefixnum(wp, rsiz >> apow_, width_);
wp += width_; wp += width_;
*(wp++) = HDBPADMAGIC; *(wp++) = HDBPADMAGIC;
*(wp++) = HDBPADMAGIC; *(wp++) = HDBPADMAGIC;
if (!file_.write_fast(off, rbuf, wp - rbuf)) { if (!file_.write_fast(off, rbuf, wp - rbuf)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
return true; return true;
} }
/** /**
* Insert a free block to the free block pool. * Insert a free block to the free block pool.
* @param off the offset of the free block. * @param off the offset of the free block.
* @param rsiz the size of the free block. * @param rsiz the size of the free block.
*/ */
void insert_free_block(int64_t off, size_t rsiz) { void insert_free_block(int64_t off, size_t rsiz) {
skipping to change at line 2792 skipping to change at line 3062
for (size_t i = 0; wp < end && i < num; i++) { for (size_t i = 0; wp < end && i < num; i++) {
wp += writevarnum(wp, blocks[i].off >> apow_); wp += writevarnum(wp, blocks[i].off >> apow_);
wp += writevarnum(wp, blocks[i].rsiz >> apow_); wp += writevarnum(wp, blocks[i].rsiz >> apow_);
} }
delete[] blocks; delete[] blocks;
} }
*(wp++) = 0; *(wp++) = 0;
*(wp++) = 0; *(wp++) = 0;
bool err = false; bool err = false;
if (!file_.write(HDBHEADSIZ, rbuf, wp - rbuf)) { if (!file_.write(HDBHEADSIZ, rbuf, wp - rbuf)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
delete[] rbuf; delete[] rbuf;
return !err; return !err;
} }
/** /**
* Dump an empty set of free blocks into the file. * Dump an empty set of free blocks into the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool dump_empty_free_blocks() { bool dump_empty_free_blocks() {
_assert_(true); _assert_(true);
if (fbpnum_ < 1) return true; if (fbpnum_ < 1) return true;
char rbuf[2]; char rbuf[2];
char* wp = rbuf; char* wp = rbuf;
*(wp++) = 0; *(wp++) = 0;
*(wp++) = 0; *(wp++) = 0;
bool err = false; bool err = false;
if (!file_.write(HDBHEADSIZ, rbuf, wp - rbuf)) { if (!file_.write(HDBHEADSIZ, rbuf, wp - rbuf)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
return !err; return !err;
} }
/** /**
* Load all free blocks from from the file. * Load all free blocks from from the file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool load_free_blocks() { bool load_free_blocks() {
_assert_(true); _assert_(true);
if (fbpnum_ < 1) return true; if (fbpnum_ < 1) return true;
size_t size = boff_ - HDBHEADSIZ; size_t size = boff_ - HDBHEADSIZ;
char* rbuf = new char[size]; char* rbuf = new char[size];
if (!file_.read(HDBHEADSIZ, rbuf, size)) { if (!file_.read(HDBHEADSIZ, rbuf, size)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long)psiz_, (long)HDBHEADSIZ, (long)file_.size()); (long long)psiz_, (long long)HDBHEADSIZ, (long long)file_.size
());
delete[] rbuf; delete[] rbuf;
return false; return false;
} }
const char* rp = rbuf; const char* rp = rbuf;
FreeBlock* blocks = new FreeBlock[fbpnum_]; FreeBlock* blocks = new FreeBlock[fbpnum_];
int32_t num = 0; int32_t num = 0;
while (num < fbpnum_ && size > 1 && *rp != '\0') { while (num < fbpnum_ && size > 1 && *rp != '\0') {
uint64_t off; uint64_t off;
size_t step = readvarnum(rp, size, &off); size_t step = readvarnum(rp, size, &off);
if (step < 1 || off < 1) { if (step < 1 || off < 1) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid free block of set_error(_KCCODELINE_, Error::BROKEN, "invalid free block offset")
fset"); ;
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
(long)psiz_, (long)off, (long)file_.size()); (long long)psiz_, (long long)off, (long long)file_.size());
delete[] rbuf; delete[] rbuf;
delete[] blocks; delete[] blocks;
return false; return false;
} }
rp += step; rp += step;
size -= step; size -= step;
uint64_t rsiz; uint64_t rsiz;
step = readvarnum(rp, size, &rsiz); step = readvarnum(rp, size, &rsiz);
if (step < 1 || rsiz < 1) { if (step < 1 || rsiz < 1) {
set_error(__FILE__, __LINE__, Error::BROKEN, "invalid free block si set_error(_KCCODELINE_, Error::BROKEN, "invalid free block size");
ze"); report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld rsiz=%lld fs
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld rsiz=%ld fsiz= iz=%lld",
%ld", (long long)psiz_, (long long)off, (long long)rsiz, (long lon
(long)psiz_, (long)off, (long)rsiz, (long)file_.size()); g)file_.size());
delete[] rbuf; delete[] rbuf;
delete[] blocks; delete[] blocks;
return false; return false;
} }
rp += step; rp += step;
size -= step; size -= step;
blocks[num].off = off << apow_; blocks[num].off = off << apow_;
blocks[num].rsiz = rsiz << apow_; blocks[num].rsiz = rsiz << apow_;
num++; num++;
} }
skipping to change at line 2953 skipping to change at line 3223
child = rec->left; child = rec->left;
} else if (rec->left < 1 && rec->right > 0) { } else if (rec->left < 1 && rec->right > 0) {
child = rec->right; child = rec->right;
} else if (rec->left < 1) { } else if (rec->left < 1) {
child = 0; child = 0;
} else { } else {
Record prec; Record prec;
prec.off = rec->left; prec.off = rec->left;
if (!read_record(&prec, rbuf)) return false; if (!read_record(&prec, rbuf)) return false;
if (prec.psiz == UINT16_MAX) { if (prec.psiz == UINT16_MAX) {
set_error(__FILE__, __LINE__, Error::BROKEN, "free block in the cha set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain");
in"); report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%lld",
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", (long long)psiz_, (long long)prec.off, (long long)file_.size
(long)psiz_, (long)prec.off, (long)file_.size()); ());
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rhsiz_); report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rhsiz_);
return false; return false;
} }
delete[] prec.bbuf; delete[] prec.bbuf;
if (prec.right > 0) { if (prec.right > 0) {
int64_t off = prec.right; int64_t off = prec.right;
int64_t pentoff = prec.off + sizeof(uint16_t) + width_; int64_t pentoff = prec.off + sizeof(uint16_t) + width_;
while (true) { while (true) {
prec.off = off; prec.off = off;
if (!read_record(&prec, rbuf)) return false; if (!read_record(&prec, rbuf)) return false;
if (prec.psiz == UINT16_MAX) { if (prec.psiz == UINT16_MAX) {
set_error(__FILE__, __LINE__, Error::BROKEN, "free block in the set_error(_KCCODELINE_, Error::BROKEN, "free block in the chain
chain"); ");
report(__FILE__, __LINE__, "info", "psiz=%ld off=%ld fsiz=%ld", report(_KCCODELINE_, Logger::WARN, "psiz=%lld off=%lld fsiz=%ll
(long)psiz_, (long)prec.off, (long)file_.size()); d",
report_binary(__FILE__, __LINE__, "info", "rbuf", rbuf, rhsiz_) (long long)psiz_, (long long)prec.off, (long long)file_.
; size());
report_binary(_KCCODELINE_, Logger::WARN, "rbuf", rbuf, rhsiz_)
;
return false; return false;
} }
delete[] prec.bbuf; delete[] prec.bbuf;
if (prec.right < 1) break; if (prec.right < 1) break;
off = prec.right; off = prec.right;
pentoff = prec.off + sizeof(uint16_t) + width_; pentoff = prec.off + sizeof(uint16_t) + width_;
} }
child = off; child = off;
if (!set_chain(pentoff, prec.left)) return false; if (!set_chain(pentoff, prec.left)) return false;
if (!set_chain(off + sizeof(uint16_t), rec->left)) return false; if (!set_chain(off + sizeof(uint16_t), rec->left)) return false;
skipping to change at line 3000 skipping to change at line 3270
if (!set_bucket(bidx, child)) return false; if (!set_bucket(bidx, child)) return false;
} }
return true; return true;
} }
/** /**
* Begin transaction. * Begin transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_impl() { bool begin_transaction_impl() {
_assert_(true); _assert_(true);
if (!dump_meta()) return false; if ((count_ != trcount_ || lsiz_ != trsize_) && !dump_meta()) return fa lse;
if (!file_.begin_transaction(trhard_, boff_)) { if (!file_.begin_transaction(trhard_, boff_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
return false; return false;
} }
if (!file_.write_transaction(HDBMOFFBNUM, HDBHEADSIZ - HDBMOFFBNUM)) { if (!file_.write_transaction(HDBMOFFBNUM, HDBHEADSIZ - HDBMOFFBNUM)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
file_.end_transaction(false); file_.end_transaction(false);
return false; return false;
} }
if (fbpnum_ > 0) { if (fbpnum_ > 0) {
FBP::const_iterator it = fbp_.end(); FBP::const_iterator it = fbp_.end();
FBP::const_iterator itbeg = fbp_.begin(); FBP::const_iterator itbeg = fbp_.begin();
for (int32_t cnt = fpow_ * 2 + 1; cnt > 0; cnt--) { for (int32_t cnt = fpow_ * 2 + 1; cnt > 0; cnt--) {
if (it == itbeg) break; if (it == itbeg) break;
it--; it--;
trfbp_.insert(*it); trfbp_.insert(*it);
skipping to change at line 3029 skipping to change at line 3299
return true; return true;
} }
/** /**
* Begin auto transaction. * Begin auto transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_auto_transaction() { bool begin_auto_transaction() {
_assert_(true); _assert_(true);
atlock_.lock(); atlock_.lock();
if (!file_.begin_transaction(autosync_, boff_)) { if (!file_.begin_transaction(autosync_, boff_)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
atlock_.unlock(); atlock_.unlock();
return false; return false;
} }
if (!file_.write_transaction(HDBMOFFCOUNT, HDBMOFFOPAQUE - HDBMOFFCOUNT )) { if (!file_.write_transaction(HDBMOFFCOUNT, HDBMOFFOPAQUE - HDBMOFFCOUNT )) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
file_.end_transaction(false); file_.end_transaction(false);
atlock_.unlock(); atlock_.unlock();
return false; return false;
} }
return true; return true;
} }
/** /**
* Commit transaction. * Commit transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool commit_transaction() { bool commit_transaction() {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
if (!dump_meta()) err = true; if ((count_ != trcount_ || lsiz_ != trsize_) && !dump_auto_meta()) err = true;
if (!file_.end_transaction(true)) { if (!file_.end_transaction(true)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
trfbp_.clear(); trfbp_.clear();
return !err; return !err;
} }
/** /**
* Commit auto transaction. * Commit auto transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool commit_auto_transaction() { bool commit_auto_transaction() {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
if (!dump_auto_meta()) err = true; if ((count_ != trcount_ || lsiz_ != trsize_) && !dump_auto_meta()) err = true;
if (!file_.end_transaction(true)) { if (!file_.end_transaction(true)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
atlock_.unlock(); atlock_.unlock();
return !err; return !err;
} }
/** /**
* Abort transaction. * Abort transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool abort_transaction() { bool abort_transaction() {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
if (!file_.end_transaction(false)) { if (!file_.end_transaction(false)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
bool flagopen = flagopen_;
if (!load_meta()) err = true; if (!load_meta()) err = true;
flagopen_ = flagopen;
calc_meta(); calc_meta();
disable_cursors(); disable_cursors();
fbp_.swap(trfbp_); fbp_.swap(trfbp_);
trfbp_.clear(); trfbp_.clear();
return !err; return !err;
} }
/** /**
* Abort auto transaction. * Abort auto transaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool abort_auto_transaction() { bool abort_auto_transaction() {
_assert_(true); _assert_(true);
bool err = false; bool err = false;
if (!file_.end_transaction(false)) { if (!file_.end_transaction(false)) {
set_error(__FILE__, __LINE__, Error::SYSTEM, file_.error()); set_error(_KCCODELINE_, Error::SYSTEM, file_.error());
err = true; err = true;
} }
if (!load_meta()) err = true; if (!load_meta()) err = true;
calc_meta(); calc_meta();
disable_cursors(); disable_cursors();
fbp_.clear(); fbp_.clear();
atlock_.unlock(); atlock_.unlock();
return !err; return !err;
} }
/** Dummy constructor to forbid the use. */
HashDB(const HashDB&);
/** Dummy Operator to forbid the use. */
HashDB& operator =(const HashDB&);
/** The method lock. */ /** The method lock. */
SpinRWLock mlock_; SpinRWLock mlock_;
/** The record locks. */ /** The record locks. */
SlottedSpinRWLock<HDBRLOCKSLOT> rlock_; SlottedSpinRWLock<HDBRLOCKSLOT> rlock_;
/** The file lock. */ /** The file lock. */
SpinLock flock_; SpinLock flock_;
/** The auto transaction lock. */ /** The auto transaction lock. */
Mutex atlock_; Mutex atlock_;
/** The last happened error. */ /** The last happened error. */
TSD<Error> error_; TSD<Error> error_;
/** The internal error reporter. */ /** The internal logger. */
std::ostream* erstrm_; Logger* logger_;
/** The flag to report all errors. */ /** The kinds of logged messages. */
bool ervbs_; uint32_t logkinds_;
/** The internal meta operation trigger. */
MetaTrigger* mtrigger_;
/** The open mode. */ /** The open mode. */
uint32_t omode_; uint32_t omode_;
/** The flag for writer. */ /** The flag for writer. */
bool writer_; bool writer_;
/** The flag for auto transaction. */ /** The flag for auto transaction. */
bool autotran_; bool autotran_;
/** The flag for auto synchronization. */ /** The flag for auto synchronization. */
bool autosync_; bool autosync_;
/** The flag for reorganized. */ /** The flag for reorganized. */
bool reorg_; bool reorg_;
skipping to change at line 3176 skipping to change at line 3454
AtomicInt64 lsiz_; AtomicInt64 lsiz_;
/** The physical size of the file. */ /** The physical size of the file. */
AtomicInt64 psiz_; AtomicInt64 psiz_;
/** The opaque data. */ /** The opaque data. */
char opaque_[HDBHEADSIZ-HDBMOFFOPAQUE]; char opaque_[HDBHEADSIZ-HDBMOFFOPAQUE];
/** The size of the internal memory-mapped region. */ /** The size of the internal memory-mapped region. */
int64_t msiz_; int64_t msiz_;
/** The unit step number of auto defragmentation. */ /** The unit step number of auto defragmentation. */
int64_t dfunit_; int64_t dfunit_;
/** The embedded data compressor. */ /** The embedded data compressor. */
Compressor *embcomp_; Compressor* embcomp_;
/** The alignment of records. */ /** The alignment of records. */
size_t align_; size_t align_;
/** The number of elements of the free block pool. */ /** The number of elements of the free block pool. */
int32_t fbpnum_; int32_t fbpnum_;
/** The width of record addressing. */ /** The width of record addressing. */
int32_t width_; int32_t width_;
/** The flag for linear collision chaining. */ /** The flag for linear collision chaining. */
bool linear_; bool linear_;
/** The data compressor. */ /** The data compressor. */
Compressor* comp_; Compressor* comp_;
skipping to change at line 3203 skipping to change at line 3481
/** The defrag cursor. */ /** The defrag cursor. */
int64_t dfcur_; int64_t dfcur_;
/** The count of fragmentation. */ /** The count of fragmentation. */
AtomicInt64 frgcnt_; AtomicInt64 frgcnt_;
/** The flag whether in transaction. */ /** The flag whether in transaction. */
bool tran_; bool tran_;
/** The flag whether hard transaction. */ /** The flag whether hard transaction. */
bool trhard_; bool trhard_;
/** The escaped free block pool for transaction. */ /** The escaped free block pool for transaction. */
FBP trfbp_; FBP trfbp_;
/** The count history for transaction. */
int64_t trcount_;
/** The size history for transaction. */
int64_t trsize_;
}; };
/** An alias of the file tree database. */
typedef PlantDB<HashDB, BasicDB::TYPETREE> TreeDB;
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 247 change blocks. 
433 lines changed or deleted 742 lines changed or added


 kclangc.h   kclangc.h 
skipping to change at line 53 skipping to change at line 53
void* db; /**< dummy member */ void* db; /**< dummy member */
} KCDB; } KCDB;
/** /**
* Error codes. * Error codes.
*/ */
enum { enum {
KCESUCCESS, /**< success */ KCESUCCESS, /**< success */
KCENOIMPL, /**< not implemented */ KCENOIMPL, /**< not implemented */
KCEINVALID, /**< invalid operation */ KCEINVALID, /**< invalid operation */
KCENOFILE, /**< file not found */ KCENOREPOS, /**< no repository */
KCENOPERM, /**< no permission */ KCENOPERM, /**< no permission */
KCEBROKEN, /**< broken file */ KCEBROKEN, /**< broken file */
KCEDUPREC, /**< record duplication */ KCEDUPREC, /**< record duplication */
KCENOREC, /**< no record */ KCENOREC, /**< no record */
KCELOGIC, /**< logical inconsistency */ KCELOGIC, /**< logical inconsistency */
KCESYSTEM, /**< system error */ KCESYSTEM, /**< system error */
KCEMISC = 15 /**< miscellaneous error */ KCEMISC = 15 /**< miscellaneous error */
}; };
/** /**
skipping to change at line 78 skipping to change at line 78
KCOWRITER = 1 << 1, /**< open as a writer */ KCOWRITER = 1 << 1, /**< open as a writer */
KCOCREATE = 1 << 2, /**< writer creating */ KCOCREATE = 1 << 2, /**< writer creating */
KCOTRUNCATE = 1 << 3, /**< writer truncating */ KCOTRUNCATE = 1 << 3, /**< writer truncating */
KCOAUTOTRAN = 1 << 4, /**< auto transaction */ KCOAUTOTRAN = 1 << 4, /**< auto transaction */
KCOAUTOSYNC = 1 << 5, /**< auto synchronization */ KCOAUTOSYNC = 1 << 5, /**< auto synchronization */
KCONOLOCK = 1 << 6, /**< open without locking */ KCONOLOCK = 1 << 6, /**< open without locking */
KCOTRYLOCK = 1 << 7, /**< lock without blocking */ KCOTRYLOCK = 1 << 7, /**< lock without blocking */
KCONOREPAIR = 1 << 8 /**< open without auto repair */ KCONOREPAIR = 1 << 8 /**< open without auto repair */
}; };
/**
* Merge modes.
*/
enum {
KCMSET, /**< overwrite the existing value
*/
KCMADD, /**< keep the existing value */
KCMREPLACE, /**< modify the existing record on
ly */
KCMAPPEND /**< append the new value */
};
/** The package version. */ /** The package version. */
extern const char* const KCVERSION; extern const char* const KCVERSION;
/** Special pointer for no operation by the visiting function. */ /** Special pointer for no operation by the visiting function. */
extern const char* const KCVISNOP; extern const char* const KCVISNOP;
/** Special pointer to remove the record by the visiting function. */ /** Special pointer to remove the record by the visiting function. */
extern const char* const KCVISREMOVE; extern const char* const KCVISREMOVE;
/** /**
skipping to change at line 137 skipping to change at line 147
typedef union { typedef union {
void* cur; /**< dummy member */ void* cur; /**< dummy member */
} KCCUR; } KCCUR;
/** /**
* Allocate a region on memory. * Allocate a region on memory.
* @param size the size of the region. * @param size the size of the region.
* @return the pointer to the allocated region. The region of the return v alue should be * @return the pointer to the allocated region. The region of the return v alue should be
* released with the kcfree function when it is no longer in use. * released with the kcfree function when it is no longer in use.
*/ */
char* kcmalloc(size_t size); void* kcmalloc(size_t size);
/** /**
* Release a region allocated in the library. * Release a region allocated in the library.
* @param ptr the pointer to the region. * @param ptr the pointer to the region.
*/ */
void kcfree(char* ptr); void kcfree(void* ptr);
/** /**
* Get the time of day in seconds. * Get the time of day in seconds.
* @return the time of day in seconds. The accuracy is in microseconds. * @return the time of day in seconds. The accuracy is in microseconds.
*/ */
double kctime(void); double kctime(void);
/** /**
* Convert a string to an integer. * Convert a string to an integer.
* @param str specifies the string. * @param str specifies the string.
skipping to change at line 242 skipping to change at line 252
* Destroy a database object. * Destroy a database object.
* @param db the database object. * @param db the database object.
*/ */
void kcdbdel(KCDB* db); void kcdbdel(KCDB* db);
/** /**
* Open a database file. * Open a database file.
* @param db a database object. * @param db a database object.
* @param path the path of a database file. If it is "-", the database wil l be a prototype * @param path the path of a database file. If it is "-", the database wil l be a prototype
* hash database. If it is "+", the database will be a prototype tree data base. If it is * hash database. If it is "+", the database will be a prototype tree data base. If it is
* "*", the database will be a cache database. If its suffix is ".kch", th * "*", the database will be a cache hash database. If it is "%", the data
e database will be base will be a
* a file hash database. If its suffix is ".kct", the database will be a f * cache tree database. If its suffix is ".kch", the database will be a fi
ile tree database. le hash database.
* If its suffix is ".kcd", the database will be a directory database. Oth * If its suffix is ".kct", the database will be a file tree database. If
erwise, this its suffix is
* function fails. Tuning parameters can trail the name, separated by "#". * ".kcd", the database will be a directory hash database. If its suffix i
Each parameter is s ".kcf", the
* composed of the name and the value, separated by "=". If the "type" par * database will be a directory tree database. Otherwise, this function fa
ameter is specified, ils. Tuning
* the database type is determined by the value in "-", "+", "*", "kch", "k * parameters can trail the name, separated by "#". Each parameter is comp
ct", and "kcd". The osed of the name
* and the value, separated by "=". If the "type" parameter is specified,
the database type
* is determined by the value in "-", "+", "*", "%", "kch", "kct", "kcd", a
nd "kcf". All
* database types support the logging parameters of "log", "logkinds", and
"logpx". The
* prototype hash database and the prototype tree database do not support a ny other tuning * prototype hash database and the prototype tree database do not support a ny other tuning
* parameter. The cache database supports "bnum", "capcount", and "capsize * parameter. The cache hash database supports "opts", "bnum", "zcomp", "c
". The file hash apcount", "capsize",
* database supports "apow", "fpow", "opts", "bnum", "msiz", "dfunit", "zco * and "zkey". The cache tree database supports all parameters of the cach
mp", "erstrm", e hash database
* "ervbs", and "zkey". The file tree database supports all parameters of * except for capacity limitation, and supports "psiz", "rcomp", "pccap" in
the file hash addition. The
* database and "psiz", "rcomp", "pccap" in addition. The directory databa * file hash database supports "apow", "fpow", "opts", "bnum", "msiz", "dfu
se supports "opts", nit", "zcomp", and
* "zcomp", and "zkey". * "zkey". The file tree database supports all parameters of the file hash
database and
* "psiz", "rcomp", "pccap" in addition. The directory hash database suppo
rts "opts", "zcomp",
* and "zkey". The directory tree database supports all parameters of the
directory hash
* database and "psiz", "rcomp", "pccap" in addition.
* @param mode the connection mode. KCOWRITER as a writer, KCOREADER as a reader. * @param mode the connection mode. KCOWRITER as a writer, KCOREADER as a reader.
* The following may be added to the writer mode by bitwise-or: KCOCREATE, which means * The following may be added to the writer mode by bitwise-or: KCOCREATE, which means
* it creates a new database if the file does not exist, KCOTRUNCATE, which means it * it creates a new database if the file does not exist, KCOTRUNCATE, which means it
* creates a new database regardless if the file exists, KCOAUTOTRAN, which means each * creates a new database regardless if the file exists, KCOAUTOTRAN, which means each
* updating operation is performed in implicit transaction, KCOAUTOSYNC, wh ich means * updating operation is performed in implicit transaction, KCOAUTOSYNC, wh ich means
* each updating operation is followed by implicit synchronization with the file system. The * each updating operation is followed by implicit synchronization with the file system. The
* following may be added to both of the reader mode and the writer mode by bitwise-or: * following may be added to both of the reader mode and the writer mode by bitwise-or:
* KCONOLOCK, which means it opens the database file without file locking, * KCONOLOCK, which means it opens the database file without file locking,
* KCOTRYLOCK, which means locking is performed without blocking, KCONOREPA IR, which * KCOTRYLOCK, which means locking is performed without blocking, KCONOREPA IR, which
* means the database file is not repaired implicitly even if file destruct ion is detected. * means the database file is not repaired implicitly even if file destruct ion is detected.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The tuning parameter "bnum" corresponds to the original "tune_buck * @note The tuning parameter "log" is for the original "tune_logger" and t
et" method. he value specifies
* "capcount" is for "cap_count". "capsize" is for "cap_size". "apow" is * the path of the log file, or "-" for the standard output, or "+" for the
for standard error.
* "tune_alignment". "fpow" is for "tune_fbp". "opts" is for "tune_option * "logkinds" specifies kinds of logged messages and the value can be "debu
s" and the value g", "info", "warn",
* can contain "s" for the small option, "l" for the linear option, and "c" * or "error". "logpx" specifies the prefix of each log message. "opts" i
for the compress s for "tune_options"
* option. "msiz" is for "tune_map". "dfunit" is for "tune_defrag". "zco * and the value can contain "s" for the small option, "l" for the linear o
mp" is for ption, and "c" for
* "tune_compressor" and the value can be "zlib" for the Zlib raw compresso * the compress option. "bnum" corresponds to "tune_bucket". "zcomp" is f
r, "def" for the or "tune_compressor"
* Zlib deflate compressor, "gz" for the Zlib gzip compressor, or "arc" for * and the value can be "zlib" for the ZLIB raw compressor, "def" for the Z
the Arcfour cipher. LIB deflate
* "erstrm" and "ervbs" are for "tune_error_reporter" and the formar value * compressor, "gz" for the ZLIB gzip compressor, "lzo" for the LZO compres
can be "stdout" or sor, "lzma" for the
* "stderr" and the latter value can be "true" or "false". "zkey" specifie * LZMA compressor, or "arc" for the Arcfour cipher. "zkey" specifies the
s the cipher key of cipher key of the
* the compressor. "psiz" is for "tune_page". "rcomp" is for "tune_compar * compressor. "capcount" is for "cap_count". "capsize" is for "cap_size"
ator" and the value . "psiz" is for
* can be "lex" for the lexical comparator or "dec" for the decimal compara * "tune_page". "rcomp" is for "tune_comparator" and the value can be "lex
tor. "pccap" is for " for the lexical
* "tune_page_cache". Every opened database must be closed by the PolyDB:: * comparator or "dec" for the decimal comparator. "pccap" is for "tune_pa
close method when it ge_cache". "apow"
* is no longer in use. It is not allowed for two or more database objects * is for "tune_alignment". "fpow" is for "tune_fbp". "msiz" is for "tune
in the same process _map". "dfunit" is
* to keep their connections to the same database file at the same time. * for "tune_defrag". Every opened database must be closed by the kcdbclos
e method when it is
* no longer in use. It is not allowed for two or more database objects in
the same process to
* keep their connections to the same database file at the same time.
*/ */
int32_t kcdbopen(KCDB* db, const char* path, uint32_t mode); int32_t kcdbopen(KCDB* db, const char* path, uint32_t mode);
/** /**
* Close the database file. * Close the database file.
* @param db a database object. * @param db a database object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
int32_t kcdbclose(KCDB* db); int32_t kcdbclose(KCDB* db);
skipping to change at line 357 skipping to change at line 375
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param vbuf the pointer to the value region. * @param vbuf the pointer to the value region.
* @param vsiz the size of the value region. * @param vsiz the size of the value region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note If no record corresponds to the key, a new record is created. If the corresponding * @note If no record corresponds to the key, a new record is created. If the corresponding
* record exists, the record is not modified and false is returned. * record exists, the record is not modified and false is returned.
*/ */
int32_t kcdbadd(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz); int32_t kcdbadd(KCDB* db, const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz);
/** /**
* Replace the value of a record.
* @param db a database object.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @param vbuf the pointer to the value region.
* @param vsiz the size of the value region.
* @return true on success, or false on failure.
* @note If no record corresponds to the key, no new record is created and
false is returned.
* If the corresponding record exists, the value is modified.
*/
int32_t kcdbreplace(KCDB* db, const char* kbuf, size_t ksiz, const char* vb
uf, size_t vsiz);
/**
* Append the value of a record. * Append the value of a record.
* @param db a database object. * @param db a database object.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param vbuf the pointer to the value region. * @param vbuf the pointer to the value region.
* @param vsiz the size of the value region. * @param vsiz the size of the value region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note If no record corresponds to the key, a new record is created. If the corresponding * @note If no record corresponds to the key, a new record is created. If the corresponding
* record exists, the given value is appended at the end of the existing va lue. * record exists, the given value is appended at the end of the existing va lue.
*/ */
skipping to change at line 543 skipping to change at line 574
* Get the miscellaneous status information. * Get the miscellaneous status information.
* @param db a database object. * @param db a database object.
* @return the result string of tab saparated values, or NULL on failure. Each line consists of * @return the result string of tab saparated values, or NULL on failure. Each line consists of
* the attribute name and its value separated by a tab character. * the attribute name and its value separated by a tab character.
* @note The region of the return value should be released with the kcfree function when it is * @note The region of the return value should be released with the kcfree function when it is
* no longer in use. * no longer in use.
*/ */
char* kcdbstatus(KCDB* db); char* kcdbstatus(KCDB* db);
/** /**
* Get keys matching a prefix string.
* @param db a database object.
* @param prefix the prefix string.
* @param strary an array to contain the result. Its size must be sufficie
nt.
* @param max the maximum number to retrieve.
* @return the number of retrieved keys or -1 on failure.
* @note The region of each element of the result should be released with t
he kcfree function
* when it is no longer in use.
*/
int64_t kcdbmatchprefix(KCDB* db, const char* prefix, char** strary, int64_
t max);
/**
* Get keys matching a regular expression string.
* @param db a database object.
* @param regex the regular expression string.
* @param strary an array to contain the result. Its size must be sufficie
nt.
* @param max the maximum number to retrieve.
* @return the number of retrieved keys or -1 on failure.
* @note The region of each element of the result should be released with t
he kcfree function
* when it is no longer in use.
*/
int64_t kcdbmatchregex(KCDB* db, const char* regex, char** strary, int64_t
max);
/**
* Merge records from other databases.
* @param db a database object.
* @param srcary an array of the source detabase objects.
* @param srcnum the number of the elements of the source array.
* @param mode the merge mode. KCMSET to overwrite the existing value, KCM
ADD to keep the
* existing value, KCMREPLACE to modify the existing record only, KCMAPPEND
to append the new
* value.
* @return true on success, or false on failure.
*/
int32_t kcdbmerge(KCDB* db, KCDB** srcary, size_t srcnum, uint32_t mode);
/**
* Create a cursor object. * Create a cursor object.
* @param db a database object. * @param db a database object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note The object of the return value should be released with the kccurde l function when it is * @note The object of the return value should be released with the kccurde l function when it is
* no longer in use. * no longer in use.
*/ */
KCCUR* kcdbcursor(KCDB* db); KCCUR* kcdbcursor(KCDB* db);
/** /**
* Destroy a cursor object. * Destroy a cursor object.
skipping to change at line 635 skipping to change at line 702
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return the pointer to the pair of the key region, or NULL on failure. * @return the pointer to the pair of the key region, or NULL on failure.
* @note If the cursor is invalidated, NULL is returned. Because an additi onal zero code is * @note If the cursor is invalidated, NULL is returned. Because an additi onal zero code is
* appended at the end of each region of the key and the value, each region can be treated * appended at the end of each region of the key and the value, each region can be treated
* as a C-style string. The region of the return value should be released with the kcfree * as a C-style string. The region of the return value should be released with the kcfree
* function when it is no longer in use. * function when it is no longer in use.
*/ */
char* kccurget(KCCUR* cur, size_t* ksp, const char** vbp, size_t* vsp, int3 2_t step); char* kccurget(KCCUR* cur, size_t* ksp, const char** vbp, size_t* vsp, int3 2_t step);
/** /**
* Jump the cursor to the first record. * Jump the cursor to the first record for forward scan.
* @param cur a cursor object. * @param cur a cursor object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
int32_t kccurjump(KCCUR* cur); int32_t kccurjump(KCCUR* cur);
/** /**
* Jump the cursor to a record. * Jump the cursor to a record for forward scan.
* @param cur a cursor object. * @param cur a cursor object.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
int32_t kccurjumpkey(KCCUR* cur, const char* kbuf, size_t ksiz); int32_t kccurjumpkey(KCCUR* cur, const char* kbuf, size_t ksiz);
/** /**
* Jump the cursor to the last record for backward scan.
* @param cur a cursor object.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database types,
especially hash
* databases, may provide a dummy implementation.
*/
int32_t kccurjumpback(KCCUR* cur);
/**
* Jump the cursor to a record for backward scan.
* @param cur a cursor object.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database types,
especially hash
* databases, will provide a dummy implementation.
*/
int32_t kccurjumpbackkey(KCCUR* cur, const char* kbuf, size_t ksiz);
/**
* Step the cursor to the next record. * Step the cursor to the next record.
* @param cur a cursor object. * @param cur a cursor object.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
int32_t kccurstep(KCCUR* cur); int32_t kccurstep(KCCUR* cur);
/** /**
* Step the cursor to the previous record.
* @param cur a cursor object.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database types,
especially hash
* databases, may provide a dummy implementation.
*/
int32_t kccurstepback(KCCUR* cur);
/**
* Get the database object. * Get the database object.
* @param cur a cursor object. * @param cur a cursor object.
* @return the database object. * @return the database object.
*/ */
KCDB* kccurdb(KCCUR* cur); KCDB* kccurdb(KCCUR* cur);
/** /**
* Get the code of the last happened error. * Get the code of the last happened error.
* @param cur a cursor object. * @param cur a cursor object.
* @return the code of the last happened error. * @return the code of the last happened error.
 End of changes. 13 change blocks. 
53 lines changed or deleted 172 lines changed or added


 kcmap.h   kcmap.h 
skipping to change at line 27 skipping to change at line 27
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation. * Constants for implementation.
*/ */
namespace { namespace {
const size_t LHMDEFBNUM = 31; ///< default bucket number of hash const size_t MAPDEFBNUM = 31; ///< default bucket number of hash
table table
const size_t LHMZMAPBNUM = 32768; ///< mininum number of buckets to const size_t MAPZMAPBNUM = 32768; ///< mininum number of buckets to
use mmap use mmap
} }
/** /**
* Double linked hash map. * Memory-saving hash map.
*/
class TinyHashMap {
public:
class Iterator;
private:
struct Record;
struct RecordComparator;
public:
/**
* Iterator of records.
*/
class Iterator {
friend class TinyHashMap;
public:
/**
* Constructor.
* @param map the container.
*/
explicit Iterator(TinyHashMap* map) : map_(map), bidx_(-1), ridx_(0), r
ecs_() {
_assert_(map);
step();
}
/**
* Destructor.
*/
~Iterator() {
_assert_(true);
free_records();
}
/**
* Get the key of the current record.
* @param sp the pointer to the variable into which the size of the reg
ion of the return
* value is assigned.
* @return the pointer to the key region of the current record, or NULL
on failure.
*/
const char* get_key(size_t* sp) {
_assert_(sp);
if (ridx_ >= recs_.size()) return NULL;
Record rec(recs_[ridx_]);
*sp = rec.ksiz_;
return rec.kbuf_;
}
/**
* Get the value of the current record.
* @param sp the pointer to the variable into which the size of the reg
ion of the return
* value is assigned.
* @return the pointer to the value region of the current record, or NU
LL on failure.
*/
const char* get_value(size_t* sp) {
_assert_(sp);
if (ridx_ >= recs_.size()) return NULL;
Record rec(recs_[ridx_]);
*sp = rec.vsiz_;
return rec.vbuf_;
}
/**
* Get a pair of the key and the value of the current record.
* @param ksp the pointer to the variable into which the size of the re
gion of the return
* value is assigned.
* @param vbp the pointer to the variable into which the pointer to the
value region is
* assigned.
* @param vsp the pointer to the variable into which the size of the va
lue region is
* assigned.
* @return the pointer to the key region, or NULL on failure.
*/
const char* get(size_t* ksp, const char** vbp, size_t* vsp) {
_assert_(ksp && vbp && vsp);
if (ridx_ >= recs_.size()) return NULL;
Record rec(recs_[ridx_]);
*ksp = rec.ksiz_;
*vbp = rec.vbuf_;
*vsp = rec.vsiz_;
return rec.kbuf_;
}
/**
* Step the cursor to the next record.
*/
void step() {
_assert_(true);
if (++ridx_ >= recs_.size()) {
ridx_ = 0;
free_records();
while (true) {
bidx_++;
if (bidx_ >= (int64_t)map_->bnum_) return;
read_records();
if (recs_.size() > 0) break;
}
}
}
private:
/**
* Read records of the current bucket.
*/
void read_records() {
char* rbuf = map_->buckets_[bidx_];
while (rbuf) {
Record rec(rbuf);
size_t rsiz = sizeof(rec.child_) + sizevarnum(rec.ksiz_) + rec.ksiz
_ +
sizevarnum(rec.vsiz_) + rec.vsiz_ + sizevarnum(rec.psiz_);
char* nbuf = new char[rsiz];
std::memcpy(nbuf, rbuf, rsiz);
recs_.push_back(nbuf);
rbuf = rec.child_;
}
}
/**
* Release recources of the current records.
*/
void free_records() {
std::vector<char*>::iterator it = recs_.begin();
std::vector<char*>::iterator itend = recs_.end();
while (it != itend) {
char* rbuf = *it;
delete[] rbuf;
it++;
}
recs_.clear();
}
/** Dummy constructor to forbid the use. */
Iterator(const Iterator&);
/** Dummy Operator to forbid the use. */
Iterator& operator =(const Iterator&);
/** The container. */
TinyHashMap* map_;
/** The current bucket index. */
int64_t bidx_;
/** The current record index. */
size_t ridx_;
/** The current records. */
std::vector<char*> recs_;
};
/**
* Sorter of records.
*/
class Sorter {
public:
/**
* Constructor.
* @param map the container.
*/
explicit Sorter(TinyHashMap* map) : map_(map), ridx_(0), recs_() {
_assert_(map);
recs_.reserve(map->count_);
char** buckets = map_->buckets_;
size_t bnum = map_->bnum_;
for (size_t i = 0; i < bnum; i++) {
char* rbuf = buckets[i];
while (rbuf) {
Record rec(rbuf);
recs_.push_back(rbuf);
rbuf = *(char**)rbuf;
}
}
std::sort(recs_.begin(), recs_.end(), RecordComparator());
}
/**
* Destructor.
*/
~Sorter() {
_assert_(true);
}
/**
* Get the key of the current record.
* @param sp the pointer to the variable into which the size of the reg
ion of the return
* value is assigned.
* @return the pointer to the key region of the current record, or NULL
on failure.
*/
const char* get_key(size_t* sp) {
_assert_(sp);
if (ridx_ >= recs_.size()) return NULL;
Record rec(recs_[ridx_]);
*sp = rec.ksiz_;
return rec.kbuf_;
}
/**
* Get the value of the current record.
* @param sp the pointer to the variable into which the size of the reg
ion of the return
* value is assigned.
* @return the pointer to the value region of the current record, or NU
LL on failure.
*/
const char* get_value(size_t* sp) {
_assert_(sp);
if (ridx_ >= recs_.size()) return NULL;
Record rec(recs_[ridx_]);
*sp = rec.vsiz_;
return rec.vbuf_;
}
/**
* Get a pair of the key and the value of the current record.
* @param ksp the pointer to the variable into which the size of the re
gion of the return
* value is assigned.
* @param vbp the pointer to the variable into which the pointer to the
value region is
* assigned.
* @param vsp the pointer to the variable into which the size of the va
lue region is
* assigned.
* @return the pointer to the key region, or NULL on failure.
*/
const char* get(size_t* ksp, const char** vbp, size_t* vsp) {
_assert_(ksp && vbp && vsp);
if (ridx_ >= recs_.size()) return NULL;
Record rec(recs_[ridx_]);
*ksp = rec.ksiz_;
*vbp = rec.vbuf_;
*vsp = rec.vsiz_;
return rec.kbuf_;
}
/**
* Step the cursor to the next record.
*/
void step() {
_assert_(true);
ridx_++;
}
/** The container. */
TinyHashMap* map_;
/** The current record index. */
size_t ridx_;
/** The current records. */
std::vector<char*> recs_;
};
/**
* Default constructor.
*/
explicit TinyHashMap() : buckets_(NULL), bnum_(MAPDEFBNUM), count_(0) {
_assert_(true);
initialize();
}
/**
* Constructor.
* @param bnum the number of buckets of the hash table.
*/
explicit TinyHashMap(size_t bnum) : buckets_(NULL), bnum_(bnum), count_(0
) {
_assert_(true);
if (bnum_ < 1) bnum_ = MAPDEFBNUM;
initialize();
}
/**
* Destructor.
*/
~TinyHashMap() {
_assert_(true);
destroy();
}
/**
* Set the value of a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @param vbuf the pointer to the value region.
* @param vsiz the size of the value region.
* @note If no record corresponds to the key, a new record is created. I
f the corresponding
* record exists, the value is overwritten.
*/
void set(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ);
size_t bidx = hash_record(kbuf, ksiz) % bnum_;
char* rbuf = buckets_[bidx];
char** entp = buckets_ + bidx;
while (rbuf) {
Record rec(rbuf);
if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) {
int32_t oh = (int32_t)sizevarnum(vsiz) - (int32_t)sizevarnum(rec.vs
iz_);
int64_t psiz = (int64_t)(rec.vsiz_ + rec.psiz_) - (int64_t)(vsiz +
oh);
if (psiz >= 0) {
rec.overwrite(rbuf, vbuf, vsiz, psiz);
} else {
Record nrec(rec.child_, kbuf, ksiz, vbuf, vsiz, 0);
delete[] rbuf;
*entp = nrec.serialize();
}
return;
}
entp = (char**)rbuf;
rbuf = rec.child_;
}
Record nrec(NULL, kbuf, ksiz, vbuf, vsiz, 0);
*entp = nrec.serialize();
count_++;
}
/**
* Add a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @param vbuf the pointer to the value region.
* @param vsiz the size of the value region.
* @return true on success, or false on failure.
* @note If no record corresponds to the key, a new record is created. I
f the corresponding
* record exists, the record is not modified and false is returned.
*/
bool add(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ);
size_t bidx = hash_record(kbuf, ksiz) % bnum_;
char* rbuf = buckets_[bidx];
char** entp = buckets_ + bidx;
while (rbuf) {
Record rec(rbuf);
if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) return
false;
entp = (char**)rbuf;
rbuf = rec.child_;
}
Record nrec(NULL, kbuf, ksiz, vbuf, vsiz, 0);
*entp = nrec.serialize();
count_++;
return true;
}
/**
* Replace the value of a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @param vbuf the pointer to the value region.
* @param vsiz the size of the value region.
* @return true on success, or false on failure.
* @note If no record corresponds to the key, no new record is created an
d false is returned.
* If the corresponding record exists, the value is modified.
*/
bool replace(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz
) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ);
size_t bidx = hash_record(kbuf, ksiz) % bnum_;
char* rbuf = buckets_[bidx];
char** entp = buckets_ + bidx;
while (rbuf) {
Record rec(rbuf);
if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) {
int32_t oh = (int32_t)sizevarnum(vsiz) - (int32_t)sizevarnum(rec.vs
iz_);
int64_t psiz = (int64_t)(rec.vsiz_ + rec.psiz_) - (int64_t)(vsiz +
oh);
if (psiz >= 0) {
rec.overwrite(rbuf, vbuf, vsiz, psiz);
} else {
Record nrec(rec.child_, kbuf, ksiz, vbuf, vsiz, 0);
delete[] rbuf;
*entp = nrec.serialize();
}
return true;
}
entp = (char**)rbuf;
rbuf = rec.child_;
}
return false;
}
/**
* Append the value of a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @param vbuf the pointer to the value region.
* @param vsiz the size of the value region.
* @note If no record corresponds to the key, a new record is created. I
f the corresponding
* record exists, the given value is appended at the end of the existing
value.
*/
void append(const char* kbuf, size_t ksiz, const char* vbuf, size_t vsiz)
{
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ);
size_t bidx = hash_record(kbuf, ksiz) % bnum_;
char* rbuf = buckets_[bidx];
char** entp = buckets_ + bidx;
while (rbuf) {
Record rec(rbuf);
if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) {
size_t nsiz = rec.vsiz_ + vsiz;
int32_t oh = (int32_t)sizevarnum(nsiz) - (int32_t)sizevarnum(rec.vs
iz_);
int64_t psiz = (int64_t)(rec.vsiz_ + rec.psiz_) - (int64_t)(nsiz +
oh);
if (psiz >= 0) {
rec.append(rbuf, oh, vbuf, vsiz, psiz);
} else {
psiz = nsiz + nsiz / 2;
Record nrec(rec.child_, kbuf, ksiz, "", 0, psiz);
char* nbuf = nrec.serialize();
oh = (int32_t)sizevarnum(nsiz) - 1;
psiz = (int64_t)psiz - (int64_t)(nsiz + oh);
rec.concatenate(nbuf, rec.vbuf_, rec.vsiz_, vbuf, vsiz, psiz);
delete[] rbuf;
*entp = nbuf;
}
return;
}
entp = (char**)rbuf;
rbuf = rec.child_;
}
Record nrec(NULL, kbuf, ksiz, vbuf, vsiz, 0);
*entp = nrec.serialize();
count_++;
}
/**
* Remove a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @return true on success, or false on failure.
* @note If no record corresponds to the key, false is returned.
*/
bool remove(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ);
size_t bidx = hash_record(kbuf, ksiz) % bnum_;
char* rbuf = buckets_[bidx];
char** entp = buckets_ + bidx;
while (rbuf) {
Record rec(rbuf);
if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) {
*entp = rec.child_;
delete[] rbuf;
count_--;
return true;
}
entp = (char**)rbuf;
rbuf = rec.child_;
}
return false;
}
/**
* Retrieve the value of a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @param sp the pointer to the variable into which the size of the regio
n of the return
* value is assigned.
* @return the pointer to the value region of the corresponding record, o
r NULL on failure.
*/
const char* get(const char* kbuf, size_t ksiz, size_t* sp) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && sp);
size_t bidx = hash_record(kbuf, ksiz) % bnum_;
char* rbuf = buckets_[bidx];
while (rbuf) {
Record rec(rbuf);
if (rec.ksiz_ == ksiz && !std::memcmp(rec.kbuf_, kbuf, ksiz)) {
*sp = rec.vsiz_;
return rec.vbuf_;
}
rbuf = rec.child_;
}
return NULL;
}
/**
* Remove all records.
*/
void clear() {
_assert_(true);
if (count_ < 1) return;
for (size_t i = 0; i < bnum_; i++) {
char* rbuf = buckets_[i];
while (rbuf) {
Record rec(rbuf);
char* child = rec.child_;
delete[] rbuf;
rbuf = child;
}
buckets_[i] = NULL;
}
count_ = 0;
}
/**
* Get the number of records.
* @return the number of records.
*/
size_t count() {
_assert_(true);
return count_;
}
private:
/**
* Record data.
*/
struct Record {
/** constructor */
Record(char* child, const char* kbuf, uint64_t ksiz,
const char* vbuf, uint64_t vsiz, uint64_t psiz) :
child_(child), kbuf_(kbuf), ksiz_(ksiz), vbuf_(vbuf), vsiz_(vsiz), ps
iz_(psiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && vbuf && vsiz <= MEMMAXSIZ && ps
iz <= MEMMAXSIZ);
}
/** constructor */
Record(const char* rbuf) :
child_(NULL), kbuf_(NULL), ksiz_(0), vbuf_(NULL), vsiz_(0), psiz_(0)
{
_assert_(rbuf);
deserialize(rbuf);
}
/** overwrite the buffer */
void overwrite(char* rbuf, const char* vbuf, size_t vsiz, size_t psiz)
{
_assert_(rbuf && vbuf && vsiz <= MEMMAXSIZ && psiz <= MEMMAXSIZ);
char* wp = rbuf + sizeof(child_) + sizevarnum(ksiz_) + ksiz_;
wp += writevarnum(wp, vsiz);
std::memcpy(wp, vbuf, vsiz);
wp += vsiz;
writevarnum(wp, psiz);
}
/** append a value */
void append(char* rbuf, int32_t oh, const char* vbuf, size_t vsiz, size
_t psiz) {
_assert_(rbuf && vbuf && vsiz <= MEMMAXSIZ && psiz <= MEMMAXSIZ);
char* wp = rbuf + sizeof(child_) + sizevarnum(ksiz_) + ksiz_;
if (oh > 0) {
char* pv = wp + sizevarnum(vsiz_);
std::memmove(pv + oh, pv, vsiz_);
wp += writevarnum(wp, vsiz_ + vsiz);
wp = pv + oh + vsiz_;
} else {
wp += writevarnum(wp, vsiz_ + vsiz);
wp += vsiz_;
}
std::memcpy(wp, vbuf, vsiz);
wp += vsiz;
writevarnum(wp, psiz);
}
/** concatenate two values */
void concatenate(char* rbuf, const char* ovbuf, size_t ovsiz,
const char* nvbuf, size_t nvsiz, size_t psiz) {
_assert_(rbuf && ovbuf && ovsiz <= MEMMAXSIZ && nvbuf && nvsiz <= MEM
MAXSIZ);
char* wp = rbuf + sizeof(child_) + sizevarnum(ksiz_) + ksiz_;
wp += writevarnum(wp, ovsiz + nvsiz);
std::memcpy(wp, ovbuf, ovsiz);
wp += ovsiz;
std::memcpy(wp, nvbuf, nvsiz);
wp += nvsiz;
writevarnum(wp, psiz);
}
/** serialize data into a buffer */
char* serialize() {
_assert_(true);
uint64_t rsiz = sizeof(child_) + sizevarnum(ksiz_) + ksiz_ + sizevarn
um(vsiz_) + vsiz_ +
sizevarnum(psiz_) + psiz_;
char* rbuf = new char[rsiz];
char* wp = rbuf;
*(char**)wp = child_;
wp += sizeof(child_);
wp += writevarnum(wp, ksiz_);
std::memcpy(wp, kbuf_, ksiz_);
wp += ksiz_;
wp += writevarnum(wp, vsiz_);
std::memcpy(wp, vbuf_, vsiz_);
wp += vsiz_;
writevarnum(wp, psiz_);
return rbuf;
}
/** deserialize a buffer into object */
void deserialize(const char* rbuf) {
_assert_(rbuf);
const char* rp = rbuf;
child_ = *(char**)rp;
rp += sizeof(child_);
rp += readvarnum(rp, sizeof(ksiz_), &ksiz_);
kbuf_ = rp;
rp += ksiz_;
rp += readvarnum(rp, sizeof(vsiz_), &vsiz_);
vbuf_ = rp;
rp += vsiz_;
readvarnum(rp, sizeof(psiz_), &psiz_);
}
char* child_; ///< region of the child
const char* kbuf_; ///< region of the key
uint64_t ksiz_; ///< size of the key
const char* vbuf_; ///< region of the value
uint64_t vsiz_; ///< size of the key
uint64_t psiz_; ///< size of the padding
};
/**
* Comparator for records.
*/
struct RecordComparator {
/** comparing operator */
bool operator ()(char* const& abuf, char* const& bbuf) {
const char* akbuf = abuf + sizeof(char**);
uint64_t aksiz;
akbuf += readvarnum(akbuf, sizeof(aksiz), &aksiz);
const char* bkbuf = bbuf + sizeof(char**);
uint64_t bksiz;
bkbuf += readvarnum(bkbuf, sizeof(bksiz), &bksiz);
uint64_t msiz = aksiz < bksiz ? aksiz : bksiz;
for (uint64_t i = 0; i < msiz; i++) {
if (((uint8_t*)akbuf)[i] != ((uint8_t*)bkbuf)[i])
return ((uint8_t*)akbuf)[i] < ((uint8_t*)bkbuf)[i];
}
return (int32_t)aksiz < (int32_t)bksiz;
}
};
/**
* Initialize fields.
*/
void initialize() {
_assert_(true);
if (bnum_ >= MAPZMAPBNUM) {
buckets_ = (char**)mapalloc(sizeof(*buckets_) * bnum_);
} else {
buckets_ = new char*[bnum_];
for (size_t i = 0; i < bnum_; i++) {
buckets_[i] = NULL;
}
}
}
/**
* Clean up fields.
*/
void destroy() {
_assert_(true);
for (size_t i = 0; i < bnum_; i++) {
char* rbuf = buckets_[i];
while (rbuf) {
Record rec(rbuf);
char* child = rec.child_;
delete[] rbuf;
rbuf = child;
}
}
if (bnum_ >= MAPZMAPBNUM) {
mapfree(buckets_);
} else {
delete[] buckets_;
}
}
/**
* Get the hash value of a record.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @return the hash value.
*/
size_t hash_record(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ);
return hashmurmur(kbuf, ksiz);
}
/** Dummy constructor to forbid the use. */
TinyHashMap(const TinyHashMap&);
/** Dummy Operator to forbid the use. */
TinyHashMap& operator =(const TinyHashMap&);
/** The bucket array. */
char** buckets_;
/** The number of buckets. */
size_t bnum_;
/** The number of records. */
size_t count_;
};
/**
* Doubly-linked hash map.
* @param KEY the key type. * @param KEY the key type.
* @param VALUE the value type. * @param VALUE the value type.
* @param HASH the hash functor. * @param HASH the hash functor.
* @param EQUALTO the equality checking functor. * @param EQUALTO the equality checking functor.
*/ */
template <class KEY, class VALUE, template <class KEY, class VALUE,
class HASH = std::hash<KEY>, class EQUALTO = std::equal_to<KEY> > class HASH = std::hash<KEY>, class EQUALTO = std::equal_to<KEY> >
class LinkedHashMap { class LinkedHashMap {
public:
class Iterator;
private: private:
struct Record; struct Record;
public: public:
/** /**
* Iterator of records. * Iterator of records.
*/ */
class Iterator { class Iterator {
private:
friend class LinkedHashMap; friend class LinkedHashMap;
public: public:
/** /**
* Copy constructor. * Copy constructor.
* @param src the source object. * @param src the source object.
*/ */
Iterator(const Iterator& src) : map_(src.map_), rec_(src.rec_) { Iterator(const Iterator& src) : map_(src.map_), rec_(src.rec_) {
_assert_(true); _assert_(true);
} }
/** /**
skipping to change at line 174 skipping to change at line 800
*/ */
enum MoveMode { enum MoveMode {
MCURRENT, ///< keep the current position MCURRENT, ///< keep the current position
MFIRST, ///< move to the first MFIRST, ///< move to the first
MLAST ///< move to the last MLAST ///< move to the last
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit LinkedHashMap() : explicit LinkedHashMap() :
buckets_(NULL), bnum_(LHMDEFBNUM), first_(NULL), last_(NULL), count_(0) { buckets_(NULL), bnum_(MAPDEFBNUM), first_(NULL), last_(NULL), count_(0) {
_assert_(true); _assert_(true);
initialize(); initialize();
} }
/** /**
* Constructor. * Constructor.
* @param bnum the number of buckets of the hash table. * @param bnum the number of buckets of the hash table.
*/ */
explicit LinkedHashMap(size_t bnum) : explicit LinkedHashMap(size_t bnum) :
buckets_(NULL), bnum_(bnum), first_(NULL), last_(NULL), count_(0) { buckets_(NULL), bnum_(bnum), first_(NULL), last_(NULL), count_(0) {
_assert_(true); _assert_(true);
if (bnum_ < 1) bnum_ = LHMDEFBNUM; if (bnum_ < 1) bnum_ = MAPDEFBNUM;
initialize(); initialize();
} }
/** /**
* Destructor. * Destructor.
*/ */
~LinkedHashMap() { ~LinkedHashMap() {
_assert_(true); _assert_(true);
destroy(); destroy();
} }
/** /**
skipping to change at line 555 skipping to change at line 1181
explicit Record(const KEY& k, const VALUE& v) : explicit Record(const KEY& k, const VALUE& v) :
key(k), value(v), child(NULL), prev(NULL), next(NULL) { key(k), value(v), child(NULL), prev(NULL), next(NULL) {
_assert_(true); _assert_(true);
} }
}; };
/** /**
* Initialize fields. * Initialize fields.
*/ */
void initialize() { void initialize() {
_assert_(true); _assert_(true);
if (bnum_ >= LHMZMAPBNUM) { if (bnum_ >= MAPZMAPBNUM) {
buckets_ = (Record**)mapalloc(sizeof(*buckets_) * bnum_); buckets_ = (Record**)mapalloc(sizeof(*buckets_) * bnum_);
} else { } else {
buckets_ = new Record*[bnum_]; buckets_ = new Record*[bnum_];
for (size_t i = 0; i < bnum_; i++) { for (size_t i = 0; i < bnum_; i++) {
buckets_[i] = NULL; buckets_[i] = NULL;
} }
} }
} }
/** /**
* Clean up fields. * Clean up fields.
*/ */
void destroy() { void destroy() {
_assert_(true); _assert_(true);
Record* rec = last_; Record* rec = last_;
while (rec) { while (rec) {
Record* prev = rec->prev; Record* prev = rec->prev;
delete rec; delete rec;
rec = prev; rec = prev;
} }
if (bnum_ >= LHMZMAPBNUM) { if (bnum_ >= MAPZMAPBNUM) {
mapfree(buckets_); mapfree(buckets_);
} else { } else {
delete[] buckets_; delete[] buckets_;
} }
} }
/** Dummy constructor to forbid the use. */
LinkedHashMap(const LinkedHashMap&);
/** Dummy Operator to forbid the use. */
LinkedHashMap& operator =(const LinkedHashMap&);
/** The functor of the hash function. */ /** The functor of the hash function. */
HASH hash_; HASH hash_;
/** The functor of the hash function. */ /** The functor of the equalto function. */
EQUALTO equalto_; EQUALTO equalto_;
/** The bucket array. */ /** The bucket array. */
Record** buckets_; Record** buckets_;
/** The number of buckets. */ /** The number of buckets. */
size_t bnum_; size_t bnum_;
/** The first record. */ /** The first record. */
Record* first_; Record* first_;
/** The last record. */ /** The last record. */
Record* last_; Record* last_;
/** The number of records. */ /** The number of records. */
 End of changes. 10 change blocks. 
11 lines changed or deleted 681 lines changed or added


 kcpolydb.h   kcpolydb.h 
skipping to change at line 20 skipping to change at line 20
* See the GNU General Public License for more details. * See the GNU General Public License for more details.
* You should have received a copy of the GNU General Public License along with this program. * You should have received a copy of the GNU General Public License along with this program.
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
************************************************************************** ***********************/ ************************************************************************** ***********************/
#ifndef _KCPOLYDB_H // duplication check #ifndef _KCPOLYDB_H // duplication check
#define _KCPOLYDB_H #define _KCPOLYDB_H
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
#include <kcdb.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h>
#include <kcdb.h>
#include <kcplantdb.h>
#include <kcprotodb.h> #include <kcprotodb.h>
#include <kcstashdb.h>
#include <kccachedb.h> #include <kccachedb.h>
#include <kchashdb.h> #include <kchashdb.h>
#include <kctreedb.h>
#include <kcdirdb.h> #include <kcdirdb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Polymorphic database. * Polymorphic database.
* @note This class is a concrete class to operate an arbitrary database wh
ose type is determined
* in runtime. This class can be inherited but overwriting methods is forb
idden. Before every
* database operation, it is necessary to call the PolyDB::open method in o
rder to open a
* database file and connect the database object to it. To avoid data miss
ing or corruption, it
* is important to close every database file by the PolyDB::close method wh
en the database is no
* longer in use. It is forbidden for multible database objects in a proce
ss to open the same
* database at the same time. It is forbidden to share a database object w
ith child processes.
*/ */
class PolyDB : public FileDB { class PolyDB : public BasicDB {
public: public:
class Cursor; class Cursor;
private:
class StreamLogger;
class StreamMetaTrigger;
struct MergeLine;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public FileDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class PolyDB; friend class PolyDB;
public: public:
/** /**
* Constructor. * Constructor.
* @param db the container database object. * @param db the container database object.
*/ */
explicit Cursor(PolyDB* db) : db_(db), cur_(NULL) { explicit Cursor(PolyDB* db) : db_(db), cur_(NULL) {
_assert_(db); _assert_(db);
if (db_->type_ == TYPEVOID) { if (db_->type_ == TYPEVOID) {
ProtoTreeDB tmpdb; ProtoHashDB tmpdb;
cur_ = tmpdb.cursor(); cur_ = tmpdb.cursor();
} else { } else {
cur_ = db->db_->cursor(); cur_ = db->db_->cursor();
} }
} }
/** /**
* Destructor. * Destructor.
*/ */
virtual ~Cursor() { virtual ~Cursor() {
_assert_(true); _assert_(true);
delete cur_; delete cur_;
} }
/** /**
* Accept a visitor to the current record. * Accept a visitor to the current record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only operation. * @param writable true for writable operation, or false for read-only operation.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and othe * @note The operation for each record is performed atomically and othe
r threads accessing r threads accessing
* the same record are blocked. * the same record are blocked. To avoid deadlock, any database operat
ion must not be
* performed in this function.
*/ */
bool accept(Visitor* visitor, bool writable = true, bool step = false) { bool accept(Visitor* visitor, bool writable = true, bool step = false) {
_assert_(visitor); _assert_(visitor);
if (db_->type_ == TYPEVOID) { if (db_->type_ == TYPEVOID) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return cur_->accept(visitor, writable, step); return cur_->accept(visitor, writable, step);
} }
/** /**
* Jump the cursor to the first record. * Jump the cursor to the first record for forward scan.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump() { bool jump() {
_assert_(true); _assert_(true);
if (db_->type_ == TYPEVOID) { if (db_->type_ == TYPEVOID) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return cur_->jump(); return cur_->jump();
} }
/** /**
* Jump the cursor onto a record. * Jump the cursor to a record for forward scan.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump(const char* kbuf, size_t ksiz) { bool jump(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
if (db_->type_ == TYPEVOID) { if (db_->type_ == TYPEVOID) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return cur_->jump(kbuf, ksiz); return cur_->jump(kbuf, ksiz);
} }
/** /**
* Jump the cursor to a record. * Jump the cursor to a record for forward scan.
* @note Equal to the original Cursor::jump method except that the para meter is std::string. * @note Equal to the original Cursor::jump method except that the para meter is std::string.
*/ */
bool jump(const std::string& key) { bool jump(const std::string& key) {
_assert_(true); _assert_(true);
if (db_->type_ == TYPEVOID) { if (db_->type_ == TYPEVOID) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return jump(key.c_str(), key.size()); return jump(key.c_str(), key.size());
} }
/** /**
* Jump the cursor to the last record for backward scan.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database typ
es, especially hash
* databases, may provide a dummy implementation.
*/
bool jump_back() {
_assert_(true);
if (db_->type_ == TYPEVOID) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
return cur_->jump_back();
}
/**
* Jump the cursor to a record for backward scan.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database typ
es, especially hash
* databases, will provide a dummy implementation.
*/
bool jump_back(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ);
if (db_->type_ == TYPEVOID) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
return cur_->jump_back(kbuf, ksiz);
}
/**
* Jump the cursor to a record for backward scan.
* @note Equal to the original Cursor::jump_back method except that the
parameter is
* std::string.
*/
bool jump_back(const std::string& key) {
_assert_(true);
return jump_back(key.c_str(), key.size());
}
/**
* Step the cursor to the next record. * Step the cursor to the next record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool step() { bool step() {
_assert_(true); _assert_(true);
if (db_->type_ == TYPEVOID) { if (db_->type_ == TYPEVOID) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return cur_->step(); return cur_->step();
} }
/** /**
* Step the cursor to the previous record.
* @return true on success, or false on failure.
* @note This method is dedicated to tree databases. Some database typ
es, especially hash
* databases, may provide a dummy implementation.
*/
bool step_back() {
_assert_(true);
if (db_->type_ == TYPEVOID) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
return cur_->step_back();
}
/**
* Get the database object. * Get the database object.
* @return the database object. * @return the database object.
*/ */
PolyDB* db() { PolyDB* db() {
_assert_(true); _assert_(true);
return db_; return db_;
} }
private: private:
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
Cursor(const Cursor&); Cursor(const Cursor&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
Cursor& operator =(const Cursor&); Cursor& operator =(const Cursor&);
/** The inner database. */ /** The inner database. */
PolyDB* db_; PolyDB* db_;
/** The inner cursor. */ /** The inner cursor. */
FileDB::Cursor* cur_; BasicDB::Cursor* cur_;
}; };
/** /**
* Default constructor. * Merge modes.
*/ */
explicit PolyDB() : type_(TYPEVOID), db_(NULL), error_(), zcomp_(NULL) { enum MergeMode {
_assert_(true); MSET, ///< overwrite the existing value
} MADD, ///< keep the existing value
MREPLACE, ///< modify the existing record on
ly
MAPPEND ///< append the new value
};
/** /**
* Constructor. * Default constructor.
* @param db the internal database object. Its possession is transferred
inside and the
* object is deleted automatically.
*/ */
explicit PolyDB(FileDB* db) : type_(TYPEMISC), db_(db), error_(), zcomp_( explicit PolyDB() :
NULL) { type_(TYPEVOID), db_(NULL), error_(),
_assert_(db); stdlogstrm_(NULL), stdlogger_(NULL), logger_(NULL), logkinds_(0),
stdmtrgstrm_(NULL), stdmtrigger_(NULL), mtrigger_(NULL), zcomp_(NULL) {
_assert_(true);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~PolyDB() { virtual ~PolyDB() {
_assert_(true); _assert_(true);
if (type_ != TYPEVOID) close(); if (type_ != TYPEVOID) close();
delete zcomp_; delete zcomp_;
delete stdmtrigger_;
delete stdmtrgstrm_;
delete stdlogger_;
delete stdlogstrm_;
}
/**
* Set the internal database object.
* @param db the internal database object. Its possession is transferred
inside and the
* object is deleted automatically.
* @return true on success, or false on failure.
*/
bool set_internal_db(BasicDB* db) {
_assert_(db);
if (type_ != TYPEVOID) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
type_ = TYPEMISC;
db_ = db;
return true;
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and other * @note The operation for each record is performed atomically and other
threads accessing the threads accessing the
* same record are blocked. * same record are blocked. To avoid deadlock, any database operation mu
st not be performed in
* this function.
*/ */
bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) { bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return db_->accept(kbuf, ksiz, visitor, writable); return db_->accept(kbuf, ksiz, visitor, writable);
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the whole iteration is performed atomically and other threads ar * @note The whole iteration is performed atomically and other threads ar
e blocked. e blocked. To avoid
* deadlock, any database operation must not be performed in this functio
n.
*/ */
bool iterate(Visitor *visitor, bool writable = true) { bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* che cker = NULL) {
_assert_(visitor); _assert_(visitor);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return db_->iterate(visitor, writable); return db_->iterate(visitor, writable, checker);
} }
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
Error error() const { Error error() const {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) return error_; if (type_ == TYPEVOID) return error_;
return db_->error(); return db_->error();
} }
/** /**
* Set the error information. * Set the error information.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param code an error code.
* @param message a supplement message.
*/
void set_error(const char* file, int32_t line, const char* func,
Error::Code code, const char* message) {
_assert_(file && line > 0 && func && message);
if (type_ == TYPEVOID) {
error_.set(code, message);
return;
}
db_->set_error(file, line, func, code, message);
}
/**
* Set the error information without source code information.
* @param code an error code. * @param code an error code.
* @param message a supplement message. * @param message a supplement message.
*/ */
void set_error(Error::Code code, const char* message) { void set_error(Error::Code code, const char* message) {
_assert_(message); _assert_(message);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
error_->set(code, message); error_.set(code, message);
return; return;
} }
db_->set_error(code, message); db_->set_error(_KCCODELINE_, code, message);
} }
/** /**
* Open a database file. * Open a database file.
* @param path the path of a database file. If it is "-", the database w ill be a prototype * @param path the path of a database file. If it is "-", the database w ill be a prototype
* hash database. If it is "+", the database will be a prototype tree da * hash database. If it is "+", the database will be a prototype tree da
tabase. If it is tabase. If it is ":",
* "*", the database will be a cache database. If its suffix is ".kch", * the database will be a stash database. If it is "*", the database wil
the database will be l be a cache hash
* a file hash database. If its suffix is ".kct", the database will be a * database. If it is "%", the database will be a cache tree database.
file tree database. If its suffix is
* If its suffix is ".kcd", the database will be a directory database. O * ".kch", the database will be a file hash database. If its suffix is "
therwise, this .kct", the database
* function fails. Tuning parameters can trail the name, separated by "# * will be a file tree database. If its suffix is ".kcd", the database w
". Each parameter is ill be a directory
* composed of the name and the value, separated by "=". If the "type" p * hash database. If its suffix is ".kcf", the database will be a direct
arameter is specified, ory tree database.
* the database type is determined by the value in "-", "+", "*", "kch", * Otherwise, this function fails. Tuning parameters can trail the name,
"kct", and "kcd". The separated by "#".
* prototype hash database and the prototype tree database do not support * Each parameter is composed of the name and the value, separated by "="
any other tuning . If the "type"
* parameter. The cache database supports "bnum", "capcount", and "capsi * parameter is specified, the database type is determined by the value i
ze". The file hash n "-", "+", ":", "*",
* database supports "apow", "fpow", "opts", "bnum", "msiz", "dfunit", "z * "%", "kch", "kct", "kcd", and "kcf". All database types support the l
comp", "erstrm", ogging parameters of
* "ervbs", and "zkey". The file tree database supports all parameters o * "log", "logkinds", and "logpx". The prototype hash database and the p
f the file hash rototype tree
* database and "psiz", "rcomp", "pccap" in addition. The directory data * database do not support any other tuning parameter. The stash databas
base supports "opts", e supports "bnum".
* "zcomp", and "zkey". * The cache hash database supports "opts", "bnum", "zcomp", "capcnt", "c
apsiz", and "zkey".
* The cache tree database supports all parameters of the cache hash data
base except for
* capacity limitation, and supports "psiz", "rcomp", "pccap" in addition
. The file hash
* database supports "apow", "fpow", "opts", "bnum", "msiz", "dfunit", "z
comp", and "zkey".
* The file tree database supports all parameters of the file hash databa
se and "psiz",
* "rcomp", "pccap" in addition. The directory hash database supports "o
pts", "zcomp", and
* "zkey". The directory tree database supports all parameters of the di
rectory hash database
* and "psiz", "rcomp", "pccap" in addition.
* @param mode the connection mode. PolyDB::OWRITER as a writer, PolyDB: :OREADER as a * @param mode the connection mode. PolyDB::OWRITER as a writer, PolyDB: :OREADER as a
* reader. The following may be added to the writer mode by bitwise-or: PolyDB::OCREATE, * reader. The following may be added to the writer mode by bitwise-or: PolyDB::OCREATE,
* which means it creates a new database if the file does not exist, Poly DB::OTRUNCATE, which * which means it creates a new database if the file does not exist, Poly DB::OTRUNCATE, which
* means it creates a new database regardless if the file exists, PolyDB: :OAUTOTRAN, which * means it creates a new database regardless if the file exists, PolyDB: :OAUTOTRAN, which
* means each updating operation is performed in implicit transaction, Po lyDB::OAUTOSYNC, * means each updating operation is performed in implicit transaction, Po lyDB::OAUTOSYNC,
* which means each updating operation is followed by implicit synchroniz ation with the file * which means each updating operation is followed by implicit synchroniz ation with the file
* system. The following may be added to both of the reader mode and the writer mode by * system. The following may be added to both of the reader mode and the writer mode by
* bitwise-or: PolyDB::ONOLOCK, which means it opens the database file wi thout file locking, * bitwise-or: PolyDB::ONOLOCK, which means it opens the database file wi thout file locking,
* PolyDB::OTRYLOCK, which means locking is performed without blocking, P olyDB::ONOREPAIR, * PolyDB::OTRYLOCK, which means locking is performed without blocking, P olyDB::ONOREPAIR,
* which means the database file is not repaired implicitly even if file destruction is * which means the database file is not repaired implicitly even if file destruction is
* detected. * detected.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note The tuning parameter "bnum" corresponds to the original "tune_bu * @note The tuning parameter "log" is for the original "tune_logger" and
cket" method. the value specifies
* "capcount" is for "cap_count". "capsize" is for "cap_size". "apow" i * the path of the log file, or "-" for the standard output, or "+" for t
s for he standard error.
* "tune_alignment". "fpow" is for "tune_fbp". "opts" is for "tune_opti * "logkinds" specifies kinds of logged messages and the value can be "de
ons" and the value bug", "info", "warn",
* can contain "s" for the small option, "l" for the linear option, and " * or "error". "logpx" specifies the prefix of each log message. "opts"
c" for the compress is for "tune_options"
* option. "msiz" is for "tune_map". "dfunit" is for "tune_defrag". "z * and the value can contain "s" for the small option, "l" for the linear
comp" is for option, and "c" for
* "tune_compressor" and the value can be "zlib" for the Zlib raw compres * the compress option. "bnum" corresponds to "tune_bucket". "zcomp" is
sor, "def" for the for "tune_compressor"
* Zlib deflate compressor, "gz" for the Zlib gzip compressor, or "arc" f * and the value can be "zlib" for the ZLIB raw compressor, "def" for the
or the Arcfour cipher. ZLIB deflate
* "erstrm" and "ervbs" are for "tune_error_reporter" and the formar valu * compressor, "gz" for the ZLIB gzip compressor, "lzo" for the LZO compr
e can be "stdout" or essor, "lzma" for the
* "stderr" and the latter value can be "true" or "false". "zkey" specif * LZMA compressor, or "arc" for the Arcfour cipher. "zkey" specifies th
ies the cipher key of e cipher key of the
* the compressor. "psiz" is for "tune_page". "rcomp" is for "tune_comp * compressor. "capcnt" is for "cap_count". "capsiz" is for "cap_size".
arator" and the value "psiz" is for
* can be "lex" for the lexical comparator or "dec" for the decimal compa * "tune_page". "rcomp" is for "tune_comparator" and the value can be "l
rator. "pccap" is for ex" for the lexical
* "tune_page_cache". Every opened database must be closed by the PolyDB * comparator or "dec" for the decimal comparator. "pccap" is for "tune_
::close method when it page_cache". "apow"
* is no longer in use. It is not allowed for two or more database objec * is for "tune_alignment". "fpow" is for "tune_fbp". "msiz" is for "tu
ts in the same process ne_map". "dfunit" is
* to keep their connections to the same database file at the same time. * for "tune_defrag". Every opened database must be closed by the PolyDB
::close method when
* it is no longer in use. It is not allowed for two or more database ob
jects in the same
* process to keep their connections to the same database file at the sam
e time.
*/ */
bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { bool open(const std::string& path = ":", uint32_t mode = OWRITER | OCREAT E) {
_assert_(true); _assert_(true);
if (type_ == TYPEMISC) return db_->open(path, mode); if (type_ == TYPEMISC) {
if (logger_) db_->tune_logger(logger_, logkinds_);
if (mtrigger_) db_->tune_meta_trigger(mtrigger_);
return db_->open(path, mode);
}
if (type_ != TYPEVOID) { if (type_ != TYPEVOID) {
set_error(Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
std::vector<std::string> elems; std::vector<std::string> elems;
strsplit(path, '#', &elems); strsplit(path, '#', &elems);
std::string fpath; std::string fpath;
Type type = TYPEVOID; Type type = TYPEVOID;
std::string logname = "";
std::string logpx = "";
uint32_t logkinds = Logger::WARN | Logger::ERROR;
std::string mtrgname = "";
std::string mtrgpx = "";
int64_t bnum = -1; int64_t bnum = -1;
int64_t capcount = -1; int64_t capcnt = -1;
int64_t capsize = -1; int64_t capsiz = -1;
int32_t apow = -1; int32_t apow = -1;
int32_t fpow = -1; int32_t fpow = -1;
bool tsmall = false; bool tsmall = false;
bool tlinear = false; bool tlinear = false;
bool tcompress = false; bool tcompress = false;
int64_t msiz = -1; int64_t msiz = -1;
int64_t dfunit = -1; int64_t dfunit = -1;
Compressor *zcomp = NULL; std::string zcompname = "";
int64_t psiz = -1; int64_t psiz = -1;
Comparator *rcomp = NULL; Comparator* rcomp = NULL;
int64_t pccap = 0; int64_t pccap = 0;
std::ostream* erstrm = NULL;
bool ervbs = false;
std::string zkey = ""; std::string zkey = "";
ArcfourCompressor *arccomp = NULL;
std::vector<std::string>::iterator it = elems.begin(); std::vector<std::string>::iterator it = elems.begin();
std::vector<std::string>::iterator itend = elems.end(); std::vector<std::string>::iterator itend = elems.end();
if (it != itend) { if (it != itend) {
fpath = *it; fpath = *it;
it++; it++;
} }
const char* fstr = fpath.c_str(); const char* fstr = fpath.c_str();
const char* pv = std::strrchr(fstr, File::PATHCHR); const char* pv = std::strrchr(fstr, File::PATHCHR);
if (pv) fstr = pv + 1; if (pv) fstr = pv + 1;
if (!std::strcmp(fstr, "-")) { if (!std::strcmp(fstr, "-")) {
type = TYPEPHASH; type = TYPEPHASH;
} else if (!std::strcmp(fstr, "+")) { } else if (!std::strcmp(fstr, "+")) {
type = TYPEPTREE; type = TYPEPTREE;
} else if (!std::strcmp(fstr, ":")) {
type = TYPESTASH;
} else if (!std::strcmp(fstr, "*")) { } else if (!std::strcmp(fstr, "*")) {
type = TYPECACHE; type = TYPECACHE;
} else if (!std::strcmp(fstr, "%")) {
type = TYPEGRASS;
} else { } else {
pv = std::strrchr(fstr, File::EXTCHR); pv = std::strrchr(fstr, File::EXTCHR);
if (pv) { if (pv) {
pv++; pv++;
if (!std::strcmp(pv, "kch") || !std::strcmp(pv, "hdb")) { if (!std::strcmp(pv, "kcph") || !std::strcmp(pv, "phdb")) {
type = TYPEPHASH;
} else if (!std::strcmp(pv, "kcpt") || !std::strcmp(pv, "ptdb")) {
type = TYPEPTREE;
} else if (!std::strcmp(pv, "kcs") || !std::strcmp(pv, "sdb")) {
type = TYPESTASH;
} else if (!std::strcmp(pv, "kcc") || !std::strcmp(pv, "cdb")) {
type = TYPECACHE;
} else if (!std::strcmp(pv, "kcg") || !std::strcmp(pv, "gdb")) {
type = TYPEGRASS;
} else if (!std::strcmp(pv, "kch") || !std::strcmp(pv, "hdb")) {
type = TYPEHASH; type = TYPEHASH;
} else if (!std::strcmp(pv, "kct") || !std::strcmp(pv, "tdb")) { } else if (!std::strcmp(pv, "kct") || !std::strcmp(pv, "tdb")) {
type = TYPETREE; type = TYPETREE;
} else if (!std::strcmp(pv, "kcd") || !std::strcmp(pv, "ddb")) { } else if (!std::strcmp(pv, "kcd") || !std::strcmp(pv, "ddb")) {
type = TYPEDIR; type = TYPEDIR;
} else if (!std::strcmp(pv, "kcf") || !std::strcmp(pv, "fdb")) {
type = TYPEFOREST;
} }
} }
} }
while (it != itend) { while (it != itend) {
std::vector<std::string> fields; std::vector<std::string> fields;
if (strsplit(*it, '=', &fields) > 1) { if (strsplit(*it, '=', &fields) > 1) {
const char* key = fields[0].c_str(); const char* key = fields[0].c_str();
const char* value = fields[1].c_str(); const char* value = fields[1].c_str();
if (!std::strcmp(key, "type")) { if (!std::strcmp(key, "type")) {
if (!std::strcmp(value, "-") || !std::strcmp(value, "phash")) { if (!std::strcmp(value, "-") || !std::strcmp(value, "kcph") ||
!std::strcmp(value, "phdb") || !std::strcmp(value, "phash"))
{
type = TYPEPHASH; type = TYPEPHASH;
} else if (!std::strcmp(value, "+") || !std::strcmp(value, "ptree } else if (!std::strcmp(value, "+") || !std::strcmp(value, "kcpt"
")) { ) ||
!std::strcmp(value, "ptdb") || !std::strcmp(value, "pt
ree")) {
type = TYPEPTREE; type = TYPEPTREE;
} else if (!std::strcmp(value, "*") || !std::strcmp(value, "cache } else if (!std::strcmp(value, ":") || !std::strcmp(value, "kcs")
")) { ||
!std::strcmp(value, "sdb") || !std::strcmp(value, "sta
sh")) {
type = TYPESTASH;
} else if (!std::strcmp(value, "*") || !std::strcmp(value, "kcc")
||
!std::strcmp(value, "cdb") || !std::strcmp(value, "cac
he")) {
type = TYPECACHE; type = TYPECACHE;
} else if (!std::strcmp(value, "%") || !std::strcmp(value, "kcg")
||
!std::strcmp(value, "gdb") || !std::strcmp(value, "gra
ss")) {
type = TYPEGRASS;
} else if (!std::strcmp(value, "kch") || !std::strcmp(value, "hdb ") || } else if (!std::strcmp(value, "kch") || !std::strcmp(value, "hdb ") ||
!std::strcmp(value, "hash")) { !std::strcmp(value, "hash")) {
type = TYPEHASH; type = TYPEHASH;
} else if (!std::strcmp(value, "kct") || !std::strcmp(value, "tdb ") || } else if (!std::strcmp(value, "kct") || !std::strcmp(value, "tdb ") ||
!std::strcmp(value, "tree")) { !std::strcmp(value, "tree")) {
type = TYPETREE; type = TYPETREE;
} else if (!std::strcmp(value, "kcd") || !std::strcmp(value, "ddb ") || } else if (!std::strcmp(value, "kcd") || !std::strcmp(value, "ddb ") ||
!std::strcmp(value, "dir")) { !std::strcmp(value, "dir") || !std::strcmp(value, "dir ectory")) {
type = TYPEDIR; type = TYPEDIR;
} else if (!std::strcmp(value, "kcf") || !std::strcmp(value, "fdb
") ||
!std::strcmp(value, "for") || !std::strcmp(value, "for
est")) {
type = TYPEFOREST;
}
} else if (!std::strcmp(key, "log") || !std::strcmp(key, "logger"))
{
logname = value;
} else if (!std::strcmp(key, "logkinds") || !std::strcmp(key, "logk
ind")) {
if (!std::strcmp(value, "debug") || !std::strcmp(value, "debuggin
g")) {
logkinds = Logger::DEBUG | Logger::INFO | Logger::WARN | Logger
::ERROR;
} else if (!std::strcmp(value, "info") || !std::strcmp(value, "in
formation")) {
logkinds = Logger::INFO | Logger::WARN | Logger::ERROR;
} else if (!std::strcmp(value, "warn") || !std::strcmp(value, "wa
rning")) {
logkinds = Logger::WARN | Logger::ERROR;
} else if (!std::strcmp(value, "error") || !std::strcmp(value, "f
atal")) {
logkinds = Logger::ERROR;
} else {
logkinds = atoix(value);
} }
} else if (!std::strcmp(key, "logpx") || !std::strcmp(key, "lpx"))
{
logpx = value;
} else if (!std::strcmp(key, "mtrg") || !std::strcmp(key, "metatrig
ger") ||
!std::strcmp(key, "meta_trigger")) {
mtrgname = value;
} else if (!std::strcmp(key, "mtrgpx") || !std::strcmp(key, "mtpx")
) {
mtrgpx = value;
} else if (!std::strcmp(key, "bnum") || !std::strcmp(key, "buckets" )) { } else if (!std::strcmp(key, "bnum") || !std::strcmp(key, "buckets" )) {
bnum = atoix(value); bnum = atoix(value);
} else if (!std::strcmp(key, "capcount") || !std::strcmp(key, "cap_ } else if (!std::strcmp(key, "capcnt") || !std::strcmp(key, "capcou
count")) { nt") ||
capcount = atoix(value); !std::strcmp(key, "cap_count")) {
} else if (!std::strcmp(key, "capsize") || !std::strcmp(key, "cap_s capcnt = atoix(value);
ize")) { } else if (!std::strcmp(key, "capsiz") || !std::strcmp(key, "capsiz
capsize = atoix(value); e") ||
!std::strcmp(key, "cap_size")) {
capsiz = atoix(value);
} else if (!std::strcmp(key, "apow") || !std::strcmp(key, "alignmen t")) { } else if (!std::strcmp(key, "apow") || !std::strcmp(key, "alignmen t")) {
apow = atoix(value); apow = atoix(value);
} else if (!std::strcmp(key, "fpow") || !std::strcmp(key, "fbp")) { } else if (!std::strcmp(key, "fpow") || !std::strcmp(key, "fbp")) {
fpow = atoix(value); fpow = atoix(value);
} else if (!std::strcmp(key, "opts") || !std::strcmp(key, "options" )) { } else if (!std::strcmp(key, "opts") || !std::strcmp(key, "options" )) {
if (std::strchr(value, 's')) tsmall = true; if (std::strchr(value, 's')) tsmall = true;
if (std::strchr(value, 'l')) tlinear = true; if (std::strchr(value, 'l')) tlinear = true;
if (std::strchr(value, 'c')) tcompress = true; if (std::strchr(value, 'c')) tcompress = true;
} else if (!std::strcmp(key, "msiz") || !std::strcmp(key, "map")) { } else if (!std::strcmp(key, "msiz") || !std::strcmp(key, "map")) {
msiz = atoix(value); msiz = atoix(value);
} else if (!std::strcmp(key, "dfunit") || !std::strcmp(key, "defrag ")) { } else if (!std::strcmp(key, "dfunit") || !std::strcmp(key, "defrag ")) {
dfunit = atoix(value); dfunit = atoix(value);
} else if (!std::strcmp(key, "zcomp") || !std::strcmp(key, "compres sor")) { } else if (!std::strcmp(key, "zcomp") || !std::strcmp(key, "compres sor")) {
delete zcomp; zcompname = value;
zcomp = NULL;
arccomp = NULL;
if (!std::strcmp(value, "zlib") || !std::strcmp(value, "raw")) {
zcomp = new ZlibCompressor<Zlib::RAW>;
} else if (!std::strcmp(value, "def") || !std::strcmp(value, "def
late")) {
zcomp = new ZlibCompressor<Zlib::DEFLATE>;
} else if (!std::strcmp(value, "gz") || !std::strcmp(value, "gzip
")) {
zcomp = new ZlibCompressor<Zlib::GZIP>;
} else if (!std::strcmp(value, "arc") || !std::strcmp(value, "rc4
")) {
arccomp = new ArcfourCompressor();
zcomp = arccomp;
}
} else if (!std::strcmp(key, "psiz") || !std::strcmp(key, "page")) { } else if (!std::strcmp(key, "psiz") || !std::strcmp(key, "page")) {
psiz = atoix(value); psiz = atoix(value);
} else if (!std::strcmp(key, "pccap") || !std::strcmp(key, "cache") ) { } else if (!std::strcmp(key, "pccap") || !std::strcmp(key, "cache") ) {
pccap = atoix(value); pccap = atoix(value);
} else if (!std::strcmp(key, "rcomp") || !std::strcmp(key, "compara tor")) { } else if (!std::strcmp(key, "rcomp") || !std::strcmp(key, "compara tor")) {
if (!std::strcmp(value, "lex") || !std::strcmp(value, "lexical")) { if (!std::strcmp(value, "lex") || !std::strcmp(value, "lexical")) {
rcomp = &LEXICALCOMP; rcomp = &LEXICALCOMP;
} else if (!std::strcmp(value, "dec") || !std::strcmp(value, "dec imal")) { } else if (!std::strcmp(value, "dec") || !std::strcmp(value, "dec imal")) {
rcomp = &DECIMALCOMP; rcomp = &DECIMALCOMP;
} }
} else if (!std::strcmp(key, "erstrm") || !std::strcmp(key, "report
er")) {
if (!std::strcmp(value, "stdout") || !std::strcmp(value, "cout")
||
atoix(value) == 1) {
erstrm = &std::cout;
} else if (!std::strcmp(value, "stderr") || !std::strcmp(value, "
cerr") ||
atoix(value) == 2) {
erstrm = &std::cerr;
}
} else if (!std::strcmp(key, "ervbs") || !std::strcmp(key, "erv"))
{
ervbs = !std::strcmp(value, "true") || atoix(value) > 0;
} else if (!std::strcmp(key, "zkey") || !std::strcmp(key, "pass") | | } else if (!std::strcmp(key, "zkey") || !std::strcmp(key, "pass") | |
!std::strcmp(key, "password")) { !std::strcmp(key, "password")) {
zkey = value; zkey = value;
} }
} }
it++; it++;
} }
if (zcomp) { delete stdlogger_;
delete zcomp_; delete stdlogstrm_;
zcomp_ = zcomp; stdlogstrm_ = NULL;
tcompress = true; stdlogger_ = NULL;
if (!logname.empty()) {
std::ostream* stdlogstrm = NULL;
if (logname == "-" || logname == "[stdout]" || logname == "[cout]") {
stdlogstrm = &std::cout;
} else if (logname == "+" || logname == "[stderr]" || logname == "[ce
rr]") {
stdlogstrm = &std::cerr;
} else {
std::ofstream *ofs = new std::ofstream;
ofs->open(logname.c_str(),
std::ios_base::out | std::ios_base::binary | std::ios_bas
e::app);
if (!*ofs) ofs->open(logname.c_str(), std::ios_base::out | std::ios
_base::binary);
if (ofs) {
stdlogstrm = ofs;
stdlogstrm_ = ofs;
} else {
delete ofs;
}
}
if (stdlogstrm) stdlogger_ = new StreamLogger(stdlogstrm, logpx.c_str
());
} }
FileDB *db; delete stdmtrigger_;
delete stdmtrgstrm_;
stdmtrgstrm_ = NULL;
stdmtrigger_ = NULL;
if (!mtrgname.empty()) {
std::ostream* stdmtrgstrm = NULL;
if (mtrgname == "-" || mtrgname == "[stdout]" || mtrgname == "[cout]"
) {
stdmtrgstrm = &std::cout;
} else if (mtrgname == "+" || mtrgname == "[stderr]" || mtrgname == "
[cerr]") {
stdmtrgstrm = &std::cerr;
} else {
std::ofstream *ofs = new std::ofstream;
ofs->open(mtrgname.c_str(),
std::ios_base::out | std::ios_base::binary | std::ios_bas
e::app);
if (!*ofs) ofs->open(mtrgname.c_str(), std::ios_base::out | std::io
s_base::binary);
if (ofs) {
stdmtrgstrm = ofs;
stdmtrgstrm_ = ofs;
} else {
delete ofs;
}
}
if (stdmtrgstrm) stdmtrigger_ = new StreamMetaTrigger(stdmtrgstrm, mt
rgpx.c_str());
}
delete zcomp_;
zcomp_ = NULL;
ArcfourCompressor* arccomp = NULL;
if (!zcompname.empty()) {
if (zcompname == "zlib" || zcompname == "raw") {
zcomp_ = new ZLIBCompressor<ZLIB::RAW>;
} else if (zcompname == "def" || zcompname == "deflate") {
zcomp_ = new ZLIBCompressor<ZLIB::DEFLATE>;
} else if (zcompname == "gz" || zcompname == "gzip") {
zcomp_ = new ZLIBCompressor<ZLIB::GZIP>;
} else if (zcompname == "lzo" || zcompname == "oz") {
zcomp_ = new LZOCompressor<LZO::RAW>;
} else if (zcompname == "lzocrc" || zcompname == "ozcrc") {
zcomp_ = new LZOCompressor<LZO::CRC>;
} else if (zcompname == "lzma" || zcompname == "xz") {
zcomp_ = new LZMACompressor<LZMA::RAW>;
} else if (zcompname == "lzmacrc" || zcompname == "xzcrc") {
zcomp_ = new LZMACompressor<LZMA::CRC>;
} else if (zcompname == "lzmasha" || zcompname == "xzsha") {
zcomp_ = new LZMACompressor<LZMA::SHA>;
} else if (zcompname == "arc" || zcompname == "rc4") {
arccomp = new ArcfourCompressor();
zcomp_ = arccomp;
} else if (zcompname == "arcz" || zcompname == "rc4z") {
arccomp = new ArcfourCompressor();
arccomp->set_compressor(&ZLIBRAWCOMP);
zcomp_ = arccomp;
}
}
BasicDB *db;
switch (type) { switch (type) {
default: { default: {
set_error(Error::INVALID, "unknown database type"); set_error(_KCCODELINE_, Error::INVALID, "unknown database type");
return false; return false;
} }
case TYPEPHASH: { case TYPEPHASH: {
ProtoHashDB* phdb = new ProtoHashDB(); ProtoHashDB* phdb = new ProtoHashDB();
if (stdlogger_) {
phdb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
phdb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
phdb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
phdb->tune_meta_trigger(mtrigger_);
}
db = phdb; db = phdb;
break; break;
} }
case TYPEPTREE: { case TYPEPTREE: {
ProtoTreeDB *ptdb = new ProtoTreeDB(); ProtoTreeDB *ptdb = new ProtoTreeDB();
if (stdlogger_) {
ptdb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
ptdb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
ptdb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
ptdb->tune_meta_trigger(mtrigger_);
}
db = ptdb; db = ptdb;
break; break;
} }
case TYPESTASH: {
StashDB* sdb = new StashDB();
if (stdlogger_) {
sdb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
sdb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
sdb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
sdb->tune_meta_trigger(mtrigger_);
}
if (bnum > 0) sdb->tune_buckets(bnum);
db = sdb;
break;
}
case TYPECACHE: { case TYPECACHE: {
int8_t opts = 0;
if (tcompress) opts |= CacheDB::TCOMPRESS;
CacheDB* cdb = new CacheDB(); CacheDB* cdb = new CacheDB();
if (stdlogger_) {
cdb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
cdb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
cdb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
cdb->tune_meta_trigger(mtrigger_);
}
if (opts > 0) cdb->tune_options(opts);
if (bnum > 0) cdb->tune_buckets(bnum); if (bnum > 0) cdb->tune_buckets(bnum);
if (capcount > 0) cdb->cap_count(capcount); if (zcomp_) cdb->tune_compressor(zcomp_);
if (capsize > 0) cdb->cap_size(capsize); if (capcnt > 0) cdb->cap_count(capcnt);
if (capsiz > 0) cdb->cap_size(capsiz);
db = cdb; db = cdb;
break; break;
} }
case TYPEGRASS: {
int8_t opts = 0;
if (tcompress) opts |= GrassDB::TCOMPRESS;
GrassDB* gdb = new GrassDB();
if (stdlogger_) {
gdb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
gdb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
gdb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
gdb->tune_meta_trigger(mtrigger_);
}
if (opts > 0) gdb->tune_options(opts);
if (bnum > 0) gdb->tune_buckets(bnum);
if (psiz > 0) gdb->tune_page(psiz);
if (zcomp_) gdb->tune_compressor(zcomp_);
if (pccap > 0) gdb->tune_page_cache(pccap);
if (rcomp) gdb->tune_comparator(rcomp);
db = gdb;
break;
}
case TYPEHASH: { case TYPEHASH: {
int8_t opts = 0; int8_t opts = 0;
if (tsmall) opts |= HashDB::TSMALL; if (tsmall) opts |= HashDB::TSMALL;
if (tlinear) opts |= HashDB::TLINEAR; if (tlinear) opts |= HashDB::TLINEAR;
if (tcompress) opts |= HashDB::TCOMPRESS; if (tcompress) opts |= HashDB::TCOMPRESS;
HashDB* hdb = new HashDB(); HashDB* hdb = new HashDB();
if (stdlogger_) {
hdb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
hdb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
hdb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
hdb->tune_meta_trigger(mtrigger_);
}
if (apow >= 0) hdb->tune_alignment(apow); if (apow >= 0) hdb->tune_alignment(apow);
if (fpow >= 0) hdb->tune_fbp(fpow); if (fpow >= 0) hdb->tune_fbp(fpow);
if (opts > 0) hdb->tune_options(opts); if (opts > 0) hdb->tune_options(opts);
if (bnum > 0) hdb->tune_buckets(bnum); if (bnum > 0) hdb->tune_buckets(bnum);
if (msiz >= 0) hdb->tune_map(msiz); if (msiz >= 0) hdb->tune_map(msiz);
if (dfunit > 0) hdb->tune_defrag(dfunit); if (dfunit > 0) hdb->tune_defrag(dfunit);
if (zcomp) hdb->tune_compressor(zcomp); if (zcomp_) hdb->tune_compressor(zcomp_);
if (erstrm) hdb->tune_error_reporter(erstrm, ervbs);
db = hdb; db = hdb;
break; break;
} }
case TYPETREE: { case TYPETREE: {
int8_t opts = 0; int8_t opts = 0;
if (tsmall) opts |= TreeDB::TSMALL; if (tsmall) opts |= TreeDB::TSMALL;
if (tlinear) opts |= TreeDB::TLINEAR; if (tlinear) opts |= TreeDB::TLINEAR;
if (tcompress) opts |= TreeDB::TCOMPRESS; if (tcompress) opts |= TreeDB::TCOMPRESS;
TreeDB* tdb = new TreeDB(); TreeDB* tdb = new TreeDB();
if (stdlogger_) {
tdb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
tdb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
tdb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
tdb->tune_meta_trigger(mtrigger_);
}
if (apow >= 0) tdb->tune_alignment(apow); if (apow >= 0) tdb->tune_alignment(apow);
if (fpow >= 0) tdb->tune_fbp(fpow); if (fpow >= 0) tdb->tune_fbp(fpow);
if (opts > 0) tdb->tune_options(opts); if (opts > 0) tdb->tune_options(opts);
if (bnum > 0) tdb->tune_buckets(bnum); if (bnum > 0) tdb->tune_buckets(bnum);
if (psiz > 0) tdb->tune_page(psiz); if (psiz > 0) tdb->tune_page(psiz);
if (msiz >= 0) tdb->tune_map(msiz); if (msiz >= 0) tdb->tune_map(msiz);
if (dfunit > 0) tdb->tune_defrag(dfunit); if (dfunit > 0) tdb->tune_defrag(dfunit);
if (zcomp) tdb->tune_compressor(zcomp); if (zcomp_) tdb->tune_compressor(zcomp_);
if (pccap > 0) tdb->tune_page_cache(pccap); if (pccap > 0) tdb->tune_page_cache(pccap);
if (rcomp) tdb->tune_comparator(rcomp); if (rcomp) tdb->tune_comparator(rcomp);
if (erstrm) tdb->tune_error_reporter(erstrm, ervbs);
db = tdb; db = tdb;
break; break;
} }
case TYPEDIR: { case TYPEDIR: {
int8_t opts = 0; int8_t opts = 0;
if (tcompress) opts |= DirDB::TCOMPRESS; if (tcompress) opts |= DirDB::TCOMPRESS;
DirDB* ddb = new DirDB(); DirDB* ddb = new DirDB();
if (stdlogger_) {
ddb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
ddb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
ddb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
ddb->tune_meta_trigger(mtrigger_);
}
if (opts > 0) ddb->tune_options(opts); if (opts > 0) ddb->tune_options(opts);
if (zcomp) ddb->tune_compressor(zcomp); if (zcomp_) ddb->tune_compressor(zcomp_);
db = ddb; db = ddb;
break; break;
} }
case TYPEFOREST: {
int8_t opts = 0;
if (tcompress) opts |= TreeDB::TCOMPRESS;
ForestDB* fdb = new ForestDB();
if (stdlogger_) {
fdb->tune_logger(stdlogger_, logkinds);
} else if (logger_) {
fdb->tune_logger(logger_, logkinds_);
}
if (stdmtrigger_) {
fdb->tune_meta_trigger(stdmtrigger_);
} else if (mtrigger_) {
fdb->tune_meta_trigger(mtrigger_);
}
if (opts > 0) fdb->tune_options(opts);
if (bnum > 0) fdb->tune_buckets(bnum);
if (psiz > 0) fdb->tune_page(psiz);
if (zcomp_) fdb->tune_compressor(zcomp_);
if (pccap > 0) fdb->tune_page_cache(pccap);
if (rcomp) fdb->tune_comparator(rcomp);
db = fdb;
break;
}
} }
if (arccomp) arccomp->set_key(zkey.c_str(), zkey.size()); if (arccomp) arccomp->set_key(zkey.c_str(), zkey.size());
if (!db->open(fpath, mode)) { if (!db->open(fpath, mode)) {
const Error& error = db->error(); const Error& error = db->error();
set_error(error.code(), error.message()); set_error(_KCCODELINE_, error.code(), error.message());
delete db; delete db;
return false; return false;
} }
if (arccomp) { if (arccomp) {
const std::string& apath = File::absolute_path(fpath); const std::string& apath = File::absolute_path(fpath);
uint64_t hash = (hashmurmur(apath.c_str(), apath.size()) >> 16) << 40 ; uint64_t hash = (hashmurmur(apath.c_str(), apath.size()) >> 16) << 40 ;
hash += (uint64_t)(time() * 256); hash += (uint64_t)(time() * 256);
arccomp->begin_cycle(hash); arccomp->begin_cycle(hash);
} }
type_ = type; type_ = type;
db_ = db; db_ = db;
return true; return true;
} }
/** /**
* Close the database file. * Close the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool close() { bool close() {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
bool err = false; bool err = false;
if (!db_->close()) { if (!db_->close()) {
const Error& error = db_->error(); const Error& error = db_->error();
set_error(error.code(), error.message()); set_error(_KCCODELINE_, error.code(), error.message());
err = true; err = true;
} }
delete zcomp_; delete zcomp_;
delete stdmtrigger_;
delete stdmtrgstrm_;
delete stdlogger_;
delete stdlogstrm_;
delete db_; delete db_;
type_ = TYPEVOID; type_ = TYPEVOID;
db_ = NULL; db_ = NULL;
stdlogstrm_ = NULL;
stdlogger_ = NULL;
stdmtrgstrm_ = NULL;
stdmtrigger_ = NULL;
zcomp_ = NULL; zcomp_ = NULL;
return !err; return !err;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @param proc a postprocessor object. If it is NULL, no postprocessing is performed. * @param proc a postprocessor object. If it is NULL, no postprocessing is performed.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize(bool hard = false, FileProcessor* proc = NULL) { bool synchronize(bool hard = false, FileProcessor* proc = NULL,
ProgressChecker* checker = NULL) {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return db_->synchronize(hard, proc); return db_->synchronize(hard, proc, checker);
} }
/** /**
* Begin transaction. * Begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction(bool hard = false) { bool begin_transaction(bool hard = false) {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return db_->begin_transaction(hard); return db_->begin_transaction(hard);
} }
/** /**
* Try to begin transaction. * Try to begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_try(bool hard = false) { bool begin_transaction_try(bool hard = false) {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return db_->begin_transaction_try(hard); return db_->begin_transaction_try(hard);
} }
/** /**
* End transaction. * End transaction.
* @param commit true to commit the transaction, or false to abort the tr ansaction. * @param commit true to commit the transaction, or false to abort the tr ansaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool end_transaction(bool commit = true) { bool end_transaction(bool commit = true) {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return db_->end_transaction(commit); return db_->end_transaction(commit);
} }
/** /**
* Remove all records. * Remove all records.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clear() { bool clear() {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return db_->clear(); return db_->clear();
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
*/ */
int64_t count() { int64_t count() {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return db_->count(); return db_->count();
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes, or -1 on failure. * @return the size of the database file in bytes, or -1 on failure.
*/ */
int64_t size() { int64_t size() {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return db_->size(); return db_->size();
} }
/** /**
* Get the path of the database file. * Get the path of the database file.
* @return the path of the database file, or an empty string on failure. * @return the path of the database file, or an empty string on failure.
*/ */
std::string path() { std::string path() {
_assert_(true); _assert_(true);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return ""; return "";
} }
return db_->path(); return db_->path();
} }
/** /**
* Get the miscellaneous status information. * Get the miscellaneous status information.
* @param strmap a string map to contain the result. * @param strmap a string map to contain the result.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool status(std::map<std::string, std::string>* strmap) { bool status(std::map<std::string, std::string>* strmap) {
_assert_(strmap); _assert_(strmap);
if (type_ == TYPEVOID) { if (type_ == TYPEVOID) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
return db_->status(strmap); return db_->status(strmap);
} }
/** /**
* Reveal the inner database object.
* @return the inner database object, or NULL on failure.
*/
BasicDB* reveal_inner_db() {
_assert_(true);
if (type_ == TYPEVOID) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return NULL;
}
return db_;
}
/**
* Get keys matching a prefix string.
* @param prefix the prefix string.
* @param strvec a string vector to contain the result.
* @param max the maximum number to retrieve. If it is negative, no limi
t is specified.
* @param checker a progress checker object. If it is NULL, no checking
is performed.
* @return the number of retrieved keys or -1 on failure.
*/
int64_t match_prefix(const std::string& prefix, std::vector<std::string>*
strvec,
int64_t max = -1, ProgressChecker* checker = NULL) {
_assert_(strvec);
const char* pbuf = prefix.data();
size_t psiz = prefix.size();
if (max < 0) max = INT64_MAX;
Comparator* comp;
switch (type_) {
case TYPEPTREE: {
comp = &LEXICALCOMP;
break;
}
case TYPEGRASS: {
comp = ((GrassDB*)db_)->rcomp();
break;
}
case TYPETREE: {
comp = ((TreeDB*)db_)->rcomp();
break;
}
case TYPEFOREST: {
comp = ((ForestDB*)db_)->rcomp();
break;
}
default: {
comp = NULL;
break;
}
}
bool err = false;
int64_t allcnt = count();
if (checker && !checker->check("match_prefix", "beginning", 0, allcnt))
{
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
strvec->clear();
Cursor* cur = cursor();
int64_t curcnt = 0;
if (comp == &LEXICALCOMP) {
if (cur->jump(pbuf, psiz)) {
while ((int64_t)strvec->size() < max) {
size_t ksiz;
char* kbuf = cur->get_key(&ksiz, true);
if (kbuf) {
if (ksiz >= psiz && !std::memcmp(kbuf, pbuf, psiz)) {
strvec->push_back(std::string(kbuf, ksiz));
} else {
delete[] kbuf;
break;
}
delete[] kbuf;
} else {
if (cur->error() != Error::NOREC) err = true;
break;
}
curcnt++;
if (checker && !checker->check("match_prefix", "processing", curc
nt, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
}
} else if (cur->error() != Error::NOREC) {
err = true;
}
} else {
if (cur->jump()) {
while ((int64_t)strvec->size() < max) {
size_t ksiz;
char* kbuf = cur->get_key(&ksiz, true);
if (kbuf) {
if (ksiz >= psiz && !std::memcmp(kbuf, pbuf, psiz))
strvec->push_back(std::string(kbuf, ksiz));
delete[] kbuf;
} else {
if (cur->error() != Error::NOREC) err = true;
break;
}
curcnt++;
if (checker && !checker->check("match_prefix", "processing", curc
nt, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
}
} else if (cur->error() != Error::NOREC) {
err = true;
}
}
if (checker && !checker->check("match_prefix", "ending", strvec->size()
, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
delete cur;
return err ? -1 : strvec->size();
}
/**
* Get keys matching a regular expression string.
* @param regex the regular expression string.
* @param strvec a string vector to contain the result.
* @param max the maximum number to retrieve. If it is negative, no limi
t is specified.
* @param checker a progress checker object. If it is NULL, no checking
is performed.
* @return the number of retrieved keys or -1 on failure.
*/
int64_t match_regex(const std::string& regex, std::vector<std::string>* s
trvec,
int64_t max = -1, ProgressChecker* checker = NULL) {
_assert_(strvec);
if (max < 0) max = INT64_MAX;
Regex reg;
if (!reg.compile(regex, Regex::MATCHONLY)) {
set_error(_KCCODELINE_, Error::LOGIC, "compilation failed");
return -1;
}
bool err = false;
int64_t allcnt = count();
if (checker && !checker->check("match_regex", "beginning", 0, allcnt))
{
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
strvec->clear();
Cursor* cur = cursor();
int64_t curcnt = 0;
if (cur->jump()) {
while ((int64_t)strvec->size() < max) {
size_t ksiz;
char* kbuf = cur->get_key(&ksiz, true);
if (kbuf) {
std::string key(kbuf, ksiz);
if (reg.match(key)) strvec->push_back(key);
delete[] kbuf;
} else {
if (cur->error() != Error::NOREC) err = true;
break;
}
curcnt++;
if (checker && !checker->check("match_regex", "processing", curcnt,
allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
}
} else if (cur->error() != Error::NOREC) {
err = true;
}
if (checker && !checker->check("match_regex", "ending", strvec->size(),
allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
delete cur;
return err ? -1 : strvec->size();
}
/**
* Merge records from other databases.
* @param srcary an array of the source detabase objects.
* @param srcnum the number of the elements of the source array.
* @param mode the merge mode. PolyDB::MSET to overwrite the existing va
lue, PolyDB::MADD to
* keep the existing value, PolyDB::MREPLACE to modify the existing recor
d only,
* PolyDB::MAPPEND to append the new value.
* @param checker a progress checker object. If it is NULL, no checking
is performed.
* @return true on success, or false on failure.
*/
bool merge(BasicDB** srcary, size_t srcnum, MergeMode mode = MSET,
ProgressChecker* checker = NULL) {
_assert_(srcary && srcnum <= MEMMAXSIZ);
if (type_ == TYPEVOID) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
bool err = false;
Comparator* comp;
switch (type_) {
case TYPEGRASS: {
comp = ((GrassDB*)db_)->rcomp();
break;
}
case TYPETREE: {
comp = ((TreeDB*)db_)->rcomp();
break;
}
case TYPEFOREST: {
comp = ((ForestDB*)db_)->rcomp();
break;
}
default: {
comp = NULL;
break;
}
}
if (!comp) comp = &LEXICALCOMP;
std::priority_queue<MergeLine> lines;
int64_t allcnt = 0;
for (size_t i = 0; i < srcnum; i++) {
MergeLine line;
line.cur = srcary[i]->cursor();
line.comp = comp;
line.cur->jump();
line.kbuf = line.cur->get(&line.ksiz, &line.vbuf, &line.vsiz, true);
if (line.kbuf) {
lines.push(line);
int64_t count = srcary[i]->count();
if (count > 0) allcnt += count;
} else {
delete line.cur;
}
}
if (checker && !checker->check("merge", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
int64_t curcnt = 0;
while (!err && !lines.empty()) {
MergeLine line = lines.top();
lines.pop();
switch (mode) {
case MSET: {
if (!set(line.kbuf, line.ksiz, line.vbuf, line.vsiz)) err = true;
break;
}
case MADD: {
if (!add(line.kbuf, line.ksiz, line.vbuf, line.vsiz) &&
error() != Error::DUPREC) err = true;
break;
}
case MREPLACE: {
if (!replace(line.kbuf, line.ksiz, line.vbuf, line.vsiz) &&
error() != Error::NOREC) err = true;
break;
}
case MAPPEND: {
if (!append(line.kbuf, line.ksiz, line.vbuf, line.vsiz)) err = tr
ue;
break;
}
}
delete[] line.kbuf;
line.kbuf = line.cur->get(&line.ksiz, &line.vbuf, &line.vsiz, true);
if (line.kbuf) {
lines.push(line);
} else {
delete line.cur;
}
curcnt++;
if (checker && !checker->check("merge", "processing", curcnt, allcnt)
) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
break;
}
}
if (checker && !checker->check("merge", "ending", -1, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
err = true;
}
while (!lines.empty()) {
MergeLine line = lines.top();
lines.pop();
delete[] line.kbuf;
delete line.cur;
}
return !err;
}
/**
* Create a cursor object. * Create a cursor object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note Because the object of the return value is allocated by the const ructor, it should be * @note Because the object of the return value is allocated by the const ructor, it should be
* released with the delete operator when it is no longer in use. * released with the delete operator when it is no longer in use.
*/ */
Cursor* cursor() { Cursor* cursor() {
_assert_(true); _assert_(true);
return new Cursor(this); return new Cursor(this);
} }
/** /**
* Reveal the inner database object. * Set the internal logger.
* @return the inner database object, or NULL on failure. * @param logger the logger object.
* @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for
debugging,
* Logger::INFO for normal information, Logger::WARN for warning, and Log
ger::ERROR for fatal
* error.
* @return true on success, or false on failure.
*/ */
FileDB* reveal_inner_db() { bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::
_assert_(true); ERROR) {
if (type_ == TYPEVOID) { _assert_(logger);
set_error(Error::INVALID, "not opened"); if (type_ != TYPEVOID) {
return NULL; set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
} }
return db_; logger_ = logger;
logkinds_ = kinds;
return true;
}
/**
* Set the internal meta operation trigger.
* @param trigger the trigger object.
* @return true on success, or false on failure.
*/
bool tune_meta_trigger(MetaTrigger* trigger) {
_assert_(trigger);
if (type_ != TYPEVOID) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
mtrigger_ = trigger;
return true;
} }
private: private:
/**
* Stream logger implementation.
*/
class StreamLogger : public Logger {
public:
/** constructor */
StreamLogger(std::ostream* strm, const char* prefix) : strm_(strm), pre
fix_(prefix) {}
/** print a log message */
void log(const char* file, int32_t line, const char* func, Kind kind,
const char* message) {
_assert_(file && line > 0 && func && message);
const char* kstr = "MISC";
switch (kind) {
case Logger::DEBUG: kstr = "DEBUG"; break;
case Logger::INFO: kstr = "INFO"; break;
case Logger::WARN: kstr = "WARN"; break;
case Logger::ERROR: kstr = "ERROR"; break;
}
if (!prefix_.empty()) *strm_ << prefix_ << ": ";
*strm_ << "[" << kstr << "]: " << file << ": " << line << ": " << fun
c << ": " <<
message << std::endl;
}
private:
std::ostream* strm_; ///< output stream
std::string prefix_; ///< prefix of each message
};
/**
* Stream meta operation trigger implementation.
*/
class StreamMetaTrigger : public MetaTrigger {
public:
/** constructor */
StreamMetaTrigger(std::ostream* strm, const char* prefix) : strm_(strm)
, prefix_(prefix) {}
/** print a meta operation */
void trigger(Kind kind, const char* message) {
_assert_(message);
const char* kstr = "unknown";
switch (kind) {
case MetaTrigger::OPEN: kstr = "OPEN"; break;
case MetaTrigger::CLOSE: kstr = "CLOSE"; break;
case MetaTrigger::CLEAR: kstr = "CLEAR"; break;
case MetaTrigger::ITERATE: kstr = "ITERATE"; break;
case MetaTrigger::SYNCHRONIZE: kstr = "SYNCHRONIZE"; break;
case MetaTrigger::BEGINTRAN: kstr = "BEGINTRAN"; break;
case MetaTrigger::COMMITTRAN: kstr = "COMMITTRAN"; break;
case MetaTrigger::ABORTTRAN: kstr = "ABORTTRAN"; break;
case MetaTrigger::MISC: kstr = "MISC"; break;
}
if (!prefix_.empty()) *strm_ << prefix_ << ": ";
*strm_ << "[" << kstr << "]: " << message << std::endl;
}
private:
std::ostream* strm_; ///< output stream
std::string prefix_; ///< prefix of each message
};
/**
* Front line of a merging list.
*/
struct MergeLine {
BasicDB::Cursor* cur; ///< cursor
Comparator* comp; ///< comparator
char* kbuf; ///< pointer to the key
size_t ksiz; ///< size of the key
const char* vbuf; ///< pointer to the value
size_t vsiz; ///< size of the value
/** comparing operator */
bool operator <(const MergeLine& right) const {
return comp->compare(kbuf, ksiz, right.kbuf, right.ksiz) > 0;
}
};
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
PolyDB(const PolyDB&); PolyDB(const PolyDB&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
PolyDB& operator =(const PolyDB&); PolyDB& operator =(const PolyDB&);
/** The database type. */ /** The database type. */
Type type_; Type type_;
/** The internal database. */ /** The internal database. */
FileDB* db_; BasicDB* db_;
/** The last happened error. */ /** The last happened error. */
TSD<Error> error_; Error error_;
/** The standard log stream. */
std::ostream* stdlogstrm_;
/** The standard logger. */
Logger* stdlogger_;
/** The internal logger. */
Logger* logger_;
/** The kinds of logged messages. */
uint32_t logkinds_;
/** The standard meta operation trigger stream. */
std::ostream* stdmtrgstrm_;
/** The standard meta operation trigger. */
MetaTrigger* stdmtrigger_;
/** The internal meta operation trigger. */
MetaTrigger* mtrigger_;
/** The custom compressor. */ /** The custom compressor. */
Compressor* zcomp_; Compressor* zcomp_;
}; };
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 103 change blocks. 
181 lines changed or deleted 991 lines changed or added


 kcprotodb.h   kcprotodb.h 
skipping to change at line 20 skipping to change at line 20
* See the GNU General Public License for more details. * See the GNU General Public License for more details.
* You should have received a copy of the GNU General Public License along with this program. * You should have received a copy of the GNU General Public License along with this program.
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
************************************************************************** ***********************/ ************************************************************************** ***********************/
#ifndef _KCPROTODB_H // duplication check #ifndef _KCPROTODB_H // duplication check
#define _KCPROTODB_H #define _KCPROTODB_H
#include <kccommon.h> #include <kccommon.h>
#include <kcutil.h> #include <kcutil.h>
#include <kcdb.h>
#include <kcthread.h> #include <kcthread.h>
#include <kcfile.h> #include <kcfile.h>
#include <kccompress.h> #include <kccompress.h>
#include <kccompare.h> #include <kccompare.h>
#include <kcmap.h> #include <kcmap.h>
#include <kcregex.h>
#include <kcdb.h>
#include <kcplantdb.h>
namespace kyotocabinet { // common namespace namespace kyotocabinet { // common namespace
/** /**
* Constants for implementation. * Constants for implementation.
*/ */
namespace { namespace {
const size_t PDBBNUM = 1048583LL; ///< bucket number of hash table const size_t PDBHASHBNUM = 1048583LL; ///< bucket number of hash table
const size_t PDBOPAQUESIZ = 16; ///< size of the opaque buffer
} }
/** /**
* Helper functions. * Helper functions.
*/ */
namespace { namespace {
template <class STRMAP> template <class STRMAP>
typename STRMAP::iterator map_find(STRMAP* map, const std::string& key) { typename STRMAP::iterator map_find(STRMAP* map, const std::string& key) {
return map->find(key); return map->find(key);
} }
template <> template <>
StringTreeMap::iterator map_find(StringTreeMap* map, const std::string& key ) { StringTreeMap::iterator map_find(StringTreeMap* map, const std::string& key ) {
StringTreeMap::iterator it = map->find(key); StringTreeMap::iterator it = map->find(key);
if (it != map->end()) return it; if (it != map->end()) return it;
return map->upper_bound(key); return map->upper_bound(key);
} }
template <class STRMAP> template <class STRMAP>
void map_tune(STRMAP* map) {} void map_tune(STRMAP* map) {}
template <> template <>
void map_tune(StringHashMap* map) { void map_tune(StringHashMap* map) {
map->rehash(PDBBNUM); map->rehash(PDBHASHBNUM);
map->max_load_factor(FLT_MAX); map->max_load_factor(FLT_MAX);
} }
template <class STRMAP> template <class ITER>
FileDB::Type map_type(STRMAP* map) { bool iter_back(ITER* itp) {
return FileDB::TYPEPMISC; return false;
}
template <>
FileDB::Type map_type(StringHashMap* map) {
return FileDB::TYPEPHASH;
} }
template <> template <>
FileDB::Type map_type(StringTreeMap* map) { bool iter_back(StringTreeMap::iterator* itp) {
return FileDB::TYPEPTREE; --(*itp);
return true;
} }
} }
/** /**
* Prototype implementation of file database with STL. * Prototype implementation of database with STL.
* @param STRMAP a map compatible class of STL. * @param STRMAP a class compatible with the map class of STL.
* @param DBTYPE the database type number of the class.
* @note This class template is a template for concrete classes which wrap data structures * @note This class template is a template for concrete classes which wrap data structures
* compatible with std::map. Template instance classes can be inherited bu t overwriting methods * compatible with std::map. Template instance classes can be inherited bu t overwriting methods
* is forbidden. The class ProtoHashDB is the instance using std::unordere d_map. The class * is forbidden. The class ProtoHashDB is the instance using std::unordere d_map. The class
* ProtoTreeDB is the instance using std::map. Before every database opera tion, it is necessary * ProtoTreeDB is the instance using std::map. Before every database opera tion, it is necessary
* to call the CacheDB::open method in order to open a database file and co nnect the database * to call the BasicDB::open method in order to open a database file and co nnect the database
* object to it. To avoid data missing or corruption, it is important to c lose every database * object to it. To avoid data missing or corruption, it is important to c lose every database
* file by the CacheDB::close method when the database is no longer in use. * file by the BasicDB::close method when the database is no longer in use.
It is forbidden for It is forbidden for
* multible database objects in a process to open the same database at the * multible database objects in a process to open the same database at the
same time. same time. It is
* forbidden to share a database object with child processes.
*/ */
template <class STRMAP> template <class STRMAP, uint8_t DBTYPE>
class ProtoDB : public FileDB { class ProtoDB : public BasicDB {
public: public:
class Cursor; class Cursor;
private: private:
struct TranLog; struct TranLog;
/** An alias of list of cursors. */ /** An alias of list of cursors. */
typedef std::list<Cursor*> CursorList; typedef std::list<Cursor*> CursorList;
/** An alias of list of transaction logs. */ /** An alias of list of transaction logs. */
typedef std::list<TranLog> TranLogList; typedef std::list<TranLog> TranLogList;
public: public:
/** /**
* Cursor to indicate a record. * Cursor to indicate a record.
*/ */
class Cursor : public FileDB::Cursor { class Cursor : public BasicDB::Cursor {
friend class ProtoDB; friend class ProtoDB;
public: public:
/** /**
* Constructor. * Constructor.
* @param db the container database object. * @param db the container database object.
*/ */
explicit Cursor(ProtoDB* db) : db_(db), it_(db->recs_.end()) { explicit Cursor(ProtoDB* db) : db_(db), it_(db->recs_.end()) {
_assert_(db); _assert_(db);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
db_->curs_.push_back(this); db_->curs_.push_back(this);
skipping to change at line 124 skipping to change at line 126
if (!db_) return; if (!db_) return;
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
db_->curs_.remove(this); db_->curs_.remove(this);
} }
/** /**
* Accept a visitor to the current record. * Accept a visitor to the current record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only operation. * @param writable true for writable operation, or false for read-only operation.
* @param step true to move the cursor to the next record, or false for no move. * @param step true to move the cursor to the next record, or false for no move.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and othe * @note The operation for each record is performed atomically and othe
r threads accessing r threads accessing
* the same record are blocked. * the same record are blocked. To avoid deadlock, any database operat
ion must not be
* performed in this function.
*/ */
bool accept(Visitor* visitor, bool writable = true, bool step = false) { bool accept(Visitor* visitor, bool writable = true, bool step = false) {
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(db_->omode_ & OWRITER)) { if (writable && !(db_->omode_ & OWRITER)) {
db_->set_error(Error::NOPERM, "permission denied"); db_->set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
if (it_ == db_->recs_.end()) { if (it_ == db_->recs_.end()) {
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
const std::string& key = it_->first; const std::string& key = it_->first;
const std::string& value = it_->second; const std::string& value = it_->second;
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(key.c_str(), key.size(), const char* vbuf = visitor->visit_full(key.c_str(), key.size(),
value.c_str(), value.size(), & vsiz); value.c_str(), value.size(), & vsiz);
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
if (db_->tran_) { if (db_->tran_) {
TranLog log(key, value); TranLog log(key, value);
skipping to change at line 178 skipping to change at line 181
db_->trlogs_.push_back(log); db_->trlogs_.push_back(log);
} }
db_->size_ -= value.size(); db_->size_ -= value.size();
db_->size_ += vsiz; db_->size_ += vsiz;
it_->second = std::string(vbuf, vsiz); it_->second = std::string(vbuf, vsiz);
if (step) it_++; if (step) it_++;
} }
return true; return true;
} }
/** /**
* Jump the cursor to the first record. * Jump the cursor to the first record for forward scan.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump() { bool jump() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
it_ = db_->recs_.begin(); it_ = db_->recs_.begin();
if (it_ == db_->recs_.end()) { if (it_ == db_->recs_.end()) {
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
return true; return true;
} }
/** /**
* Jump the cursor onto a record. * Jump the cursor to a record for forward scan.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool jump(const char* kbuf, size_t ksiz) { bool jump(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ); _assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
std::string key(kbuf, ksiz); std::string key(kbuf, ksiz);
it_ = map_find(&db_->recs_, key); it_ = map_find(&db_->recs_, key);
if (it_ == db_->recs_.end()) { if (it_ == db_->recs_.end()) {
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
return true; return true;
} }
/** /**
* Jump the cursor to a record. * Jump the cursor to a record for forward scan.
* @note Equal to the original Cursor::jump method except that the para meter is std::string. * @note Equal to the original Cursor::jump method except that the para meter is std::string.
*/ */
bool jump(const std::string& key) { bool jump(const std::string& key) {
_assert_(true); _assert_(true);
return jump(key.c_str(), key.size()); return jump(key.c_str(), key.size());
} }
/** /**
* Jump the cursor to the last record for backward scan.
* @return true on success, or false on failure.
*/
bool jump_back() {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
it_ = db_->recs_.end();
if (it_ == db_->recs_.begin()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false;
}
if (!iter_back(&it_)) {
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
return true;
}
/**
* Jump the cursor to a record for backward scan.
* @param kbuf the pointer to the key region.
* @param ksiz the size of the key region.
* @return true on success, or false on failure.
*/
bool jump_back(const char* kbuf, size_t ksiz) {
_assert_(kbuf && ksiz <= MEMMAXSIZ);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
std::string key(kbuf, ksiz);
it_ = map_find(&db_->recs_, key);
if (it_ == db_->recs_.end()) {
if (it_ == db_->recs_.begin()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false;
}
if (!iter_back(&it_)) {
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
} else {
std::string key(kbuf, ksiz);
if (key < it_->first) {
if (it_ == db_->recs_.begin()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
it_ = db_->recs_.end();
return false;
}
if (!iter_back(&it_)) {
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
it_ = db_->recs_.end();
return false;
}
}
}
return true;
}
/**
* Jump the cursor to a record for backward scan.
* @note Equal to the original Cursor::jump_back method except that the
parameter is
* std::string.
*/
bool jump_back(const std::string& key) {
_assert_(true);
return jump_back(key.c_str(), key.size());
}
/**
* Step the cursor to the next record. * Step the cursor to the next record.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool step() { bool step() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true); ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) { if (db_->omode_ == 0) {
db_->set_error(Error::INVALID, "not opened"); db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (it_ == db_->recs_.end()) { if (it_ == db_->recs_.end()) {
db_->set_error(Error::NOREC, "no record"); db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false; return false;
} }
it_++; it_++;
if (it_ == db_->recs_.end()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
return false;
}
return true;
}
/**
* Step the cursor to the previous record.
* @return true on success, or false on failure.
*/
bool step_back() {
_assert_(true);
ScopedSpinRWLock lock(&db_->mlock_, true);
if (db_->omode_ == 0) {
db_->set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
if (it_ == db_->recs_.begin()) {
db_->set_error(_KCCODELINE_, Error::NOREC, "no record");
it_ = db_->recs_.end();
return false;
}
if (!iter_back(&it_)) {
db_->set_error(_KCCODELINE_, Error::NOIMPL, "not implemented");
return false;
}
return true; return true;
} }
/** /**
* Get the database object. * Get the database object.
* @return the database object. * @return the database object.
*/ */
ProtoDB* db() { ProtoDB* db() {
_assert_(true); _assert_(true);
return db_; return db_;
} }
skipping to change at line 263 skipping to change at line 364
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
Cursor& operator =(const Cursor&); Cursor& operator =(const Cursor&);
/** The inner database. */ /** The inner database. */
ProtoDB* db_; ProtoDB* db_;
/** The inner iterator. */ /** The inner iterator. */
typename STRMAP::iterator it_; typename STRMAP::iterator it_;
}; };
/** /**
* Default constructor. * Default constructor.
*/ */
explicit ProtoDB() : mlock_(), error_(), omode_(0), recs_(), explicit ProtoDB() :
curs_(), path_(""), size_(0), tran_(false), trlogs_( mlock_(), error_(), logger_(NULL), logkinds_(0), mtrigger_(NULL),
), trsize_(0) { omode_(0), recs_(), curs_(), path_(""), size_(0), opaque_(),
tran_(false), trlogs_(), trsize_(0) {
_assert_(true); _assert_(true);
map_tune(&recs_); map_tune(&recs_);
} }
/** /**
* Destructor. * Destructor.
* @note If the database is not closed, it is closed implicitly. * @note If the database is not closed, it is closed implicitly.
*/ */
virtual ~ProtoDB() { virtual ~ProtoDB() {
_assert_(true); _assert_(true);
if (omode_ != 0) close(); if (omode_ != 0) close();
skipping to change at line 292 skipping to change at line 395
} }
} }
} }
/** /**
* Accept a visitor to a record. * Accept a visitor to a record.
* @param kbuf the pointer to the key region. * @param kbuf the pointer to the key region.
* @param ksiz the size of the key region. * @param ksiz the size of the key region.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the operation for each record is performed atomically and other * @note The operation for each record is performed atomically and other
threads accessing the threads accessing the
* same record are blocked. * same record are blocked. To avoid deadlock, any database operation mu
st not be performed in
* this function.
*/ */
bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) { bool accept(const char* kbuf, size_t ksiz, Visitor* visitor, bool writabl e = true) {
_assert_(kbuf && ksiz <= MEMMAXSIZ && visitor); _assert_(kbuf && ksiz <= MEMMAXSIZ && visitor);
if (writable) { if (writable) {
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
std::string key(kbuf, ksiz); std::string key(kbuf, ksiz);
typename STRMAP::iterator it = recs_.find(key); typename STRMAP::iterator it = recs_.find(key);
if (it == recs_.end()) { if (it == recs_.end()) {
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz);
if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) {
if (tran_) { if (tran_) {
TranLog log(key); TranLog log(key);
skipping to change at line 353 skipping to change at line 457
trlogs_.push_back(log); trlogs_.push_back(log);
} }
size_ -= value.size(); size_ -= value.size();
size_ += vsiz; size_ += vsiz;
it->second = std::string(vbuf, vsiz); it->second = std::string(vbuf, vsiz);
} }
} }
} else { } else {
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
std::string key(kbuf, ksiz); std::string key(kbuf, ksiz);
const STRMAP& rrecs = recs_; const STRMAP& rrecs = recs_;
typename STRMAP::const_iterator it = rrecs.find(key); typename STRMAP::const_iterator it = rrecs.find(key);
if (it == rrecs.end()) { if (it == rrecs.end()) {
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz); const char* vbuf = visitor->visit_empty(kbuf, ksiz, &vsiz);
if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
} else { } else {
const std::string& value = it->second; const std::string& value = it->second;
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(kbuf, ksiz, value.c_str(), v alue.size(), &vsiz); const char* vbuf = visitor->visit_full(kbuf, ksiz, value.c_str(), v alue.size(), &vsiz);
if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) { if (vbuf != Visitor::NOP && vbuf != Visitor::REMOVE) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false; return false;
} }
} }
} }
return true; return true;
} }
/** /**
* Iterate to accept a visitor for each record. * Iterate to accept a visitor for each record.
* @param visitor a visitor object. * @param visitor a visitor object.
* @param writable true for writable operation, or false for read-only op eration. * @param writable true for writable operation, or false for read-only op eration.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note the whole iteration is performed atomically and other threads ar * @note The whole iteration is performed atomically and other threads ar
e blocked. e blocked. To avoid
* deadlock, any database operation must not be performed in this functio
n.
*/ */
bool iterate(Visitor *visitor, bool writable = true) { bool iterate(Visitor *visitor, bool writable = true, ProgressChecker* che cker = NULL) {
_assert_(visitor); _assert_(visitor);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (writable && !(omode_ & OWRITER)) { if (writable && !(omode_ & OWRITER)) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false;
}
int64_t allcnt = recs_.size();
if (checker && !checker->check("iterate", "beginning", 0, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
typename STRMAP::iterator it = recs_.begin(); typename STRMAP::iterator it = recs_.begin();
typename STRMAP::iterator itend = recs_.end(); typename STRMAP::iterator itend = recs_.end();
int64_t curcnt = 0;
while (it != itend) { while (it != itend) {
const std::string& key = it->first; const std::string& key = it->first;
const std::string& value = it->second; const std::string& value = it->second;
size_t vsiz; size_t vsiz;
const char* vbuf = visitor->visit_full(key.c_str(), key.size(), const char* vbuf = visitor->visit_full(key.c_str(), key.size(),
value.c_str(), value.size(), & vsiz); value.c_str(), value.size(), & vsiz);
if (vbuf == Visitor::REMOVE) { if (vbuf == Visitor::REMOVE) {
size_ -= key.size() + value.size(); size_ -= key.size() + value.size();
recs_.erase(it++); recs_.erase(it++);
} else if (vbuf == Visitor::NOP) { } else if (vbuf == Visitor::NOP) {
it++; it++;
} else { } else {
size_ -= value.size(); size_ -= value.size();
size_ += vsiz; size_ += vsiz;
it->second = std::string(vbuf, vsiz); it->second = std::string(vbuf, vsiz);
it++; it++;
} }
curcnt++;
if (checker && !checker->check("iterate", "processing", curcnt, allcn
t)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
}
if (checker && !checker->check("iterate", "ending", -1, allcnt)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
} }
trigger_meta(MetaTrigger::ITERATE, "iterate");
return true; return true;
} }
/** /**
* Get the last happened error. * Get the last happened error.
* @return the last happened error. * @return the last happened error.
*/ */
Error error() const { Error error() const {
_assert_(true); _assert_(true);
return error_; return error_;
} }
/** /**
* Set the error information. * Set the error information.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param code an error code. * @param code an error code.
* @param message a supplement message. * @param message a supplement message.
*/ */
void set_error(Error::Code code, const char* message) { void set_error(const char* file, int32_t line, const char* func,
_assert_(message); Error::Code code, const char* message) {
_assert_(file && line > 0 && func && message);
error_->set(code, message); error_->set(code, message);
if (logger_) {
Logger::Kind kind = code == Error::BROKEN || code == Error::SYSTEM ?
Logger::ERROR : Logger::INFO;
if (kind & logkinds_)
report(file, line, func, kind, "%d: %s: %s", code, Error::codename(
code), message);
}
} }
/** /**
* Open a database file. * Open a database file.
* @param path the path of a database file. * @param path the path of a database file.
* @param mode the connection mode. FileDB::OWRITER as a writer, FileDB: * @param mode the connection mode. BasicDB::OWRITER as a writer, BasicD
:OREADER as a B::OREADER as a
* reader. The following may be added to the writer mode by bitwise-or: * reader. The following may be added to the writer mode by bitwise-or:
FileDB::OCREATE, BasicDB::OCREATE,
* which means it creates a new database if the file does not exist, File * which means it creates a new database if the file does not exist, Basi
DB::OTRUNCATE, which cDB::OTRUNCATE, which
* means it creates a new database regardless if the file exists, FileDB: * means it creates a new database regardless if the file exists, BasicDB
:OAUTOTRAN, which ::OAUTOTRAN, which
* means each updating operation is performed in implicit transaction, Fi * means each updating operation is performed in implicit transaction, Ba
leDB::OAUTOSYNC, sicDB::OAUTOSYNC,
* which means each updating operation is followed by implicit synchroniz ation with the file * which means each updating operation is followed by implicit synchroniz ation with the file
* system. The following may be added to both of the reader mode and the writer mode by * system. The following may be added to both of the reader mode and the writer mode by
* bitwise-or: FileDB::ONOLOCK, which means it opens the database file wi * bitwise-or: BasicDB::ONOLOCK, which means it opens the database file w
thout file locking, ithout file locking,
* FileDB::OTRYLOCK, which means locking is performed without blocking, F * BasicDB::OTRYLOCK, which means locking is performed without blocking,
ileDB::ONOREPAIR, BasicDB::ONOREPAIR,
* which means the database file is not repaired implicitly even if file destruction is * which means the database file is not repaired implicitly even if file destruction is
* detected. * detected.
* @return true on success, or false on failure. * @return true on success, or false on failure.
* @note Every opened database must be closed by the FileDB::close method when it is no * @note Every opened database must be closed by the BasicDB::close metho d when it is no
* longer in use. It is not allowed for two or more database objects in the same process to * longer in use. It is not allowed for two or more database objects in the same process to
* keep their connections to the same database file at the same time. * keep their connections to the same database file at the same time.
*/ */
bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) { bool open(const std::string& path, uint32_t mode = OWRITER | OCREATE) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) { if (omode_ != 0) {
set_error(Error::INVALID, "already opened"); set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "opening the database (path=%s)", p ath.c_str());
omode_ = mode; omode_ = mode;
path_.append(path); path_.append(path);
std::memset(opaque_, 0, sizeof(opaque_));
trigger_meta(MetaTrigger::OPEN, "open");
return true; return true;
} }
/** /**
* Close the database file. * Close the database file.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool close() { bool close() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
report(_KCCODELINE_, Logger::DEBUG, "closing the database (path=%s)", p ath_.c_str());
tran_ = false; tran_ = false;
trlogs_.clear(); trlogs_.clear();
recs_.clear(); recs_.clear();
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->it_ = recs_.end(); cur->it_ = recs_.end();
cit++; cit++;
} }
} }
path_.clear(); path_.clear();
omode_ = 0; omode_ = 0;
trigger_meta(MetaTrigger::CLOSE, "close");
return true; return true;
} }
/** /**
* Synchronize updated contents with the file and the device. * Synchronize updated contents with the file and the device.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @param proc a postprocessor object. If it is NULL, no postprocessing is performed. * @param proc a postprocessor object. If it is NULL, no postprocessing is performed.
* @param checker a progress checker object. If it is NULL, no checking is performed.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool synchronize(bool hard = false, FileProcessor* proc = NULL) { bool synchronize(bool hard = false, FileProcessor* proc = NULL,
ProgressChecker* checker = NULL) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!(omode_ & OWRITER)) { bool err = false;
set_error(Error::NOPERM, "permission denied"); if ((omode_ & OWRITER) && checker &&
!checker->check("synchronize", "nothing to be synchronized", -1, -1
)) {
set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false; return false;
} }
bool err = false; if (proc) {
if (proc && !proc->process(path_, recs_.size(), size_)) { if (checker && !checker->check("synchronize", "running the post proce
set_error(Error::LOGIC, "postprocessing failed"); ssor", -1, -1)) {
err = true; set_error(_KCCODELINE_, Error::LOGIC, "checker failed");
return false;
}
if (!proc->process(path_, recs_.size(), size_)) {
set_error(_KCCODELINE_, Error::LOGIC, "postprocessing failed");
err = true;
}
} }
trigger_meta(MetaTrigger::SYNCHRONIZE, "synchronize");
return !err; return !err;
} }
/** /**
* Begin transaction. * Begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction(bool hard = false) { bool begin_transaction(bool hard = false) {
_assert_(true); _assert_(true);
for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) { for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) {
mlock_.lock_writer(); mlock_.lock_writer();
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!tran_) break; if (!tran_) break;
mlock_.unlock(); mlock_.unlock();
if (wsec > 1.0) wsec = 1.0; if (wsec > 1.0) wsec = 1.0;
Thread::sleep(wsec); Thread::sleep(wsec);
} }
tran_ = true; tran_ = true;
trsize_ = size_; trsize_ = size_;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction");
mlock_.unlock(); mlock_.unlock();
return true; return true;
} }
/** /**
* Try to begin transaction. * Try to begin transaction.
* @param hard true for physical synchronization with the device, or fals e for logical * @param hard true for physical synchronization with the device, or fals e for logical
* synchronization with the file system. * synchronization with the file system.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool begin_transaction_try(bool hard = false) { bool begin_transaction_try(bool hard = false) {
_assert_(true); _assert_(true);
mlock_.lock_writer(); mlock_.lock_writer();
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (!(omode_ & OWRITER)) { if (!(omode_ & OWRITER)) {
set_error(Error::NOPERM, "permission denied"); set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
if (tran_) { if (tran_) {
set_error(Error::LOGIC, "competition avoided"); set_error(_KCCODELINE_, Error::LOGIC, "competition avoided");
mlock_.unlock(); mlock_.unlock();
return false; return false;
} }
tran_ = true; tran_ = true;
trsize_ = size_; trsize_ = size_;
trigger_meta(MetaTrigger::BEGINTRAN, "begin_transaction_try");
mlock_.unlock(); mlock_.unlock();
return true; return true;
} }
/** /**
* End transaction. * End transaction.
* @param commit true to commit the transaction, or false to abort the tr ansaction. * @param commit true to commit the transaction, or false to abort the tr ansaction.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool end_transaction(bool commit = true) { bool end_transaction(bool commit = true) {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
if (!tran_) { if (!tran_) {
set_error(Error::INVALID, "not in transaction"); set_error(_KCCODELINE_, Error::INVALID, "not in transaction");
return false; return false;
} }
if (!commit) { if (!commit) {
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->it_ = recs_.end(); cur->it_ = recs_.end();
cit++; cit++;
skipping to change at line 617 skipping to change at line 766
if (lit->full) { if (lit->full) {
recs_[lit->key] = lit->value; recs_[lit->key] = lit->value;
} else { } else {
recs_.erase(lit->key); recs_.erase(lit->key);
} }
} }
size_ = trsize_; size_ = trsize_;
} }
trlogs_.clear(); trlogs_.clear();
tran_ = false; tran_ = false;
trigger_meta(commit ? MetaTrigger::COMMITTRAN : MetaTrigger::ABORTTRAN, "end_transaction");
return true; return true;
} }
/** /**
* Remove all records. * Remove all records.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool clear() { bool clear() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
recs_.clear(); recs_.clear();
if (!curs_.empty()) { if (!curs_.empty()) {
typename CursorList::const_iterator cit = curs_.begin(); typename CursorList::const_iterator cit = curs_.begin();
typename CursorList::const_iterator citend = curs_.end(); typename CursorList::const_iterator citend = curs_.end();
while (cit != citend) { while (cit != citend) {
Cursor* cur = *cit; Cursor* cur = *cit;
cur->it_ = recs_.end(); cur->it_ = recs_.end();
cit++; cit++;
} }
} }
std::memset(opaque_, 0, sizeof(opaque_));
trigger_meta(MetaTrigger::CLEAR, "clear");
return true; return true;
} }
/** /**
* Get the number of records. * Get the number of records.
* @return the number of records, or -1 on failure. * @return the number of records, or -1 on failure.
*/ */
int64_t count() { int64_t count() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return recs_.size(); return recs_.size();
} }
/** /**
* Get the size of the database file. * Get the size of the database file.
* @return the size of the database file in bytes, or -1 on failure. * @return the size of the database file in bytes, or -1 on failure.
*/ */
int64_t size() { int64_t size() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return -1; return -1;
} }
return size_; return size_;
} }
/** /**
* Get the path of the database file. * Get the path of the database file.
* @return the path of the database file, or an empty string on failure. * @return the path of the database file, or an empty string on failure.
*/ */
std::string path() { std::string path() {
_assert_(true); _assert_(true);
ScopedSpinRWLock lock(&mlock_, false); ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return ""; return "";
} }
return path_; return path_;
} }
/** /**
* Get the miscellaneous status information. * Get the miscellaneous status information.
* @param strmap a string map to contain the result. * @param strmap a string map to contain the result.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
bool status(std::map<std::string, std::string>* strmap) { bool status(std::map<std::string, std::string>* strmap) {
_assert_(strmap); _assert_(strmap);
ScopedSpinRWLock lock(&mlock_, true); ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) { if (omode_ == 0) {
set_error(Error::INVALID, "not opened"); set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false; return false;
} }
(*strmap)["type"] = "ProtoDB"; (*strmap)["type"] = strprintf("%u", (unsigned)DBTYPE);
(*strmap)["realtype"] = strprintf("%u", (unsigned)map_type(&recs_)); (*strmap)["realtype"] = strprintf("%u", (unsigned)DBTYPE);
(*strmap)["path"] = path_; (*strmap)["path"] = path_;
if (strmap->count("opaque") > 0)
(*strmap)["opaque"] = std::string(opaque_, sizeof(opaque_));
(*strmap)["count"] = strprintf("%lld", (long long)recs_.size()); (*strmap)["count"] = strprintf("%lld", (long long)recs_.size());
(*strmap)["size"] = strprintf("%lld", (long long)size_); (*strmap)["size"] = strprintf("%lld", (long long)size_);
return true; return true;
} }
/** /**
* Create a cursor object. * Create a cursor object.
* @return the return value is the created cursor object. * @return the return value is the created cursor object.
* @note Because the object of the return value is allocated by the const ructor, it should be * @note Because the object of the return value is allocated by the const ructor, it should be
* released with the delete operator when it is no longer in use. * released with the delete operator when it is no longer in use.
*/ */
Cursor* cursor() { Cursor* cursor() {
_assert_(true); _assert_(true);
return new Cursor(this); return new Cursor(this);
} }
/**
* Set the internal logger.
* @param logger the logger object.
* @param kinds kinds of logged messages by bitwise-or: Logger::DEBUG for
debugging,
* Logger::INFO for normal information, Logger::WARN for warning, and Log
ger::ERROR for fatal
* error.
* @return true on success, or false on failure.
*/
bool tune_logger(Logger* logger, uint32_t kinds = Logger::WARN | Logger::
ERROR) {
_assert_(logger);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
logger_ = logger;
logkinds_ = kinds;
return true;
}
/**
* Set the internal meta operation trigger.
* @param trigger the trigger object.
* @return true on success, or false on failure.
*/
bool tune_meta_trigger(MetaTrigger* trigger) {
_assert_(trigger);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ != 0) {
set_error(_KCCODELINE_, Error::INVALID, "already opened");
return false;
}
mtrigger_ = trigger;
return true;
}
/**
* Get the opaque data.
* @return the pointer to the opaque data region, whose size is 16 bytes.
*/
char* opaque() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, false);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return NULL;
}
return opaque_;
}
/**
* Synchronize the opaque data.
* @return true on success, or false on failure.
*/
bool synchronize_opaque() {
_assert_(true);
ScopedSpinRWLock lock(&mlock_, true);
if (omode_ == 0) {
set_error(_KCCODELINE_, Error::INVALID, "not opened");
return false;
}
if (!(omode_ & OWRITER)) {
set_error(_KCCODELINE_, Error::NOPERM, "permission denied");
return false;
}
return true;
}
protected:
/**
* Report a message for debugging.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param format the printf-like format string.
* @param ... used according to the format string.
*/
void report(const char* file, int32_t line, const char* func, Logger::Kin
d kind,
const char* format, ...) {
_assert_(file && line > 0 && func && format);
if (!logger_ || !(kind & logkinds_)) return;
std::string message;
strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str());
va_list ap;
va_start(ap, format);
vstrprintf(&message, format, ap);
va_end(ap);
logger_->log(file, line, func, kind, message.c_str());
}
/**
* Report a message for debugging with variable number of arguments.
* @param file the file name of the program source code.
* @param line the line number of the program source code.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param format the printf-like format string.
* @param ap used according to the format string.
*/
void report_valist(const char* file, int32_t line, const char* func, Logg
er::Kind kind,
const char* format, va_list ap) {
_assert_(file && line > 0 && func && format);
if (!logger_ || !(kind & logkinds_)) return;
std::string message;
strprintf(&message, "%s: ", path_.empty() ? "-" : path_.c_str());
vstrprintf(&message, format, ap);
logger_->log(file, line, func, kind, message.c_str());
}
/**
* Report the content of a binary buffer for debugging.
* @param file the file name of the epicenter.
* @param line the line number of the epicenter.
* @param func the function name of the program source code.
* @param kind the kind of the event. Logger::DEBUG for debugging, Logge
r::INFO for normal
* information, Logger::WARN for warning, and Logger::ERROR for fatal err
or.
* @param name the name of the information.
* @param buf the binary buffer.
* @param size the size of the binary buffer
*/
void report_binary(const char* file, int32_t line, const char* func, Logg
er::Kind kind,
const char* name, const char* buf, size_t size) {
_assert_(file && line > 0 && func && name && buf && size <= MEMMAXSIZ);
if (!logger_) return;
char* hex = hexencode(buf, size);
report(file, line, func, kind, "%s=%s", name, hex);
delete[] hex;
}
/**
* Trigger a meta database operation.
* @param kind the kind of the event. MetaTrigger::OPEN for opening, Met
aTrigger::CLOSE for
* closing, MetaTrigger::CLEAR for clearing, MetaTrigger::ITERATE for ite
ration,
* MetaTrigger::SYNCHRONIZE for synchronization, MetaTrigger::BEGINTRAN f
or beginning
* transaction, MetaTrigger::COMMITTRAN for committing transaction, MetaT
rigger::ABORTTRAN
* for aborting transaction, and MetaTrigger::MISC for miscellaneous oper
ations.
* @param message the supplement message.
*/
void trigger_meta(MetaTrigger::Kind kind, const char* message) {
_assert_(message);
if (mtrigger_) mtrigger_->trigger(kind, message);
}
private: private:
/** /**
* Transaction log. * Transaction log.
*/ */
struct TranLog { struct TranLog {
bool full; ///< flag whether full bool full; ///< flag whether full
std::string key; ///< old key std::string key; ///< old key
std::string value; ///< old value std::string value; ///< old value
/** constructor for a full record */ /** constructor for a full record */
explicit TranLog(const std::string& pkey, const std::string& pvalue) : explicit TranLog(const std::string& pkey, const std::string& pvalue) :
full(true), key(pkey), value(pvalue) {} full(true), key(pkey), value(pvalue) {
_assert_(true);
}
/** constructor for an empty record */ /** constructor for an empty record */
explicit TranLog(const std::string& pkey) : full(false), key(pkey) {} explicit TranLog(const std::string& pkey) : full(false), key(pkey) {
_assert_(true);
}
}; };
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
ProtoDB(const ProtoDB&); ProtoDB(const ProtoDB&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
ProtoDB& operator =(const ProtoDB&); ProtoDB& operator =(const ProtoDB&);
/** The method lock. */ /** The method lock. */
SpinRWLock mlock_; SpinRWLock mlock_;
/** The last happened error. */ /** The last happened error. */
TSD<Error> error_; TSD<Error> error_;
/** The internal logger. */
Logger* logger_;
/** The kinds of logged messages. */
uint32_t logkinds_;
/** The internal meta operation trigger. */
MetaTrigger* mtrigger_;
/** The open mode. */ /** The open mode. */
uint32_t omode_; uint32_t omode_;
/** The map of records. */ /** The map of records. */
STRMAP recs_; STRMAP recs_;
/** The cursor objects. */ /** The cursor objects. */
CursorList curs_; CursorList curs_;
/** The path of the database file. */ /** The path of the database file. */
std::string path_; std::string path_;
/** The total size of records. */ /** The total size of records. */
int64_t size_; int64_t size_;
/** The opaque data. */
char opaque_[PDBOPAQUESIZ];
/** The flag whether in transaction. */ /** The flag whether in transaction. */
bool tran_; bool tran_;
/** The transaction logs. */ /** The transaction logs. */
TranLogList trlogs_; TranLogList trlogs_;
/** The old size before transaction. */ /** The old size before transaction. */
size_t trsize_; size_t trsize_;
}; };
/** An alias of the prototype hash database. */ /** An alias of the prototype hash database. */
typedef ProtoDB<StringHashMap> ProtoHashDB; typedef ProtoDB<StringHashMap, BasicDB::TYPEPHASH> ProtoHashDB;
/** An alias of the prototype tree database. */ /** An alias of the prototype tree database. */
typedef ProtoDB<StringTreeMap> ProtoTreeDB; typedef ProtoDB<StringTreeMap, BasicDB::TYPEPTREE> ProtoTreeDB;
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 84 change blocks. 
98 lines changed or deleted 426 lines changed or added


 kcthread.h   kcthread.h 
skipping to change at line 53 skipping to change at line 53
void start(); void start();
/** /**
* Wait for the thread to finish. * Wait for the thread to finish.
*/ */
void join(); void join();
/** /**
* Put the thread in the detached state. * Put the thread in the detached state.
*/ */
void detach(); void detach();
/** /**
* Yield the processor from the running thread.
*/
static void yield();
/**
* Terminate the running thread. * Terminate the running thread.
*/ */
static void exit(); static void exit();
/** /**
* Yield the processor from the current thread.
*/
static void yield();
/**
* Suspend execution of the current thread. * Suspend execution of the current thread.
* @param sec the interval of the suspension in seconds. * @param sec the interval of the suspension in seconds.
* @return true on success, or false on failure. * @return true on success, or false on failure.
*/ */
static bool sleep(double sec); static bool sleep(double sec);
/** /**
* Get the hash value of the current thread. * Get the hash value of the current thread.
* @return the hash value of the current thread. * @return the hash value of the current thread.
*/ */
static int64_t hash(); static int64_t hash();
skipping to change at line 686 skipping to change at line 686
void wait(Mutex* mutex); void wait(Mutex* mutex);
/** /**
* Wait for the signal. * Wait for the signal.
* @param mutex a locked mutex. * @param mutex a locked mutex.
* @param sec the interval of the suspension in seconds. * @param sec the interval of the suspension in seconds.
* @return true on catched signal, or false on timeout. * @return true on catched signal, or false on timeout.
*/ */
bool wait(Mutex* mutex, double sec); bool wait(Mutex* mutex, double sec);
/** /**
* Send the wake-up signal to another waiting thread. * Send the wake-up signal to another waiting thread.
* @note The mutex used for the wait method should be locked by the calle r.
*/ */
void signal(); void signal();
/** /**
* Send the wake-up signal to all waiting threads. * Send the wake-up signal to all waiting threads.
* @note The mutex used for the wait method should be locked by the calle r.
*/ */
void broadcast(); void broadcast();
private: private:
/** Dummy constructor to forbid the use. */ /** Dummy constructor to forbid the use. */
CondVar(const CondVar&); CondVar(const CondVar&);
/** Dummy Operator to forbid the use. */ /** Dummy Operator to forbid the use. */
CondVar& operator =(const CondVar&); CondVar& operator =(const CondVar&);
/** Opaque pointer. */ /** Opaque pointer. */
void* opq_; void* opq_;
}; };
skipping to change at line 935 skipping to change at line 937
} }
return val; return val;
} }
private: private:
/** The value. */ /** The value. */
volatile int64_t value_; volatile int64_t value_;
/** The alternative lock. */ /** The alternative lock. */
mutable SpinLock lock_; mutable SpinLock lock_;
}; };
/**
* Task queue device.
*/
class TaskQueue {
public:
class Task;
private:
class WorkerThread;
/** An alias of list of tasks. */
typedef std::list<Task*> TaskList;
public:
/**
* Interface of a task.
*/
class Task {
friend class TaskQueue;
public:
/**
* Default constructor.
*/
explicit Task() : id_(0), thid_(0), aborted_(false) {
_assert_(true);
}
/**
* Destructor.
*/
virtual ~Task() {
_assert_(true);
}
/**
* Get the ID number of the task.
* @return the ID number of the task, which is incremented from 1.
*/
uint64_t id() {
_assert_(true);
return id_;
}
/**
* Get the ID number of the worker thread.
* @return the ID number of the worker thread. It is from 0 to less th
an the number of
* worker threads.
*/
uint32_t thread_id() {
_assert_(true);
return thid_;
}
/**
* Check whether the thread is to be aborted.
* @return true if the thread is to be aborted, or false if not.
*/
bool aborted() {
_assert_(true);
return aborted_;
}
private:
/** The task ID number. */
uint64_t id_;
/** The thread ID number. */
uint64_t thid_;
/** The flag to be aborted. */
bool aborted_;
};
/**
* Default Constructor.
*/
TaskQueue() : thary_(NULL), thnum_(0), tasks_(), count_(0), mutex_(), con
d_(), seed_(0) {
_assert_(true);
}
/**
* Destructor.
*/
virtual ~TaskQueue() {
_assert_(true);
}
/**
* Process a task.
* @param task a task object.
*/
virtual void do_task(Task* task) = 0;
/**
* Start the task queue.
* @param thnum the number of worker threads.
*/
void start(size_t thnum) {
_assert_(thnum > 0 && thnum <= MEMMAXSIZ);
thary_ = new WorkerThread[thnum];
for (size_t i = 0; i < thnum; i++) {
thary_[i].id_ = i;
thary_[i].queue_ = this;
thary_[i].start();
}
thnum_ = thnum;
}
/**
* Finish the task queue.
* @note This function blocks until all tasks in the queue are popped.
*/
void finish() {
_assert_(true);
mutex_.lock();
TaskList::iterator it = tasks_.begin();
TaskList::iterator itend = tasks_.end();
while (it != itend) {
Task* task = *it;
task->aborted_ = true;
it++;
}
cond_.broadcast();
mutex_.unlock();
Thread::yield();
for (double wsec = 1.0 / CLOCKTICK; true; wsec *= 2) {
mutex_.lock();
if (tasks_.empty()) {
mutex_.unlock();
break;
}
mutex_.unlock();
if (wsec > 1.0) wsec = 1.0;
Thread::sleep(wsec);
}
mutex_.lock();
for (size_t i = 0; i < thnum_; i++) {
thary_[i].aborted_ = true;
}
cond_.broadcast();
mutex_.unlock();
for (size_t i = 0; i < thnum_; i++) {
thary_[i].join();
}
delete[] thary_;
}
/**
* Add a task.
* @param task a task object.
* @return the number of tasks in the queue.
*/
int64_t add_task(Task* task) {
_assert_(task);
mutex_.lock();
task->id_ = ++seed_;
tasks_.push_back(task);
int64_t count = ++count_;
cond_.signal();
mutex_.unlock();
return count;
}
/**
* Get the number of tasks in the queue.
* @return the number of tasks in the queue.
*/
int64_t count() {
_assert_(true);
mutex_.lock();
int64_t count = count_;
mutex_.unlock();
return count;
}
private:
/**
* Implementation of the worker thread.
*/
class WorkerThread : public Thread {
friend class TaskQueue;
public:
explicit WorkerThread() : id_(0), queue_(NULL), aborted_(false) {
_assert_(true);
}
private:
void run() {
_assert_(true);
bool empty = false;
while (true) {
queue_->mutex_.lock();
if (aborted_) {
queue_->mutex_.unlock();
break;
}
if (empty) queue_->cond_.wait(&queue_->mutex_, 1.0);
Task * task = NULL;
if (queue_->tasks_.empty()) {
empty = true;
} else {
task = queue_->tasks_.front();
task->thid_ = id_;
queue_->tasks_.pop_front();
queue_->count_--;
empty = false;
}
queue_->mutex_.unlock();
if (task) queue_->do_task(task);
}
}
uint32_t id_;
TaskQueue* queue_;
Task* task_;
bool aborted_;
};
/** Dummy constructor to forbid the use. */
TaskQueue(const TaskQueue&);
/** Dummy Operator to forbid the use. */
TaskQueue& operator =(const TaskQueue&);
/** The array of worker threads. */
WorkerThread* thary_;
/** The number of worker threads. */
size_t thnum_;
/** The list of tasks. */
TaskList tasks_;
/** The number of the tasks. */
int64_t count_;
/** The mutex for the task list. */
Mutex mutex_;
/** The condition variable for the task list. */
CondVar cond_;
/** The seed of ID numbers. */
uint64_t seed_;
};
} // common namespace } // common namespace
#endif // duplication check #endif // duplication check
// END OF FILE // END OF FILE
 End of changes. 5 change blocks. 
4 lines changed or deleted 225 lines changed or added


 kcutil.h   kcutil.h 
skipping to change at line 52 skipping to change at line 52
/** The flag for big endian environments. */ /** The flag for big endian environments. */
extern const bool BIGEND; extern const bool BIGEND;
/** The clock tick of interruption. */ /** The clock tick of interruption. */
extern const int32_t CLOCKTICK; extern const int32_t CLOCKTICK;
/** The size of a page. */ /** The size of a page. */
extern const int32_t PAGESIZE; extern const int32_t PAGESIZE;
/** The extra feature list. */
extern const char* const FEATURES;
/** The buffer size for numeric data. */ /** The buffer size for numeric data. */
const size_t NUMBUFSIZ = 32; const size_t NUMBUFSIZ = 32;
/** The maximum memory size for debugging. */ /** The maximum memory size for debugging. */
const size_t MEMMAXSIZ = INT32_MAX / 2; const size_t MEMMAXSIZ = INT32_MAX / 2;
/** /**
* Convert a string to an integer. * Convert a decimal string to an integer.
* @param str specifies the string. * @param str the decimal string.
* @return the integer. If the string does not contain numeric expression, 0 is returned. * @return the integer. If the string does not contain numeric expression, 0 is returned.
*/ */
int64_t atoi(const char* str); int64_t atoi(const char* str);
/** /**
* Convert a string with a metric prefix to an integer. * Convert a decimal string with a metric prefix to an integer.
* @param str the string, which can be trailed by a binary metric prefix. * @param str the decimal string, which can be trailed by a binary metric p
"K", "M", "G", "T", refix. "K", "M", "G",
* "P", and "E" are supported. They are case-insensitive. * "T", "P", and "E" are supported. They are case-insensitive.
* @return the integer. If the string does not contain numeric expression, 0 is returned. If * @return the integer. If the string does not contain numeric expression, 0 is returned. If
* the integer overflows the domain, INT64_MAX or INT64_MIN is returned acc ording to the * the integer overflows the domain, INT64_MAX or INT64_MIN is returned acc ording to the
* sign. * sign.
*/ */
int64_t atoix(const char* str); int64_t atoix(const char* str);
/** /**
* Convert a string to a real number. * Convert a hexadecimal string to an integer.
* @param str specifies the string. * @param str the hexadecimal string.
* @return the real number. If the string does not contain numeric express * @return the integer. If the string does not contain numeric expression,
ion, 0.0 is 0 is returned.
* returned. */
int64_t atoih(const char* str);
/**
* Convert a decimal byte array to an integer.
* @param ptr the decimal byte array.
* @param size the size of the decimal byte array.
* @return the integer. If the string does not contain numeric expression,
0 is returned.
*/
int64_t atoin(const char* ptr, size_t size);
/**
* Convert a decimal string to a real number.
* @param str the decimal string.
* @return the real number. If the string does not contain numeric express
ion, 0.0 is returned.
*/ */
double atof(const char* str); double atof(const char* str);
/** /**
* Convert a decimal byte array to a real number.
* @param ptr the decimal byte array.
* @param size the size of the decimal byte array.
* @return the real number. If the string does not contain numeric express
ion, 0.0 is returned.
*/
double atofn(const char* ptr, size_t size);
/**
* Normalize a 16-bit number in the native order into the network byte orde r. * Normalize a 16-bit number in the native order into the network byte orde r.
* @param num the 16-bit number in the native order. * @param num the 16-bit number in the native order.
* @return the number in the network byte order. * @return the number in the network byte order.
*/ */
uint16_t hton16(uint16_t num); uint16_t hton16(uint16_t num);
/** /**
* Normalize a 32-bit number in the native order into the network byte orde r. * Normalize a 32-bit number in the native order into the network byte orde r.
* @param num the 32-bit number in the native order. * @param num the 32-bit number in the native order.
* @return the number in the network byte order. * @return the number in the network byte order.
skipping to change at line 159 skipping to change at line 184
/** /**
* Read a number in variable length format from a buffer. * Read a number in variable length format from a buffer.
* @param buf the source buffer. * @param buf the source buffer.
* @param size the size of the source buffer. * @param size the size of the source buffer.
* @param np the pointer to the variable into which the read number is assi gned. * @param np the pointer to the variable into which the read number is assi gned.
* @return the length of the read region, or 0 on failure. * @return the length of the read region, or 0 on failure.
*/ */
size_t readvarnum(const void* buf, size_t size, uint64_t* np); size_t readvarnum(const void* buf, size_t size, uint64_t* np);
/** /**
* Check the size of variable length format of a number.
* @return the size of variable length format.
*/
size_t sizevarnum(uint64_t num);
/**
* Get the hash value by MurMur hashing. * Get the hash value by MurMur hashing.
* @param buf the source buffer. * @param buf the source buffer.
* @param size the size of the source buffer. * @param size the size of the source buffer.
* @return the hash value. * @return the hash value.
*/ */
uint64_t hashmurmur(const void* buf, size_t size); uint64_t hashmurmur(const void* buf, size_t size);
/** /**
* Get the hash value by FNV hashing. * Get the hash value by FNV hashing.
* @param buf the source buffer. * @param buf the source buffer.
skipping to change at line 220 skipping to change at line 251
* Check a number is an infinity value. * Check a number is an infinity value.
* @return true for the number is an infinity value, or false if not. * @return true for the number is an infinity value, or false if not.
*/ */
bool chkinf(double num); bool chkinf(double num);
/** /**
* Append a formatted string at the end of a string. * Append a formatted string at the end of a string.
* @param dest the destination string. * @param dest the destination string.
* @param format the printf-like format string. The conversion character ` %' can be used with * @param format the printf-like format string. The conversion character ` %' can be used with
* such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f' , `g', `G', and `%'. * such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f' , `g', `G', and `%'.
* `S' treats the pointer to a std::string object.
* @param ap used according to the format string. * @param ap used according to the format string.
*/ */
void strprintf(std::string* dest, const char* format, va_list ap); void vstrprintf(std::string* dest, const char* format, va_list ap);
/** /**
* Append a formatted string at the end of a string. * Append a formatted string at the end of a string.
* @param dest the destination string. * @param dest the destination string.
* @param format the printf-like format string. The conversion character ` %' can be used with * @param format the printf-like format string. The conversion character ` %' can be used with
* such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f' , `g', `G', and `%'. * such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f' , `g', `G', and `%'.
* `S' treats the pointer to a std::string object.
* @param ... used according to the format string. * @param ... used according to the format string.
*/ */
void strprintf(std::string* dest, const char* format, ...); void strprintf(std::string* dest, const char* format, ...);
/** /**
* Generate a formatted string. * Generate a formatted string.
* @param format the printf-like format string. The conversion character ` %' can be used with * @param format the printf-like format string. The conversion character ` %' can be used with
* such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f' , `g', `G', and `%'. * such flag characters as `s', `d', `o', `u', `x', `X', `c', `e', `E', `f' , `g', `G', and `%'.
* `S' treats the pointer to a std::string object.
* @param ... used according to the format string. * @param ... used according to the format string.
* @return the result string. * @return the result string.
*/ */
std::string strprintf(const char* format, ...); std::string strprintf(const char* format, ...);
/** /**
* Split a string with a delimiter * Split a string with a delimiter.
* @param str the string. * @param str the string.
* @param delim the delimiter. * @param delim the delimiter.
* @param elems a vector object into which the result elements are pushed. * @param elems a vector object into which the result elements are pushed.
* @return the number of result elements. * @return the number of result elements.
*/ */
size_t strsplit(const std::string& str, char delim, std::vector<std::string >* elems); size_t strsplit(const std::string& str, char delim, std::vector<std::string >* elems);
/** /**
* Encode a serial object with hexadecimal encoding. * Split a string with delimiters.
* @param str the string.
* @param delims the delimiters.
* @param elems a vector object into which the result elements are pushed.
* @return the number of result elements.
*/
size_t strsplit(const std::string& str, const std::string& delims,
std::vector<std::string>* elems);
/**
* Encode a serial object by hexadecimal encoding.
* @param buf the pointer to the region. * @param buf the pointer to the region.
* @param size the size of the region. * @param size the size of the region.
* @return the result string. * @return the result string.
* @note Because the region of the return value is allocated with the the n ew[] operator, it * @note Because the region of the return value is allocated with the the n ew[] operator, it
* should be released with the delete[] operator when it is no longer in us e. * should be released with the delete[] operator when it is no longer in us e.
*/ */
char* hexencode(const void* buf, size_t size); char* hexencode(const void* buf, size_t size);
/** /**
* Decode a string encoded with hexadecimal encoding. * Decode a string encoded by hexadecimal encoding.
* @param str specifies the encoded string. * @param str specifies the encoded string.
* @param sp the pointer to the variable into which the size of the region of the return value * @param sp the pointer to the variable into which the size of the region of the return value
* is assigned. * is assigned.
* @return the pointer to the region of the result. * @return the pointer to the region of the result.
* @note Because an additional zero code is appended at the end of the regi on of the return * @note Because an additional zero code is appended at the end of the regi on of the return
* value, the return value can be treated as a character string. Because t he region of the * value, the return value can be treated as a character string. Because t he region of the
* return value is allocated with the the new[] operator, it should be rele ased with the delete[] * return value is allocated with the the new[] operator, it should be rele ased with the delete[]
* operator when it is no longer in use. * operator when it is no longer in use.
*/ */
char* hexdecode(const char* str, size_t* sp); char* hexdecode(const char* str, size_t* sp);
/** /**
* Encode a serial object by URL encoding.
* @param buf the pointer to the region.
* @param size the size of the region.
* @return the result string.
* @note Because the region of the return value is allocated with the the n
ew[] operator, it
* should be released with the delete[] operator when it is no longer in us
e.
*/
char* urlencode(const void* buf, size_t size);
/**
* Decode a string encoded by URL encoding.
* @param str specifies the encoded string.
* @param sp the pointer to the variable into which the size of the region
of the return value
* is assigned.
* @return the pointer to the region of the result.
* @note Because an additional zero code is appended at the end of the regi
on of the return
* value, the return value can be treated as a character string. Because t
he region of the
* return value is allocated with the the new[] operator, it should be rele
ased with the delete[]
* operator when it is no longer in use.
*/
char* urldecode(const char* str, size_t* sp);
/**
* Encode a serial object by Quoted-printable encoding.
* @param buf the pointer to the region.
* @param size the size of the region.
* @return the result string.
* @note Because the region of the return value is allocated with the the n
ew[] operator, it
* should be released with the delete[] operator when it is no longer in us
e.
*/
char* quoteencode(const void* buf, size_t size);
/**
* Decode a string encoded by Quoted-printable encoding.
* @param str specifies the encoded string.
* @param sp the pointer to the variable into which the size of the region
of the return value
* is assigned.
* @return the pointer to the region of the result.
* @note Because an additional zero code is appended at the end of the regi
on of the return
* value, the return value can be treated as a character string. Because t
he region of the
* return value is allocated with the the new[] operator, it should be rele
ased with the delete[]
* operator when it is no longer in use.
*/
char* quotedecode(const char* str, size_t* sp);
/**
* Encode a serial object by Base64 encoding.
* @param buf the pointer to the region.
* @param size the size of the region.
* @return the result string.
* @note Because the region of the return value is allocated with the the n
ew[] operator, it
* should be released with the delete[] operator when it is no longer in us
e.
*/
char* baseencode(const void* buf, size_t size);
/**
* Decode a string encoded by Base64 encoding.
* @param str specifies the encoded string.
* @param sp the pointer to the variable into which the size of the region
of the return value
* is assigned.
* @return the pointer to the region of the result.
* @note Because an additional zero code is appended at the end of the regi
on of the return
* value, the return value can be treated as a character string. Because t
he region of the
* return value is allocated with the the new[] operator, it should be rele
ased with the delete[]
* operator when it is no longer in use.
*/
char* basedecode(const char* str, size_t* sp);
/**
* Cipher or decipher a serial object with the Arcfour stream cipher. * Cipher or decipher a serial object with the Arcfour stream cipher.
* @param ptr the pointer to the region. * @param ptr the pointer to the region.
* @param size the size of the region. * @param size the size of the region.
* @param kbuf the pointer to the region of the cipher key. * @param kbuf the pointer to the region of the cipher key.
* @param ksiz the size of the region of the cipher key. * @param ksiz the size of the region of the cipher key.
* @param obuf the pointer to the region into which the result data is writ ten. The size of the * @param obuf the pointer to the region into which the result data is writ ten. The size of the
* buffer should be equal to or more than the input region. The region can be the same as the * buffer should be equal to or more than the input region. The region can be the same as the
* source region. * source region.
*/ */
void arccipher(const void* ptr, size_t size, const void* kbuf, size_t ksiz, void* obuf); void arccipher(const void* ptr, size_t size, const void* kbuf, size_t ksiz, void* obuf);
/** /**
* Duplicate a region on memory.
* @param ptr the source buffer.
* @param size the size of the source buffer.
* @note Because the region of the return value is allocated with the the n
ew[] operator, it
* should be released with the delete[] operator when it is no longer in us
e.
*/
char* memdup(const char* ptr, size_t size);
/**
* Duplicate a string on memory.
* @param str the source string.
* @note Because the region of the return value is allocated with the the n
ew[] operator, it
* should be released with the delete[] operator when it is no longer in us
e.
*/
char* strdup(const char* str);
/**
* Convert the letters of a string into upper case.
* @param str the string to convert.
* @return the string itself.
*/
char* strtoupper(char* str);
/**
* Convert the letters of a string into lower case.
* @param str the string to convert.
* @return the string itself.
*/
char* strtolower(char* str);
/**
* Cut space characters at head or tail of a string.
* @param str the string to convert.
* @return the string itself.
*/
char* strtrim(char* str);
/**
* Squeeze space characters in a string and trim it.
* @param str the string to convert.
* @return the string itself.
*/
char* strsqzspc(char* str);
/**
* Normalize space characters in a string and trim it.
* @param str the string to convert.
* @return the string itself.
*/
char* strnrmspc(char* str);
/**
* Compare two strings with case insensitive evaluation.
* @param astr a string.
* @param bstr the other string.
* @return positive if the former is big, negative if the latter is big, 0
if both are
* equivalent.
*/
int32_t stricmp(const char* astr, const char* bstr);
/**
* Check whether a string begins with a key.
* @param str the string.
* @param key the forward matching key string.
* @return true if the target string begins with the key, else, it is false
.
*/
bool strfwm(const char* str, const char* key);
/**
* Check whether a string begins with a key by case insensitive evaluation.
* @param str the string.
* @param key the forward matching key string.
* @return true if the target string begins with the key, else, it is false
.
*/
bool strifwm(const char* str, const char* key);
/**
* Check whether a string ends with a key.
* @param str the string.
* @param key the backward matching key string.
* @return true if the target string ends with the key, else, it is false.
*/
bool strbwm(const char* str, const char* key);
/**
* Check whether a string ends with a key by case insensitive evaluation.
* @param str the string.
* @param key the backward matching key string.
* @return true if the target string ends with the key, else, it is false.
*/
bool stribwm(const char* str, const char* key);
/**
* Allocate a region on memory. * Allocate a region on memory.
* @param size the size of the region. * @param size the size of the region.
* @return the pointer to the allocated region. * @return the pointer to the allocated region.
*/ */
void* xmalloc(size_t size); void* xmalloc(size_t size);
/** /**
* Allocate a nullified region on memory. * Allocate a nullified region on memory.
* @param nmemb the number of elements. * @param nmemb the number of elements.
* @param size the size of each element. * @param size the size of each element.
skipping to change at line 350 skipping to change at line 550
*/ */
int64_t getpid(); int64_t getpid();
/** /**
* Get the value of an environment variable. * Get the value of an environment variable.
* @return the value of the environment variable, or NULL on failure. * @return the value of the environment variable, or NULL on failure.
*/ */
const char* getenv(const char* name); const char* getenv(const char* name);
/** /**
* Convert a string to an integer. * Get system information of the environment.
* @param strmap a string map to contain the result.
*/
void getsysinfo(std::map<std::string, std::string>* strmap);
/**
* Set the standard streams into the binary mode.
*/
void setstdiobin();
/**
* Convert a decimal string to an integer.
*/ */
inline int64_t atoi(const char* str) { inline int64_t atoi(const char* str) {
_assert_(str); _assert_(str);
while (*str > '\0' && *str <= ' ') { while (*str > '\0' && *str <= ' ') {
str++; str++;
} }
int32_t sign = 1; int32_t sign = 1;
int64_t num = 0; int64_t num = 0;
if (*str == '-') { if (*str == '-') {
str++; str++;
skipping to change at line 374 skipping to change at line 585
} }
while (*str != '\0') { while (*str != '\0') {
if (*str < '0' || *str > '9') break; if (*str < '0' || *str > '9') break;
num = num * 10 + *str - '0'; num = num * 10 + *str - '0';
str++; str++;
} }
return num * sign; return num * sign;
} }
/** /**
* Convert a string with a metric prefix to an integer. * Convert a decimal string with a metric prefix to an integer.
*/ */
inline int64_t atoix(const char* str) { inline int64_t atoix(const char* str) {
_assert_(str); _assert_(str);
while (*str > '\0' && *str <= ' ') { while (*str > '\0' && *str <= ' ') {
str++; str++;
} }
int32_t sign = 1; int32_t sign = 1;
if (*str == '-') { if (*str == '-') {
str++; str++;
sign = -1; sign = -1;
skipping to change at line 427 skipping to change at line 638
num *= 1LL << 50; num *= 1LL << 50;
} else if (*str == 'e' || *str == 'E') { } else if (*str == 'e' || *str == 'E') {
num *= 1LL << 60; num *= 1LL << 60;
} }
if (num > INT64_MAX) return INT64_MAX; if (num > INT64_MAX) return INT64_MAX;
if (num < INT64_MIN) return INT64_MIN; if (num < INT64_MIN) return INT64_MIN;
return (int64_t)num; return (int64_t)num;
} }
/** /**
* Convert a string to a real number. * Convert a hexadecimal string to an integer.
*/
inline int64_t atoih(const char* str) {
_assert_(str);
while (*str > '\0' && *str <= ' ') {
str++;
}
if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) {
str += 2;
}
int64_t num = 0;
while (true) {
if (*str >= '0' && *str <= '9') {
num = num * 0x10 + *str - '0';
} else if (*str >= 'a' && *str <= 'f') {
num = num * 0x10 + *str - 'a' + 10;
} else if (*str >= 'A' && *str <= 'F') {
num = num * 0x10 + *str - 'A' + 10;
} else {
break;
}
str++;
}
return num;
}
/**
* Convert a decimal byte array to an integer.
*/
inline int64_t atoin(const char* ptr, size_t size) {
_assert_(ptr && size <= MEMMAXSIZ);
while (size > 0 && *ptr >= '\0' && *ptr <= ' ') {
ptr++;
size--;
}
int32_t sign = 1;
int64_t num = 0;
if (size > 0) {
if (*ptr == '-') {
ptr++;
size--;
sign = -1;
} else if (*ptr == '+') {
ptr++;
size--;
}
}
while (size > 0) {
if (*ptr < '0' || *ptr > '9') break;
num = num * 10 + *ptr - '0';
ptr++;
size--;
}
return num * sign;
}
/**
* Convert a decimal string to a real number.
*/ */
inline double atof(const char* str) { inline double atof(const char* str) {
_assert_(str); _assert_(str);
while (*str > '\0' && *str <= ' ') { while (*str > '\0' && *str <= ' ') {
str++; str++;
} }
int32_t sign = 1; int32_t sign = 1;
if (*str == '-') { if (*str == '-') {
str++; str++;
sign = -1; sign = -1;
skipping to change at line 474 skipping to change at line 742
num += fract; num += fract;
} }
if (*str == 'e' || *str == 'E') { if (*str == 'e' || *str == 'E') {
str++; str++;
num *= std::pow((long double)10, (long double)atoi(str)); num *= std::pow((long double)10, (long double)atoi(str));
} }
return num * sign; return num * sign;
} }
/** /**
* Convert a decimal byte array to a real number.
*/
inline double atofn(const char* ptr, size_t size) {
_assert_(ptr && size <= MEMMAXSIZ);
while (size > 0 && *ptr >= '\0' && *ptr <= ' ') {
ptr++;
size--;
}
int32_t sign = 1;
if (size > 0) {
if (*ptr == '-') {
ptr++;
size--;
sign = -1;
} else if (*ptr == '+') {
ptr++;
size--;
}
}
if (size > 2) {
if ((ptr[0] == 'i' || ptr[0] == 'I') && (ptr[1] == 'n' || ptr[1] == 'N'
) &&
(ptr[2] == 'f' || ptr[2] == 'F')) return HUGE_VAL * sign;
if ((ptr[0] == 'n' || ptr[0] == 'N') && (ptr[1] == 'a' || ptr[1] == 'A'
) &&
(ptr[2] == 'n' || ptr[2] == 'N')) return nan();
}
long double num = 0;
int32_t col = 0;
while (size > 0) {
if (*ptr < '0' || *ptr > '9') break;
num = num * 10 + *ptr - '0';
ptr++;
size--;
if (num > 0) col++;
}
if (size > 0 && *ptr == '.') {
ptr++;
size--;
long double fract = 0.0;
long double base = 10;
while (col < 16 && size > 0) {
if (*ptr < '0' || *ptr > '9') break;
fract += (*ptr - '0') / base;
ptr++;
size--;
col++;
base *= 10;
}
num += fract;
}
if (size > 0 && (*ptr == 'e' || *ptr == 'E')) {
ptr++;
size--;
num *= std::pow((long double)10, (long double)atoin(ptr, size));
}
return num * sign;
}
/**
* Normalize a 16-bit number in the native order into the network byte orde r. * Normalize a 16-bit number in the native order into the network byte orde r.
*/ */
inline uint16_t hton16(uint16_t num) { inline uint16_t hton16(uint16_t num) {
_assert_(true); _assert_(true);
if (BIGEND) return num; if (BIGEND) return num;
return ((num & 0x00ffU) << 8) | ((num & 0xff00U) >> 8); return ((num & 0x00ffU) << 8) | ((num & 0xff00U) >> 8);
} }
/** /**
* Normalize a 32-bit number in the native order into the network byte orde r. * Normalize a 32-bit number in the native order into the network byte orde r.
skipping to change at line 645 skipping to change at line 971
} }
c = *rp; c = *rp;
num = (num << 7) + (c & 0x7f); num = (num << 7) + (c & 0x7f);
rp++; rp++;
} while (c >= 0x80); } while (c >= 0x80);
*np = num; *np = num;
return rp - (const unsigned char*)buf; return rp - (const unsigned char*)buf;
} }
/** /**
* Check the size of variable length format of a number.
*/
inline size_t sizevarnum(uint64_t num) {
_assert_(true);
if (num < (1ULL << 7)) return 1;
if (num < (1ULL << 14)) return 2;
if (num < (1ULL << 21)) return 3;
if (num < (1ULL << 28)) return 4;
if (num < (1ULL << 35)) return 5;
if (num < (1ULL << 42)) return 6;
if (num < (1ULL << 49)) return 7;
if (num < (1ULL << 56)) return 8;
if (num < (1ULL << 63)) return 9;
return 10;
}
/**
* Get the hash value by MurMur hashing. * Get the hash value by MurMur hashing.
*/ */
inline uint64_t hashmurmur(const void* buf, size_t size) { inline uint64_t hashmurmur(const void* buf, size_t size) {
_assert_(buf && size <= MEMMAXSIZ); _assert_(buf && size <= MEMMAXSIZ);
const uint64_t mul = 0xc6a4a7935bd1e995ULL; const uint64_t mul = 0xc6a4a7935bd1e995ULL;
const int32_t rtt = 47; const int32_t rtt = 47;
uint64_t hash = 19780211ULL ^ (size * mul); uint64_t hash = 19780211ULL ^ (size * mul);
const unsigned char* rp = (const unsigned char*)buf; const unsigned char* rp = (const unsigned char*)buf;
while (size >= sizeof(uint64_t)) { while (size >= sizeof(uint64_t)) {
uint64_t num = ((uint64_t)rp[0] << 0) | ((uint64_t)rp[1] << 8) | uint64_t num = ((uint64_t)rp[0] << 0) | ((uint64_t)rp[1] << 8) |
skipping to change at line 730 skipping to change at line 1073
rp++; rp++;
} }
} else { } else {
*(wp++) = '0'; *(wp++) = '0';
} }
uint64_t hash = hashmurmur(buf, size); uint64_t hash = hashmurmur(buf, size);
rv = (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff00000 000ULL) >> 16)) ^ rv = (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff00000 000ULL) >> 16)) ^
(((hash & 0x000000000000ffffULL) << 16) | ((hash & 0x00000000ffff0000 ULL) >> 16)); (((hash & 0x000000000000ffffULL) << 16) | ((hash & 0x00000000ffff0000 ULL) >> 16));
} else { } else {
*(wp++) = 'f' + 1 + (size & 0x0f); *(wp++) = 'f' + 1 + (size & 0x0f);
int32_t num = (rp[0] >> 4) + (rp[1] & 0x01 ? 0x10 : 0); for (int32_t i = 0; i <= 6; i += 3) {
if (num < 10) { uint32_t num = (rp[i] ^ rp[i+1] ^ rp[i+2] ^
*(wp++) = '0' + num; rp[size-i-1] ^ rp[size-i-2] ^ rp[size-i-3]) % 36;
} else { if (num < 10) {
*(wp++) = 'a' + num - 10; *(wp++) = '0' + num;
} } else {
num = (rp[0] & 0x0f) + (rp[1] & 0x02 ? 0x10 : 0); *(wp++) = 'a' + num - 10;
if (num < 10) { }
*(wp++) = '0' + num;
} else {
*(wp++) = 'a' + num - 10;
}
num = (rp[1] >> 2) & 0x1f;
if (num < 10) {
*(wp++) = '0' + num;
} else {
*(wp++) = 'a' + num - 10;
} }
uint64_t hash = hashmurmur(buf, size); uint64_t hash = hashmurmur(buf, size);
rv = (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff00000 000ULL) >> 16)) ^ rv = (((hash & 0xffff000000000000ULL) >> 48) | ((hash & 0x0000ffff00000 000ULL) >> 16)) ^
(((hash & 0x000000000000ffffULL) << 16) | ((hash & 0x00000000ffff0000 ULL) >> 16)); (((hash & 0x000000000000ffffULL) << 16) | ((hash & 0x00000000ffff0000 ULL) >> 16));
uint64_t inc = hashfnv(buf, size); uint64_t inc = hashfnv(buf, size);
inc = (((inc & 0xffff000000000000ULL) >> 48) | ((inc & 0x0000ffff000000 00ULL) >> 16)) ^ inc = (((inc & 0xffff000000000000ULL) >> 48) | ((inc & 0x0000ffff000000 00ULL) >> 16)) ^
(((inc & 0x000000000000ffffULL) << 16) | ((inc & 0x00000000ffff0000UL L) >> 16)); (((inc & 0x000000000000ffffULL) << 16) | ((inc & 0x00000000ffff0000UL L) >> 16));
for (size_t i = 0; i < sizeof(hash); i++) { for (size_t i = 0; i < sizeof(hash); i++) {
uint32_t least = hash >> ((sizeof(hash) - 1) * 8); uint32_t least = hash >> ((sizeof(hash) - 1) * 8);
num = least >> 4; uint64_t num = least >> 4;
if (inc & 0x01) num += 0x10; if (inc & 0x01) num += 0x10;
inc = inc >> 1; inc = inc >> 1;
if (num < 10) { if (num < 10) {
*(wp++) = '0' + num; *(wp++) = '0' + num;
} else { } else {
*(wp++) = 'a' + num - 10; *(wp++) = 'a' + num - 10;
} }
num = least & 0x0f; num = least & 0x0f;
if (inc & 0x01) num += 0x10; if (inc & 0x01) num += 0x10;
inc = inc >> 1; inc = inc >> 1;
skipping to change at line 860 skipping to change at line 1194
* Check a number is an infinity value. * Check a number is an infinity value.
*/ */
inline bool chkinf(double num) { inline bool chkinf(double num) {
_assert_(true); _assert_(true);
return num == inf() || num == -inf(); return num == inf() || num == -inf();
} }
/** /**
* Append a formatted string at the end of a string. * Append a formatted string at the end of a string.
*/ */
inline void strprintf(std::string* dest, const char* format, va_list ap) { inline void vstrprintf(std::string* dest, const char* format, va_list ap) {
_assert_(dest && format); _assert_(dest && format);
while (*format != '\0') { while (*format != '\0') {
if (*format == '%') { if (*format == '%') {
char cbuf[NUMBUFSIZ]; char cbuf[NUMBUFSIZ];
cbuf[0] = '%'; cbuf[0] = '%';
size_t cbsiz = 1; size_t cbsiz = 1;
int32_t lnum = 0; int32_t lnum = 0;
format++; format++;
while (std::strchr("0123456789 .+-hlLz", *format) && *format != '\0' && while (std::strchr("0123456789 .+-hlLz", *format) && *format != '\0' &&
cbsiz < NUMBUFSIZ - 1) { cbsiz < NUMBUFSIZ - 1) {
skipping to change at line 886 skipping to change at line 1220
switch (*format) { switch (*format) {
case 's': { case 's': {
const char* tmp = va_arg(ap, const char*); const char* tmp = va_arg(ap, const char*);
if (tmp) { if (tmp) {
dest->append(tmp); dest->append(tmp);
} else { } else {
dest->append("(null)"); dest->append("(null)");
} }
break; break;
} }
case 'S': {
const std::string* tmp = va_arg(ap, const std::string*);
if (tmp) {
dest->append(*tmp);
} else {
dest->append("(null)");
}
break;
}
case 'd': { case 'd': {
char tbuf[NUMBUFSIZ*4]; char tbuf[NUMBUFSIZ*4];
size_t tsiz; size_t tsiz;
if (lnum >= 2) { if (lnum >= 2) {
tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, long long)); tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, long long));
} else if (lnum >= 1) { } else if (lnum >= 1) {
tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, long)); tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, long));
} else { } else {
tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, int)); tsiz = std::sprintf(tbuf, cbuf, va_arg(ap, int));
} }
skipping to change at line 929 skipping to change at line 1254
break; break;
} }
case 'e': case 'E': case 'f': case 'g': case 'G': { case 'e': case 'E': case 'f': case 'g': case 'G': {
char tbuf[NUMBUFSIZ*4]; char tbuf[NUMBUFSIZ*4];
size_t tsiz; size_t tsiz;
if (lnum >= 1) { if (lnum >= 1) {
tsiz = std::snprintf(tbuf, sizeof(tbuf), cbuf, va_arg(ap, long double)); tsiz = std::snprintf(tbuf, sizeof(tbuf), cbuf, va_arg(ap, long double));
} else { } else {
tsiz = std::snprintf(tbuf, sizeof(tbuf), cbuf, va_arg(ap, doubl e)); tsiz = std::snprintf(tbuf, sizeof(tbuf), cbuf, va_arg(ap, doubl e));
} }
if (tsiz < 0 || tsiz &gt; sizeof(tbuf)) { if (tsiz > sizeof(tbuf)) {
tbuf[sizeof(tbuf)-1] = '*'; tbuf[sizeof(tbuf)-1] = '*';
tsiz = sizeof(tbuf); tsiz = sizeof(tbuf);
} }
dest->append(tbuf, tsiz); dest->append(tbuf, tsiz);
break; break;
} }
case 'p': { case 'p': {
char tbuf[NUMBUFSIZ*4]; char tbuf[NUMBUFSIZ*4];
size_t tsiz = std::sprintf(tbuf, "%p", va_arg(ap, void*)); size_t tsiz = std::sprintf(tbuf, "%p", va_arg(ap, void*));
dest->append(tbuf, tsiz); dest->append(tbuf, tsiz);
skipping to change at line 961 skipping to change at line 1286
} }
} }
/** /**
* Append a formatted string at the end of a string. * Append a formatted string at the end of a string.
*/ */
inline void strprintf(std::string* dest, const char* format, ...) { inline void strprintf(std::string* dest, const char* format, ...) {
_assert_(dest && format); _assert_(dest && format);
va_list ap; va_list ap;
va_start(ap, format); va_start(ap, format);
strprintf(dest, format, ap); vstrprintf(dest, format, ap);
va_end(ap); va_end(ap);
} }
/** /**
* Generate a formatted string on memory. * Generate a formatted string on memory.
*/ */
inline std::string strprintf(const char* format, ...) { inline std::string strprintf(const char* format, ...) {
_assert_(format); _assert_(format);
std::string str; std::string str;
va_list ap; va_list ap;
va_start(ap, format); va_start(ap, format);
strprintf(&str, format, ap); vstrprintf(&str, format, ap);
va_end(ap); va_end(ap);
return str; return str;
} }
/** /**
* Split a string with a delimiter * Split a string with a delimiter
*/ */
inline size_t strsplit(const std::string& str, char delim, std::vector<std: :string>* elems) { inline size_t strsplit(const std::string& str, char delim, std::vector<std: :string>* elems) {
_assert_(elems); _assert_(elems);
elems->clear(); elems->clear();
std::string::const_iterator it = str.begin(); std::string::const_iterator it = str.begin();
std::string field; std::string::const_iterator pv = it;
while (it != str.end()) { while (it != str.end()) {
if (*it == delim) { if (*it == delim) {
elems->push_back(field); std::string col(pv, it);
field.clear(); elems->push_back(col);
} else { pv = it + 1;
field.append(1, *it);
} }
it++; it++;
} }
elems->push_back(field); std::string col(pv, it);
elems->push_back(col);
return elems->size(); return elems->size();
} }
/** /**
* Encode a serial object with hexadecimal encoding. * Split a string with delimiters.
*/
inline size_t strsplit(const std::string& str, const std::string& delims,
std::vector<std::string>* elems) {
_assert_(elems);
elems->clear();
std::string::const_iterator it = str.begin();
std::string::const_iterator pv = it;
while (it != str.end()) {
while (delims.find(*it, 0) != std::string::npos) {
std::string col(pv, it);
elems->push_back(col);
pv = it + 1;
break;
}
it++;
}
std::string col(pv, it);
elems->push_back(col);
return elems->size();
}
/**
* Encode a serial object by hexadecimal encoding.
*/ */
inline char* hexencode(const void* buf, size_t size) { inline char* hexencode(const void* buf, size_t size) {
_assert_(buf && size <= MEMMAXSIZ); _assert_(buf && size <= MEMMAXSIZ);
const unsigned char* rp = (const unsigned char*)buf; const unsigned char* rp = (const unsigned char*)buf;
char* zbuf = new char[size*2+1]; char* zbuf = new char[size*2+1];
char* wp = zbuf; char* wp = zbuf;
for (const unsigned char* ep = rp + size; rp < ep; rp++) { for (const unsigned char* ep = rp + size; rp < ep; rp++) {
int32_t num = (*rp >> 4); int32_t num = *rp >> 4;
if (num < 10) { if (num < 10) {
*(wp++) = '0' + num; *(wp++) = '0' + num;
} else { } else {
*(wp++) = 'a' + num - 10; *(wp++) = 'a' + num - 10;
} }
num = (*rp & 0x0f); num = *rp & 0x0f;
if (num < 10) { if (num < 10) {
*(wp++) = '0' + num; *(wp++) = '0' + num;
} else { } else {
*(wp++) = 'a' + num - 10; *(wp++) = 'a' + num - 10;
} }
} }
*wp = '\0'; *wp = '\0';
return zbuf; return zbuf;
} }
/** /**
* Decode a string encoded with hexadecimal encoding. * Decode a string encoded by hexadecimal encoding.
*/ */
inline char* hexdecode(const char* str, size_t* sp) { inline char* hexdecode(const char* str, size_t* sp) {
_assert_(str && sp); _assert_(str && sp);
size_t zsiz = std::strlen(str); size_t zsiz = std::strlen(str);
char* zbuf = new char[zsiz+1]; char* zbuf = new char[zsiz+1];
char* wp = zbuf; char* wp = zbuf;
for (size_t i = 0; i < zsiz; i += 2) { for (size_t i = 0; i < zsiz; i += 2) {
while (str[i] >= '\0' && str[i] <= ' ') { while (str[i] >= '\0' && str[i] <= ' ') {
i++; i++;
} }
skipping to change at line 1067 skipping to change at line 1415
break; break;
} }
*(wp++) = num; *(wp++) = num;
} }
*wp = '\0'; *wp = '\0';
*sp = wp - zbuf; *sp = wp - zbuf;
return zbuf; return zbuf;
} }
/** /**
* Encode a serial object by URL encoding.
*/
inline char* urlencode(const void* buf, size_t size) {
_assert_(buf && size <= MEMMAXSIZ);
const unsigned char* rp = (const unsigned char*)buf;
char* zbuf = new char[size*3+1];
char* wp = zbuf;
for (const unsigned char* ep = rp + size; rp < ep; rp++) {
int32_t c = *rp;
if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
(c >= '0' && c <= '9') || (c != '\0' && std::strchr("_-.!~*'()", c)
)) {
*(wp++) = c;
} else {
*(wp++) = '%';
int32_t num = c >> 4;
if (num < 10) {
*(wp++) = '0' + num;
} else {
*(wp++) = 'a' + num - 10;
}
num = c & 0x0f;
if (num < 10) {
*(wp++) = '0' + num;
} else {
*(wp++) = 'a' + num - 10;
}
}
}
*wp = '\0';
return zbuf;
}
/**
* Decode a string encoded by URL encoding.
*/
inline char* urldecode(const char* str, size_t* sp) {
_assert_(str && sp);
size_t zsiz = std::strlen(str);
char* zbuf = new char[zsiz+1];
char* wp = zbuf;
const char* ep = str + zsiz;
while (str < ep) {
int32_t c = *str;
if (c == '%') {
int32_t num = 0;
if (++str >= ep) break;
c = *str;
if (c >= '0' && c <= '9') {
num = c - '0';
} else if (c >= 'a' && c <= 'f') {
num = c - 'a' + 10;
} else if (c >= 'A' && c <= 'F') {
num = c - 'A' + 10;
}
if (++str >= ep) break;
c = *str;
if (c >= '0' && c <= '9') {
num = num * 0x10 + c - '0';
} else if (c >= 'a' && c <= 'f') {
num = num * 0x10 + c - 'a' + 10;
} else if (c >= 'A' && c <= 'F') {
num = num * 0x10 + c - 'A' + 10;
}
*(wp++) = num;
str++;
} else if (c == '+') {
*(wp++) = ' ';
str++;
} else if (c <= ' ' || c == 0x7f) {
str++;
} else {
*(wp++) = c;
str++;
}
}
*wp = '\0';
*sp = wp - zbuf;
return zbuf;
}
/**
* Encode a serial object by Quoted-printable encoding.
*/
inline char* quoteencode(const void* buf, size_t size) {
_assert_(buf && size <= MEMMAXSIZ);
const unsigned char* rp = (const unsigned char*)buf;
char* zbuf = new char[size*3+1];
char* wp = zbuf;
for (const unsigned char* ep = rp + size; rp < ep; rp++) {
int32_t c = *rp;
if (c == '=' || c < ' ' || c > 0x7e) {
*(wp++) = '=';
int32_t num = c >> 4;
if (num < 10) {
*(wp++) = '0' + num;
} else {
*(wp++) = 'A' + num - 10;
}
num = c & 0x0f;
if (num < 10) {
*(wp++) = '0' + num;
} else {
*(wp++) = 'A' + num - 10;
}
} else {
*(wp++) = c;
}
}
*wp = '\0';
return zbuf;
}
/**
* Decode a string encoded by Quoted-printable encoding.
*/
inline char* quotedecode(const char* str, size_t* sp) {
_assert_(str && sp);
size_t zsiz = std::strlen(str);
char* zbuf = new char[zsiz+1];
char* wp = zbuf;
const char* ep = str + zsiz;
while (str < ep) {
int32_t c = *str;
if (c == '=') {
int32_t num = 0;
if (++str >= ep) break;
c = *str;
if (c == '\r') {
if (++str >= ep) break;
if (*str == '\n') str++;
} else if (c == '\n') {
str++;
} else {
if (c >= '0' && c <= '9') {
num = c - '0';
} else if (c >= 'a' && c <= 'f') {
num = c - 'a' + 10;
} else if (c >= 'A' && c <= 'F') {
num = c - 'A' + 10;
}
if (++str >= ep) break;
c = *str;
if (c >= '0' && c <= '9') {
num = num * 0x10 + c - '0';
} else if (c >= 'a' && c <= 'f') {
num = num * 0x10 + c - 'a' + 10;
} else if (c >= 'A' && c <= 'F') {
num = num * 0x10 + c - 'A' + 10;
}
*(wp++) = num;
str++;
}
} else if (c < ' ' || c == 0x7f) {
str++;
} else {
*(wp++) = c;
str++;
}
}
*wp = '\0';
*sp = wp - zbuf;
return zbuf;
}
/**
* Encode a serial object by Base64 encoding.
*/
inline char* baseencode(const void* buf, size_t size) {
_assert_(buf && size <= MEMMAXSIZ);
const char* tbl = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01
23456789+/";
const unsigned char* rp = (const unsigned char*)buf;
char* zbuf = new char[size*4/3+5];
char* wp = zbuf;
for (size_t i = 0; i < size; i += 3) {
switch (size - i) {
case 1: {
*(wp++) = tbl[rp[0] >> 2];
*(wp++) = tbl[(rp[0] & 3) << 4];
*(wp++) = '=';
*(wp++) = '=';
break;
}
case 2: {
*(wp++) = tbl[rp[0] >> 2];
*(wp++) = tbl[((rp[0] & 3) << 4) + (rp[1] >> 4)];
*(wp++) = tbl[(rp[1] & 0xf) << 2];
*(wp++) = '=';
break;
}
default: {
*(wp++) = tbl[rp[0] >> 2];
*(wp++) = tbl[((rp[0] & 3) << 4) + (rp[1] >> 4)];
*(wp++) = tbl[((rp[1] & 0xf) << 2) + (rp[2] >> 6)];
*(wp++) = tbl[rp[2] & 0x3f];
break;
}
}
rp += 3;
}
*wp = '\0';
return zbuf;
}
/**
* Decode a string encoded by Base64 encoding.
*/
inline char* basedecode(const char* str, size_t* sp) {
_assert_(str && sp);
size_t bpos = 0;
size_t eqcnt = 0;
size_t len = std::strlen(str);
unsigned char* zbuf = new unsigned char[len+4];
unsigned char* wp = zbuf;
size_t zsiz = 0;
while (bpos < len && eqcnt == 0) {
size_t bits = 0;
size_t i;
for (i = 0; bpos < len && i < 4; bpos++) {
if (str[bpos] >= 'A' && str[bpos] <= 'Z') {
bits = (bits << 6) | (str[bpos] - 'A');
i++;
} else if (str[bpos] >= 'a' && str[bpos] <= 'z') {
bits = (bits << 6) | (str[bpos] - 'a' + 26);
i++;
} else if (str[bpos] >= '0' && str[bpos] <= '9') {
bits = (bits << 6) | (str[bpos] - '0' + 52);
i++;
} else if (str[bpos] == '+') {
bits = (bits << 6) | 62;
i++;
} else if (str[bpos] == '/') {
bits = (bits << 6) | 63;
i++;
} else if (str[bpos] == '=') {
bits <<= 6;
i++;
eqcnt++;
}
}
if (i == 0 && bpos >= len) continue;
switch (eqcnt) {
case 0: {
*wp++ = (bits >> 16) & 0xff;
*wp++ = (bits >> 8) & 0xff;
*wp++ = bits & 0xff;
zsiz += 3;
break;
}
case 1: {
*wp++ = (bits >> 16) & 0xff;
*wp++ = (bits >> 8) & 0xff;
zsiz += 2;
break;
}
case 2: {
*wp++ = (bits >> 16) & 0xff;
zsiz += 1;
break;
}
}
}
zbuf[zsiz] = '\0';
*sp = zsiz;
return (char*)zbuf;
}
/**
* Cipher or decipher a serial object with the Arcfour stream cipher. * Cipher or decipher a serial object with the Arcfour stream cipher.
*/ */
inline void arccipher(const void* ptr, size_t size, const void* kbuf, size_ t ksiz, void* obuf) { inline void arccipher(const void* ptr, size_t size, const void* kbuf, size_ t ksiz, void* obuf) {
_assert_(ptr && size <= MEMMAXSIZ && kbuf && ksiz <= MEMMAXSIZ && obuf); _assert_(ptr && size <= MEMMAXSIZ && kbuf && ksiz <= MEMMAXSIZ && obuf);
if (ksiz < 1) { if (ksiz < 1) {
kbuf = ""; kbuf = "";
ksiz = 1; ksiz = 1;
} }
uint32_t sbox[0x100], kbox[0x100]; uint32_t sbox[0x100], kbox[0x100];
for(int32_t i = 0; i < 0x100; i++){ for (int32_t i = 0; i < 0x100; i++) {
sbox[i] = i; sbox[i] = i;
kbox[i] = ((uint8_t*)kbuf)[i%ksiz]; kbox[i] = ((uint8_t*)kbuf)[i%ksiz];
} }
uint32_t sidx = 0; uint32_t sidx = 0;
for(int i = 0; i < 0x100; i++){ for (int32_t i = 0; i < 0x100; i++) {
sidx = (sidx + sbox[i] + kbox[i]) & 0xff; sidx = (sidx + sbox[i] + kbox[i]) & 0xff;
uint32_t swap = sbox[i]; uint32_t swap = sbox[i];
sbox[i] = sbox[sidx]; sbox[i] = sbox[sidx];
sbox[sidx] = swap; sbox[sidx] = swap;
} }
uint32_t x = 0; uint32_t x = 0;
uint32_t y = 0; uint32_t y = 0;
for(size_t i = 0; i < size; i++){ for (size_t i = 0; i < size; i++) {
x = (x + 1) & 0xff; x = (x + 1) & 0xff;
y = (y + sbox[x]) & 0xff; y = (y + sbox[x]) & 0xff;
uint32_t swap = sbox[x]; uint32_t swap = sbox[x];
sbox[x] = sbox[y]; sbox[x] = sbox[y];
sbox[y] = swap; sbox[y] = swap;
((uint8_t*)obuf)[i] = ((uint8_t*)ptr)[i] ^ sbox[(sbox[x]+sbox[y])&0xff] ; ((uint8_t*)obuf)[i] = ((uint8_t*)ptr)[i] ^ sbox[(sbox[x]+sbox[y])&0xff] ;
} }
} }
/** /**
* Duplicate a region on memory.
*/
inline char* memdup(const char* ptr, size_t size) {
_assert_(ptr && size <= MEMMAXSIZ);
char* obuf = new char[size+1];
std::memcpy(obuf, ptr, size);
return obuf;
}
/**
* Duplicate a string on memory.
*/
inline char* strdup(const char* str) {
_assert_(str);
size_t size = std::strlen(str);
char* obuf = memdup(str, size);
obuf[size] = '\0';
return obuf;
}
/**
* Convert the letters of a string into upper case.
*/
inline char* strtoupper(char* str) {
_assert_(str);
char* wp = str;
while (*wp != '\0') {
if (*wp >= 'a' && *wp <= 'z') *wp -= 'a' - 'A';
wp++;
}
return str;
}
/**
* Convert the letters of a string into lower case.
*/
inline char* strtolower(char* str) {
_assert_(str);
char* wp = str;
while (*wp != '\0') {
if (*wp >= 'A' && *wp <= 'Z') *wp += 'a' - 'A';
wp++;
}
return str;
}
/**
* Cut space characters at head or tail of a string.
*/
inline char* strtrim(char* str) {
_assert_(str);
const char* rp = str;
char* wp = str;
bool head = true;
while (*rp != '\0') {
if (*rp > '\0' && *rp <= ' ') {
if (!head) *(wp++) = *rp;
} else {
*(wp++) = *rp;
head = false;
}
rp++;
}
*wp = '\0';
while (wp > str && wp[-1] > '\0' && wp[-1] <= ' ') {
*(--wp) = '\0';
}
return str;
}
/**
* Squeeze space characters in a string and trim it.
*/
inline char* strsqzspc(char* str) {
_assert_(str);
const char* rp = str;
char* wp = str;
bool spc = true;
while (*rp != '\0') {
if (*rp > '\0' && *rp <= ' ') {
if (!spc) *(wp++) = *rp;
spc = true;
} else {
*(wp++) = *rp;
spc = false;
}
rp++;
}
*wp = '\0';
for (wp--; wp >= str; wp--) {
if (*wp > '\0' && *wp <= ' ') {
*wp = '\0';
} else {
break;
}
}
return str;
}
/**
* Normalize space characters in a string and trim it.
*/
inline char* strnrmspc(char* str) {
_assert_(str);
const char* rp = str;
char* wp = str;
bool spc = true;
while (*rp != '\0') {
if ((*rp > '\0' && *rp <= ' ') || *rp == 0x7f) {
if (!spc) *(wp++) = ' ';
spc = true;
} else {
*(wp++) = *rp;
spc = false;
}
rp++;
}
*wp = '\0';
for (wp--; wp >= str; wp--) {
if (*wp == ' ') {
*wp = '\0';
} else {
break;
}
}
return str;
}
/**
* Compare two strings with case insensitive evaluation.
*/
inline int32_t stricmp(const char* astr, const char* bstr) {
_assert_(astr && bstr);
while (*astr != '\0') {
if (*bstr == '\0') return 1;
int32_t ac = (*astr >= 'A' && *astr <= 'Z') ? *astr + ('a' - 'A') : *(u
nsigned char*)astr;
int32_t bc = (*bstr >= 'A' && *bstr <= 'Z') ? *bstr + ('a' - 'A') : *(u
nsigned char*)bstr;
if (ac != bc) return ac - bc;
astr++;
bstr++;
}
return (*bstr == '\0') ? 0 : -1;
}
/**
* Check whether a string begins with a key.
*/
inline bool strfwm(const char* str, const char* key) {
_assert_(str && key);
while (*key != '\0') {
if (*str != *key || *str == '\0') return false;
key++;
str++;
}
return true;
}
/**
* Check whether a string begins with a key by case insensitive evaluation.
*/
inline bool strifwm(const char* str, const char* key) {
_assert_(str && key);
while (*key != '\0') {
if (*str == '\0') return false;
int32_t sc = *str;
if (sc >= 'A' && sc <= 'Z') sc += 'a' - 'A';
int32_t kc = *key;
if (kc >= 'A' && kc <= 'Z') kc += 'a' - 'A';
if (sc != kc) return false;
key++;
str++;
}
return true;
}
/**
* Check whether a string ends with a key.
*/
inline bool strbwm(const char* str, const char* key) {
_assert_(str && key);
size_t slen = std::strlen(str);
size_t klen = std::strlen(key);
for (size_t i = 1; i <= klen; i++) {
if (i > slen || str[slen-i] != key[klen-i]) return false;
}
return true;
}
/**
* Check whether a string ends with a key by case insensitive evaluation.
*/
inline bool stribwm(const char* str, const char* key) {
_assert_(str && key);
size_t slen = std::strlen(str);
size_t klen = std::strlen(key);
for (size_t i = 1; i <= klen; i++) {
if (i > slen) return false;
int32_t sc = str[slen-i];
if (sc >= 'A' && sc <= 'Z') sc += 'a' - 'A';
int32_t kc = key[klen-i];
if (kc >= 'A' && kc <= 'Z') kc += 'a' - 'A';
if (sc != kc) return false;
}
return true;
}
/**
* Allocate a region on memory. * Allocate a region on memory.
*/ */
inline void* xmalloc(size_t size) { inline void* xmalloc(size_t size) {
_assert_(size <= MEMMAXSIZ); _assert_(size <= MEMMAXSIZ);
void* ptr = std::malloc(size); void* ptr = std::malloc(size);
if (!ptr) throw std::bad_alloc(); if (!ptr) throw std::bad_alloc();
return ptr; return ptr;
} }
/** /**
 End of changes. 39 change blocks. 
65 lines changed or deleted 921 lines changed or added

This html diff was produced by rfcdiff 1.41. The latest version is available from http://tools.ietf.org/tools/rfcdiff/