From b1fd0ab73425e682a0a1404da6963f47a033bb34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Mar 2016 15:50:45 +0200 Subject: daemon: Improve the SQLite wrapper API. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In particular, this eliminates a bunch of boilerplate code. Also integrates these Nix commits: 80da7a6 Probably fix SQLITE_BUSY errors 37a337b throwSQLiteError(): Check for SIGINT so we don't loop forever Co-authored-by: Ludovic Courtès --- nix/libstore/local-store.cc | 243 +++++++++++++------------------------------- nix/libstore/local-store.hh | 7 +- nix/libstore/sqlite.cc | 85 ++++++++++------ nix/libstore/sqlite.hh | 57 +++++++---- nix/libstore/store-api.hh | 4 +- 5 files changed, 168 insertions(+), 228 deletions(-) (limited to 'nix/libstore') diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 6d298cda43..882bce1f40 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -329,6 +329,7 @@ void LocalStore::openDB(bool create) // ensure efficient lookup. stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -523,23 +524,16 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.hash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -555,13 +549,12 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - foreach (DerivationOutputs::iterator, i, drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i->first); - stmtAddDerivationOutput.bind(i->second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + for (auto & i : drv.outputs) { + stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } @@ -569,24 +562,16 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che } -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) +void LocalStore::addReference(uint64_t referrer, uint64_t reference) { - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); + stmtAddReference.use()(referrer)(reference).exec(); } void LocalStore::registerFailedPath(const Path & path) { retrySQLite([&]() { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path `%1%'") % path); + stmtRegisterFailedPath.use()(path)(time(0)).step(); }); } @@ -594,12 +579,7 @@ void LocalStore::registerFailedPath(const Path & path) bool LocalStore::hasPathFailed(const Path & path) { return retrySQLite([&]() { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; + return stmtHasPathFailed.use()(path).next(); }); } @@ -607,18 +587,11 @@ bool LocalStore::hasPathFailed(const Path & path) PathSet LocalStore::queryFailedPaths() { return retrySQLite([&]() { - SQLiteStmtUse use(stmtQueryFailedPaths); + auto useQueryFailedPaths(stmtQueryFailedPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); + while (useQueryFailedPaths.next()) + res.insert(useQueryFailedPaths.getStr(0)); return res; }); @@ -630,12 +603,8 @@ void LocalStore::clearFailedPaths(const PathSet & paths) retrySQLite([&]() { SQLiteTxn txn(db); - foreach (PathSet::const_iterator, i, paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(*i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i); - } + for (auto & path : paths) + stmtClearFailedPath.use()(path).exec(); txn.commit(); }); @@ -666,41 +635,28 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) return retrySQLite([&]() { /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); - - stmtQueryPathInfo.bind(path); + auto useQueryPathInfo(stmtQueryPathInfo.use()(path)); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + if (!useQueryPathInfo.next()) + throw Error(format("path `%1%' is not valid") % path); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info.id = useQueryPathInfo.getInt(0); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.hash = parseHashField(path, s); + info.hash = parseHashField(path, useQueryPathInfo.getStr(1)); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + info.registrationTime = useQueryPathInfo.getInt(2); - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); if (s) info.deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info.narSize = useQueryPathInfo.getInt(4); /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); - - stmtQueryReferences.bind(info.id); + auto useQueryReferences(stmtQueryReferences.use()(info.id)); - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of `%1%'") % path); + while (useQueryReferences.next()) + info.references.insert(useQueryReferences.getStr(0)); return info; }); @@ -711,37 +667,26 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) narSize field. */ void LocalStore::updatePathInfo(const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); + stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.hash)) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } bool LocalStore::isValidPath_(const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return stmtQueryPathInfo.use()(path).next(); } @@ -767,20 +712,9 @@ PathSet LocalStore::queryValidPaths(const PathSet & paths) PathSet LocalStore::queryAllValidPaths() { return retrySQLite([&]() { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + auto use(stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; }); } @@ -796,19 +730,10 @@ void LocalStore::queryReferences(const Path & path, void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); - - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } + auto useQueryReferrers(stmtQueryReferrers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of `%1%'") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } @@ -832,19 +757,11 @@ PathSet LocalStore::queryValidDerivers(const Path & path) assertStorePath(path); return retrySQLite([&]() { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path)); PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; }); @@ -854,19 +771,11 @@ PathSet LocalStore::queryValidDerivers(const Path & path) PathSet LocalStore::queryDerivationOutputs(const Path & path) { return retrySQLite([&]() { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path))); PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of `%1%'") % path); + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; }); @@ -876,19 +785,11 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) StringSet LocalStore::queryDerivationOutputNames(const Path & path) { return retrySQLite([&]() { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path))); StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of `%1%'") % path); + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; }); @@ -902,12 +803,9 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) Path prefix = settings.nixStore + "/" + hashPart; return retrySQLite([&]() -> Path { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); + auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix)); - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + if (!useQueryPathFromHashPart.next()) return ""; const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; @@ -1154,10 +1052,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) paths.insert(i->path); } - foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryValidPathId(i->path); - foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryValidPathId(*j)); + for (auto & i : infos) { + auto referrer = queryValidPathId(i.path); + for (auto & j : i.references) + addReference(referrer, queryValidPathId(j)); } /* Check that the derivation outputs are correct. We can't do @@ -1190,12 +1088,7 @@ void LocalStore::invalidatePath(const Path & path) drvHashes.erase(path); - SQLiteStmtUse use(stmtInvalidatePath); - - stmtInvalidatePath.bind(path); - - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path `%1%' in database") % path); + stmtInvalidatePath.use()(path).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh index f591589838..6110468498 100644 --- a/nix/libstore/local-store.hh +++ b/nix/libstore/local-store.hh @@ -207,6 +207,7 @@ private: SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryDerivationOutputs; SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; /* Cache for pathContentsGood(). */ std::map pathContentsGoodCache; @@ -223,11 +224,11 @@ private: void makeStoreWritable(); - unsigned long long queryValidPathId(const Path & path); + uint64_t queryValidPathId(const Path & path); - unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); + uint64_t addValidPath(const ValidPathInfo & info, bool checkOutputs = true); - void addReference(unsigned long long referrer, unsigned long long reference); + void addReference(uint64_t referrer, uint64_t reference); void appendReferrer(const Path & from, const Path & to, bool lock); diff --git a/nix/libstore/sqlite.cc b/nix/libstore/sqlite.cc index 8646ff5b12..e08c67f40e 100644 --- a/nix/libstore/sqlite.cc +++ b/nix/libstore/sqlite.cc @@ -53,15 +53,6 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) this->db = db; } -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - SQLiteStmt::~SQLiteStmt() { try { @@ -72,43 +63,79 @@ SQLiteStmt::~SQLiteStmt() } } -void SQLiteStmt::bind(const string & value) +SQLiteStmt::Use::Use(SQLiteStmt & stmt) + : stmt(stmt) +{ + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() { - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); + sqlite3_reset(stmt); } -void SQLiteStmt::bind(int value) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) { - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; } -void SQLiteStmt::bind64(long long value) +SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; } -void SQLiteStmt::bind() +SQLiteStmt::Use & SQLiteStmt::Use::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); + throwSQLiteError(stmt.db, "binding argument"); + return *this; } -SQLiteStmtUse::SQLiteStmtUse(SQLiteStmt & stmt) - : stmt(stmt) +int SQLiteStmt::Use::step() { - stmt.reset(); + return sqlite3_step(stmt); } -SQLiteStmtUse::~SQLiteStmtUse() +void SQLiteStmt::Use::exec() { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, "executing SQLite statement"); +} + +bool SQLiteStmt::Use::next() +{ + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, "executing SQLite query"); + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) +{ + auto s = (const char *) sqlite3_column_text(stmt, col); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) +{ + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); } SQLiteTxn::SQLiteTxn(sqlite3 * db) diff --git a/nix/libstore/sqlite.hh b/nix/libstore/sqlite.hh index 0abdb74631..326e4a4855 100644 --- a/nix/libstore/sqlite.hh +++ b/nix/libstore/sqlite.hh @@ -22,29 +22,48 @@ struct SQLite /* RAII wrapper to create and destroy SQLite prepared statements. */ struct SQLiteStmt { - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } + sqlite3 * db = 0; + sqlite3_stmt * stmt = 0; + SQLiteStmt() { } void create(sqlite3 * db, const std::string & s); - void reset(); ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - void bind(const std::string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt); - ~SQLiteStmtUse(); + /* Helper for binding / executing statements. */ + class Use + { + friend struct SQLiteStmt; + private: + SQLiteStmt & stmt; + unsigned int curArg = 1; + Use(SQLiteStmt & stmt); + + public: + + ~Use(); + + /* Bind the next parameter. */ + Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (int64_t value, bool notNull = true); + Use & bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + }; + + Use use() + { + return Use(*this); + } }; /* RAII helper that ensures transactions are aborted unless explicitly diff --git a/nix/libstore/store-api.hh b/nix/libstore/store-api.hh index 3e982f6dd3..fa78d595f2 100644 --- a/nix/libstore/store-api.hh +++ b/nix/libstore/store-api.hh @@ -89,8 +89,8 @@ struct ValidPathInfo Hash hash; PathSet references; time_t registrationTime = 0; - unsigned long long narSize = 0; // 0 = unknown - unsigned long long id; // internal use only + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only bool operator == (const ValidPathInfo & i) const { -- cgit v1.2.3