ETM-Schurig/powersync-core
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

don't use QSqlQuery at all outside the job queue

Browse Source 
On some backends, such as MySQL, a prepare() call will access
the database
This commit is contained in:
Michael Zanetti 2019-11-28 14:20:37 +01:00
parent cee2ed1542
commit 619bbf76f3
2 changed files with 85 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
libnymea-core/logging/logengine.cpp
View File

@ -139,6 +139,13 @@ namespace nymeaserver {
\a dbName is the name of the database. In case of SQLITE this should contain a file path. The Driver will create the file if required. In case of using a \a dbName is the name of the database. In case of SQLITE this should contain a file path. The Driver will create the file if required. In case of using a
database server like MYSQL, the database must exist on the host given by \a hostname and be accessible with the given \a username and \a password. database server like MYSQL, the database must exist on the host given by \a hostname and be accessible with the given \a username and \a password.
*/ */
// IMPORTANT:
// DatabaseJobs run threaded, however, QSql is *not* threadsafe.
// It is crucial to *not* access m_db while the job queue is being processed.
// That is, entire setup of the DB must happen before processQueue() is called
// and teardown must happen only after the job queue is empty.
LogEngine::LogEngine(const QString &driver, const QString &dbName, const QString &hostname, const QString &username, const QString &password, int maxDBSize, QObject *parent): LogEngine::LogEngine(const QString &driver, const QString &dbName, const QString &hostname, const QString &username, const QString &password, int maxDBSize, QObject *parent):
QObject(parent), QObject(parent),
m_username(username), m_username(username),
@ -195,7 +202,6 @@ LogEngine::~LogEngine()
LogEntriesFetchJob *LogEngine::fetchLogEntries(const LogFilter &filter) LogEntriesFetchJob *LogEngine::fetchLogEntries(const LogFilter &filter)
{ {
QList<LogEntry> results; QList<LogEntry> results;
QSqlQuery query(m_db);
QString limitString; QString limitString;
if (filter.limit() >= 0) { if (filter.limit() >= 0) {
@ -211,40 +217,33 @@ LogEntriesFetchJob *LogEngine::fetchLogEntries(const LogFilter &filter)
} else { } else {
queryString = QString("SELECT * FROM entries WHERE %1 ORDER BY timestamp DESC %2;").arg(filter.queryString()).arg(limitString); queryString = QString("SELECT * FROM entries WHERE %1 ORDER BY timestamp DESC %2;").arg(filter.queryString()).arg(limitString);
} }
// qCDebug(dcLogEngine()) << "Preparing query:" << queryString;
query.prepare(queryString);
foreach (const QString &value, filter.values()) { DatabaseJob *job = new DatabaseJob(m_db, queryString, filter.values());
query.addBindValue(LogValueTool::serializeValue(value));
qCDebug(dcLogEngine()) << "Binding value to query:" << LogValueTool::serializeValue(value);
}
DatabaseJob *job = new DatabaseJob(query);
LogEntriesFetchJob *fetchJob = new LogEntriesFetchJob(this); LogEntriesFetchJob *fetchJob = new LogEntriesFetchJob(this);
connect(job, &DatabaseJob::finished, this, [this, job, fetchJob](){ connect(job, &DatabaseJob::finished, this, [this, job, fetchJob](){
fetchJob->deleteLater(); fetchJob->deleteLater();
if (job->query().lastError().isValid()) { if (job->error().isValid()) {
qCWarning(dcLogEngine) << "Error fetching log entries. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText(); qCWarning(dcLogEngine) << "Error fetching log entries. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
fetchJob->finished(); fetchJob->finished();
return; return;
} }
while (job->query().next()) { foreach (const QSqlRecord &result, job->results()) {
LogEntry entry( LogEntry entry(
QDateTime::fromTime_t(job->query().value("timestamp").toLongLong()), QDateTime::fromTime_t(result.value("timestamp").toUInt()),
(Logging::LoggingLevel)job->query().value("loggingLevel").toInt(), static_cast<Logging::LoggingLevel>(result.value("loggingLevel").toInt()),
(Logging::LoggingSource)job->query().value("sourceType").toInt(), static_cast<Logging::LoggingSource>(result.value("sourceType").toInt()),
job->query().value("errorCode").toInt()); result.value("errorCode").toInt());
entry.setTypeId(job->query().value("typeId").toUuid()); entry.setTypeId(result.value("typeId").toUuid());
entry.setDeviceId(DeviceId(job->query().value("deviceId").toString())); entry.setDeviceId(DeviceId(result.value("deviceId").toString()));
entry.setValue(LogValueTool::convertVariantToString(LogValueTool::deserializeValue(job->query().value("value").toString()))); entry.setValue(LogValueTool::convertVariantToString(LogValueTool::deserializeValue(result.value("value").toString())));
entry.setEventType((Logging::LoggingEventType)job->query().value("loggingEventType").toInt()); entry.setEventType(static_cast<Logging::LoggingEventType>(result.value("loggingEventType").toInt()));
entry.setActive(job->query().value("active").toBool()); entry.setActive(result.value("active").toBool());
fetchJob->m_results.append(entry); fetchJob->m_results.append(entry);
} }
qCDebug(dcLogEngine) << "Fetched" << fetchJob->results().count() << "entries for db query:" << job->query().executedQuery(); qCDebug(dcLogEngine) << "Fetched" << fetchJob->results().count() << "entries for db query:" << job->executedQuery();
fetchJob->finished(); fetchJob->finished();
}); });
@ -257,23 +256,19 @@ DevicesFetchJob *LogEngine::fetchDevices()
{ {
QString queryString = QString("SELECT deviceId FROM entries WHERE deviceId != \"%1\" GROUP BY deviceId;").arg(QUuid().toString()); QString queryString = QString("SELECT deviceId FROM entries WHERE deviceId != \"%1\" GROUP BY deviceId;").arg(QUuid().toString());
DatabaseJob *job = new DatabaseJob(queryString, m_db); DatabaseJob *job = new DatabaseJob(m_db, queryString);
DevicesFetchJob *fetchJob = new DevicesFetchJob(this); DevicesFetchJob *fetchJob = new DevicesFetchJob(this);
connect(job, &DatabaseJob::finished, this, [this, job, fetchJob](){ connect(job, &DatabaseJob::finished, this, [this, job, fetchJob](){
fetchJob->deleteLater(); fetchJob->deleteLater();
if (job->query().lastError().type() != QSqlError::NoError) { if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Error fetching device entries from log database:" << m_db.lastError().driverText() << m_db.lastError().databaseText(); qCWarning(dcLogEngine()) << "Error fetching device entries from log database:" << job->error().driverText() << job->error().databaseText();
fetchJob->finished(); fetchJob->finished();
return; return;
} }
if (!job->query().first()) { foreach (const QSqlRecord &result, job->results()) {
fetchJob->finished(); fetchJob->m_results.append(DeviceId::fromUuid(result.value("deviceId").toUuid()));
return;
} }
do {
fetchJob->m_results.append(DeviceId::fromUuid(job->query().value("deviceId").toUuid()));
} while (job->query().next());
fetchJob->finished(); fetchJob->finished();
}); });
return fetchJob; return fetchJob;
@ -293,11 +288,11 @@ void LogEngine::clearDatabase()
QString queryDeleteString = QString("DELETE FROM entries;"); QString queryDeleteString = QString("DELETE FROM entries;");
DatabaseJob *job = new DatabaseJob(queryDeleteString, m_db); DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job](){ connect(job, &DatabaseJob::finished, this, [this, job](){
if (job->query().lastError().type() != QSqlError::NoError) { if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Could not clear logging database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText(); qCWarning(dcLogEngine) << "Could not clear logging database. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
} }
emit logDatabaseUpdated(); emit logDatabaseUpdated();
}); });
@ -426,10 +421,10 @@ void LogEngine::removeDeviceLogs(const DeviceId &deviceId)
QString queryDeleteString = QString("DELETE FROM entries WHERE deviceId = '%1';").arg(deviceId.toString()); QString queryDeleteString = QString("DELETE FROM entries WHERE deviceId = '%1';").arg(deviceId.toString());
DatabaseJob *job = new DatabaseJob(queryDeleteString, m_db); DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job, deviceId](){ connect(job, &DatabaseJob::finished, this, [this, job, deviceId](){
if (job->query().lastError().type() != QSqlError::NoError) { if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting log entries from device" << deviceId.toString() << ". Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText(); qCWarning(dcLogEngine) << "Error deleting log entries from device" << deviceId.toString() << ". Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
} else { } else {
emit logDatabaseUpdated(); emit logDatabaseUpdated();
} }
@ -444,12 +439,12 @@ void LogEngine::removeRuleLogs(const RuleId &ruleId)
QString queryDeleteString = QString("DELETE FROM entries WHERE typeId = '%1';").arg(ruleId.toString()); QString queryDeleteString = QString("DELETE FROM entries WHERE typeId = '%1';").arg(ruleId.toString());
DatabaseJob *job = new DatabaseJob(queryDeleteString, m_db); DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job, ruleId](){ connect(job, &DatabaseJob::finished, this, [this, job, ruleId](){
if (job->query().lastError().type() != QSqlError::NoError) { if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting log entries from rule" << ruleId.toString() << ". Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText(); qCWarning(dcLogEngine) << "Error deleting log entries from rule" << ruleId.toString() << ". Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
} else { } else {
emit logDatabaseUpdated(); emit logDatabaseUpdated();
} }
@ -471,12 +466,12 @@ void LogEngine::appendLogEntry(const LogEntry &entry)
.arg(entry.active()) .arg(entry.active())
.arg(entry.errorCode()); .arg(entry.errorCode());
DatabaseJob *job = new DatabaseJob(queryString, m_db); DatabaseJob *job = new DatabaseJob(m_db, queryString);
connect(job, &DatabaseJob::finished, this, [this, job, entry](){ connect(job, &DatabaseJob::finished, this, [this, job, entry](){
if (job->query().lastError().type() != QSqlError::NoError) { if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error writing log entry. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText(); qCWarning(dcLogEngine) << "Error writing log entry. Driver error:" << job->error().driverText() << "Database error:" << job->error().number() << job->error().databaseText();
qCWarning(dcLogEngine) << entry; qCWarning(dcLogEngine) << entry;
m_dbMalformed = true; m_dbMalformed = true;
processQueue(); processQueue();
@ -502,20 +497,18 @@ void LogEngine::checkDBSize()
QDateTime startTime = QDateTime::currentDateTime(); QDateTime startTime = QDateTime::currentDateTime();
QString queryString = "SELECT COUNT(*) FROM entries;"; QString queryString = "SELECT COUNT(*) FROM entries;";
DatabaseJob *job = new DatabaseJob(queryString, m_db); DatabaseJob *job = new DatabaseJob(m_db, queryString);
connect(job, &DatabaseJob::finished, this, [this, job, startTime](){ connect(job, &DatabaseJob::finished, this, [this, job, startTime](){
QSqlQuery result = job->query(); if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Failed to query entry count in db:" << job->error().databaseText();
if (m_db.lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Failed to query entry count in db:" << m_db.lastError().databaseText();
return; return;
} }
if (!result.first()) { if (job->results().isEmpty()) {
qCWarning(dcLogEngine()) << "Failed retrieving entry count."; qCWarning(dcLogEngine()) << "Failed retrieving entry count.";
return; return;
} }
m_entryCount = result.value(0).toInt(); m_entryCount = job->results().first().value(0).toInt();
if (m_entryCount <= m_dbMaxSize) { if (m_entryCount <= m_dbMaxSize) {
return; return;
@ -528,11 +521,11 @@ void LogEngine::checkDBSize()
} }
QString queryDeleteString = QString("DELETE FROM entries WHERE ROWID IN (SELECT ROWID FROM entries ORDER BY timestamp DESC LIMIT -1 OFFSET %1);").arg(QString::number(m_dbMaxSize)); QString queryDeleteString = QString("DELETE FROM entries WHERE ROWID IN (SELECT ROWID FROM entries ORDER BY timestamp DESC LIMIT -1 OFFSET %1);").arg(QString::number(m_dbMaxSize));
DatabaseJob *deleteJob = new DatabaseJob(queryDeleteString, m_db); DatabaseJob *deleteJob = new DatabaseJob(m_db, queryDeleteString);
connect(deleteJob, &DatabaseJob::finished, this, [this, deleteJob,startTime](){ connect(deleteJob, &DatabaseJob::finished, this, [this, deleteJob,startTime](){
if (deleteJob->query().lastError().type() != QSqlError::NoError) { if (deleteJob->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting oldest log entries to keep size. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText(); qCWarning(dcLogEngine) << "Error deleting oldest log entries to keep size. Driver error:" << deleteJob->error().driverText() << "Database error:" << deleteJob->error().databaseText();
} }
m_entryCount = m_dbMaxSize; m_entryCount = m_dbMaxSize;
qCDebug(dcLogEngine()) << "Ran housekeeping on log database in" << startTime.msecsTo(QDateTime::currentDateTime()) << "ms."; qCDebug(dcLogEngine()) << "Ran housekeeping on log database in" << startTime.msecsTo(QDateTime::currentDateTime()) << "ms.";
@ -573,8 +566,25 @@ void LogEngine::processQueue()
m_currentJob = job; m_currentJob = job;
QFuture<DatabaseJob*> future = QtConcurrent::run([job](){ QFuture<DatabaseJob*> future = QtConcurrent::run([job](){
job->query().exec(); QSqlQuery query(job->m_db);
return job; query.prepare(job->m_queryString);
foreach (const QString &value, job->m_bindValues) {
query.addBindValue(LogValueTool::serializeValue(value));
}
query.exec();
job->m_error = query.lastError();
job->m_executedQuery = query.executedQuery();
if (!query.lastError().isValid()) {
while (query.next()) {
job->m_results.append(query.record());
}
}
return job;
}); });
m_jobWatcher.setFuture(future); m_jobWatcher.setFuture(future);
@ -752,6 +762,7 @@ bool LogEngine::initDB(const QString &username, const QString &password)
} }
if (!m_db.tables().contains("entries")) { if (!m_db.tables().contains("entries")) {
qCDebug(dcLogEngine()) << "No \"entries\" table in database. Creating it.";
m_db.exec("CREATE TABLE entries " m_db.exec("CREATE TABLE entries "
"(" "("
"timestamp int," "timestamp int,"

28
libnymea-core/logging/logengine.h
View File

@ -33,6 +33,8 @@
#include <QObject> #include <QObject>
#include <QSqlDatabase> #include <QSqlDatabase>
#include <QSqlQuery> #include <QSqlQuery>
#include <QSqlError>
#include <QSqlRecord>
#include <QTimer> #include <QTimer>
#include <QFutureWatcher> #include <QFutureWatcher>
@ -106,22 +108,30 @@ class DatabaseJob: public QObject
{ {
Q_OBJECT Q_OBJECT
public: public:
DatabaseJob(const QString &queryString, const QSqlDatabase &db) { DatabaseJob(const QSqlDatabase &db, const QString &queryString, const QStringList &bindValues = QStringList()):
m_query = QSqlQuery(db); m_db(db),
m_query.prepare(queryString); m_queryString(queryString),
m_bindValues(bindValues)
{
} }
// IMPORTANT: Make sure it only prepare()d but not executed QString executedQuery() const { return m_executedQuery; }
// QSQlQuery(QString, QSqlDatabase) implicitly executes! QSqlError error() const { return m_error; }
DatabaseJob(const QSqlQuery &query): m_query(query) {} QList<QSqlRecord> results() const { return m_results; }
QSqlQuery query() const { return m_query; }
signals: signals:
void finished(); void finished();
private: private:
QSqlQuery m_query; QSqlDatabase m_db;
QString m_queryString;
QStringList m_bindValues;
QString m_executedQuery;
QSqlError m_error;
QList<QSqlRecord> m_results;
friend class LogEngine;
}; };
class LogEntriesFetchJob: public QObject class LogEntriesFetchJob: public QObject