ETM-Schurig/powersync-core
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

don't use QSqlQuery at all outside the job queue

Browse Source 
On some backends, such as MySQL, a prepare() call will access
the database
This commit is contained in:
Michael Zanetti 2019-11-28 14:20:37 +01:00
parent cee2ed1542
commit 619bbf76f3
2 changed files with 85 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
libnymea-core/logging/logengine.cpp
View File

@ -139,6 +139,13 @@ namespace nymeaserver {
\a dbName is the name of the database. In case of SQLITE this should contain a file path. The Driver will create the file if required. In case of using a
database server like MYSQL, the database must exist on the host given by \a hostname and be accessible with the given \a username and \a password.
*/
// IMPORTANT:
// DatabaseJobs run threaded, however, QSql is *not* threadsafe.
// It is crucial to *not* access m_db while the job queue is being processed.
// That is, entire setup of the DB must happen before processQueue() is called
// and teardown must happen only after the job queue is empty.
LogEngine::LogEngine(const QString &driver, const QString &dbName, const QString &hostname, const QString &username, const QString &password, int maxDBSize, QObject *parent):
QObject(parent),
m_username(username),
@ -195,7 +202,6 @@ LogEngine::~LogEngine()
LogEntriesFetchJob *LogEngine::fetchLogEntries(const LogFilter &filter)
{
QList<LogEntry> results;
QSqlQuery query(m_db);
QString limitString;
if (filter.limit() >= 0) {
@ -211,40 +217,33 @@ LogEntriesFetchJob *LogEngine::fetchLogEntries(const LogFilter &filter)
} else {
queryString = QString("SELECT * FROM entries WHERE %1 ORDER BY timestamp DESC %2;").arg(filter.queryString()).arg(limitString);
}
// qCDebug(dcLogEngine()) << "Preparing query:" << queryString;
query.prepare(queryString);
foreach (const QString &value, filter.values()) {
query.addBindValue(LogValueTool::serializeValue(value));
qCDebug(dcLogEngine()) << "Binding value to query:" << LogValueTool::serializeValue(value);
}
DatabaseJob *job = new DatabaseJob(query);
DatabaseJob *job = new DatabaseJob(m_db, queryString, filter.values());
LogEntriesFetchJob *fetchJob = new LogEntriesFetchJob(this);
connect(job, &DatabaseJob::finished, this, [this, job, fetchJob](){
fetchJob->deleteLater();
if (job->query().lastError().isValid()) {
qCWarning(dcLogEngine) << "Error fetching log entries. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
if (job->error().isValid()) {
qCWarning(dcLogEngine) << "Error fetching log entries. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
fetchJob->finished();
return;
}
while (job->query().next()) {
foreach (const QSqlRecord &result, job->results()) {
LogEntry entry(
QDateTime::fromTime_t(job->query().value("timestamp").toLongLong()),
(Logging::LoggingLevel)job->query().value("loggingLevel").toInt(),
(Logging::LoggingSource)job->query().value("sourceType").toInt(),
job->query().value("errorCode").toInt());
entry.setTypeId(job->query().value("typeId").toUuid());
entry.setDeviceId(DeviceId(job->query().value("deviceId").toString()));
entry.setValue(LogValueTool::convertVariantToString(LogValueTool::deserializeValue(job->query().value("value").toString())));
entry.setEventType((Logging::LoggingEventType)job->query().value("loggingEventType").toInt());
entry.setActive(job->query().value("active").toBool());
QDateTime::fromTime_t(result.value("timestamp").toUInt()),
static_cast<Logging::LoggingLevel>(result.value("loggingLevel").toInt()),
static_cast<Logging::LoggingSource>(result.value("sourceType").toInt()),
result.value("errorCode").toInt());
entry.setTypeId(result.value("typeId").toUuid());
entry.setDeviceId(DeviceId(result.value("deviceId").toString()));
entry.setValue(LogValueTool::convertVariantToString(LogValueTool::deserializeValue(result.value("value").toString())));
entry.setEventType(static_cast<Logging::LoggingEventType>(result.value("loggingEventType").toInt()));
entry.setActive(result.value("active").toBool());
fetchJob->m_results.append(entry);
}
qCDebug(dcLogEngine) << "Fetched" << fetchJob->results().count() << "entries for db query:" << job->query().executedQuery();
qCDebug(dcLogEngine) << "Fetched" << fetchJob->results().count() << "entries for db query:" << job->executedQuery();
fetchJob->finished();
});
@ -257,23 +256,19 @@ DevicesFetchJob *LogEngine::fetchDevices()
{
QString queryString = QString("SELECT deviceId FROM entries WHERE deviceId != \"%1\" GROUP BY deviceId;").arg(QUuid().toString());
DatabaseJob *job = new DatabaseJob(queryString, m_db);
DatabaseJob *job = new DatabaseJob(m_db, queryString);
DevicesFetchJob *fetchJob = new DevicesFetchJob(this);
connect(job, &DatabaseJob::finished, this, [this, job, fetchJob](){
fetchJob->deleteLater();
if (job->query().lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Error fetching device entries from log database:" << m_db.lastError().driverText() << m_db.lastError().databaseText();
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Error fetching device entries from log database:" << job->error().driverText() << job->error().databaseText();
fetchJob->finished();
return;
}
if (!job->query().first()) {
fetchJob->finished();
return;
foreach (const QSqlRecord &result, job->results()) {
fetchJob->m_results.append(DeviceId::fromUuid(result.value("deviceId").toUuid()));
}
do {
fetchJob->m_results.append(DeviceId::fromUuid(job->query().value("deviceId").toUuid()));
} while (job->query().next());
fetchJob->finished();
});
return fetchJob;
@ -293,11 +288,11 @@ void LogEngine::clearDatabase()
QString queryDeleteString = QString("DELETE FROM entries;");
DatabaseJob *job = new DatabaseJob(queryDeleteString, m_db);
DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job](){
if (job->query().lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Could not clear logging database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Could not clear logging database. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
}
emit logDatabaseUpdated();
});
@ -426,10 +421,10 @@ void LogEngine::removeDeviceLogs(const DeviceId &deviceId)
QString queryDeleteString = QString("DELETE FROM entries WHERE deviceId = '%1';").arg(deviceId.toString());
DatabaseJob *job = new DatabaseJob(queryDeleteString, m_db);
DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job, deviceId](){
if (job->query().lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting log entries from device" << deviceId.toString() << ". Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting log entries from device" << deviceId.toString() << ". Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
} else {
emit logDatabaseUpdated();
}
@ -444,12 +439,12 @@ void LogEngine::removeRuleLogs(const RuleId &ruleId)
QString queryDeleteString = QString("DELETE FROM entries WHERE typeId = '%1';").arg(ruleId.toString());
DatabaseJob *job = new DatabaseJob(queryDeleteString, m_db);
DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job, ruleId](){
if (job->query().lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting log entries from rule" << ruleId.toString() << ". Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting log entries from rule" << ruleId.toString() << ". Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
} else {
emit logDatabaseUpdated();
}
@ -471,12 +466,12 @@ void LogEngine::appendLogEntry(const LogEntry &entry)
.arg(entry.active())
.arg(entry.errorCode());
DatabaseJob *job = new DatabaseJob(queryString, m_db);
DatabaseJob *job = new DatabaseJob(m_db, queryString);
connect(job, &DatabaseJob::finished, this, [this, job, entry](){
if (job->query().lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error writing log entry. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error writing log entry. Driver error:" << job->error().driverText() << "Database error:" << job->error().number() << job->error().databaseText();
qCWarning(dcLogEngine) << entry;
m_dbMalformed = true;
processQueue();
@ -502,20 +497,18 @@ void LogEngine::checkDBSize()
QDateTime startTime = QDateTime::currentDateTime();
QString queryString = "SELECT COUNT(*) FROM entries;";
DatabaseJob *job = new DatabaseJob(queryString, m_db);
DatabaseJob *job = new DatabaseJob(m_db, queryString);
connect(job, &DatabaseJob::finished, this, [this, job, startTime](){
QSqlQuery result = job->query();
if (m_db.lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Failed to query entry count in db:" << m_db.lastError().databaseText();
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Failed to query entry count in db:" << job->error().databaseText();
return;
}
if (!result.first()) {
if (job->results().isEmpty()) {
qCWarning(dcLogEngine()) << "Failed retrieving entry count.";
return;
}
m_entryCount = result.value(0).toInt();
m_entryCount = job->results().first().value(0).toInt();
if (m_entryCount <= m_dbMaxSize) {
return;
@ -528,11 +521,11 @@ void LogEngine::checkDBSize()
}
QString queryDeleteString = QString("DELETE FROM entries WHERE ROWID IN (SELECT ROWID FROM entries ORDER BY timestamp DESC LIMIT -1 OFFSET %1);").arg(QString::number(m_dbMaxSize));
DatabaseJob *deleteJob = new DatabaseJob(queryDeleteString, m_db);
DatabaseJob *deleteJob = new DatabaseJob(m_db, queryDeleteString);
connect(deleteJob, &DatabaseJob::finished, this, [this, deleteJob,startTime](){
if (deleteJob->query().lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting oldest log entries to keep size. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
if (deleteJob->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting oldest log entries to keep size. Driver error:" << deleteJob->error().driverText() << "Database error:" << deleteJob->error().databaseText();
}
m_entryCount = m_dbMaxSize;
qCDebug(dcLogEngine()) << "Ran housekeeping on log database in" << startTime.msecsTo(QDateTime::currentDateTime()) << "ms.";
@ -573,8 +566,25 @@ void LogEngine::processQueue()
m_currentJob = job;
QFuture<DatabaseJob*> future = QtConcurrent::run([job](){
job->query().exec();
return job;
QSqlQuery query(job->m_db);
query.prepare(job->m_queryString);
foreach (const QString &value, job->m_bindValues) {
query.addBindValue(LogValueTool::serializeValue(value));
}
query.exec();
job->m_error = query.lastError();
job->m_executedQuery = query.executedQuery();
if (!query.lastError().isValid()) {
while (query.next()) {
job->m_results.append(query.record());
}
}
return job;
});
m_jobWatcher.setFuture(future);
@ -752,6 +762,7 @@ bool LogEngine::initDB(const QString &username, const QString &password)
}
if (!m_db.tables().contains("entries")) {
qCDebug(dcLogEngine()) << "No \"entries\" table in database. Creating it.";
m_db.exec("CREATE TABLE entries "
"("
"timestamp int,"

28
libnymea-core/logging/logengine.h
View File

@ -33,6 +33,8 @@
#include <QObject>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QSqlError>
#include <QSqlRecord>
#include <QTimer>
#include <QFutureWatcher>
@ -106,22 +108,30 @@ class DatabaseJob: public QObject
{
Q_OBJECT
public:
DatabaseJob(const QString &queryString, const QSqlDatabase &db) {
m_query = QSqlQuery(db);
m_query.prepare(queryString);
DatabaseJob(const QSqlDatabase &db, const QString &queryString, const QStringList &bindValues = QStringList()):
m_db(db),
m_queryString(queryString),
m_bindValues(bindValues)
{
}
// IMPORTANT: Make sure it only prepare()d but not executed
// QSQlQuery(QString, QSqlDatabase) implicitly executes!
DatabaseJob(const QSqlQuery &query): m_query(query) {}
QSqlQuery query() const { return m_query; }
QString executedQuery() const { return m_executedQuery; }
QSqlError error() const { return m_error; }
QList<QSqlRecord> results() const { return m_results; }
signals:
void finished();
private:
QSqlQuery m_query;
QSqlDatabase m_db;
QString m_queryString;
QStringList m_bindValues;
QString m_executedQuery;
QSqlError m_error;
QList<QSqlRecord> m_results;
friend class LogEngine;
};
class LogEntriesFetchJob: public QObject