Add support for configuring the used database backend

libnymea-core/debugserverhandler.cpp

@@ -228,12 +228,6 @@ QByteArray DebugServerHandler::createDebugXmlDocument()
     writer.writeTextElement("td", NymeaSettings(NymeaSettings::SettingsRoleGlobal).translationsPath());
     writer.writeEndElement(); // tr
 
-    writer.writeStartElement("tr");
-    //: The log database path description in the server infromation section of the debug interface
-    writer.writeTextElement("th", tr("Log database"));
-    writer.writeTextElement("td", NymeaSettings(NymeaSettings::SettingsRoleGlobal).logPath());
-    writer.writeEndElement(); // tr
-
     for (int i = 0; i < NymeaCore::instance()->deviceManager()->pluginSearchDirs().count(); i++) {
         writer.writeStartElement("tr");
         writer.writeEndElement(); // tr
@@ -271,10 +265,12 @@ QByteArray DebugServerHandler::createDebugXmlDocument()
     writer.writeTextElement("p", tr("Log database"));
     writer.writeEndElement(); // div download-name-column
 
-    writer.writeStartElement("div");
+    if (QFileInfo(NymeaCore::instance()->configuration()->logDBName()).exists()) {
-    writer.writeAttribute("class", "download-path-column");
+        writer.writeStartElement("div");
-    writer.writeTextElement("p", NymeaSettings::logPath());
+        writer.writeAttribute("class", "download-path-column");
-    writer.writeEndElement(); // div download-path-column
+        writer.writeTextElement("p", NymeaCore::instance()->configuration()->logDBName());
+        writer.writeEndElement(); // div download-path-column
+    }
 
     writer.writeStartElement("div");
     writer.writeAttribute("class", "download-button-column");
@@ -643,10 +639,10 @@ HttpReply *DebugServerHandler::processDebugRequest(const QString &requestPath)
 
     // Check if this is a logdb requested
     if (requestPath.startsWith("/debug/logdb.sql")) {
-        qCDebug(dcWebServer()) << "Loading" << NymeaSettings::logPath();
+        qCDebug(dcWebServer()) << "Loading" << NymeaCore::instance()->configuration()->logDBName();
-        QFile logDatabaseFile(NymeaSettings::logPath());
+        QFile logDatabaseFile(NymeaCore::instance()->configuration()->logDBName());
         if (!logDatabaseFile.exists()) {
-            qCWarning(dcWebServer()) << "Could not read log database file for debug download" << NymeaSettings::logPath() << "file does not exist.";
+            qCWarning(dcWebServer()) << "Could not read log database file for debug download" << NymeaCore::instance()->configuration()->logDBName() << "file does not exist.";
             HttpReply *reply = RestResource::createErrorReply(HttpReply::NotFound);
             reply->setHeader(HttpReply::ContentTypeHeader, "text/html");
             //: The HTTP error message of the debug interface. The %1 represents the file name.
@@ -655,7 +651,7 @@ HttpReply *DebugServerHandler::processDebugRequest(const QString &requestPath)
         }
 
         if (!logDatabaseFile.open(QFile::ReadOnly)) {
-            qCWarning(dcWebServer()) << "Could not read log database file for debug download" << NymeaSettings::logPath();
+            qCWarning(dcWebServer()) << "Could not read log database file for debug download" << NymeaCore::instance()->configuration()->logDBName();
             HttpReply *reply = RestResource::createErrorReply(HttpReply::Forbidden);
             reply->setHeader(HttpReply::ContentTypeHeader, "text/html");
             //: The HTTP error message of the debug interface. The %1 represents the file name.

libnymea-core/logging/logengine.cpp

@@ -131,13 +131,18 @@
 
 namespace nymeaserver {
 
-/*! Constructs the log engine with the given \a parent. */
+/*! Constructs the log engine with the given parameters.
-LogEngine::LogEngine(const QString &logPath, QObject *parent):
+    \a The Qt Database backend to be used. Depending on the installed Qt modules this can be any of QDB2 QIBASE QMYSQL QOCI QODBC QPSQL QSQLITE QSQLITE2 QTDS.
-    QObject(parent)
+    \a dbName is the name of the database. In case of SQLITE this should contain a file path. The Driver will create the file if required. In case of using a
+    database server like MYSQL, the database must exist on the host given by \a hostname and be accessible with the given \a username and \a password.
+*/
+LogEngine::LogEngine(const QString &driver, const QString &dbName, const QString &hostname, const QString &username, const QString &password, int maxDBSize, QObject *parent):
+    QObject(parent),
+    m_dbMaxSize(maxDBSize)
 {
-    m_db = QSqlDatabase::addDatabase("QSQLITE", "logs");
+    m_db = QSqlDatabase::addDatabase(driver, "logs");
-    m_db.setDatabaseName(logPath);
+    m_db.setDatabaseName(dbName);
-    m_dbMaxSize = 50000;
+    m_db.setHostName(hostname);
     m_overflow = 100;
 
     if (QCoreApplication::instance()->organizationName() == "nymea-test") {
@@ -151,16 +156,14 @@ LogEngine::LogEngine(const QString &logPath, QObject *parent):
         qCWarning(dcLogEngine) << "Database not valid:" << m_db.lastError().driverText() << m_db.lastError().databaseText();
         rotate(m_db.databaseName());
     }
-    if (!m_db.open()) {
-        qCWarning(dcLogEngine) << "Error opening log database:" << m_db.lastError().driverText() << m_db.lastError().databaseText();
-        rotate(m_db.databaseName());
-    }
 
-    if (!initDB()) {
+    if (!initDB(username, password)) {
         qCWarning(dcLogEngine()) << "Error initializing database. Trying to correct it.";
-        rotate(m_db.databaseName());
+        if (QFileInfo(m_db.databaseName()).exists()) {
-        if (!initDB()) {
+            rotate(m_db.databaseName());
-            qCWarning(dcLogEngine()) << "Error fixing log database. Giving up. Logs can't be stored.";
+            if (!initDB(username, password)) {
+                qCWarning(dcLogEngine()) << "Error fixing log database. Giving up. Logs can't be stored.";
+            }
         }
     }
 
@@ -405,6 +408,10 @@ void LogEngine::appendLogEntry(const LogEntry &entry)
 
 void LogEngine::checkDBSize()
 {
+    if (m_dbMaxSize == -1) {
+        // No tripping required
+        return;
+    }
     QDateTime startTime = QDateTime::currentDateTime();
     QString queryString = "SELECT COUNT(*) FROM entries;";
     QSqlQuery result = m_db.exec(queryString);
@@ -529,17 +536,26 @@ bool LogEngine::migrateDatabaseVersion2to3()
     return true;
 }
 
-bool LogEngine::initDB()
+bool LogEngine::initDB(const QString &username, const QString &password)
 {
     m_db.close();
-    m_db.open();
+    bool opened = m_db.open(username, password);
-
+    if (!opened) {
-    if (!m_db.tables().contains("metadata")) {
+        qCWarning(dcLogEngine()) << "Can't open Log DB. Init failed.";
-        m_db.exec("CREATE TABLE metadata (key varchar(10), data varchar(40));");
+        return false;
-        m_db.exec(QString("INSERT INTO metadata (key, data) VALUES('version', '%1');").arg(DB_SCHEMA_VERSION));
     }
 
-    QSqlQuery query = m_db.exec("SELECT data FROM metadata WHERE key = 'version';");
+    if (!m_db.tables().contains("metadata")) {
+        qCDebug(dcLogEngine()) << "Empty Database. Setting up metadata...";
+        m_db.exec("CREATE TABLE metadata (`key` VARCHAR(10), data VARCHAR(40));");
+        if (m_db.lastError().isValid()) {
+            qCWarning(dcLogEngine) << "Error initualizing database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
+            return false;
+        }
+        m_db.exec(QString("INSERT INTO metadata (`key`, data) VALUES('version', '%1');").arg(DB_SCHEMA_VERSION));
+    }
+
+    QSqlQuery query = m_db.exec("SELECT data FROM metadata WHERE `key` = 'version';");
     if (query.next()) {
         int version = query.value("data").toInt();
 
@@ -575,12 +591,15 @@ bool LogEngine::initDB()
     }
 
     if (!m_db.tables().contains("loggingEventTypes")) {
-        m_db.exec("CREATE TABLE loggingEventTypes (id int, name varchar(20), PRIMARY KEY(id));");
+        m_db.exec("CREATE TABLE loggingEventTypes (id int, name varchar(40), PRIMARY KEY(id));");
         //qCDebug(dcLogEngine) << m_db.lastError().databaseText();
         QMetaEnum logTypes = Logging::staticMetaObject.enumerator(Logging::staticMetaObject.indexOfEnumerator("LoggingEventType"));
         Q_ASSERT_X(logTypes.isValid(), "LogEngine", "Logging has no enum LoggingEventType");
         for (int i = 0; i < logTypes.keyCount(); i++) {
             m_db.exec(QString("INSERT INTO loggingEventTypes (id, name) VALUES(%1, '%2');").arg(i).arg(logTypes.key(i)));
+            if (m_db.lastError().isValid()) {
+                qCWarning(dcLogEngine()) << "Failed to insert loggingEventTypes into DB. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
+            }
         }
     }
 

libnymea-core/logging/logengine.h

@@ -27,7 +27,6 @@
 #include "types/event.h"
 #include "types/action.h"
 #include "rule.h"
-#include "nymeasettings.h"
 
 #include <QObject>
 #include <QSqlDatabase>
@@ -39,7 +38,7 @@ class LogEngine: public QObject
 {
     Q_OBJECT
 public:
-    LogEngine(const QString &logPath = NymeaSettings::logPath(), QObject *parent = 0);
+    LogEngine(const QString &driver, const QString &dbName, const QString &hostname = QString("127.0.0.1"), const QString &username = QString(), const QString &password = QString(), int maxDBSize = 50000, QObject *parent = 0);
     ~LogEngine();
 
     QList<LogEntry> logEntries(const LogFilter &filter = LogFilter()) const;
@@ -64,7 +63,7 @@ signals:
     void logDatabaseUpdated();
 
 private:
-    bool initDB();
+    bool initDB(const QString &username, const QString &password);
     void appendLogEntry(const LogEntry &entry);
     void rotate(const QString &dbName);
 

libnymea-core/nymeaconfiguration.cpp

@@ -25,6 +25,7 @@
 #include <QTimeZone>
 #include <QCoreApplication>
 #include <QFile>
+#include <QDir>
 
 namespace nymeaserver {
 
@@ -158,6 +159,16 @@ NymeaConfiguration::NymeaConfiguration(QObject *parent) :
         m_webSocketServerConfigs[config.id] = config;
         storeServerConfig("WebSocketServer", config);
     }
+
+    // Write defaults for log settings
+    settings.beginGroup("Logs");
+    settings.setValue("logDBDriver", logDBDriver());
+    settings.setValue("logDBName", logDBName());
+    settings.setValue("logDBHost", logDBHost());
+    settings.setValue("logDBUser", logDBUser());
+    settings.setValue("logDBPassword", logDBPassword());
+    settings.setValue("logDBMaxEntries", logDBMaxEntries());
+    settings.endGroup();
 }
 
 QUuid NymeaConfiguration::serverUuid() const
@@ -368,6 +379,61 @@ QString NymeaConfiguration::cloudCertificateKey() const
     return settings.value("cloudCertificateKey").toString();
 }
 
+QString NymeaConfiguration::logDBDriver() const
+{
+    NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
+    settings.beginGroup("Logs");
+    return settings.value("logDBDriver", "QSQLITE").toString();
+}
+
+QString NymeaConfiguration::logDBHost() const
+{
+    NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
+    settings.beginGroup("Logs");
+    return settings.value("logDBHost", "127.0.0.1").toString();
+}
+
+QString NymeaConfiguration::logDBName() const
+{
+    QString defaultLogPath;
+    QString organisationName = QCoreApplication::instance()->organizationName();
+
+    if (!qgetenv("SNAP").isEmpty()) {
+        defaultLogPath = QString(qgetenv("SNAP_COMMON")) + "/nymead.sqlite";
+    } else if (organisationName == "nymea-test") {
+        defaultLogPath = "/tmp/" + organisationName + "/nymead-test.sqlite";
+    } else if (NymeaSettings::isRoot()) {
+        defaultLogPath = "/var/log/nymead.sqlite";
+    } else {
+        defaultLogPath = QDir::homePath() + "/.config/" + organisationName + "/nymead.sqlite";
+    }
+
+    NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
+    settings.beginGroup("Logs");
+    return settings.value("logDBName", defaultLogPath).toString();
+}
+
+QString NymeaConfiguration::logDBUser() const
+{
+    NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
+    settings.beginGroup("Logs");
+    return settings.value("logDBUser").toString();
+}
+
+QString NymeaConfiguration::logDBPassword() const
+{
+    NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
+    settings.beginGroup("Logs");
+    return settings.value("logDBPassword").toString();
+}
+
+int NymeaConfiguration::logDBMaxEntries() const
+{
+    NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
+    settings.beginGroup("Logs");
+    return settings.value("logDBMaxEntries", 200000).toInt();
+}
+
 QString NymeaConfiguration::sslCertificate() const
 {
     NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);

libnymea-core/nymeaconfiguration.h

@@ -123,6 +123,14 @@ public:
     QString cloudCertificate() const;
     QString cloudCertificateKey() const;
 
+    // Logging
+    QString logDBDriver() const;
+    QString logDBName() const;
+    QString logDBHost() const;
+    QString logDBUser() const;
+    QString logDBPassword() const;
+    int logDBMaxEntries() const;
+
 private:
     QHash<QString, ServerConfiguration> m_tcpServerConfigs;
     QHash<QString, WebServerConfiguration> m_webServerConfigs;

libnymea-core/nymeacore.cpp

@@ -502,7 +502,7 @@ void NymeaCore::init() {
     m_timeManager = new TimeManager(QTimeZone::systemTimeZoneId(), this);
 
     qCDebug(dcApplication) << "Creating Log Engine";
-    m_logger = new LogEngine(NymeaSettings::logPath(), this);
+    m_logger = new LogEngine(m_configuration->logDBDriver(), m_configuration->logDBName(), m_configuration->logDBHost(), m_configuration->logDBUser(), m_configuration->logDBPassword(), m_configuration->logDBMaxEntries(), this);
 
     qCDebug(dcApplication) << "Creating Hardware Manager";
     m_hardwareManager = new HardwareManagerImplementation(this);

libnymea-core/servermanager.cpp

@@ -36,6 +36,7 @@
 #include "servermanager.h"
 #include "nymeacore.h"
 #include "certificategenerator.h"
+#include "nymeasettings.h"
 
 #include <QSslCertificate>
 #include <QSslConfiguration>

libnymea/nymeasettings.cpp

@@ -125,28 +125,6 @@ bool NymeaSettings::isRoot()
     return true;
 }
 
-/*! Returns the path where the logging database will be stored.
-
-  \sa nymeaserver::LogEngine
-*/
-QString NymeaSettings::logPath()
-{
-    QString logPath;
-    QString organisationName = QCoreApplication::instance()->organizationName();
-
-    if (!qgetenv("SNAP").isEmpty()) {
-        logPath = QString(qgetenv("SNAP_COMMON")) + "/nymead.sqlite";
-    } else if (organisationName == "nymea-test") {
-        logPath = "/tmp/" + organisationName + "/nymead-test.sqlite";
-    } else if (NymeaSettings::isRoot()) {
-        logPath = "/var/log/nymead.sqlite";
-    } else {
-        logPath = QDir::homePath() + "/.config/" + organisationName + "/nymead.sqlite";
-    }
-
-    return logPath;
-}
-
 /*! Returns the path to the folder where the NymeaSettings will be saved i.e. \tt{/etc/nymea}. */
 QString NymeaSettings::settingsPath()
 {

libnymea/nymeasettings.h

@@ -49,7 +49,6 @@ public:
     SettingsRole settingsRole() const;
 
     static bool isRoot();
-    static QString logPath();
     static QString settingsPath();
     static QString translationsPath();
     static QString storagePath();

tests/auto/configurations/testconfigurations.cpp

@@ -23,6 +23,7 @@
 #include "devicemanager.h"
 #include "mocktcpserver.h"
 #include "nymeacore.h"
+#include "nymeasettings.h"
 
 #include <QtTest/QtTest>
 #include <QCoreApplication>

tests/auto/loggingdirect/testloggingdirect.cpp

@@ -35,11 +35,12 @@ private slots:
     void benchmarkDB();
 
 private:
-    LogEngine engine;
+    LogEngine *engine;
 };
 
 TestLoggingDirect::TestLoggingDirect(QObject *parent): QObject(parent)
 {
+    engine = new LogEngine("QSQLITE", "/tmp/nymea-test/nymea.sqlite");
     // Setting timeout to 20 mins
     qputenv("QTEST_FUNCTION_TIMEOUT", "1200000");
     QCoreApplication::instance()->setOrganizationName("nymea-test");
@@ -73,25 +74,25 @@ void TestLoggingDirect::benchmarkDB()
 
     // setting max log entries to "prefill" to trim it down to what this test needs.
     int overflow = 10;
-    engine.setMaxLogEntries(prefill, overflow);
+    engine->setMaxLogEntries(prefill, overflow);
-    engine.setMaxLogEntries(maxSize, overflow);
+    engine->setMaxLogEntries(maxSize, overflow);
 
-    for (int i = engine.logEntries().count(); i < prefill; i++) {
+    for (int i = engine->logEntries().count(); i < prefill; i++) {
-        engine.logSystemEvent(QDateTime::currentDateTime(), true);
+        engine->logSystemEvent(QDateTime::currentDateTime(), true);
     }
 
-    qDebug() << "Starting benchmark with" << engine.logEntries().count() << "entries in the db";
+    qDebug() << "Starting benchmark with" << engine->logEntries().count() << "entries in the db";
     QBENCHMARK {
-        engine.logSystemEvent(QDateTime::currentDateTime(), true);
+        engine->logSystemEvent(QDateTime::currentDateTime(), true);
     }
     QDateTime now = QDateTime::currentDateTime();
-    while (engine.logEntries().count() > maxSize + overflow) {
+    while (engine->logEntries().count() > maxSize + overflow) {
         qApp->processEvents();
         if (now.addSecs(5) < QDateTime::currentDateTime()) {
-            QVERIFY2(false, QString("Housekeeping didn't work. Have %1 entries but expected to have max %2").arg(engine.logEntries().count()).arg(QString::number(maxSize)).toLocal8Bit());
+            QVERIFY2(false, QString("Housekeeping didn't work. Have %1 entries but expected to have max %2").arg(engine->logEntries().count()).arg(QString::number(maxSize)).toLocal8Bit());
         }
     }
-    qDebug() << "Ended benchmark with" << engine.logEntries().count() << "entries in the db";
+    qDebug() << "Ended benchmark with" << engine->logEntries().count() << "entries in the db";
 }
 
 #include "testloggingdirect.moc"

tests/auto/loggingloading/testloggingloading.cpp

@@ -58,7 +58,7 @@ void TestLoggingLoading::initTestCase()
 void TestLoggingLoading::testLogMigration()
 {
     // Create LogEngine with log db from resource file
-    QString temporaryDbName = NymeaSettings::settingsPath() + "/nymead-v2.sqlite";
+    QString temporaryDbName = "/tmp/nymea-test/nymead-v2.sqlite";
 
     if (QFile::exists(temporaryDbName))
         QVERIFY(QFile(temporaryDbName).remove());
@@ -68,7 +68,7 @@ void TestLoggingLoading::testLogMigration()
     QVERIFY(QFile::copy(":/nymead-v2.sqlite", temporaryDbName));
     QVERIFY(QFile::setPermissions(temporaryDbName, QFile::ReadOwner | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOther));
 
-    LogEngine *logEngine = new LogEngine(temporaryDbName, this);
+    LogEngine *logEngine = new LogEngine("QSQLITE", temporaryDbName);
     // Check there is no rotated logfile
     QVERIFY(!QFile::exists(temporaryDbName + ".1"));
 
@@ -80,8 +80,8 @@ void TestLoggingLoading::testLogMigration()
 void TestLoggingLoading::testLogfileRotation()
 {
     // Create LogEngine with log db from resource file
-    QString temporaryDbName = NymeaSettings::settingsPath() + "/nymead-broken.sqlite";
+    QString temporaryDbName = "/tmp/nymea-test/nymead-broken.sqlite";
-    QString rotatedDbName = NymeaSettings::settingsPath() + "/nymead-broken.sqlite.1";
+    QString rotatedDbName = "/tmp/nymea-test/nymead-broken.sqlite.1";
 
     // Remove the files if there are some left
     if (QFile::exists(temporaryDbName))
@@ -96,7 +96,7 @@ void TestLoggingLoading::testLogfileRotation()
     QVERIFY(QFile::setPermissions(temporaryDbName, QFile::ReadOwner | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOther));
 
     QVERIFY(!QFile::exists(rotatedDbName));
-    LogEngine *logEngine = new LogEngine(temporaryDbName, this);
+    LogEngine *logEngine = new LogEngine("QSQLITE", temporaryDbName);
     QVERIFY(QFile::exists(rotatedDbName));
 
     delete logEngine;