add database migration and variant serialization for log db

libguh-core/logging/logengine.cpp

@@ -124,8 +124,11 @@
 #include <QMetaEnum>
 #include <QDateTime>
 #include <QFileInfo>
+#include <QTime>
+#include <QBuffer>
+#include <QDataStream>
 
-#define DB_SCHEMA_VERSION 2
+#define DB_SCHEMA_VERSION 3
 
 namespace guhserver {
 
@@ -208,7 +211,7 @@ QList<LogEntry> LogEngine::logEntries(const LogFilter &filter) const
                     query.value("errorCode").toInt());
         entry.setTypeId(query.value("typeId").toUuid());
         entry.setDeviceId(DeviceId(query.value("deviceId").toString()));
-        entry.setValue(query.value("value").toString());
+        entry.setValue(LogEngine::convertVariantToString(deserializeValue(query.value("value").toString())));
         entry.setEventType((Logging::LoggingEventType)query.value("loggingEventType").toInt());
         entry.setActive(query.value("active").toBool());
         results.append(entry);
@@ -254,12 +257,12 @@ void LogEngine::logEvent(const Event &event)
         sourceType = Logging::LoggingSourceStates;
         // There should only be one param
         if (!event.params().isEmpty())
-            valueList << event.params().first().value().toString();
+            valueList << serializeValue(event.params().first().value());
 
     } else {
         sourceType = Logging::LoggingSourceEvents;
         foreach (const Param &param, event.params()) {
-            valueList << param.value().toString();
+            valueList << serializeValue(param.value());
         }
     }
 
@@ -435,7 +438,119 @@ void LogEngine::rotate(const QString &dbName)
     if (!f.rename(QString("%1.%2").arg(dbName).arg(index))) {
         qCWarning(dcLogEngine()) << "Error backing up old database.";
     } else {
-        qCDebug(dcLogEngine()) << "Successfully moved old databse";
+        qCDebug(dcLogEngine()) << "Successfully moved old database";
+    }
+}
+
+QString LogEngine::serializeValue(const QVariant &value)
+{
+    QByteArray byteArray;
+    QBuffer writeBuffer(&byteArray);
+    writeBuffer.open(QIODevice::WriteOnly);
+    QDataStream out(&writeBuffer);
+    out << value;
+    writeBuffer.close();
+    return QString(byteArray.toBase64());
+}
+
+QVariant LogEngine::deserializeValue(const QString &serializedValue)
+{
+    QByteArray data = QByteArray::fromBase64(serializedValue.toUtf8());
+    QBuffer readBuffer(&data);
+    readBuffer.open(QIODevice::ReadOnly);
+    QDataStream inputStream(&readBuffer);
+    QVariant value;
+    inputStream >> value;
+    readBuffer.close();
+    return value;
+}
+
+bool LogEngine::migrateDatabaseVersion2to3()
+{
+    // Changelog: serialize values of logentries in order to prevent typecast errors
+    qCDebug(dcLogEngine()) << "Start migration of log database from version 2 to version 3";
+
+    QDateTime startTime = QDateTime::currentDateTime();
+
+    int migrationCounter = 0;
+    int migrationProgress = 0;
+    int entryCount = 0;
+
+    // Count entries we have to migrate
+    QString queryString = "SELECT COUNT(*) FROM entries;";
+    QSqlQuery countQuery = m_db.exec(queryString);
+    if (m_db.lastError().type() != QSqlError::NoError) {
+        qCWarning(dcLogEngine()) << "Failed to query entry count in db:" << m_db.lastError().databaseText();
+        return false;
+    }
+    if (!countQuery.first()) {
+        qCWarning(dcLogEngine()) << "Migration: Failed retrieving entry count.";
+        return false;
+    }
+    entryCount = countQuery.value(0).toInt();
+
+    // Select all entries
+    QSqlQuery selectQuery = m_db.exec("SELECT * FROM entries;");
+    if (m_db.lastError().isValid()) {
+        qCWarning(dcLogEngine) << "Error migrating database verion 2 -> 3. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
+        return false;
+    }
+
+    // Migrate all selected entries
+    while (selectQuery.next()) {
+        QString oldValue = selectQuery.value("value").toString();
+        QString newValue = serializeValue(QVariant(oldValue));
+        if (oldValue.isEmpty())
+            continue;
+
+        QString updateCall = QString("UPDATE entries SET value = '%1' WHERE timestamp = '%2' AND loggingLevel = '%3' AND sourceType = '%4' AND errorCode = '%5' AND typeId = '%6' AND deviceId = '%7' AND value = '%8' AND loggingEventType = '%9'AND active = '%10';")
+                .arg(newValue)
+                .arg(selectQuery.value("timestamp").toLongLong())
+                .arg(selectQuery.value("loggingLevel").toInt())
+                .arg(selectQuery.value("sourceType").toInt())
+                .arg(selectQuery.value("errorCode").toInt())
+                .arg(selectQuery.value("typeId").toUuid().toString())
+                .arg(selectQuery.value("deviceId").toString())
+                .arg(selectQuery.value("value").toString())
+                .arg(selectQuery.value("loggingEventType").toInt())
+                .arg(selectQuery.value("active").toBool());
+
+        m_db.exec(updateCall);
+        if (m_db.lastError().isValid()) {
+            qCWarning(dcLogEngine) << "Error migrating database verion 2 -> 3. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
+            return false;
+        }
+
+        migrationCounter++;
+
+        double percentage = migrationCounter * 100.0 / entryCount;
+        if (qRound(percentage) != migrationProgress) {
+            migrationProgress = qRound(percentage);
+            qCDebug(dcLogEngine()) << QString("Migration progress: %1\%").arg(migrationProgress);
+        }
+    }
+
+    qCDebug(dcLogEngine()) << "Migration of" << migrationCounter << "done in" << QTime().addMSecs(startTime.msecsTo(QDateTime::currentDateTime())).toString("mm:ss.zzz");
+    qCDebug(dcLogEngine()) << "Updating database version to" << DB_SCHEMA_VERSION;
+    m_db.exec(QString("UPDATE metadata SET data = %1 WHERE key = 'version';").arg(DB_SCHEMA_VERSION));
+    if (m_db.lastError().isValid()) {
+        qCWarning(dcLogEngine) << "Error updating database verion 2 -> 3. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
+        return false;
+    }
+
+    qCDebug(dcLogEngine()) << "Migrated" << migrationCounter << "entries from database verion 2 -> 3 successfully.";
+    return true;
+}
+
+QString LogEngine::convertVariantToString(QVariant value)
+{
+    switch (value.type()) {
+    case QVariant::Double:
+        return QString::number(value.toDouble());
+        break;
+    default:
+        return value.toString();
+        break;
     }
 }
 
@@ -444,7 +559,6 @@ bool LogEngine::initDB()
     m_db.close();
     m_db.open();
 
-
     if (!m_db.tables().contains("metadata")) {
         m_db.exec("CREATE TABLE metadata (key varchar(10), data varchar(40));");
         m_db.exec(QString("INSERT INTO metadata (key, data) VALUES('version', '%1');").arg(DB_SCHEMA_VERSION));
@@ -453,6 +567,13 @@ bool LogEngine::initDB()
     QSqlQuery query = m_db.exec("SELECT data FROM metadata WHERE key = 'version';");
     if (query.next()) {
         int version = query.value("data").toInt();
+        if (DB_SCHEMA_VERSION == 3 && version == 2) {
+            if (!migrateDatabaseVersion2to3()) {
+                qCWarning(dcLogEngine()) << "Migration process failed.";
+                return false;
+            }
+        }
+
         if (version != DB_SCHEMA_VERSION) {
             qCWarning(dcLogEngine) << "Log schema version not matching! Schema upgrade not implemented yet. Logging might fail.";
         } else {
@@ -485,19 +606,19 @@ bool LogEngine::initDB()
 
     if (!m_db.tables().contains("entries")) {
         m_db.exec("CREATE TABLE entries "
-                   "("
+                  "("
-                   "timestamp int,"
+                  "timestamp int,"
-                   "loggingLevel int,"
+                  "loggingLevel int,"
-                   "sourceType int,"
+                  "sourceType int,"
-                   "typeId varchar(38),"
+                  "typeId varchar(38),"
-                   "deviceId varchar(38),"
+                  "deviceId varchar(38),"
-                   "value varchar(100),"
+                  "value varchar(100),"
-                   "loggingEventType int,"
+                  "loggingEventType int,"
-                   "active bool,"
+                  "active bool,"
-                   "errorCode int,"
+                  "errorCode int,"
-                   "FOREIGN KEY(sourceType) REFERENCES sourceTypes(id),"
+                  "FOREIGN KEY(sourceType) REFERENCES sourceTypes(id),"
-                   "FOREIGN KEY(loggingEventType) REFERENCES loggingEventTypes(id)"
+                  "FOREIGN KEY(loggingEventType) REFERENCES loggingEventTypes(id)"
-                   ");");
+                  ");");
 
         if (m_db.lastError().isValid()) {
             qCWarning(dcLogEngine) << "Error creating log table in database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();

libguh-core/logging/logengine.h

@@ -67,6 +67,12 @@ private:
     void appendLogEntry(const LogEntry &entry);
     void rotate(const QString &dbName);
 
+    static QString serializeValue(const QVariant &value);
+    static QVariant deserializeValue(const QString &serializedValue);
+
+    bool migrateDatabaseVersion2to3();
+    static QString convertVariantToString(QVariant value);
+
 private slots:
     void checkDBSize();
 

tests/auto/logging/testlogging.cpp

@@ -56,6 +56,8 @@ private slots:
 
     void deviceLogs();
 
+    void testDoubleValues();
+
     void testHouseKeeping();
 
     // this has to be the last test
@@ -404,6 +406,103 @@ void TestLogging::deviceLogs()
 
 }
 
+void TestLogging::testDoubleValues()
+{
+    QCOMPARE(enableNotifications(), true);
+
+    // Add display pin device which contains a double value
+
+    // Discover device
+    QVariantList discoveryParams;
+    QVariantMap resultCountParam;
+    resultCountParam.insert("paramTypeId", resultCountParamTypeId);
+    resultCountParam.insert("value", 1);
+    discoveryParams.append(resultCountParam);
+
+    QVariantMap params;
+    params.insert("deviceClassId", mockDisplayPinDeviceClassId);
+    params.insert("discoveryParams", discoveryParams);
+    QVariant response = injectAndWait("Devices.GetDiscoveredDevices", params);
+
+    verifyDeviceError(response, DeviceManager::DeviceErrorNoError);
+
+    // Pair device
+    DeviceDescriptorId descriptorId = DeviceDescriptorId(response.toMap().value("params").toMap().value("deviceDescriptors").toList().first().toMap().value("id").toString());
+    params.clear();
+    params.insert("deviceClassId", mockDisplayPinDeviceClassId);
+    params.insert("name", "Display pin mock device");
+    params.insert("deviceDescriptorId", descriptorId.toString());
+    response = injectAndWait("Devices.PairDevice", params);
+
+    verifyDeviceError(response);
+
+    PairingTransactionId pairingTransactionId(response.toMap().value("params").toMap().value("pairingTransactionId").toString());
+    QString displayMessage = response.toMap().value("params").toMap().value("displayMessage").toString();
+
+    qDebug() << "displayMessage" << displayMessage;
+
+    params.clear();
+    params.insert("pairingTransactionId", pairingTransactionId.toString());
+    params.insert("secret", "243681");
+    response = injectAndWait("Devices.ConfirmPairing", params);
+
+    verifyDeviceError(response);
+
+    DeviceId deviceId(response.toMap().value("params").toMap().value("deviceId").toString());
+
+    QSignalSpy notificationSpy(m_mockTcpServer, SIGNAL(outgoingData(QUuid,QByteArray)));
+
+    // Set the double state value and sniff for LogEntryAdded notification
+    double value = 23.80;
+    QVariantMap actionParam;
+    actionParam.insert("paramTypeId", doubleStateParamTypeId.toString());
+    actionParam.insert("value", value);
+
+    params.clear(); response.clear();
+    params.insert("deviceId", deviceId);
+    params.insert("actionTypeId", doubleStateParamTypeId.toString());
+    params.insert("params", QVariantList() << actionParam);
+
+    response = injectAndWait("Actions.ExecuteAction", params);
+    verifyDeviceError(response);
+
+    notificationSpy.wait(1000);
+    QVariantList logNotificationsList = checkNotifications(notificationSpy, "Logging.LogEntryAdded");
+    QVERIFY2(!logNotificationsList.isEmpty(), "Did not get Logging.LogEntryAdded notification.");
+
+    foreach (const QVariant &logNotificationVariant, logNotificationsList) {
+        QVariantMap logNotification = logNotificationVariant.toMap().value("params").toMap().value("logEntry").toMap();
+
+        printJson(logNotification);
+
+        if (logNotification.value("typeId").toString() == doubleStateParamTypeId.toString()) {
+            if (logNotification.value("typeId").toString() == doubleStateParamTypeId.toString()) {
+
+                // If state source
+                if (logNotification.value("source").toString() == JsonTypes::loggingSourceToString(Logging::LoggingSourceStates)) {
+                    QString logValue = logNotification.value("value").toString();
+                    qDebug() << QString::number(value) << logValue;
+                    QCOMPARE(logValue, QString::number(value));
+                }
+
+                // If action source notification
+                if (logNotification.value("source").toString() == JsonTypes::loggingSourceToString(Logging::LoggingSourceActions)) {
+                    QString logValue = logNotification.value("value").toString();
+                    qDebug() << QString::number(value) << logValue;
+                    QCOMPARE(logValue, QString::number(value));
+                }
+            }
+        }
+    }
+
+
+    // Remove device
+    params.clear();
+    params.insert("deviceId", deviceId.toString());
+    response = injectAndWait("Devices.RemoveConfiguredDevice", params);
+    verifyDeviceError(response);
+}
+
 void TestLogging::testHouseKeeping()
 {
     QVariantMap params;