ETM-Schurig/powersync-core
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add database serialization and variant string converting to jsonrpc

Browse Source
This commit is contained in:
Simon Stürz 2017-10-11 13:08:58 +02:00 committed by Michael Zanetti
parent 163402e158
commit ba526eee11
10 changed files with 197 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libguh-core/jsonrpc/jsontypes.cpp
View File

@ -55,6 +55,7 @@
#include "guhcore.h"
#include "ruleengine.h"
#include "loggingcategories.h"
#include "logging/logvaluetool.h"
#include <QStringList>
#include <QJsonDocument>
@ -887,7 +888,7 @@ QVariantMap JsonTypes::packLogEntry(const LogEntry &logEntry)
case Logging::LoggingSourceStates:
logEntryMap.insert("typeId", logEntry.typeId().toString());
logEntryMap.insert("deviceId", logEntry.deviceId().toString());
logEntryMap.insert("value", logEntry.value());
logEntryMap.insert("value", LogValueTool::convertVariantToString(logEntry.value()));
break;
case Logging::LoggingSourceSystem:
logEntryMap.insert("active", logEntry.active());

6
libguh-core/libguh-core.pro
View File

@ -65,7 +65,8 @@ HEADERS += $$top_srcdir/libguh-core/guhcore.h \
$$top_srcdir/libguh-core/networkmanager/wirednetworkdevice.h \
$$top_srcdir/libguh-core/usermanager.h \
$$top_srcdir/libguh-core/tokeninfo.h \
$$PWD/certificategenerator.h
$$PWD/certificategenerator.h \
logging/logvaluetool.h
SOURCES += $$top_srcdir/libguh-core/guhcore.cpp \
@ -118,5 +119,6 @@ SOURCES += $$top_srcdir/libguh-core/guhcore.cpp \
$$top_srcdir/libguh-core/networkmanager/wirednetworkdevice.cpp \
$$top_srcdir/libguh-core/usermanager.cpp \
$$top_srcdir/libguh-core/tokeninfo.cpp \
$$PWD/certificategenerator.cpp
$$PWD/certificategenerator.cpp \
logging/logvaluetool.cpp

84
libguh-core/logging/logengine.cpp
View File

@ -1,7 +1,7 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* *
* Copyright (C) 2014 Michael Zanetti <michael_zanetti@gmx.net> *
* Copyright (C) 2015 Simon Stürz <simon.stuerz@guh.io> *
* Copyright (C) 2015-2017 Simon Stürz <simon.stuerz@guh.io> *
* *
* This file is part of guh. *
* *
@ -114,6 +114,7 @@
#include "logengine.h"
#include "loggingcategories.h"
#include "logging.h"
#include "logvaluetool.h"
#include <QCoreApplication>
#include <QSqlDatabase>
@ -125,8 +126,6 @@
#include <QDateTime>
#include <QFileInfo>
#include <QTime>
#include <QBuffer>
#include <QDataStream>
#define DB_SCHEMA_VERSION 3
@ -211,7 +210,7 @@ QList<LogEntry> LogEngine::logEntries(const LogFilter &filter) const
query.value("errorCode").toInt());
entry.setTypeId(query.value("typeId").toUuid());
entry.setDeviceId(DeviceId(query.value("deviceId").toString()));
entry.setValue(LogEngine::convertVariantToString(deserializeValue(query.value("value").toString())));
entry.setValue(LogValueTool::convertVariantToString(LogValueTool::deserializeValue(query.value("value").toString())));
entry.setEventType((Logging::LoggingEventType)query.value("loggingEventType").toInt());
entry.setActive(query.value("active").toBool());
results.append(entry);
@ -251,38 +250,49 @@ void LogEngine::logSystemEvent(const QDateTime &dateTime, bool active, Logging::
void LogEngine::logEvent(const Event &event)
{
QStringList valueList;
QVariantList valueList;
Logging::LoggingSource sourceType;
if (event.isStateChangeEvent()) {
sourceType = Logging::LoggingSourceStates;
// There should only be one param
if (!event.params().isEmpty())
valueList << serializeValue(event.params().first().value());
valueList << event.params().first().value();
} else {
sourceType = Logging::LoggingSourceEvents;
foreach (const Param &param, event.params()) {
valueList << serializeValue(param.value());
valueList << param.value();
}
}
LogEntry entry(sourceType);
entry.setTypeId(event.eventTypeId());
entry.setDeviceId(event.deviceId());
entry.setValue(valueList.join(", "));
if (valueList.count() == 1) {
entry.setValue(valueList.first());
} else {
entry.setValue(valueList);
}
appendLogEntry(entry);
}
void LogEngine::logAction(const Action &action, Logging::LoggingLevel level, int errorCode)
{
QStringList valueList;
foreach (const Param &param, action.params()) {
valueList << param.value().toString();
}
LogEntry entry(level, Logging::LoggingSourceActions, errorCode);
entry.setTypeId(action.actionTypeId());
entry.setDeviceId(action.deviceId());
entry.setValue(valueList.join(", "));
if (action.params().isEmpty()) {
entry.setValue(QVariant());
} else if (action.params().count() == 1) {
entry.setValue(action.params().first().value());
} else {
QVariantList valueList;
foreach (const Param &param, action.params()) {
valueList << param.value();
}
entry.setValue(valueList);
}
appendLogEntry(entry);
}
@ -379,7 +389,7 @@ void LogEngine::appendLogEntry(const LogEntry &entry)
.arg(entry.source())
.arg(entry.typeId().toString())
.arg(entry.deviceId().toString())
.arg(entry.value())
.arg(LogValueTool::serializeValue(entry.value()))
.arg(entry.active())
.arg(entry.errorCode());
@ -442,29 +452,6 @@ void LogEngine::rotate(const QString &dbName)
}
}
QString LogEngine::serializeValue(const QVariant &value)
{
QByteArray byteArray;
QBuffer writeBuffer(&byteArray);
writeBuffer.open(QIODevice::WriteOnly);
QDataStream out(&writeBuffer);
out << value;
writeBuffer.close();
return QString(byteArray.toBase64());
}
QVariant LogEngine::deserializeValue(const QString &serializedValue)
{
QByteArray data = QByteArray::fromBase64(serializedValue.toUtf8());
QBuffer readBuffer(&data);
readBuffer.open(QIODevice::ReadOnly);
QDataStream inputStream(&readBuffer);
QVariant value;
inputStream >> value;
readBuffer.close();
return value;
}
bool LogEngine::migrateDatabaseVersion2to3()
{
// Changelog: serialize values of logentries in order to prevent typecast errors
@ -477,7 +464,7 @@ bool LogEngine::migrateDatabaseVersion2to3()
int entryCount = 0;
// Count entries we have to migrate
QString queryString = "SELECT COUNT(*) FROM entries;";
QString queryString = "SELECT COUNT(*) FROM entries WHERE value != '';";
QSqlQuery countQuery = m_db.exec(queryString);
if (m_db.lastError().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Failed to query entry count in db:" << m_db.lastError().databaseText();
@ -489,6 +476,8 @@ bool LogEngine::migrateDatabaseVersion2to3()
}
entryCount = countQuery.value(0).toInt();
qCDebug(dcLogEngine()) << "Entries to migrate:" << entryCount;
// Select all entries
QSqlQuery selectQuery = m_db.exec("SELECT * FROM entries;");
if (m_db.lastError().isValid()) {
@ -499,7 +488,7 @@ bool LogEngine::migrateDatabaseVersion2to3()
// Migrate all selected entries
while (selectQuery.next()) {
QString oldValue = selectQuery.value("value").toString();
QString newValue = serializeValue(QVariant(oldValue));
QString newValue = LogValueTool::serializeValue(QVariant(oldValue));
if (oldValue.isEmpty())
continue;
@ -526,11 +515,12 @@ bool LogEngine::migrateDatabaseVersion2to3()
double percentage = migrationCounter * 100.0 / entryCount;
if (qRound(percentage) != migrationProgress) {
migrationProgress = qRound(percentage);
qCDebug(dcLogEngine()) << QString("Migration progress: %1\%").arg(migrationProgress);
qCDebug(dcLogEngine()) << QString("Migration progress: %1\%").arg(migrationProgress).toLatin1().data();
}
}
qCDebug(dcLogEngine()) << "Migration of" << migrationCounter << "done in" << QTime().addMSecs(startTime.msecsTo(QDateTime::currentDateTime())).toString("mm:ss.zzz");
QTime runTime = QTime(0,0,0,0).addMSecs(startTime.msecsTo(QDateTime::currentDateTime()));
qCDebug(dcLogEngine()) << "Migration of" << migrationCounter << "done in" << runTime.toString("mm:ss.zzz");
qCDebug(dcLogEngine()) << "Updating database version to" << DB_SCHEMA_VERSION;
m_db.exec(QString("UPDATE metadata SET data = %1 WHERE key = 'version';").arg(DB_SCHEMA_VERSION));
if (m_db.lastError().isValid()) {
@ -542,18 +532,6 @@ bool LogEngine::migrateDatabaseVersion2to3()
return true;
}
QString LogEngine::convertVariantToString(QVariant value)
{
switch (value.type()) {
case QVariant::Double:
return QString::number(value.toDouble());
break;
default:
return value.toString();
break;
}
}
bool LogEngine::initDB()
{
m_db.close();

3
libguh-core/logging/logengine.h
View File

@ -67,11 +67,8 @@ private:
void appendLogEntry(const LogEntry &entry);
void rotate(const QString &dbName);
static QString serializeValue(const QVariant &value);
static QVariant deserializeValue(const QString &serializedValue);
bool migrateDatabaseVersion2to3();
static QString convertVariantToString(QVariant value);
private slots:
void checkDBSize();

4
libguh-core/logging/logentry.cpp
View File

@ -112,13 +112,13 @@ void LogEntry::setDeviceId(const DeviceId &deviceId)
}
/*! Returns the value of this \l{LogEntry}. */
QString LogEntry::value() const
QVariant LogEntry::value() const
{
return m_value;
}
/*! Sets the \a value of this \l{LogEntry}. */
void LogEntry::setValue(const QString &value)
void LogEntry::setValue(const QVariant &value)
{
m_value = value;
}

7
libguh-core/logging/logentry.h
View File

@ -26,6 +26,7 @@
#include "typeutils.h"
#include <QObject>
#include <QVariant>
#include <QDateTime>
namespace guhserver {
@ -56,8 +57,8 @@ public:
void setDeviceId(const DeviceId &deviceId);
// Valid for LoggingSourceStates
QString value() const;
void setValue(const QString &value);
QVariant value() const;
void setValue(const QVariant &value);
// Valid for LoggingEventTypeActiveChanged
bool active() const;
@ -75,7 +76,7 @@ private:
// FIXME: If it turns out we need many more of those, we should subclass LogEntry with specific ones.
QUuid m_typeId;
DeviceId m_deviceId;
QString m_value;
QVariant m_value;
Logging::LoggingEventType m_eventType;
bool m_active;
int m_errorCode;

2
libguh-core/logging/logfilter.cpp
View File

@ -336,6 +336,8 @@ QString LogFilter::createDeviceIdString() const
QString LogFilter::createValuesString() const
{
// FIXME: check how to filter for serialized values
QString query;
if (!m_values.isEmpty()) {
if (m_values.count() == 1) {

72
libguh-core/logging/logvaluetool.cpp Normal file
View File

@ -0,0 +1,72 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* *
* Copyright (C) 2017 Simon Stürz <simon.stuerz@guh.io> *
* *
* This file is part of guh. *
* *
* Guh is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, version 2 of the License. *
* *
* Guh is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with guh. If not, see <http://www.gnu.org/licenses/>. *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include "logvaluetool.h"
#include <QBuffer>
#include <QByteArray>
#include <QDataStream>
LogValueTool::LogValueTool(QObject *parent) : QObject(parent)
{
}
QString LogValueTool::convertVariantToString(const QVariant &value)
{
switch (value.type()) {
case QVariant::Double:
return QString::number(value.toDouble());
break;
case QVariant::List: {
QStringList valueStringList;
foreach (const QVariant &variantValue, value.toList()) {
valueStringList.append(convertVariantToString(variantValue));
}
return valueStringList.join(", ");
}
default:
return value.toString();
break;
}
}
QString LogValueTool::serializeValue(const QVariant &value)
{
QByteArray byteArray;
QBuffer writeBuffer(&byteArray);
writeBuffer.open(QIODevice::WriteOnly);
QDataStream out(&writeBuffer);
out << value;
writeBuffer.close();
return QString(byteArray.toBase64());
}
QVariant LogValueTool::deserializeValue(const QString &serializedValue)
{
QByteArray data = QByteArray::fromBase64(serializedValue.toUtf8());
QBuffer readBuffer(&data);
readBuffer.open(QIODevice::ReadOnly);
QDataStream inputStream(&readBuffer);
QVariant value;
inputStream >> value;
readBuffer.close();
return value;
}

38
libguh-core/logging/logvaluetool.h Normal file
View File

@ -0,0 +1,38 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* *
* Copyright (C) 2017 Simon Stürz <simon.stuerz@guh.io> *
* *
* This file is part of guh. *
* *
* Guh is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, version 2 of the License. *
* *
* Guh is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with guh. If not, see <http://www.gnu.org/licenses/>. *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef LOGVALUETOOL_H
#define LOGVALUETOOL_H
#include <QObject>
#include <QVariant>
class LogValueTool : public QObject
{
Q_OBJECT
public:
explicit LogValueTool(QObject *parent = nullptr);
static QString convertVariantToString(const QVariant &value);
static QString serializeValue(const QVariant &value);
static QVariant deserializeValue(const QString &serializedValue);
};
#endif // LOGVALUETOOL_H

44
tests/auto/logging/testlogging.cpp
View File

@ -24,6 +24,7 @@
#include "devicemanager.h"
#include "guhsettings.h"
#include "logging/logentry.h"
#include "logging/logvaluetool.h"
#include "plugin/deviceplugin.h"
#include <QDebug>
@ -43,6 +44,9 @@ private:
private slots:
void initLogs();
void databaseSerializationTest_data();
void databaseSerializationTest();
void coverageCalls();
void systemLogs();
@ -83,6 +87,42 @@ void TestLogging::initLogs()
restartServer();
}
void TestLogging::databaseSerializationTest_data()
{
QUuid uuid = QUuid("3782732b-61b4-48e8-8d6d-b5205159d7cd");
QVariantMap variantMap;
variantMap.insert("string", "value");
variantMap.insert("int", 5);
variantMap.insert("double", 3.14);
variantMap.insert("uuid", uuid);
QTest::addColumn<QVariant>("value");
QTest::newRow("QString") << QVariant(QString("Hello"));
QTest::newRow("Integer") << QVariant((int)2);
QTest::newRow("Double") << QVariant((double)2.34);
QTest::newRow("Float") << QVariant((float)2.34);
QTest::newRow("QColor") << QVariant(QColor(0,255,128));
QTest::newRow("QByteArray") << QVariant(QByteArray("\nthisisatestarray\n"));
QTest::newRow("QUuid") << QVariant(uuid);
QTest::newRow("QVariantMap") << QVariant(variantMap);
}
void TestLogging::databaseSerializationTest()
{
QFETCH(QVariant, value);
QString serializedValue = LogValueTool::serializeValue(value);
QVariant deserializedValue = LogValueTool::deserializeValue(serializedValue);
qDebug() << "Stored:" << value;
qDebug() << "Loaded:" << deserializedValue;
QCOMPARE(deserializedValue, value);
}
void TestLogging::coverageCalls()
{
LogEntry entry(QDateTime::currentDateTime(), Logging::LoggingLevelInfo, Logging::LoggingSourceSystem);
@ -245,7 +285,7 @@ void TestLogging::actionLog()
verifyDeviceError(response);
// Lets wait for the notification
clientSpy.wait(200);
clientSpy.wait(500);
QVariantList loggEntryAddedVariants = checkNotifications(clientSpy, "Logging.LogEntryAdded");
QVERIFY2(!loggEntryAddedVariants.isEmpty(), "Did not get Logging.LogEntryAdded notification.");
@ -287,7 +327,7 @@ void TestLogging::actionLog()
params.insert("deviceIds", QVariantList() << m_mockDeviceId);
params.insert("loggingSources", QVariantList() << JsonTypes::loggingSourceToString(Logging::LoggingSourceActions));
params.insert("eventTypes", QVariantList() << JsonTypes::loggingEventTypeToString(Logging::LoggingEventTypeTrigger));
params.insert("values", QVariantList() << "7, true");
//params.insert("values", QVariantList() << "7, true");
response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);