ETM-Schurig/powersync-core
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for configuring the used database backend

Browse Source
This commit is contained in:
Michael Zanetti 2018-03-15 17:48:40 +01:00
parent 3a862e2050
commit 8db7e050db
12 changed files with 146 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
libnymea-core/debugserverhandler.cpp
View File

@ -228,12 +228,6 @@ QByteArray DebugServerHandler::createDebugXmlDocument()
writer.writeTextElement("td", NymeaSettings(NymeaSettings::SettingsRoleGlobal).translationsPath());
writer.writeEndElement(); // tr
writer.writeStartElement("tr");
//: The log database path description in the server infromation section of the debug interface
writer.writeTextElement("th", tr("Log database"));
writer.writeTextElement("td", NymeaSettings(NymeaSettings::SettingsRoleGlobal).logPath());
writer.writeEndElement(); // tr
for (int i = 0; i < NymeaCore::instance()->deviceManager()->pluginSearchDirs().count(); i++) {
writer.writeStartElement("tr");
writer.writeEndElement(); // tr
@ -271,10 +265,12 @@ QByteArray DebugServerHandler::createDebugXmlDocument()
writer.writeTextElement("p", tr("Log database"));
writer.writeEndElement(); // div download-name-column
writer.writeStartElement("div");
writer.writeAttribute("class", "download-path-column");
writer.writeTextElement("p", NymeaSettings::logPath());
writer.writeEndElement(); // div download-path-column
if (QFileInfo(NymeaCore::instance()->configuration()->logDBName()).exists()) {
writer.writeStartElement("div");
writer.writeAttribute("class", "download-path-column");
writer.writeTextElement("p", NymeaCore::instance()->configuration()->logDBName());
writer.writeEndElement(); // div download-path-column
}
writer.writeStartElement("div");
writer.writeAttribute("class", "download-button-column");
@ -643,10 +639,10 @@ HttpReply *DebugServerHandler::processDebugRequest(const QString &requestPath)
// Check if this is a logdb requested
if (requestPath.startsWith("/debug/logdb.sql")) {
qCDebug(dcWebServer()) << "Loading" << NymeaSettings::logPath();
QFile logDatabaseFile(NymeaSettings::logPath());
qCDebug(dcWebServer()) << "Loading" << NymeaCore::instance()->configuration()->logDBName();
QFile logDatabaseFile(NymeaCore::instance()->configuration()->logDBName());
if (!logDatabaseFile.exists()) {
qCWarning(dcWebServer()) << "Could not read log database file for debug download" << NymeaSettings::logPath() << "file does not exist.";
qCWarning(dcWebServer()) << "Could not read log database file for debug download" << NymeaCore::instance()->configuration()->logDBName() << "file does not exist.";
HttpReply *reply = RestResource::createErrorReply(HttpReply::NotFound);
reply->setHeader(HttpReply::ContentTypeHeader, "text/html");
//: The HTTP error message of the debug interface. The %1 represents the file name.
@ -655,7 +651,7 @@ HttpReply *DebugServerHandler::processDebugRequest(const QString &requestPath)
}
if (!logDatabaseFile.open(QFile::ReadOnly)) {
qCWarning(dcWebServer()) << "Could not read log database file for debug download" << NymeaSettings::logPath();
qCWarning(dcWebServer()) << "Could not read log database file for debug download" << NymeaCore::instance()->configuration()->logDBName();
HttpReply *reply = RestResource::createErrorReply(HttpReply::Forbidden);
reply->setHeader(HttpReply::ContentTypeHeader, "text/html");
//: The HTTP error message of the debug interface. The %1 represents the file name.

63
libnymea-core/logging/logengine.cpp
View File

@ -131,13 +131,18 @@
namespace nymeaserver {
/*! Constructs the log engine with the given \a parent. */
LogEngine::LogEngine(const QString &logPath, QObject *parent):
QObject(parent)
/*! Constructs the log engine with the given parameters.
\a The Qt Database backend to be used. Depending on the installed Qt modules this can be any of QDB2 QIBASE QMYSQL QOCI QODBC QPSQL QSQLITE QSQLITE2 QTDS.
\a dbName is the name of the database. In case of SQLITE this should contain a file path. The Driver will create the file if required. In case of using a
database server like MYSQL, the database must exist on the host given by \a hostname and be accessible with the given \a username and \a password.
*/
LogEngine::LogEngine(const QString &driver, const QString &dbName, const QString &hostname, const QString &username, const QString &password, int maxDBSize, QObject *parent):
QObject(parent),
m_dbMaxSize(maxDBSize)
{
m_db = QSqlDatabase::addDatabase("QSQLITE", "logs");
m_db.setDatabaseName(logPath);
m_dbMaxSize = 50000;
m_db = QSqlDatabase::addDatabase(driver, "logs");
m_db.setDatabaseName(dbName);
m_db.setHostName(hostname);
m_overflow = 100;
if (QCoreApplication::instance()->organizationName() == "nymea-test") {
@ -151,16 +156,14 @@ LogEngine::LogEngine(const QString &logPath, QObject *parent):
qCWarning(dcLogEngine) << "Database not valid:" << m_db.lastError().driverText() << m_db.lastError().databaseText();
rotate(m_db.databaseName());
}
if (!m_db.open()) {
qCWarning(dcLogEngine) << "Error opening log database:" << m_db.lastError().driverText() << m_db.lastError().databaseText();
rotate(m_db.databaseName());
}
if (!initDB()) {
if (!initDB(username, password)) {
qCWarning(dcLogEngine()) << "Error initializing database. Trying to correct it.";
rotate(m_db.databaseName());
if (!initDB()) {
qCWarning(dcLogEngine()) << "Error fixing log database. Giving up. Logs can't be stored.";
if (QFileInfo(m_db.databaseName()).exists()) {
rotate(m_db.databaseName());
if (!initDB(username, password)) {
qCWarning(dcLogEngine()) << "Error fixing log database. Giving up. Logs can't be stored.";
}
}
}
@ -405,6 +408,10 @@ void LogEngine::appendLogEntry(const LogEntry &entry)
void LogEngine::checkDBSize()
{
if (m_dbMaxSize == -1) {
// No tripping required
return;
}
QDateTime startTime = QDateTime::currentDateTime();
QString queryString = "SELECT COUNT(*) FROM entries;";
QSqlQuery result = m_db.exec(queryString);
@ -529,17 +536,26 @@ bool LogEngine::migrateDatabaseVersion2to3()
return true;
}
bool LogEngine::initDB()
bool LogEngine::initDB(const QString &username, const QString &password)
{
m_db.close();
m_db.open();
if (!m_db.tables().contains("metadata")) {
m_db.exec("CREATE TABLE metadata (key varchar(10), data varchar(40));");
m_db.exec(QString("INSERT INTO metadata (key, data) VALUES('version', '%1');").arg(DB_SCHEMA_VERSION));
bool opened = m_db.open(username, password);
if (!opened) {
qCWarning(dcLogEngine()) << "Can't open Log DB. Init failed.";
return false;
}
QSqlQuery query = m_db.exec("SELECT data FROM metadata WHERE key = 'version';");
if (!m_db.tables().contains("metadata")) {
qCDebug(dcLogEngine()) << "Empty Database. Setting up metadata...";
m_db.exec("CREATE TABLE metadata (`key` VARCHAR(10), data VARCHAR(40));");
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine) << "Error initualizing database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
return false;
}
m_db.exec(QString("INSERT INTO metadata (`key`, data) VALUES('version', '%1');").arg(DB_SCHEMA_VERSION));
}
QSqlQuery query = m_db.exec("SELECT data FROM metadata WHERE `key` = 'version';");
if (query.next()) {
int version = query.value("data").toInt();
@ -575,12 +591,15 @@ bool LogEngine::initDB()
}
if (!m_db.tables().contains("loggingEventTypes")) {
m_db.exec("CREATE TABLE loggingEventTypes (id int, name varchar(20), PRIMARY KEY(id));");
m_db.exec("CREATE TABLE loggingEventTypes (id int, name varchar(40), PRIMARY KEY(id));");
//qCDebug(dcLogEngine) << m_db.lastError().databaseText();
QMetaEnum logTypes = Logging::staticMetaObject.enumerator(Logging::staticMetaObject.indexOfEnumerator("LoggingEventType"));
Q_ASSERT_X(logTypes.isValid(), "LogEngine", "Logging has no enum LoggingEventType");
for (int i = 0; i < logTypes.keyCount(); i++) {
m_db.exec(QString("INSERT INTO loggingEventTypes (id, name) VALUES(%1, '%2');").arg(i).arg(logTypes.key(i)));
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine()) << "Failed to insert loggingEventTypes into DB. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
}
}
}

5
libnymea-core/logging/logengine.h
View File

@ -27,7 +27,6 @@
#include "types/event.h"
#include "types/action.h"
#include "rule.h"
#include "nymeasettings.h"
#include <QObject>
#include <QSqlDatabase>
@ -39,7 +38,7 @@ class LogEngine: public QObject
{
Q_OBJECT
public:
LogEngine(const QString &logPath = NymeaSettings::logPath(), QObject *parent = 0);
LogEngine(const QString &driver, const QString &dbName, const QString &hostname = QString("127.0.0.1"), const QString &username = QString(), const QString &password = QString(), int maxDBSize = 50000, QObject *parent = 0);
~LogEngine();
QList<LogEntry> logEntries(const LogFilter &filter = LogFilter()) const;
@ -64,7 +63,7 @@ signals:
void logDatabaseUpdated();
private:
bool initDB();
bool initDB(const QString &username, const QString &password);
void appendLogEntry(const LogEntry &entry);
void rotate(const QString &dbName);

66
libnymea-core/nymeaconfiguration.cpp
View File

@ -25,6 +25,7 @@
#include <QTimeZone>
#include <QCoreApplication>
#include <QFile>
#include <QDir>
namespace nymeaserver {
@ -158,6 +159,16 @@ NymeaConfiguration::NymeaConfiguration(QObject *parent) :
m_webSocketServerConfigs[config.id] = config;
storeServerConfig("WebSocketServer", config);
}
// Write defaults for log settings
settings.beginGroup("Logs");
settings.setValue("logDBDriver", logDBDriver());
settings.setValue("logDBName", logDBName());
settings.setValue("logDBHost", logDBHost());
settings.setValue("logDBUser", logDBUser());
settings.setValue("logDBPassword", logDBPassword());
settings.setValue("logDBMaxEntries", logDBMaxEntries());
settings.endGroup();
}
QUuid NymeaConfiguration::serverUuid() const
@ -368,6 +379,61 @@ QString NymeaConfiguration::cloudCertificateKey() const
return settings.value("cloudCertificateKey").toString();
}
QString NymeaConfiguration::logDBDriver() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBDriver", "QSQLITE").toString();
}
QString NymeaConfiguration::logDBHost() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBHost", "127.0.0.1").toString();
}
QString NymeaConfiguration::logDBName() const
{
QString defaultLogPath;
QString organisationName = QCoreApplication::instance()->organizationName();
if (!qgetenv("SNAP").isEmpty()) {
defaultLogPath = QString(qgetenv("SNAP_COMMON")) + "/nymead.sqlite";
} else if (organisationName == "nymea-test") {
defaultLogPath = "/tmp/" + organisationName + "/nymead-test.sqlite";
} else if (NymeaSettings::isRoot()) {
defaultLogPath = "/var/log/nymead.sqlite";
} else {
defaultLogPath = QDir::homePath() + "/.config/" + organisationName + "/nymead.sqlite";
}
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBName", defaultLogPath).toString();
}
QString NymeaConfiguration::logDBUser() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBUser").toString();
}
QString NymeaConfiguration::logDBPassword() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBPassword").toString();
}
int NymeaConfiguration::logDBMaxEntries() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBMaxEntries", 200000).toInt();
}
QString NymeaConfiguration::sslCertificate() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);

8
libnymea-core/nymeaconfiguration.h
View File

@ -123,6 +123,14 @@ public:
QString cloudCertificate() const;
QString cloudCertificateKey() const;
// Logging
QString logDBDriver() const;
QString logDBName() const;
QString logDBHost() const;
QString logDBUser() const;
QString logDBPassword() const;
int logDBMaxEntries() const;
private:
QHash<QString, ServerConfiguration> m_tcpServerConfigs;
QHash<QString, WebServerConfiguration> m_webServerConfigs;

2
libnymea-core/nymeacore.cpp
View File

@ -502,7 +502,7 @@ void NymeaCore::init() {
m_timeManager = new TimeManager(QTimeZone::systemTimeZoneId(), this);
qCDebug(dcApplication) << "Creating Log Engine";
m_logger = new LogEngine(NymeaSettings::logPath(), this);
m_logger = new LogEngine(m_configuration->logDBDriver(), m_configuration->logDBName(), m_configuration->logDBHost(), m_configuration->logDBUser(), m_configuration->logDBPassword(), m_configuration->logDBMaxEntries(), this);
qCDebug(dcApplication) << "Creating Hardware Manager";
m_hardwareManager = new HardwareManagerImplementation(this);

1
libnymea-core/servermanager.cpp
View File

@ -36,6 +36,7 @@
#include "servermanager.h"
#include "nymeacore.h"
#include "certificategenerator.h"
#include "nymeasettings.h"
#include <QSslCertificate>
#include <QSslConfiguration>

22
libnymea/nymeasettings.cpp
View File

@ -125,28 +125,6 @@ bool NymeaSettings::isRoot()
return true;
}
/*! Returns the path where the logging database will be stored.
\sa nymeaserver::LogEngine
*/
QString NymeaSettings::logPath()
{
QString logPath;
QString organisationName = QCoreApplication::instance()->organizationName();
if (!qgetenv("SNAP").isEmpty()) {
logPath = QString(qgetenv("SNAP_COMMON")) + "/nymead.sqlite";
} else if (organisationName == "nymea-test") {
logPath = "/tmp/" + organisationName + "/nymead-test.sqlite";
} else if (NymeaSettings::isRoot()) {
logPath = "/var/log/nymead.sqlite";
} else {
logPath = QDir::homePath() + "/.config/" + organisationName + "/nymead.sqlite";
}
return logPath;
}
/*! Returns the path to the folder where the NymeaSettings will be saved i.e. \tt{/etc/nymea}. */
QString NymeaSettings::settingsPath()
{

1
libnymea/nymeasettings.h
View File

@ -49,7 +49,6 @@ public:
SettingsRole settingsRole() const;
static bool isRoot();
static QString logPath();
static QString settingsPath();
static QString translationsPath();
static QString storagePath();

1
tests/auto/configurations/testconfigurations.cpp
View File

@ -23,6 +23,7 @@
#include "devicemanager.h"
#include "mocktcpserver.h"
#include "nymeacore.h"
#include "nymeasettings.h"
#include <QtTest/QtTest>
#include <QCoreApplication>

21
tests/auto/loggingdirect/testloggingdirect.cpp
View File

@ -35,11 +35,12 @@ private slots:
void benchmarkDB();
private:
LogEngine engine;
LogEngine *engine;
};
TestLoggingDirect::TestLoggingDirect(QObject *parent): QObject(parent)
{
engine = new LogEngine("QSQLITE", "/tmp/nymea-test/nymea.sqlite");
// Setting timeout to 20 mins
qputenv("QTEST_FUNCTION_TIMEOUT", "1200000");
QCoreApplication::instance()->setOrganizationName("nymea-test");
@ -73,25 +74,25 @@ void TestLoggingDirect::benchmarkDB()
// setting max log entries to "prefill" to trim it down to what this test needs.
int overflow = 10;
engine.setMaxLogEntries(prefill, overflow);
engine.setMaxLogEntries(maxSize, overflow);
engine->setMaxLogEntries(prefill, overflow);
engine->setMaxLogEntries(maxSize, overflow);
for (int i = engine.logEntries().count(); i < prefill; i++) {
engine.logSystemEvent(QDateTime::currentDateTime(), true);
for (int i = engine->logEntries().count(); i < prefill; i++) {
engine->logSystemEvent(QDateTime::currentDateTime(), true);
}
qDebug() << "Starting benchmark with" << engine.logEntries().count() << "entries in the db";
qDebug() << "Starting benchmark with" << engine->logEntries().count() << "entries in the db";
QBENCHMARK {
engine.logSystemEvent(QDateTime::currentDateTime(), true);
engine->logSystemEvent(QDateTime::currentDateTime(), true);
}
QDateTime now = QDateTime::currentDateTime();
while (engine.logEntries().count() > maxSize + overflow) {
while (engine->logEntries().count() > maxSize + overflow) {
qApp->processEvents();
if (now.addSecs(5) < QDateTime::currentDateTime()) {
QVERIFY2(false, QString("Housekeeping didn't work. Have %1 entries but expected to have max %2").arg(engine.logEntries().count()).arg(QString::number(maxSize)).toLocal8Bit());
QVERIFY2(false, QString("Housekeeping didn't work. Have %1 entries but expected to have max %2").arg(engine->logEntries().count()).arg(QString::number(maxSize)).toLocal8Bit());
}
}
qDebug() << "Ended benchmark with" << engine.logEntries().count() << "entries in the db";
qDebug() << "Ended benchmark with" << engine->logEntries().count() << "entries in the db";
}
#include "testloggingdirect.moc"

10
tests/auto/loggingloading/testloggingloading.cpp
View File

@ -58,7 +58,7 @@ void TestLoggingLoading::initTestCase()
void TestLoggingLoading::testLogMigration()
{
// Create LogEngine with log db from resource file
QString temporaryDbName = NymeaSettings::settingsPath() + "/nymead-v2.sqlite";
QString temporaryDbName = "/tmp/nymea-test/nymead-v2.sqlite";
if (QFile::exists(temporaryDbName))
QVERIFY(QFile(temporaryDbName).remove());
@ -68,7 +68,7 @@ void TestLoggingLoading::testLogMigration()
QVERIFY(QFile::copy(":/nymead-v2.sqlite", temporaryDbName));
QVERIFY(QFile::setPermissions(temporaryDbName, QFile::ReadOwner | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOther));
LogEngine *logEngine = new LogEngine(temporaryDbName, this);
LogEngine *logEngine = new LogEngine("QSQLITE", temporaryDbName);
// Check there is no rotated logfile
QVERIFY(!QFile::exists(temporaryDbName + ".1"));
@ -80,8 +80,8 @@ void TestLoggingLoading::testLogMigration()
void TestLoggingLoading::testLogfileRotation()
{
// Create LogEngine with log db from resource file
QString temporaryDbName = NymeaSettings::settingsPath() + "/nymead-broken.sqlite";
QString rotatedDbName = NymeaSettings::settingsPath() + "/nymead-broken.sqlite.1";
QString temporaryDbName = "/tmp/nymea-test/nymead-broken.sqlite";
QString rotatedDbName = "/tmp/nymea-test/nymead-broken.sqlite.1";
// Remove the files if there are some left
if (QFile::exists(temporaryDbName))
@ -96,7 +96,7 @@ void TestLoggingLoading::testLogfileRotation()
QVERIFY(QFile::setPermissions(temporaryDbName, QFile::ReadOwner | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOther));
QVERIFY(!QFile::exists(rotatedDbName));
LogEngine *logEngine = new LogEngine(temporaryDbName, this);
LogEngine *logEngine = new LogEngine("QSQLITE", temporaryDbName);
QVERIFY(QFile::exists(rotatedDbName));
delete logEngine;