ETM-Schurig/powersync-core
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework the log engine to use influxdb instead of sql

Browse Source
This commit is contained in:
Michael Zanetti 2023-02-24 22:18:47 +01:00
parent a41b82afbe
commit 4e509841b0
82 changed files with 2094 additions and 3412 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
debian/control vendored
View File

@ -29,6 +29,7 @@ Build-Depends: debhelper (>= 9.0.0),
qtdeclarative5-dev,
libqt5serialport5-dev,
libqt5serialbus5-dev,
influxdb:native,
Package: nymea
@ -62,6 +63,7 @@ Recommends: nymea-cli,
nymea-zeroconf-plugin-impl,
nymea-apikeysprovider-plugin-impl,
nymea-zwave-plugin-impl,
influxdb,
Description: An open source IoT server - daemon
The nymea daemon is a plugin based IoT (Internet of Things) server.
The server works like a translator for devices, things and services
@ -128,7 +130,8 @@ Multi-Arch: same
Depends: nymea (= ${binary:Version}),
${shlibs:Depends},
${misc:Depends},
libnymea-tests (= ${binary:Version})
libnymea-tests (= ${binary:Version}),
influxdb,
Description: nymea automated tests - tests
Automated tests for the nymea daemon.

1
debian/libnymea-dev.install.in vendored
View File

@ -9,4 +9,5 @@ usr/include/nymea/network/*
usr/include/nymea/platform/*
usr/include/nymea/time/*
usr/include/nymea/types/*
usr/include/nymea/logging/*
usr/lib/@DEB_HOST_MULTIARCH@/pkgconfig/nymea.pc

1
libnymea-core/debugreportgenerator.cpp
View File

@ -210,7 +210,6 @@ void DebugReportGenerator::saveConfigs()
copyFileToReportDirectory(NymeaSettings(NymeaSettings::SettingsRoleRules).fileName(), "config");
copyFileToReportDirectory(NymeaSettings(NymeaSettings::SettingsRolePlugins).fileName(), "config");
copyFileToReportDirectory(NymeaSettings(NymeaSettings::SettingsRoleTags).fileName(), "config");
copyFileToReportDirectory(NymeaCore::instance()->configuration()->logDBName(), "config");
}
void DebugReportGenerator::saveEnv()

69
libnymea-core/debugserverhandler.cpp
View File

@ -76,38 +76,6 @@ HttpReply *DebugServerHandler::processDebugRequest(const QString &requestPath, c
return reply;
}
// Check if this is a logdb requested
if (requestPath.startsWith("/debug/logdb.sql")) {
qCDebug(dcDebugServer()) << "Loading" << NymeaCore::instance()->configuration()->logDBName();
QFile logDatabaseFile(NymeaCore::instance()->configuration()->logDBName());
if (!logDatabaseFile.exists()) {
qCWarning(dcDebugServer()) << "Could not read log database file for debug download" << NymeaCore::instance()->configuration()->logDBName() << "file does not exist.";
HttpReply *reply = HttpReply::createErrorReply(HttpReply::NotFound);
reply->setHeader(HttpReply::ContentTypeHeader, "text/html");
//: The HTTP error message of the debug interface. The %1 represents the file name.
reply->setPayload(createErrorXmlDocument(HttpReply::NotFound, tr("Could not find file \"%1\".").arg(logDatabaseFile.fileName())));
return reply;
}
if (!logDatabaseFile.open(QFile::ReadOnly)) {
qCWarning(dcDebugServer()) << "Could not read log database file for debug download" << NymeaCore::instance()->configuration()->logDBName();
HttpReply *reply = HttpReply::createErrorReply(HttpReply::Forbidden);
reply->setHeader(HttpReply::ContentTypeHeader, "text/html");
//: The HTTP error message of the debug interface. The %1 represents the file name.
reply->setPayload(createErrorXmlDocument(HttpReply::NotFound, tr("Could not open file \"%1\".").arg(logDatabaseFile.fileName())));
return reply;
}
QByteArray logDatabaseRawData = logDatabaseFile.readAll();
logDatabaseFile.close();
HttpReply *reply = HttpReply::createSuccessReply();
reply->setHeader(HttpReply::ContentTypeHeader, "application/sql");
reply->setPayload(logDatabaseRawData);
return reply;
}
// Check if this is a syslog requested
if (requestPath.startsWith("/debug/syslog")) {
QString syslogFileName = "/var/log/syslog";
@ -1223,43 +1191,6 @@ QByteArray DebugServerHandler::createDebugXmlDocument()
writer.writeTextElement("h3", tr("Logs"));
writer.writeEmptyElement("hr");
// Download row logdb
writer.writeStartElement("div");
writer.writeAttribute("class", "download-row");
writer.writeStartElement("div");
writer.writeAttribute("class", "download-name-column");
//: The log databse download description of the debug interface
writer.writeTextElement("p", tr("Log database"));
writer.writeEndElement(); // div download-name-column
if (QFileInfo(NymeaCore::instance()->configuration()->logDBName()).exists()) {
writer.writeStartElement("div");
writer.writeAttribute("class", "download-path-column");
writer.writeTextElement("p", NymeaCore::instance()->configuration()->logDBName());
writer.writeEndElement(); // div download-path-column
}
writer.writeStartElement("div");
writer.writeAttribute("class", "download-button-column");
writer.writeStartElement("form");
writer.writeAttribute("class", "download-button");
writer.writeStartElement("button");
writer.writeAttribute("class", "button");
writer.writeAttribute("type", "button");
if (!QFile::exists(NymeaCore::instance()->configuration()->logDBName())) {
writer.writeAttribute("disabled", "disabled");
}
writer.writeAttribute("onClick", "downloadFile('/debug/logdb.sql', 'logdb.sql')");
//: The download button description of the debug interface
writer.writeCharacters(tr("Download"));
writer.writeEndElement(); // button
writer.writeEndElement(); // form
writer.writeEndElement(); // div download-button-column
writer.writeEndElement(); // div download-row
// Download row syslog
writer.writeStartElement("div");
writer.writeAttribute("class", "download-row");

1
libnymea-core/integrations/pythonintegrationplugin.cpp
View File

@ -286,6 +286,7 @@ void PythonIntegrationPlugin::deinitPython()
// Our main thread state is destroyed now
s_mainThreadState = nullptr;
qCDebug(dcPythonIntegrations()) << "Python engine finalized";
}
bool PythonIntegrationPlugin::loadScript(const QString &scriptFile)

229
libnymea-core/integrations/thingmanagerimplementation.cpp
View File

@ -70,6 +70,7 @@
#include <QStandardPaths>
#include <QDir>
#include <QJsonDocument>
#include <QMetaEnum>
ThingManagerImplementation::ThingManagerImplementation(HardwareManager *hardwareManager, LogEngine *logEngine, const QLocale &locale, QObject *parent) :
ThingManager(parent),
@ -531,7 +532,7 @@ Thing::ThingError ThingManagerImplementation::setStateLogging(const ThingId &thi
qCWarning(dcThingManager()) << "Cannot configure event logging. Thing" << thingId.toString() << "not found";
return Thing::ThingErrorThingNotFound;
}
if (!thing->thingClass().stateTypes().findById(stateTypeId).isValid()) {
if (!thing->thingClass().hasStateType(stateTypeId)) {
qCWarning(dcThingManager()) << "Cannot configure state logging. Thing" << thing << "has no state type with id" << stateTypeId;
return Thing::ThingErrorStateTypeNotFound;
}
@ -539,10 +540,18 @@ Thing::ThingError ThingManagerImplementation::setStateLogging(const ThingId &thi
if (enabled && !loggedStateTypes.contains(stateTypeId)) {
loggedStateTypes.append(stateTypeId);
thing->setLoggedStateTypeIds(loggedStateTypes);
storeConfiguredThings();
registerStateLogger(thing, stateTypeId);
emit thingChanged(thing);
} else if (!enabled && loggedStateTypes.contains(stateTypeId)) {
loggedStateTypes.removeAll(stateTypeId);
thing->setLoggedStateTypeIds(loggedStateTypes);
storeConfiguredThings();
unregisterStateLogger(thing, stateTypeId);
emit thingChanged(thing);
}
return Thing::ThingErrorNoError;
@ -563,15 +572,56 @@ Thing::ThingError ThingManagerImplementation::setEventLogging(const ThingId &thi
if (enabled && !loggedEventTypes.contains(eventTypeId)) {
loggedEventTypes.append(eventTypeId);
thing->setLoggedEventTypeIds(loggedEventTypes);
storeConfiguredThings();
registerEventLogger(thing, eventTypeId);
emit thingChanged(thing);
} else if (!enabled && loggedEventTypes.contains(eventTypeId)) {
loggedEventTypes.removeAll(eventTypeId);
thing->setLoggedEventTypeIds(loggedEventTypes);
storeConfiguredThings();
unregisterEventLogger(thing, eventTypeId);
emit thingChanged(thing);
}
return Thing::ThingErrorNoError;
}
Thing::ThingError ThingManagerImplementation::setActionLogging(const ThingId &thingId, const ActionTypeId &actionTypeId, bool enabled)
{
Thing *thing = m_configuredThings.value(thingId);
if (!thing) {
qCWarning(dcThingManager()) << "Cannot configure action logging. Thing" << thingId.toString() << "not found";
return Thing::ThingErrorThingNotFound;
}
if (!thing->thingClass().hasActionType(actionTypeId)) {
qCWarning(dcThingManager()) << "Cannot configure action logging. Thing" << thing << "has no action type with id" << actionTypeId;
return Thing::ThingErrorEventTypeNotFound;
}
QList<ActionTypeId> loggedActionTypes = thing->loggedActionTypeIds();
if (enabled && !loggedActionTypes.contains(actionTypeId)) {
loggedActionTypes.append(actionTypeId);
thing->setLoggedActionTypeIds(loggedActionTypes);
storeConfiguredThings();
registerActionLogger(thing, actionTypeId);
emit thingChanged(thing);
} else if (!enabled && loggedActionTypes.contains(actionTypeId)) {
loggedActionTypes.removeAll(actionTypeId);
thing->setLoggedActionTypeIds(loggedActionTypes);
storeConfiguredThings();
unregisterActionLogger(thing, actionTypeId);
emit thingChanged(thing);
}
return Thing::ThingErrorNoError;
}
Thing::ThingError ThingManagerImplementation::setStateFilter(const ThingId &thingId, const StateTypeId &stateTypeId, Types::StateValueFilter filter)
{
Thing *thing = m_configuredThings.value(thingId);
@ -911,7 +961,15 @@ void ThingManagerImplementation::removeConfiguredThingInternal(Thing *thing)
}
}
m_logEngine->removeThingLogs(thing->id());
foreach (const StateTypeId &stateTypeId, thing->loggedStateTypeIds()) {
unregisterStateLogger(thing, stateTypeId);
}
foreach (const EventTypeId &eventTypeId, thing->loggedEventTypeIds()) {
unregisterEventLogger(thing, eventTypeId);
}
foreach (const ActionTypeId &actionTypeId, thing->loggedActionTypeIds()) {
unregisterActionLogger(thing, actionTypeId);
}
emit thingRemoved(t->id());
}
@ -998,9 +1056,6 @@ BrowserActionInfo* ThingManagerImplementation::executeBrowserItem(const BrowserA
Thing *thing = m_configuredThings.value(browserAction.thingId());
BrowserActionInfo *info = new BrowserActionInfo(thing, this, browserAction, this, 30000);
connect(info, &BrowserActionInfo::finished, info->thing(), [this, info](){
m_logEngine->logBrowserAction(info->browserAction(), info->status() == Thing::ThingErrorNoError ? Logging::LoggingLevelInfo : Logging::LoggingLevelAlert, info->status());
});
if (!thing) {
info->finish(Thing::ThingErrorThingNotFound);
@ -1033,9 +1088,6 @@ BrowserItemActionInfo* ThingManagerImplementation::executeBrowserItemAction(cons
Thing *thing = m_configuredThings.value(browserItemAction.thingId());
BrowserItemActionInfo *info = new BrowserItemActionInfo(thing, this, browserItemAction, this, 30000);
connect(info, &BrowserItemActionInfo::finished, info->thing(), [this, info](){
m_logEngine->logBrowserItemAction(info->browserItemAction(), info->status() == Thing::ThingErrorNoError ? Logging::LoggingLevelInfo : Logging::LoggingLevelAlert, info->status());
});
if (!thing) {
info->finish(Thing::ThingErrorThingNotFound);
@ -1386,7 +1438,21 @@ ThingActionInfo *ThingManagerImplementation::executeAction(const Action &action)
ThingActionInfo *info = new ThingActionInfo(thing, finalAction, this, 15000);
connect(info, &ThingActionInfo::finished, this, [=](){
m_logEngine->logAction(finalAction, info->status());
if (thing->loggedActionTypeIds().contains(actionType.id())) {
QVariantMap params;
foreach (const ParamType &paramType, actionType.paramTypes()) {
params.insert(paramType.name(), action.paramValue(paramType.id()));
}
m_actionLoggers.value(thing->id().toString() + "-" + actionType.name())->log({},
{
{"status", QMetaEnum::fromType<Thing::ThingError>().valueToKey(info->status())},
{"triggeredBy", QMetaEnum::fromType<Action::TriggeredBy>().valueToKey(action.triggeredBy())},
{"params", QJsonDocument::fromVariant(params).toJson(QJsonDocument::Compact)}
});
}
emit actionExecuted(action, info->status());
});
@ -1676,6 +1742,29 @@ void ThingManagerImplementation::loadConfiguredThings()
initThing(thing);
// Overriding logging settings from thingClass/interfaces with user preferences
if (settings.contains("loggedStateTypeIds")) {
QList<StateTypeId> loggedStateTypeIds;
foreach (const QString &stateTypeId, settings.value("loggedStateTypeIds").toStringList()) {
loggedStateTypeIds.append(StateTypeId(stateTypeId));
}
thing->setLoggedStateTypeIds(loggedStateTypeIds);
}
if (settings.contains("loggedEventTypeIds")) {
QList<EventTypeId> loggedEventTypeIds;
foreach (const QString &eventTypeId, settings.value("loggedEventTypeIds").toStringList()) {
loggedEventTypeIds.append(EventTypeId(eventTypeId));
}
thing->setLoggedEventTypeIds(loggedEventTypeIds);
}
if (settings.contains("loggedActionTypeIds")) {
QList<ActionTypeId> loggedActionTypeIds;
foreach (const QString &actionTypeId, settings.value("loggedActionTypeIds").toStringList()) {
loggedActionTypeIds.append(ActionTypeId(actionTypeId));
}
thing->setLoggedActionTypeIds(loggedActionTypeIds);
}
settings.endGroup(); // ThingId
// We always add the thing to the list in this case. If it's in the stored things
@ -1738,6 +1827,21 @@ void ThingManagerImplementation::storeConfiguredThings()
}
settings.endGroup(); // Settings
QStringList loggedStateTypeIds;
foreach (const StateTypeId &stateTypeId, thing->loggedStateTypeIds()) {
loggedStateTypeIds.append(stateTypeId.toString());
}
settings.setValue("loggedStateTypeIds", loggedStateTypeIds);
QStringList loggedEventTypeIds;
foreach (const EventTypeId &eventTypeId, thing->loggedEventTypeIds()) {
loggedEventTypeIds.append(eventTypeId.toString());
}
settings.setValue("loggedEventTypeIds", loggedEventTypeIds);
QStringList loggedActionTypeIds;
foreach (const ActionTypeId &actionTypeId, thing->loggedActionTypeIds()) {
loggedActionTypeIds.append(actionTypeId.toString());
}
settings.setValue("loggedActionTypeIds", loggedActionTypeIds);
settings.endGroup(); // ThingId
}
@ -1875,9 +1979,13 @@ void ThingManagerImplementation::onEventTriggered(Event event)
qCWarning(dcThingManager()) << "The given thing" << thing << "does not have an event type of id " + event.eventTypeId().toString() + ". Not forwarding event.";
return;
}
// configure logging
if (thing->loggedEventTypeIds().contains(event.eventTypeId())) {
m_logEngine->logEvent(event);
QVariantMap params;
foreach (const ParamType &paramType, eventType.paramTypes()) {
params.insert(paramType.name(), event.paramValue(paramType.id()));
}
m_eventLoggers.value(thing->id().toString() + "-" + eventType.name())->log({}, {{"params", QJsonDocument::fromVariant(params).toJson(QJsonDocument::Compact)}});
}
// Forward the event
@ -1891,12 +1999,13 @@ void ThingManagerImplementation::slotThingStateValueChanged(const StateTypeId &s
qCWarning(dcThingManager()) << "Invalid thing id in state change. Not forwarding event. Thing setup not complete yet?";
return;
}
if (thing->thingClass().getStateType(stateTypeId).cached()) {
StateType stateType = thing->thingClass().getStateType(stateTypeId);
if (stateType.cached()) {
storeThingState(thing, stateTypeId);
}
if (thing->loggedStateTypeIds().contains(stateTypeId)) {
m_logEngine->logStateChange(thing, stateTypeId, value);
m_stateLoggers.value(thing->id().toString() + "-" + stateType.name())->log({}, {{stateType.name(), value}});
}
emit thingStateChanged(thing, stateTypeId, value, minValue, maxValue);
@ -2150,13 +2259,6 @@ void ThingManagerImplementation::initThing(Thing *thing)
thing->setStates(states);
loadThingStates(thing);
QList<EventTypeId> loggedEventTypeIds;
foreach (const EventType &eventType, thingClass.eventTypes()) {
if (eventType.suggestLogging()) {
loggedEventTypeIds.append(eventType.id());
}
}
thing->setLoggedEventTypeIds(loggedEventTypeIds);
QList<StateTypeId> loggedStateTypeIds;
foreach (const StateType &stateType, thingClass.stateTypes()) {
if (stateType.suggestLogging()) {
@ -2164,6 +2266,21 @@ void ThingManagerImplementation::initThing(Thing *thing)
}
}
thing->setLoggedStateTypeIds(loggedStateTypeIds);
QList<EventTypeId> loggedEventTypeIds;
foreach (const EventType &eventType, thingClass.eventTypes()) {
if (eventType.suggestLogging()) {
loggedEventTypeIds.append(eventType.id());
}
}
thing->setLoggedEventTypeIds(loggedEventTypeIds);
// By default all action types are logged
QList<ActionTypeId> loggedActionTypeIds;
foreach (const ActionType &actionType, thingClass.actionTypes()) {
loggedActionTypeIds.append(actionType.id());
}
thing->setLoggedActionTypeIds(loggedActionTypeIds);
}
void ThingManagerImplementation::postSetupThing(Thing *thing)
@ -2274,6 +2391,62 @@ QVariant ThingManagerImplementation::mapValue(const QVariant &value, const State
return toValue;
}
void ThingManagerImplementation::registerStateLogger(Thing *thing, const StateTypeId &stateTypeId)
{
StateType stateType = thing->thingClass().getStateType(stateTypeId);
QString name = thing->id().toString() + "-" + stateType.name();
QList<QVariant::Type> sampledTypes {
QVariant::Int,
QVariant::UInt,
QVariant::LongLong,
QVariant::ULongLong,
QVariant::Double
};
Types::LoggingType loggingType = sampledTypes.contains(stateType.type()) ? Types::LoggingTypeSampled : Types::LoggingTypeDiscrete;
Logger *logger = m_logEngine->registerLogSource("state-" + name, {}, loggingType, stateType.name());
m_stateLoggers.insert(name, logger);
}
void ThingManagerImplementation::unregisterStateLogger(Thing *thing, const StateTypeId &stateTypeId)
{
StateType stateType = thing->thingClass().getStateType(stateTypeId);
QString name = thing->id().toString() + "-" + stateType.name();
m_logEngine->unregisterLogSource("state-" + name);
m_stateLoggers.remove(name);
}
void ThingManagerImplementation::registerEventLogger(Thing *thing, const EventTypeId &eventTypeId)
{
EventType eventType = thing->thingClass().eventTypes().findById(eventTypeId);
QString name = thing->id().toString() + "-" + eventType.name();
Logger *logger = m_logEngine->registerLogSource("event-" + name, {});
m_eventLoggers.insert(name, logger);
}
void ThingManagerImplementation::unregisterEventLogger(Thing *thing, const EventTypeId &eventTypeId)
{
EventType eventType = thing->thingClass().eventTypes().findById(eventTypeId);
QString name = thing->id().toString() + "-" + eventType.name();
m_logEngine->unregisterLogSource("event-" + name);
m_eventLoggers.remove(name);
}
void ThingManagerImplementation::registerActionLogger(Thing *thing, const ActionTypeId &actionTypeId)
{
ActionType actionType = thing->thingClass().actionTypes().findById(actionTypeId);
QString name = thing->id().toString() + "-" + actionType.name();
Logger *logger = m_logEngine->registerLogSource("action-" + name, {});
m_actionLoggers.insert(name, logger);
}
void ThingManagerImplementation::unregisterActionLogger(Thing *thing, const ActionTypeId &actionTypeId)
{
ActionType actionType = thing->thingClass().actionTypes().findById(actionTypeId);
QString name = thing->id().toString() + "-" + actionType.name();
m_logEngine->unregisterLogSource("action-" + name);
m_actionLoggers.remove(name);
}
void ThingManagerImplementation::trySetupThing(Thing *thing)
{
thing->setSetupStatus(Thing::ThingSetupStatusInProgress, Thing::ThingErrorNoError);
@ -2312,6 +2485,22 @@ void ThingManagerImplementation::registerThing(Thing *thing)
connect(thing, &Thing::stateValueChanged, this, &ThingManagerImplementation::slotThingStateValueChanged);
connect(thing, &Thing::settingChanged, this, &ThingManagerImplementation::slotThingSettingChanged);
connect(thing, &Thing::nameChanged, this, &ThingManagerImplementation::slotThingNameChanged);
foreach (const StateType &stateType, thing->thingClass().stateTypes()) {
if (thing->loggedStateTypeIds().contains(stateType.id())) {
registerStateLogger(thing, stateType.id());
}
}
foreach (const EventType &eventType, thing->thingClass().eventTypes()) {
if (thing->loggedEventTypeIds().contains(eventType.id())) {
registerEventLogger(thing, eventType.id());
}
}
foreach (const ActionType &actionType, thing->thingClass().actionTypes()) {
if (thing->loggedActionTypeIds().contains(actionType.id())) {
registerActionLogger(thing, actionType.id());
}
}
}
IntegrationPlugin *ThingManagerImplementation::createCppIntegrationPlugin(const QString &absoluteFilePath)

18
libnymea-core/integrations/thingmanagerimplementation.h
View File

@ -60,11 +60,8 @@ class ThingPairingInfo;
class HardwareManager;
class Translator;
class ApiKeysProvidersLoader;
namespace nymeaserver {
class LogEngine;
}
using namespace nymeaserver;
class Logger;
class ThingManagerImplementation: public ThingManager
{
@ -113,6 +110,7 @@ public:
Thing::ThingError setStateLogging(const ThingId &thingId, const StateTypeId &stateTypeId, bool enabled) override;
Thing::ThingError setEventLogging(const ThingId &thingId, const EventTypeId &eventTypeId, bool enabled) override;
Thing::ThingError setActionLogging(const ThingId &thingId, const ActionTypeId &actionTypeId, bool enabled) override;
Thing::ThingError setStateFilter(const ThingId &thingId, const StateTypeId &stateTypeId, Types::StateValueFilter filter) override;
Thing::ThingError removeConfiguredThing(const ThingId &thingId) override;
@ -175,11 +173,18 @@ private:
void syncIOConnection(Thing *inputThing, const StateTypeId &stateTypeId);
QVariant mapValue(const QVariant &value, const State &fromState, const State &toState, bool inverted) const;
void registerStateLogger(Thing *thing, const StateTypeId &stateTypeId);
void unregisterStateLogger(Thing *thing, const StateTypeId &stateTypeId);
void registerEventLogger(Thing *thing, const EventTypeId &eventTypeId);
void unregisterEventLogger(Thing *thing, const EventTypeId &eventTypeId);
void registerActionLogger(Thing *thing, const ActionTypeId &actionTypeId);
void unregisterActionLogger(Thing *thing, const ActionTypeId &actionTypeId);
IntegrationPlugin *createCppIntegrationPlugin(const QString &absoluteFilePath);
private:
HardwareManager *m_hardwareManager;
nymeaserver::LogEngine *m_logEngine;
LogEngine *m_logEngine;
QLocale m_locale;
Translator *m_translator = nullptr;
@ -189,6 +194,9 @@ private:
QHash<ThingClassId, ThingClass> m_supportedThings;
QHash<ThingId, Thing*> m_configuredThings;
QHash<ThingDescriptorId, ThingDescriptor> m_discoveredThings;
QHash<QString, Logger*> m_stateLoggers;
QHash<QString, Logger*> m_actionLoggers;
QHash<QString, Logger*> m_eventLoggers;
QHash<PluginId, IntegrationPlugin*> m_integrationPlugins;

17
libnymea-core/jsonrpc/integrationshandler.cpp
View File

@ -259,6 +259,14 @@ IntegrationsHandler::IntegrationsHandler(ThingManager *thingManager, QObject *pa
returns.insert("thingError", enumRef<Thing::ThingError>());
registerMethod("SetEventLogging", description, params, returns, Types::PermissionScopeConfigureThings);
params.clear(); returns.clear();
description = "Enable/disable logging for the given action type on the given thing.";
params.insert("thingId", enumValueName(Uuid));
params.insert("actionTypeId", enumValueName(Uuid));
params.insert("enabled", enumValueName(Bool));
returns.insert("thingError", enumRef<Thing::ThingError>());
registerMethod("SetActionLogging", description, params, returns, Types::PermissionScopeConfigureThings);
params.clear(); returns.clear();
description = "Set the filter for the given state on the given thing.";
params.insert("thingId", enumValueName(Uuid));
@ -875,6 +883,15 @@ JsonReply *IntegrationsHandler::SetEventLogging(const QVariantMap &params)
return createReply(statusToReply(status));
}
JsonReply *IntegrationsHandler::SetActionLogging(const QVariantMap &params)
{
ThingId thingId = ThingId(params.value("thingId").toString());
ActionTypeId actionTypeId = ActionTypeId(params.value("actionTypeId").toUuid());
bool enabled = params.value("enabled").toBool();
Thing::ThingError status = m_thingManager->setActionLogging(thingId, actionTypeId, enabled);
return createReply(statusToReply(status));
}
JsonReply *IntegrationsHandler::SetStateFilter(const QVariantMap &params)
{
ThingId thingId = ThingId(params.value("thingId").toString());

1
libnymea-core/jsonrpc/integrationshandler.h
View File

@ -61,6 +61,7 @@ public:
Q_INVOKABLE JsonReply *SetThingSettings(const QVariantMap &params);
Q_INVOKABLE JsonReply *SetStateLogging(const QVariantMap &params);
Q_INVOKABLE JsonReply *SetEventLogging(const QVariantMap &params);
Q_INVOKABLE JsonReply *SetActionLogging(const QVariantMap &params);
Q_INVOKABLE JsonReply *SetStateFilter(const QVariantMap &params);
Q_INVOKABLE JsonReply *GetEventTypes(const QVariantMap &params, const JsonContext &context) const;

2
libnymea-core/jsonrpc/jsonrpcserverimplementation.cpp
View File

@ -530,7 +530,7 @@ void JsonRPCServerImplementation::setup()
registerHandler(this);
registerHandler(new IntegrationsHandler(NymeaCore::instance()->thingManager(), this));
registerHandler(new RulesHandler(NymeaCore::instance()->ruleEngine(), this));
registerHandler(new LoggingHandler(this));
registerHandler(new LoggingHandler(NymeaCore::instance()->logEngine(), this));
registerHandler(new ConfigurationHandler(this));
registerHandler(new NetworkManagerHandler(NymeaCore::instance()->networkManager(), this));
registerHandler(new TagsHandler(this));

261
libnymea-core/jsonrpc/logginghandler.cpp
View File

@ -30,56 +30,44 @@
#include "logginghandler.h"
#include "logging/logengine.h"
#include "logging/logfilter.h"
#include "logging/logentry.h"
#include "logging/logvaluetool.h"
#include "loggingcategories.h"
#include "nymeacore.h"
namespace nymeaserver {
LoggingHandler::LoggingHandler(QObject *parent) :
JsonHandler(parent)
LoggingHandler::LoggingHandler(LogEngine *logEngine, QObject *parent) :
JsonHandler(parent),
m_logEngine(logEngine)
{
// Enums
registerEnum<Logging::LoggingSource>();
registerEnum<Logging::LoggingLevel>();
registerEnum<Logging::LoggingEventType>();
registerEnum<Logging::LoggingError>();
registerEnum<Types::SampleRate>();
registerEnum<Qt::SortOrder>();
// Objects
registerObject<LogEntry, LogEntries>();
// Methods
QString description; QVariantMap params; QVariantMap returns;
description = "Get the LogEntries matching the given filter. "
"The result set will contain entries matching all filter rules combined. "
"If multiple options are given for a single filter type, the result set will "
"contain entries matching any of those. The offset starts at the newest entry "
"in the result set. By default all items are returned. Example: If the specified "
"filter returns a total amount of 100 entries:\n"
"- a offset value of 10 would include the oldest 90 entries\n"
"- a offset value of 0 would return all 100 entries\n\n"
"The offset is particularly useful in combination with the maxCount property and "
"can be used for pagination. E.g. A result set of 10000 entries can be fetched in "
" batches of 1000 entries by fetching\n"
"1) offset 0, maxCount 1000: Entries 0 to 9999\n"
"2) offset 10000, maxCount 1000: Entries 10000 - 19999\n"
"3) offset 20000, maxCount 1000: Entries 20000 - 29999\n"
"...";
QVariantMap timeFilter;
timeFilter.insert("o:startDate", enumValueName(Int));
timeFilter.insert("o:endDate", enumValueName(Int));
params.insert("o:timeFilters", QVariantList() << timeFilter);
params.insert("o:loggingSources", QVariantList() << enumRef<Logging::LoggingSource>());
params.insert("o:loggingLevels", QVariantList() << enumRef<Logging::LoggingLevel>());
params.insert("o:eventTypes", QVariantList() << enumRef<Logging::LoggingEventType>());
params.insert("o:typeIds", QVariantList() << enumValueName(Uuid));
params.insert("o:thingIds", QVariantList() << enumValueName(Uuid));
params.insert("o:values", QVariantList() << enumValueName(Variant));
description = "Get the LogEntries matching the given filter. \n"
"\"sources\": Builtin sources are: \"core\", \"rules\", \"scripts\", \"integrations\". May be extended by experience plugins.\n"
"\"columns\": Columns to be returned.\n"
"\"filter\": A map of column:value entries. Only = is supported currently.\n"
"\"startTime\": The datetime of the oldest entry, in ms.\n"
"\"endTime\": The datetime of the newest entry, in ms.\n"
"\"sampleRate\": If given, returns a sampled series of the values, filling in gaps with the previous value.\n"
"\"sortOrder\": Sort order of results. Note that this impacts the filling of gaps when resampling.\n"
"\"limit\": Maximum amount of entries to be returned.\n"
"\"offset\": Offset to be skipped before returning entries.";
params.insert("sources", QVariantList() << enumValueName(String));
params.insert("o:columns", QVariantList() << enumValueName(String));
params.insert("o:filter", enumValueName(Variant));
params.insert("o:startTime", enumValueName(Uint));
params.insert("o:endTime", enumValueName(Uint));
params.insert("o:sampleRate", enumRef<Types::SampleRate>());
params.insert("o:sortOrder", enumRef<Qt::SortOrder>());
params.insert("o:limit", enumValueName(Int));
params.insert("o:offset", enumValueName(Int));
returns.insert("loggingError", enumRef<Logging::LoggingError>());
returns.insert("o:logEntries", objectRef<LogEntries>());
returns.insert("count", enumValueName(Int));
returns.insert("offset", enumValueName(Int));
@ -87,20 +75,13 @@ LoggingHandler::LoggingHandler(QObject *parent) :
// Notifications
params.clear();
description = "Emitted whenever an entry is appended to the logging system. ";
description = "Emitted when a log entry is added. This will only be emitted for discrete series, not for resampled entries";
params.insert("logEntry", objectRef<LogEntry>());
registerNotification("LogEntryAdded", description, params);
params.clear();
description = "Emitted whenever the database was updated. "
"The database will be updated when a log entry was deleted. A log "
"entry will be deleted when the corresponding thing or a rule will "
"be removed, or when the oldest entry of the database was deleted to "
"keep to database in the size limits.";
registerNotification("LogDatabaseUpdated", description, params);
connect(NymeaCore::instance()->logEngine(), &LogEngine::logEntryAdded, this, &LoggingHandler::logEntryAdded);
connect(NymeaCore::instance()->logEngine(), &LogEngine::logDatabaseUpdated, this, &LoggingHandler::logDatabaseUpdated);
connect(m_logEngine, &LogEngine::logEntryAdded, this, [this](const LogEntry &logEntry){
emit LogEntryAdded({{"logEntry", packLogEntry(logEntry)}});
});
}
QString LoggingHandler::name() const
@ -108,42 +89,61 @@ QString LoggingHandler::name() const
return "Logging";
}
void LoggingHandler::logEntryAdded(const LogEntry &logEntry)
{
QVariantMap params;
params.insert("logEntry", packLogEntry(logEntry));
emit LogEntryAdded(params);
}
void LoggingHandler::logDatabaseUpdated()
{
emit LogDatabaseUpdated(QVariantMap());
}
JsonReply* LoggingHandler::GetLogEntries(const QVariantMap &params) const
{
LogFilter filter = unpackLogFilter(params);
LogEntriesFetchJob *job = NymeaCore::instance()->logEngine()->fetchLogEntries(filter);
JsonReply *reply = createAsyncReply("GetLogEntries");
connect(job, &LogEntriesFetchJob::finished, reply, [reply, job, filter](){
QVariantMap filter = params.value("filter").toMap();
QDateTime startTime;
if (params.contains("startTime")) {
startTime = QDateTime::fromMSecsSinceEpoch(params.value("startTime").toULongLong());
}
QDateTime endTime;
if (params.contains("endTime")) {
endTime = QDateTime::fromMSecsSinceEpoch(params.value("endTime").toULongLong());
}
Types::SampleRate sampleRate = Types::SampleRateAny;
if (params.contains("sampleRate")) {
QMetaEnum sampleRateEnum = QMetaEnum::fromType<Types::SampleRate>();
sampleRate = static_cast<Types::SampleRate>(sampleRateEnum.keyToValue(params.value("sampleRate").toByteArray()));
}
QStringList columns;
if (params.contains("columns")) {
columns = params.value("columns").toStringList();
}
QVariantList entries;
foreach (const LogEntry &entry, job->results()) {
entries.append(packLogEntry(entry));
int offset = params.value("offset").toInt();
int limit = params.value("limit").toInt();
Qt::SortOrder sortOrder = Qt::AscendingOrder;
if (params.contains("sortOrder")) {
sortOrder = enumNameToValue<Qt::SortOrder>(params.value("sortOrder").toString());
}
QStringList sources = params.value("sources").toStringList();
LogFetchJob *job = m_logEngine->fetchLogEntries(sources, columns, startTime, endTime, filter, sampleRate, sortOrder, offset, limit);
connect(job, &LogFetchJob::finished, reply, [reply](const LogEntries &entries){
QVariantList entryMaps;
foreach (const LogEntry &logEntry, entries) {
QVariantMap logEntryMap;
logEntryMap.insert("timestamp", logEntry.timestamp().toMSecsSinceEpoch());
logEntryMap.insert("source", logEntry.source());
QVariantMap values;
foreach (const QString &valueKey, logEntry.values().keys()) {
values.insert(valueKey, logEntry.values().value(valueKey));
}
logEntryMap.insert("values", values);
entryMaps.append(logEntryMap);
}
QVariantMap returns;
returns.insert("loggingError", enumValueName<Logging::LoggingError>(Logging::LoggingErrorNoError));
returns.insert("logEntries", entries);
returns.insert("offset", filter.offset());
returns.insert("count", entries.count());
reply->setData(returns);
QVariantMap params {
{"count", entries.count()},
{"offset", 0},
{"logEntries", entryMaps}
};
reply->setData(params);
reply->finished();
});
});
return reply;
}
@ -151,118 +151,13 @@ QVariantMap LoggingHandler::packLogEntry(const LogEntry &logEntry)
{
QVariantMap logEntryMap;
logEntryMap.insert("timestamp", logEntry.timestamp().toMSecsSinceEpoch());
logEntryMap.insert("loggingLevel", enumValueName<Logging::LoggingLevel>(logEntry.level()));
logEntryMap.insert("source", enumValueName<Logging::LoggingSource>(logEntry.source()));
logEntryMap.insert("eventType", enumValueName<Logging::LoggingEventType>(logEntry.eventType()));
if (logEntry.eventType() == Logging::LoggingEventTypeActiveChange)
logEntryMap.insert("active", logEntry.active());
if (logEntry.eventType() == Logging::LoggingEventTypeEnabledChange)
logEntryMap.insert("active", logEntry.active());
if (logEntry.level() == Logging::LoggingLevelAlert) {
switch (logEntry.source()) {
case Logging::LoggingSourceRules:
logEntryMap.insert("errorCode", enumValueName<RuleEngine::RuleError>(static_cast<RuleEngine::RuleError>(logEntry.errorCode())));
break;
case Logging::LoggingSourceActions:
case Logging::LoggingSourceEvents:
case Logging::LoggingSourceStates:
case Logging::LoggingSourceBrowserActions:
logEntryMap.insert("errorCode", enumValueName<Thing::ThingError>(static_cast<Thing::ThingError>(logEntry.errorCode())));
break;
case Logging::LoggingSourceSystem:
// FIXME: Update this once we support error codes for the general system
// logEntryMap.insert("errorCode", "");
break;
}
logEntryMap.insert("source", logEntry.source());
QVariantMap values;
foreach (const QString &valueKey, logEntry.values().keys()) {
values.insert(valueKey, logEntry.values().value(valueKey));
}
switch (logEntry.source()) {
case Logging::LoggingSourceActions:
case Logging::LoggingSourceEvents:
case Logging::LoggingSourceStates:
case Logging::LoggingSourceBrowserActions:
if (!logEntry.typeId().isNull()) {
logEntryMap.insert("typeId", logEntry.typeId());
}
logEntryMap.insert("thingId", logEntry.thingId());
logEntryMap.insert("value", LogValueTool::convertVariantToString(logEntry.value()));
break;
case Logging::LoggingSourceSystem:
logEntryMap.insert("active", logEntry.active());
break;
case Logging::LoggingSourceRules:
logEntryMap.insert("typeId", logEntry.typeId().toString());
break;
}
logEntryMap.insert("values", values);
return logEntryMap;
}
LogFilter LoggingHandler::unpackLogFilter(const QVariantMap &logFilterMap)
{
LogFilter filter;
if (logFilterMap.contains("timeFilters")) {
QVariantList timeFilters = logFilterMap.value("timeFilters").toList();
foreach (const QVariant &timeFilter, timeFilters) {
QVariantMap timeFilterMap = timeFilter.toMap();
QDateTime startDate; QDateTime endDate;
if (timeFilterMap.contains("startDate"))
startDate = QDateTime::fromTime_t(timeFilterMap.value("startDate").toUInt());
if (timeFilterMap.contains("endDate"))
endDate = QDateTime::fromTime_t(timeFilterMap.value("endDate").toUInt());
filter.addTimeFilter(startDate, endDate);
}
}
if (logFilterMap.contains("loggingSources")) {
QVariantList loggingSources = logFilterMap.value("loggingSources").toList();
foreach (const QVariant &source, loggingSources) {
filter.addLoggingSource(enumNameToValue<Logging::LoggingSource>(source.toString()));
}
}
if (logFilterMap.contains("loggingLevels")) {
QVariantList loggingLevels = logFilterMap.value("loggingLevels").toList();
foreach (const QVariant &level, loggingLevels) {
filter.addLoggingLevel(enumNameToValue<Logging::LoggingLevel>(level.toString()));
}
}
if (logFilterMap.contains("eventTypes")) {
QVariantList eventTypes = logFilterMap.value("eventTypes").toList();
foreach (const QVariant &eventType, eventTypes) {
filter.addLoggingEventType(enumNameToValue<Logging::LoggingEventType>(eventType.toString()));
}
}
if (logFilterMap.contains("typeIds")) {
QVariantList typeIds = logFilterMap.value("typeIds").toList();
foreach (const QVariant &typeId, typeIds) {
filter.addTypeId(typeId.toUuid());
}
}
if (logFilterMap.contains("thingIds")) {
QVariantList thingIds = logFilterMap.value("thingIds").toList();
foreach (const QVariant &thingId, thingIds) {
filter.addThingId(ThingId(thingId.toString()));
}
}
if (logFilterMap.contains("values")) {
QVariantList values = logFilterMap.value("values").toList();
foreach (const QVariant &value, values) {
filter.addValue(value.toString());
}
}
if (logFilterMap.contains("limit")) {
filter.setLimit(logFilterMap.value("limit", -1).toInt());
}
if (logFilterMap.contains("offset")) {
filter.setOffset(logFilterMap.value("offset").toInt());
}
return filter;
}
}

16
libnymea-core/jsonrpc/logginghandler.h
View File

@ -33,7 +33,8 @@
#include "jsonrpc/jsonhandler.h"
#include "logging/logentry.h"
#include "logging/logfilter.h"
class LogEngine;
namespace nymeaserver {
@ -41,24 +42,19 @@ class LoggingHandler : public JsonHandler
{
Q_OBJECT
public:
explicit LoggingHandler(QObject *parent = nullptr);
explicit LoggingHandler(LogEngine *logEngine, QObject *parent = nullptr);
QString name() const override;
Q_INVOKABLE JsonReply *GetLogEntries(const QVariantMap &params) const;
signals:
void LogEntryAdded(const QVariantMap &params);
void LogDatabaseUpdated(const QVariantMap &params);
private:
static QVariantMap packLogEntry(const LogEntry &logEntry);
static LogFilter unpackLogFilter(const QVariantMap &logFilterMap);
private slots:
void logEntryAdded(const LogEntry &entry);
void logDatabaseUpdated();
QVariantMap packLogEntry(const LogEntry &logEntry);
private:
LogEngine *m_logEngine = nullptr;
};
}

11
libnymea-core/libnymea-core.pro
View File

@ -61,6 +61,7 @@ HEADERS += nymeacore.h \
hardware/network/macaddressdatabasereplyimpl.h \
hardware/serialport/serialportmonitor.h \
hardware/zwave/zwavehardwareresourceimplementation.h \
logging/logengineinfluxdb.h \
scriptengine/scriptthing.h \
scriptengine/scriptthings.h \
zwave/zwavedevicedatabase.h \
@ -119,11 +120,6 @@ HEADERS += nymeacore.h \
jsonrpc/systemhandler.h \
jsonrpc/scriptshandler.h \
jsonrpc/usershandler.h \
logging/logging.h \
logging/logengine.h \
logging/logfilter.h \
logging/logentry.h \
logging/logvaluetool.h \
time/timemanager.h \
usermanager/userautorizer.h \
usermanager/userinfo.h \
@ -172,6 +168,7 @@ SOURCES += nymeacore.cpp \
hardware/network/macaddressdatabasereplyimpl.cpp \
hardware/serialport/serialportmonitor.cpp \
hardware/zwave/zwavehardwareresourceimplementation.cpp \
logging/logengineinfluxdb.cpp \
scriptengine/scriptthing.cpp \
scriptengine/scriptthings.cpp \
zwave/zwavedevicedatabase.cpp \
@ -223,10 +220,6 @@ SOURCES += nymeacore.cpp \
jsonrpc/systemhandler.cpp \
jsonrpc/scriptshandler.cpp \
jsonrpc/usershandler.cpp \
logging/logengine.cpp \
logging/logfilter.cpp \
logging/logentry.cpp \
logging/logvaluetool.cpp \
time/timemanager.cpp \
usermanager/userautorizer.cpp \
usermanager/userinfo.cpp \

830
libnymea-core/logging/logengine.cpp
View File

@ -1,830 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include "nymeasettings.h"
#include "logengine.h"
#include "loggingcategories.h"
#include "logging.h"
#include "logvaluetool.h"
#include "integrations/thingmanager.h"
#include <QCoreApplication>
#include <QSqlDatabase>
#include <QSqlDriver>
#include <QSqlQuery>
#include <QSqlRecord>
#include <QSqlError>
#include <QMetaEnum>
#include <QDateTime>
#include <QFileInfo>
#include <QTime>
#include <QtConcurrent/QtConcurrent>
#define DB_SCHEMA_VERSION 4
namespace nymeaserver {
// IMPORTANT:
// DatabaseJobs run threaded, however, QSql is *not* threadsafe.
// It is crucial to *not* access m_db while the job queue is being processed.
// That is, entire setup of the DB must happen before processQueue() is called
// and teardown must happen only after the job queue is empty.
LogEngine::LogEngine(const QString &driver, const QString &dbName, const QString &hostname, const QString &username, const QString &password, int maxDBSize, QObject *parent):
QObject(parent),
m_username(username),
m_password(password),
m_dbMaxSize(maxDBSize)
{
m_db = QSqlDatabase::addDatabase(driver, "logs");
m_db.setDatabaseName(dbName);
m_db.setHostName(hostname);
m_trimSize = qRound(0.01 * m_dbMaxSize);
m_maxQueueLength = 1000;
qCDebug(dcLogEngine) << "Opening logging database" << m_db.databaseName() << "(Max size:" << m_dbMaxSize << "trim size:" << m_trimSize << ")";
if (!m_db.isValid()) {
qCWarning(dcLogEngine) << "Database not valid:" << m_db.lastError().driverText() << m_db.lastError().databaseText();
rotate(m_db.databaseName());
}
if (!initDB(username, password)) {
qCWarning(dcLogEngine()) << "Error initializing database. Trying to correct it.";
if (QFileInfo(m_db.databaseName()).exists()) {
rotate(m_db.databaseName());
if (!initDB(username, password)) {
qCWarning(dcLogEngine()) << "Error fixing log database. Giving up. Logs can't be stored.";
return;
}
}
}
connect(&m_jobWatcher, SIGNAL(finished()), this, SLOT(handleJobFinished()));
checkDBSize();
}
LogEngine::~LogEngine()
{
// Process the job queue before allowing to shut down
while (m_currentJob) {
qCDebug(dcLogEngine()) << "Waiting for job to finish... (" << (m_priorityJobQueue.count() + m_jobQueue.count()) << "jobs left in queue)";
m_jobWatcher.waitForFinished();
// Make sure that the job queue is processes
// We can't call processQueue ourselves because thread synchronisation is done via queued connections
qApp->processEvents();
}
qCDebug(dcLogEngine()) << "Closing Database";
m_db.close();
}
LogEntriesFetchJob *LogEngine::fetchLogEntries(const LogFilter &filter)
{
QList<LogEntry> results;
QString limitString;
if (filter.limit() >= 0) {
limitString.append(QString("LIMIT %1 ").arg(filter.limit()));
}
if (filter.offset() > 0) {
limitString.append(QString("OFFSET %1").arg(QString::number(filter.offset())));
}
QString queryString;
if (filter.isEmpty()) {
queryString = QString("SELECT * FROM entries ORDER BY timestamp DESC %1;").arg(limitString);
} else {
queryString = QString("SELECT * FROM entries WHERE %1 ORDER BY timestamp DESC %2;").arg(filter.queryString()).arg(limitString);
}
DatabaseJob *job = new DatabaseJob(m_db, queryString, filter.values());
LogEntriesFetchJob *fetchJob = new LogEntriesFetchJob(this);
connect(job, &DatabaseJob::finished, this, [job, fetchJob](){
fetchJob->deleteLater();
if (job->error().isValid()) {
qCWarning(dcLogEngine) << "Error fetching log entries. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
fetchJob->finished();
return;
}
foreach (const QSqlRecord &result, job->results()) {
LogEntry entry(
QDateTime::fromMSecsSinceEpoch(result.value("timestamp").toLongLong()),
static_cast<Logging::LoggingLevel>(result.value("loggingLevel").toInt()),
static_cast<Logging::LoggingSource>(result.value("sourceType").toInt()),
result.value("errorCode").toInt());
entry.setTypeId(result.value("typeId").toUuid());
entry.setThingId(ThingId(result.value("thingId").toString()));
entry.setValue(result.value("value").toString());
entry.setEventType(static_cast<Logging::LoggingEventType>(result.value("loggingEventType").toInt()));
entry.setActive(result.value("active").toBool());
fetchJob->m_results.append(entry);
}
qCDebug(dcLogEngine) << "Fetched" << fetchJob->results().count() << "entries for db query:" << job->executedQuery();
fetchJob->finished();
});
enqueJob(job, true);
return fetchJob;
}
ThingsFetchJob *LogEngine::fetchThings()
{
QString queryString = QString("SELECT thingId FROM entries WHERE thingId != \"%1\" GROUP BY thingId;").arg(QUuid().toString());
DatabaseJob *job = new DatabaseJob(m_db, queryString);
ThingsFetchJob *fetchJob = new ThingsFetchJob(this);
connect(job, &DatabaseJob::finished, this, [job, fetchJob](){
fetchJob->deleteLater();
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine()) << "Error fetching device entries from log database:" << job->error().driverText() << job->error().databaseText();
fetchJob->finished();
return;
}
foreach (const QSqlRecord &result, job->results()) {
fetchJob->m_results.append(ThingId(result.value("thingId").toUuid()));
}
fetchJob->finished();
});
enqueJob(job, true);
return fetchJob;
}
bool LogEngine::jobsRunning() const
{
return !m_jobQueue.isEmpty() || !m_priorityJobQueue.isEmpty() || m_currentJob;
}
void LogEngine::setMaxLogEntries(int maxLogEntries, int trimSize)
{
m_dbMaxSize = maxLogEntries;
m_trimSize = trimSize;
trim();
}
void LogEngine::clearDatabase()
{
qCWarning(dcLogEngine) << "Clearing logging database.";
QString queryDeleteString = QString("DELETE FROM entries;");
DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job](){
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error clearing log database. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
return;
}
m_entryCount = 0;
emit logDatabaseUpdated();
});
enqueJob(job);
}
void LogEngine::logSystemEvent(const QDateTime &dateTime, bool active, Logging::LoggingLevel level)
{
qCDebug(dcLogEngine()) << "Logging system event:" << active;
LogEntry entry(dateTime, level, Logging::LoggingSourceSystem);
entry.setEventType(Logging::LoggingEventTypeActiveChange);
entry.setActive(active);
appendLogEntry(entry);
}
void LogEngine::logEvent(const Event &event)
{
QVariantList valueList;
foreach (const Param &param, event.params()) {
valueList << param.value();
}
LogEntry entry(Logging::LoggingSourceEvents);
entry.setTypeId(event.eventTypeId());
entry.setThingId(event.thingId());
if (valueList.count() == 1) {
entry.setValue(valueList.first());
} else {
entry.setValue(valueList);
}
appendLogEntry(entry);
}
void LogEngine::logStateChange(Thing *thing, const StateTypeId &stateTypeId, const QVariant &value)
{
LogEntry entry(Logging::LoggingSourceStates);
entry.setTypeId(stateTypeId);
entry.setThingId(thing->id());
entry.setValue(value);
appendLogEntry(entry);
}
void LogEngine::logAction(const Action &action, Thing::ThingError status)
{
Logging::LoggingLevel level = status == Thing::ThingErrorNoError ? Logging::LoggingLevelInfo : Logging::LoggingLevelAlert;
LogEntry entry(QDateTime::currentDateTime(), level, Logging::LoggingSourceActions, status);
entry.setTypeId(action.actionTypeId());
entry.setThingId(action.thingId());
if (action.params().isEmpty()) {
entry.setValue(QVariant());
} else if (action.params().count() == 1) {
entry.setValue(action.params().first().value());
} else {
QVariantList valueList;
foreach (const Param &param, action.params()) {
valueList << param.value();
}
entry.setValue(valueList);
}
appendLogEntry(entry);
}
void LogEngine::logBrowserAction(const BrowserAction &browserAction, Logging::LoggingLevel level, int errorCode)
{
LogEntry entry(level, Logging::LoggingSourceBrowserActions, errorCode);
entry.setThingId(browserAction.thingId());
entry.setValue(browserAction.itemId());
appendLogEntry(entry);
}
void LogEngine::logBrowserItemAction(const BrowserItemAction &browserItemAction, Logging::LoggingLevel level, int errorCode)
{
LogEntry entry(level, Logging::LoggingSourceBrowserActions, errorCode);
entry.setThingId(browserItemAction.thingId());
entry.setTypeId(browserItemAction.actionTypeId());
entry.setValue(browserItemAction.itemId());
appendLogEntry(entry);
}
void LogEngine::logRuleTriggered(const Rule &rule)
{
LogEntry entry(Logging::LoggingSourceRules);
entry.setTypeId(rule.id());
entry.setEventType(Logging::LoggingEventTypeTrigger);
appendLogEntry(entry);
}
void LogEngine::logRuleActiveChanged(const Rule &rule)
{
LogEntry entry(Logging::LoggingSourceRules);
entry.setTypeId(rule.id());
entry.setActive(rule.active());
entry.setEventType(Logging::LoggingEventTypeActiveChange);
appendLogEntry(entry);
}
void LogEngine::logRuleEnabledChanged(const Rule &rule, const bool &enabled)
{
LogEntry entry(Logging::LoggingSourceRules);
entry.setTypeId(rule.id());
entry.setEventType(Logging::LoggingEventTypeEnabledChange);
entry.setActive(enabled);
appendLogEntry(entry);
}
void LogEngine::logRuleActionsExecuted(const Rule &rule)
{
LogEntry entry(Logging::LoggingSourceRules);
entry.setTypeId(rule.id());
entry.setEventType(Logging::LoggingEventTypeActionsExecuted);
appendLogEntry(entry);
}
void LogEngine::logRuleExitActionsExecuted(const Rule &rule)
{
LogEntry entry(Logging::LoggingSourceRules);
entry.setTypeId(rule.id());
entry.setEventType(Logging::LoggingEventTypeExitActionsExecuted);
appendLogEntry(entry);
}
void LogEngine::removeThingLogs(const ThingId &thingId)
{
qCDebug(dcLogEngine) << "Deleting log entries from device" << thingId.toString();
QString queryDeleteString = QString("DELETE FROM entries WHERE thingId = '%1';").arg(thingId.toString());
DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job, thingId](){
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting log entries from device" << thingId.toString() << ". Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
return;
}
emit logDatabaseUpdated();
checkDBSize();
});
enqueJob(job);
}
void LogEngine::removeRuleLogs(const RuleId &ruleId)
{
qCDebug(dcLogEngine) << "Deleting log entries from rule" << ruleId.toString();
QString queryDeleteString = QString("DELETE FROM entries WHERE typeId = '%1';").arg(ruleId.toString());
DatabaseJob *job = new DatabaseJob(m_db, queryDeleteString);
connect(job, &DatabaseJob::finished, this, [this, job, ruleId](){
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting log entries from rule" << ruleId.toString() << ". Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
return;
}
emit logDatabaseUpdated();
checkDBSize();
});
enqueJob(job);
}
void LogEngine::appendLogEntry(const LogEntry &entry)
{
qCDebug(dcLogEngine()) << "Adding log entry:" << entry;
QString queryString = QString("INSERT INTO entries (timestamp, loggingEventType, loggingLevel, sourceType, typeId, thingId, value, active, errorCode) values (?, ?, ?, ?, ?, ?, ?, ?, ?);");
QVariantList bindValues;
bindValues.append(entry.timestamp().toMSecsSinceEpoch());
bindValues.append(entry.eventType());
bindValues.append(entry.level());
bindValues.append(entry.source());
bindValues.append(entry.typeId().toString());
bindValues.append(entry.thingId().toString());
bindValues.append(LogValueTool::convertVariantToString(entry.value()));
bindValues.append(entry.active());
bindValues.append(entry.errorCode());
DatabaseJob *job = new DatabaseJob(m_db, queryString, bindValues);
// Check for log flooding. If we are exceeding the queue we'll start flagging log events of a certain type.
// If we'll get more log events of the same type while the queue is still exceededd, we'll discard the old
// ones and queue up the new one instead. The most recent one is more important (i.e. we don't want to lose
// the last event in a series).
if (m_jobQueue.count() > m_maxQueueLength) {
if (!m_initialized) {
qCDebug(dcLogEngine()) << "Log DB not initialized and queue is full. Discarding log entry.";
delete job;
return;
}
qCDebug(dcLogEngine()) << "An excessive amount of data is being logged. (" << m_jobQueue.length() << "jobs in the queue)";
if (m_flaggedJobs.contains(entry.typeId().toString() + entry.thingId().toString())) {
if (m_flaggedJobs.value(entry.typeId().toString() + entry.thingId().toString()).count() > 10) {
qCWarning(dcLogEngine()) << "Discarding log entry because of excessive log flooding.";
DatabaseJob *job = m_flaggedJobs[entry.typeId().toString() + entry.thingId().toString()].takeFirst();
int jobIdx = m_jobQueue.indexOf(job);
m_jobQueue.takeAt(jobIdx)->deleteLater();
}
}
m_flaggedJobs[entry.typeId().toString() + entry.thingId().toString()].append(job);
}
connect(job, &DatabaseJob::finished, this, [this, job, entry](){
m_flaggedJobs[entry.typeId().toString() + entry.thingId().toString()].removeAll(job);
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error writing log entry. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
qCWarning(dcLogEngine) << entry;
m_dbMalformed = true;
return;
}
emit logEntryAdded(entry);
m_entryCount++;
trim();
});
enqueJob(job);
}
void LogEngine::checkDBSize()
{
DatabaseJob *job = new DatabaseJob(m_db, "SELECT COUNT(*) FROM entries;");
connect(job, &DatabaseJob::finished, this, [this, job](){
if (job->error().type() != QSqlError::NoError || job->results().count() == 0) {
qCWarning(dcLogEngine()) << "Error fetching log DB size. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
m_entryCount = 0;
return;
}
m_entryCount = job->results().first().value(0).toInt();
});
enqueJob(job, true);
}
void LogEngine::trim()
{
if (m_dbMaxSize == -1 || m_entryCount < m_dbMaxSize) {
// No trimming required
return;
}
QDateTime startTime = QDateTime::currentDateTime();
QString queryDeleteString = QString("DELETE FROM entries WHERE ROWID IN (SELECT ROWID FROM entries ORDER BY timestamp DESC LIMIT -1 OFFSET %1);").arg(QString::number(m_dbMaxSize - m_trimSize));
DatabaseJob *deleteJob = new DatabaseJob(m_db, queryDeleteString);
connect(deleteJob, &DatabaseJob::finished, this, [this, deleteJob, startTime](){
if (deleteJob->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting oldest log entries to keep size. Driver error:" << deleteJob->error().driverText() << "Database error:" << deleteJob->error().databaseText();
}
qCDebug(dcLogEngine()) << "Ran housekeeping on log database in" << startTime.msecsTo(QDateTime::currentDateTime()) << "ms. (Deleted" << m_entryCount - (m_dbMaxSize - m_trimSize) << "entries)";
m_entryCount = m_dbMaxSize - m_trimSize;
emit logDatabaseUpdated();
});
qCDebug(dcLogEngine()) << "Scheduling housekeeping job.";
enqueJob(deleteJob, true);
}
void LogEngine::enqueJob(DatabaseJob *job, bool priority)
{
if (priority) {
m_priorityJobQueue.append(job);
qCDebug(dcLogEngine()) << "Scheduled priority job at position" << (m_priorityJobQueue.count() - 1) << "(" << m_priorityJobQueue.count() << "jobs in the queue)";
} else {
m_jobQueue.append(job);
qCDebug(dcLogEngine()) << "Scheduled job at position" << (m_jobQueue.count() - 1) << "(" << m_jobQueue.count() << "jobs in the queue)";
}
processQueue();
}
void LogEngine::processQueue()
{
if (!m_initialized) {
return;
}
if (m_priorityJobQueue.isEmpty() && m_jobQueue.isEmpty()) {
emit jobsRunningChanged();
return;
}
if (m_currentJob) {
return;
}
emit jobsRunningChanged();
if (m_dbMalformed) {
qCWarning(dcLogEngine()) << "Database is malformed. Trying to recover...";
m_db.close();
rotate(m_db.databaseName());
initDB(m_username, m_password);
m_dbMalformed = false;
}
DatabaseJob *job = nullptr;
if (!m_priorityJobQueue.isEmpty()) {
job = m_priorityJobQueue.takeFirst();
qCDebug(dcLogEngine()) << "Processing DB priority queue. (" << m_priorityJobQueue.count() << "jobs left in queue," << m_entryCount << "entries in DB)";
} else {
job = m_jobQueue.takeFirst();
qCDebug(dcLogEngine()) << "Processing DB queue. (" << m_jobQueue.count() << "jobs left in queue," << m_entryCount << "entries in DB)";
}
m_currentJob = job;
QFuture<DatabaseJob*> future = QtConcurrent::run([job](){
QSqlQuery query(job->m_db);
query.prepare(job->m_queryString);
foreach (const QVariant &value, job->m_bindValues) {
query.addBindValue(value);
}
query.exec();
job->m_error = query.lastError();
job->m_executedQuery = query.executedQuery();
if (!query.lastError().isValid()) {
while (query.next()) {
job->m_results.append(query.record());
}
}
return job;
});
m_jobWatcher.setFuture(future);
}
void LogEngine::handleJobFinished()
{
DatabaseJob *job = m_jobWatcher.result();
job->finished();
job->deleteLater();
m_currentJob = nullptr;
qCDebug(dcLogEngine()) << "DB job finished. (" << m_entryCount << "entries in DB)";
processQueue();
}
void LogEngine::rotate(const QString &dbName)
{
int index = 1;
while (QFileInfo(QString("%1.%2").arg(dbName).arg(index)).exists()) {
index++;
}
qCDebug(dcLogEngine()) << "Backing up old database file to" << QString("%1.%2").arg(dbName).arg(index);
QFile f(dbName);
if (!f.rename(QString("%1.%2").arg(dbName).arg(index))) {
qCWarning(dcLogEngine()) << "Error backing up old database.";
} else {
qCDebug(dcLogEngine()) << "Successfully moved old database";
}
}
bool LogEngine::migrateDatabaseVersion3to4()
{
QSqlQuery renameQuery = m_db.exec("ALTER TABLE entries RENAME TO _entries_v3;");
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine) << "Error migrating database verion 3 -> 4 (renaming table). Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
return false;
}
qCDebug(dcLogEngine()) << "Renamed entries table to entries_v3:" << m_db.lastError().text();
m_db.close();
m_db.open(m_username, m_password);
QSqlQuery createQuery = m_db.exec("CREATE TABLE entries "
"("
"timestamp BIGINT,"
"loggingLevel INT,"
"sourceType INT,"
"typeId VARCHAR(38),"
"thingId VARCHAR(38),"
"value VARCHAR(100),"
"loggingEventType INT,"
"active BOOL,"
"errorCode INT,"
"FOREIGN KEY(sourceType) REFERENCES sourceTypes(id),"
"FOREIGN KEY(loggingEventType) REFERENCES loggingEventTypes(id)"
");");
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine) << "Error migrating database verion 3 -> 4 (creating new table). Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
return false;
}
qCDebug(dcLogEngine()) << "Created new entries table:" << m_db.lastError().text();
qCDebug(dcLogEngine()) << "Updating database version to" << DB_SCHEMA_VERSION;
m_db.exec(QString("UPDATE metadata SET data = %1 WHERE `key` = 'version';").arg(DB_SCHEMA_VERSION));
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine) << "Error updating database verion 3 -> 4. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
return false;
}
qCDebug(dcLogEngine()) << "Migrated database schema from version 3 to 4.";
return true;
}
void LogEngine::migrateEntries3to4()
{
QString selectQuery = QString("SELECT * FROM _entries_v3;");
DatabaseJob *job = new DatabaseJob(m_db, selectQuery);
connect(job, &DatabaseJob::finished, this, [this, job](){
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error fetching entries to migrate. Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
m_dbMalformed = true;
return;
}
if (job->results().isEmpty()) {
qCDebug(dcLogEngine()) << "No items to migrate from schema 3 to 4 remaining.";
finalizeMigration3To4();
return;
}
int count = job->results().count();
QSqlRecord result = job->results().first();
QString encodedValue = result.value("value").toByteArray();
QString decodedValue = LogValueTool::convertVariantToString(LogValueTool::deserializeValue(encodedValue));
QString insertCall = QString("INSERT INTO entries (timestamp, loggingEventType, loggingLevel, sourceType, typeId, thingId, value, active, errorCode) values ('%1', '%2', '%3', '%4', '%5', '%6', '%7', '%8', '%9');")
.arg(result.value("timestamp").toLongLong() * 1000)
.arg(result.value("loggingEventType").toInt())
.arg(result.value("loggingLevel").toInt())
.arg(result.value("sourceType").toInt())
.arg(result.value("typeId").toString())
.arg(result.value("deviceId").toString())
.arg(decodedValue)
.arg(result.value("active").toBool())
.arg(result.value("errorCode").toInt());
DatabaseJob *insertJob = new DatabaseJob(m_db, insertCall);
connect(insertJob, &DatabaseJob::finished, this, [this, insertJob, count, result](){
if (insertJob->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error fetching entries to migrate. Driver error:" << insertJob->error().driverText() << "Database error:" << insertJob->error().databaseText();
m_dbMalformed = true;
return;
}
QString deleteCall = QString("DELETE FROM _entries_v3 WHERE timestamp = '%1' AND loggingEventType = '%2' AND loggingLevel = '%3' AND sourceType = '%4' AND typeId = '%5' AND deviceId = '%6' AND value = '%7' AND active = '%8' AND errorCode = '%9';")
.arg(result.value("timestamp").toLongLong())
.arg(result.value("loggingEventType").toInt())
.arg(result.value("loggingLevel").toInt())
.arg(result.value("sourceType").toInt())
.arg(result.value("typeId").toString())
.arg(result.value("deviceId").toString())
.arg(result.value("value").toString())
.arg(result.value("active").toBool())
.arg(result.value("errorCode").toInt());
DatabaseJob *deleteJob = new DatabaseJob(m_db, deleteCall);
connect(deleteJob, &DatabaseJob::finished, this, [this, deleteJob, count, result](){
if (deleteJob->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error deleting old entry during migration. Driver error:" << deleteJob->error().driverText() << "Database error:" << deleteJob->error().databaseText();
finalizeMigration3To4();
return;
}
qCDebug(dcLogEngine()) << "Migrated log entry from version 3 to 4." << (count - 1) << "items left to migrate";
if (count - 1 > 0) {
migrateEntries3to4();
} else {
finalizeMigration3To4();
}
});
enqueJob(deleteJob);
});
enqueJob(insertJob);
});
enqueJob(job);
}
void LogEngine::finalizeMigration3To4()
{
qCDebug(dcLogEngine()) << "Finalizing migration of database version 3 to 4.";
QString selectQuery = QString("DROP TABLE _entries_v3;");
DatabaseJob *job = new DatabaseJob(m_db, selectQuery);
enqueJob(job);
connect(job, &DatabaseJob::finished, this, [job](){
if (job->error().type() != QSqlError::NoError) {
qCWarning(dcLogEngine) << "Error finalizing migration from 3 to 4 (drop entries_v3). Driver error:" << job->error().driverText() << "Database error:" << job->error().databaseText();
return;
}
});
}
bool LogEngine::initDB(const QString &username, const QString &password)
{
m_db.close();
m_initialized = false;
bool opened = m_db.open(username, password);
if (!opened) {
qCWarning(dcLogEngine()) << "Can't open Log DB. Init failed.";
return false;
}
if (!m_db.tables().contains("metadata")) {
qCDebug(dcLogEngine()) << "Empty Database. Setting up metadata...";
m_db.exec("CREATE TABLE metadata (`key` VARCHAR(10), data VARCHAR(40));");
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine) << "Error initualizing database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
m_db.close();
return false;
}
m_db.exec(QString("INSERT INTO metadata (`key`, data) VALUES('version', '%1');").arg(DB_SCHEMA_VERSION));
}
QSqlQuery query = m_db.exec("SELECT data FROM metadata WHERE `key` = 'version';");
if (query.next()) {
int version = query.value("data").toInt();
// Migration from 3 -> 4
if (version == 3) {
if (!migrateDatabaseVersion3to4()) {
qCWarning(dcLogEngine()) << "Migration process failed.";
m_db.close();
return false;
} else {
// Successfully migrated
version = 4;
}
}
if (version != DB_SCHEMA_VERSION) {
qCWarning(dcLogEngine) << "Log schema version not matching! Schema upgrade not implemented for this version change.";
m_db.close();
return false;
} else {
qCDebug(dcLogEngine) << QString("Log database schema version \"%1\" matches").arg(DB_SCHEMA_VERSION).toLatin1().data();
// If there is still a deviceId column, schedule items to be migrated in the
// background with low priority as this might take hours
if (m_db.tables().contains("_entries_v3")) {
migrateEntries3to4();
}
}
} else {
qCWarning(dcLogEngine) << "Broken log database. Version not found in metadata table.";
m_db.close();
return false;
}
if (!m_db.tables().contains("sourceTypes")) {
m_db.exec("CREATE TABLE sourceTypes (id int, name varchar(20), PRIMARY KEY(id));");
//qCDebug(dcLogEngine) << m_db.lastError().databaseText();
QMetaEnum logTypes = Logging::staticMetaObject.enumerator(Logging::staticMetaObject.indexOfEnumerator("LoggingSource"));
Q_ASSERT_X(logTypes.isValid(), "LogEngine", "Logging has no enum LoggingSource");
for (int i = 0; i < logTypes.keyCount(); i++) {
m_db.exec(QString("INSERT INTO sourceTypes (id, name) VALUES(%1, '%2');").arg(i).arg(logTypes.key(i)));
}
}
if (!m_db.tables().contains("loggingEventTypes")) {
m_db.exec("CREATE TABLE loggingEventTypes (id int, name varchar(40), PRIMARY KEY(id));");
//qCDebug(dcLogEngine) << m_db.lastError().databaseText();
QMetaEnum logTypes = Logging::staticMetaObject.enumerator(Logging::staticMetaObject.indexOfEnumerator("LoggingEventType"));
Q_ASSERT_X(logTypes.isValid(), "LogEngine", "Logging has no enum LoggingEventType");
for (int i = 0; i < logTypes.keyCount(); i++) {
m_db.exec(QString("INSERT INTO loggingEventTypes (id, name) VALUES(%1, '%2');").arg(i).arg(logTypes.key(i)));
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine()) << "Failed to insert loggingEventTypes into DB. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
}
}
}
if (!m_db.tables().contains("entries")) {
qCDebug(dcLogEngine()) << "No \"entries\" table in database. Creating it.";
m_db.exec("CREATE TABLE entries "
"("
"timestamp BIGINT,"
"loggingLevel INT,"
"sourceType INT,"
"typeId VARCHAR(38),"
"thingId VARCHAR(38),"
"value VARCHAR(100),"
"loggingEventType INT,"
"active BOOL,"
"errorCode INT,"
"FOREIGN KEY(sourceType) REFERENCES sourceTypes(id),"
"FOREIGN KEY(loggingEventType) REFERENCES loggingEventTypes(id)"
");");
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine) << "Error creating log table in database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
m_db.close();
return false;
}
}
m_db.exec("CREATE INDEX IF NOT EXISTS idx_query_single_thing ON entries (thingId);");
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine()) << "Error creating entries table thing index in log database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
return false;
}
m_db.exec("CREATE INDEX IF NOT EXISTS idx_query_single_type ON entries (typeId, thingId);");
if (m_db.lastError().isValid()) {
qCWarning(dcLogEngine()) << "Error creating entries table state index in log database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText();
return false;
}
qCDebug(dcLogEngine) << "Initialized logging DB successfully. (maximum DB size:" << m_dbMaxSize << ")";
m_initialized = true;
return true;
}
}

189
libnymea-core/logging/logengine.h
View File

@ -1,189 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef LOGENGINE_H
#define LOGENGINE_H
#include "logentry.h"
#include "logfilter.h"
#include "types/event.h"
#include "types/action.h"
#include "types/browseritemaction.h"
#include "types/browseraction.h"
#include "ruleengine/rule.h"
#include "integrations/thingmanager.h"
#include <QObject>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QSqlError>
#include <QSqlRecord>
#include <QTimer>
#include <QFutureWatcher>
namespace nymeaserver {
class DatabaseJob;
class LogEntriesFetchJob;
class ThingsFetchJob;
class LogEngine: public QObject
{
Q_OBJECT
public:
LogEngine(const QString &driver, const QString &dbName, const QString &hostname = QString("127.0.0.1"), const QString &username = QString(), const QString &password = QString(), int maxDBSize = 50000, QObject *parent = nullptr);
~LogEngine();
LogEntriesFetchJob *fetchLogEntries(const LogFilter &filter = LogFilter());
ThingsFetchJob *fetchThings();
bool jobsRunning() const;
void setMaxLogEntries(int maxLogEntries, int trimSize);
void clearDatabase();
void removeThingLogs(const ThingId &thingId);
void removeRuleLogs(const RuleId &ruleId);
public slots:
void logSystemEvent(const QDateTime &dateTime, bool active, Logging::LoggingLevel level = Logging::LoggingLevelInfo);
void logEvent(const Event &event);
void logStateChange(Thing *thing, const StateTypeId &stateTypeId, const QVariant &value);
void logAction(const Action &action, Thing::ThingError status);
void logBrowserAction(const BrowserAction &browserAction, Logging::LoggingLevel level = Logging::LoggingLevelInfo, int errorCode = 0);
void logBrowserItemAction(const BrowserItemAction &browserItemAction, Logging::LoggingLevel level = Logging::LoggingLevelInfo, int errorCode = 0);
void logRuleTriggered(const Rule &rule);
void logRuleActiveChanged(const Rule &rule);
void logRuleEnabledChanged(const Rule &rule, const bool &enabled);
void logRuleActionsExecuted(const Rule &rule);
void logRuleExitActionsExecuted(const Rule &rule);
signals:
void logEntryAdded(const LogEntry &logEntry);
void logDatabaseUpdated();
void jobsRunningChanged();
private:
bool initDB(const QString &username, const QString &password);
void appendLogEntry(const LogEntry &entry);
void rotate(const QString &dbName);
bool migrateDatabaseVersion3to4();
void migrateEntries3to4();
void finalizeMigration3To4();
private slots:
void checkDBSize();
void trim();
void enqueJob(DatabaseJob *job, bool priority = false);
void processQueue();
void handleJobFinished();
private:
QSqlDatabase m_db;
QString m_username;
QString m_password;
int m_dbMaxSize;
int m_trimSize;
int m_entryCount = 0;
bool m_initialized = false;
bool m_dbMalformed = false;
// When maxQueueLength is exceeded, jobs will be flagged and discarded if this source logs more events
int m_maxQueueLength;
QHash<QString, QList<DatabaseJob*>> m_flaggedJobs;
QList<DatabaseJob*> m_jobQueue;
QList<DatabaseJob*> m_priorityJobQueue;
DatabaseJob *m_currentJob = nullptr;
QFutureWatcher<DatabaseJob*> m_jobWatcher;
};
class DatabaseJob: public QObject
{
Q_OBJECT
public:
DatabaseJob(const QSqlDatabase &db, const QString &queryString, const QVariantList &bindValues = QVariantList()):
m_db(db),
m_queryString(queryString),
m_bindValues(bindValues)
{
}
QString executedQuery() const { return m_executedQuery; }
QSqlError error() const { return m_error; }
QList<QSqlRecord> results() const { return m_results; }
signals:
void finished();
private:
QSqlDatabase m_db;
QString m_queryString;
QVariantList m_bindValues;
QString m_executedQuery;
QSqlError m_error;
QList<QSqlRecord> m_results;
friend class LogEngine;
};
class LogEntriesFetchJob: public QObject
{
Q_OBJECT
public:
LogEntriesFetchJob(QObject *parent): QObject(parent) {}
QList<LogEntry> results() { return m_results; }
signals:
void finished();
private:
QList<LogEntry> m_results;
friend class LogEngine;
};
class ThingsFetchJob: public QObject
{
Q_OBJECT
public:
ThingsFetchJob(QObject *parent): QObject(parent) {}
QList<ThingId> results() { return m_results; }
signals:
void finished();
private:
QList<ThingId> m_results;
friend class LogEngine;
};
}
#endif

791
libnymea-core/logging/logengineinfluxdb.cpp Normal file
View File

@ -0,0 +1,791 @@
#include "logengineinfluxdb.h"
#include <QNetworkReply>
#include <QUrlQuery>
#include <QJsonDocument>
#include <QCoreApplication>
#include <QTimer>
LogEngineInfluxDB::LogEngineInfluxDB(const QString &host, const QString &dbName, const QString &username, const QString &password, QObject *parent)
: LogEngine{parent},
m_host(host),
m_dbName(dbName),
m_username(username),
m_password(password)
{
m_nam = new QNetworkAccessManager(this);
initDB();
}
LogEngineInfluxDB::~LogEngineInfluxDB()
{
if (m_initStatus == InitStatusStarting) {
m_initStatus = InitStatusFailure;
}
if (jobsRunning()) {
qCInfo(dcLogEngine()) << "Waiting for" << (m_initQueryQueue.count() + m_queryQueue.count() + m_writeQueue.count()) << "jobs to finish... Init status:" << m_initStatus;
}
while (jobsRunning()) {
// qCDebug(dcLogEngine()) << "Waiting for logs to finish processing." << m_writeQueue.count() << "jobs pending...";
processQueues();
qApp->processEvents();
}
}
Logger *LogEngineInfluxDB::registerLogSource(const QString &name, const QStringList &tagNames, Types::LoggingType loggingType, const QString &sampleColumn)
{
if (m_loggers.contains(name)) {
qCCritical(dcLogEngine()) << "Log source" << name << "already registerd. Not registering a second time.";
return nullptr;
}
// qCDebug(dcLogEngine()) << "Registering log source" << name << "with tags" << tagNames;
Logger *logger = createLogger(name, tagNames, loggingType);
m_loggers.insert(name, logger);
if (loggingType == Types::LoggingTypeSampled) {
qCDebug(dcLogEngine()) << "Setting up log sampling on" << sampleColumn;
if (sampleColumn.isEmpty()) {
qCCritical(dcLogEngine()) << "Sample type != None but no sample column given. Unable to create samples for" << name;
} else {
QStringList columns;
columns.append(QString("MIN(\"%1\") AS min_%1").arg(sampleColumn));
columns.append(QString("MAX(\"%1\") AS max_%1").arg(sampleColumn));
columns.append(QString("MEAN(\"%1\") AS %1").arg(sampleColumn));
QString target = columns.join(", ");
QueryJob *minutesJob = query(QString("CREATE CONTINUOUS QUERY \"minutes-%1\" "
"ON \"nymea\" "
"BEGIN "
"SELECT %2 "
"INTO minutes.\"%1\" "
"FROM live.\"%1\" "
"GROUP BY time(1m) "
"fill(previous) "
"END").arg(name).arg(target), true);
connect(minutesJob, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status, const QVariantList &response){
if (status == QNetworkReply::NoError) {
qCDebug(dcLogEngine()) << "Created minute based continuous query for" << name << qUtf8Printable(QJsonDocument::fromVariant(response).toJson());
} else {
qCWarning(dcLogEngine()) << "Unable to create minute based continuous query for" << name << qUtf8Printable(QJsonDocument::fromVariant(response).toJson());
}
});
QueryJob *hoursJob = query(QString("CREATE CONTINUOUS QUERY \"hours-%1\" "
"ON \"nymea\" "
"BEGIN "
"SELECT %2 "
"INTO hours.\"%1\" "
"FROM minutes.\"%1\" "
"GROUP BY time(1h) "
"fill(previous) "
"END").arg(name).arg(target), true);
connect(hoursJob, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status, const QVariantList &response){
if (status == QNetworkReply::NoError) {
qCDebug(dcLogEngine()) << "Created hour based continuous query for" << name << qUtf8Printable(QJsonDocument::fromVariant(response).toJson());
} else {
qCWarning(dcLogEngine()) << "Unable to create hour based continuous query for" << name << qUtf8Printable(QJsonDocument::fromVariant(response).toJson());
}
});
QueryJob *daysJob = query(QString("CREATE CONTINUOUS QUERY \"days-%1\" "
"ON \"nymea\" "
"BEGIN "
"SELECT %2 "
"INTO days.\"%1\" "
"FROM hours.\"%1\" "
"GROUP BY time(24h) "
"fill(previous) "
"END").arg(name).arg(target), true);
connect(daysJob, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status, const QVariantList &response){
if (status == QNetworkReply::NoError) {
qCDebug(dcLogEngine()) << "Created day based continuous query for" << name << qUtf8Printable(QJsonDocument::fromVariant(response).toJson());
} else {
qCWarning(dcLogEngine()) << "Unable to create days based continuous query for" << name << qUtf8Printable(QJsonDocument::fromVariant(response).toJson());
}
});
}
}
return logger;
}
void LogEngineInfluxDB::unregisterLogSource(const QString &name)
{
Logger *logger = m_loggers.take(name);
if (!logger) {
qCWarning(dcLogEngine()) << "Log source" << name << "unknown. Cannot unregister.";
return;
}
QString queryString = QString("DROP MEASUREMENT \"%1\"").arg(name);
qCInfo(dcLogEngine()) << "Removing log entries:" << queryString;
QueryJob *job = query(queryString);
connect(job, &QueryJob::finished, this, [name](bool success){
if (success) {
qCDebug(dcLogEngine()) << "Removed log entries for source" << name;
} else {
qCWarning(dcLogEngine()) << "Removing log entries for source" << name << "failed";
}
});
}
void LogEngineInfluxDB::logEvent(Logger *logger, const QStringList &tags, const QVariantMap &values)
{
QString measurement = logger->name();
QStringList tagsList;
QStringList fieldsList;
QDateTime timestamp = QDateTime::currentDateTime();
QVariantMap combinedValues;
for (int i = 0; i < qMin(logger->tagNames().count(), tags.count()); i++) {
tagsList.append(QString("%1=%2").arg(logger->tagNames().at(i)).arg(tags.at(i)));
combinedValues.insert(logger->tagNames().at(i), tags.at(i));
}
foreach (const QString &key, values.keys()) {
combinedValues.insert(key, values.value(key));
}
foreach (const QString &name, values.keys()) {
QVariant value = values.value(name);
switch (value.type()) {
case QVariant::String:
case QVariant::ByteArray:
fieldsList.append(QString("%1=\"%2\"").arg(name).arg(QString(value.toByteArray().toPercentEncoding())));
break;
case QVariant::Uuid:
fieldsList.append(QString("%1=\"%2\"").arg(name).arg(value.toString()));
break;
case QVariant::Int:
case QVariant::UInt:
case QVariant::LongLong:
case QVariant::ULongLong:
fieldsList.append(QString("%1=%2i").arg(name).arg(value.toString()));
break;
default:
fieldsList.append(QString("%1=%2").arg(name).arg(value.toString()));
}
}
tagsList.prepend(measurement);
QString measurementAndTags = tagsList.join(',').trimmed();
QString fieldsString = fieldsList.join(',').trimmed();
if (!fieldsList.isEmpty()) {
fieldsString.append(' ');
}
QString data = measurementAndTags + " " + fieldsString + QString::number(timestamp.toMSecsSinceEpoch());
QString retentionPolicy = logger->loggingType() == Types::LoggingTypeSampled ? "live" : "discrete";
LogEntry entry(timestamp, logger->name(), combinedValues);
QueueEntry queueEntry;
queueEntry.request = createWriteRequest(retentionPolicy);
queueEntry.request.setHeader(QNetworkRequest::ContentTypeHeader, "application/text");
queueEntry.data = data.toUtf8();
queueEntry.entry = entry;
m_writeQueue.append(queueEntry);
processQueues();
}
void LogEngineInfluxDB::processQueues()
{
if (m_initStatus == InitStatusFailure) {
m_writeQueue.clear();
qDeleteAll(m_queryQueue);
m_queryQueue.clear();
qDeleteAll(m_initQueryQueue);
m_initQueryQueue.clear();
return;
}
// qCDebug(dcLogEngine()) << "Processing queue:" << m_initStatus << "init count:" << m_initQueryQueue.count() << "query count:" << m_queryQueue.count() << "write count:" << m_writeQueue.count();
if (!m_currentInitQuery && !m_initQueryQueue.isEmpty()) {
QueryJob *job = m_initQueryQueue.takeFirst();
QNetworkReply *reply;
qCDebug(dcLogEngine()) << "Sending init query to influx" << job->m_request.url();
if (job->m_post) {
reply = m_nam->post(job->m_request, QByteArray());
} else {
reply = m_nam->get(job->m_request);
}
m_currentInitQuery = job;
connect(reply, &QNetworkReply::finished, job, [=](){
m_currentInitQuery = nullptr;
qCDebug(dcLogEngine()) << "Init query job finished";
reply->deleteLater();
if (reply->error() == QNetworkReply::ProtocolInvalidOperationError) {
qCWarning(dcLogEngine()) << "Influx DB protocol error:" << reply->readAll();
job->finish(reply->error());
return;
}
if (reply->error() != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Error in influxdb communication:" << reply->error() << reply->errorString() << "for query:" << job->m_request.url().toString();
job->finish(reply->error());
return;
}
QByteArray data = reply->readAll();
QJsonParseError error;
QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &error);
if (error.error != QJsonParseError::NoError) {
qCWarning(dcLogEngine) << "Unable to process response from influxdb:" << error.errorString() << qUtf8Printable(data);
job->finish(QNetworkReply::ProtocolFailure);
return;
}
job->finish(QNetworkReply::NoError, jsonDoc.toVariant().toMap().value("results").toList());
});
}
if (m_initStatus != InitStatusOK ) {
return;
}
// process query queue
if (!m_currentQuery && !m_queryQueue.isEmpty()) {
QueryJob *job = m_queryQueue.takeFirst();
QNetworkReply *reply;
qCDebug(dcLogEngine()) << "Sending query to influx" << job->m_request.url();
if (job->m_post) {
reply = m_nam->post(job->m_request, QByteArray());
} else {
reply = m_nam->get(job->m_request);
}
m_currentQuery = job;
connect(reply, &QNetworkReply::finished, job, [=](){
qCDebug(dcLogEngine()) << "Query finished";
m_currentQuery = nullptr;
reply->deleteLater();
if (reply->error() == QNetworkReply::ProtocolInvalidOperationError) {
qCWarning(dcLogEngine()) << "Influx DB protocol error:" << reply->readAll();
job->finish(reply->error());
processQueues();
return;
}
if (reply->error() != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Error in influxdb communication:" << reply->error() << reply->errorString();
job->finish(reply->error());
processQueues();
return;
}
QByteArray data = reply->readAll();
QJsonParseError error;
QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &error);
if (error.error != QJsonParseError::NoError) {
qCWarning(dcLogEngine) << "Unable to process response from influxdb:" << error.errorString() << qUtf8Printable(data);
job->finish(QNetworkReply::ProtocolFailure);
processQueues();
return;
}
// qCDebug(dcLogEngine()) << "Reply" << qUtf8Printable(jsonDoc.toJson(QJsonDocument::Indented));
job->finish(QNetworkReply::NoError, jsonDoc.toVariant().toMap().value("results").toList());
processQueues();
});
return;
}
// Process write queue
if (!m_writeQueue.isEmpty() && !m_currentWriteReply) {
QueueEntry entry = m_writeQueue.takeFirst();
QNetworkRequest request = entry.request;
QByteArray data = entry.data;
LogEntry logEntry = entry.entry;
qCDebug(dcLogEngine()) << "Sending log write event to influx" << request.url().toString() << data;
QNetworkReply *reply = m_nam->post(request, data);
qCDebug(dcLogEngine()) << "Started:" << reply->isRunning() << reply->isFinished();
m_currentWriteReply = reply;
connect(reply, &QNetworkReply::finished, this, [=](){
m_currentWriteReply = nullptr;
reply->deleteLater();
if (reply->error() != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Unable to connect to influxdb. Cannot log events." << reply->error() << reply->readAll();
processQueues();
return;
}
emit logEntryAdded(logEntry);
QByteArray result = reply->readAll();
if (!result.isEmpty()) {
qCDebug(dcLogEngine()) << "Log reply:" << result;
}
processQueues();
});
}
}
LogFetchJob *LogEngineInfluxDB::fetchLogEntries(const QStringList &sources, const QStringList &columns, const QDateTime &startTime, const QDateTime &endTime, const QVariantMap &filter, Types::SampleRate sampleRate, Qt::SortOrder sortOrder, int offset, int limit)
{
LogFetchJob *job = new LogFetchJob(this);
// FIXME: injection attacks possible?
QString what = "*";
if (sampleRate == Types::SampleRateAny) {
if (!columns.isEmpty()) {
what = columns.join(", ");
}
} else {
if (!columns.isEmpty()) {
QStringList meanColumns;
foreach (const QString &column, columns) {
meanColumns.append(QString("MEAN(%1)").arg(column));
}
what = meanColumns.join(", ");
} else {
what = "MEAN(*)";
}
}
QStringList escapedSourced;
foreach (const QString &source, sources) {
QString retentionPolicy;
switch (sampleRate) {
case Types::SampleRate1Min:
case Types::SampleRate15Mins:
retentionPolicy = "minutes";
break;
case Types::SampleRate1Hour:
case Types::SampleRate3Hours:
retentionPolicy = "hours";
break;
case Types::SampleRate1Day:
case Types::SampleRate1Week:
case Types::SampleRate1Month:
case Types::SampleRate1Year:
retentionPolicy = "days";
break;
case Types::SampleRateAny:
if (m_loggers.contains(source) && m_loggers.value(source)->loggingType() == Types::LoggingTypeSampled) {
retentionPolicy = "live";
} else {
retentionPolicy = "discrete";
}
break;
}
escapedSourced.append(QString("%1.\"%2\"").arg(retentionPolicy).arg(source));
}
QString query = QString("SELECT %1 FROM %2").arg(what).arg(escapedSourced.join(", "));
QStringList parts;
if (!startTime.isNull()) {
parts.append(QString("time >= %1ms").arg(startTime.toMSecsSinceEpoch()));
}
if (!endTime.isNull()) {
parts.append(QString("time <= %1ms").arg(endTime.toMSecsSinceEpoch()));
}
if (!filter.isEmpty()) {
foreach (const QString &column, filter.keys()) {
parts.append(QString("%1 = '%2'").arg(column).arg(filter.value(column).toString()));
}
}
if (parts.count() > 0) {
query.append(" WHERE ");
query.append(parts.join(" AND "));
}
if (sampleRate != Types::SampleRateAny) {
// When resampling, we need to go from oldest to newest to properly "fill(previous)"
query.append(QString(" GROUP BY time(%1m) fill(previous)").arg(sampleRate));
}
if (sortOrder == Qt::AscendingOrder) {
query.append(" ORDER BY time ASC");
} else {
query.append(" ORDER BY time DESC");
}
if (limit > 0) {
query.append(QString(" LIMIT %1").arg(limit));
}
if (offset > 0) {
query.append(QString(" OFFSET %1").arg(offset));
}
qCDebug(dcLogEngine()) << "Running query:" << query;
qCDebug(dcLogEngine()) << "start" << startTime << "end" << endTime;
QNetworkRequest request = createQueryRequest(query);
qCDebug(dcLogEngine()) << "Request:" << request.url() << filter;
QNetworkReply *reply = m_nam->get(request);
connect(reply, &QNetworkReply::finished, this, [=](){
reply->deleteLater();
if (reply->error() != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Unable to obtain entries from influxdb" << reply->error() << reply->readAll();
finishFetchJob(job, QList<LogEntry>());
return;
}
QByteArray data = reply->readAll();
QJsonParseError error;
QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &error);
if (error.error != QJsonParseError::NoError) {
qCWarning(dcLogEngine) << "Unable to process response from influxdb:" << error.errorString() << qUtf8Printable(data);
finishFetchJob(job, QList<LogEntry>());
return;
}
qCDebug(dcLogEngine()) << "Reply" << qUtf8Printable(jsonDoc.toJson(QJsonDocument::Indented));
QList<LogEntry> entries;
foreach (const QVariant &resultsVariant, jsonDoc.toVariant().toMap().value("results").toList()) {
QVariantMap resultMap = resultsVariant.toMap();
foreach (const QVariant &seriesVariant, resultMap.value("series").toList()) {
QVariantMap seriesMap = seriesVariant.toMap();
QStringList columns = seriesMap.value("columns").toStringList();
foreach (const QVariant &valueVariant, seriesMap.value("values").toList()) {
QVariantMap valuesMap;
QVariantList values = valueVariant.toList();
QDateTime timestamp = QDateTime::fromMSecsSinceEpoch(values.first().toULongLong());
for (int i = 1; i < columns.count(); i++) {
QString column = columns.at(i);
if (sampleRate != Types::SampleRateAny) {
column.remove(QRegExp("^mean_"));
}
QVariant value = values.at(i);
if (value.type() == QVariant::String || value.type() == QVariant::ByteArray) {
valuesMap.insert(column, QByteArray::fromPercentEncoding(value.toByteArray()));
} else {
valuesMap.insert(column, values.at(i));
}
}
LogEntry entry(timestamp, seriesMap.value("name").toString(), valuesMap);
entries.append(entry);
}
}
}
finishFetchJob(job, entries);
});
return job;
}
bool LogEngineInfluxDB::jobsRunning() const
{
// qCDebug(dcLogEngine()) << "Jobs running:" << m_initStatus << m_writeQueue.count() << m_initQueryQueue.count() << m_queryQueue.count() << m_currentWriteReply;
return m_currentInitQuery
|| !m_initQueryQueue.isEmpty()
|| m_currentQuery
|| !m_queryQueue.isEmpty()
|| m_currentWriteReply
|| !m_writeQueue.isEmpty();
}
void LogEngineInfluxDB::clear(const QString &source)
{
qCDebug(dcLogEngine()) << "Clearing entries for source:" << source;
QueryJob *job = query(QString("DROP MEASUREMENT \"%1\"").arg(source));
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status, const QVariantList &results){
if (status != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Unable to clear log entries for" << source << ":" << qUtf8Printable(QJsonDocument::fromVariant(results).toJson());
}
});
}
void LogEngineInfluxDB::initDB()
{
m_initStatus = InitStatusStarting;
qCInfo(dcLogEngine()) << "Initializing influx DB connection";
createDB();
}
void LogEngineInfluxDB::createDB()
{
QueryJob *job = query("SHOW DATABASES", false, true);
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status, const QVariantList &results){
if (status != QNetworkReply::NoError) {
if (status == QNetworkReply::ConnectionRefusedError) {
// Influx not up yet? trying again in 5 secs...
qCInfo(dcLogEngine) << "Failed to connect to influx... retrying in 5 seconds...";
QTimer::singleShot(5000, this, [=](){
initDB();
});
return;
}
qCCritical(dcLogEngine()) << "Unable to connect to InfluxDB";
m_initStatus = InitStatusFailure;
return;
}
if (results.count() != 1) {
qCWarning(dcLogEngine()) << "Unable to read databases from influxdb. No result set.";
m_initStatus = InitStatusFailure;
return;
}
QVariantList series = results.first().toMap().value("series").toList();
if (series.count() != 1) {
qCWarning(dcLogEngine()) << "Unable to read databases from influxdb. No series set.";
m_initStatus = InitStatusFailure;
return;
}
QVariantList values = series.first().toMap().value("values").toList();
qCDebug(dcLogEngine()) << "Databases in influx:" << values;
bool nymeaDBfound = false;
foreach (const QVariant &value, values) {
QVariantList valueList = value.toList();
if (valueList.count() > 0) {
if (valueList.first().toString() == m_dbName) {
nymeaDBfound = true;
break;
}
}
}
if (nymeaDBfound) {
qCDebug(dcLogEngine()) << "influxdb database already set up.";
createRetentionPolicies();
return;
}
qCInfo(dcLogEngine()) << "Creating" << m_dbName << "database in influxdb";
QueryJob *job = query(QString("CREATE DATABASE %1").arg(m_dbName), true, true);
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status, const QVariantList &result) {
if (status != QNetworkReply::NoError) {
qCCritical(dcLogEngine()) << "Unable to create" << m_dbName << "database in influxdb:" << QJsonDocument::fromVariant(result).toJson();
m_initStatus = InitStatusFailure;
return;
}
qCInfo(dcLogEngine()) << m_dbName << "database created in influxdb.";
createRetentionPolicies();
});
});
}
void LogEngineInfluxDB::createRetentionPolicies()
{
QueryJob *job = query("SHOW RETENTION POLICIES", false, true);
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status, const QVariantList &results){
if (status != QNetworkReply::NoError) {
qCCritical(dcLogEngine()) << "Unable to query retention policies.";
m_initStatus = InitStatusFailure;
return;
}
if (results.count() != 1) {
qCWarning(dcLogEngine()) << "Unable to read retention policies from influxdb. No result set.";
m_initStatus = InitStatusFailure;
return;
}
QVariantList series = results.first().toMap().value("series").toList();
if (series.count() != 1) {
qCWarning(dcLogEngine()) << "Unable to read retention policies from influxdb. No series set.";
m_initStatus = InitStatusFailure;
return;
}
QVariantList values = series.first().toMap().value("values").toList();
qCDebug(dcLogEngine()) << "Retention policies in influx:" << values;
bool discreteRPFound = false, liveRPFound = false, minutesRPFound = false, hoursRPFound = false, daysRPFound = false;
foreach (const QVariant &value, values) {
QVariantList valueList = value.toList();
if (valueList.count() > 0) {
if (valueList.first().toString() == "discrete") {
discreteRPFound = true;
continue;
}
}
if (valueList.count() > 0) {
if (valueList.first().toString() == "live") {
liveRPFound = true;
continue;
}
}
if (valueList.count() > 0) {
if (valueList.first().toString() == "minutes") {
minutesRPFound = true;
continue;
}
}
if (valueList.count() > 0) {
if (valueList.first().toString() == "hours") {
hoursRPFound = true;
continue;
}
}
if (valueList.count() > 0) {
if (valueList.first().toString() == "days") {
daysRPFound = true;
continue;
}
}
}
if (!discreteRPFound) {
qCInfo(dcLogEngine()) << "Creating discrete nymea retention policy in influxdb";
QueryJob *job = query(QString("CREATE RETENTION POLICY discrete ON %1 DURATION 8760h REPLICATION 1").arg(m_dbName), true, true);
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status){
if (status != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Unable to create discrete retention policy in influxdb.";
m_initStatus = InitStatusFailure;
return;
}
createRetentionPolicies();
});
return;
}
if (!liveRPFound) {
qCInfo(dcLogEngine()) << "Creating live nymea retention policy in influxdb";
QueryJob *job = query(QString("CREATE RETENTION POLICY live ON %1 DURATION 24h REPLICATION 1").arg(m_dbName), true, true);
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status){
if (status != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Unable to create live retention policy in influxdb.";
m_initStatus = InitStatusFailure;
return;
}
createRetentionPolicies();
});
return;
}
if (!minutesRPFound) {
qCInfo(dcLogEngine()) << "Creating minutes nymea retention policy in influxdb";
QueryJob *job = query(QString("CREATE RETENTION POLICY minutes ON %1 DURATION 168h REPLICATION 1").arg(m_dbName), true, true);
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status){
if (status != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Unable to create minutes retention policy in influxdb.";
m_initStatus = InitStatusFailure;
return;
}
createRetentionPolicies();
});
return;
}
if (!hoursRPFound) {
qCInfo(dcLogEngine()) << "Creating hours nymea retention policy in influxdb";
QueryJob *job = query(QString("CREATE RETENTION POLICY hours ON %1 DURATION 26280h REPLICATION 1").arg(m_dbName), true, true);
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status){
if (status != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Unable to create hours retention policy in influxdb.";
m_initStatus = InitStatusFailure;
return;
}
createRetentionPolicies();
});
return;
}
if (!daysRPFound) {
qCInfo(dcLogEngine()) << "Creating days nymea retention policy in influxdb";
QueryJob *job = query(QString("CREATE RETENTION POLICY days ON %1 DURATION 175200h REPLICATION 1").arg(m_dbName), true, true);
connect(job, &QueryJob::finished, this, [=](QNetworkReply::NetworkError status){
if (status != QNetworkReply::NoError) {
qCWarning(dcLogEngine()) << "Unable to create days retention policy in influxdb.";
m_initStatus = InitStatusFailure;
return;
}
createRetentionPolicies();
});
return;
}
m_initStatus = InitStatusOK;
qCDebug(dcLogEngine()) << "Influx initialized. Starting to process log entries (" << m_initQueryQueue.count() << m_queryQueue.count() << m_writeQueue.count() << "in queue)";
processQueues();
});
}
QNetworkRequest LogEngineInfluxDB::createQueryRequest(const QString &quer)
{
QUrl url;
url.setScheme("http");
url.setHost(m_host);
url.setPort(8086);
url.setPath("/query");
QUrlQuery urlQuery;
urlQuery.addQueryItem("db", m_dbName);
urlQuery.addQueryItem("q", quer);
urlQuery.addQueryItem("epoch", "ms");
url.setQuery(urlQuery);
QNetworkRequest request(url);
if (!m_username.isEmpty() || !m_password.isEmpty()) {
QByteArray auth = QByteArray(m_username.toLatin1() + ':' + m_password.toLatin1()).toBase64(QByteArray::Base64Encoding | QByteArray::KeepTrailingEquals);
request.setRawHeader("Authorization", QString("Basic %1").arg(QString(auth)).toUtf8());
}
return request;
}
QNetworkRequest LogEngineInfluxDB::createWriteRequest(const QString &retentionPolicy)
{
QUrl url;
url.setScheme("http");
url.setHost(m_host);
url.setPort(8086);
url.setPath("/write");
QUrlQuery urlQuery;
urlQuery.addQueryItem("db", m_dbName);
urlQuery.addQueryItem("precision", "ms");
urlQuery.addQueryItem("rp", retentionPolicy);
url.setQuery(urlQuery);
QNetworkRequest request(url);
if (!m_username.isEmpty() || !m_password.isEmpty()) {
QByteArray auth = QByteArray(m_username.toLatin1() + ':' + m_password.toLatin1()).toBase64(QByteArray::Base64Encoding | QByteArray::KeepTrailingEquals);
request.setRawHeader("Authorization", QString("Basic %1").arg(QString(auth)).toUtf8());
}
return request;
}
QueryJob *LogEngineInfluxDB::query(const QString &query, bool post, bool isInit)
{
QNetworkRequest request = createQueryRequest(query);
if (post) {
request.setHeader(QNetworkRequest::ContentTypeHeader, "application/x-www-form-urlencoded");
}
QueryJob *job = new QueryJob(request, post, isInit, this);
if (isInit) {
m_initQueryQueue.append(job);
} else {
m_queryQueue.append(job);
}
QMetaObject::invokeMethod(this, "processQueues", Qt::QueuedConnection);
return job;
}
QueryJob::QueryJob(const QNetworkRequest &request, bool post, bool isInit, QObject *parent):
QObject(parent),
m_request(request),
m_post(post),
m_isInit(isInit)
{
}
void QueryJob::finish(QNetworkReply::NetworkError status, const QVariantList &results)
{
QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection, Q_ARG(QNetworkReply::NetworkError, status), Q_ARG(QVariantList, results));
QMetaObject::invokeMethod(this, "deleteLater", Qt::QueuedConnection);
}

93
libnymea-core/logging/logengineinfluxdb.h Normal file
View File

@ -0,0 +1,93 @@
#ifndef LOGENGINEINFLUXDB_H
#define LOGENGINEINFLUXDB_H
#include "logging/logengine.h"
#include <QObject>
#include <QQueue>
#include <QHostAddress>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QNetworkAccessManager>
class QueryJob: public QObject {
Q_OBJECT
explicit QueryJob(const QNetworkRequest &request, bool post, bool isInit, QObject *parent = nullptr);
signals:
void finished(QNetworkReply::NetworkError status, const QVariantList &response);
private:
friend class LogEngineInfluxDB;
QNetworkRequest m_request;
bool m_post = false;
bool m_isInit = false;
void finish(QNetworkReply::NetworkError status, const QVariantList &results = QVariantList());
};
class LogEngineInfluxDB : public LogEngine
{
Q_OBJECT
public:
enum InitStatus {
InitStatusNone,
InitStatusStarting,
InitStatusOK,
InitStatusFailure
};
Q_ENUM(InitStatus)
explicit LogEngineInfluxDB(const QString &host, const QString &dbName, const QString &username = QString(), const QString &password = QString(), QObject *parent = nullptr);
~LogEngineInfluxDB();
Logger *registerLogSource(const QString &name, const QStringList &tagNames, Types::LoggingType loggingType = Types::LoggingTypeDiscrete, const QString &sampleColumn = QString()) override;
void unregisterLogSource(const QString &name) override;
void logEvent(Logger *logger, const QStringList &tags, const QVariantMap &values) override;
LogFetchJob *fetchLogEntries(const QStringList &sources, const QStringList &columns, const QDateTime &startTime = QDateTime(), const QDateTime &endTime = QDateTime(), const QVariantMap &filter = QVariantMap(), Types::SampleRate sampleRate = Types::SampleRateAny, Qt::SortOrder sortOrder = Qt::AscendingOrder, int offset = 0, int limit = 0) override;
bool jobsRunning() const override;
void clear(const QString &source) override;
private:
void initDB();
void createRetentionPolicies();
void createDB();
QNetworkRequest createQueryRequest(const QString &quer);
QNetworkRequest createWriteRequest(const QString &retentionPolicy);
QueryJob *query(const QString &query, bool post = false, bool isInit = false);
private slots:
void processQueues();
private:
struct QueueEntry {
QNetworkRequest request;
QByteArray data;
LogEntry entry;
};
InitStatus m_initStatus = InitStatusNone;
QNetworkAccessManager *m_nam = nullptr;
QString m_host;
QString m_dbName;
QString m_username;
QString m_password;
QHash<QString, Logger*> m_loggers;
QQueue<QueryJob*> m_initQueryQueue;
QueryJob *m_currentInitQuery = nullptr;
QQueue<QueryJob*> m_queryQueue;
QueryJob *m_currentQuery = nullptr;
QQueue<QueueEntry> m_writeQueue;
QNetworkReply* m_currentWriteReply = nullptr;
};
#endif // LOGENGINEINFLUXDB_H

211
libnymea-core/logging/logentry.cpp
View File

@ -1,211 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*!
\class nymeaserver::LogEntry
\brief Represents an entry of the log database.
\ingroup logs
\inmodule core
A \l{LogEntry} represents an a nymea event which can be stored from the \l{LogEngine} to the database.
Each LogEntry has a timestamp an can be loaded from the database and stored in the database.
\sa LogEngine, LogFilter, LogsResource, LoggingHandler
*/
/*! \fn QDebug nymeaserver::operator<< (QDebug dbg, const LogEntry &entry);;
Writes the \l{LogEntry} \a entry to the given \a dbg. This method gets used just for debugging.
*/
#include "logentry.h"
#include "nymeacore.h"
#include <QDebug>
#include <QMetaEnum>
namespace nymeaserver {
LogEntry::LogEntry()
{
}
/*! Constructs a \l{LogEntry} with the given \a timestamp, \a level, \a source and \a errorCode.*/
LogEntry::LogEntry(QDateTime timestamp, Logging::LoggingLevel level, Logging::LoggingSource source, int errorCode):
m_timestamp(timestamp),
m_level(level),
m_source(source),
m_eventType(Logging::LoggingEventTypeTrigger),
m_active(false),
m_errorCode(errorCode)
{
}
/*! Constructs a \l{LogEntry} with the given \a level, \a source and \a errorCode.*/
LogEntry::LogEntry(Logging::LoggingLevel level, Logging::LoggingSource source, int errorCode):
LogEntry(NymeaCore::instance()->timeManager()->currentDateTime(), level, source, errorCode)
{
}
/*! Constructs a \l{LogEntry} with the given \a source.*/
LogEntry::LogEntry(Logging::LoggingSource source):
LogEntry(Logging::LoggingLevelInfo, source)
{
}
/*! Returns the timestamp of this \l{LogEntry}. */
QDateTime LogEntry::timestamp() const
{
return m_timestamp;
}
/*! Returns the level of this \l{LogEntry}. */
Logging::LoggingLevel LogEntry::level() const
{
return m_level;
}
/*! Returns the source of this \l{LogEntry}. */
Logging::LoggingSource LogEntry::source() const
{
return m_source;
}
/*! Returns the type ID of this \l{LogEntry}. */
QUuid LogEntry::typeId() const
{
return m_typeId;
}
/*! Sets the \a typeId of this \l{LogEntry}. */
void LogEntry::setTypeId(const QUuid &typeId) {
m_typeId = typeId;
}
/*! Returns the thingId of this \l{LogEntry}. */
ThingId LogEntry::thingId() const
{
return m_thingId;
}
/*! Sets the \a thingId of this \l{LogEntry}. */
void LogEntry::setThingId(const ThingId &thingId)
{
m_thingId = thingId;
}
/*! Returns the value of this \l{LogEntry}. */
QVariant LogEntry::value() const
{
return m_value;
}
/*! Sets the \a value of this \l{LogEntry}. */
void LogEntry::setValue(const QVariant &value)
{
m_value = value;
}
/*! Returns the event type of this \l{LogEntry}. */
Logging::LoggingEventType LogEntry::eventType() const
{
return m_eventType;
}
/*! Sets the \a eventType of this \l{LogEntry}. */
void LogEntry::setEventType(const Logging::LoggingEventType &eventType)
{
m_eventType = eventType;
}
/*! Returns true if this \l{LogEntry} is a system active type. */
bool LogEntry::active() const
{
return m_active;
}
/*! Sets this \l{LogEntry} to \a active. */
void LogEntry::setActive(bool active)
{
m_active = active;
}
/*! Returns the error code of this \l{LogEntry}. */
int LogEntry::errorCode() const
{
return m_errorCode;
}
QDebug operator<<(QDebug dbg, const LogEntry &entry)
{
QDebugStateSaver saver(dbg);
QMetaEnum metaEnum;
dbg.nospace() << "LogEntry (" << entry.timestamp().toString() << ")" << endl;
dbg.nospace() << " time stamp: " << entry.timestamp().toTime_t() << endl;
dbg.nospace() << " ThingId: " << entry.thingId().toString() << endl;
dbg.nospace() << " type id: " << entry.typeId().toString() << endl;
metaEnum = QMetaEnum::fromType<Logging::LoggingSource>();
dbg.nospace() << " source: " << metaEnum.valueToKey(entry.source()) << endl;
metaEnum = QMetaEnum::fromType<Logging::LoggingLevel>();
dbg.nospace() << " level: " << metaEnum.valueToKey(entry.level()) << endl;
metaEnum = QMetaEnum::fromType<Logging::LoggingEventType>();
dbg.nospace() << " eventType: " << metaEnum.valueToKey(entry.eventType()) << endl;
dbg.nospace() << " error code: " << entry.errorCode() << endl;
dbg.nospace() << " active: " << entry.active() << endl;
dbg.nospace() << " value: " << entry.value() << endl;
return dbg;
}
LogEntries::LogEntries()
{
}
LogEntries::LogEntries(const QList<LogEntry> &other): QList<LogEntry>(other)
{
}
QVariant LogEntries::get(int index) const
{
return QVariant::fromValue(at(index));
}
void LogEntries::put(const QVariant &variant)
{
append(variant.value<LogEntry>());
}
}

121
libnymea-core/logging/logentry.h
View File

@ -1,121 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef LOGENTRY_H
#define LOGENTRY_H
#include "logging.h"
#include "typeutils.h"
#include <QObject>
#include <QVariant>
#include <QDateTime>
namespace nymeaserver {
class LogEntry
{
Q_GADGET
Q_PROPERTY(QDateTime timestamp READ timestamp)
Q_PROPERTY(Logging::LoggingLevel loggingLevel READ level)
Q_PROPERTY(Logging::LoggingSource source READ source)
Q_PROPERTY(QUuid typeId READ typeId USER true)
Q_PROPERTY(QUuid thingId READ thingId USER true)
Q_PROPERTY(QVariant value READ value USER true)
Q_PROPERTY(bool active READ active USER true)
Q_PROPERTY(Logging::LoggingEventType eventType READ eventType USER true)
Q_PROPERTY(QString errorCode READ errorCode USER true)
public:
LogEntry();
LogEntry(QDateTime timestamp, Logging::LoggingLevel level, Logging::LoggingSource source, int errorCode = 0);
LogEntry(Logging::LoggingLevel level, Logging::LoggingSource source, int errorCode = 0);
LogEntry(Logging::LoggingSource source);
// Valid for all LoggingSources
QDateTime timestamp() const;
Logging::LoggingLevel level() const;
Logging::LoggingSource source() const;
Logging::LoggingEventType eventType() const;
void setEventType(const Logging::LoggingEventType &eventType);
// Valid for LoggingSourceStates, LoggingSourceEvents, LoggingSourceActions, LoggingSourceRules
QUuid typeId() const;
void setTypeId(const QUuid &typeId);
// Valid for LoggingSourceStates, LoggingSourceEvents, LoggingSourceActions
ThingId thingId() const;
void setThingId(const ThingId &thingId);
// Valid for LoggingSourceStates, LoggingSourceBrowserActions
QVariant value() const;
void setValue(const QVariant &value);
// Valid for LoggingEventTypeActiveChanged
bool active() const;
void setActive(bool active);
// Valid for LoggingLevelAlert
int errorCode() const;
private:
QDateTime m_timestamp;
Logging::LoggingLevel m_level;
Logging::LoggingSource m_source;
// RuleSource specific properties.
// FIXME: If it turns out we need many more of those, we should subclass LogEntry with specific ones.
QUuid m_typeId;
ThingId m_thingId;
QVariant m_value;
Logging::LoggingEventType m_eventType;
bool m_active;
int m_errorCode;
};
class LogEntries: QList<LogEntry>
{
Q_GADGET
Q_PROPERTY(int count READ count)
public:
LogEntries();
LogEntries(const QList<LogEntry> &other);
Q_INVOKABLE QVariant get(int index) const;
Q_INVOKABLE void put(const QVariant &variant);
};
QDebug operator<<(QDebug dbg, const LogEntry &entry);
}
Q_DECLARE_METATYPE(nymeaserver::LogEntry)
Q_DECLARE_METATYPE(nymeaserver::LogEntries)
#endif

407
libnymea-core/logging/logfilter.cpp
View File

@ -1,407 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*!
\class nymeaserver::LogFilter
\brief Represents a filter to access the logging databse.
\ingroup logs
\inmodule core
A \l{LogFilter} can be used to get \l{LogEntry}{LogEntries} from the \l{LogEngine} matching
a certain pattern.
\sa LogEngine, LogEntry, LogsResource, LoggingHandler
*/
#include "logfilter.h"
#include "loggingcategories.h"
namespace nymeaserver {
/*! Constructs a new \l{LogFilter}.*/
LogFilter::LogFilter()
{
}
/*! Returns the database query string for this \l{LogFilter}.*/
QString LogFilter::queryString() const
{
if (isEmpty()) {
return QString();
}
QString query;
query.append(createDateString());
if (!query.isEmpty() && !loggingSources().isEmpty()) {
query.append("AND ");
}
query.append(createSourcesString());
if (!query.isEmpty() && !loggingLevels().isEmpty()) {
query.append("AND ");
}
query.append(createLevelsString());
if (!query.isEmpty() && !loggingEventTypes().isEmpty()) {
query.append("AND ");
}
query.append(createEventTypesString());
if (!query.isEmpty() && !typeIds().isEmpty()) {
query.append("AND ");
}
query.append(createTypeIdsString());
if (!query.isEmpty() && !thingIds().isEmpty()) {
query.append("AND ");
}
query.append(createThingIdString());
if (!query.isEmpty() && !values().isEmpty()) {
query.append("AND ");
}
query.append(createValuesString());
return query;
}
/*! Add a new time filter with the given \a startDate and \a endDate. */
void LogFilter::addTimeFilter(const QDateTime &startDate, const QDateTime &endDate)
{
QPair<QDateTime, QDateTime> timeFilter(startDate, endDate);
if (!m_timeFilters.contains(timeFilter))
m_timeFilters.append(timeFilter);
}
/*! Returns the list of time filters from this \l{LogFilter}. */
QList<QPair<QDateTime, QDateTime> > LogFilter::timeFilters() const
{
return m_timeFilters;
}
/*! Add a new \a source to this \l{LogFilter}. */
void LogFilter::addLoggingSource(const Logging::LoggingSource &source)
{
if (!m_sources.contains(source))
m_sources.append(source);
}
/*! Returns the list of logging sources from this \l{LogFilter}. */
QList<Logging::LoggingSource> LogFilter::loggingSources() const
{
return m_sources;
}
/*! Add a new \a level to this \l{LogFilter}. */
void LogFilter::addLoggingLevel(const Logging::LoggingLevel &level)
{
if (!m_levels.contains(level))
m_levels.append(level);
}
/*! Returns the list of logging levels from this \l{LogFilter}. */
QList<Logging::LoggingLevel> LogFilter::loggingLevels() const
{
return m_levels;
}
/*! Add a new \a eventType to this \l{LogFilter}. */
void LogFilter::addLoggingEventType(const Logging::LoggingEventType &eventType)
{
if (!m_eventTypes.contains(eventType))
m_eventTypes.append(eventType);
}
/*! Returns the list of event types from this \l{LogFilter}. */
QList<Logging::LoggingEventType> LogFilter::loggingEventTypes() const
{
return m_eventTypes;
}
/*! Add a new \a typeId to this \l{LogFilter}. */
void LogFilter::addTypeId(const QUuid &typeId)
{
if (!m_typeIds.contains(typeId))
m_typeIds.append(typeId);
}
/*! Returns the list of type id's from this \l{LogFilter}. */
QList<QUuid> LogFilter::typeIds() const
{
return m_typeIds;
}
/*! Add a new \a thingId to this \l{LogFilter}. */
void LogFilter::addThingId(const ThingId &thingId)
{
if (!m_thingIds.contains(thingId))
m_thingIds.append(thingId);
}
/*! Returns the list of thing id's from this \l{LogFilter}. */
QList<ThingId> LogFilter::thingIds() const
{
return m_thingIds;
}
/*! Add a new \a value to this \l{LogFilter}. */
void LogFilter::addValue(const QString &value)
{
if (!m_values.contains(value))
m_values.append(value);
}
/*! Returns the list of values from this \l{LogFilter}. */
QVariantList LogFilter::values() const
{
return m_values;
}
/*! Set the maximum count for the result set. Unless a \l{offset} is specified,
* the newest \a count entries will be returned. \sa{setOffset}
*/
void LogFilter::setLimit(int limit)
{
m_limit = limit;
}
/*! Returns the maximum count for the result set. \sa{setOffset} */
int LogFilter::limit() const
{
return m_limit;
}
/*! Set the offset for the result set.
* The offset starts at the newest entry in the result set.
* 0 (default) means "all items"
* Example: If the specified filter returns a total amount of 100 entries:
* - a offset value of 10 would include the oldest 90 entries
* - a offset value of 0 would return all 100 entries
*
* The offset is particularly useful in combination with the \l{limit} property and
* can be used for pagination.
*
* E.g. A result set of 10000 entries can be fetched in batches of 1000 entries by fetching
* 1) offset 0, limit 1000: Entries 0 to 9999
* 2) offset 10000, limit 1000: Entries 10000 - 19999
* 3) offset 20000, limit 1000: Entries 20000 - 29999
* ...
*/
void LogFilter::setOffset(int offset)
{
m_offset = offset;
}
/*! Returns the offset for the result set. \sa{setOffset} */
int LogFilter::offset() const
{
return m_offset;
}
/*! Returns true if this \l{LogFilter} is empty. */
bool LogFilter::isEmpty() const
{
return m_timeFilters.isEmpty() &&
m_sources.isEmpty() &&
m_levels.isEmpty() &&
m_eventTypes.isEmpty() &&
m_typeIds.isEmpty() &&
m_thingIds.isEmpty() &&
m_values.isEmpty();
}
QString LogFilter::createDateString() const
{
QString query;
if (!m_timeFilters.isEmpty()) {
if (m_timeFilters.count() == 1) {
QPair<QDateTime, QDateTime> timeFilter = m_timeFilters.first();
query.append(createTimeFilterString(timeFilter));
} else {
query.append("( ");
QPair<QDateTime, QDateTime> timeFilter;
foreach (timeFilter, m_timeFilters) {
query.append(createTimeFilterString(timeFilter));
if (timeFilter != m_timeFilters.last())
query.append("OR ");
}
query.append(") ");
}
}
return query;
}
QString LogFilter::createTimeFilterString(QPair<QDateTime, QDateTime> timeFilter) const
{
QString query;
QDateTime startDate = timeFilter.first;
QDateTime endDate = timeFilter.second;
qCDebug(dcLogEngine) << "create timefiler for" << startDate.toString() << endDate.toString();
query.append("( ");
if (startDate.isValid() && !endDate.isValid()) {
// only start date is valid
query.append(QString("timestamp BETWEEN '%1' AND '%2' ")
.arg(startDate.toMSecsSinceEpoch())
.arg(QDateTime::currentDateTime().toMSecsSinceEpoch()));
} else if (!startDate.isValid() && endDate.isValid()) {
// only end date is valid
query.append(QString("timestamp NOT BETWEEN '%1' AND '%2' ")
.arg(endDate.toMSecsSinceEpoch())
.arg(QDateTime::currentDateTime().toMSecsSinceEpoch()));
} else if (startDate.isValid() && endDate.isValid()) {
// both dates are valid
query.append(QString("timestamp BETWEEN '%1' AND '%2' ")
.arg(startDate.toMSecsSinceEpoch())
.arg(endDate.toMSecsSinceEpoch()));
}
query.append(") ");
return query;
}
QString LogFilter::createSourcesString() const
{
QString query;
if (!m_sources.isEmpty()) {
if (m_sources.count() == 1) {
query.append(QString("sourceType = '%1' ").arg(m_sources.first()));
} else {
query.append("( ");
foreach (const Logging::LoggingSource &source, m_sources) {
query.append(QString("sourceType = '%1' ").arg(source));
if (source != m_sources.last())
query.append("OR ");
}
query.append(") ");
}
}
return query;
}
QString LogFilter::createLevelsString() const
{
QString query;
if (!m_levels.isEmpty()) {
if (m_levels.count() == 1) {
query.append(QString("loggingLevel = '%1' ").arg(m_levels.first()));
} else {
query.append("( ");
foreach (const Logging::LoggingLevel &level, m_levels) {
query.append(QString("loggingLevel = '%1' ").arg(level));
if (level != m_levels.last())
query.append("OR ");
}
query.append(") ");
}
}
return query;
}
QString LogFilter::createEventTypesString() const
{
QString query;
if (!m_eventTypes.isEmpty()) {
if (m_eventTypes.count() == 1) {
query.append(QString("loggingEventType = '%1' ").arg(m_eventTypes.first()));
} else {
query.append("( ");
foreach (const Logging::LoggingEventType &eventType, m_eventTypes) {
query.append(QString("loggingEventType = '%1' ").arg(eventType));
if (eventType != m_eventTypes.last())
query.append("OR ");
}
query.append(") ");
}
}
return query;
}
QString LogFilter::createTypeIdsString() const
{
QString query;
if (!m_typeIds.isEmpty()) {
if (m_typeIds.count() == 1) {
query.append(QString("typeId = '%1' ").arg(m_typeIds.first().toString()));
} else {
query.append("( ");
foreach (const QUuid &typeId, m_typeIds) {
query.append(QString("typeId = '%1' ").arg(typeId.toString()));
if (typeId != m_typeIds.last())
query.append("OR ");
}
query.append(") ");
}
}
return query;
}
QString LogFilter::createThingIdString() const
{
QString query;
if (!m_thingIds.isEmpty()) {
if (m_thingIds.count() == 1) {
query.append(QString("thingId = '%1' ").arg(m_thingIds.first().toString()));
} else {
query.append("( ");
foreach (const ThingId &thingId, m_thingIds) {
query.append(QString("thingId = '%1' ").arg(thingId.toString()));
if (thingId != m_thingIds.last())
query.append("OR ");
}
query.append(") ");
}
}
return query;
}
QString LogFilter::createValuesString() const
{
QString query;
if (!m_values.isEmpty()) {
if (m_values.count() == 1) {
query.append("value = ? ");
} else {
query.append("( ");
foreach (const QVariant &value, m_values) {
query.append("value = ? ");
if (value != m_values.last())
query.append("OR ");
}
query.append(") ");
}
}
return query;
}
}

105
libnymea-core/logging/logfilter.h
View File

@ -1,105 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef LOGFILTER_H
#define LOGFILTER_H
#include <QPair>
#include <QDateTime>
#include "logging.h"
#include "typeutils.h"
namespace nymeaserver {
class LogFilter
{
public:
LogFilter();
QString queryString() const;
void addTimeFilter(const QDateTime &startDate = QDateTime(), const QDateTime &endDate = QDateTime());
QList<QPair<QDateTime, QDateTime> > timeFilters() const;
void addLoggingSource(const Logging::LoggingSource &source) ;
QList<Logging::LoggingSource> loggingSources() const;
void addLoggingLevel(const Logging::LoggingLevel &level);
QList<Logging::LoggingLevel> loggingLevels() const;
void addLoggingEventType(const Logging::LoggingEventType &eventType);
QList<Logging::LoggingEventType> loggingEventTypes() const;
// Valid for LoggingSourceStates, LoggingSourceEvents, LoggingSourceActions, LoggingSourceRules
void addTypeId(const QUuid &typeId);
QList<QUuid> typeIds() const;
// Valid for LoggingSourceStates, LoggingSourceEvents, LoggingSourceActions
void addThingId(const ThingId &thingId);
QList<ThingId> thingIds() const;
// Valid for LoggingSourceStates
void addValue(const QString &value);
QVariantList values() const;
void setLimit(int limit);
int limit() const;
void setOffset(int offset);
int offset() const;
bool isEmpty() const;
private:
QList<QPair<QDateTime, QDateTime > > m_timeFilters;
QList<Logging::LoggingSource> m_sources;
QList<Logging::LoggingLevel> m_levels;
QList<Logging::LoggingEventType> m_eventTypes;
QList<QUuid> m_typeIds;
QList<ThingId> m_thingIds;
QVariantList m_values;
int m_limit = -1;
int m_offset = 0;
QString createDateString() const;
QString createTimeFilterString(QPair<QDateTime, QDateTime> timeFilter) const;
QString createSourcesString() const;
QString createLevelsString() const;
QString createEventTypesString() const;
QString createTypeIdsString() const;
QString createThingIdString() const;
QString createValuesString() const;
};
}
#endif

82
libnymea-core/logging/logging.h
View File

@ -1,82 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef LOGGING_H
#define LOGGING_H
#include <QObject>
namespace nymeaserver {
class Logging
{
Q_GADGET
public:
enum LoggingError {
LoggingErrorNoError,
LoggingErrorLogEntryNotFound,
LoggingErrorInvalidFilterParameter
};
Q_ENUM(LoggingError)
enum LoggingSource {
LoggingSourceSystem,
LoggingSourceEvents,
LoggingSourceActions,
LoggingSourceStates,
LoggingSourceRules,
LoggingSourceBrowserActions,
};
Q_ENUM(LoggingSource)
Q_FLAGS(LoggingSources)
Q_DECLARE_FLAGS(LoggingSources, LoggingSource)
enum LoggingLevel {
LoggingLevelInfo,
LoggingLevelAlert
};
Q_ENUM(LoggingLevel)
enum LoggingEventType {
LoggingEventTypeTrigger,
LoggingEventTypeActiveChange,
LoggingEventTypeEnabledChange,
LoggingEventTypeActionsExecuted,
LoggingEventTypeExitActionsExecuted
};
Q_ENUM(LoggingEventType)
Logging(QObject *parent = nullptr);
};
}
#endif // LOGGING_H

0
libnymea-core/logging/logsource.cpp
View File

0
libnymea-core/logging/logsource.h
View File

80
libnymea-core/logging/logvaluetool.cpp
View File

@ -1,80 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include "logvaluetool.h"
#include <QBuffer>
#include <QByteArray>
#include <QDataStream>
LogValueTool::LogValueTool(QObject *parent) : QObject(parent)
{
}
QString LogValueTool::convertVariantToString(const QVariant &value)
{
switch (value.type()) {
case QVariant::Double:
return QString::number(value.toDouble());
case QVariant::List: {
QStringList valueStringList;
foreach (const QVariant &variantValue, value.toList()) {
valueStringList.append(convertVariantToString(variantValue));
}
return valueStringList.join(", ");
}
default:
return value.toString();
}
}
QString LogValueTool::serializeValue(const QVariant &value)
{
QByteArray byteArray;
QBuffer writeBuffer(&byteArray);
writeBuffer.open(QIODevice::WriteOnly);
QDataStream out(&writeBuffer);
out << value;
writeBuffer.close();
return QString(byteArray.toBase64());
}
QVariant LogValueTool::deserializeValue(const QString &serializedValue)
{
QByteArray data = QByteArray::fromBase64(serializedValue.toUtf8());
QBuffer readBuffer(&data);
readBuffer.open(QIODevice::ReadOnly);
QDataStream inputStream(&readBuffer);
QVariant value;
inputStream >> value;
readBuffer.close();
return value;
}

48
libnymea-core/logging/logvaluetool.h
View File

@ -1,48 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef LOGVALUETOOL_H
#define LOGVALUETOOL_H
#include <QObject>
#include <QVariant>
class LogValueTool : public QObject
{
Q_OBJECT
public:
explicit LogValueTool(QObject *parent = nullptr);
static QString convertVariantToString(const QVariant &value);
static QString serializeValue(const QVariant &value);
static QVariant deserializeValue(const QString &serializedValue);
};
#endif // LOGVALUETOOL_H

38
libnymea-core/nymeaconfiguration.cpp
View File

@ -223,12 +223,10 @@ NymeaConfiguration::NymeaConfiguration(QObject *parent) :
// Write defaults for log settings
settings.beginGroup("Logs");
settings.setValue("logDBDriver", logDBDriver());
settings.setValue("logDBName", logDBName());
settings.setValue("logDBHost", logDBHost());
settings.setValue("logDBUser", logDBUser());
settings.setValue("logDBPassword", logDBPassword());
settings.setValue("logDBMaxEntries", logDBMaxEntries());
settings.endGroup();
}
@ -455,13 +453,6 @@ void NymeaConfiguration::setBluetoothServerEnabled(bool enabled)
emit bluetoothServerEnabledChanged();
}
QString NymeaConfiguration::logDBDriver() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBDriver", "QSQLITE").toString();
}
QString NymeaConfiguration::logDBHost() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
@ -471,22 +462,16 @@ QString NymeaConfiguration::logDBHost() const
QString NymeaConfiguration::logDBName() const
{
QString defaultLogPath;
QString organisationName = QCoreApplication::instance()->organizationName();
if (!qgetenv("SNAP").isEmpty()) {
defaultLogPath = QString(qgetenv("SNAP_COMMON")) + "/nymead.sqlite";
} else if (organisationName == "nymea-test") {
defaultLogPath = "/tmp/" + organisationName + "/nymead-test.sqlite";
} else if (NymeaSettings::isRoot()) {
defaultLogPath = "/var/log/nymead.sqlite";
} else {
defaultLogPath = QDir::homePath() + "/.config/" + organisationName + "/nymead.sqlite";
}
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBName", defaultLogPath).toString();
// Migration from < 1.8. Switching the driver is not supported any more.
// As sqlite used an absolute filename which won't work as DB name in other databases
// we'll reset the config if the user has explicitly set it.
if (settings.value("logDBDriver").toString() == "QSQLITE") {
settings.remove("logDBName");
settings.remove("logDBDriver");
}
return settings.value("logDBName", "nymea").toString();
}
QString NymeaConfiguration::logDBUser() const
@ -503,13 +488,6 @@ QString NymeaConfiguration::logDBPassword() const
return settings.value("logDBPassword").toString();
}
int NymeaConfiguration::logDBMaxEntries() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);
settings.beginGroup("Logs");
return settings.value("logDBMaxEntries", 200000).toInt();
}
QString NymeaConfiguration::sslCertificate() const
{
NymeaSettings settings(NymeaSettings::SettingsRoleGlobal);

2
libnymea-core/nymeaconfiguration.h
View File

@ -189,12 +189,10 @@ public:
void setBluetoothServerEnabled(bool enabled);
// Logging
QString logDBDriver() const;
QString logDBName() const;
QString logDBHost() const;
QString logDBUser() const;
QString logDBPassword() const;
int logDBMaxEntries() const;
private:
QHash<QString, ServerConfiguration> m_tcpServerConfigs;

48
libnymea-core/nymeacore.cpp
View File

@ -38,9 +38,10 @@
#include "platform/platform.h"
#include "experiences/experiencemanager.h"
#include "platform/platformsystemcontroller.h"
#include "logging/logengineinfluxdb.h"
#include "scriptengine/scriptengine.h"
#include "jsonrpc/scriptshandler.h"
#include "version.h"
#include "integrations/thingmanagerimplementation.h"
#include "integrations/thing.h"
@ -64,6 +65,7 @@ NYMEA_LOGGING_CATEGORY(dcCore, "Core")
namespace nymeaserver {
NymeaCore* NymeaCore::s_instance = nullptr;
NymeaCore::ShutdownReason NymeaCore::s_shutdownReason = NymeaCore::ShutdownReasonTerm;
/*! Returns a pointer to the single \l{NymeaCore} instance. */
NymeaCore *NymeaCore::instance()
@ -122,16 +124,18 @@ void NymeaCore::init(const QStringList &additionalInterfaces) {
m_hardwareManager = new HardwareManagerImplementation(m_platform, m_serverManager->mqttBroker(), m_zigbeeManager, m_zwaveManager, m_modbusRtuManager, this);
qCDebug(dcCore) << "Creating Log Engine";
m_logger = new LogEngine(m_configuration->logDBDriver(), m_configuration->logDBName(), m_configuration->logDBHost(), m_configuration->logDBUser(), m_configuration->logDBPassword(), m_configuration->logDBMaxEntries(), this);
// m_logger = new Logger(m_configuration->logDBDriver(), m_configuration->logDBName(), m_configuration->logDBHost(), m_configuration->logDBUser(), m_configuration->logDBPassword(), this);
m_logEngine = new LogEngineInfluxDB(m_configuration->logDBHost(), m_configuration->logDBName(), m_configuration->logDBUser(), m_configuration->logDBPassword(), this);
m_logger = m_logEngine->registerLogSource("core", {"event"});
qCDebug(dcCore) << "Creating Thing Manager (locale:" << m_configuration->locale() << ")";
m_thingManager = new ThingManagerImplementation(m_hardwareManager, m_logger, m_configuration->locale(), this);
m_thingManager = new ThingManagerImplementation(m_hardwareManager, m_logEngine, m_configuration->locale(), this);
qCDebug(dcCore) << "Creating Rule Engine";
m_ruleEngine = new RuleEngine(m_thingManager, m_timeManager, m_logger, this);
m_ruleEngine = new RuleEngine(m_thingManager, m_timeManager, m_logEngine, this);
qCDebug(dcCore()) << "Creating Script Engine";
m_scriptEngine = new scriptengine::ScriptEngine(m_thingManager, this);
m_scriptEngine = new scriptengine::ScriptEngine(m_thingManager, m_logEngine, this);
m_serverManager->jsonServer()->registerHandler(new ScriptsHandler(m_scriptEngine, m_scriptEngine));
qCDebug(dcCore()) << "Creating Tags Storage";
@ -152,14 +156,17 @@ void NymeaCore::init(const QStringList &additionalInterfaces) {
connect(m_thingManager, &ThingManagerImplementation::loaded, this, &NymeaCore::thingManagerLoaded);
m_logger->logSystemEvent(m_timeManager->currentDateTime(), true);
m_logger->log({"started"}, {{"version", NYMEA_VERSION_STRING}});
}
/*! Destructor of the \l{NymeaCore}. */
NymeaCore::~NymeaCore()
{
qCDebug(dcCore()) << "Shutting down NymeaCore";
m_logger->logSystemEvent(m_timeManager->currentDateTime(), false);
m_logger->log({"stopped"}, {
{"version", NYMEA_VERSION_STRING},
{"shutdownReason", QMetaEnum::fromType<ShutdownReason>().valueToKey(s_shutdownReason)}
});
// Disconnect all signals/slots, we're going down now
m_timeManager->disconnect(this);
@ -186,14 +193,15 @@ NymeaCore::~NymeaCore()
// Now go ahead and clean up stuff.
qCDebug(dcCore) << "Shutting down \"Log Engine\"";
delete m_logger;
delete m_logEngine;
qCDebug(dcCore) << "Done shutting down NymeaCore";
}
void NymeaCore::destroy()
void NymeaCore::destroy(ShutdownReason reason)
{
if (s_instance) {
s_shutdownReason = reason;
delete s_instance;
}
@ -334,7 +342,7 @@ ExperienceManager *NymeaCore::experienceManager() const
LogEngine* NymeaCore::logEngine() const
{
return m_logger;
return m_logEngine;
}
JsonRPCServerImplementation *NymeaCore::jsonRPCServer() const
@ -353,16 +361,16 @@ void NymeaCore::thingManagerLoaded()
// Do some houskeeping...
qCDebug(dcCore()) << "Starting housekeeping...";
QDateTime startTime = QDateTime::currentDateTime();
ThingsFetchJob *job = m_logger->fetchThings();
connect(job, &ThingsFetchJob::finished, m_thingManager, [this, job, startTime](){
foreach (const ThingId &thingId, job->results()) {
if (!m_thingManager->findConfiguredThing(thingId)) {
qCDebug(dcCore()) << "Cleaning stale thing entries from log DB for thing id" << thingId;
m_logger->removeThingLogs(thingId);
}
}
qCDebug(dcCore()) << "Housekeeping done in" << startTime.msecsTo(QDateTime::currentDateTime()) << "ms.";
});
// ThingsFetchJob *job = m_logger->fetchThings();
// connect(job, &ThingsFetchJob::finished, m_thingManager, [this, job, startTime](){
// foreach (const ThingId &thingId, job->results()) {
// if (!m_thingManager->findConfiguredThing(thingId)) {
// qCDebug(dcCore()) << "Cleaning stale thing entries from log DB for thing id" << thingId;
// m_logger->removeThingLogs(thingId);
// }
// }
// qCDebug(dcCore()) << "Housekeeping done in" << startTime.msecsTo(QDateTime::currentDateTime()) << "ms.";
// });
foreach (const ThingId &thingId, m_ruleEngine->thingsInRules()) {
if (!m_thingManager->findConfiguredThing(thingId)) {

55
libnymea-core/nymeacore.h
View File

@ -40,7 +40,6 @@
#include "ruleengine/rule.h"
#include "ruleengine/ruleengine.h"
#include "logging/logengine.h"
#include "servermanager.h"
#include "time/timemanager.h"
@ -51,13 +50,14 @@
#include <QObject>
class Thing;
class LogEngine;
class Logger;
class NetworkManager;
namespace nymeaserver {
class JsonRPCServerImplementation;
class LogEngine;
class NymeaConfiguration;
class TagsStorage;
class UserManager;
@ -81,11 +81,19 @@ class NymeaCore : public QObject
friend class NymeaTestBase;
public:
enum ShutdownReason {
ShutdownReasonQuit,
ShutdownReasonTerm,
ShutdownReasonFailure,
ShutdownReasonRestart
};
Q_ENUM(ShutdownReason)
static NymeaCore* instance();
~NymeaCore();
void init(const QStringList &additionalInterfaces = QStringList());
void destroy();
void destroy(nymeaserver::NymeaCore::ShutdownReason reason);
RuleEngine::RuleError removeRule(const RuleId &id);
@ -117,31 +125,34 @@ signals:
void initialized();
private:
explicit NymeaCore(QObject *parent = nullptr);
static NymeaCore *s_instance;
static ShutdownReason s_shutdownReason;
Platform *m_platform = nullptr;
NymeaConfiguration *m_configuration;
ServerManager *m_serverManager;
ThingManagerImplementation *m_thingManager;
RuleEngine *m_ruleEngine;
ScriptEngine *m_scriptEngine;
LogEngine *m_logger;
TimeManager *m_timeManager;
CloudManager *m_cloudManager;
HardwareManagerImplementation *m_hardwareManager;
DebugServerHandler *m_debugServerHandler;
TagsStorage *m_tagsStorage;
NymeaConfiguration *m_configuration = nullptr;
ServerManager *m_serverManager = nullptr;
ThingManagerImplementation *m_thingManager = nullptr;
RuleEngine *m_ruleEngine = nullptr;
ScriptEngine *m_scriptEngine = nullptr;
LogEngine *m_logEngine = nullptr;
Logger *m_logger = nullptr;
TimeManager *m_timeManager = nullptr;
CloudManager *m_cloudManager = nullptr;
HardwareManagerImplementation *m_hardwareManager = nullptr;
DebugServerHandler *m_debugServerHandler = nullptr;
TagsStorage *m_tagsStorage = nullptr;
NetworkManager *m_networkManager;
UserManager *m_userManager;
System *m_system;
ExperienceManager *m_experienceManager;
ZigbeeManager *m_zigbeeManager;
ZWaveManager *m_zwaveManager;
SerialPortMonitor *m_serialPortMonitor;
ModbusRtuManager *m_modbusRtuManager;
NetworkManager *m_networkManager = nullptr;
UserManager *m_userManager = nullptr;
System *m_system = nullptr;
ExperienceManager *m_experienceManager = nullptr;
ZigbeeManager *m_zigbeeManager = nullptr;
ZWaveManager *m_zwaveManager = nullptr;
SerialPortMonitor *m_serialPortMonitor = nullptr;
ModbusRtuManager *m_modbusRtuManager = nullptr;
private slots:

112
libnymea-core/ruleengine/ruleengine.cpp
View File

@ -127,6 +127,7 @@
#include <QStringList>
#include <QStandardPaths>
#include <QCoreApplication>
#include <QMetaEnum>
NYMEA_LOGGING_CATEGORY(dcRuleEngine, "RuleEngine")
NYMEA_LOGGING_CATEGORY(dcRuleEngineDebug, "RuleEngineDebug")
@ -139,9 +140,9 @@ namespace nymeaserver {
RuleEngine::RuleEngine(ThingManager *thingManager, TimeManager *timeManager, LogEngine *logEngine, QObject *parent) :
QObject(parent),
m_thingManager(thingManager),
m_timeManager(timeManager),
m_logEngine(logEngine)
m_timeManager(timeManager)
{
m_logger = logEngine->registerLogSource("rules", {"id", "event"});
connect(m_thingManager, &ThingManager::eventTriggered, this, &RuleEngine::onEventTriggered);
@ -451,6 +452,7 @@ RuleEngine::RuleError RuleEngine::addRule(const Rule &rule, bool fromEdit)
appendRule(rule);
saveRule(rule);
m_logger->log({rule.id().toString(), "created"}, {{"name", rule.name()}});
if (!fromEdit)
emit ruleAdded(rule);
@ -494,6 +496,7 @@ RuleEngine::RuleError RuleEngine::editRule(const Rule &rule)
emit ruleConfigurationChanged(rule);
qCDebug(dcRuleEngine()) << "Rule" << rule.id().toString() << "updated.";
m_logger->log({rule.id().toString(), "changed"}, {{"name", rule.name()}});
return RuleErrorNoError;
}
@ -526,7 +529,7 @@ RuleEngine::RuleError RuleEngine::removeRule(const RuleId &ruleId, bool fromEdit
}
m_ruleIds.takeAt(index);
m_rules.remove(ruleId);
Rule rule = m_rules.take(ruleId);
m_activeRules.removeAll(ruleId);
NymeaSettings settings(NymeaSettings::SettingsRoleRules);
@ -534,7 +537,7 @@ RuleEngine::RuleError RuleEngine::removeRule(const RuleId &ruleId, bool fromEdit
settings.remove("");
settings.endGroup();
m_logEngine->removeRuleLogs(ruleId);
m_logger->log({ruleId.toString(), "removed"}, {{"name", rule.name()}});
if (!fromEdit)
emit ruleRemoved(ruleId);
@ -565,7 +568,7 @@ RuleEngine::RuleError RuleEngine::enableRule(const RuleId &ruleId)
saveRule(rule);
emit ruleConfigurationChanged(rule);
m_logEngine->logRuleEnabledChanged(rule, true);
m_logger->log({rule.id().toString(), "enabled"}, {{"name", rule.name()}});
qCDebug(dcRuleEngine()) << "Rule" << rule.name() << rule.id().toString() << "enabled.";
return RuleErrorNoError;
@ -591,7 +594,7 @@ RuleEngine::RuleError RuleEngine::disableRule(const RuleId &ruleId)
saveRule(rule);
emit ruleConfigurationChanged(rule);
m_logEngine->logRuleEnabledChanged(rule, false);
m_logger->log({rule.id().toString(), "disabled"}, {{"name", rule.name()}});
qCDebug(dcRuleEngine()) << "Rule" << rule.name() << rule.id().toString() << "disabled.";
return RuleErrorNoError;
}
@ -626,8 +629,8 @@ RuleEngine::RuleError RuleEngine::executeActions(const RuleId &ruleId)
}
qCDebug(dcRuleEngine) << "Executing rule actions of rule" << rule.name() << rule.id().toString();
m_logEngine->logRuleActionsExecuted(rule);
executeRuleActions(rule.actions());
m_logger->log({rule.id().toString(), "executed"}, {{"name", rule.name()}});
executeRuleActions(rule.id(), rule.actions());
return RuleErrorNoError;
}
@ -658,8 +661,9 @@ RuleEngine::RuleError RuleEngine::executeExitActions(const RuleId &ruleId)
}
qCDebug(dcRuleEngine) << "Executing rule exit actions of rule" << rule.name() << rule.id().toString();
m_logEngine->logRuleExitActionsExecuted(rule);
executeRuleActions(rule.exitActions());
m_logger->log({rule.id().toString(), "executed"}, {{"name", rule.name()}});
// m_logEngine->logRuleExitActionsExecuted(rule);
executeRuleActions(rule.id(), rule.exitActions());
return RuleErrorNoError;
}
@ -1400,7 +1404,7 @@ QList<RuleAction> RuleEngine::loadRuleActions(NymeaSettings *settings)
return actions;
}
void RuleEngine::executeRuleActions(const QList<RuleAction> ruleActions)
void RuleEngine::executeRuleActions(const RuleId &ruleId, const QList<RuleAction> &ruleActions)
{
QList<Action> actions;
QList<BrowserAction> browserActions;
@ -1500,19 +1504,31 @@ void RuleEngine::executeRuleActions(const QList<RuleAction> ruleActions)
foreach (const Action &action, actions) {
qCDebug(dcRuleEngine) << "Executing action" << action.actionTypeId() << action.params();
ThingActionInfo *info = m_thingManager->executeAction(action);
connect(info, &ThingActionInfo::finished, this, [info](){
connect(info, &ThingActionInfo::finished, this, [=](){
if (info->status() != Thing::ThingErrorNoError) {
qCWarning(dcRuleEngine) << "Error executing action:" << info->status() << info->displayMessage();
}
ActionType actionType = m_thingManager->findConfiguredThing(action.thingId())->thingClass().actionTypes().findById(action.actionTypeId());
m_logger->log({ruleId.toString(), "executed"}, {
{"name", m_rules.value(ruleId).name()},
{"status", QMetaEnum::fromType<Thing::ThingError>().valueToKey(info->status())},
{"thingId", info->action().thingId()},
{"action", actionType.name()}
});
});
}
foreach (const BrowserAction &browserAction, browserActions) {
BrowserActionInfo *info = m_thingManager->executeBrowserItem(browserAction);
connect(info, &BrowserActionInfo::finished, this, [info, this](){
m_logEngine->logBrowserAction(info->browserAction(), info->status() == Thing::ThingErrorNoError ? Logging::LoggingLevelInfo : Logging::LoggingLevelAlert, info->status());
connect(info, &BrowserActionInfo::finished, this, [this, ruleId, info](){
if (info->status() != Thing::ThingErrorNoError) {
qCWarning(dcRuleEngine) << "Error executing browser action:" << info->status();
m_logger->log({ruleId.toString(), "executed"}, {
{"name", m_rules.value(ruleId).name()},
{"status", QMetaEnum::fromType<Thing::ThingError>().valueToKey(info->status())},
{"thingId", info->browserAction().thingId()},
{"browserItem", info->browserAction().itemId()}
});
}
});
}
@ -1520,8 +1536,6 @@ void RuleEngine::executeRuleActions(const QList<RuleAction> ruleActions)
void RuleEngine::onEventTriggered(const Event &event)
{
QList<RuleAction> actions;
QList<RuleAction> eventBasedActions;
foreach (const Rule &rule, evaluateEvent(event)) {
if (m_executingRules.contains(rule.id())) {
qCWarning(dcRuleEngine()) << "WARNING: Loop detected in rule execution for rule" << rule.id().toString() << rule.name();
@ -1531,7 +1545,11 @@ void RuleEngine::onEventTriggered(const Event &event)
// Event based
if (!rule.eventDescriptors().isEmpty()) {
m_logEngine->logRuleTriggered(rule);
m_logger->log({rule.id().toString()}, {
{"name", rule.name()},
{"state", "triggered"}
});
QList<RuleAction> tmp;
if (rule.statesActive() && rule.timeActive()) {
qCDebug(dcRuleEngineDebug()) << "Executing actions";
@ -1541,56 +1559,56 @@ void RuleEngine::onEventTriggered(const Event &event)
tmp = rule.exitActions();
}
// check if we have an event based action or a normal action
foreach (const RuleAction &action, tmp) {
QList<RuleAction> actions;
foreach (RuleAction action, tmp) {
if (action.isEventBased()) {
eventBasedActions.append(action);
RuleActionParams newParams;
foreach (RuleActionParam ruleActionParam, action.ruleActionParams()) {
// if this event param should be taken over in this action
if (event.eventTypeId() == ruleActionParam.eventTypeId()) {
QVariant eventValue = event.params().paramValue(ruleActionParam.eventParamTypeId());
// TODO: limits / scale calculation -> actionValue = eventValue * x
// something like a EventParamDescriptor
ruleActionParam.setValue(eventValue);
qCDebug(dcRuleEngine) << "Using param value from event:" << ruleActionParam.value();
}
newParams.append(ruleActionParam);
}
action.setRuleActionParams(newParams);
actions.append(action);
} else {
actions.append(action);
}
}
executeRuleActions(rule.id(), actions);
} else {
// State based rule
m_logEngine->logRuleActiveChanged(rule);
m_logger->log({rule.id().toString()}, {
{"name", rule.name()},
{"state", rule.active() ? "active" : "inactive"}
});
emit ruleActiveChanged(rule);
if (rule.active()) {
actions.append(rule.actions());
executeRuleActions(rule.id(), rule.actions());
} else {
actions.append(rule.exitActions());
executeRuleActions(rule.id(), rule.exitActions());
}
}
}
// Set action params, depending on the event value
foreach (RuleAction ruleAction, eventBasedActions) {
RuleActionParams newParams;
foreach (RuleActionParam ruleActionParam, ruleAction.ruleActionParams()) {
// if this event param should be taken over in this action
if (event.eventTypeId() == ruleActionParam.eventTypeId()) {
QVariant eventValue = event.params().paramValue(ruleActionParam.eventParamTypeId());
// TODO: limits / scale calculation -> actionValue = eventValue * x
// something like a EventParamDescriptor
ruleActionParam.setValue(eventValue);
qCDebug(dcRuleEngine) << "Using param value from event:" << ruleActionParam.value();
}
newParams.append(ruleActionParam);
}
ruleAction.setRuleActionParams(newParams);
actions.append(ruleAction);
}
executeRuleActions(actions);
m_executingRules.clear();
}
void RuleEngine::onDateTimeChanged(const QDateTime &dateTime)
{
QList<RuleAction> actions;
foreach (const Rule &rule, evaluateTime(dateTime)) {
// TimeEvent based
QList<RuleAction> actions;
if (!rule.timeDescriptor().timeEventItems().isEmpty()) {
m_logEngine->logRuleTriggered(rule);
m_logger->log({rule.id().toString(), "triggered"}, {{"name", rule.name()}});
if (rule.statesActive() && rule.timeActive()) {
actions.append(rule.actions());
} else {
@ -1598,7 +1616,7 @@ void RuleEngine::onDateTimeChanged(const QDateTime &dateTime)
}
} else {
// Calendar based rule
m_logEngine->logRuleActiveChanged(rule);
m_logger->log({rule.id().toString(), "triggered"}, {{"name", rule.name()}});
emit ruleActiveChanged(rule);
if (rule.active()) {
actions.append(rule.actions());
@ -1606,8 +1624,8 @@ void RuleEngine::onDateTimeChanged(const QDateTime &dateTime)
actions.append(rule.exitActions());
}
}
executeRuleActions(rule.id(), actions);
}
executeRuleActions(actions);
}
void RuleEngine::onThingRemoved(const ThingId &thingId)

7
libnymea-core/ruleengine/ruleengine.h
View File

@ -46,10 +46,11 @@ Q_DECLARE_LOGGING_CATEGORY(dcRuleEngine)
Q_DECLARE_LOGGING_CATEGORY(dcRuleEngineDebug)
class ThingManager;
class LogEngine;
class Logger;
namespace nymeaserver {
class LogEngine;
class TimeManager;
class RuleEngine : public QObject
@ -134,13 +135,13 @@ private:
void saveRuleActions(NymeaSettings *settings, const QList<RuleAction> &ruleActions);
QList<RuleAction> loadRuleActions(NymeaSettings *settings);
void executeRuleActions(const QList<RuleAction> ruleActions);
void executeRuleActions(const RuleId &ruleId, const QList<RuleAction> &ruleActions);
private:
ThingManager *m_thingManager = nullptr;
TimeManager *m_timeManager = nullptr;
LogEngine *m_logEngine = nullptr;
Logger *m_logger = nullptr;
QList<RuleId> m_ruleIds; // Keeping a list of RuleIds to keep sorting order...
QHash<RuleId, Rule> m_rules; // ...but use a Hash for faster finding

17
libnymea-core/scriptengine/scriptaction.cpp
View File

@ -32,9 +32,11 @@
#include "integrations/thingmanager.h"
#include "types/action.h"
#include "logging/logengine.h"
#include <QQmlEngine>
#include <qqml.h>
#include <QQmlContext>
#include <QLoggingCategory>
Q_DECLARE_LOGGING_CATEGORY(dcScriptEngine)
@ -50,6 +52,10 @@ ScriptAction::ScriptAction(QObject *parent) : QObject(parent)
void ScriptAction::classBegin()
{
m_thingManager = reinterpret_cast<ThingManager*>(qmlEngine(this)->property("thingManager").toULongLong());
m_scriptId = qmlEngine(this)->contextForObject(this)->contextProperty("scriptId").toUuid();
m_logger = qmlEngine(this)->contextForObject(this)->contextProperty("logger").value<Logger*>();
}
void ScriptAction::componentComplete()
@ -156,7 +162,16 @@ void ScriptAction::execute(const QVariantMap &params)
}
action.setParams(paramList);
qCDebug(dcScriptEngine()) << "Executing action:" << action.thingId() << action.actionTypeId() << action.params();
m_thingManager->executeAction(action);
ThingActionInfo *actionInfo = m_thingManager->executeAction(action);
connect(actionInfo, &ThingActionInfo::finished, this, [this, actionInfo, thing, action](){
ActionType actionType = thing->thingClass().actionTypes().findById(action.actionTypeId());
m_logger->log({m_scriptId.toString(), "action"}, {
{"thingId", thing->id()},
{"action", actionType.name()},
{"status", QMetaEnum::fromType<Thing::ThingError>().valueToKey(actionInfo->status())}
});
});
}
}

5
libnymea-core/scriptengine/scriptaction.h
View File

@ -34,7 +34,9 @@
#include <QObject>
#include <QQmlParserStatus>
#include <QVariantMap>
#include <QUuid>
class Logger;
class ThingManager;
namespace nymeaserver {
@ -77,6 +79,9 @@ signals:
public:
ThingManager *m_thingManager = nullptr;
Logger *m_logger = nullptr;
QUuid m_scriptId;
QString m_thingId;
QString m_interfaceName;
QString m_actionTypeId;

8
libnymea-core/scriptengine/scriptengine.cpp
View File

@ -42,6 +42,7 @@
#include "scriptthings.h"
#include "nymeasettings.h"
#include "logging/logengine.h"
#include <QQmlApplicationEngine>
#include <QQmlContext>
@ -63,7 +64,7 @@ QtMessageHandler ScriptEngine::s_upstreamMessageHandler;
QLoggingCategory::CategoryFilter ScriptEngine::s_oldCategoryFilter = nullptr;
QMutex ScriptEngine::s_loggerMutex;
ScriptEngine::ScriptEngine(ThingManager *thingManager, QObject *parent) : QObject(parent),
ScriptEngine::ScriptEngine(ThingManager *thingManager, LogEngine *logEngine, QObject *parent) : QObject(parent),
m_thingManager(thingManager)
{
qmlRegisterType<ScriptEvent>("nymea", 1, 0, "ThingEvent");
@ -76,6 +77,8 @@ ScriptEngine::ScriptEngine(ThingManager *thingManager, QObject *parent) : QObjec
qmlRegisterType<ScriptThing>("nymea", 1, 0, "Thing");
qmlRegisterType<ScriptThings>("nymea", 1, 0, "Things");
m_logger = logEngine->registerLogSource("scripts", {"id", "event"});
m_engine = new QQmlEngine(this);
m_engine->setProperty("thingManager", reinterpret_cast<quint64>(m_thingManager));
@ -401,8 +404,11 @@ bool ScriptEngine::loadScript(Script *script)
script->errors.clear();
script->component = new QQmlComponent(m_engine, QUrl::fromLocalFile(fileName), this);
script->context = new QQmlContext(m_engine, this);
script->context->setContextProperty("logger", QVariant::fromValue(m_logger));
script->context->setContextProperty("scriptId", script->id().toString());
script->object = script->component->create(script->context);
if (!script->object) {

8
libnymea-core/scriptengine/scriptengine.h
View File

@ -38,9 +38,12 @@
#include <QLoggingCategory>
#include <QMutex>
#include "integrations/thingmanager.h"
#include "script.h"
class ThingManager;
class LogEngine;
class Logger;
namespace nymeaserver {
namespace scriptengine {
@ -76,7 +79,7 @@ public:
QByteArray content;
};
explicit ScriptEngine(ThingManager *thingManager, QObject *parent = nullptr);
explicit ScriptEngine(ThingManager *thingManager, LogEngine *logEngine, QObject *parent = nullptr);
~ScriptEngine();
Scripts scripts();
@ -105,6 +108,7 @@ private:
private:
ThingManager *m_thingManager = nullptr;
QQmlEngine *m_engine = nullptr;
Logger *m_logger = nullptr;
QHash<QUuid, Script*> m_scripts;

18
libnymea-core/scriptengine/scriptstate.cpp
View File

@ -33,6 +33,9 @@
#include <QColor>
#include <qqml.h>
#include <QQmlEngine>
#include <QQmlContext>
#include "logging/logengine.h"
#include <QLoggingCategory>
Q_DECLARE_LOGGING_CATEGORY(dcScriptEngine)
@ -56,6 +59,9 @@ void ScriptState::classBegin()
connectToThing();
}
});
m_scriptId = qmlEngine(this)->contextForObject(this)->contextProperty("scriptId").toUuid();
m_logger = qmlEngine(this)->contextForObject(this)->contextProperty("logger").value<Logger*>();
}
void ScriptState::componentComplete()
@ -175,13 +181,23 @@ void ScriptState::setValue(const QVariant &value)
action.setParams(params);
qCDebug(dcScriptEngine()) << "Executing action on" << thing->name();
m_valueCache = QVariant();
m_pendingActionInfo = m_thingManager->executeAction(action);
connect(m_pendingActionInfo, &ThingActionInfo::finished, this, [this](){
connect(m_pendingActionInfo, &ThingActionInfo::finished, this, [this, thing, actionTypeId](){
ActionType actionType = thing->thingClass().actionTypes().findById(actionTypeId);
m_logger->log({m_scriptId.toString(), "action"}, {
{"thingId", thing->id()},
{"action", actionType.name()},
{"status", QMetaEnum::fromType<Thing::ThingError>().valueToKey(m_pendingActionInfo->status())}
});
m_pendingActionInfo = nullptr;
if (!m_valueCache.isNull()) {
setValue(m_valueCache);
}
});
}

4
libnymea-core/scriptengine/scriptstate.h
View File

@ -38,6 +38,8 @@
#include "integrations/thingmanager.h"
#include "integrations/thingactioninfo.h"
class Logger;
namespace nymeaserver {
namespace scriptengine {
@ -89,6 +91,8 @@ private slots:
private:
ThingManager *m_thingManager = nullptr;
Logger *m_logger = nullptr;
QUuid m_scriptId;
QString m_thingId;
QString m_stateTypeId;

15
libnymea-core/zwave/zwavedevicedatabase.cpp
View File

@ -34,10 +34,10 @@
#include <QSqlError>
#include <QSqlQuery>
#include <QSqlResult>
#include <QDataStream>
#include "hardware/zwave/zwavenode.h"
#include "zwavenodeimplementation.h"
#include "logging/logvaluetool.h"
#include "loggingcategories.h"
Q_DECLARE_LOGGING_CATEGORY(dcZWave)
@ -197,6 +197,11 @@ void ZWaveDeviceDatabase::removeNode(quint8 nodeId)
void ZWaveDeviceDatabase::storeValue(ZWaveNode *node, quint64 valueId)
{
ZWaveValue value = node->value(valueId);
QByteArray byteArray;
QDataStream out(&byteArray, QIODevice::WriteOnly);
out << value.value();
QSqlQuery query(m_db);
query.prepare("INSERT OR REPLACE INTO nodevalues(valueId, nodeId, valueGenre, commandClass, instance, idx, type, value, valueSelection, description) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);");
query.addBindValue(value.id());
@ -206,7 +211,7 @@ void ZWaveDeviceDatabase::storeValue(ZWaveNode *node, quint64 valueId)
query.addBindValue(value.instance());
query.addBindValue(value.index());
query.addBindValue(value.type());
query.addBindValue(LogValueTool::serializeValue(value.value()));
query.addBindValue(byteArray.toBase64());
query.addBindValue(value.valueListSelection());
query.addBindValue(value.description());
if (!query.exec()) {
@ -269,7 +274,11 @@ ZWaveNodes ZWaveDeviceDatabase::createNodes(ZWaveManager *manager)
valueQuery.value("idx").toUInt(),
static_cast<ZWaveValue::Type>(valueQuery.value("type").toInt()),
valueQuery.value("description").toString());
value.setValue(LogValueTool::deserializeValue(valueQuery.value("value").toString()), valueQuery.value("valueSelection").toInt());
QByteArray data = QByteArray::fromBase64(valueQuery.value("value").toString().toUtf8());
QDataStream inputStream(data);
QVariant deseriealizedValue;
inputStream >> deseriealizedValue;
value.setValue(deseriealizedValue, valueQuery.value("valueSelection").toInt());
node->updateValue(value);
}

10
libnymea/integrations/thing.cpp
View File

@ -600,6 +600,11 @@ QList<EventTypeId> Thing::loggedEventTypeIds() const
return m_loggedEventTypeIds;
}
QList<ActionTypeId> Thing::loggedActionTypeIds() const
{
return m_loggedActionTypeIds;
}
/*! Returns the \l{ThingId} of the parent of this thing. If the parentId
is not set, this thing does not have a parent.
*/
@ -676,6 +681,11 @@ void Thing::setLoggedEventTypeIds(const QList<EventTypeId> loggedEventTypeIds)
m_loggedEventTypeIds = loggedEventTypeIds;
}
void Thing::setLoggedActionTypeIds(const QList<ActionTypeId> loggedActionTypeIds)
{
m_loggedActionTypeIds = loggedActionTypeIds;
}
void Thing::setStateValueFilter(const StateTypeId &stateTypeId, Types::StateValueFilter filter)
{
for (int i = 0; i < m_states.count(); i++) {

4
libnymea/integrations/thing.h
View File

@ -63,6 +63,7 @@ class LIBNYMEA_EXPORT Thing: public QObject
Q_PROPERTY(QUuid parentId READ parentId USER true)
Q_PROPERTY(QList<StateTypeId> loggedStateTypeIds READ loggedStateTypeIds USER true)
Q_PROPERTY(QList<EventTypeId> loggedEventTypeIds READ loggedEventTypeIds USER true)
Q_PROPERTY(QList<ActionTypeId> loggedActionTypeIds READ loggedActionTypeIds USER true)
public:
enum ThingError {
@ -154,6 +155,7 @@ public:
QList<StateTypeId> loggedStateTypeIds() const;
QList<EventTypeId> loggedEventTypeIds() const;
QList<ActionTypeId> loggedActionTypeIds() const;
ThingId parentId() const;
void setParentId(const ThingId &parentId);
@ -186,6 +188,7 @@ private:
void setSetupStatus(ThingSetupStatus status, ThingError setupError, const QString &displayMessage = QString());
void setLoggedStateTypeIds(const QList<StateTypeId> loggedStateTypeIds);
void setLoggedEventTypeIds(const QList<EventTypeId> loggedEventTypeIds);
void setLoggedActionTypeIds(const QList<ActionTypeId> loggedActionTypeIds);
void setStateValueFilter(const StateTypeId &stateTypeId, Types::StateValueFilter filter);
private:
@ -205,6 +208,7 @@ private:
QList<StateTypeId> m_loggedStateTypeIds;
QList<EventTypeId> m_loggedEventTypeIds;
QList<ActionTypeId> m_loggedActionTypeIds;
QHash<StateTypeId, StateValueFilter*> m_stateValueFilters;
};

1
libnymea/integrations/thingmanager.h
View File

@ -83,6 +83,7 @@ public:
virtual Thing::ThingError setStateLogging(const ThingId &thingId, const StateTypeId &stateTypeId, bool enabled) = 0;
virtual Thing::ThingError setEventLogging(const ThingId &thingId, const EventTypeId &eventTypeId, bool enabled) = 0;
virtual Thing::ThingError setActionLogging(const ThingId &thingId, const ActionTypeId &actionTypeId, bool enabled) = 0;
virtual Thing::ThingError setStateFilter(const ThingId &thingId, const StateTypeId &stateTypeId, Types::StateValueFilter filter) = 0;
virtual Thing::ThingError removeConfiguredThing(const ThingId &thingId) = 0;

49
libnymea/integrations/thingutils.cpp
View File

@ -193,42 +193,45 @@ Interface ThingUtils::loadInterface(const QString &name)
InterfaceActionTypes actionTypes;
InterfaceEventTypes eventTypes;
foreach (const QVariant &stateVariant, content.value("states").toList()) {
QVariantMap stateMap = stateVariant.toMap();
InterfaceStateType stateType;
stateType.setName(stateVariant.toMap().value("name").toString());
stateType.setType(QVariant::nameToType(stateVariant.toMap().value("type").toByteArray()));
stateType.setPossibleValues(stateVariant.toMap().value("allowedValues").toList());
stateType.setMinValue(stateVariant.toMap().value("minValue"));
stateType.setMaxValue(stateVariant.toMap().value("maxValue"));
stateType.setOptional(stateVariant.toMap().value("optional", false).toBool());
if (stateVariant.toMap().contains("logged")) {
stateType.setLoggingOverride(true);
stateType.setSuggestLogging(stateVariant.toMap().value("logged", false).toBool());
}
if (stateVariant.toMap().contains("unit")) {
stateType.setName(stateMap.value("name").toString());
stateType.setType(QVariant::nameToType(stateMap.value("type").toByteArray()));
stateType.setPossibleValues(stateMap.value("allowedValues").toList());
stateType.setMinValue(stateMap.value("minValue"));
stateType.setMaxValue(stateMap.value("maxValue"));
stateType.setOptional(stateMap.value("optional", false).toBool());
if (stateMap.contains("unit")) {
QMetaEnum unitEnum = QMetaEnum::fromType<Types::Unit>();
int enumValue = unitEnum.keyToValue("Unit" + stateVariant.toMap().value("unit").toByteArray());
int enumValue = unitEnum.keyToValue("Unit" + stateMap.value("unit").toByteArray());
if (enumValue == -1) {
qCWarning(dcThingManager) << "Invalid unit" << stateVariant.toMap().value("unit").toString() << "in interface" << name;
qCWarning(dcThingManager) << "Invalid unit" << stateMap.value("unit").toString() << "in interface" << name;
} else {
stateType.setUnit(static_cast<Types::Unit>(unitEnum.keyToValue("Unit" + stateVariant.toMap().value("unit").toByteArray())));
stateType.setUnit(static_cast<Types::Unit>(enumValue));
}
}
stateTypes.append(stateType);
ParamType stateChangeEventParamType;
stateChangeEventParamType.setName(stateType.name());
stateChangeEventParamType.setType(stateType.type());
stateChangeEventParamType.setAllowedValues(stateType.possibleValues());
stateChangeEventParamType.setMinValue(stateType.minValue());
stateChangeEventParamType.setMaxValue(stateType.maxValue());
ParamType stateChangeActionParamType;
stateChangeActionParamType.setName(stateType.name());
stateChangeActionParamType.setType(stateType.type());
stateChangeActionParamType.setAllowedValues(stateType.possibleValues());
stateChangeActionParamType.setMinValue(stateType.minValue());
stateChangeActionParamType.setMaxValue(stateType.maxValue());
if (stateVariant.toMap().value("writable", false).toBool()) {
if (stateMap.value("writable", false).toBool()) {
InterfaceActionType stateChangeActionType;
stateChangeActionType.setName(stateType.name());
stateChangeActionType.setOptional(stateType.optional());
stateChangeActionType.setParamTypes(ParamTypes() << stateChangeEventParamType);
stateChangeActionType.setParamTypes(ParamTypes() << stateChangeActionParamType);
actionTypes.append(stateChangeActionType);
}
if (stateMap.contains("logged")) {
stateType.setLoggingOverride(true);
stateType.setSuggestLogging(stateMap.value("logged", false).toBool());
}
stateTypes.append(stateType);
}
foreach (const QVariant &actionVariant, content.value("actions").toList()) {

3
libnymea/interfaces/update.json
View File

@ -8,7 +8,8 @@
},
{
"name": "currentVersion",
"type": "QString"
"type": "QString",
"logged": true
},
{
"name": "availableVersion",

3
libnymea/interfaces/ventilation.json
View File

@ -8,7 +8,8 @@
"minValue": "any",
"maxValue": "any",
"writable": true,
"optional": true
"optional": true,
"logged": true
}
]
}

4
libnymea/jsonrpc/jsonhandler.cpp
View File

@ -363,11 +363,11 @@ QVariant JsonHandler::pack(const QMetaObject &metaObject, const void *value) con
list << entry;
}
} else if (propertyTypeName == "QList<StateTypeId>") {
foreach (const EventTypeId &entry, propertyValue.value<QList<EventTypeId>>()) {
foreach (const EventTypeId &entry, propertyValue.value<QList<StateTypeId>>()) {
list << entry;
}
} else if (propertyTypeName == "QList<ActionTypeId>") {
foreach (const EventTypeId &entry, propertyValue.value<QList<EventTypeId>>()) {
foreach (const EventTypeId &entry, propertyValue.value<QList<ActionTypeId>>()) {
list << entry;
}
} else if (propertyTypeName == "QList<QDateTime>") {

6
libnymea/libnymea.pro
View File

@ -48,6 +48,9 @@ HEADERS += \
jsonrpc/jsonreply.h \
jsonrpc/jsonrpcserver.h \
libnymea.h \
logging/logengine.h \
logging/logentry.h \
logging/logger.h \
network/apikeys/apikey.h \
network/apikeys/apikeysprovider.h \
network/apikeys/apikeystorage.h \
@ -161,6 +164,9 @@ SOURCES += \
jsonrpc/jsonhandler.cpp \
jsonrpc/jsonreply.cpp \
jsonrpc/jsonrpcserver.cpp \
logging/logengine.cpp \
logging/logentry.cpp \
logging/logger.cpp \
loggingcategories.cpp \
network/apikeys/apikey.cpp \
network/apikeys/apikeysprovider.cpp \

41
libnymea/logging/logengine.cpp Normal file
View File

@ -0,0 +1,41 @@
#include "logengine.h"
#include "logger.h"
#include "loggingcategories.h"
NYMEA_LOGGING_CATEGORY(dcLogEngine, "LogEngine")
LogFetchJob::LogFetchJob(QObject *parent): QObject(parent)
{
}
LogEntries LogFetchJob::entries() const
{
return m_entries;
}
void LogFetchJob::finish(const LogEntries &entries)
{
m_entries = entries;
emit finished(entries);
QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection, Q_ARG(LogEntries, entries));
}
LogEngine::LogEngine(QObject *parent)
: QObject{parent}
{
}
Logger *LogEngine::createLogger(const QString &name, const QStringList &tags, Types::LoggingType loggingType)
{
return new Logger(this, name, tags, loggingType);
}
void LogEngine::finishFetchJob(LogFetchJob *job, const LogEntries &entries)
{
job->finish(entries);
}

71
libnymea/logging/logengine.h Normal file
View File

@ -0,0 +1,71 @@
#ifndef LOGENGINE_H
#define LOGENGINE_H
#include "logentry.h"
#include "types/param.h"
#include "typeutils.h"
#include "logger.h"
#include <QObject>
#include <QDateTime>
#include <QLoggingCategory>
Q_DECLARE_LOGGING_CATEGORY(dcLogEngine)
class LogFetchJob: public QObject
{
Q_OBJECT
public:
LogFetchJob(QObject *parent = nullptr);
LogEntries entries() const;
signals:
void finished(const LogEntries &entries);
private:
friend class LogEngine;
void finish(const LogEntries &entries);
LogEntries m_entries;
};
class LogEngine : public QObject
{
Q_OBJECT
public:
explicit LogEngine(QObject *parent = nullptr);
virtual ~LogEngine() = default;
// LogEngine keeps ownership:
// * name: must be unique (table name)
// * tags: optional, indexed column names for faster queries
// Values for tagged columns must be non-null.
// * loggingType defines wheter a source will log discrete values or should be resampled
// * sampleColumn is required for LoggingTypeSampled. Values in the given column will be sampled
virtual Logger *registerLogSource(const QString &name, const QStringList &tags = QStringList(), Types::LoggingType loggingType = Types::LoggingTypeDiscrete, const QString &sampleColumn = QString()) = 0;
// Unregistering will discard all related entries from the log database.
// It is not required to unregister or clean up a log source on application shutdown as the engine keeps ownership.
virtual void unregisterLogSource(const QString &name) = 0;
virtual LogFetchJob *fetchLogEntries(const QStringList &sources, const QStringList &columns = QStringList(), const QDateTime &from = QDateTime(), const QDateTime &to = QDateTime(), const QVariantMap &filer = QVariantMap(), Types::SampleRate sampleRate = Types::SampleRateAny, Qt::SortOrder sortOrder = Qt::AscendingOrder, int offset = 0, int limit = 0) = 0;
virtual bool jobsRunning() const = 0;
virtual void clear(const QString &source) = 0;
signals:
void logEntryAdded(const LogEntry &entry);
protected:
Logger *createLogger(const QString &name, const QStringList &tags, Types::LoggingType loggingType);
void finishFetchJob(LogFetchJob *job, const LogEntries &entries);
private:
friend class Logger;
virtual void logEvent(Logger *logger, const QStringList &tags, const QVariantMap &values) = 0;
};
#endif // LOGENGINE_H

50
libnymea/logging/logentry.cpp Normal file
View File

@ -0,0 +1,50 @@
#include "logentry.h"
LogEntry::LogEntry()
{
}
LogEntry::LogEntry(const QDateTime &timestamp, const QString &source, const QVariantMap &values):
m_timestamp(timestamp),
m_source(source),
m_values(values)
{
}
QDateTime LogEntry::timestamp() const
{
return m_timestamp;
}
QString LogEntry::source() const
{
return m_source;
}
QVariantMap LogEntry::values() const
{
return m_values;
}
LogEntries::LogEntries(): QList<LogEntry>()
{
}
LogEntries::LogEntries(const QList<LogEntry> &other):
QList<LogEntry>(other)
{
}
QVariant LogEntries::get(int index) const
{
return QVariant::fromValue(at(index));
}
void LogEntries::put(const QVariant &variant)
{
append(variant.value<LogEntry>());
}

42
libnymea/logging/logentry.h Normal file
View File

@ -0,0 +1,42 @@
#ifndef LOGENTRY_H
#define LOGENTRY_H
#include <QDateTime>
#include <QObject>
#include <QVariant>
class LogEntry
{
Q_GADGET
Q_PROPERTY(QDateTime timestamp READ timestamp)
Q_PROPERTY(QString source READ source )
Q_PROPERTY(QVariantMap values READ values)
public:
LogEntry();
LogEntry(const QDateTime &timestamp, const QString &source, const QVariantMap &values);
QDateTime timestamp() const;
QString source() const;
QVariantMap values() const;
private:
QDateTime m_timestamp;
QString m_source;
QVariantMap m_values;
};
Q_DECLARE_METATYPE(LogEntry)
class LogEntries: public QList<LogEntry>
{
Q_GADGET
Q_PROPERTY(int count READ count)
public:
LogEntries();
LogEntries(const QList<LogEntry> &other);
LogEntries(std::initializer_list<LogEntry> args):QList(args) {}
Q_INVOKABLE QVariant get(int index) const;
Q_INVOKABLE void put(const QVariant &variant);
};
Q_DECLARE_METATYPE(LogEntries)
#endif // LOGENTRY_H

32
libnymea/logging/logger.cpp Normal file
View File

@ -0,0 +1,32 @@
#include "logger.h"
#include "logengine.h"
#include <QLoggingCategory>
Logger::Logger(LogEngine *engine, const QString &name, const QStringList &tagNames, Types::LoggingType loggingType):
m_engine(engine),
m_name(name),
m_tagNames(tagNames),
m_loggingType(loggingType)
{
}
QString Logger::name() const
{
return m_name;
}
QStringList Logger::tagNames() const
{
return m_tagNames;
}
Types::LoggingType Logger::loggingType() const
{
return m_loggingType;
}
void Logger::log(const QStringList &tags, const QVariantMap &values)
{
m_engine->logEvent(this, tags, values);
}

31
libnymea/logging/logger.h Normal file
View File

@ -0,0 +1,31 @@
#ifndef LOGGER_H
#define LOGGER_H
#include <QStringList>
#include <QVariant>
#include "typeutils.h"
class LogEngine;
class Logger
{
public:
QString name() const;
QStringList tagNames() const;
Types::LoggingType loggingType() const;
void log(const QStringList &tags, const QVariantMap &values);
private:
friend class LogEngine;
Logger(LogEngine *engine, const QString &name, const QStringList &tagNames, Types::LoggingType loggingType);
LogEngine *m_engine = nullptr;
QString m_name;
QStringList m_tagNames;
Types::LoggingType m_loggingType = Types::LoggingTypeDiscrete;
};
Q_DECLARE_METATYPE(Logger*)
#endif // LOGGER_H

1
libnymea/loggingcategories.cpp
View File

@ -49,7 +49,6 @@ NYMEA_LOGGING_CATEGORY(dcPlatformZeroConf, "PlatformZeroConf")
NYMEA_LOGGING_CATEGORY(dcExperiences, "Experiences")
NYMEA_LOGGING_CATEGORY(dcTimeManager, "TimeManager")
NYMEA_LOGGING_CATEGORY(dcHardware, "Hardware")
NYMEA_LOGGING_CATEGORY(dcLogEngine, "LogEngine")
NYMEA_LOGGING_CATEGORY(dcServerManager, "ServerManager")
NYMEA_LOGGING_CATEGORY(dcTcpServer, "TcpServer")
NYMEA_LOGGING_CATEGORY(dcTcpServerTraffic, "TcpServerTraffic")

1
libnymea/loggingcategories.h
View File

@ -57,7 +57,6 @@ Q_DECLARE_LOGGING_CATEGORY(dcPlatformZeroConf)
Q_DECLARE_LOGGING_CATEGORY(dcExperiences)
Q_DECLARE_LOGGING_CATEGORY(dcTimeManager)
Q_DECLARE_LOGGING_CATEGORY(dcHardware)
Q_DECLARE_LOGGING_CATEGORY(dcLogEngine)
Q_DECLARE_LOGGING_CATEGORY(dcServerManager)
Q_DECLARE_LOGGING_CATEGORY(dcTcpServer)
Q_DECLARE_LOGGING_CATEGORY(dcTcpServerTraffic)

2
libnymea/types/action.h
View File

@ -50,6 +50,8 @@ public:
TriggeredByRule,
TriggeredByScript
};
Q_ENUM(TriggeredBy)
explicit Action(const ActionTypeId &actionTypeId = ActionTypeId(), const ThingId &thingId = ThingId(), TriggeredBy triggeredBy = TriggeredByUser);
Action(const Action &other);

4
libnymea/types/statetype.cpp
View File

@ -213,9 +213,9 @@ bool StateType::suggestLogging() const
return m_logged;
}
void StateType::setSuggestLogging(bool logged)
void StateType::setSuggestLogging(bool suggestLogging)
{
m_logged = logged;
m_logged = suggestLogging;
}
Types::StateValueFilter StateType::filter() const

2
libnymea/types/statetype.h
View File

@ -94,7 +94,7 @@ public:
void setCached(bool cached);
bool suggestLogging() const;
void setSuggestLogging(bool logged);
void setSuggestLogging(bool suggestLogging);
Types::StateValueFilter filter() const;
void setFilter(Types::StateValueFilter filter);

20
libnymea/typeutils.h
View File

@ -201,6 +201,26 @@ public:
static PermissionScope scopeFromString(const QString &scopeString);
static QStringList scopesToStringList(PermissionScopes scopes);
static QString scopeToString(PermissionScope scope);
enum LoggingType {
LoggingTypeDiscrete,
LoggingTypeSampled,
};
Q_ENUM(LoggingType)
enum SampleRate {
SampleRateAny = 0,
SampleRate1Min = 1,
SampleRate15Mins = 15,
SampleRate1Hour = 60,
SampleRate3Hours = 180,
SampleRate1Day = 1440,
SampleRate1Week = 10080,
SampleRate1Month = 43200,
SampleRate1Year = 525600
};
Q_ENUM(SampleRate)
};
Q_DECLARE_METATYPE(Types::InputType)

4
nymea.pro
View File

@ -4,8 +4,8 @@ include(nymea.pri)
NYMEA_VERSION_STRING=$$system('dpkg-parsechangelog | sed -n -e "s/^Version: //p"')
# define protocol versions
JSON_PROTOCOL_VERSION_MAJOR=7
JSON_PROTOCOL_VERSION_MINOR=1
JSON_PROTOCOL_VERSION_MAJOR=8
JSON_PROTOCOL_VERSION_MINOR=0
JSON_PROTOCOL_VERSION="$${JSON_PROTOCOL_VERSION_MAJOR}.$${JSON_PROTOCOL_VERSION_MINOR}"
LIBNYMEA_API_VERSION_MAJOR=7
LIBNYMEA_API_VERSION_MINOR=4

4
plugins/mock/integrationpluginmock.cpp
View File

@ -628,13 +628,15 @@ void IntegrationPluginMock::executeAction(ThingActionInfo *info)
if (info->thing()->thingClassId() == autoMockThingClassId) {
if (info->action().actionTypeId() == autoMockMockActionAsyncActionTypeId || info->action().actionTypeId() == autoMockMockActionAsyncBrokenActionTypeId) {
QTimer::singleShot(1000, info->thing(), [info](){
QTimer::singleShot(1000, info->thing(), [this, info](){
if (info->action().actionTypeId() == autoMockMockActionAsyncBrokenActionTypeId) {
info->finish(Thing::ThingErrorSetupFailed, QT_TR_NOOP("This mock action is intentionally broken."));
} else {
m_daemons.value(info->thing())->actionExecuted(info->action().actionTypeId());
info->finish(Thing::ThingErrorNoError);
}
});
return;
}
if (info->action().actionTypeId() == autoMockMockActionBrokenActionTypeId) {

1
plugins/mock/integrationpluginmock.json
View File

@ -210,6 +210,7 @@
"id": "863d5920-b1cf-4eb9-88bd-8f7b8583b1cf",
"name": "event2",
"displayName": "Mock Event 2",
"suggestLogging": true,
"paramTypes": [
{
"id": "0550e16d-60b9-4ba5-83f4-4d3cee656121",

2
plugins/mock/plugininfo.h
View File

@ -14,7 +14,7 @@
#include <QLoggingCategory>
#include <QObject>
extern "C" const QString libnymea_api_version() { return QString("7.3.0");}
extern "C" const QString libnymea_api_version() { return QString("7.4.0");}
Q_DECLARE_LOGGING_CATEGORY(dcMock)
Q_LOGGING_CATEGORY(dcMock, "Mock")

104
server/nymeaapplication.cpp
View File

@ -59,75 +59,64 @@ Q_DECLARE_LOGGING_CATEGORY(dcApplication)
namespace nymeaserver {
static bool s_aboutToShutdown = false;
static bool s_multipleShutdownDetected = false;
static int s_shutdownCounter = 0;
static void catchUnixSignals(const std::vector<int>& quitSignals, const std::vector<int>& ignoreSignals = std::vector<int>())
{
auto handler = [](int sig) ->void {
switch (sig) {
case SIGQUIT:
qCDebug(dcApplication) << "Cought SIGQUIT quit signal...";
break;
case SIGINT:
qCDebug(dcApplication) << "Cought SIGINT quit signal...";
break;
case SIGTERM:
qCDebug(dcApplication) << "Cought SIGTERM quit signal...";
break;
case SIGHUP:
qCDebug(dcApplication) << "Cought SIGHUP quit signal...";
break;
default:
break;
}
if (s_aboutToShutdown) {
switch (s_shutdownCounter) {
case 0:
qCWarning(dcApplication()) << "Already shutting down. Be nice and give me some time to clean up, please.";
break;
case 1:
qCCritical(dcApplication()) << "I told you, I'm already shutting down. Be nice and give me some time to clean up, PLEASE.";
break;
case 2:
qCCritical(dcApplication()) << "Still shutting down...";
break;
case 3:
qCCritical(dcApplication()) << "Hmpf...";
break;
case 4:
qCCritical(dcApplication()) << "It's getting boring...";
break;
case 5:
qCCritical(dcApplication()) << "S H U T T I N G DOWN";
break;
case 6:
qCCritical(dcApplication()) << "S H U T T I N G DOWN";
break;
case 7:
qCCritical(dcApplication()) << "S H U T T I N G DOWN";
break;
default:
qCCritical(dcApplication()) << "Fuck this shit. I'm out...";
NymeaApplication::quit();
break;
// forecefully exit() if repeated signals come in.
if (s_shutdownCounter > 0) {
if (s_shutdownCounter < 4) {
qCCritical(dcApplication()) << "Shutdown in progress." << (4 - s_shutdownCounter) << "more times to abort.";
s_shutdownCounter++;
return;
}
s_shutdownCounter++;
exit(EXIT_FAILURE);
return;
}
NymeaCore::ShutdownReason reason = NymeaCore::ShutdownReasonQuit;
switch (sig) {
case SIGQUIT:
qCDebug(dcApplication) << "Cought SIGQUIT signal...";
reason = NymeaCore::ShutdownReasonQuit;
break;
case SIGINT:
qCDebug(dcApplication) << "Cought SIGINT signal...";
reason = NymeaCore::ShutdownReasonTerm;
break;
case SIGTERM:
qCDebug(dcApplication) << "Cought SIGTERM signal...";
reason = NymeaCore::ShutdownReasonTerm;
break;
case SIGHUP:
qCDebug(dcApplication) << "Cought SIGHUP signal...";
reason = NymeaCore::ShutdownReasonTerm;
break;
case SIGKILL:
qCDebug(dcApplication) << "Cought SIGKILL signal...";
reason = NymeaCore::ShutdownReasonTerm;
break;
case SIGSEGV:
qCDebug(dcApplication) << "Cought SIGSEGV quit signal...";
reason = NymeaCore::ShutdownReasonFailure;
break;
case SIGFPE:
qCDebug(dcApplication) << "Cought SIGFPE quit signal...";
reason = NymeaCore::ShutdownReasonFailure;
break;
default:
qCDebug(dcApplication) << "Cought signal" << sig;
break;
}
qCInfo(dcApplication) << "=====================================";
qCInfo(dcApplication) << "Shutting down nymea daemon";
qCInfo(dcApplication) << "Shutting down nymea:core";
qCInfo(dcApplication) << "=====================================";
s_aboutToShutdown = true;
NymeaCore::instance()->destroy();
if (s_multipleShutdownDetected)
qCDebug(dcApplication) << "Ok, ok, I'm done! :)";
s_shutdownCounter++;
NymeaCore::instance()->destroy(reason);
NymeaApplication::quit();
};
@ -144,7 +133,8 @@ static void catchUnixSignals(const std::vector<int>& quitSignals, const std::vec
NymeaApplication::NymeaApplication(int &argc, char **argv) :
QCoreApplication(argc, argv)
{
catchUnixSignals({SIGQUIT, SIGINT, SIGTERM, SIGHUP});
// Catching SIGSEGV messes too much with various tools...
catchUnixSignals({SIGQUIT, SIGINT, SIGTERM, SIGHUP, SIGKILL, /*SIGSEGV,*/ SIGFPE});
}
}

109
tests/auto/api.json
View File

@ -1,4 +1,4 @@
7.1
8.0
{
"enums": {
"BasicType": [
@ -64,30 +64,6 @@
"InputTypeUrl",
"InputTypeMacAddress"
],
"LoggingError": [
"LoggingErrorNoError",
"LoggingErrorLogEntryNotFound",
"LoggingErrorInvalidFilterParameter"
],
"LoggingEventType": [
"LoggingEventTypeTrigger",
"LoggingEventTypeActiveChange",
"LoggingEventTypeEnabledChange",
"LoggingEventTypeActionsExecuted",
"LoggingEventTypeExitActionsExecuted"
],
"LoggingLevel": [
"LoggingLevelInfo",
"LoggingLevelAlert"
],
"LoggingSource": [
"LoggingSourceSystem",
"LoggingSourceEvents",
"LoggingSourceActions",
"LoggingSourceStates",
"LoggingSourceRules",
"LoggingSourceBrowserActions"
],
"MediaBrowserIcon": [
"MediaBrowserIconNone",
"MediaBrowserIconPlaylist",
@ -200,6 +176,17 @@
"RuleErrorNoExitActions",
"RuleErrorInterfaceNotFound"
],
"SampleRate": [
"SampleRateAny",
"SampleRate1Min",
"SampleRate15Mins",
"SampleRate1Hour",
"SampleRate3Hours",
"SampleRate1Day",
"SampleRate1Week",
"SampleRate1Month",
"SampleRate1Year"
],
"ScriptError": [
"ScriptErrorNoError",
"ScriptErrorScriptNotFound",
@ -239,6 +226,10 @@
"SetupMethodUserAndPassword",
"SetupMethodOAuth"
],
"SortOrder": [
"AscendingOrder",
"DescendingOrder"
],
"StateOperator": [
"StateOperatorAnd",
"StateOperatorOr"
@ -1114,6 +1105,18 @@
"thingError": "$ref:ThingError"
}
},
"Integrations.SetActionLogging": {
"description": "Enable/disable logging for the given action type on the given thing.",
"params": {
"actionTypeId": "Uuid",
"enabled": "Bool",
"thingId": "Uuid"
},
"permissionScope": "PermissionScopeConfigureThings",
"returns": {
"thingError": "$ref:ThingError"
}
},
"Integrations.SetEventLogging": {
"description": "Enable/disable logging for the given event type on the given thing.",
"params": {
@ -1286,39 +1289,25 @@
}
},
"Logging.GetLogEntries": {
"description": "Get the LogEntries matching the given filter. The result set will contain entries matching all filter rules combined. If multiple options are given for a single filter type, the result set will contain entries matching any of those. The offset starts at the newest entry in the result set. By default all items are returned. Example: If the specified filter returns a total amount of 100 entries:\n- a offset value of 10 would include the oldest 90 entries\n- a offset value of 0 would return all 100 entries\n\nThe offset is particularly useful in combination with the maxCount property and can be used for pagination. E.g. A result set of 10000 entries can be fetched in batches of 1000 entries by fetching\n1) offset 0, maxCount 1000: Entries 0 to 9999\n2) offset 10000, maxCount 1000: Entries 10000 - 19999\n3) offset 20000, maxCount 1000: Entries 20000 - 29999\n...",
"description": "Get the LogEntries matching the given filter. \n\"sources\": Builtin sources are: \"core\", \"rules\", \"scripts\", \"integrations\". May be extended by experience plugins.\n\"columns\": Columns to be returned.\n\"filter\": A map of column:value entries. Only = is supported currently.\n\"startTime\": The datetime of the oldest entry, in ms.\n\"endTime\": The datetime of the newest entry, in ms.\n\"sampleRate\": If given, returns a sampled series of the values, filling in gaps with the previous value.\n\"sortOrder\": Sort order of results. Note that this impacts the filling of gaps when resampling.\n\"limit\": Maximum amount of entries to be returned.\n\"offset\": Offset to be skipped before returning entries.",
"params": {
"o:eventTypes": [
"$ref:LoggingEventType"
"o:columns": [
"String"
],
"o:endTime": "Uint",
"o:filter": "Variant",
"o:limit": "Int",
"o:loggingLevels": [
"$ref:LoggingLevel"
],
"o:loggingSources": [
"$ref:LoggingSource"
],
"o:offset": "Int",
"o:thingIds": [
"Uuid"
],
"o:timeFilters": [
{
"o:endDate": "Int",
"o:startDate": "Int"
}
],
"o:typeIds": [
"Uuid"
],
"o:values": [
"Variant"
"o:sampleRate": "$ref:SampleRate",
"o:sortOrder": "$ref:SortOrder",
"o:startTime": "Uint",
"sources": [
"String"
]
},
"permissionScope": "PermissionScopeControlThings",
"returns": {
"count": "Int",
"loggingError": "$ref:LoggingError",
"o:logEntries": "$ref:LogEntries",
"offset": "Int"
}
@ -2431,13 +2420,8 @@
"transactionId": "Int"
}
},
"Logging.LogDatabaseUpdated": {
"description": "Emitted whenever the database was updated. The database will be updated when a log entry was deleted. A log entry will be deleted when the corresponding thing or a rule will be removed, or when the oldest entry of the database was deleted to keep to database in the size limits.",
"params": {
}
},
"Logging.LogEntryAdded": {
"description": "Emitted whenever an entry is appended to the logging system. ",
"description": "Emitted when a log entry is added. This will only be emitted for discrete series, not for resampled entries",
"params": {
"logEntry": "$ref:LogEntry"
}
@ -2866,15 +2850,9 @@
"$ref:LogEntry"
],
"LogEntry": {
"r:loggingLevel": "$ref:LoggingLevel",
"r:o:active": "Bool",
"r:o:errorCode": "String",
"r:o:eventType": "$ref:LoggingEventType",
"r:o:thingId": "Uuid",
"r:o:typeId": "Uuid",
"r:o:value": "Variant",
"r:source": "$ref:LoggingSource",
"r:timestamp": "Uint"
"r:source": "String",
"r:timestamp": "Uint",
"r:values": "Object"
},
"ModbusRtuMaster": {
"baudrate": "Uint",
@ -3089,6 +3067,9 @@
"o:name": "String",
"o:settings": "$ref:ParamList",
"r:id": "Uuid",
"r:o:loggedActionTypeIds": [
"Uuid"
],
"r:o:loggedEventTypeIds": [
"Uuid"
],

2
tests/auto/auto.pro
View File

@ -6,8 +6,6 @@ SUBDIRS = \
ioconnections \
jsonrpc \
logging \
loggingdirect \
loggingloading \
macaddress \
mqttbroker \
plugins \

8
tests/auto/integrations/testintegrations.cpp
View File

@ -167,6 +167,7 @@ void TestIntegrations::initTestCase()
"Mock.debug=true\n"
"Translations.debug=true\n"
"PythonIntegrations.debug=true\n"
"LogEngine.debug=true\n"
);
// Adding an async mock to be used in tests below
@ -617,7 +618,7 @@ void TestIntegrations::storedThings()
ThingId addedThingId = ThingId(response.toMap().value("params").toMap().value("thingId").toString());
QVERIFY(!addedThingId.isNull());
clearLoggingDatabase();
clearLoggingDatabase("state-" + addedThingId.toString() + "-int");
// Restart the core instance to check if settings are loaded at startup
restartServer();
@ -640,8 +641,7 @@ void TestIntegrations::storedThings()
waitForDBSync();
params.clear();
params.insert("thingIds", QVariantList() << addedThingId);
params.insert("loggingSources", QVariantList() << "LoggingSourceStates");
params.insert("sources", QStringList{"state-" + addedThingId.toString() + "-int"});
response = injectAndWait("Logging.GetLogEntries", params);
QVERIFY2(response.toMap().value("params").toMap().contains("logEntries"), "Huh? GetLogEntries failed!");
qCDebug(dcTests()) << "log response:" << response.toMap().value("params").toMap().value("logEntries");
@ -683,7 +683,7 @@ void TestIntegrations::stateCache()
connect(reply, &QNetworkReply::finished, reply, &QNetworkReply::deleteLater);
spy.wait();
// For completeness, verify through JSONRPC that they were actually yet.
// For completeness, verify through JSONRPC that they were actually set.
QVariantMap params;
params.insert("thingId", thing->id());

9
tests/auto/jsonrpc/testjsonrpc.cpp
View File

@ -34,6 +34,7 @@
#include "version.h"
#include "servers/mocktcpserver.h"
#include "usermanager/usermanager.h"
#include "logging/logengine.h"
#include "nymeadbusservice.h"
#include "../plugins/mock/extern-plugininfo.h"
@ -153,9 +154,10 @@ void TestJSONRPC::initTestCase()
NymeaDBusService::setBusType(QDBusConnection::SessionBus);
NymeaTestBase::initTestCase("*.debug=false\n"
// "JsonRpcTraffic.debug=true\n"
"JsonRpc.debug=true\n"
"Translations.debug=true\n"
"Tests.debug=true");
"JsonRpc.debug=true\n"
"Translations.debug=true\n"
"Tests.debug=true\n"
"PushButtonAgent.debug=true\n");
}
void TestJSONRPC::cleanup()
@ -1003,6 +1005,7 @@ void TestJSONRPC::testPushButtonAuth()
if (clientSpy.count() == 0) clientSpy.wait();
QVariantMap rsp = checkNotification(clientSpy, "JSONRPC.PushButtonAuthFinished").toMap();
qCDebug(dcTests()) << "rsp" << rsp;
QCOMPARE(rsp.value("params").toMap().value("transactionId").toInt(), transactionId);
QVERIFY2(!rsp.value("params").toMap().value("token").toByteArray().isEmpty(), "Token not in push button auth notification");

623
tests/auto/logging/testlogging.cpp
View File

@ -31,62 +31,39 @@
#include "nymeatestbase.h"
#include "nymeacore.h"
#include "nymeasettings.h"
#include "logging/logvaluetool.h"
#include "logging/logengine.h"
#include "servers/mocktcpserver.h"
#include "../plugins/mock/extern-plugininfo.h"
#include <qglobal.h>
#include "version.h"
using namespace nymeaserver;
class TestLogging : public NymeaTestBase
{
Q_OBJECT
private:
inline void verifyLoggingError(const QVariant &response, Logging::LoggingError error = Logging::LoggingErrorNoError) {
verifyError(response, "loggingError", enumValueName(error));
}
inline void verifyThingError(const QVariant &response, Thing::ThingError error = Thing::ThingErrorNoError) {
verifyError(response, "thingError", enumValueName(error));
}
// DEPRECTATED
inline void verifyDeviceError(const QVariant &response, Thing::ThingError error = Thing::ThingErrorNoError) {
verifyError(response, "deviceError", enumValueName(error).replace("Thing", "Device"));
}
private slots:
void initTestCase();
void init();
void initLogs();
void databaseSerializationTest_data();
void databaseSerializationTest();
void coverageCalls();
void systemLogs();
void invalidFilter_data();
void invalidFilter();
void eventLogs_data();
void eventLogs();
void stateChangeLogs_data();
void stateChangeLogs();
void eventLog();
void actionLog();
void thingLogs();
void testDoubleValues();
void testHouseKeeping();
void testLimits();
// this has to be the last test
void removeThing();
};
@ -108,86 +85,34 @@ void TestLogging::init()
void TestLogging::initLogs()
{
QVariant response = injectAndWait("Logging.GetLogEntries");
verifyLoggingError(response);
QVariantMap params;
params.insert("sources", QStringList{"core"});
QVariant response = injectAndWait("Logging.GetLogEntries", params);
QVariantList logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY2(logEntries.count() > 0,
QString("Expected at least one log entry.")
.toUtf8());
clearLoggingDatabase();
clearLoggingDatabase("core");
waitForDBSync();
response = injectAndWait("Logging.GetLogEntries");
verifyLoggingError(response);
logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY(logEntries.count() == 0);
restartServer();
NymeaCore::instance()->logEngine()->setMaxLogEntries(1000, 10);
}
void TestLogging::databaseSerializationTest_data()
{
QUuid uuid = QUuid("3782732b-61b4-48e8-8d6d-b5205159d7cd");
QVariantMap variantMap;
variantMap.insert("string", "value");
variantMap.insert("int", 5);
variantMap.insert("double", 3.14);
variantMap.insert("uuid", uuid);
QVariantList variantList;
variantList.append(variantMap);
variantList.append("String");
variantList.append(3.14);
variantList.append(uuid);
QTest::addColumn<QVariant>("value");
QTest::newRow("QString") << QVariant(QString("Hello"));
QTest::newRow("Integer") << QVariant((int)2);
QTest::newRow("Double") << QVariant((double)2.34);
QTest::newRow("Float") << QVariant((float)2.34);
QTest::newRow("QColor") << QVariant(QColor(0,255,128));
QTest::newRow("QByteArray") << QVariant(QByteArray("\nthisisatestarray\n"));
QTest::newRow("QUuid") << QVariant(uuid);
QTest::newRow("QVariantMap") << QVariant(variantMap);
QTest::newRow("QVariantList") << QVariant(variantList);
}
void TestLogging::databaseSerializationTest()
{
QFETCH(QVariant, value);
QString serializedValue = LogValueTool::serializeValue(value);
QVariant deserializedValue = LogValueTool::deserializeValue(serializedValue);
qDebug() << "Stored:" << value;
qDebug() << "Loaded:" << deserializedValue;
QCOMPARE(deserializedValue, value);
}
void TestLogging::coverageCalls()
{
LogEntry entry(QDateTime::currentDateTime(), Logging::LoggingLevelInfo, Logging::LoggingSourceSystem);
qDebug() << entry;
LogFilter filter;
qDebug() << filter.queryString() << filter.timeFilters();
}
void TestLogging::systemLogs()
{
qWarning() << "Clearing logging DB";
clearLoggingDatabase();
qCDebug(dcTests()) << "Clearing logging DB";
waitForDBSync();
clearLoggingDatabase("core");
QVariantMap params;
params.insert("loggingSources", QVariantList() << enumValueName(Logging::LoggingSourceSystem));
params.insert("eventTypes", QVariantList() << enumValueName(Logging::LoggingEventTypeActiveChange));
params.insert("sources", QStringList{"core"});
params.insert("sortOrder", enumValueName(Qt::DescendingOrder));
// there should be 0 log entries
QVariant response = injectAndWait("Logging.GetLogEntries", params);
@ -198,14 +123,13 @@ void TestLogging::systemLogs()
.toUtf8());
// check the active system log at boot
qWarning() << "Restarting server";
qCDebug(dcTests) << "Restarting server";
restartServer();
qWarning() << "Restart done";
qCDebug(dcTests) << "Restart done";
waitForDBSync();
// there should be 2 log entries, one for shutdown, one for startup (from server restart)
response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY2(logEntries.count() == 2,
QString("Expected 2 log entries but got:\n%1")
@ -214,44 +138,24 @@ void TestLogging::systemLogs()
QVariantMap logEntryStartup = logEntries.first().toMap();
QVariantMap logEntryShutdown = logEntries.last().toMap();
// We cannot rely on the order those events
if (!logEntryStartup.value("active").toBool()) {
logEntryStartup = logEntries.last().toMap();
logEntryShutdown = logEntries.first().toMap();
}
QCOMPARE(logEntryShutdown.value("active").toBool(), false);
QCOMPARE(logEntryShutdown.value("eventType").toString(), enumValueName(Logging::LoggingEventTypeActiveChange));
QCOMPARE(logEntryShutdown.value("source").toString(), enumValueName(Logging::LoggingSourceSystem));
QCOMPARE(logEntryShutdown.value("loggingLevel").toString(), enumValueName(Logging::LoggingLevelInfo));
QCOMPARE(logEntryShutdown.value("values").toMap().value("event").toString(), QString("stopped"));
QCOMPARE(logEntryShutdown.value("values").toMap().value("shutdownReason").toString(), enumValueName(NymeaCore::ShutdownReasonRestart));
QCOMPARE(logEntryShutdown.value("values").toMap().value("version").toString(), QString(NYMEA_VERSION_STRING));
QCOMPARE(logEntryStartup.value("active").toBool(), true);
QCOMPARE(logEntryStartup.value("eventType").toString(), enumValueName(Logging::LoggingEventTypeActiveChange));
QCOMPARE(logEntryStartup.value("source").toString(), enumValueName(Logging::LoggingSourceSystem));
QCOMPARE(logEntryStartup.value("loggingLevel").toString(), enumValueName(Logging::LoggingLevelInfo));
QCOMPARE(logEntryStartup.value("values").toMap().value("event").toString(), QString("started"));
QCOMPARE(logEntryStartup.value("values").toMap().value("version").toString(), QString(NYMEA_VERSION_STRING));
}
void TestLogging::invalidFilter_data()
{
QVariantMap invalidSourcesFilter;
invalidSourcesFilter.insert("loggingSources", QVariantList() << "bla");
QVariantMap invalidFilterValue;
invalidFilterValue.insert("loggingSource", QVariantList() << "bla");
QVariantMap invalidTypeIds;
invalidTypeIds.insert("typeId", QVariantList() << "bla" << "blub");
QVariantMap invalidEventTypes;
invalidEventTypes.insert("eventTypes", QVariantList() << enumValueName(Logging::LoggingEventTypeTrigger) << "blub");
invalidSourcesFilter.insert("sources", QStringList{"bla"});
QTest::addColumn<QVariantMap>("filter");
QTest::newRow("Invalid source") << invalidSourcesFilter;
QTest::newRow("Invalid filter value") << invalidFilterValue;
QTest::newRow("Invalid typeIds") << invalidTypeIds;
QTest::newRow("Invalid eventTypes") << invalidEventTypes;
}
void TestLogging::invalidFilter()
@ -261,36 +165,42 @@ void TestLogging::invalidFilter()
QVERIFY(!response.isNull());
// verify json error
QVERIFY(response.toMap().value("status").toString() == "error");
QVERIFY(response.toMap().value("status").toString() == "success");
QVERIFY(response.toMap().value("params").toMap().contains("logEntries"));
QVERIFY(response.toMap().value("params").toMap().value("logEntries").toList().isEmpty());
qDebug() << response.toMap().value("error").toString();
}
void TestLogging::eventLogs_data()
void TestLogging::stateChangeLogs_data()
{
QTest::addColumn<StateTypeId>("stateTypeId");
QTest::addColumn<QString>("stateName");
QTest::addColumn<QVariant>("initValue");
QTest::addColumn<QVariant>("newValue");
QTest::addColumn<bool>("expectLogEntry");
QTest::newRow("logged event") << mockConnectedStateTypeId << QVariant(false) << QVariant(true) << true;
QTest::newRow("not logged event") << mockSignalStrengthStateTypeId << QVariant(10) << QVariant(20) << false;
QTest::newRow("logged state") << mockConnectedStateTypeId << "connected" << QVariant(false) << QVariant(true) << true;
QTest::newRow("not logged state") << mockSignalStrengthStateTypeId << "signalStrength" << QVariant(10) << QVariant(20) << false;
}
void TestLogging::eventLogs()
void TestLogging::stateChangeLogs()
{
QFETCH(StateTypeId, stateTypeId);
QFETCH(QString, stateName);
QFETCH(QVariant, initValue);
QFETCH(QVariant, newValue);
QFETCH(bool, expectLogEntry);
QList<Thing*> devices = NymeaCore::instance()->thingManager()->findConfiguredThings(mockThingClassId);
QVERIFY2(devices.count() > 0, "There needs to be at least one configured Mock Device for this test");
Thing *device = devices.first();
QList<Thing*> things = NymeaCore::instance()->thingManager()->findConfiguredThings(mockThingClassId);
QVERIFY2(things.count() > 0, "There needs to be at least one configured Mock Device for this test");
Thing *thing = things.first();
// Setup connection to mock client
QNetworkAccessManager nam;
int port = device->paramValue(mockThingHttpportParamTypeId).toInt();
int port = thing->paramValue(mockThingHttpportParamTypeId).toInt();
QString logSourceName = "state-" + thing->id().toString() + "-" + stateName;
// init state in mock device
QNetworkRequest request(QUrl(QString("http://localhost:%1/setstate?%2=%3").arg(port).arg(stateTypeId.toString()).arg(initValue.toString())));
@ -300,9 +210,12 @@ void TestLogging::eventLogs()
finishedSpy.wait();
}
waitForDBSync();
// Now snoop in for the events
clearLoggingDatabase();
enableNotifications({"Integrations", "Logging"});
clearLoggingDatabase(logSourceName);
enableNotifications({"Integrations"});
QSignalSpy clientSpy(m_mockTcpServer, SIGNAL(outgoingData(QUuid,QByteArray)));
// trigger state change in mock device
@ -318,43 +231,70 @@ void TestLogging::eventLogs()
clientSpy.wait(1000);
reply->deleteLater();
// Make sure the logg notification contains all the stuff we expect
QVariantList logEntryAddedVariants = checkNotifications(clientSpy, "Logging.LogEntryAdded");
qDebug() << "got" << logEntryAddedVariants.count() << "Logging.LogEntryAdded notifications";
QVariantList stateChangeNotification = checkNotifications(clientSpy, "Integrations.StateChanged");
bool found = false;
qDebug() << "got" << logEntryAddedVariants.count() << "Logging.LogEntryAdded";
foreach (const QVariant &logEntryAddedVariant, logEntryAddedVariants) {
QVariantMap logEntry = logEntryAddedVariant.toMap().value("params").toMap().value("logEntry").toMap();
if (logEntry.value("thingId").toUuid() == device->id()) {
found = true;
// Make sure the notification contains all the stuff we expect
QCOMPARE(logEntry.value("typeId").toUuid().toString(), stateTypeId.toString());
QCOMPARE(logEntry.value("eventType").toString(), enumValueName(Logging::LoggingEventTypeTrigger));
QCOMPARE(logEntry.value("source").toString(), enumValueName(Logging::LoggingSourceStates));
QCOMPARE(logEntry.value("loggingLevel").toString(), enumValueName(Logging::LoggingLevelInfo));
break;
}
}
waitForDBSync();
QVERIFY2(found == expectLogEntry, "Could not find the corresponding Logging.LogEntryAdded notification");
QVariant response = injectAndWait("Logging.GetLogEntries", {{"sources", QStringList{logSourceName}}});
QVERIFY(!response.isNull());
QVariantList logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY2(logEntries.count() == (expectLogEntry ? 1 : 0), "Unexpected amount of log entries in DB");
if (expectLogEntry) {
// get this logentry with filter
QVariantMap params;
params.insert("thingIds", QVariantList() << device->id());
params.insert("loggingSources", QVariantList() << enumValueName(Logging::LoggingSourceStates));
params.insert("eventTypes", QVariantList() << enumValueName(Logging::LoggingEventTypeTrigger));
params.insert("typeIds", QVariantList() << stateTypeId);
QVariant response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
QVariantList logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
qCDebug(dcTests()) << qUtf8Printable(QJsonDocument::fromVariant(logEntries).toJson());
QCOMPARE(logEntries.count(), 1);
QVariantMap entry = logEntries.first().toMap();
QVERIFY2(entry.value("values").toMap().value(stateName) == QVariant(newValue), "Log entry value not matching");
}
// disable notifications
QCOMPARE(disableNotifications(), true);
}
void TestLogging::eventLog()
{
QList<Thing*> things = NymeaCore::instance()->thingManager()->findConfiguredThings(mockThingClassId);
QVERIFY2(things.count() > 0, "There needs to be at least one configured Mock Device for this test");
Thing *thing = things.first();
EventTypeId eventTypeId = mockEvent2EventTypeId;
QString eventName = "event2";
QString logSourceName = "event-" + thing->id().toString() + "-" + eventName;
clearLoggingDatabase(logSourceName);
enableNotifications({"Integrations"});
QSignalSpy clientSpy(m_mockTcpServer, SIGNAL(outgoingData(QUuid,QByteArray)));
// trigger state change in mock device
QNetworkAccessManager nam;
int port = thing->paramValue(mockThingHttpportParamTypeId).toInt();
QNetworkRequest request = QNetworkRequest(QUrl(QString("http://localhost:%1/generateevent?eventtypeid=%2&%3=%4").arg(port).arg(eventTypeId.toString()).arg(mockEvent2EventIntParamParamTypeId.toString()).arg(42)));
QNetworkReply *reply = nam.get(request);
{
QSignalSpy finishedSpy(reply, &QNetworkReply::finished);
finishedSpy.wait();
}
clientSpy.wait();
reply->deleteLater();
QVariantList stateChangeNotification = checkNotifications(clientSpy, "Integrations.EventTriggered");
waitForDBSync();
QVariant response = injectAndWait("Logging.GetLogEntries", {{"sources", QStringList{logSourceName}}});
QVERIFY(!response.isNull());
QVariantList logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY2(logEntries.count() == 1, "Unexpected amount of log entries in DB");
QVariantMap entry = logEntries.first().toMap();
QCOMPARE(entry.value("source").toString(), logSourceName);
QByteArray paramsJson = entry.value("values").toMap().value("params").toByteArray();
QJsonParseError error;
QJsonDocument jsonDoc = QJsonDocument::fromJson(paramsJson, &error);
QCOMPARE(error.error, QJsonParseError::NoError);
QCOMPARE(jsonDoc.toVariant().toMap().value("intParam").toInt(), 42);
// disable notifications
QCOMPARE(disableNotifications(), true);
@ -362,8 +302,6 @@ void TestLogging::eventLogs()
void TestLogging::actionLog()
{
clearLoggingDatabase();
QVariantList actionParams;
QVariantMap param1;
param1.insert("paramTypeId", mockWithParamsActionParam1ParamTypeId);
@ -385,7 +323,6 @@ void TestLogging::actionLog()
// EXECUTE with params
QVariant response = injectAndWait("Integrations.ExecuteAction", params);
verifyThingError(response);
// wait for the outgoing data
// 3 packets: ExecuteAction reply, LogDatabaseUpdated signal and LogEntryAdded signal
@ -403,14 +340,15 @@ void TestLogging::actionLog()
bool found = false;
foreach (const QVariant &loggEntryAddedVariant, logEntryAddedVariants) {
QVariantMap logEntry = loggEntryAddedVariant.toMap().value("params").toMap().value("logEntry").toMap();
if (logEntry.value("thingId").toUuid() == m_mockThingId) {
if (logEntry.value("source").toString() == "action-" + m_mockThingId.toString() + "-withParams" ) {
found = true;
// Make sure the notification contains all the stuff we expect
QCOMPARE(logEntry.value("typeId").toUuid().toString(), mockWithParamsActionTypeId.toString());
QCOMPARE(logEntry.value("eventType").toString(), enumValueName(Logging::LoggingEventTypeTrigger));
QCOMPARE(logEntry.value("source").toString(), enumValueName(Logging::LoggingSourceActions));
QCOMPARE(logEntry.value("loggingLevel").toString(), enumValueName(Logging::LoggingLevelInfo));
break;
QCOMPARE(logEntry.value("values").toMap().value("status").toString(), enumValueName(Thing::ThingErrorNoError));
QCOMPARE(logEntry.value("values").toMap().value("triggeredBy").toString(), enumValueName(Action::TriggeredByUser));
QJsonParseError error;
QVariantMap actionParams = QJsonDocument::fromJson(logEntry.value("values").toMap().value("params").toByteArray(), &error).toVariant().toMap();
QCOMPARE(error.error, QJsonParseError::NoError);
QCOMPARE(actionParams.value("param1").toInt(), 7);
QCOMPARE(actionParams.value("param2").toBool(), true);
}
}
@ -421,24 +359,16 @@ void TestLogging::actionLog()
params.insert("actionTypeId", mockWithoutParamsActionTypeId);
params.insert("thingId", m_mockThingId);
response = injectAndWait("Integrations.ExecuteAction", params);
verifyThingError(response);
clientSpy.wait(200);
clientSpy.wait();
logEntryAddedVariants = checkNotifications(clientSpy, "Logging.LogEntryAdded");
QVERIFY(!logEntryAddedVariants.isEmpty());
// get this logentry with filter
params.clear();
params.insert("thingIds", QVariantList() << m_mockThingId);
params.insert("loggingSources", QVariantList() << enumValueName(Logging::LoggingSourceActions));
params.insert("eventTypes", QVariantList() << enumValueName(Logging::LoggingEventTypeTrigger));
// FIXME: currently is filtering for values not supported
//params.insert("values", QVariantList() << "7, true");
params.insert("sources", QStringList{"action-" + m_mockThingId.toString() + "-withoutParams"});
response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
QVariantList logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY2(!logEntries.isEmpty(), "No logs received");
@ -448,9 +378,8 @@ void TestLogging::actionLog()
params.insert("actionTypeId", mockFailingActionTypeId);
params.insert("thingId", m_mockThingId);
response = injectAndWait("Integrations.ExecuteAction", params);
verifyThingError(response, Thing::ThingErrorSetupFailed);
clientSpy.wait(200);
clientSpy.wait();
logEntryAddedVariants = checkNotifications(clientSpy, "Logging.LogEntryAdded");
QVERIFY2(!logEntryAddedVariants.isEmpty(), "Did not get Logging.LogEntryAdded notification.");
@ -458,14 +387,14 @@ void TestLogging::actionLog()
found = false;
foreach (const QVariant &loggEntryAddedVariant, logEntryAddedVariants) {
QVariantMap logEntry = loggEntryAddedVariant.toMap().value("params").toMap().value("logEntry").toMap();
if (logEntry.value("thingId").toUuid() == m_mockThingId) {
if (logEntry.value("source").toString() == "action-" + m_mockThingId.toString() + "-failing") {
found = true;
// Make sure the notification contains all the stuff we expect
QCOMPARE(logEntry.value("typeId").toUuid().toString(), mockFailingActionTypeId.toString());
QCOMPARE(logEntry.value("eventType").toString(), enumValueName(Logging::LoggingEventTypeTrigger));
QCOMPARE(logEntry.value("source").toString(), enumValueName(Logging::LoggingSourceActions));
QCOMPARE(logEntry.value("loggingLevel").toString(), enumValueName(Logging::LoggingLevelAlert));
QCOMPARE(logEntry.value("errorCode").toString(), enumValueName(Thing::ThingErrorSetupFailed));
QCOMPARE(logEntry.value("values").toMap().value("status").toString(), enumValueName(Thing::ThingErrorSetupFailed));
QCOMPARE(logEntry.value("values").toMap().value("triggeredBy").toString(), enumValueName(Action::TriggeredByUser));
QJsonParseError error;
QVariantMap actionParams = QJsonDocument::fromJson(logEntry.value("values").toMap().value("params").toByteArray(), &error).toVariant().toMap();
QCOMPARE(error.error, QJsonParseError::NoError);
QVERIFY(actionParams.isEmpty());
break;
}
}
@ -474,309 +403,73 @@ void TestLogging::actionLog()
// get this logentry with filter
params.clear();
params.insert("thingIds", QVariantList() << m_mockThingId);
params.insert("loggingSources", QVariantList() << enumValueName(Logging::LoggingSourceActions));
params.insert("eventTypes", QVariantList() << enumValueName(Logging::LoggingEventTypeTrigger));
// FIXME: filter for values currently not working
//params.insert("values", QVariantList() << "7, true");
params.insert("sources", QStringList{"action-" + m_mockThingId.toString() + "-failing"});
response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY2(!logEntries.isEmpty(), "No logs received");
// check different filters
// Get all action logs in one go
params.clear();
params.insert("thingIds", QVariantList() << m_mockThingId);
params.insert("loggingSources", QVariantList() << enumValueName(Logging::LoggingSourceActions));
params.insert("eventTypes", QVariantList() << enumValueName(Logging::LoggingEventTypeTrigger));
params.insert("typeIds", QVariantList() << mockWithoutParamsActionTypeId);
params.insert("sources", QStringList{
"action-" + m_mockThingId.toString() + "-withParams",
"action-" + m_mockThingId.toString() + "-withoutParams",
"action-" + m_mockThingId.toString() + "-failing"
});
response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY2(!logEntries.isEmpty(), "No logs received");
params.clear();
params.insert("thingIds", QVariantList() << m_mockThingId);
params.insert("loggingSources", QVariantList() << enumValueName(Logging::LoggingSourceActions));
params.insert("eventTypes", QVariantList() << enumValueName(Logging::LoggingEventTypeTrigger));
params.insert("typeIds", QVariantList() << mockWithoutParamsActionTypeId << mockWithParamsActionTypeId << mockFailingActionTypeId);
response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY2(!logEntries.isEmpty(), "No logs received");
QCOMPARE(logEntries.count(), 3);
// disable notifications
QCOMPARE(disableNotifications(), true);
}
void TestLogging::thingLogs()
{
QVariantMap params;
params.insert("thingClassId", parentMockThingClassId);
params.insert("name", "Parent thing");
QVariant response = injectAndWait("Integrations.AddThing", params);
verifyThingError(response);
ThingId thingId = ThingId(response.toMap().value("params").toMap().value("thingId").toString());
QVERIFY(!thingId.isNull());
// get this logentry with filter
params.clear();
params.insert("thingIds", QVariantList() << m_mockThingId << thingId);
params.insert("loggingSources", QVariantList() << enumValueName(Logging::LoggingSourceActions)
<< enumValueName(Logging::LoggingSourceEvents)
<< enumValueName(Logging::LoggingSourceStates));
params.insert("loggingLevels", QVariantList() << enumValueName(Logging::LoggingLevelInfo)
<< enumValueName(Logging::LoggingLevelAlert));
params.insert("values", QVariantList() << "7, true" << "9, false");
QVariantMap timeFilter;
timeFilter.insert("startDate", QDateTime::currentDateTime().toTime_t() - 5);
timeFilter.insert("endDate", QDateTime::currentDateTime().toTime_t());
QVariantMap timeFilter2;
timeFilter2.insert("endDate", QDateTime::currentDateTime().toTime_t() - 20);
params.insert("timeFilters", QVariantList() << timeFilter << timeFilter2);
response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
}
void TestLogging::testDoubleValues()
{
enableNotifications({"Logging"});
// Add display pin device which contains a double value
// Discover device
QVariantList discoveryParams;
QVariantMap resultCountParam;
resultCountParam.insert("paramTypeId", displayPinMockDiscoveryResultCountParamTypeId);
resultCountParam.insert("value", 1);
discoveryParams.append(resultCountParam);
QVariantMap params;
params.insert("thingClassId", displayPinMockThingClassId);
params.insert("discoveryParams", discoveryParams);
QVariant response = injectAndWait("Integrations.DiscoverThings", params);
verifyThingError(response, Thing::ThingErrorNoError);
// Pair device
ThingDescriptorId descriptorId = ThingDescriptorId(response.toMap().value("params").toMap().value("thingDescriptors").toList().first().toMap().value("id").toString());
params.clear();
params.insert("thingClassId", displayPinMockThingClassId);
params.insert("name", "Display pin mock device");
params.insert("thingDescriptorId", descriptorId.toString());
response = injectAndWait("Integrations.PairThing", params);
verifyThingError(response);
PairingTransactionId pairingTransactionId(response.toMap().value("params").toMap().value("pairingTransactionId").toString());
QString displayMessage = response.toMap().value("params").toMap().value("displayMessage").toString();
qCDebug(dcTests) << "displayMessage" << displayMessage;
params.clear();
params.insert("pairingTransactionId", pairingTransactionId.toString());
params.insert("secret", "243681");
response = injectAndWait("Integrations.ConfirmPairing", params);
verifyThingError(response);
ThingId thingId(response.toMap().value("params").toMap().value("thingId").toString());
QSignalSpy notificationSpy(m_mockTcpServer, SIGNAL(outgoingData(QUuid,QByteArray)));
// Set the double state value and sniff for LogEntryAdded notification
double value = 23.80;
QVariantMap actionParam;
actionParam.insert("paramTypeId", displayPinMockDoubleActionDoubleParamTypeId.toString());
actionParam.insert("value", value);
params.clear(); response.clear();
params.insert("thingId", thingId);
params.insert("actionTypeId", displayPinMockDoubleActionTypeId.toString());
params.insert("params", QVariantList() << actionParam);
response = injectAndWait("Integrations.ExecuteAction", params);
verifyThingError(response);
notificationSpy.wait();
QVariantList logNotificationsList = checkNotifications(notificationSpy, "Logging.LogEntryAdded");
QVERIFY2(!logNotificationsList.isEmpty(), "Did not get Logging.LogEntryAdded notification.");
foreach (const QVariant &logNotificationVariant, logNotificationsList) {
QVariantMap logNotification = logNotificationVariant.toMap().value("params").toMap().value("logEntry").toMap();
if (logNotification.value("typeId").toString() == displayPinMockDoubleActionDoubleParamTypeId.toString()) {
if (logNotification.value("typeId").toString() == displayPinMockDoubleActionDoubleParamTypeId.toString()) {
// If state source
if (logNotification.value("source").toString() == enumValueName(Logging::LoggingSourceStates)) {
QString logValue = logNotification.value("value").toString();
qDebug() << QString::number(value) << logValue;
QCOMPARE(logValue, QString::number(value));
}
// If action source notification
if (logNotification.value("source").toString() == enumValueName(Logging::LoggingSourceActions)) {
QString logValue = logNotification.value("value").toString();
qDebug() << QString::number(value) << logValue;
QCOMPARE(logValue, QString::number(value));
}
}
}
}
// Remove device
params.clear();
params.insert("thingId", thingId.toString());
response = injectAndWait("Integrations.RemoveThing", params);
verifyThingError(response);
}
void TestLogging::testHouseKeeping()
{
QVariantMap params;
params.insert("thingClassId", mockThingClassId);
params.insert("name", "TestDeviceToBeRemoved");
QVariantList thingParams;
QVariantMap httpParam;
httpParam.insert("paramTypeId", mockThingHttpportParamTypeId);
httpParam.insert("value", 6667);
thingParams.append(httpParam);
params.insert("thingParams", thingParams);
QVariant response = injectAndWait("Integrations.AddThing", params);
ThingId thingId = ThingId(response.toMap().value("params").toMap().value("thingId").toUuid());
QVERIFY2(!thingId.isNull(), "Something went wrong creating the thing for testing.");
// Trigger something that creates a logging entry
QNetworkAccessManager nam;
QSignalSpy spy(&nam, SIGNAL(finished(QNetworkReply*)));
QNetworkRequest request(QUrl(QString("http://localhost:%1/setstate?%2=%3").arg(6667).arg(mockConnectedStateTypeId.toString()).arg(false)));
QNetworkReply *reply = nam.get(request);
connect(reply, SIGNAL(finished()), reply, SLOT(deleteLater()));
spy.wait();
waitForDBSync();
params.clear();
params.insert("thingIds", QVariantList() << thingId);
response = injectAndWait("Logging.GetLogEntries", params);
QVERIFY2(response.toMap().value("params").toMap().value("logEntries").toList().count() > 0, "Couldn't find state change event in log...");
// Manually delete this device from config
NymeaSettings settings(NymeaSettings::SettingsRoleThings);
settings.beginGroup("ThingConfig");
settings.remove(thingId.toString());
settings.endGroup();
restartServer();
waitForDBSync();
params.clear();
params.insert("thingIds", QVariantList() << thingId);
response = injectAndWait("Logging.GetLogEntries", params);
qCDebug(dcTests()) << qUtf8Printable(QJsonDocument::fromVariant(response).toJson());
QVERIFY2(response.toMap().value("status").toString() == QString("success"), "GetLogEntries failed");
QVERIFY2(response.toMap().value("params").toMap().value("logEntries").toList().count() == 0, "Device state change event still in log. Should've been cleaned by housekeeping.");
}
void TestLogging::testLimits()
{
clearLoggingDatabase();
for (int i = 0; i < 50; i++) {
QVariantList actionParams;
QVariantMap param1;
param1.insert("paramTypeId", mockWithParamsActionParam1ParamTypeId);
param1.insert("value", i);
actionParams.append(param1);
QVariantMap param2;
param2.insert("paramTypeId", mockWithParamsActionParam2ParamTypeId);
param2.insert("value", true);
actionParams.append(param2);
QVariantMap params;
params.insert("actionTypeId", mockWithParamsActionTypeId);
params.insert("thingId", m_mockThingId);
params.insert("params", actionParams);
// EXECUTE with params
QVariant response = injectAndWait("Integrations.ExecuteAction", params);
verifyThingError(response);
}
waitForDBSync();
QVariantMap params;
QVariantMap response;
// No limits, should be all 50 entries
params.clear();
response = injectAndWait("Logging.GetLogEntries", params).toMap();
QCOMPARE(response.value("params").toMap().value("count").toInt(), 50);
QCOMPARE(response.value("params").toMap().value("logEntries").toList().count(), 50);
// Add a limit of 20
params.clear();
params.insert("limit", 20);
response = injectAndWait("Logging.GetLogEntries", params).toMap();
QCOMPARE(response.value("params").toMap().value("count").toInt(), 20);
QCOMPARE(response.value("params").toMap().value("logEntries").toList().count(), 20);
// Add a offset of 40, keeping a limit of 20. should return 10 entries
params.clear();
params.insert("limit", 20);
params.insert("offset", 40);
response = injectAndWait("Logging.GetLogEntries", params).toMap();
QCOMPARE(response.value("params").toMap().value("count").toInt(), 10);
QCOMPARE(response.value("params").toMap().value("logEntries").toList().count(), 10);
}
void TestLogging::removeThing()
{
QList<Thing*> things = NymeaCore::instance()->thingManager()->findConfiguredThings(mockThingClassId);
QVERIFY2(things.count() > 0, "There needs to be at least one configured Mock Device for this test");
Thing *thing = things.first();
QString stateName = "int";
StateTypeId stateTypeId = mockIntStateTypeId;
qCDebug(dcTests) << "Using mock:" << things.first()->id();
// Setup connection to mock client
QNetworkAccessManager nam;
int port = thing->paramValue(mockThingHttpportParamTypeId).toInt();
QString logSourceName = "state-" + thing->id().toString() + "-" + stateName;
// init state in mock device
QNetworkRequest request(QUrl(QString("http://localhost:%1/setstate?%2=%3").arg(port).arg(stateTypeId.toString()).arg("12")));
QNetworkReply *reply = nam.get(request);
{
QSignalSpy finishedSpy(reply, &QNetworkReply::finished);
finishedSpy.wait();
}
waitForDBSync();
// enable notifications
enableNotifications({"Logging"});
// get this logentry with filter
QVariantMap params;
params.insert("thingIds", QVariantList() << m_mockThingId);
QVariant response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
QVariant response = injectAndWait("Logging.GetLogEntries", {{"sources", QStringList{logSourceName}}});
QVariantList logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QVERIFY(logEntries.count() > 0);
QSignalSpy clientSpy(m_mockTcpServer, SIGNAL(outgoingData(QUuid,QByteArray)));
// Remove the device
params.clear();
params.insert("thingId", m_mockThingId);
response = injectAndWait("Integrations.RemoveThing", params);
verifyThingError(response);
response = injectAndWait("Integrations.RemoveThing", {{"thingId", m_mockThingId}});
clientSpy.wait(200);
QVariant notification = checkNotification(clientSpy, "Logging.LogDatabaseUpdated");
QVERIFY(!notification.isNull());
waitForDBSync();
// verify that the logs from this device where removed from the db
params.clear();
params.insert("thingIds", QVariantList() << m_mockThingId);
response = injectAndWait("Logging.GetLogEntries", params);
verifyLoggingError(response);
response = injectAndWait("Logging.GetLogEntries", {{"sources", QStringList{logSourceName}}});
logEntries = response.toMap().value("params").toMap().value("logEntries").toList();
QCOMPARE(logEntries.count(), 0);

5
tests/auto/loggingdirect/loggingdirect.pro
View File

@ -1,5 +0,0 @@
include(../../../nymea.pri)
include(../autotests.pri)
TARGET = nymeatestloggingdirect
SOURCES += testloggingdirect.cpp

BIN
tests/auto/loggingdirect/testlogging
View File

Binary file not shown.

134
tests/auto/loggingdirect/testloggingdirect.cpp
View File

@ -1,134 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <QtTest>
#include "logging/logengine.h"
using namespace nymeaserver;
class TestLoggingDirect: public QObject
{
Q_OBJECT
public:
TestLoggingDirect(QObject* parent = nullptr);
private slots:
void benchmarkDB_data();
void benchmarkDB();
private:
LogEngine *engine;
};
TestLoggingDirect::TestLoggingDirect(QObject *parent): QObject(parent)
{
engine = new LogEngine("QSQLITE", "/tmp/nymea-test/nymea.sqlite");
// Setting timeout to 20 mins
qputenv("QTEST_FUNCTION_TIMEOUT", "1200000");
QCoreApplication::instance()->setOrganizationName("nymea-test");
}
void TestLoggingDirect::benchmarkDB_data() {
QTest::addColumn<int>("prefill");
QTest::addColumn<int>("maxSize");
QTest::newRow("empty, no trim") << 0 << 20000;
QTest::newRow("empty, trim") << 1 << 1;
QTest::newRow("10000, no trim") << 10000 << 20000;
QTest::newRow("10000, trim") << 10000 << 10000;
// QTest::newRow("20000, no trim") << 20000 << 30000;
// QTest::newRow("20000, trim") << 20000 << 20000;
// QTest::newRow("30000, no trim") << 30000 << 40000;
// QTest::newRow("30000, trim") << 30000 << 30000;
// QTest::newRow("40000, no trim") << 40000 << 50000;
// QTest::newRow("40000, trim") << 40000 << 40000;
// QTest::newRow("50000, no trim") << 50000 << 60000;
// QTest::newRow("50000, trim") << 50000 << 50000;
// QTest::newRow("60000, no trim") << 60000 << 70000;
// QTest::newRow("60000, trim") << 60000 << 60000;
}
void TestLoggingDirect::benchmarkDB()
{
if (qgetenv("WITH_BENCHMARK").isEmpty()) {
QSKIP("Skipping benchmark tests: export WITH_BENCHMARK=1 to enable it.");
}
QFETCH(int, prefill);
QFETCH(int, maxSize);
// setting max log entries to "prefill" to trim it down to what this test needs.
int overflow = 10;
qDebug() << "Flushing DB for test";
engine->setMaxLogEntries(prefill, overflow);
engine->setMaxLogEntries(maxSize, overflow);
LogEntriesFetchJob *job = engine->fetchLogEntries();
QSignalSpy fetchSpy(job, &LogEntriesFetchJob::finished);
fetchSpy.wait();
QList<LogEntry> entries = job->results();
qDebug() << "DB has" << entries.count() << "entries";
qDebug() << "Prefilling DB for test";
for (int i = entries.count(); i < prefill; i++) {
engine->logSystemEvent(QDateTime::currentDateTime(), true);
}
job = engine->fetchLogEntries();
QSignalSpy fetchSpy2(job, &LogEntriesFetchJob::finished);
fetchSpy2.wait();
entries = job->results();
qDebug() << "DB has" << entries.count() << "entries";
qDebug() << "Starting benchmark with" << entries.count() << "entries in the db";
QBENCHMARK {
engine->logSystemEvent(QDateTime::currentDateTime(), true);
}
QDateTime now = QDateTime::currentDateTime();
job = engine->fetchLogEntries();
QSignalSpy fetchSpy3(job, &LogEntriesFetchJob::finished);
fetchSpy3.wait();
entries = job->results();
while (entries.count() > maxSize + overflow) {
qApp->processEvents();
if (now.addSecs(5) < QDateTime::currentDateTime()) {
QVERIFY2(false, QString("Housekeeping didn't work. Have %1 entries but expected to have max %2").arg(entries.count()).arg(QString::number(maxSize)).toLocal8Bit());
}
}
qDebug() << "Ended benchmark with" << entries.count() << "entries in the db";
}
#include "testloggingdirect.moc"
QTEST_MAIN(TestLoggingDirect)

6
tests/auto/loggingloading/loggingloading.pro
View File

@ -1,6 +0,0 @@
include(../../../nymea.pri)
include(../autotests.pri)
RESOURCES += loggingloading.qrc
TARGET = nymeatestloggingloading
SOURCES += testloggingloading.cpp

6
tests/auto/loggingloading/loggingloading.qrc
View File

@ -1,6 +0,0 @@
<!DOCTYPE RCC><RCC version="1.0">
<qresource>
<file>nymead-v2.sqlite</file>
<file>nymead-broken.sqlite</file>
</qresource>
</RCC>

BIN
tests/auto/loggingloading/nymead-broken.sqlite
View File

Binary file not shown.

BIN
tests/auto/loggingloading/nymead-v2.sqlite
View File

Binary file not shown.

93
tests/auto/loggingloading/testloggingloading.cpp
View File

@ -1,93 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2020, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU General Public License as published by the Free Software
* Foundation, GNU version 3. This project is distributed in the hope that it
* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along with
* this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include <QtTest>
#include "logging/logengine.h"
#include "logging/logvaluetool.h"
using namespace nymeaserver;
class TestLoggingLoading: public QObject
{
Q_OBJECT
public:
TestLoggingLoading(QObject* parent = nullptr);
protected slots:
void initTestCase();
private slots:
void testLogfileRotation();
};
TestLoggingLoading::TestLoggingLoading(QObject *parent): QObject(parent)
{
Q_INIT_RESOURCE(loggingloading);
}
void TestLoggingLoading::initTestCase()
{
// Important for settings
QCoreApplication::instance()->setOrganizationName("nymea-test");
}
void TestLoggingLoading::testLogfileRotation()
{
// Create LogEngine with log db from resource file
QString temporaryDbName = "/tmp/nymea-test/nymead-broken.sqlite";
QString rotatedDbName = "/tmp/nymea-test/nymead-broken.sqlite.1";
// Remove the files if there are some left
if (QFile::exists(temporaryDbName))
QVERIFY(QFile(temporaryDbName).remove());
if (QFile::exists(rotatedDbName))
QVERIFY(QFile(rotatedDbName).remove());
// Copy broken log db from resources to default settings path and set permissions
qDebug() << "Copy broken log db to" << temporaryDbName;
QVERIFY(QFile::copy(":/nymead-broken.sqlite", temporaryDbName));
QVERIFY(QFile::setPermissions(temporaryDbName, QFile::ReadOwner | QFile::WriteOwner | QFile::ReadGroup | QFile::ReadOther));
QVERIFY(!QFile::exists(rotatedDbName));
LogEngine *logEngine = new LogEngine("QSQLITE", temporaryDbName);
QVERIFY(QFile::exists(rotatedDbName));
delete logEngine;
QVERIFY(QFile(temporaryDbName).remove());
QVERIFY(QFile(rotatedDbName).remove());
}
#include "testloggingloading.moc"
QTEST_MAIN(TestLoggingLoading)

61
tests/auto/rules/testrules.cpp
View File

@ -33,6 +33,7 @@
#include "servers/mocktcpserver.h"
#include "nymeacore.h"
#include "jsonrpc/jsonhandler.h"
#include "logging/logengine.h"
#include "../plugins/mock/extern-plugininfo.h"
using namespace nymeaserver;
@ -411,7 +412,9 @@ void TestRules::initTestCase()
"RuleEngine.debug=true\n"
"RuleEngineDebug.debug=true\n"
"JsonRpc.debug=true\n"
"Mock.*=true");
"Mock.*=true\n"
"LogEngine.debug=true\n"
);
}
void TestRules::addRemoveRules_data()
@ -2341,6 +2344,11 @@ void TestRules::testStateBasedAction()
QVariant response = injectAndWait("Rules.AddRule", addRuleParams);
verifyRuleError(response);
QString logSourceName = "action-" + m_mockThingId.toString() + "-withParams";
waitForDBSync(); // Waiting for sync before clearning to make sure we also clear any on process entries
clearLoggingDatabase(logSourceName);
waitForDBSync();
// trigger event
spy.clear();
request = QNetworkRequest(QUrl(QString("http://localhost:%1/generateevent?eventtypeid=%2").arg(m_mockThing1Port).arg(mockEvent1EventTypeId.toString())));
@ -2349,15 +2357,32 @@ void TestRules::testStateBasedAction()
QCOMPARE(spy.count(), 1);
reply->deleteLater();
LogFilter filter;
filter.addThingId(m_mockThingId);
filter.addTypeId(mockWithParamsActionTypeId);
// LogFilter filter;
// filter.addThingId(m_mockThingId);
// filter.addTypeId(mockWithParamsActionTypeId);
LogEntriesFetchJob *job = NymeaCore::instance()->logEngine()->fetchLogEntries(filter);
QSignalSpy fetchSpy(job, &LogEntriesFetchJob::finished);
waitForDBSync();
LogFetchJob *job = NymeaCore::instance()->logEngine()->fetchLogEntries({logSourceName});
QSignalSpy fetchSpy(job, &LogFetchJob::finished);
fetchSpy.wait();
QList<LogEntry> entries = job->results();
qCDebug(dcTests()) << "Log entries:" << entries;
QVERIFY(!fetchSpy.isEmpty());
LogEntries entries = job->entries();
qCDebug(dcTests()) << "Got log entries:" << entries.count();
foreach (const LogEntry &entry, job->entries()) {
qCDebug(dcTests()) << entry.source() << entry.timestamp() << entry.values();
}
QCOMPARE(entries.count(), 1);
QCOMPARE(entries.first().values().value("triggeredBy").toString(), enumValueName(Action::TriggeredByRule));
QCOMPARE(entries.first().values().value("status").toString(), enumValueName(Thing::ThingErrorNoError));
QJsonParseError error;
QJsonDocument jsonDoc = QJsonDocument::fromJson(entries.first().values().value("params").toByteArray(), &error);
QCOMPARE(error.error, QJsonParseError::NoError);
QCOMPARE(jsonDoc.toVariant().toMap().value("param1").toInt(), 11);
QCOMPARE(jsonDoc.toVariant().toMap().value("param2").toBool(), true);
QCOMPARE(jsonDoc.toVariant().toMap().value("param3").toString(), QString());
// set bool state to false
spy.clear();
@ -2375,12 +2400,22 @@ void TestRules::testStateBasedAction()
QCOMPARE(spy.count(), 1);
reply->deleteLater();
job = NymeaCore::instance()->logEngine()->fetchLogEntries(filter);
QSignalSpy fetchSpy2(job, &LogEntriesFetchJob::finished);
fetchSpy2.wait();
entries = job->results();
qCDebug(dcTests()) << "Log entries:" << entries;
job = NymeaCore::instance()->logEngine()->fetchLogEntries({logSourceName});
{
QSignalSpy fetchSpy(job, &LogFetchJob::finished);
fetchSpy.wait();
QVERIFY(!fetchSpy.isEmpty());
}
entries = job->entries();
QCOMPARE(entries.count(), 1);
QCOMPARE(entries.first().values().value("triggeredBy").toString(), enumValueName(Action::TriggeredByRule));
QCOMPARE(entries.first().values().value("status").toString(), enumValueName(Thing::ThingErrorNoError));
jsonDoc = QJsonDocument::fromJson(entries.first().values().value("params").toByteArray(), &error);
QCOMPARE(error.error, QJsonParseError::NoError);
QCOMPARE(jsonDoc.toVariant().toMap().value("param1").toInt(), 11);
QCOMPARE(jsonDoc.toVariant().toMap().value("param2").toBool(), true);
QCOMPARE(jsonDoc.toVariant().toMap().value("param3").toString(), QString());
}
void TestRules::removeThingCleansRule()

1
tests/auto/userloading/testuserloading.cpp
View File

@ -31,7 +31,6 @@
#include <QtTest>
#include "logging/logengine.h"
#include "logging/logvaluetool.h"
#include "usermanager/usermanager.h"

14
tests/auto/webserver/testwebserver.cpp
View File

@ -473,20 +473,6 @@ void TestWebserver::getDebugServer_data()
QTest::newRow("POST /debug/styles.css | server disabled | 404") << "post" << "/debug/styles.css" << false << 404;
QTest::newRow("DELETE /debug/styles.css | server disabled | 404") << "delete" << "/debug/styles.css" << false << 404;
// logdb enabled
QTest::newRow("GET /debug/logdb.sql | server enabled | 200") << "get" << "/debug/logdb.sql" << true << 200;
QTest::newRow("OPTIONS /debug/logdb.sql | server enabled | 200") << "options" << "/debug/logdb.sql" << true << 200;
QTest::newRow("PUT /debug/logdb.sql | server enabled | 405") << "put" << "/debug/logdb.sql" << true << 405;
QTest::newRow("POST /debug/logdb.sql | server enabled | 405") << "post" << "/debug/logdb.sql" << true << 405;
QTest::newRow("DELETE /debug/logdb.sql | server enabled | 405") << "delete" << "/debug/logdb.sql" << true << 405;
// logdb disabled
QTest::newRow("GET /debug/logdb.sql | server disabled | 404") << "get" << "/debug/logdb.sql" << false << 404;
QTest::newRow("OPTIONS /debug/logdb.sql | server disabled | 404") << "options" << "/debug/logdb.sql" << false << 404;
QTest::newRow("PUT /debug/logdb.sql | server disabled | 404") << "put" << "/debug/logdb.sql" << false << 404;
QTest::newRow("POST /debug/logdb.sql | server disabled | 404") << "post" << "/debug/logdb.sql" << false << 404;
QTest::newRow("DELETE /debug/logdb.sql | server disabled | 404") << "delete" << "/debug/logdb.sql" << false << 404;
// Check if syslog is accessable
QFileInfo syslogFileInfo("/var/log/syslog");

15
tests/testlib/nymeatestbase.cpp
View File

@ -33,6 +33,7 @@
#include "nymeasettings.h"
#include "servers/mocktcpserver.h"
#include "usermanager/usermanager.h"
#include "logging/logengine.h"
using namespace nymeaserver;
@ -71,6 +72,10 @@ void NymeaTestBase::initTestCase(const QString &loggingRules)
NymeaSettings nymeadSettings(NymeaSettings::SettingsRoleGlobal);
nymeadSettings.clear();
nymeadSettings.beginGroup("Logs");
nymeadSettings.setValue("logDBName", "nymeatest");
nymeadSettings.endGroup();
if (loggingRules.isEmpty()) {
QLoggingCategory::setFilterRules("*.debug=false\nApplication.debug=true\nTests.debug=true\nMock.debug=true");
} else {
@ -122,7 +127,7 @@ void NymeaTestBase::initTestCase(const QString &loggingRules)
void NymeaTestBase::cleanupTestCase()
{
NymeaCore::instance()->destroy();
NymeaCore::instance()->destroy(NymeaCore::ShutdownReasonTerm);
}
void NymeaTestBase::cleanup()
@ -417,6 +422,7 @@ void NymeaTestBase::waitForDBSync()
{
while (NymeaCore::instance()->logEngine()->jobsRunning()) {
qApp->processEvents();
QTest::qWait(500);
}
}
@ -424,7 +430,7 @@ void NymeaTestBase::restartServer()
{
// Destroy and recreate the core instance...
qCDebug(dcTests()) << "Tearing down server instance";
NymeaCore::instance()->destroy();
NymeaCore::instance()->destroy(NymeaCore::ShutdownReasonRestart);
qCDebug(dcTests()) << "Restarting server instance";
NymeaCore::instance()->init();
QSignalSpy coreSpy(NymeaCore::instance(), SIGNAL(initialized()));
@ -435,9 +441,10 @@ void NymeaTestBase::restartServer()
injectAndWait("JSONRPC.Hello");
}
void NymeaTestBase::clearLoggingDatabase()
void NymeaTestBase::clearLoggingDatabase(const QString &source)
{
NymeaCore::instance()->logEngine()->clearDatabase();
NymeaCore::instance()->logEngine()->clear(source);
waitForDBSync();
}
void NymeaTestBase::createMock()

2
tests/testlib/nymeatestbase.h
View File

@ -124,7 +124,7 @@ protected:
void waitForDBSync();
void restartServer();
void clearLoggingDatabase();
void clearLoggingDatabase(const QString &source);
private:
void createMock();