From eeb1feade05e8f82d47bc1eebafb5d05b1c44564 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sat, 2 Jan 2021 14:32:03 +0100 Subject: [PATCH] Add a jitter filtering mechanism --- .../thingmanagerimplementation.cpp | 6 +- libnymea/integrations/pluginmetadata.cpp | 20 ++- .../statevaluefilters/statevaluefilter.cpp | 9 + .../statevaluefilters/statevaluefilter.h | 3 + .../statevaluefilteradaptive.cpp | 161 +++++++++++++++++- .../statevaluefilteradaptive.h | 19 +++ libnymea/types/statetype.cpp | 14 -- libnymea/types/statetype.h | 3 - plugins/mock/extern-plugininfo.h | 7 +- plugins/mock/integrationpluginmock.json | 3 +- plugins/mock/plugininfo.h | 7 +- 11 files changed, 227 insertions(+), 25 deletions(-) diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index 14e7a992..6ec1820d 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -1796,7 +1796,7 @@ void ThingManagerImplementation::slotThingStateValueChanged(const StateTypeId &s Param valueParam(ParamTypeId(stateTypeId.toString()), value); Event event(EventTypeId(stateTypeId.toString()), thing->id(), ParamList() << valueParam, true); - emit eventTriggered(event); + onEventTriggered(event); syncIOConnection(thing, stateTypeId); } @@ -2063,6 +2063,8 @@ void ThingManagerImplementation::loadThingStates(Thing *thing) } else { thing->setStateValue(stateType.id(), stateType.defaultValue()); } + qWarning() << "-----" << stateType.name() << stateType.filter(); + thing->setStateValueFilter(stateType.id(), stateType.filter()); } settings.endGroup(); } @@ -2220,7 +2222,7 @@ IntegrationPlugin *ThingManagerImplementation::createCppIntegrationPlugin(const return nullptr; } - pluginIface->setMetaData(PluginMetadata(pluginInfo)); + pluginIface->setMetaData(metaData); return pluginIface; } diff --git a/libnymea/integrations/pluginmetadata.cpp b/libnymea/integrations/pluginmetadata.cpp index 5e2b1f7d..5bbd997e 100644 --- a/libnymea/integrations/pluginmetadata.cpp +++ b/libnymea/integrations/pluginmetadata.cpp @@ -318,7 +318,11 @@ void PluginMetadata::parse(const QJsonObject &jsonObject) QJsonObject st = stateTypesJson.toObject(); bool writableState = false; - QPair verificationResult = verifyFields(StateType::typeProperties(), StateType::mandatoryTypeProperties(), st); + QStringList stateTypeProperties = {"id", "name", "displayName", "displayNameEvent", "type", "defaultValue", "cached", + "unit", "minValue", "maxValue", "possibleValues", "writable", "displayNameAction", + "ioType", "logged", "filter"}; + QStringList mandatoryStateTypeProperties = {"id", "name", "displayName", "displayNameEvent", "type", "defaultValue"}; + QPair verificationResult = verifyFields(stateTypeProperties, mandatoryStateTypeProperties, st); // Check mandatory fields if (!verificationResult.first.isEmpty()) { @@ -469,7 +473,19 @@ void PluginMetadata::parse(const QJsonObject &jsonObject) break; } stateType.setIOType(ioType); - stateType.setSuggestLogging(st.value("suggestLogging").toBool()); + } + + stateType.setSuggestLogging(st.value("suggestLogging").toBool()); + + if (st.contains("filter")) { + QString filter = st.value("filter").toString(); + if (filter == "adaptive") { + qWarning() << "++++++++++++++++++++++++++++" << stateType.name(); + stateType.setFilter(Types::StateValueFilterAdaptive); + } else if (!filter.isEmpty()) { + m_validationErrors.append("Thing class \"" + thingClass.name() + "\" state type \"" + stateTypeName + "\" has invalid filter value \"" + filter + "\". Supported filters are: \"adaptive\""); + hasError = true; + } } stateTypes.append(stateType); diff --git a/libnymea/integrations/statevaluefilters/statevaluefilter.cpp b/libnymea/integrations/statevaluefilters/statevaluefilter.cpp index 68e88626..c1f37c32 100644 --- a/libnymea/integrations/statevaluefilters/statevaluefilter.cpp +++ b/libnymea/integrations/statevaluefilters/statevaluefilter.cpp @@ -1,6 +1,15 @@ #include "statevaluefilter.h" +#include "loggingcategories.h" + +NYMEA_LOGGING_CATEGORY(dcStateValueFilter, "StateValueFilter") + StateValueFilter::StateValueFilter() { } + +StateValueFilter::~StateValueFilter() +{ + +} diff --git a/libnymea/integrations/statevaluefilters/statevaluefilter.h b/libnymea/integrations/statevaluefilters/statevaluefilter.h index 83b59a88..bd1e2da0 100644 --- a/libnymea/integrations/statevaluefilters/statevaluefilter.h +++ b/libnymea/integrations/statevaluefilters/statevaluefilter.h @@ -2,6 +2,9 @@ #define STATEVALUEFILTER_H #include +#include + +Q_DECLARE_LOGGING_CATEGORY(dcStateValueFilter) class StateValueFilter { diff --git a/libnymea/integrations/statevaluefilters/statevaluefilteradaptive.cpp b/libnymea/integrations/statevaluefilters/statevaluefilteradaptive.cpp index 26f5217b..c2273ffb 100644 --- a/libnymea/integrations/statevaluefilters/statevaluefilteradaptive.cpp +++ b/libnymea/integrations/statevaluefilters/statevaluefilteradaptive.cpp @@ -1,5 +1,7 @@ #include "statevaluefilteradaptive.h" +#include + StateValueFilterAdaptive::StateValueFilterAdaptive() { @@ -7,5 +9,162 @@ StateValueFilterAdaptive::StateValueFilterAdaptive() void StateValueFilterAdaptive::addValue(const QVariant &value) { - + qCDebug(dcStateValueFilter()) << "Adding value:" << value.toDouble(); + m_values.prepend(value.toDouble()); + m_inputValues++; + update(); +} + +QVariant StateValueFilterAdaptive::filteredValue() const +{ + return m_filteredValue; +} + +void StateValueFilterAdaptive::update() +{ + +// while (m_values.count() > m_windowSize + 1) { +// m_values.removeLast(); +// } + +// if (m_values.isEmpty()) { +// m_filteredValue = 0; +// return; +// } + +// if (m_values.count() == 1) { +// m_filteredValue = m_values.first(); +// m_outputValues++; +// return; +// } + + + +//// m_filteredValue = m_values.first(); +//// m_outputValues++; + + +// double currentValue = m_values.first(); +// if (currentValue == 0) { +// m_filteredValue = 0; +// return; +// } + + +// // Calculate average of history, for all values and for all but the last one +// double sum = 0; +// for (int i = 1; i < m_values.count(); i++) { +// sum += m_values.at(i); +// } +// double average = sum / (m_values.count() - 1); + +// double absoluteJitter = currentValue - average; +// double relativeJitter = absoluteJitter / currentValue; + + +// // Outside of jitter window... Forward value directly +// if (qAbs(relativeJitter) > m_averageJitter * 3) { +// m_filteredValue = m_values.first(); +// m_values.clear(); +// m_values.prepend(m_filteredValue); +// m_outputValues++; +// qCDebug(dcStateValueFilter()) << "Updating output"; +// } else { + +// } +// // Adjust average jitter +// m_averageJitter = ((m_averageJitter * m_windowSize) + qAbs(relativeJitter)) / (m_windowSize + 1); + + +// qCDebug(dcStateValueFilter()) << "input" << currentValue << "output" << m_filteredValue << "average" << average << "jitter:" << absoluteJitter << "relative" << relativeJitter << "avg" << m_averageJitter; +// qCDebug(dcStateValueFilter()) << "Filter input values:" << m_inputValues << "output values:" << m_outputValues << "compression ratio:" << (1.0 * m_inputValues / m_outputValues); + + +// return; + + + + + + while (m_values.count() > m_windowSize) { + m_values.removeLast(); + } + + if (m_values.isEmpty()) { + m_filteredValue = 0; + return; + } + + if (m_values.count() == 1) { + // Not enough data + m_filteredValue = m_values.first(); + m_outputValues++; + return; + } + + + // Calculate average of history, for all values and for all but the last one + double sum = 0; + for (int i = 0; i < m_values.count(); i++) { + sum += m_values.at(i); + } + + double currentValue = m_values.first(); + if (qFuzzyCompare(currentValue, 0)) { + m_filteredValue = 0; + return; + } + + double filteredValue = sum / m_values.count(); + double previousFilteredValue = (sum - m_values.first()) / (m_values.count() - 1); + + if (qFuzzyCompare(previousFilteredValue, 0)) { + m_filteredValue = m_values.first(); + m_outputValues++; + return; + } + + // Calculate change ratio of the last value compared to the previous one, unflitered and filtered + double changeRatio = 1 - qAbs(currentValue / previousFilteredValue); + double changeRatioFiltered = 1 - qAbs(filteredValue / previousFilteredValue); + + // Add deviation of actual value vs filtered value up to have an idea how much we're off + m_totalDeviation += changeRatioFiltered - changeRatio; + + + // If the unfiltered value changes for more than 3 times the standard deviation of the jittering values + // it's a 99% chance a big change happened that's not jitter (e.g turned on/off) + // Discard the history and follow the new value right away + if (qAbs(changeRatio) > m_standardDeviation * 3) { + m_values.clear(); + m_values.prepend(currentValue); + m_totalDeviation = 0; + if (!qFuzzyCompare(m_filteredValue, filteredValue)) { + m_filteredValue = currentValue; + qCDebug(dcStateValueFilter()) << "Updating output value:" << m_filteredValue << "(input exceeds max jitter)"; + m_outputValues++; + } + + // If the filtered value changed for 5 percent or more, follow slowly + // In order to not get stuck on being off for 5% forever, also move closer + // to the new value when the deviation exceeds max deviation + } else if (qAbs(changeRatioFiltered) > m_standardDeviation || qAbs(m_totalDeviation) > m_maxTotalDeviation) { + m_totalDeviation = 0; + if (!qFuzzyCompare(m_filteredValue, filteredValue)) { + qCDebug(dcStateValueFilter()) << "Updating output value:" << filteredValue << "(drift compensation)"; + m_filteredValue = filteredValue; + m_outputValues++; + } + } + + // Poor mans solution to calculate standard deviation. Not as precise, but much faster than looping over history again + m_standardDeviation = ((m_standardDeviation * m_windowSize) + qAbs(changeRatio)) / (m_windowSize + 1); + qWarning(dcStateValueFilter()) << "New:" << currentValue << "Old:" << previousFilteredValue << "Filtered:" << filteredValue << "ratio:" << changeRatio << "filteredRatio" << changeRatioFiltered << "deviation" << m_totalDeviation << "averageJitter" << m_averageJitter; + + // correct stats on overflow of counters + if (m_inputValues < m_outputValues) { + m_outputValues = 0; + } + + qCDebug(dcStateValueFilter()) << "Filter input values:" << m_inputValues << "output values:" << m_outputValues << "compression ratio:" << (1.0 * m_inputValues / m_outputValues); } diff --git a/libnymea/integrations/statevaluefilters/statevaluefilteradaptive.h b/libnymea/integrations/statevaluefilters/statevaluefilteradaptive.h index dbb85b84..c9ce5b6e 100644 --- a/libnymea/integrations/statevaluefilters/statevaluefilteradaptive.h +++ b/libnymea/integrations/statevaluefilters/statevaluefilteradaptive.h @@ -10,6 +10,25 @@ public: void addValue(const QVariant &value) override; QVariant filteredValue() const override; + +private: + void update(); + +private: + QList m_values; + + int m_windowSize = 20; + double m_standardDeviation = 0.05; + double m_maxTotalDeviation = 1; + + double m_filteredValue = 0; + double m_totalDeviation = 0; + + // Stats for debugging + quint64 m_inputValues = 0; + quint64 m_outputValues = 0; + + }; #endif // STATEVALUEFILTERADAPTIVE_H diff --git a/libnymea/types/statetype.cpp b/libnymea/types/statetype.cpp index 74b70ad5..997e7592 100644 --- a/libnymea/types/statetype.cpp +++ b/libnymea/types/statetype.cpp @@ -228,20 +228,6 @@ void StateType::setFilter(Types::StateValueFilter filter) m_filter = filter; } -/*! Returns a list of all valid properties a DeviceClass definition can have. */ -QStringList StateType::typeProperties() -{ - return QStringList() << "id" << "name" << "displayName" << "displayNameEvent" << "type" << "defaultValue" - << "cached" << "unit" << "minValue" << "maxValue" << "possibleValues" << "writable" - << "displayNameAction" << "ioType" << "logged"; -} - -/*! Returns a list of mandatory properties a DeviceClass definition must have. */ -QStringList StateType::mandatoryTypeProperties() -{ - return QStringList() << "id" << "name" << "displayName" << "displayNameEvent" << "type" << "defaultValue"; -} - /*! Returns true if this state type has an ID, a type and a name set. */ bool StateType::isValid() const { diff --git a/libnymea/types/statetype.h b/libnymea/types/statetype.h index b1504e55..aff27735 100644 --- a/libnymea/types/statetype.h +++ b/libnymea/types/statetype.h @@ -99,9 +99,6 @@ public: Types::StateValueFilter filter() const; void setFilter(Types::StateValueFilter filter); - static QStringList typeProperties(); - static QStringList mandatoryTypeProperties(); - bool isValid() const; private: diff --git a/plugins/mock/extern-plugininfo.h b/plugins/mock/extern-plugininfo.h index b7ba912a..e31016e9 100644 --- a/plugins/mock/extern-plugininfo.h +++ b/plugins/mock/extern-plugininfo.h @@ -1,5 +1,10 @@ /* This file is generated by the nymea build system. Any changes to this file will * - * be lost. If you want to change this file, edit the plugin's json file. */ + * be lost. If you want to change this file, edit the plugin's json file. * + * * + * NOTE: This file can be included only once per plugin. If you need to access * + * definitions from this file in multiple source files, use * + * #include extern-plugininfo.h * + * instead and re-run qmake. */ #ifndef EXTERNPLUGININFO_H #define EXTERNPLUGININFO_H diff --git a/plugins/mock/integrationpluginmock.json b/plugins/mock/integrationpluginmock.json index c3afe9e9..9bf6fd6c 100644 --- a/plugins/mock/integrationpluginmock.json +++ b/plugins/mock/integrationpluginmock.json @@ -152,7 +152,8 @@ "minValue": 0, "maxValue": 100, "defaultValue": 50, - "writable": true + "writable": true, + "filter": "adaptive" }, { "id": "ebc41327-53d5-40c2-8e7b-1164a8ff359e", diff --git a/plugins/mock/plugininfo.h b/plugins/mock/plugininfo.h index 51ee9c17..e7b3de50 100644 --- a/plugins/mock/plugininfo.h +++ b/plugins/mock/plugininfo.h @@ -1,5 +1,10 @@ /* This file is generated by the nymea build system. Any changes to this file will * - * be lost. If you want to change this file, edit the plugin's json file. */ + * be lost. If you want to change this file, edit the plugin's json file. * + * * + * NOTE: This file can be included only once per plugin. If you need to access * + * definitions from this file in multiple source files, use * + * #include extern-plugininfo.h * + * instead and re-run qmake. */ #ifndef PLUGININFO_H #define PLUGININFO_H