Add a jitter filtering mechanism

libnymea-core/integrations/thingmanagerimplementation.cpp

@@ -1796,7 +1796,7 @@ void ThingManagerImplementation::slotThingStateValueChanged(const StateTypeId &s
 
     Param valueParam(ParamTypeId(stateTypeId.toString()), value);
     Event event(EventTypeId(stateTypeId.toString()), thing->id(), ParamList() << valueParam, true);
-    emit eventTriggered(event);
+    onEventTriggered(event);
 
     syncIOConnection(thing, stateTypeId);
 }
@@ -2063,6 +2063,8 @@ void ThingManagerImplementation::loadThingStates(Thing *thing)
         } else {
             thing->setStateValue(stateType.id(), stateType.defaultValue());
         }
+        qWarning() << "-----" << stateType.name() <<  stateType.filter();
+        thing->setStateValueFilter(stateType.id(), stateType.filter());
     }
     settings.endGroup();
 }
@@ -2220,7 +2222,7 @@ IntegrationPlugin *ThingManagerImplementation::createCppIntegrationPlugin(const
         return nullptr;
     }
 
-    pluginIface->setMetaData(PluginMetadata(pluginInfo));
+    pluginIface->setMetaData(metaData);
 
     return pluginIface;
 }

libnymea/integrations/pluginmetadata.cpp

@@ -318,7 +318,11 @@ void PluginMetadata::parse(const QJsonObject &jsonObject)
                 QJsonObject st = stateTypesJson.toObject();
                 bool writableState = false;
 
-                QPair<QStringList, QStringList> verificationResult = verifyFields(StateType::typeProperties(), StateType::mandatoryTypeProperties(), st);
+                QStringList stateTypeProperties = {"id", "name", "displayName", "displayNameEvent", "type", "defaultValue", "cached",
+                                                   "unit", "minValue", "maxValue", "possibleValues", "writable", "displayNameAction",
+                                                   "ioType", "logged", "filter"};
+                QStringList mandatoryStateTypeProperties = {"id", "name", "displayName", "displayNameEvent", "type", "defaultValue"};
+                QPair<QStringList, QStringList> verificationResult = verifyFields(stateTypeProperties, mandatoryStateTypeProperties, st);
 
                 // Check mandatory fields
                 if (!verificationResult.first.isEmpty()) {
@@ -469,7 +473,19 @@ void PluginMetadata::parse(const QJsonObject &jsonObject)
                         break;
                     }
                     stateType.setIOType(ioType);
-                    stateType.setSuggestLogging(st.value("suggestLogging").toBool());
+                }
+
+                stateType.setSuggestLogging(st.value("suggestLogging").toBool());
+
+                if (st.contains("filter")) {
+                    QString filter = st.value("filter").toString();
+                    if (filter == "adaptive") {
+                        qWarning() << "++++++++++++++++++++++++++++" << stateType.name();
+                        stateType.setFilter(Types::StateValueFilterAdaptive);
+                    } else if (!filter.isEmpty()) {
+                        m_validationErrors.append("Thing class \"" + thingClass.name() + "\" state type \"" + stateTypeName + "\" has invalid filter value \"" + filter + "\". Supported filters are: \"adaptive\"");
+                        hasError = true;
+                    }
                 }
                 stateTypes.append(stateType);
 

libnymea/integrations/statevaluefilters/statevaluefilter.cpp

@@ -1,6 +1,15 @@
 #include "statevaluefilter.h"
 
+#include "loggingcategories.h"
+
+NYMEA_LOGGING_CATEGORY(dcStateValueFilter, "StateValueFilter")
+
 StateValueFilter::StateValueFilter()
 {
 
 }
+
+StateValueFilter::~StateValueFilter()
+{
+
+}

libnymea/integrations/statevaluefilters/statevaluefilter.h

@@ -2,6 +2,9 @@
 #define STATEVALUEFILTER_H
 
 #include <QVariant>
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(dcStateValueFilter)
 
 class StateValueFilter
 {

libnymea/integrations/statevaluefilters/statevaluefilteradaptive.cpp

@@ -1,5 +1,7 @@
 #include "statevaluefilteradaptive.h"
 
+#include <qmath.h>
+
 StateValueFilterAdaptive::StateValueFilterAdaptive()
 {
 
@@ -7,5 +9,162 @@ StateValueFilterAdaptive::StateValueFilterAdaptive()
 
 void StateValueFilterAdaptive::addValue(const QVariant &value)
 {
-
+    qCDebug(dcStateValueFilter()) << "Adding value:" << value.toDouble();
+    m_values.prepend(value.toDouble());
+    m_inputValues++;
+    update();
+}
+
+QVariant StateValueFilterAdaptive::filteredValue() const
+{
+    return m_filteredValue;
+}
+
+void StateValueFilterAdaptive::update()
+{
+
+//    while (m_values.count() > m_windowSize + 1) {
+//        m_values.removeLast();
+//    }
+
+//    if (m_values.isEmpty()) {
+//        m_filteredValue = 0;
+//        return;
+//    }
+
+//    if (m_values.count() == 1) {
+//        m_filteredValue = m_values.first();
+//        m_outputValues++;
+//        return;
+//    }
+
+
+
+////    m_filteredValue = m_values.first();
+////    m_outputValues++;
+
+
+//    double currentValue = m_values.first();
+//    if (currentValue == 0) {
+//        m_filteredValue = 0;
+//        return;
+//    }
+
+
+//    // Calculate average of history, for all values and for all but the last one
+//    double sum = 0;
+//    for (int i = 1; i < m_values.count(); i++) {
+//        sum += m_values.at(i);
+//    }
+//    double average = sum / (m_values.count() - 1);
+
+//    double absoluteJitter = currentValue - average;
+//    double relativeJitter = absoluteJitter / currentValue;
+
+
+//    // Outside of jitter window... Forward value directly
+//    if (qAbs(relativeJitter) > m_averageJitter * 3) {
+//        m_filteredValue = m_values.first();
+//        m_values.clear();
+//        m_values.prepend(m_filteredValue);
+//        m_outputValues++;
+//        qCDebug(dcStateValueFilter()) << "Updating output";
+//    } else {
+
+//    }
+//    // Adjust average jitter
+//    m_averageJitter = ((m_averageJitter * m_windowSize) + qAbs(relativeJitter)) / (m_windowSize + 1);
+
+
+//    qCDebug(dcStateValueFilter()) << "input" << currentValue << "output" << m_filteredValue << "average" << average << "jitter:" << absoluteJitter << "relative" << relativeJitter << "avg" << m_averageJitter;
+//    qCDebug(dcStateValueFilter()) << "Filter input values:" << m_inputValues << "output values:" << m_outputValues << "compression ratio:" << (1.0 * m_inputValues / m_outputValues);
+
+
+//    return;
+
+
+
+
+
+    while (m_values.count() > m_windowSize) {
+        m_values.removeLast();
+    }
+
+    if (m_values.isEmpty()) {
+        m_filteredValue = 0;
+        return;
+    }
+
+    if (m_values.count() == 1) {
+        // Not enough data
+        m_filteredValue = m_values.first();
+        m_outputValues++;
+        return;
+    }
+
+
+    // Calculate average of history, for all values and for all but the last one
+    double sum = 0;
+    for (int i = 0; i < m_values.count(); i++) {
+        sum += m_values.at(i);
+    }
+
+    double currentValue = m_values.first();
+    if (qFuzzyCompare(currentValue, 0)) {
+        m_filteredValue = 0;
+        return;
+    }
+
+    double filteredValue = sum / m_values.count();
+    double previousFilteredValue = (sum - m_values.first()) / (m_values.count() - 1);
+
+    if (qFuzzyCompare(previousFilteredValue, 0)) {
+        m_filteredValue = m_values.first();
+        m_outputValues++;
+        return;
+    }
+
+    // Calculate change ratio of the last value compared to the previous one, unflitered and filtered
+    double changeRatio = 1 - qAbs(currentValue / previousFilteredValue);
+    double changeRatioFiltered = 1 - qAbs(filteredValue / previousFilteredValue);
+
+    // Add deviation of actual value vs filtered value up to have an idea how much we're off
+    m_totalDeviation += changeRatioFiltered - changeRatio;
+
+
+    // If the unfiltered value changes for more than 3 times the standard deviation of the jittering values
+    // it's a 99% chance a big change happened that's not jitter (e.g turned on/off)
+    // Discard the history and follow the new value right away
+    if (qAbs(changeRatio) > m_standardDeviation * 3) {
+        m_values.clear();
+        m_values.prepend(currentValue);
+        m_totalDeviation = 0;
+        if (!qFuzzyCompare(m_filteredValue, filteredValue)) {
+            m_filteredValue = currentValue;
+            qCDebug(dcStateValueFilter()) << "Updating output value:" << m_filteredValue << "(input exceeds max jitter)";
+            m_outputValues++;
+        }
+
+    // If the filtered value changed for 5 percent or more, follow slowly
+    // In order to not get stuck on being off for 5% forever, also move closer
+    // to the new value when the deviation exceeds max deviation
+    } else if (qAbs(changeRatioFiltered) > m_standardDeviation || qAbs(m_totalDeviation) > m_maxTotalDeviation) {
+        m_totalDeviation = 0;
+        if (!qFuzzyCompare(m_filteredValue, filteredValue)) {
+            qCDebug(dcStateValueFilter()) << "Updating output value:" << filteredValue << "(drift compensation)";
+            m_filteredValue = filteredValue;
+            m_outputValues++;
+        }
+    }
+
+    // Poor mans solution to calculate standard deviation. Not as precise, but much faster than looping over history again
+    m_standardDeviation = ((m_standardDeviation * m_windowSize) + qAbs(changeRatio)) / (m_windowSize + 1);
+    qWarning(dcStateValueFilter()) << "New:" << currentValue << "Old:" << previousFilteredValue << "Filtered:" << filteredValue << "ratio:" << changeRatio << "filteredRatio" << changeRatioFiltered << "deviation" << m_totalDeviation << "averageJitter" << m_averageJitter;
+
+    // correct stats on overflow of counters
+    if (m_inputValues < m_outputValues) {
+        m_outputValues = 0;
+    }
+
+    qCDebug(dcStateValueFilter()) << "Filter input values:" << m_inputValues << "output values:" << m_outputValues << "compression ratio:" << (1.0 * m_inputValues / m_outputValues);
 }

libnymea/integrations/statevaluefilters/statevaluefilteradaptive.h

@@ -10,6 +10,25 @@ public:
 
     void addValue(const QVariant &value) override;
     QVariant filteredValue() const override;
+
+private:
+    void update();
+
+private:
+    QList<double> m_values;
+
+    int m_windowSize = 20;
+    double m_standardDeviation = 0.05;
+    double m_maxTotalDeviation = 1;
+
+    double m_filteredValue = 0;
+    double m_totalDeviation = 0;
+
+    // Stats for debugging
+    quint64 m_inputValues = 0;
+    quint64 m_outputValues = 0;
+
+
 };
 
 #endif // STATEVALUEFILTERADAPTIVE_H

libnymea/types/statetype.cpp

@@ -228,20 +228,6 @@ void StateType::setFilter(Types::StateValueFilter filter)
     m_filter = filter;
 }
 
-/*! Returns a list of all valid properties a DeviceClass definition can have. */
-QStringList StateType::typeProperties()
-{
-    return QStringList() << "id" << "name" << "displayName" << "displayNameEvent" << "type" << "defaultValue"
-                         << "cached" << "unit" << "minValue" << "maxValue" << "possibleValues" << "writable"
-                         << "displayNameAction" << "ioType" << "logged";
-}
-
-/*! Returns a list of mandatory properties a DeviceClass definition must have. */
-QStringList StateType::mandatoryTypeProperties()
-{
-    return QStringList() << "id" << "name" << "displayName" << "displayNameEvent" << "type" << "defaultValue";
-}
-
 /*! Returns true if this state type has an ID, a type and a name set. */
 bool StateType::isValid() const
 {

libnymea/types/statetype.h

@@ -99,9 +99,6 @@ public:
     Types::StateValueFilter filter() const;
     void setFilter(Types::StateValueFilter filter);
 
-    static QStringList typeProperties();
-    static QStringList mandatoryTypeProperties();
-
     bool isValid() const;
 
 private:

plugins/mock/extern-plugininfo.h

@@ -1,5 +1,10 @@
 /* This file is generated by the nymea build system. Any changes to this file will *
- * be lost. If you want to change this file, edit the plugin's json file.          */
+ * be lost. If you want to change this file, edit the plugin's json file.          *
+ *                                                                                 *
+ * NOTE: This file can be included only once per plugin. If you need to access     *
+ * definitions from this file in multiple source files, use                        *
+ * #include extern-plugininfo.h                                                    *
+ * instead and re-run qmake.                                                       */
 
 #ifndef EXTERNPLUGININFO_H
 #define EXTERNPLUGININFO_H

plugins/mock/integrationpluginmock.json

@@ -152,7 +152,8 @@
                             "minValue": 0,
                             "maxValue": 100,
                             "defaultValue": 50,
-                            "writable": true
+                            "writable": true,
+                            "filter": "adaptive"
                         },
                         {
                             "id": "ebc41327-53d5-40c2-8e7b-1164a8ff359e",

plugins/mock/plugininfo.h

@@ -1,5 +1,10 @@
 /* This file is generated by the nymea build system. Any changes to this file will *
- * be lost. If you want to change this file, edit the plugin's json file.          */
+ * be lost. If you want to change this file, edit the plugin's json file.          *
+ *                                                                                 *
+ * NOTE: This file can be included only once per plugin. If you need to access     *
+ * definitions from this file in multiple source files, use                        *
+ * #include extern-plugininfo.h                                                    *
+ * instead and re-run qmake.                                                       */
 
 #ifndef PLUGININFO_H
 #define PLUGININFO_H