Fix energy stats lookup and deduplicate fetched log samples

libnymea-app/energy/energylogs.cpp

@@ -216,38 +216,42 @@ int EnergyLogs::indexOf(const QDateTime &timestamp)
     if (m_list.isEmpty()) {
         return -1;
     }
-    QDateTime first = m_list.first()->timestamp();
 
-    int index = qRound(1.0 * first.secsTo(timestamp) / (m_sampleRate * 60));
+    const qint64 target = timestamp.toMSecsSinceEpoch();
-    if (index < 0 || index >= m_list.count()) {
+    const qint64 firstTimestamp = m_list.first()->timestamp().toMSecsSinceEpoch();
-        qCDebug(dcEnergyLogs()) << "finding:" << timestamp << index << first.toString() << "NOT FOUND" << m_list.last()->timestamp() << m_list.count();
+    const qint64 lastTimestamp = m_list.last()->timestamp().toMSecsSinceEpoch();
+    if (target < firstTimestamp || target > lastTimestamp) {
         return -1;
     }
-    qCDebug(dcEnergyLogs()) << "finding:" << timestamp << index << first.toString() << m_list.at(index)->timestamp();
 
+    int low = 0;
+    int high = m_list.count() - 1;
 
-    // Normally, if the DB is in a consistent state, we can rely that the above finds the correct entry.
+    // Use timestamp-based lookup instead of sample-rate math. This is robust against
-    // However, if the user changes the timezone, during the lifetime, or other woes may appear like NTP
+    // duplicate/missing rows (e.g. after resampling glitches or timezone jumps).
-    // changing time which may cause inconsistent entries like passing the same time twice, we could end up
+    while (low <= high) {
-    // off by one. In order to compensate for that, we'll see if the next or previous entries may be closer
+        const int mid = low + (high - low) / 2;
-    // In theory we could even be off by some more samples in very rare circumstances, but unlikely enough
+        const qint64 midTimestamp = m_list.at(mid)->timestamp().toMSecsSinceEpoch();
-    // to not bother with that at this point.
+        if (midTimestamp < target) {
-    QDateTime found = m_list.at(index)->timestamp();
+            low = mid + 1;
-    QDateTime previous = index > 0 ? m_list.at(index-1)->timestamp() : found;
+        } else if (midTimestamp > target) {
-    QDateTime next = index < m_list.count() - 1 ? m_list.at(index+1)->timestamp() : found;
+            high = mid - 1;
-
+        } else {
-    int diffToFound = qAbs(timestamp.secsTo(found));
+            return mid;
-    int diffToPrevious = qAbs(timestamp.secsTo(previous));
+        }
-    int diffToNext = qAbs(timestamp.secsTo(next));
-    if (diffToPrevious < diffToFound && diffToPrevious < diffToNext) {
-//        qWarning() << "Correcting to previous" << index << m_list.count() << found << previous << diffToPrevious << diffToFound;
-        return index - 1;
     }
-    if (diffToNext < diffToFound) {
+
-//        qWarning() << "Correcting to next" << index << m_list.count() << found << next << diffToNext << diffToFound;
+    const int previousIndex = low - 1;
-        return index + 1;
+    const int nextIndex = low;
+    const qint64 previousTimestamp = m_list.at(previousIndex)->timestamp().toMSecsSinceEpoch();
+    const qint64 nextTimestamp = m_list.at(nextIndex)->timestamp().toMSecsSinceEpoch();
+    const qint64 diffToPrevious = qAbs(target - previousTimestamp);
+    const qint64 diffToNext = qAbs(target - nextTimestamp);
+
+    if (diffToPrevious <= diffToNext) {
+        return previousIndex;
     }
-    return index;
+    return nextIndex;
 }
 
 EnergyLogEntry *EnergyLogs::find(const QDateTime &timestamp)

libnymea-app/energy/powerbalancelogs.cpp

@@ -24,6 +24,7 @@
 
 #include "powerbalancelogs.h"
 
+#include <QMap>
 #include <QMetaEnum>
 
 PowerBalanceLogEntry::PowerBalanceLogEntry(QObject *parent): EnergyLogEntry(parent)
@@ -98,8 +99,15 @@ QString PowerBalanceLogs::logsName() const
 QList<EnergyLogEntry *> PowerBalanceLogs::unpackEntries(const QVariantMap &params, double *minValue, double *maxValue)
 {
     QList<EnergyLogEntry*> ret;
+    QMap<qint64, QVariantMap> deduplicatedEntries;
     foreach (const QVariant &variant, params.value("powerBalanceLogEntries").toList()) {
         QVariantMap map = variant.toMap();
+        // Keep the last row for a timestamp if the backend returned duplicates.
+        deduplicatedEntries.insert(map.value("timestamp").toLongLong(), map);
+    }
+
+    for (auto it = deduplicatedEntries.constBegin(); it != deduplicatedEntries.constEnd(); ++it) {
+        const QVariantMap &map = it.value();
         QDateTime timestamp = QDateTime::fromSecsSinceEpoch(map.value("timestamp").toLongLong());
         double consumption = map.value("consumption").toDouble();
         double production = map.value("production").toDouble();

libnymea-app/energy/thingpowerlogs.cpp

@@ -24,6 +24,8 @@
 
 #include "thingpowerlogs.h"
 
+#include <limits>
+#include <QMap>
 #include <QMetaEnum>
 
 #include <QLoggingCategory>
@@ -142,25 +144,37 @@ QVariantMap ThingPowerLogs::fetchParams() const
 
 QList<EnergyLogEntry *> ThingPowerLogs::unpackEntries(const QVariantMap &params, double *minValue, double *maxValue)
 {
+    qint64 newestCurrentTimestamp = std::numeric_limits<qint64>::min();
     foreach (const QVariant &variant, params.value("currentEntries").toList()) {
         QVariantMap map = variant.toMap();
         if (map.value("thingId").toUuid() != m_thingId) {
             continue;
         }
+        const qint64 timestamp = map.value("timestamp").toLongLong();
+        if (timestamp < newestCurrentTimestamp) {
+            continue;
+        }
+        newestCurrentTimestamp = timestamp;
         if (m_liveEntry) {
             m_liveEntry->deleteLater();
         }
         m_liveEntry = unpack(map);
         emit liveEntryChanged(m_liveEntry);
-        break;
     }
 
     QList<EnergyLogEntry*> ret;
+    QMap<qint64, QVariantMap> deduplicatedEntries;
     foreach (const QVariant &variant, params.value("thingPowerLogEntries").toList()) {
         QVariantMap map = variant.toMap();
         if (map.value("thingId").toUuid() != m_thingId) {
             continue;
         }
+        // Keep the last row for a timestamp if the backend returned duplicates.
+        deduplicatedEntries.insert(map.value("timestamp").toLongLong(), map);
+    }
+
+    for (auto it = deduplicatedEntries.constBegin(); it != deduplicatedEntries.constEnd(); ++it) {
+        const QVariantMap &map = it.value();
         QDateTime timestamp = QDateTime::fromSecsSinceEpoch(map.value("timestamp").toLongLong());
         QUuid thingId = map.value("thingId").toUuid();
         double currentPower = map.value("currentPower").toDouble();

nymea-app/ui/mainviews/energy/ConsumerStats.qml

@@ -80,6 +80,14 @@ StatsBase {
                 d.selectedThing = thing
             }
         }
+
+        function sampleMatches(entry, expectedTimestamp) {
+            if (!entry) {
+                return false
+            }
+            var maxDistance = d.config.sampleRate * 60 * 1000 / 2
+            return Math.abs(entry.timestamp.getTime() - expectedTimestamp.getTime()) <= maxDistance
+        }
     }
 
     ThingPowerLogsLoader {
@@ -133,22 +141,18 @@ StatsBase {
 //                    print("timestamp:", timestamp, "previous:", previousTimestamp)
                     var entry = thingPowerLogs.find(timestamp)
                     var previousEntry = thingPowerLogs.find(previousTimestamp);
-                    if (timestamp < upcomingTimestamp && entry && (previousEntry || !d.loading)) {
+                    var hasEntry = d.sampleMatches(entry, timestamp)
+                    var hasPreviousEntry = d.sampleMatches(previousEntry, previousTimestamp)
+                    if (timestamp < upcomingTimestamp && hasEntry && hasPreviousEntry) {
 //                        print("found entry:", entry.timestamp, previousEntry)
-                        var consumption = entry.totalConsumption
+                        var consumption = entry.totalConsumption - previousEntry.totalConsumption
-                        if (previousEntry) {
-                            consumption -= previousEntry.totalConsumption
-                        }
                         barSet.replace(i, consumption)
                         valueAxis.adjustMax(consumption)
 
-                    } else if (timestamp.getTime() == upcomingTimestamp.getTime() && (previousEntry || !d.loading) && thingPowerLogs.liveEntry()) {
+                    } else if (timestamp.getTime() == upcomingTimestamp.getTime() && hasPreviousEntry && thingPowerLogs.liveEntry()) {
                         var consumption = thingPowerLogs.liveEntry().totalConsumption
 //                        print("it's today for thing", thing.name, consumption, previousEntry)
-                        if (previousEntry) {
+                        consumption -= previousEntry.totalConsumption
-//                            print("previous timestamp", previousEntry.timestamp, previousEntry.totalConsumption)
-                            consumption -= previousEntry.totalConsumption
-                        }
                         barSet.replace(i, consumption)
                         valueAxis.adjustMax(consumption)
                     } else {

nymea-app/ui/mainviews/energy/PowerBalanceStats.qml

@@ -79,6 +79,14 @@ StatsBase {
             }
         }
 
+        function sampleMatches(entry, expectedTimestamp) {
+            if (!entry) {
+                return false
+            }
+            var maxDistance = d.config.sampleRate * 60 * 1000 / 2
+            return Math.abs(entry.timestamp.getTime() - expectedTimestamp.getTime()) <= maxDistance
+        }
+
         function refresh() {
             if (powerBalanceLogs.loadingInhibited) {
                 return;
@@ -90,25 +98,19 @@ StatsBase {
             for (var i = 0; i < d.config.count; i++) {
                 var timestamp = root.calculateTimestamp(d.config.startTime(), d.config.sampleRate, d.startOffset + i + 1)
                 var previousTimestamp = root.calculateTimestamp(timestamp, d.config.sampleRate, -1)
-                var idx = powerBalanceLogs.indexOf(timestamp);
+                var entry = powerBalanceLogs.find(timestamp)
-                var entry = powerBalanceLogs.get(idx)
-//                print("timestamp:", timestamp, "previousTimestamp:", previousTimestamp)
                 var previousEntry = powerBalanceLogs.find(previousTimestamp);
-                if (timestamp < upcomingTimestamp && entry && (previousEntry || !d.loading)) {
+                var hasEntry = sampleMatches(entry, timestamp)
-//                    print("found entry:", entry.timestamp, entry.totalConsumption)
+                var hasPreviousEntry = sampleMatches(previousEntry, previousTimestamp)
-//                    if (previousEntry) {
+                if (timestamp < upcomingTimestamp && hasEntry && hasPreviousEntry) {
-//                        print("found previous:", previousEntry.timestamp, previousEntry.totalConsumption)
-//                    }
                     var consumption = entry.totalConsumption
                     var production = entry.totalProduction
                     var acquisition = entry.totalAcquisition
                     var returned = entry.totalReturn
-                    if (previousEntry) {
+                    consumption -= previousEntry.totalConsumption
-                        consumption -= previousEntry.totalConsumption
+                    production -= previousEntry.totalProduction
-                        production -= previousEntry.totalProduction
+                    acquisition -= previousEntry.totalAcquisition
-                        acquisition -= previousEntry.totalAcquisition
+                    returned -= previousEntry.totalReturn
-                        returned -= previousEntry.totalReturn
-                    }
                     consumptionSet.replace(i, consumption)
                     productionSet.replace(i, production)
                     acquisitionSet.replace(i, acquisition)
@@ -117,18 +119,16 @@ StatsBase {
                     valueAxis.adjustMax(production)
                     valueAxis.adjustMax(acquisition)
                     valueAxis.adjustMax(returned)
-                } else if (timestamp.getTime() == upcomingTimestamp.getTime() && (previousEntry || !d.loading)) {
+                } else if (timestamp.getTime() == upcomingTimestamp.getTime() && hasPreviousEntry) {
 //                    print("it's today!")
                     var consumption = energyManager.totalConsumption
                     var production = energyManager.totalProduction
                     var acquisition = energyManager.totalAcquisition
                     var returned = energyManager.totalReturn
-                    if (previousEntry) {
+                    consumption -= previousEntry.totalConsumption
-                        consumption -= previousEntry.totalConsumption
+                    production -= previousEntry.totalProduction
-                        production -= previousEntry.totalProduction
+                    acquisition -= previousEntry.totalAcquisition
-                        acquisition -= previousEntry.totalAcquisition
+                    returned -= previousEntry.totalReturn
-                        returned -= previousEntry.totalReturn
-                    }
                     consumptionSet.replace(i, consumption)
                     productionSet.replace(i, production)
                     acquisitionSet.replace(i, acquisition)