ETM-Schurig
/
nymea-app
mirror of https://github.com/nymea/nymea-app.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix energy stats lookup and deduplicate fetched log samples

improve-energy-logging
Simon Stürz 2026-02-20 15:57:02 +01:00
parent 8401af9529
commit f8ed67c4d3
5 changed files with 87 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
libnymea-app/energy/energylogs.cpp
View File

@ -216,38 +216,42 @@ int EnergyLogs::indexOf(const QDateTime &timestamp)
if (m_list.isEmpty()) {
return -1;
}
QDateTime first = m_list.first()->timestamp();
int index = qRound(1.0 * first.secsTo(timestamp) / (m_sampleRate * 60));
if (index < 0 || index >= m_list.count()) {
qCDebug(dcEnergyLogs()) << "finding:" << timestamp << index << first.toString() << "NOT FOUND" << m_list.last()->timestamp() << m_list.count();
const qint64 target = timestamp.toMSecsSinceEpoch();
const qint64 firstTimestamp = m_list.first()->timestamp().toMSecsSinceEpoch();
const qint64 lastTimestamp = m_list.last()->timestamp().toMSecsSinceEpoch();
if (target < firstTimestamp || target > lastTimestamp) {
return -1;
}
qCDebug(dcEnergyLogs()) << "finding:" << timestamp << index << first.toString() << m_list.at(index)->timestamp();
int low = 0;
int high = m_list.count() - 1;
// Normally, if the DB is in a consistent state, we can rely that the above finds the correct entry.
// However, if the user changes the timezone, during the lifetime, or other woes may appear like NTP
// changing time which may cause inconsistent entries like passing the same time twice, we could end up
// off by one. In order to compensate for that, we'll see if the next or previous entries may be closer
// In theory we could even be off by some more samples in very rare circumstances, but unlikely enough
// to not bother with that at this point.
QDateTime found = m_list.at(index)->timestamp();
QDateTime previous = index > 0 ? m_list.at(index-1)->timestamp() : found;
QDateTime next = index < m_list.count() - 1 ? m_list.at(index+1)->timestamp() : found;
int diffToFound = qAbs(timestamp.secsTo(found));
int diffToPrevious = qAbs(timestamp.secsTo(previous));
int diffToNext = qAbs(timestamp.secsTo(next));
if (diffToPrevious < diffToFound && diffToPrevious < diffToNext) {
// qWarning() << "Correcting to previous" << index << m_list.count() << found << previous << diffToPrevious << diffToFound;
return index - 1;
// Use timestamp-based lookup instead of sample-rate math. This is robust against
// duplicate/missing rows (e.g. after resampling glitches or timezone jumps).
while (low <= high) {
const int mid = low + (high - low) / 2;
const qint64 midTimestamp = m_list.at(mid)->timestamp().toMSecsSinceEpoch();
if (midTimestamp < target) {
low = mid + 1;
} else if (midTimestamp > target) {
high = mid - 1;
} else {
return mid;
}
}
if (diffToNext < diffToFound) {
// qWarning() << "Correcting to next" << index << m_list.count() << found << next << diffToNext << diffToFound;
return index + 1;
const int previousIndex = low - 1;
const int nextIndex = low;
const qint64 previousTimestamp = m_list.at(previousIndex)->timestamp().toMSecsSinceEpoch();
const qint64 nextTimestamp = m_list.at(nextIndex)->timestamp().toMSecsSinceEpoch();
const qint64 diffToPrevious = qAbs(target - previousTimestamp);
const qint64 diffToNext = qAbs(target - nextTimestamp);
if (diffToPrevious <= diffToNext) {
return previousIndex;
}
return index;
return nextIndex;
}
EnergyLogEntry *EnergyLogs::find(const QDateTime &timestamp)

8
libnymea-app/energy/powerbalancelogs.cpp
View File

@ -24,6 +24,7 @@
#include "powerbalancelogs.h"
#include <QMap>
#include <QMetaEnum>
PowerBalanceLogEntry::PowerBalanceLogEntry(QObject *parent): EnergyLogEntry(parent)
@ -98,8 +99,15 @@ QString PowerBalanceLogs::logsName() const
QList<EnergyLogEntry *> PowerBalanceLogs::unpackEntries(const QVariantMap &params, double *minValue, double *maxValue)
{
QList<EnergyLogEntry*> ret;
QMap<qint64, QVariantMap> deduplicatedEntries;
foreach (const QVariant &variant, params.value("powerBalanceLogEntries").toList()) {
QVariantMap map = variant.toMap();
// Keep the last row for a timestamp if the backend returned duplicates.
deduplicatedEntries.insert(map.value("timestamp").toLongLong(), map);
}
for (auto it = deduplicatedEntries.constBegin(); it != deduplicatedEntries.constEnd(); ++it) {
const QVariantMap &map = it.value();
QDateTime timestamp = QDateTime::fromSecsSinceEpoch(map.value("timestamp").toLongLong());
double consumption = map.value("consumption").toDouble();
double production = map.value("production").toDouble();

16
libnymea-app/energy/thingpowerlogs.cpp
View File

@ -24,6 +24,8 @@
#include "thingpowerlogs.h"
#include <limits>
#include <QMap>
#include <QMetaEnum>
#include <QLoggingCategory>
@ -142,25 +144,37 @@ QVariantMap ThingPowerLogs::fetchParams() const
QList<EnergyLogEntry *> ThingPowerLogs::unpackEntries(const QVariantMap &params, double *minValue, double *maxValue)
{
qint64 newestCurrentTimestamp = std::numeric_limits<qint64>::min();
foreach (const QVariant &variant, params.value("currentEntries").toList()) {
QVariantMap map = variant.toMap();
if (map.value("thingId").toUuid() != m_thingId) {
continue;
}
const qint64 timestamp = map.value("timestamp").toLongLong();
if (timestamp < newestCurrentTimestamp) {
continue;
}
newestCurrentTimestamp = timestamp;
if (m_liveEntry) {
m_liveEntry->deleteLater();
}
m_liveEntry = unpack(map);
emit liveEntryChanged(m_liveEntry);
break;
}
QList<EnergyLogEntry*> ret;
QMap<qint64, QVariantMap> deduplicatedEntries;
foreach (const QVariant &variant, params.value("thingPowerLogEntries").toList()) {
QVariantMap map = variant.toMap();
if (map.value("thingId").toUuid() != m_thingId) {
continue;
}
// Keep the last row for a timestamp if the backend returned duplicates.
deduplicatedEntries.insert(map.value("timestamp").toLongLong(), map);
}
for (auto it = deduplicatedEntries.constBegin(); it != deduplicatedEntries.constEnd(); ++it) {
const QVariantMap &map = it.value();
QDateTime timestamp = QDateTime::fromSecsSinceEpoch(map.value("timestamp").toLongLong());
QUuid thingId = map.value("thingId").toUuid();
double currentPower = map.value("currentPower").toDouble();

24
nymea-app/ui/mainviews/energy/ConsumerStats.qml
View File

@ -80,6 +80,14 @@ StatsBase {
d.selectedThing = thing
}
}
function sampleMatches(entry, expectedTimestamp) {
if (!entry) {
return false
}
var maxDistance = d.config.sampleRate * 60 * 1000 / 2
return Math.abs(entry.timestamp.getTime() - expectedTimestamp.getTime()) <= maxDistance
}
}
ThingPowerLogsLoader {
@ -133,22 +141,18 @@ StatsBase {
// print("timestamp:", timestamp, "previous:", previousTimestamp)
var entry = thingPowerLogs.find(timestamp)
var previousEntry = thingPowerLogs.find(previousTimestamp);
if (timestamp < upcomingTimestamp && entry && (previousEntry || !d.loading)) {
var hasEntry = d.sampleMatches(entry, timestamp)
var hasPreviousEntry = d.sampleMatches(previousEntry, previousTimestamp)
if (timestamp < upcomingTimestamp && hasEntry && hasPreviousEntry) {
// print("found entry:", entry.timestamp, previousEntry)
var consumption = entry.totalConsumption
if (previousEntry) {
consumption -= previousEntry.totalConsumption
}
var consumption = entry.totalConsumption - previousEntry.totalConsumption
barSet.replace(i, consumption)
valueAxis.adjustMax(consumption)
} else if (timestamp.getTime() == upcomingTimestamp.getTime() && (previousEntry || !d.loading) && thingPowerLogs.liveEntry()) {
} else if (timestamp.getTime() == upcomingTimestamp.getTime() && hasPreviousEntry && thingPowerLogs.liveEntry()) {
var consumption = thingPowerLogs.liveEntry().totalConsumption
// print("it's today for thing", thing.name, consumption, previousEntry)
if (previousEntry) {
// print("previous timestamp", previousEntry.timestamp, previousEntry.totalConsumption)
consumption -= previousEntry.totalConsumption
}
consumption -= previousEntry.totalConsumption
barSet.replace(i, consumption)
valueAxis.adjustMax(consumption)
} else {

42
nymea-app/ui/mainviews/energy/PowerBalanceStats.qml
View File

@ -79,6 +79,14 @@ StatsBase {
}
}
function sampleMatches(entry, expectedTimestamp) {
if (!entry) {
return false
}
var maxDistance = d.config.sampleRate * 60 * 1000 / 2
return Math.abs(entry.timestamp.getTime() - expectedTimestamp.getTime()) <= maxDistance
}
function refresh() {
if (powerBalanceLogs.loadingInhibited) {
return;
@ -90,25 +98,19 @@ StatsBase {
for (var i = 0; i < d.config.count; i++) {
var timestamp = root.calculateTimestamp(d.config.startTime(), d.config.sampleRate, d.startOffset + i + 1)
var previousTimestamp = root.calculateTimestamp(timestamp, d.config.sampleRate, -1)
var idx = powerBalanceLogs.indexOf(timestamp);
var entry = powerBalanceLogs.get(idx)
// print("timestamp:", timestamp, "previousTimestamp:", previousTimestamp)
var entry = powerBalanceLogs.find(timestamp)
var previousEntry = powerBalanceLogs.find(previousTimestamp);
if (timestamp < upcomingTimestamp && entry && (previousEntry || !d.loading)) {
// print("found entry:", entry.timestamp, entry.totalConsumption)
// if (previousEntry) {
// print("found previous:", previousEntry.timestamp, previousEntry.totalConsumption)
// }
var hasEntry = sampleMatches(entry, timestamp)
var hasPreviousEntry = sampleMatches(previousEntry, previousTimestamp)
if (timestamp < upcomingTimestamp && hasEntry && hasPreviousEntry) {
var consumption = entry.totalConsumption
var production = entry.totalProduction
var acquisition = entry.totalAcquisition
var returned = entry.totalReturn
if (previousEntry) {
consumption -= previousEntry.totalConsumption
production -= previousEntry.totalProduction
acquisition -= previousEntry.totalAcquisition
returned -= previousEntry.totalReturn
}
consumption -= previousEntry.totalConsumption
production -= previousEntry.totalProduction
acquisition -= previousEntry.totalAcquisition
returned -= previousEntry.totalReturn
consumptionSet.replace(i, consumption)
productionSet.replace(i, production)
acquisitionSet.replace(i, acquisition)
@ -117,18 +119,16 @@ StatsBase {
valueAxis.adjustMax(production)
valueAxis.adjustMax(acquisition)
valueAxis.adjustMax(returned)
} else if (timestamp.getTime() == upcomingTimestamp.getTime() && (previousEntry || !d.loading)) {
} else if (timestamp.getTime() == upcomingTimestamp.getTime() && hasPreviousEntry) {
// print("it's today!")
var consumption = energyManager.totalConsumption
var production = energyManager.totalProduction
var acquisition = energyManager.totalAcquisition
var returned = energyManager.totalReturn
if (previousEntry) {
consumption -= previousEntry.totalConsumption
production -= previousEntry.totalProduction
acquisition -= previousEntry.totalAcquisition
returned -= previousEntry.totalReturn
}
consumption -= previousEntry.totalConsumption
production -= previousEntry.totalProduction
acquisition -= previousEntry.totalAcquisition
returned -= previousEntry.totalReturn
consumptionSet.replace(i, consumption)
productionSet.replace(i, production)
acquisitionSet.replace(i, acquisition)