From 68bc1d648fd437828727a200ae405298407a593d Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sat, 11 Dec 2021 00:28:43 +0100 Subject: [PATCH] Make internal counters independent from thing counters --- plugin/energylogger.cpp | 67 +++++++++++++++++++- plugin/energylogger.h | 6 ++ plugin/energymanagerimpl.cpp | 114 ++++++++++++++++++++++++++++++----- plugin/energymanagerimpl.h | 16 ++++- 4 files changed, 186 insertions(+), 17 deletions(-) diff --git a/plugin/energylogger.cpp b/plugin/energylogger.cpp index 9119a6d..6ddfff6 100644 --- a/plugin/energylogger.cpp +++ b/plugin/energylogger.cpp @@ -79,7 +79,6 @@ void EnergyLogger::logPowerBalance(double consumption, double production, double void EnergyLogger::logThingPower(const ThingId &thingId, double currentPower, double totalConsumption, double totalProduction) { - qCDebug(dcEnergyExperience()) << "Logging thing power for" << thingId.toString() << "Current power:" << currentPower << "Total consumption:" << totalConsumption << "Total production:" << totalProduction; ThingPowerLogEntry entry(QDateTime::currentDateTime(), thingId, currentPower, totalConsumption, totalProduction); m_thingsPowerLiveLogs[thingId].prepend(entry); @@ -177,6 +176,11 @@ ThingPowerLogEntries EnergyLogger::thingPowerLogs(SampleRate sampleRate, const Q PowerBalanceLogEntry EnergyLogger::latestLogEntry(SampleRate sampleRate) { + if (sampleRate == SampleRateAny) { + if (m_balanceLiveLog.count() > 0) { + return m_balanceLiveLog.first(); + } + } QSqlQuery query(m_db); QString queryString = "SELECT MAX(timestamp), consumption, production, acquisition, storage, totalConsumption, totalProduction, totalAcquisition, totalReturn FROM powerBalance"; QVariantList bindValues; @@ -203,6 +207,12 @@ PowerBalanceLogEntry EnergyLogger::latestLogEntry(SampleRate sampleRate) ThingPowerLogEntry EnergyLogger::latestLogEntry(SampleRate sampleRate, const ThingId &thingId) { + if (sampleRate == SampleRateAny) { + if (m_thingsPowerLiveLogs.value(thingId).count() > 0) { + return m_thingsPowerLiveLogs.value(thingId).first(); + } + } + QSqlQuery query(m_db); query.prepare("SELECT MAX(timestamp), currentPower, totalConsumption, totalProduction from thingPower WHERE sampleRate = ? AND thingId = ?;"); query.addBindValue(sampleRate); @@ -247,6 +257,36 @@ QList EnergyLogger::loggedThings() const return ret; } +void EnergyLogger::cacheThingEntry(const ThingId &thingId, double totalEnergyConsumed, double totalEnergyProduced) +{ + QSqlQuery query(m_db); + query.prepare("INSERT OR REPLACE INTO thingCache (thingId, totalEnergyConsumed, totalEnergyProduced) VALUES (?, ?, ?);"); + query.addBindValue(thingId); + query.addBindValue(totalEnergyConsumed); + query.addBindValue(totalEnergyProduced); + query.exec(); + if (query.lastError().isValid()) { + qCWarning(dcEnergyExperience()) << "Failed to store thing cache entry:" << query.lastError() << query.executedQuery(); + } +} + +ThingPowerLogEntry EnergyLogger::cachedThingEntry(const ThingId &thingId) +{ + QSqlQuery query(m_db); + query.prepare("SELECT * FROM thingCache WHERE thingId = ?;"); + query.addBindValue(thingId); + query.exec(); + if (query.lastError().isValid()) { + qCWarning(dcEnergyExperience()) << "Failed to retrieve thing cache entry:" << query.lastError() << query.executedQuery(); + return ThingPowerLogEntry(); + } + if (!query.next()) { + qCDebug(dcEnergyExperience()) << "No cached thing entry for" << thingId; + return ThingPowerLogEntry(); + } + return ThingPowerLogEntry(QDateTime(), thingId, 0, query.value("totalEnergyConsumed").toDouble(), query.value("totalEnergyProduced").toDouble()); +} + void EnergyLogger::sample() { QDateTime now = QDateTime::currentDateTime(); @@ -375,6 +415,17 @@ bool EnergyLogger::initDB() return false; } + if (!m_db.tables().contains("metadata")) { + qCDebug(dcEnergyExperience()) << "No \metadata\" table in database. Creating it."; + m_db.exec("CREATE TABLE metadata (version INT);"); + m_db.exec("INSERT INTO metadata (version) VALUES (1);"); + + if (m_db.lastError().isValid()) { + qCWarning(dcEnergyExperience()) << "Error creating metadata table in energy log database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText(); + return false; + } + } + if (!m_db.tables().contains("powerBalance")) { qCDebug(dcEnergyExperience()) << "No \"powerBalance\" table in database. Creating it."; m_db.exec("CREATE TABLE powerBalance " @@ -414,6 +465,20 @@ bool EnergyLogger::initDB() } } + if (!m_db.tables().contains("thingCache")) { + qCDebug(dcEnergyExperience()) << "No \"thingCache\" table in database. Creating it."; + m_db.exec("CREATE TABLE thingCache " + "(" + "thingId VARCHAR(38) PRIMARY KEY," + "totalEnergyConsumed FLOAT," + "totalEnergyProduced FLOAT" + ");"); + if (m_db.lastError().isValid()) { + qCWarning(dcEnergyExperience()) << "Error creating thingCache table in energy log database. Driver error:" << m_db.lastError().driverText() << "Database error:" << m_db.lastError().databaseText(); + return false; + } + } + qCDebug(dcEnergyExperience()) << "Initialized logging DB successfully." << m_db.databaseName(); return true; } diff --git a/plugin/energylogger.h b/plugin/energylogger.h index 07378de..6c279d8 100644 --- a/plugin/energylogger.h +++ b/plugin/energylogger.h @@ -30,6 +30,12 @@ public: void removeThingLogs(const ThingId &thingId); QList loggedThings() const; + // For internal use, the energymanager needs to cache some values to track things total values + // This is really only here to have a single storage and not keep a separate cache file. Shouldn't be used for anything else + // Note that the returned ThingPowerLogEntry will be incomplete. It won't have a timestamp nor a currentPower value! + void cacheThingEntry(const ThingId &thingId, double totalEnergyConsumed, double totalEnergyProduced); + ThingPowerLogEntry cachedThingEntry(const ThingId &thingId); + private slots: void sample(); diff --git a/plugin/energymanagerimpl.cpp b/plugin/energymanagerimpl.cpp index ea664f3..5eca4c5 100644 --- a/plugin/energymanagerimpl.cpp +++ b/plugin/energymanagerimpl.cpp @@ -124,7 +124,7 @@ void EnergyManagerImpl::watchThing(Thing *thing) setRootMeter(thing->id()); } - qCDebug(dcEnergyExperience()) << "Wathing thing:" << thing->name(); + qCDebug(dcEnergyExperience()) << "Watching thing:" << thing->name(); // React on things that require us updating the power balance if (thing->thingClass().interfaces().contains("energymeter") @@ -143,14 +143,69 @@ void EnergyManagerImpl::watchThing(Thing *thing) || thing->thingClass().interfaces().contains("smartmeterproducer") || thing->thingClass().interfaces().contains("energystorage")) { - ThingPowerLogEntry entry = m_logger->latestLogEntry(EnergyLogs::SampleRate1Min, {thing->id()}); - m_totalEnergyConsumedCache[thing] = entry.totalConsumption(); - m_totalEnergyProducedCache[thing] = entry.totalProduction(); - qCDebug(dcEnergyExperience()) << "Loaded thing power totals for" << thing->name() << "Consumption:" << entry.totalConsumption() << "Production:" << entry.totalProduction(); + // Initialize caches used to calculate diffs + ThingPowerLogEntry entry = m_logger->latestLogEntry(EnergyLogs::SampleRateAny, {thing->id()}); + ThingPowerLogEntry stateEntry = m_logger->cachedThingEntry(thing->id()); + + m_powerBalanceTotalEnergyConsumedCache[thing] = stateEntry.totalConsumption(); + m_powerBalanceTotalEnergyProducedCache[thing] = stateEntry.totalProduction(); + + m_thingsTotalEnergyConsumedCache[thing] = qMakePair(stateEntry.totalConsumption(), entry.totalConsumption()); + m_thingsTotalEnergyProducedCache[thing] = qMakePair(stateEntry.totalProduction(), entry.totalProduction()); + qCDebug(dcEnergyExperience()) << "Loaded thing power totals for" << thing->name() << "Consumption:" << entry.totalConsumption() << "Production:" << entry.totalProduction() << "Last thing state consumption:" << stateEntry.totalConsumption() << "production:" << stateEntry.totalProduction(); connect(thing, &Thing::stateValueChanged, this, [=](const StateTypeId &stateTypeId, const QVariant &/*value*/){ if (QStringList({"currentPower", "totalEnergyConsumed", "totalEnergyProduced"}).contains(thing->thingClass().getStateType(stateTypeId).name())) { - m_logger->logThingPower(thing->id(), thing->state("currentPower").value().toDouble(), thing->state("totalEnergyConsumed").value().toDouble(), thing->state("totalEnergyProduced").value().toDouble()); + + // We'll be keeping our own counters, starting from 0 at the time they're added to nymea and increasing with the things counters. + // This way we'll have proper logs even if the thing counter is reset (some things may reset their counter on power loss, factory reset etc) + // and also won't start with huge values if the thing has been counting for a while and only added to nymea later on + + + // Consumption + double oldThingConsumptionState = m_thingsTotalEnergyConsumedCache.value(thing).first; + double oldThingConsumptionInternal = m_thingsTotalEnergyConsumedCache.value(thing).second; + double newThingConsumptionState = thing->stateValue("totalEnergyConsumed").toDouble(); + // For the very first cycle (oldConsumption is 0) we'll sync up on the meter, without actually adding it to our diff + if (oldThingConsumptionState == 0 && newThingConsumptionState != 0) { + qInfo(dcEnergyExperience()) << "Don't have a consumption counter for" << thing->name() << "Synching internal counters to initial value:" << newThingConsumptionState; + oldThingConsumptionState = newThingConsumptionState; + } + // If the thing's meter has been reset in the meantime (newConsumption < oldConsumption) we'll sync down, taking the whole diff from 0 to new value + if (newThingConsumptionState < oldThingConsumptionState) { + qCInfo(dcEnergyExperience()) << "Thing meter for" << thing->name() << "seems to have been reset. Re-synching internal consumption counter."; + oldThingConsumptionState = newThingConsumptionState; + } + double consumptionDiff = newThingConsumptionState - oldThingConsumptionState; + double newThingConsumptionInternal = oldThingConsumptionInternal + consumptionDiff; + m_thingsTotalEnergyConsumedCache[thing] = qMakePair(newThingConsumptionState, newThingConsumptionInternal); + + + // Production + double oldThingProductionState = m_thingsTotalEnergyProducedCache.value(thing).first; + double oldThingProductionInternal = m_thingsTotalEnergyProducedCache.value(thing).second; + double newThingProductionState = thing->stateValue("totalEnergyProduced").toDouble(); + // For the very first cycle (oldProductino is 0) we'll sync up on the meter, without actually adding it to our diff + if (oldThingProductionState == 0 && newThingProductionState != 0) { + qInfo(dcEnergyExperience()) << "Don't have a production counter for" << thing->name() << "Synching internal counter to initial value:" << newThingProductionState; + oldThingProductionState = newThingProductionState; + } + // If the thing's meter has been reset in the meantime (newProduction < oldProduction) we'll sync down, taking the whole diff from 0 to new value + if (newThingProductionState < oldThingProductionState) { + qCInfo(dcEnergyExperience()) << "Thing meter for" << thing->name() << "seems to have been reset. Re-synching internal production counter."; + oldThingProductionState = newThingProductionState; + } + double productionDiff = newThingProductionState - oldThingProductionState; + double newThingProductionInternal = oldThingProductionInternal + productionDiff; + m_thingsTotalEnergyProducedCache[thing] = qMakePair(newThingProductionState, newThingProductionInternal); + + + // Write to log + qCDebug(dcEnergyExperience()) << "Logging thing" << thing->name() << "total consumption:" << newThingConsumptionInternal << "production:" << newThingProductionInternal; + m_logger->logThingPower(thing->id(), thing->state("currentPower").value().toDouble(), newThingConsumptionInternal, newThingProductionInternal); + + // Cache the thing state values in case nymea is restarted + m_logger->cacheThingEntry(thing->id(), newThingConsumptionState, newThingProductionState); } }); } @@ -172,38 +227,69 @@ void EnergyManagerImpl::updatePowerBalance() if (m_rootMeter) { currentPowerAcquisition = m_rootMeter->stateValue("currentPower").toDouble(); - double oldAcquisition = m_totalEnergyConsumedCache.value(m_rootMeter); + double oldAcquisition = m_powerBalanceTotalEnergyConsumedCache.value(m_rootMeter); double newAcquisition = m_rootMeter->stateValue("totalEnergyConsumed").toDouble(); + // For the very first cycle (oldAcquisition is 0) we'll sync up on the meter values without actually adding them to our balance. + if (oldAcquisition == 0) { + oldAcquisition = newAcquisition; + } + // If the root meter has been reset in the meantime (newConsumption < oldConsumption) we'll sync down, taking the whole diff from 0 to new value + if (newAcquisition < oldAcquisition) { + qCInfo(dcEnergyExperience()) << "Root meter seems to have been reset. Re-synching internal consumption counter."; + oldAcquisition = newAcquisition; + } qCDebug(dcEnergyExperience()) << "Root meter total consumption: Previous value:" << oldAcquisition << "New value:" << newAcquisition << "Diff:" << (newAcquisition -oldAcquisition); m_totalAcquisition += newAcquisition - oldAcquisition; - m_totalEnergyConsumedCache[m_rootMeter] = newAcquisition; + m_powerBalanceTotalEnergyConsumedCache[m_rootMeter] = newAcquisition; - double oldReturn = m_totalEnergyProducedCache.value(m_rootMeter); + double oldReturn = m_powerBalanceTotalEnergyProducedCache.value(m_rootMeter); double newReturn = m_rootMeter->stateValue("totalEnergyProduced").toDouble(); + // For the very first cycle (oldReturn is 0) we'll sync up on the meter values without actually adding them to our balance. + if (oldReturn == 0) { + oldReturn = newReturn; + } + if (newReturn < oldReturn) { + qCInfo(dcEnergyExperience()) << "Root meter seems to have been reset. Re-synching internal production counter."; + oldReturn = newReturn; + } qCDebug(dcEnergyExperience()) << "Root meter total production: Previous value:" << oldReturn << "New value:" << newReturn << "Diff:" << (newReturn - oldReturn); m_totalReturn += newReturn - oldReturn; - m_totalEnergyProducedCache[m_rootMeter] = newReturn; + m_powerBalanceTotalEnergyProducedCache[m_rootMeter] = newReturn; } double currentPowerProduction = 0; foreach (Thing* thing, m_thingManager->configuredThings().filterByInterface("smartmeterproducer")) { currentPowerProduction += thing->stateValue("currentPower").toDouble(); - double oldProduction = m_totalEnergyProducedCache.value(thing); + double oldProduction = m_powerBalanceTotalEnergyProducedCache.value(thing); double newProduction = thing->stateValue("totalEnergyProduced").toDouble(); + // For the very first cycle (oldProduction is 0) we'll sync up on the producer values without actually adding them to our balance. + if (oldProduction == 0) { + oldProduction = newProduction; + } + if (newProduction < oldProduction) { + oldProduction = newProduction; + } qCDebug(dcEnergyExperience()) << "Producer" << thing->name() << "total production: Previous value:" << oldProduction << "New value:" << newProduction << "Diff:" << (newProduction - oldProduction); m_totalProduction += newProduction - oldProduction; - m_totalEnergyProducedCache[thing] = newProduction; + m_powerBalanceTotalEnergyProducedCache[thing] = newProduction; } double currentPowerStorage = 0; double totalFromStorage = 0; foreach (Thing *thing, m_thingManager->configuredThings().filterByInterface("energystorage")) { currentPowerStorage += thing->stateValue("currentPower").toDouble(); - double oldProduction = m_totalEnergyProducedCache.value(thing); + double oldProduction = m_powerBalanceTotalEnergyProducedCache.value(thing); double newProduction = thing->stateValue("totalEnergyProduced").toDouble(); + // For the very first cycle (oldProdction is 0) we'll sync up on the meter values without actually adding them to our balance. + if (oldProduction == 0) { + oldProduction = newProduction; + } + if (newProduction < oldProduction) { + oldProduction = newProduction; + } qCDebug(dcEnergyExperience()) << "Storage" << thing->name() << "total storage: Previous value:" << oldProduction << "New value:" << newProduction << "Diff:" << (newProduction - oldProduction); totalFromStorage += newProduction - oldProduction; - m_totalEnergyProducedCache[thing] = newProduction; + m_powerBalanceTotalEnergyProducedCache[thing] = newProduction; } double currentPowerConsumption = currentPowerAcquisition + qAbs(qMin(0.0, currentPowerProduction)) - currentPowerStorage; diff --git a/plugin/energymanagerimpl.h b/plugin/energymanagerimpl.h index 6b172a9..8964e9e 100644 --- a/plugin/energymanagerimpl.h +++ b/plugin/energymanagerimpl.h @@ -57,8 +57,20 @@ private: EnergyLogger *m_logger = nullptr; - QHash m_totalEnergyConsumedCache; - QHash m_totalEnergyProducedCache; + // Caching some values so we don't have to look them up on the DB all the time: + // We use different caches for power balance and thing logs because they are calculated independently + // and one must not update the others cache for the diffs to be correct + + // For things totals we need to cache 2 values: + // The last thing state values we've processed + QHash m_powerBalanceTotalEnergyConsumedCache; + QHash m_powerBalanceTotalEnergyProducedCache; + + // - The last thing state value we've read and processed + // - The last entry in our internal counters we've processed and logged + // QHash> + QHash> m_thingsTotalEnergyConsumedCache; + QHash> m_thingsTotalEnergyProducedCache; }; #endif // ENERGYMANAGERIMPL_H