Improve reachable handling and buffered message handling

libnymea-zigbee/backends/deconz/zigbeenetworkdeconz.cpp

@@ -593,7 +593,7 @@ void ZigbeeNetworkDeconz::onApsDataConfirmReceived(const Zigbee::ApsdeDataConfir
         return;
     }
 
-    setReplyResponseError(reply, static_cast<Zigbee::ZigbeeApsStatus>(confirm.zigbeeStatusCode));
+    setReplyResponseError(reply, confirm.zigbeeStatusCode);
 }
 
 void ZigbeeNetworkDeconz::onApsDataIndicationReceived(const Zigbee::ApsdeDataIndication &indication)

libnymea-zigbee/backends/nxp/zigbeenetworknxp.cpp

@@ -77,12 +77,17 @@ ZigbeeNetworkReply *ZigbeeNetworkNxp::sendRequest(const ZigbeeNetworkRequest &re
     ZigbeeNetworkReply *reply = createNetworkReply(request);
     // Send the request, and keep the reply until transposrt, zigbee trasmission and response arrived
     connect(reply, &ZigbeeNetworkReply::finished, this, [this, reply](){
-        if (!m_pendingReplies.values().contains(reply)) {
+        if (m_pendingReplies.values().contains(reply)) {
-            //qCWarning(dcZigbeeNetwork()) << "#### Reply finished but not in the pending replies list" << reply;
-            return;
-        }
             quint8 requestId = m_pendingReplies.key(reply);
             m_pendingReplies.remove(requestId);
+            return;
+        }
+
+        if (m_bufferedReplies.values().contains(reply)) {
+            quint8 requestId = m_pendingReplies.key(reply);
+            m_pendingReplies.remove(requestId);
+            return;
+        }
         //qCWarning(dcZigbeeNetwork()) << "#### Removed network reply" << reply << "ID:" << requestId << "Current reply count" << m_pendingReplies.count();
     });
 
@@ -193,21 +198,30 @@ void ZigbeeNetworkNxp::sendNextReply()
         }
 
         // Note: this is a special case for nxp coordinator requests, they don't send a confirm because the request will not be sent trough the network
-        if (reply->request().destinationShortAddress() == 0x0000 && reply->request().profileId() == Zigbee::ZigbeeProfileDevice) {
+        if ((reply->request().destinationAddressMode() == Zigbee::DestinationAddressModeShortAddress &&
+                reply->request().destinationShortAddress() == 0x0000 &&
+                reply->request().profileId() == Zigbee::ZigbeeProfileDevice) ||
+                (reply->request().destinationAddressMode() == Zigbee::DestinationAddressModeIeeeAddress &&
+                 m_coordinatorNode &&
+                 reply->request().destinationIeeeAddress() == m_coordinatorNode->extendedAddress() &&
+                 reply->request().profileId() == Zigbee::ZigbeeProfileDevice)) {
+
             qCDebug(dcZigbeeNetwork()) << "Finish reply since there will be no CONFIRM for local node requests.";
             finishReplyInternally(reply);
             return;
         }
 
         quint8 networkRequestId = interfaceReply->responseData().at(0);
-        //qCDebug(dcZigbeeNetwork()) << "Request has network SQN" << networkRequestId;
+        ZigbeeNetworkRequest request = reply->request();
-        reply->request().setRequestId(networkRequestId);
+        request.setRequestId(networkRequestId);
+        updateReplyRequest(reply, request);
+
+        qCDebug(dcZigbeeAps()) << "Request SQN updated:" << reply->request();
         //qCWarning(dcZigbeeNetwork()) << "#### Insert network reply" << reply << "ID:" << networkRequestId << "Current reply count" << m_pendingReplies.count();
         m_pendingReplies.insert(networkRequestId, reply);
         // The request has been sent successfully to the device, start the timeout timer now
         startWaitingReply(reply);
     });
-
 }
 
 void ZigbeeNetworkNxp::finishReplyInternally(ZigbeeNetworkReply *reply, ZigbeeNetworkReply::Error error)
@@ -539,7 +553,17 @@ void ZigbeeNetworkNxp::onApsDataConfirmReceived(const Zigbee::ApsdeDataConfirm &
         return;
     }
 
-    setReplyResponseError(reply, static_cast<Zigbee::ZigbeeApsStatus>(confirm.zigbeeStatusCode));
+    if (confirm.zigbeeStatusCode == Zigbee::ZigbeeNwkLayerStatusFrameBuffered) {
+        // Move the reply to the buffered hash
+        m_pendingReplies.remove(confirm.requestId);
+        m_bufferedReplies.insert(confirm.requestId, reply);
+        // The frame has been buffered and will be sent once the route has been discovered.
+        // If the ACK will arrive, the frame was sent successfully, otherwise on timeout the request failed
+        qCWarning(dcZigbeeNetwork()) << "Request frame buffered" << reply->request();
+        return;
+    }
+
+    setReplyResponseError(reply, confirm.zigbeeStatusCode);
 }
 
 void ZigbeeNetworkNxp::onApsDataIndicationReceived(const Zigbee::ApsdeDataIndication &indication)
@@ -556,10 +580,25 @@ void ZigbeeNetworkNxp::onApsDataIndicationReceived(const Zigbee::ApsdeDataIndica
 
 void ZigbeeNetworkNxp::onApsDataAckReceived(const Zigbee::ApsdeDataAck &acknowledgement)
 {
-    ZigbeeNetworkReply *reply = m_pendingReplies.value(acknowledgement.requestId);
+    // Check first if we received an ACK from a buffered node...if so, the network reply can be finished with the given ACK status
-    if (reply && reply->buffered()) {
+    ZigbeeNetworkReply *reply = m_bufferedReplies.value(acknowledgement.requestId);
-        qCDebug(dcZigbeeNetwork()) << "Buffered frame from network request has been acknowledged" << acknowledgement;
+    if (reply) {
-        setReplyResponseError(reply, static_cast<Zigbee::ZigbeeApsStatus>(acknowledgement.zigbeeStatusCode));
+        if (acknowledgement.zigbeeStatusCode != Zigbee::ZigbeeApsStatusSuccess) {
+            qCWarning(dcZigbeeNetwork()) << "Buffered frame from network request has been acknowledged with error" << acknowledgement;
+        } else {
+            qCDebug(dcZigbeeNetwork()) << "Buffered frame from network request has been acknowledged successfully" << acknowledgement;
+        }
+        setReplyResponseError(reply, acknowledgement.zigbeeStatusCode);
+    } else {
+        if (acknowledgement.zigbeeStatusCode != Zigbee::ZigbeeApsStatusSuccess) {
+            qCWarning(dcZigbeeNetwork()) << acknowledgement;
+        } else {
+            ZigbeeNode *node = getZigbeeNode(acknowledgement.destinationAddress);
+            if (node) {
+                // We received a successfull ACk from this node...it is reachable in any case
+                setNodeReachable(node, true);
+            }
+        }
     }
 }
 

libnymea-zigbee/backends/nxp/zigbeenetworknxp.h

@@ -53,6 +53,8 @@ private:
     bool m_networkRunning = false;
 
     QHash<quint8, ZigbeeNetworkReply *> m_pendingReplies;
+    QHash<quint8, ZigbeeNetworkReply *> m_bufferedReplies;
+
     QQueue<ZigbeeNetworkReply *> m_replyQueue;
     ZigbeeNetworkReply *m_currentReply = nullptr;
 

libnymea-zigbee/zigbee.cpp

@@ -45,7 +45,7 @@ QDebug operator<<(QDebug debug, const Zigbee::ApsdeDataConfirm &confirm)
 
     debug.nospace() << "Destination EP:" << ZigbeeUtils::convertByteToHexString(confirm.destinationEndpoint) << ", ";
     debug.nospace() << "Source EP:" << ZigbeeUtils::convertByteToHexString(confirm.sourceEndpoint) << ", ";
-    debug.nospace() << static_cast<ZigbeeClusterLibrary::Status>(confirm.zigbeeStatusCode);
+    debug.nospace() << "Status:" << ZigbeeUtils::zigbeeStatusToString(confirm.zigbeeStatusCode);
     debug.nospace() << ")";
 
     return debug.space();
@@ -102,7 +102,7 @@ QDebug operator<<(QDebug debug, const Zigbee::ApsdeDataAck &acknowledgement)
     } else {
         debug.nospace() << static_cast<ZigbeeClusterLibrary::ClusterId>(acknowledgement.clusterId) << ", ";
     }
-    debug.nospace() << static_cast<ZigbeeClusterLibrary::Status>(acknowledgement.zigbeeStatusCode);
+    debug.nospace() << "Status:" << ZigbeeUtils::zigbeeStatusToString(acknowledgement.zigbeeStatusCode);
     debug.nospace() << ")";
     return debug.space();
 }

libnymea-zigbee/zigbee.h

@@ -145,14 +145,26 @@ public:
         HomeAutomationDeviceExtendedColourLight         = 0x010D,
         HomeAutomationDeviceLightLevelSensor            = 0x010E,
 
+        // Closures
+        HomeAutomationDeviceShade                       = 0x02000,
+        HomeAutomationDeviceShadeController             = 0x02001,
+        HomeAutomationWindowCoveringDevice              = 0x02002,
+        HomeAutomationWindowCoveringController          = 0x02003,
+
         // Heating, Ventilation and Air-Conditioning (HVAC) devices
+        HomeAutomationDeviceHeatingCoolingUnit          = 0x0300,
         HomeAutomationDeviceThermostat                  = 0x0301,
+        HomeAutomationDeviceTemperatureSensor           = 0x0302,
+        HomeAutomationDevicePump                        = 0x0303,
+        HomeAutomationDevicePumpController              = 0x0304,
+        HomeAutomationDevicePressureSensor              = 0x0305,
+        HomeAutomationDeviceFlowSensor                  = 0x0306,
 
         // Intruder Alarm System (IAS) devices
         HomeAutomationDeviceIsaControlEquipment             = 0x0400, // CIE
         HomeAutomationDeviceIsaAncillaryControlEquipment    = 0x0401, // ACE
-        HomeAutomationDeviceIsaZone                         = 0x0401,
+        HomeAutomationDeviceIsaZone                         = 0x0402,
-        HomeAutomationDeviceIsaWarningDevice                = 0x0401  // WD
+        HomeAutomationDeviceIsaWarningDevice                = 0x0403  // WD
     };
     Q_ENUM(HomeAutomationDevice)
 

libnymea-zigbee/zigbeenetwork.cpp

@@ -55,7 +55,7 @@ ZigbeeNetwork::ZigbeeNetwork(const QUuid &networkUuid, QObject *parent) :
 
 
     m_reachableRefreshTimer = new QTimer(this);
-    m_reachableRefreshTimer->setInterval(300000);
+    m_reachableRefreshTimer->setInterval(120000);
     m_reachableRefreshTimer->setSingleShot(false);
     connect(m_reachableRefreshTimer, &QTimer::timeout, this, &ZigbeeNetwork::evaluateNodeReachableStates);
 
@@ -472,8 +472,7 @@ void ZigbeeNetwork::evaluateNextNodeReachableState()
     if (m_reachableRefreshAddresses.isEmpty())
         return;
 
-    ZigbeeAddress address = m_reachableRefreshAddresses.takeFirst();
+    ZigbeeNode *node = getZigbeeNode(m_reachableRefreshAddresses.takeFirst());
-    ZigbeeNode *node = getZigbeeNode(address);
     if (!node) {
         // Not does not exit any more...continue
         evaluateNextNodeReachableState();
@@ -481,14 +480,17 @@ void ZigbeeNetwork::evaluateNextNodeReachableState()
     }
 
     // Make a lqi request in order to check if the node is reachable
-    ZigbeeDeviceObjectReply *zdoReply = node->deviceObject()->requestMgmtLqi();
+    ZigbeeDeviceObjectReply *zdoReply = node->deviceObject()->requestNetworkAddress();
     connect(zdoReply, &ZigbeeDeviceObjectReply::finished, this, [=](){
         if (zdoReply->error()) {
             qCWarning(dcZigbeeNetwork()) << node << "seems not to be reachable" << zdoReply->error();
             setNodeReachable(node, false);
+        } else {
+            setNodeReachable(node, true);
         }
 
-        evaluateNextNodeReachableState();
+        // Give some time for other requests to be processed
+        QTimer::singleShot(5000, this, &ZigbeeNetwork::evaluateNextNodeReachableState);
     });
 }
 
@@ -610,6 +612,11 @@ void ZigbeeNetwork::setNodeReachable(ZigbeeNode *node, bool reachable)
     node->setReachable(reachable);
 }
 
+void ZigbeeNetwork::updateReplyRequest(ZigbeeNetworkReply *reply, const ZigbeeNetworkRequest &request)
+{
+    reply->m_request = request;
+}
+
 void ZigbeeNetwork::setNodeInformation(ZigbeeNode *node, const QString &manufacturerName, const QString &modelName, const QString &version)
 {
     node->m_manufacturerName = manufacturerName;
@@ -836,29 +843,21 @@ ZigbeeNetworkReply *ZigbeeNetwork::createNetworkReply(const ZigbeeNetworkRequest
     return reply;
 }
 
-void ZigbeeNetwork::setReplyResponseError(ZigbeeNetworkReply *reply, Zigbee::ZigbeeApsStatus zigbeeApsStatus)
+void ZigbeeNetwork::setReplyResponseError(ZigbeeNetworkReply *reply, quint8 zigbeeStatus)
 {
-    if (zigbeeApsStatus == Zigbee::ZigbeeApsStatusSuccess) {
+    if (zigbeeStatus == Zigbee::ZigbeeApsStatusSuccess) {
         // The request has been sent successfully to the device
         finishNetworkReply(reply);
     } else {
         // There has been an error while transporting the request to the device
-        if (zigbeeApsStatus >= 0xc1 && zigbeeApsStatus <= 0xd4) {
+        if (zigbeeStatus >= 0xc1 && zigbeeStatus <= 0xd4) {
-            reply->m_zigbeeNwkStatus = static_cast<Zigbee::ZigbeeNwkLayerStatus>(static_cast<quint8>(zigbeeApsStatus));
+            reply->m_zigbeeNwkStatus = static_cast<Zigbee::ZigbeeNwkLayerStatus>(static_cast<quint8>(zigbeeStatus));
-            if (reply->zigbeeNwkStatus() == Zigbee::ZigbeeNwkLayerStatusFrameBuffered) {
-                // The frame has been buffered and will be sent once the route has been discovered.
-                // If the ACK will arrive, the frame was sent successfully, otherwise on timeout the request failed
-                reply->m_buffered = true;
-                // Restart the timer and wait for ack
-                reply->m_timer->start();
-                return;
-            }
             finishNetworkReply(reply, ZigbeeNetworkReply::ErrorZigbeeNwkStatusError);
-        } else if (zigbeeApsStatus >= 0xE0 && zigbeeApsStatus <= 0xF4) {
+        } else if (zigbeeStatus >= 0xE0 && zigbeeStatus <= 0xF4) {
-            reply->m_zigbeeMacStatus = static_cast<Zigbee::ZigbeeMacLayerStatus>(static_cast<quint8>(zigbeeApsStatus));
+            reply->m_zigbeeMacStatus = static_cast<Zigbee::ZigbeeMacLayerStatus>(static_cast<quint8>(zigbeeStatus));
             finishNetworkReply(reply, ZigbeeNetworkReply::ErrorZigbeeMacStatusError);
         } else {
-            reply->m_zigbeeApsStatus = zigbeeApsStatus;
+            reply->m_zigbeeApsStatus = static_cast<Zigbee::ZigbeeApsStatus>(zigbeeStatus);
             finishNetworkReply(reply, ZigbeeNetworkReply::ErrorZigbeeApsStatusError);
         }
     }
@@ -920,8 +919,27 @@ void ZigbeeNetwork::evaluateNodeReachableStates()
     m_reachableRefreshAddresses.clear();
 
     foreach (ZigbeeNode *node, m_nodes) {
+        // Skip the coordinator
+        if (node->shortAddress() == 0x0000)
+            continue;
+
         if (node->macCapabilities().receiverOnWhenIdle && node->shortAddress() != 0x0000) {
+
+            // Lets send a request to all things which are not reachable
+            if (!node->reachable()) {
+                qCDebug(dcZigbeeNetwork()) << node << "enqueue evaluating reachable state";
                 m_reachableRefreshAddresses.append(node->extendedAddress());
+                continue;
+            }
+
+            // Lets send a request to nodes which have not been seen more than 10 min
+            int msSinceLastSeen = node->lastSeen().msecsTo(QDateTime::currentDateTimeUtc());
+            qCDebug(dcZigbeeNetwork()) << node << "has been seen the last time" << QTime::fromMSecsSinceStartOfDay(msSinceLastSeen).toString() << "ago.";
+            // 10 min = 10 * 60 * 1000 = 600000 ms
+            if (msSinceLastSeen > 600000) {
+                qCDebug(dcZigbeeNetwork()) << node << "enqueue evaluating reachable state";
+                m_reachableRefreshAddresses.append(node->extendedAddress());
+            }
         } else {
             // Note: sleeping devices should send some message within 6 hours,
             // otherwise the device might not be reachable any more

libnymea-zigbee/zigbeenetwork.h

@@ -194,6 +194,7 @@ protected:
     void removeUninitializedNode(ZigbeeNode *node);
 
     void setNodeReachable(ZigbeeNode *node, bool reachable);
+    void updateReplyRequest(ZigbeeNetworkReply *reply, const ZigbeeNetworkRequest &request);
 
     // Set the coordinator infromation since they cannot be fetched
     void setNodeInformation(ZigbeeNode *node, const QString &manufacturerName, const QString &modelName, const QString &version);
@@ -218,7 +219,7 @@ protected:
 
     // Network reply methods
     ZigbeeNetworkReply *createNetworkReply(const ZigbeeNetworkRequest &request = ZigbeeNetworkRequest());
-    void setReplyResponseError(ZigbeeNetworkReply *reply, Zigbee::ZigbeeApsStatus zigbeeApsStatus = Zigbee::ZigbeeApsStatusSuccess);
+    void setReplyResponseError(ZigbeeNetworkReply *reply, quint8 zigbeeStatus = Zigbee::ZigbeeApsStatusSuccess);
     void finishNetworkReply(ZigbeeNetworkReply *reply, ZigbeeNetworkReply::Error error = ZigbeeNetworkReply::ErrorNoError);
     void startWaitingReply(ZigbeeNetworkReply *reply);
 

libnymea-zigbee/zigbeenetworkreply.cpp

@@ -47,11 +47,6 @@ Zigbee::ZigbeeApsStatus ZigbeeNetworkReply::zigbeeApsStatus() const
     return m_zigbeeApsStatus;
 }
 
-bool ZigbeeNetworkReply::buffered() const
-{
-    return m_buffered;
-}
-
 Zigbee::ZigbeeNwkLayerStatus ZigbeeNetworkReply::zigbeeNwkStatus() const
 {
     return m_zigbeeNwkStatus;
@@ -65,13 +60,7 @@ ZigbeeNetworkReply::ZigbeeNetworkReply(const ZigbeeNetworkRequest &request, QObj
     m_timer->setSingleShot(true);
     m_timer->setInterval(10000);
     connect(m_timer, &QTimer::timeout, this, [this](){
-        if (m_buffered) {
-            // We did not receive any reply from the buffered message, assuming the route could not be discovered to the device
-            m_zigbeeNwkStatus = Zigbee::ZigbeeNwkLayerStatusRouteDiscoveryFailed;
-            m_error = ErrorZigbeeNwkStatusError;
-        } else {
         m_error = ErrorTimeout;
-        }
         emit finished();
     });
 }

libnymea-zigbee/zigbeenetworkreply.h

@@ -60,13 +60,10 @@ public:
     Zigbee::ZigbeeNwkLayerStatus zigbeeNwkStatus() const;
     Zigbee::ZigbeeApsStatus zigbeeApsStatus() const;
 
-    bool buffered() const;
-
 private:
     explicit ZigbeeNetworkReply(const ZigbeeNetworkRequest &request, QObject *parent = nullptr);
     ZigbeeNetworkRequest m_request;
     QTimer *m_timer = nullptr;
-    bool m_buffered = false;
 
     Error m_error = ErrorNoError;
     Zigbee::ZigbeeMacLayerStatus m_zigbeeMacStatus = Zigbee::ZigbeeMacLayerStatusSuccess;

libnymea-zigbee/zigbeenode.cpp

@@ -162,7 +162,12 @@ void ZigbeeNode::setReachable(bool reachable)
     if (m_reachable == reachable)
         return;
 
+    if (!reachable) {
+        qCWarning(dcZigbeeNode()) << "Reachable changed"  << this << reachable;
+    } else {
         qCDebug(dcZigbeeNode()) << "Reachable changed"  << this << reachable;
+    }
+
     m_reachable = reachable;
     emit reachableChanged(m_reachable);
 }
@@ -806,6 +811,12 @@ QDebug operator<<(QDebug debug, ZigbeeNode *node)
 {
     debug.nospace().noquote() << "ZigbeeNode(" << ZigbeeUtils::convertUint16ToHexString(node->shortAddress());
     debug.nospace().noquote() << ", " << node->extendedAddress().toString();
+    if (!node->manufacturerName().isEmpty())
+        debug.nospace().noquote() << ", " << node->manufacturerName();
+
+    if (!node->modelName().isEmpty())
+        debug.nospace().noquote() << ", " << node->modelName();
+
     switch (node->nodeDescriptor().nodeType) {
     case ZigbeeDeviceProfile::NodeTypeCoordinator:
         debug.nospace().noquote() << ", Coordinator";

libnymea-zigbee/zigbeeutils.cpp

@@ -243,6 +243,34 @@ QString ZigbeeUtils::convertUint64ToHexString(const quint64 &value)
     return QString("0x%1").arg(convertByteArrayToHexString(data).remove(" ").remove("0x"));
 }
 
+QString ZigbeeUtils::zigbeeStatusToString(quint8 status)
+{
+    QString statusString;
+
+    if (status == 0) {
+        statusString = "Success";
+    } else if (status >= 0xc1 && status <= 0xd4) {
+        // NWK layer status
+        QMetaEnum metaEnum = QMetaEnum::fromType<Zigbee::ZigbeeNwkLayerStatus>();
+        QString enumString = QString(metaEnum.valueToKey(status));
+        statusString = QString("%1(%2)").arg(enumString).arg(ZigbeeUtils::convertByteToHexString(status));
+    } else if (status >= 0xE0 && status <= 0xF4) {
+        //MAC layer
+        QMetaEnum metaEnum = QMetaEnum::fromType<Zigbee::ZigbeeMacLayerStatus>();
+        QString enumString = QString(metaEnum.valueToKey(status));
+        statusString = QString("%1(%2)").arg(enumString).arg(ZigbeeUtils::convertByteToHexString(status));
+    } else if (status >= 0xa0 && status <= 0xb0) {
+        // APS layer
+        QMetaEnum metaEnum = QMetaEnum::fromType<Zigbee::ZigbeeApsStatus>();
+        QString enumString = QString(metaEnum.valueToKey(status));
+        statusString = QString("%1(%2)").arg(enumString).arg(ZigbeeUtils::convertByteToHexString(status));
+    } else {
+        statusString = QString("Unknown status (%1)").arg(status);
+    }
+
+    return statusString;
+}
+
 QString ZigbeeUtils::clusterIdToString(const ZigbeeClusterLibrary::ClusterId &clusterId)
 {
     QMetaEnum metaEnum = QMetaEnum::fromType<ZigbeeClusterLibrary::ClusterId>();

libnymea-zigbee/zigbeeutils.h

@@ -61,6 +61,8 @@ public:
     static QString convertUint32ToHexString(const quint32 &value);
     static QString convertUint64ToHexString(const quint64 &value);
 
+    static QString zigbeeStatusToString(quint8 status);
+
     // Enum prittify print methods
     //static QString messageTypeToString(const Zigbee::InterfaceMessageType &type);
     static QString clusterIdToString(const ZigbeeClusterLibrary::ClusterId &clusterId);
@@ -69,6 +71,7 @@ public:
     // Generate random data
     static quint64 generateRandomPanId();
 
+
     // Color converter
     static QPointF convertColorToXY(const QColor &color);
     static QPoint convertColorToXYInt(const QColor &color);