Move entirely to db storage and fix attribute loading

libnymea-zigbee/zigbeenetwork.cpp

@@ -266,6 +266,11 @@ void ZigbeeNetwork::addNodeInternally(ZigbeeNode *node)
     // FIXME: check when and how the note will be reachable
     //node->setConnected(state() == StateRunning);
 
+    // Note: if a cluster shows up after initialization (out of spec devices), save the cluster and it's attributes
+    foreach (ZigbeeNodeEndpoint *endpoint, node->endpoints()) {
+        connect(endpoint, &ZigbeeNodeEndpoint::clusterAttributeChanged, this, &ZigbeeNetwork::onNodeClusterAttributeChanged);
+    }
+
     m_nodes.append(node);
     emit nodeAdded(node);
 }
@@ -306,7 +311,7 @@ void ZigbeeNetwork::saveNetwork()
     settings.endGroup();
 
     foreach (ZigbeeNode *node, nodes()) {
-        saveNode(node);
+        m_database->saveNode(node);
     }
 }
 
@@ -320,7 +325,6 @@ void ZigbeeNetwork::loadNetwork()
     }
 
     QSettings settings(m_settingsFileName, QSettings::IniFormat, this);
-
     settings.beginGroup("Network");
     quint16 panId = static_cast<quint16>(settings.value("panId", 0).toUInt());
     setPanId(panId);
@@ -335,87 +339,11 @@ void ZigbeeNetwork::loadNetwork()
 
     settings.endGroup(); // Network
 
-    // Load nodes
+    QList<ZigbeeNode *> nodes = m_database->loadNodes();
-    settings.beginGroup("Nodes");
+    foreach (ZigbeeNode *node, nodes) {
-    foreach (const QString ieeeAddressString, settings.childGroups()) {
-        settings.beginGroup(ieeeAddressString);
-        quint16 shortAddress = static_cast<quint16>(settings.value("nwkAddress", 0).toUInt());
-        ZigbeeNode *node = createNode(shortAddress, ZigbeeAddress(ieeeAddressString), this);
-        node->m_nodeDescriptor = ZigbeeDeviceProfile::parseNodeDescriptor(settings.value("nodeDescriptorRaw").toByteArray());
-        node->m_macCapabilities = node->nodeDescriptor().macCapabilities;
-        node->m_powerDescriptor = ZigbeeDeviceProfile::parsePowerDescriptor(static_cast<quint16>(settings.value("powerDescriptorFlag", 0).toUInt()));
-
-        int endpointsCount = settings.beginReadArray("endpoints");
-        //qCDebug(dcZigbeeNetwork()) << "loading endpoints" << endpointsCount << settings.childKeys() << settings.childGroups();
-        for (int i = 0; i < endpointsCount; i++) {
-            settings.setArrayIndex(i);
-            quint8 endpointId = static_cast<quint8>(settings.value("id", 0).toUInt());
-            ZigbeeNodeEndpoint *endpoint = new ZigbeeNodeEndpoint(this, node, endpointId, node);
-            endpoint->m_profile = static_cast<Zigbee::ZigbeeProfile>(settings.value("profile", 0).toUInt());
-            endpoint->m_deviceId = static_cast<quint16>(settings.value("deviceId", 0).toUInt());
-            endpoint->m_deviceVersion = static_cast<quint8>(settings.value("deviceId", 0).toUInt());
-            endpoint->m_manufacturerName = settings.value("manufacturerName").toString();
-            endpoint->m_modelIdentifier = settings.value("modelIdentifier").toString();
-            endpoint->m_softwareBuildId = settings.value("softwareBuildId").toString();
-
-            int inputClustersCount = settings.beginReadArray("inputClusters");
-            for (int n = 0; n < inputClustersCount; n ++) {
-                settings.setArrayIndex(n);
-                Zigbee::ClusterId clusterId = static_cast<Zigbee::ClusterId>(settings.value("clusterId", 0).toUInt());
-                ZigbeeCluster *cluster = endpoint->createCluster(clusterId, ZigbeeCluster::Server);
-                endpoint->addInputCluster(cluster);
-            }
-            settings.endArray(); // inputClusters
-
-            int outputClustersCount = settings.beginReadArray("outputClusters");
-            for (int n = 0; n < outputClustersCount; n ++) {
-                settings.setArrayIndex(n);
-                Zigbee::ClusterId clusterId = static_cast<Zigbee::ClusterId>(settings.value("clusterId", 0).toUInt());
-                ZigbeeCluster *cluster = endpoint->createCluster(clusterId, ZigbeeCluster::Client);
-                endpoint->addOutputCluster(cluster);
-            }
-            settings.endArray(); // outputClusters
-
-            node->m_endpoints.append(endpoint);
-        }
-
-        settings.endArray(); // endpoints
-
-        settings.endGroup(); // ieeeAddress
-
         node->setState(ZigbeeNode::StateInitialized);
         addNodeInternally(node);
     }
-    settings.endGroup(); // Nodes
-
-    qCWarning(dcZigbeeNetwork()) << this;
-
-    // FIXME: for testing
-    foreach(ZigbeeNode *node, m_nodes) {
-        m_database->saveNode(node);
-    }
-
-    // Now load and print the nodes again for checking
-    QList<ZigbeeNode *> dbNodes = m_database->loadNodes();
-
-    foreach (ZigbeeNode *node, dbNodes) {
-        qCWarning(dcZigbeeNetwork()) << " ---> " << node << endl;
-        qCWarning(dcZigbeeNetwork()) << "  " << node->nodeDescriptor();
-        qCWarning(dcZigbeeNetwork()) << "  " << node->powerDescriptor();
-        qCWarning(dcZigbeeNetwork()) << "  Endpoints: " << node->endpoints().count() << endl;
-        foreach (ZigbeeNodeEndpoint *endpoint, node->endpoints()) {
-            qCWarning(dcZigbeeNetwork()) << "    - " << endpoint << endl;
-            qCWarning(dcZigbeeNetwork()) << "      Input clusters:" << endl;
-            foreach (ZigbeeCluster *cluster, endpoint->inputClusters()) {
-                qCWarning(dcZigbeeNetwork()) << "      - " << cluster << endl;
-            }
-            qCWarning(dcZigbeeNetwork()) << "      Output clusters:" << endl;
-            foreach (ZigbeeCluster *cluster, endpoint->outputClusters()) {
-                qCWarning(dcZigbeeNetwork()) << "      - " << cluster << endl;
-            }
-        }
-    }
-
 }
 
 void ZigbeeNetwork::clearSettings()
@@ -493,20 +421,6 @@ void ZigbeeNetwork::saveNode(ZigbeeNode *node)
     settings.endGroup(); // Nodes
 }
 
-void ZigbeeNetwork::removeNodeFromSettings(ZigbeeNode *node)
-{
-    qCDebug(dcZigbeeNetwork()) << "Remove node" << node << "from settings" << m_settingsFileName;
-    QSettings settings(m_settingsFileName, QSettings::IniFormat, this);
-    settings.beginGroup("Nodes");
-
-    // Clear settings for this node before storing it
-    settings.beginGroup(node->extendedAddress().toString());
-    settings.remove("");
-    settings.endGroup(); // Node ieee address
-
-    settings.endGroup(); // Nodes
-}
-
 void ZigbeeNetwork::addNode(ZigbeeNode *node)
 {
     qCDebug(dcZigbeeNetwork()) << "Add node" << node;
@@ -515,8 +429,8 @@ void ZigbeeNetwork::addNode(ZigbeeNode *node)
         return;
     }
 
+    m_database->saveNode(node);
     addNodeInternally(node);
-    saveNode(node);
 }
 
 void ZigbeeNetwork::addUnitializedNode(ZigbeeNode *node)
@@ -534,7 +448,7 @@ void ZigbeeNetwork::removeNode(ZigbeeNode *node)
 {
     qCDebug(dcZigbeeNetwork()) << "Remove node" << node;
     removeNodeInternally(node);
-    removeNodeFromSettings(node);
+    m_database->removeNode(node);
 }
 
 void ZigbeeNetwork::setState(ZigbeeNetwork::State state)
@@ -626,11 +540,7 @@ void ZigbeeNetwork::onNodeStateChanged(ZigbeeNode::State state)
 
 void ZigbeeNetwork::onNodeClusterAttributeChanged(ZigbeeCluster *cluster, const ZigbeeClusterAttribute &attribute)
 {
-    Q_UNUSED(cluster)
+    m_database->saveAttribute(cluster, attribute);
-    Q_UNUSED(attribute)
-
-    ZigbeeNode *node = qobject_cast<ZigbeeNode *>(sender());
-    saveNode(node);
 }
 
 QDebug operator<<(QDebug debug, ZigbeeNetwork *network)
@@ -648,10 +558,16 @@ QDebug operator<<(QDebug debug, ZigbeeNetwork *network)
             debug.nospace().noquote() << "      Input clusters:" << endl;
             foreach (ZigbeeCluster *cluster, endpoint->inputClusters()) {
                 debug.nospace().noquote() << "      - " << cluster << endl;
+                foreach (const ZigbeeClusterAttribute &attribute, cluster->attributes()) {
+                    debug.nospace().noquote() << "        - " << attribute << endl;
+                }
             }
             debug.nospace().noquote() << "      Output clusters:" << endl;
             foreach (ZigbeeCluster *cluster, endpoint->outputClusters()) {
                 debug.nospace().noquote() << "      - " << cluster << endl;
+                foreach (const ZigbeeClusterAttribute &attribute, cluster->attributes()) {
+                    debug.nospace().noquote() << "        - " << attribute << endl;
+                }
             }
         }
     }

libnymea-zigbee/zigbeenetwork.h

@@ -155,9 +155,8 @@ protected:
     void saveNetwork();
     void loadNetwork();
     void clearSettings();
-
     void saveNode(ZigbeeNode *node);
-    void removeNodeFromSettings(ZigbeeNode *node);
+
 
     void addNode(ZigbeeNode *node);
     void addUnitializedNode(ZigbeeNode *node);

libnymea-zigbee/zigbeenetworkdatabase.cpp

@@ -70,9 +70,10 @@ QList<ZigbeeNode *> ZigbeeNetworkDatabase::loadNodes()
         // Build the node object
         ZigbeeNode *node = new ZigbeeNode(m_network, shortAddress, ZigbeeAddress(ieeeAddress), m_network);
         node->m_nodeDescriptor = ZigbeeDeviceProfile::parseNodeDescriptor(nodeDescriptor);
+        node->m_macCapabilities = node->nodeDescriptor().macCapabilities;
         node->m_powerDescriptor = ZigbeeDeviceProfile::parsePowerDescriptor(powerDescriptor);
 
-        qCDebug(dcZigbeeNetworkDatabase()) << "Loaded node" << node;
+        qCDebug(dcZigbeeNetworkDatabase()) << "Loaded" << node;
 
         // Now load all endpoints for this node
         query = QString("SELECT * FROM endpoints WHERE ieeeAddress = \"%1\";").arg(ieeeAddress);
@@ -84,7 +85,7 @@ QList<ZigbeeNode *> ZigbeeNetworkDatabase::loadNodes()
             endpoint->setDeviceId(static_cast<Zigbee::ZigbeeProfile>(endpointsQuery.value("deviceId").toUInt()));
             endpoint->setDeviceVersion(static_cast<Zigbee::ZigbeeProfile>(endpointsQuery.value("deviceVersion").toUInt()));
 
-            qCDebug(dcZigbeeNetworkDatabase()) << "Loaded endpoint" << endpoint;
+            qCDebug(dcZigbeeNetworkDatabase()) << "Loaded" << endpoint;
 
             // Load input clusters for this endpoint
             query = QString("SELECT * FROM serverClusters WHERE endpointId = (SELECT id FROM endpoints WHERE ieeeAddress = \"%1\" AND endpointId = \"%2\");")
@@ -96,20 +97,45 @@ QList<ZigbeeNode *> ZigbeeNetworkDatabase::loadNodes()
                 ZigbeeCluster *cluster = endpoint->createCluster(clusterId, ZigbeeCluster::Server);
                 endpoint->addInputCluster(cluster);
 
-                // Load cluster attributes for the server cluster
+                qCDebug(dcZigbeeNetworkDatabase()) << "Loaded" << cluster;
-                query = QString("SELECT * FROM attributes WHERE clusterId = (SELECT id FROM serverClusters WHERE endpointId = (SELECT id FROM endpoints WHERE ieeeAddress = \"%1\" AND endpointId = \"%2\") AND clusterId = \"%3\"));")
+
+                // Load cluster attributes for this server cluster
+                query = QString("SELECT * FROM attributes WHERE clusterId = (SELECT id FROM serverClusters WHERE endpointId = (SELECT id FROM endpoints WHERE ieeeAddress = \"%1\" AND endpointId = \"%2\") AND clusterId = \"%3\");")
                         .arg(ieeeAddress)
                         .arg(endpointId)
                         .arg(cluster->clusterId());
                 QSqlQuery attributesQuery = m_db.exec(query);
+                if (m_db.lastError().type() != QSqlError::NoError) {
+                    qCWarning(dcZigbeeNetworkDatabase()) << "Could not fetch attributes from database entries." << query << m_db.lastError().databaseText() << m_db.lastError().driverText();
+                    continue;
+                }
+
+
                 while (attributesQuery.next()) {
                     quint16 attributeId = attributesQuery.value("attributeId").toUInt();
                     Zigbee::DataType type = static_cast<Zigbee::DataType>(attributesQuery.value("dataType").toUInt());
-                    QByteArray data = attributesQuery.value("data").toByteArray();
+                    QByteArray data = QByteArray::fromBase64(attributesQuery.value("data").toByteArray());
-                    cluster->setAttribute(ZigbeeClusterAttribute(attributeId, ZigbeeDataType(type, data)));
+                    ZigbeeClusterAttribute attribute(attributeId, ZigbeeDataType(type, data));
+                    qCDebug(dcZigbeeNetworkDatabase()) << "Loaded" << attribute;
+                    cluster->setAttribute(attribute);
                 }
             }
 
+            // Set the basic cluster attributes if present
+            if (endpoint->hasInputCluster(Zigbee::ClusterIdBasic)) {
+                ZigbeeClusterBasic *basicCluster = endpoint->inputCluster<ZigbeeClusterBasic>(Zigbee::ClusterIdBasic);
+
+                if (basicCluster->hasAttribute(ZigbeeClusterBasic::AttributeManufacturerName))
+                    endpoint->m_manufacturerName = basicCluster->attribute(ZigbeeClusterBasic::AttributeManufacturerName).dataType().toString();
+
+                if (basicCluster->hasAttribute(ZigbeeClusterBasic::AttributeModelIdentifier))
+                    endpoint->m_modelIdentifier = basicCluster->attribute(ZigbeeClusterBasic::AttributeManufacturerName).dataType().toString();
+
+                if (basicCluster->hasAttribute(ZigbeeClusterBasic::AttributeSwBuildId))
+                    endpoint->m_softwareBuildId = basicCluster->attribute(ZigbeeClusterBasic::AttributeSwBuildId).dataType().toString();
+
+            }
+
             // Load output clusters for this endpoint
             query = QString("SELECT * FROM clientClusters WHERE endpointId = (SELECT id FROM endpoints WHERE ieeeAddress = \"%1\" AND endpointId = \"%2\");")
                     .arg(ieeeAddress)
@@ -118,6 +144,7 @@ QList<ZigbeeNode *> ZigbeeNetworkDatabase::loadNodes()
             while (outputClustersQuery.next()) {
                 Zigbee::ClusterId clusterId = static_cast<Zigbee::ClusterId>(outputClustersQuery.value("clusterId").toUInt());
                 ZigbeeCluster *cluster = endpoint->createCluster(clusterId, ZigbeeCluster::Client);
+                qCDebug(dcZigbeeNetworkDatabase()) << "Loaded" << cluster;
                 endpoint->addOutputCluster(cluster);
             }
 
@@ -129,6 +156,19 @@ QList<ZigbeeNode *> ZigbeeNetworkDatabase::loadNodes()
     return nodes;
 }
 
+bool ZigbeeNetworkDatabase::wipeDatabase()
+{
+    qCDebug(dcZigbeeNetworkDatabase()) << "Wipe all database entries from" << m_db.databaseName();
+    // Note: cascade will clean all other tables
+    m_db.exec("DELETE FROM nodes;");
+    if (m_db.lastError().type() != QSqlError::NoError) {
+        qCWarning(dcZigbeeNetworkDatabase()) << "Could not delete all node database entries." << m_db.lastError().databaseText() << m_db.lastError().driverText();
+        return false;
+    }
+
+    return true;
+}
+
 bool ZigbeeNetworkDatabase::initDatabase()
 {
     // Reopen the db to make sure it can be written
@@ -141,6 +181,8 @@ bool ZigbeeNetworkDatabase::initDatabase()
     // Write pragmas
     m_db.exec("PRAGMA foreign_keys = ON;");
 
+    // FIXME: check schema version
+
     // Create nodes table
     createTable("nodes",
                 "(ieeeAddress INTEGER PRIMARY KEY, " // uint64
@@ -205,7 +247,7 @@ void ZigbeeNetworkDatabase::createIndices(const QString &indexName, const QStrin
 
 bool ZigbeeNetworkDatabase::saveNodeEndpoint(ZigbeeNodeEndpoint *endpoint)
 {
-    qCDebug(dcZigbeeNetworkDatabase()) << "Store" << endpoint;
+    qCDebug(dcZigbeeNetworkDatabase()) << "Save" << endpoint;
     QString queryString = QString("INSERT OR REPLACE INTO endpoints (ieeeAddress, endpointId, profileId, deviceId, deviceVersion) "
                                   "VALUES (\"%1\", \"%2\", \"%3\", \"%4\", \"%5\");")
             .arg(endpoint->node()->extendedAddress().toUInt64())
@@ -244,7 +286,7 @@ bool ZigbeeNetworkDatabase::saveNodeEndpoint(ZigbeeNodeEndpoint *endpoint)
 
 bool ZigbeeNetworkDatabase::saveInputCluster(ZigbeeCluster *cluster)
 {
-    qCDebug(dcZigbeeNetworkDatabase()) << "Store" << cluster;
+    qCDebug(dcZigbeeNetworkDatabase()) << "Save" << cluster;
     QString endpointIdReferenceQuery = QString("(SELECT id FROM endpoints WHERE ieeeAddress = \"%1\" AND endpointId = \"%2\")")
             .arg(cluster->node()->extendedAddress().toUInt64())
             .arg(cluster->endpoint()->endpointId());
@@ -262,7 +304,7 @@ bool ZigbeeNetworkDatabase::saveInputCluster(ZigbeeCluster *cluster)
 
 bool ZigbeeNetworkDatabase::saveOutputCluster(ZigbeeCluster *cluster)
 {
-    qCDebug(dcZigbeeNetworkDatabase()) << "Store" << cluster;
+    qCDebug(dcZigbeeNetworkDatabase()) << "Save" << cluster;
     QString endpointIdReferenceQuery = QString("(SELECT id FROM endpoints WHERE ieeeAddress = \"%1\" AND endpointId = \"%2\")")
             .arg(cluster->node()->extendedAddress().toUInt64())
             .arg(cluster->endpoint()->endpointId());
@@ -281,7 +323,7 @@ bool ZigbeeNetworkDatabase::saveOutputCluster(ZigbeeCluster *cluster)
 
 bool ZigbeeNetworkDatabase::saveAttribute(ZigbeeCluster *cluster, const ZigbeeClusterAttribute &attribute)
 {
-    qCDebug(dcZigbeeNetworkDatabase()) << "Store" << cluster << attribute;
+    qCDebug(dcZigbeeNetworkDatabase()) << "Save" << attribute;
     QString serverClusterIdReferenceQuery = QString("(SELECT id FROM serverClusters "
                                                     "WHERE endpointId = (SELECT id FROM endpoints WHERE ieeeAddress = \"%1\" AND endpointId = \"%2\")"
                                                     "AND clusterId = \"%3\")")
@@ -307,7 +349,7 @@ bool ZigbeeNetworkDatabase::saveAttribute(ZigbeeCluster *cluster, const ZigbeeCl
 
 bool ZigbeeNetworkDatabase::saveNode(ZigbeeNode *node)
 {
-    qCDebug(dcZigbeeNetworkDatabase()) << "Store" << node;
+    qCDebug(dcZigbeeNetworkDatabase()) << "Save" << node;
     QString queryString = QString("INSERT OR REPLACE INTO nodes (ieeeAddress, shortAddress, nodeDescriptor, powerDescriptor) "
                                   "VALUES (\"%1\", \"%2\", \"%3\", \"%4\");")
             .arg(node->extendedAddress().toUInt64())

libnymea-zigbee/zigbeenetworkdatabase.h

@@ -46,6 +46,7 @@ public:
     explicit ZigbeeNetworkDatabase(ZigbeeNetwork *network, const QString &databaseName, QObject *parent = nullptr);
 
     QList<ZigbeeNode *> loadNodes();
+    bool wipeDatabase();
 
 private:
     ZigbeeNetwork *m_network = nullptr;
@@ -55,13 +56,13 @@ private:
     void createTable(const QString &tableName, const QString &schema);
     void createIndices(const QString &indexName, const QString &tableName, const QString &columns);
 
+public slots:
     bool saveNodeEndpoint(ZigbeeNodeEndpoint *endpoint);
     bool saveInputCluster(ZigbeeCluster *cluster);
     bool saveOutputCluster(ZigbeeCluster *cluster);
     bool saveAttribute(ZigbeeCluster *cluster, const ZigbeeClusterAttribute &attribute);
-
-public slots:
     bool saveNode(ZigbeeNode *node);
+
     bool removeNode(ZigbeeNode *node);
 
 };