Add mac address database and build tool

This commit is contained in:
Simon Stürz 2021-06-05 22:56:19 +02:00
parent 08aae83a00
commit 60de7e5c45
13 changed files with 482 additions and 32 deletions

3
.gitignore vendored
View File

@ -9,6 +9,9 @@ snap/prime/
snap/stage/
doc/interfacelist.qdoc
# Never add downloaded json data
data/mac-database/macaddress.io-db.json
.crossbuilder/
debs*.tar

View File

@ -0,0 +1,16 @@
# MAC address dabase
The MAC address database has been created using the dowloaded JSON data from [https://macaddress.io](https://macaddress.io).
## Build database
Before you can start the `build-database.py` script, please make sure you downloaded the the online database in JSON format.
Once the `macaddress.io-db.json` file has been downloaded and placed into this folder, the python tool can be started in order to generate a read performance optimized and minimal database for searching mac address OUIs (Organizationally Unique Identifiers).
$ python3 build-database.py
The final database will be named `mac-addresses.db`.
In nymea the `MacAddressDatabase` will search for this database file in `/usr/share/nymea/mac-addresses.db` and provides an asynch mechanism to provide the manufacturer name for a given mac address.

View File

@ -0,0 +1,88 @@
#!/usr/bin/env python
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Copyright 2013 - 2021, nymea GmbH
# Contact: contact@nymea.io
#
# This file is part of nymea.
# This project including source code and documentation is protected by
# copyright law, and remains the property of nymea GmbH. All rights, including
# reproduction, publication, editing and translation, are reserved. The use of
# this project is subject to the terms of a license agreement to be concluded
# with nymea GmbH in accordance with the terms of use of nymea GmbH, available
# under https://nymea.io/license
#
# GNU Lesser General Public License Usage
# Alternatively, this project may be redistributed and/or modified under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; version 3. This project is distributed in the hope that
# it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this project. If not, see <https://www.gnu.org/licenses/>.
#
# For any further details and any questions please contact us under
# contact@nymea.io or see our FAQ/Licensing Information on
# https://nymea.io/license/faq
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
import os
import sys
import json
import sqlite3
jsonDataFileName = 'macaddress.io-db.json'
print('Loading JSON data from', jsonDataFileName)
jsonDataFile = open(jsonDataFileName, 'r')
lines = jsonDataFile.readlines()
vendorInfoHash = {}
for line in lines:
vendorMap = json.loads(line)
vendorInfoHash[vendorMap['oui']] = vendorMap['companyName']
jsonDataFile.close()
# Sort by oui for good binary search in the db
sortedOuiHash = sorted(vendorInfoHash.items(), key=lambda x: x[0], reverse=False)
databaseFileName = 'mac-addresses.db'
if os.path.exists(databaseFileName):
print('Delete old db file and create a new one', databaseFileName)
os.remove(databaseFileName)
connection = sqlite3.connect(databaseFileName)
cursor = connection.cursor()
cursor.execute('CREATE TABLE companyNames (companyName TEXT PRIMARY KEY, UNIQUE(companyName));')
cursor.execute('CREATE TABLE oui (oui TEXT PRIMARY KEY, companyNameIndex INTEGER, UNIQUE(oui)) WITHOUT ROWID;')
#cursor.execute('CREATE UNIQUE INDEX ouiIndex ON `oui` (`oui`);')
# Insert all vendor names alphabetically
sortedVendorHash = sorted(vendorInfoHash.items(), key=lambda x: x[1], reverse=False)
vendorCount = 0
for vendorInfo in sortedVendorHash:
insertQuery = 'INSERT OR IGNORE INTO companyNames (companyName) VALUES(?);'
cursor.execute(insertQuery, [vendorInfo[1]])
cursor.execute('SELECT COUNT(companyName) FROM companyNames');
countResult = cursor.fetchall()
vendorCount = countResult[0][0]
connection.commit()
# Insert all oui with reference to company name
ouiCount = 0
for vendorInfo in sortedOuiHash:
insertQuery = 'INSERT OR IGNORE INTO oui (oui, companyNameIndex) VALUES(?, (SELECT rowid FROM companyNames WHERE companyName = ?));'
cursor.execute(insertQuery, [vendorInfo[0].replace(':', ''), vendorInfo[1]])
cursor.execute('SELECT COUNT(oui) FROM oui');
countResult = cursor.fetchall()
ouiCount = countResult[0][0]
connection.commit()
connection.close()
print('Loaded', ouiCount, 'OUI values from', vendorCount, 'manufacturers into', databaseFileName)

Binary file not shown.

View File

@ -3,8 +3,9 @@ include(../nymea.pri)
TARGET = nymea
TEMPLATE = lib
QT += network bluetooth dbus serialport
QT += network bluetooth dbus serialport sql
QT -= gui
DEFINES += LIBNYMEA_LIBRARY
CONFIG += link_pkgconfig
@ -43,6 +44,7 @@ HEADERS += \
network/apikeys/apikeysprovider.h \
network/apikeys/apikeystorage.h \
network/arpsocket.h \
network/macaddressdatabase.h \
network/networkdevice.h \
network/networkdevicediscovery.h \
network/networkdevicediscoveryreply.h \
@ -149,6 +151,7 @@ SOURCES += \
network/apikeys/apikeysprovider.cpp \
network/apikeys/apikeystorage.cpp \
network/arpsocket.cpp \
network/macaddressdatabase.cpp \
network/networkdevice.cpp \
network/networkdevicediscovery.cpp \
network/networkdevicediscoveryreply.cpp \

View File

@ -0,0 +1,200 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2021, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU Lesser General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU Lesser General Public License as published by the Free
* Software Foundation; version 3. This project is distributed in the hope that
* it will be useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include "macaddressdatabase.h"
#include "loggingcategories.h"
#include <QSqlQuery>
#include <QSqlError>
#include <QFileInfo>
#include <QTimer>
#include <QSqlDatabase>
#include <QtConcurrent/QtConcurrent>
NYMEA_LOGGING_CATEGORY(dcMacAddressDatabase, "MacAddressDatabase")
MacAddressDatabase::MacAddressDatabase(QObject *parent) : QObject(parent)
{
m_available = initDatabase();
if (m_available) {
m_futureWatcher = new QFutureWatcher<QString>(this);
connect(m_futureWatcher, &QFutureWatcher<QString>::finished, this, &MacAddressDatabase::onLookupFinished);
}
}
MacAddressDatabase::MacAddressDatabase(const QString &databaseName, QObject *parent) :
QObject(parent),
m_databaseName(databaseName)
{
m_available = initDatabase();
if (m_available) {
m_futureWatcher = new QFutureWatcher<QString>(this);
connect(m_futureWatcher, &QFutureWatcher<QString>::finished, this, &MacAddressDatabase::onLookupFinished);
}
}
MacAddressDatabase::~MacAddressDatabase()
{
m_db.close();
m_db = QSqlDatabase();
QSqlDatabase::removeDatabase(m_connectionName);
}
bool MacAddressDatabase::available() const
{
return m_available;
}
MacAddressDatabaseReply *MacAddressDatabase::lookupMacAddress(const QString &macAddress)
{
MacAddressDatabaseReply *reply = new MacAddressDatabaseReply(this);
connect(reply, &MacAddressDatabaseReply::finished, reply, &MacAddressDatabaseReply::deleteLater);
reply->m_macAddress = macAddress;
if (!m_available) {
QTimer::singleShot(0, this, [=](){ emit reply->finished(); });
return reply;
}
m_pendingReplies.enqueue(reply);
runNextLookup();
return reply;
}
bool MacAddressDatabase::initDatabase()
{
m_connectionName = QFileInfo(m_databaseName).baseName();
m_db = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), m_connectionName);
m_db.setDatabaseName(m_databaseName);
if (!m_db.isValid()) {
qCWarning(dcMacAddressDatabase()) << "The network database is not valid" << m_db.databaseName();
return false;
}
m_db.close();
if (!m_db.open()) {
qCWarning(dcMacAddressDatabase()) << "Could not open database" << m_db.databaseName() << "Initialization failed.";
return false;
}
// Verify the tables we need exist
qCDebug(dcMacAddressDatabase()) << "Tables" << m_db.tables();
if (!m_db.tables().contains("oui")) {
qCWarning(dcMacAddressDatabase()) << "Invalid database. Could not find \"oui\" table in" << m_db.databaseName();
return false;
}
if (!m_db.tables().contains("companyNames")) {
qCWarning(dcMacAddressDatabase()) << "Invalid database. Could not find \"companyNames\" table in" << m_db.databaseName();
return false;
}
return true;
}
void MacAddressDatabase::runNextLookup()
{
if (m_pendingReplies.isEmpty())
return;
if (m_futureWatcher->isRunning() || m_currentReply)
return;
m_currentReply = m_pendingReplies.dequeue();
m_currentReply->m_startTimestamp = QDateTime::currentMSecsSinceEpoch();
QFuture<QString> future = QtConcurrent::run(this, &MacAddressDatabase::lookupMacAddressVendorInternal, m_currentReply->macAddress());
m_futureWatcher->setFuture(future);
}
void MacAddressDatabase::onLookupFinished()
{
if (m_currentReply) {
QString manufacturer = m_futureWatcher->future().result();
qCDebug(dcMacAddressDatabase()) << "Manufacturer lookup for" << m_currentReply->macAddress() << "finished:" << manufacturer << QDateTime::currentMSecsSinceEpoch() - m_currentReply->m_startTimestamp << "ms";
m_currentReply->m_manufacturer = manufacturer;
emit m_currentReply->finished();
m_currentReply = nullptr;
}
runNextLookup();
}
QString MacAddressDatabase::lookupMacAddressVendorInternal(const QString &macAddress)
{
qCDebug(dcMacAddressDatabase()) << "Start looking up vendor for" << macAddress;
// Convert the mac address string to upper like in the database and remove : since they have been removed for size reasons
QString fullMacAddressString = QString(macAddress).toUpper().remove(":");
QString manufacturer;
int length = 6;
while (true) {
QString searchString = fullMacAddressString.left(length);
QString queryString = QString("SELECT COUNT(oui) FROM oui WHERE oui LIKE \'%1%\';").arg(searchString);
qCDebug(dcMacAddressDatabase()) << "Query:" << queryString;
QSqlQuery countQuery = m_db.exec(queryString);
if (countQuery.lastError().isValid()) {
qCWarning(dcMacAddressDatabase()) << "Query finished with error" << countQuery.lastError().text();
break;
}
if (!countQuery.next())
break;
int rowCount = countQuery.value(0).toInt();
qCDebug(dcMacAddressDatabase()) << "Found" << rowCount << "with" << searchString;
// If we have found the one...
if (rowCount == 1) {
// Query the name
queryString = QString("SELECT companyName from companyNames WHERE rowid IS (SELECT companyNameIndex FROM oui WHERE oui=\'%1\');").arg(searchString);
qCDebug(dcMacAddressDatabase()) << "Query:" << queryString;
countQuery = m_db.exec(queryString);
if (!countQuery.next())
break;
manufacturer = countQuery.value(0).toString();
break;
}
// If nothing found
if (rowCount == 0)
break;
// Found to many results, lets add a value until we find the matching vendor
length += 1;
if (length > fullMacAddressString)
break;
// Search with one addition digit
}
return manufacturer;
}

View File

@ -0,0 +1,93 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
*
* Copyright 2013 - 2021, nymea GmbH
* Contact: contact@nymea.io
*
* This file is part of nymea.
* This project including source code and documentation is protected by
* copyright law, and remains the property of nymea GmbH. All rights, including
* reproduction, publication, editing and translation, are reserved. The use of
* this project is subject to the terms of a license agreement to be concluded
* with nymea GmbH in accordance with the terms of use of nymea GmbH, available
* under https://nymea.io/license
*
* GNU Lesser General Public License Usage
* Alternatively, this project may be redistributed and/or modified under the
* terms of the GNU Lesser General Public License as published by the Free
* Software Foundation; version 3. This project is distributed in the hope that
* it will be useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this project. If not, see <https://www.gnu.org/licenses/>.
*
* For any further details and any questions please contact us under
* contact@nymea.io or see our FAQ/Licensing Information on
* https://nymea.io/license/faq
*
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#ifndef MACADDRESSDATABASE_H
#define MACADDRESSDATABASE_H
#include <QQueue>
#include <QObject>
#include <QSqlDatabase>
#include <QFutureWatcher>
#include "libnymea.h"
class LIBNYMEA_EXPORT MacAddressDatabaseReply : public QObject
{
Q_OBJECT
friend class MacAddressDatabase;
public:
QString macAddress() const { return m_macAddress; };
QString manufacturer() const { return m_manufacturer; };
private:
explicit MacAddressDatabaseReply(QObject *parent = nullptr) : QObject(parent) { };
QString m_macAddress;
QString m_manufacturer;
qint64 m_startTimestamp;
signals:
void finished();
};
class LIBNYMEA_EXPORT MacAddressDatabase : public QObject
{
Q_OBJECT
public:
explicit MacAddressDatabase(QObject *parent = nullptr);
MacAddressDatabase(const QString &databaseName, QObject *parent = nullptr);
~MacAddressDatabase();
bool available() const;
MacAddressDatabaseReply *lookupMacAddress(const QString &macAddress);
private:
QSqlDatabase m_db;
bool m_available = false;
QString m_connectionName;
QString m_databaseName = "/usr/share/nymea/mac-addresses.db";
MacAddressDatabaseReply *m_currentReply = nullptr;
QFutureWatcher<QString> *m_futureWatcher = nullptr;
QQueue<MacAddressDatabaseReply *> m_pendingReplies;
bool initDatabase();
void runNextLookup();
private slots:
void onLookupFinished();
QString lookupMacAddressVendorInternal(const QString &macAddress);
};
#endif // MACADDRESSDATABASE_H

View File

@ -31,6 +31,9 @@
#include "networkdevicediscovery.h"
#include "loggingcategories.h"
#include "networkutils.h"
#include "macaddressdatabase.h"
#include <QDateTime>
NYMEA_LOGGING_CATEGORY(dcNetworkDeviceDiscovery, "NetworkDeviceDiscovery")
@ -49,6 +52,10 @@ NetworkDeviceDiscovery::NetworkDeviceDiscovery(QObject *parent) :
if (!m_ping->available())
qCWarning(dcNetworkDeviceDiscovery()) << "Failed to create ping tool" << m_ping->error();
m_macAddressDatabase = new MacAddressDatabase(this);
if (!m_macAddressDatabase->available())
qCWarning(dcNetworkDeviceDiscovery()) << "The mac address database is not available. Network discovery will not lookup mac address manufacturer";
m_discoveryTimer = new QTimer(this);
m_discoveryTimer->setInterval(5000);
m_discoveryTimer->setSingleShot(true);
@ -72,6 +79,7 @@ NetworkDeviceDiscoveryReply *NetworkDeviceDiscovery::discover()
NetworkDeviceDiscoveryReply *reply = new NetworkDeviceDiscoveryReply(this);
m_currentReply = reply;
m_currentReply->m_startTimestamp = QDateTime::currentMSecsSinceEpoch();
if (m_ping->available()) {
pingAllNetworkDevices();
@ -179,13 +187,53 @@ void NetworkDeviceDiscovery::finishDiscovery()
m_running = false;
emit runningChanged(m_running);
qCDebug(dcNetworkDeviceDiscovery()) << "Discovery finished. Found" << m_currentReply->networkDevices().count() << "network devices.";
qint64 durationMilliSeconds = QDateTime::currentMSecsSinceEpoch() - m_currentReply->m_startTimestamp;
qCDebug(dcNetworkDeviceDiscovery()) << "Discovery finished. Found" << m_currentReply->networkDevices().count() << "network devices in" << QTime::fromMSecsSinceStartOfDay(durationMilliSeconds).toString("mm:ss.zzz");
m_arpSocket->closeSocket();
emit m_currentReply->finished();
m_currentReply->deleteLater();
m_currentReply = nullptr;
}
void NetworkDeviceDiscovery::updateOrAddNetworkDeviceArp(const QNetworkInterface &interface, const QHostAddress &address, const QString &macAddress, const QString &manufacturer)
{
int index = m_currentReply->networkDevices().indexFromHostAddress(address);
if (index >= 0) {
// Update the network device
m_currentReply->networkDevices()[index].setMacAddress(macAddress);
if (!manufacturer.isEmpty())
m_currentReply->networkDevices()[index].setMacAddressManufacturer(manufacturer);
if (interface.isValid()) {
m_currentReply->networkDevices()[index].setNetworkInterface(interface);
}
} else {
index = m_currentReply->networkDevices().indexFromMacAddress(macAddress);
if (index >= 0) {
// Update the network device
m_currentReply->networkDevices()[index].setAddress(address);
if (!manufacturer.isEmpty())
m_currentReply->networkDevices()[index].setMacAddressManufacturer(manufacturer);
if (interface.isValid()) {
m_currentReply->networkDevices()[index].setNetworkInterface(interface);
}
} else {
// Add the network device
NetworkDevice networkDevice;
networkDevice.setAddress(address);
networkDevice.setMacAddress(macAddress);
if (!manufacturer.isEmpty())
networkDevice.setMacAddressManufacturer(manufacturer);
if (interface.isValid())
networkDevice.setNetworkInterface(interface);
m_currentReply->networkDevices().append(networkDevice);
}
}
}
void NetworkDeviceDiscovery::onArpResponseRceived(const QNetworkInterface &interface, const QHostAddress &address, const QString &macAddress)
{
if (!m_currentReply) {
@ -194,31 +242,15 @@ void NetworkDeviceDiscovery::onArpResponseRceived(const QNetworkInterface &inter
}
qCDebug(dcNetworkDeviceDiscovery()) << "ARP reply received" << address.toString() << macAddress << interface.name();
// Update or add network device
int index = m_currentReply->networkDevices().indexFromHostAddress(address);
if (index >= 0) {
// Update the network device
m_currentReply->networkDevices()[index].setMacAddress(macAddress);
if (interface.isValid()) {
m_currentReply->networkDevices()[index].setNetworkInterface(interface);
}
} else {
index = m_currentReply->networkDevices().indexFromMacAddress(macAddress);
if (index >= 0) {
// Update the network device
m_currentReply->networkDevices()[index].setAddress(address);
if (interface.isValid()) {
m_currentReply->networkDevices()[index].setNetworkInterface(interface);
}
} else {
// Add the network device
NetworkDevice networkDevice;
networkDevice.setAddress(address);
networkDevice.setMacAddress(macAddress);
if (interface.isValid())
networkDevice.setNetworkInterface(interface);
m_currentReply->networkDevices().append(networkDevice);
}
// Lookup the mac address vendor if possible
if (m_macAddressDatabase->available()) {
MacAddressDatabaseReply *reply = m_macAddressDatabase->lookupMacAddress(macAddress);
connect(reply, &MacAddressDatabaseReply::finished, this, [=](){
qCDebug(dcNetworkDeviceDiscovery()) << "MAC manufacturer lookup finished for" << macAddress << ":" << reply->manufacturer();
updateOrAddNetworkDeviceArp(interface, address, macAddress, reply->manufacturer());
});
} else {
updateOrAddNetworkDeviceArp(interface, address, macAddress);
}
}

View File

@ -40,6 +40,8 @@
#include "arpsocket.h"
#include "networkdevicediscoveryreply.h"
class MacAddressDatabase;
Q_DECLARE_LOGGING_CATEGORY(dcNetworkDeviceDiscovery)
class LIBNYMEA_EXPORT NetworkDeviceDiscovery : public QObject
@ -59,6 +61,7 @@ signals:
void runningChanged(bool running);
private:
MacAddressDatabase *m_macAddressDatabase = nullptr;
ArpSocket *m_arpSocket = nullptr;
Ping *m_ping = nullptr;
bool m_running = false;
@ -70,6 +73,8 @@ private:
void pingAllNetworkDevices();
void finishDiscovery();
void updateOrAddNetworkDeviceArp(const QNetworkInterface &interface, const QHostAddress &address, const QString &macAddress, const QString &manufacturer = QString());
private slots:
void onArpResponseRceived(const QNetworkInterface &interface, const QHostAddress &address, const QString &macAddress);

View File

@ -43,15 +43,15 @@ class LIBNYMEA_EXPORT NetworkDeviceDiscoveryReply : public QObject
friend class NetworkDeviceDiscovery;
public:
explicit NetworkDeviceDiscoveryReply(QObject *parent = nullptr);
NetworkDevices &networkDevices();
signals:
void finished();
private:
explicit NetworkDeviceDiscoveryReply(QObject *parent = nullptr);
NetworkDevices m_networkDevices;
qint64 m_startTimestamp;
};

View File

@ -200,6 +200,10 @@ void Ping::performPing(PingReply *reply)
// Start reply timer and handle timeout
m_pendingReplies.insert(reply->requestId(), reply);
reply->m_timer->start(8000);
connect(reply, &PingReply::timeout, this, [=](){
finishReply(reply, PingReply::ErrorTimeout);
});
}
void Ping::verifyErrno(int error)
@ -336,7 +340,7 @@ void Ping::onSocketReadyRead(int socketDescriptor)
// Verify sequence number
if (responsePacket->icmp_seq != reply->sequenceNumber()) {
qCWarning(dcPing()) << "Received echo reply with different sequence number" << htons(responsePacket->icmp_seq);
finishReply(reply, PingReply::ErrorInvalidSequenceNumberResponse);
finishReply(reply, PingReply::ErrorInvalidResponse);
return;
}

View File

@ -32,7 +32,10 @@
PingReply::PingReply(QObject *parent) : QObject(parent)
{
m_timer = new QTimer(this);
m_timer->setSingleShot(true);
connect(m_timer, &QTimer::timeout, this, &PingReply::timeout);
connect(this, &PingReply::finished, m_timer, &QTimer::stop);
}
QHostAddress PingReply::targetHostAddress() const

View File

@ -51,11 +51,12 @@ class LIBNYMEA_EXPORT PingReply : public QObject
public:
enum Error {
ErrorNoError,
ErrorInvalidSequenceNumberResponse,
ErrorInvalidResponse,
ErrorNetworkDown,
ErrorNetworkUnreachable,
ErrorPermissionDenied,
ErrorSocketError,
ErrorTimeout,
ErrorHostUnreachable
};
Q_ENUM(Error)
@ -74,8 +75,10 @@ public:
signals:
void finished();
void timeout();
private:
QTimer *m_timer = nullptr;
QHostAddress m_targetHostAddress;
quint16 m_sequenceNumber = 0;
quint16 m_requestId = 0;