diff --git a/data/mac-database/README.md b/data/mac-database/README.md index dc2f77bd..b353ba07 100644 --- a/data/mac-database/README.md +++ b/data/mac-database/README.md @@ -1,8 +1,14 @@ # Building the MAC address database The MAC address database can be created using the `build-database.py` script. + +There are 2 online services supported: + +* [https://maclookup.app](https://maclookup.app) default +* [https://macaddress.io](https://macaddress.io) not free any more, but still supported + The script will download the latest registered MAC address block information -from [https://macaddress.io](https://macaddress.io) and creates a size and access optimized +from [https://maclookup.app](https://maclookup.app) and creates a size and access optimized SQLITE database file. The generated database is read performance optimized and tried to keep as small as possible for @@ -12,8 +18,7 @@ searching MAC address OUIs (Organizationally Unique Identifiers) blocks and retu The final database will be named `mac-addresses.db`. -In nymea the `MacAddressDatabase` class will provide access to this generated database and provides an asynch threaded mechanism to -get the company name for a given MAC address. +In nymea the `MacAddressDatabase` class will provide access to this generated database and provides an asynch threaded mechanism to get the company name for a given MAC address. The database will be searched in the system default data location `${XDG_DATA_DIRS}/nymead/`. diff --git a/data/mac-database/build-database.py b/data/mac-database/build-database.py index 193457de..fd261145 100644 --- a/data/mac-database/build-database.py +++ b/data/mac-database/build-database.py @@ -2,7 +2,7 @@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# Copyright 2013 - 2021, nymea GmbH +# Copyright 2013 - 2022, nymea GmbH # Contact: contact@nymea.io # # This file is part of nymea. @@ -35,24 +35,56 @@ import sys import json import sqlite3 import requests +import argparse -downloadUrl='https://macaddress.io/database/macaddress.io-db.json' -jsonDataFileName = 'macaddress.io-db.json' +# Note: this db is no longer free databaseFileName = 'mac-addresses.db' -print('Downloading', downloadUrl, '...') -downloadRequest = requests.get(downloadUrl) -open(jsonDataFileName, 'wb').write(downloadRequest.content) +parser = argparse.ArgumentParser(description='Build mac address database from source.') +parser.add_argument('--macaddressio', action='store_true', help='Download and generate using macaddress.io as source (note: not free any more)') + +args = parser.parse_args() -print('Reading JSON data..') vendorInfoHash = {} -jsonDataFile = open(jsonDataFileName, 'r') -lines = jsonDataFile.readlines() -for line in lines: - vendorMap = json.loads(line) - vendorInfoHash[vendorMap['oui']] = vendorMap['companyName'] -jsonDataFile.close() +if args.macaddressio: + # Deprecated: not free any more, but would still work if you buy the db + downloadUrl='https://macaddress.io/database/macaddress.io-db.json' + jsonDataFileName = 'macaddress.io-db.json' + print('Downloading', downloadUrl, '...') + downloadRequest = requests.get(downloadUrl) + open(jsonDataFileName, 'wb').write(downloadRequest.content) + + print('Reading JSON data..') + jsonDataFile = open(jsonDataFileName, 'r') + lines = jsonDataFile.readlines() + for line in lines: + vendorMap = json.loads(line) + vendorInfoHash[vendorMap['oui']] = vendorMap['companyName'] + + jsonDataFile.close() + +else: + downloadUrl="https://maclookup.app/downloads/json-database/get-db" + jsonDataFileName = 'mac-vendors-export.json' + print('Downloading', downloadUrl, '...') + downloadRequest = requests.get(downloadUrl) + open(jsonDataFileName, 'wb').write(downloadRequest.content) + + if not os.path.exists(jsonDataFileName): + print('Could not find %s. Please download the json data from https://maclookup.app/downloads/json-database and place it into this folder.') + + print('Reading JSON data..') + # Example: {"macPrefix":"00:00:0C","vendorName":"Cisco Systems, Inc","private":false,"blockType":"MA-L","lastUpdate":"2015/11/17"} + + jsonDataFile = open(jsonDataFileName, 'r') + jsonData = json.load(jsonDataFile) + jsonDataFile.close() + for vendorMap in jsonData: + vendorInfoHash[vendorMap['macPrefix']] = vendorMap['vendorName'] + print(vendorMap['macPrefix'], vendorMap['vendorName']) + + if os.path.exists(databaseFileName): print('Delete old db file', databaseFileName) @@ -72,7 +104,7 @@ vendorCount = 0 for vendorInfo in sortedVendorHash: insertQuery = 'INSERT OR IGNORE INTO companyNames (companyName) VALUES(?);' cursor.execute(insertQuery, [vendorInfo[1]]) - cursor.execute('SELECT COUNT(companyName) FROM companyNames'); + cursor.execute('SELECT COUNT(companyName) FROM companyNames;') countResult = cursor.fetchall() vendorCount = countResult[0][0] @@ -86,12 +118,10 @@ ouiCount = 0 for vendorInfo in sortedOuiHash: insertQuery = 'INSERT OR IGNORE INTO oui (oui, companyNameIndex) VALUES(?, (SELECT rowid FROM companyNames WHERE companyName = ?));' cursor.execute(insertQuery, [vendorInfo[0].replace(':', ''), vendorInfo[1]]) - cursor.execute('SELECT COUNT(oui) FROM oui'); + cursor.execute('SELECT COUNT(oui) FROM oui;') countResult = cursor.fetchall() ouiCount = countResult[0][0] connection.commit() connection.close() -print('Finished successfully. Loaded', ouiCount, 'OUI values from', vendorCount, 'manufacturers into', databaseFileName) - - +print('Finished successfully. Loaded', ouiCount, 'OUI values from', vendorCount, 'manufacturers into', databaseFileName) \ No newline at end of file diff --git a/data/mac-database/mac-addresses.db b/data/mac-database/mac-addresses.db index 913c13f1..23289571 100644 Binary files a/data/mac-database/mac-addresses.db and b/data/mac-database/mac-addresses.db differ