From d17b44c83d58479fd7d34a7127b8850521f9363d Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Wed, 27 May 2020 00:07:16 +0200 Subject: [PATCH 01/46] Initial attempt to support python plugins --- .../integrations/pythonintegrationplugin.cpp | 180 ++++++++++++++++++ .../integrations/pythonintegrationplugin.h | 34 ++++ .../thingmanagerimplementation.cpp | 15 ++ libnymea-core/jsonrpc/devicehandler.cpp | 1 + libnymea-core/libnymea-core.pro | 4 +- nymea.pro | 1 - 6 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 libnymea-core/integrations/pythonintegrationplugin.cpp create mode 100644 libnymea-core/integrations/pythonintegrationplugin.h diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp new file mode 100644 index 00000000..ea3885fa --- /dev/null +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -0,0 +1,180 @@ +#include + +#include "pythonintegrationplugin.h" + +#include "loggingcategories.h" + +#include + +PyObject* aview_write(PyObject* /*self*/, PyObject* args) +{ + const char *what; + if (!PyArg_ParseTuple(args, "s", &what)) + return nullptr; + qCDebug(dcThingManager()) << what; + return Py_BuildValue(""); +} + + +PyObject* aview_flush(PyObject* /*self*/, PyObject* /*args*/) +{ + return Py_BuildValue(""); +} + +static PyMethodDef aview_methods[] = +{ + {"write", aview_write, METH_VARARGS, "doc for write"}, + {"flush", aview_flush, METH_VARARGS, "doc for flush"}, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyModuleDef aview_module = +{ + PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; + "aview", // const char* m_name; + "doc for aview", // const char* m_doc; + -1, // Py_ssize_t m_size; + aview_methods, // PyMethodDef *m_methods + // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; + nullptr, nullptr, nullptr, nullptr +}; + +PyMODINIT_FUNC PyInit_aview(void) +{ + PyObject* m = PyModule_Create(&aview_module); + PySys_SetObject("stdout", m); + PySys_SetObject("stderr", m); + return m; +} + +PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationPlugin(parent) +{ + +} + +bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) +{ + // TODO: Call this just once and call Py_Finalize() + PyImport_AppendInittab("aview", PyInit_aview); + Py_Initialize(); + PyImport_ImportModule("aview"); + + QFileInfo fi(scriptFile); + qCDebug(dcThingManager()) << "Importing" << fi.absolutePath() << fi.fileName() << fi.baseName(); + + PyObject* sysPath = PySys_GetObject("path"); + PyList_Append(sysPath, PyUnicode_FromString(fi.absolutePath().toUtf8())); + + + PyObject *pName = PyUnicode_FromString(fi.baseName().toUtf8()); + qCDebug(dcThingManager()) << "Importing python plugin from" << scriptFile; + m_module = PyImport_Import(pName); + + if (!m_module) { + qCWarning(dcThingManager()) << "Error importing plugin"; + return false; + } + return true; +} + +QJsonObject PythonIntegrationPlugin::metaData() const +{ + QVariantMap pluginMetaData; + pluginMetaData.insert("id", "ccc6dbc8-e352-48a1-8e87-3c89a4669fc2"); + pluginMetaData.insert("name", "CloudNotifications"); + pluginMetaData.insert("displayName", tr("Cloud Notifications")); + + QVariantList interfaces; +// interfaces.append("notifications"); + interfaces.append("connectable"); + + QVariantList createMethods; + createMethods.append("discovery"); + + QVariantMap testActionParamTitle; + testActionParamTitle.insert("id", "c9545e1c-55cd-42ca-a00f-43f21dfdf05a"); + testActionParamTitle.insert("name", "title"); + testActionParamTitle.insert("displayName", tr("Title")); + testActionParamTitle.insert("type", "QString"); + + QVariantList notifyActionParamTypes; + notifyActionParamTypes.append(testActionParamTitle); + + QVariantMap notifyAction; + notifyAction.insert("id", "cc6ad463-0a63-4570-ae13-956f50faa3a6"); + notifyAction.insert("name", "notify"); + notifyAction.insert("displayName", tr("Send notification")); + notifyAction.insert("paramTypes", notifyActionParamTypes); + + QVariantList actionTypes; + actionTypes.append(notifyAction); + + QVariantMap connectedState; + connectedState.insert("id", "292b5c5d-a6fc-43b6-a59e-f3e1a3ab42b4"); + connectedState.insert("name", "connected"); + connectedState.insert("displayName", tr("connected")); + connectedState.insert("type", "bool"); + connectedState.insert("displayNameEvent", tr("Connected changed")); + connectedState.insert("defaultValue", false); + + QVariantList stateTypes; + stateTypes.append(connectedState); + + + QVariantMap cloudNotificationsThingClass; + cloudNotificationsThingClass.insert("id", "0f1a441a-3793-4a1c-91fc-35d752443aff"); + cloudNotificationsThingClass.insert("name", "PyTest"); + cloudNotificationsThingClass.insert("displayName", tr("Python test")); + cloudNotificationsThingClass.insert("createMethods", createMethods); + cloudNotificationsThingClass.insert("interfaces", interfaces); + cloudNotificationsThingClass.insert("actionTypes", actionTypes); + cloudNotificationsThingClass.insert("stateTypes", stateTypes); + + QVariantList thingClasses; + thingClasses.append(cloudNotificationsThingClass); + + QVariantMap guhVendor; + guhVendor.insert("id", "2062d64d-3232-433c-88bc-0d33c0ba2ba6"); // nymea's id + guhVendor.insert("name", "nymea"); + guhVendor.insert("displayName", "nymea"); + guhVendor.insert("thingClasses", thingClasses); + + QVariantList vendors; + vendors.append(guhVendor); + pluginMetaData.insert("vendors", vendors); + + return QJsonObject::fromVariantMap(pluginMetaData); + +} + +void PythonIntegrationPlugin::init() +{ + qCDebug(dcThingManager()) << "Python wrapper: init()"; + PyObject *pFunc = PyObject_GetAttrString(m_module, "init"); + if(!pFunc || !PyCallable_Check(pFunc)) { + qCDebug(dcThingManager()) << "Python plugin does not implement \"init()\" method."; + return; + } + PyObject_CallObject(pFunc, nullptr); +} + +void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) +{ + qCDebug(dcThingManager()) << "Python wrapper: discoverThings()" << info; + PyObject *pFunc = PyObject_GetAttrString(m_module, "init"); + if(!pFunc || !PyCallable_Check(pFunc)) { + qCWarning(dcThingManager()) << "Python plugin does not implement \"discoverThings()\" method."; + return; + } +// PyObject* args = Py_BuildValue("(s)", ln.c_str()); +// if (!args) +// { +// PyErr_Print(); +// Py_DECREF(filterFunc); +// return "Error building args tuple"; +// } + +// PyObject_CallObject(pFunc, nullptr); + +} + diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h new file mode 100644 index 00000000..562ec5a4 --- /dev/null +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -0,0 +1,34 @@ +#ifndef PYTHONINTEGRATIONPLUGIN_H +#define PYTHONINTEGRATIONPLUGIN_H + +#include "integrations/integrationplugin.h" + +#include +#include + +extern "C" { +typedef struct _object PyObject; +} + + +class PythonIntegrationPlugin : public IntegrationPlugin +{ + Q_OBJECT +public: + explicit PythonIntegrationPlugin(QObject *parent = nullptr); + + bool loadScript(const QString &scriptFile); + + QJsonObject metaData() const; + + + void init() override; + void discoverThings(ThingDiscoveryInfo *info) override; + + +private: + PyObject *m_module = nullptr; + +}; + +#endif // PYTHONINTEGRATIONPLUGIN_H diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index f8bf1184..9f8bfa66 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -33,6 +33,7 @@ #if QT_VERSION >= QT_VERSION_CHECK(5,12,0) #include "scriptintegrationplugin.h" #endif +#include "pythonintegrationplugin.h" #include "loggingcategories.h" #include "typeutils.h" @@ -1333,12 +1334,26 @@ void ThingManagerImplementation::loadPlugins() qCWarning(dcThingManager()) << "Not loading JS plugin. Invalid metadata."; foreach (const QString &error, metaData.validationErrors()) { qCWarning(dcThingManager()) << error; + delete plugin; + continue; } } loadPlugin(plugin, metaData); } } #endif + + PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); + p->loadScript("/home/micha/Develop/nymea-plugin-pytest/pytest.py"); + PluginMetadata metaData(p->metaData()); + if (!metaData.isValid()) { + qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; + foreach (const QString &error, metaData.validationErrors()) { + qCWarning(dcThingManager()) << error; + } + return; + } + loadPlugin(p, metaData); } void ThingManagerImplementation::loadPlugin(IntegrationPlugin *pluginIface, const PluginMetadata &metaData) diff --git a/libnymea-core/jsonrpc/devicehandler.cpp b/libnymea-core/jsonrpc/devicehandler.cpp index 120992b1..9e306d95 100644 --- a/libnymea-core/jsonrpc/devicehandler.cpp +++ b/libnymea-core/jsonrpc/devicehandler.cpp @@ -865,6 +865,7 @@ JsonReply *DeviceHandler::ExecuteAction(const QVariantMap ¶ms, const JsonCon ThingActionInfo *info = NymeaCore::instance()->executeAction(action); connect(info, &ThingActionInfo::finished, jsonReply, [info, jsonReply, locale](){ + qWarning() << "finished..."; QVariantMap data; data.insert("deviceError", enumValueName(info->status()).replace("Thing", "Device")); if (!info->displayMessage().isEmpty()) { diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index 1842a5a6..8e149afe 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -8,7 +8,7 @@ INCLUDEPATH += $$top_srcdir/libnymea $$top_builddir LIBS += -L$$top_builddir/libnymea/ -lnymea -lssl -lcrypto CONFIG += link_pkgconfig -PKGCONFIG += nymea-mqtt nymea-networkmanager +PKGCONFIG += nymea-mqtt nymea-networkmanager python3-embed target.path = $$[QT_INSTALL_LIBS] INSTALLS += target @@ -22,6 +22,7 @@ HEADERS += nymeacore.h \ integrations/plugininfocache.h \ integrations/thingmanagerimplementation.h \ integrations/translator.h \ + integrations/pythonintegrationplugin.h \ experiences/experiencemanager.h \ ruleengine/ruleengine.h \ ruleengine/rule.h \ @@ -101,6 +102,7 @@ SOURCES += nymeacore.cpp \ integrations/plugininfocache.cpp \ integrations/thingmanagerimplementation.cpp \ integrations/translator.cpp \ + integrations/pythonintegrationplugin.cpp \ experiences/experiencemanager.cpp \ ruleengine/ruleengine.cpp \ ruleengine/rule.cpp \ diff --git a/nymea.pro b/nymea.pro index 82cacc52..1b2879eb 100644 --- a/nymea.pro +++ b/nymea.pro @@ -105,4 +105,3 @@ coverage { ccache { message("Using ccache.") } - From a21315efc57750f0591e29f6d613a8dab5034916 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sun, 31 May 2020 00:16:58 +0200 Subject: [PATCH 02/46] some more hacking --- .../python/PyThingDiscoveryInfo.h | 79 ++++++++++++++++++ .../integrations/pythonintegrationplugin.cpp | 82 +++++++++++++------ libnymea-core/libnymea-core.pro | 1 + 3 files changed, 139 insertions(+), 23 deletions(-) create mode 100644 libnymea-core/integrations/python/PyThingDiscoveryInfo.h diff --git a/libnymea-core/integrations/python/PyThingDiscoveryInfo.h b/libnymea-core/integrations/python/PyThingDiscoveryInfo.h new file mode 100644 index 00000000..a98fcd0e --- /dev/null +++ b/libnymea-core/integrations/python/PyThingDiscoveryInfo.h @@ -0,0 +1,79 @@ +#ifndef PYTHINGDISCOVERYINFO_H +#define PYTHINGDISCOVERYINFO_H + + +#include + +#include "integrations/thingdiscoveryinfo.h" + +#include + +typedef struct { + PyObject_HEAD + ThingDiscoveryInfo* ptrObj; +} PyThingDiscoveryInfo; + + +static int PyThingDiscoveryInfo_init(PyThingDiscoveryInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) +// initialize PyVoice Object +{ + return 0; +} + + +static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) +// destruct the object +{ + // FIXME: Why is this not called? Seems we're leaking... + Q_ASSERT(false); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * PyThingDiscoveryInfo_finish(PyThingDiscoveryInfo* self, PyObject* args) +{ + int status; + char *message; + + if (PyArg_ParseTuple(args, "is", &status, &message)) { + (self->ptrObj)->finish(static_cast(status), QString(message)); + return Py_BuildValue(""); + } + + if (PyArg_ParseTuple(args, "i", &status)) { + (self->ptrObj)->finish(static_cast(status)); + return Py_BuildValue(""); + } + + return Py_False; +} + +static PyMethodDef PyThingDiscoveryInfo_methods[] = { + { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyTypeObject PyThingDiscoveryInfoType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.ThingDiscoveryInfo", /* tp_name */ + sizeof(PyThingDiscoveryInfo), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Noddy objects", /* tp_doc */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; +#endif // PYTHINGDISCOVERYINFO_H diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index ea3885fa..9fe77af0 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -1,4 +1,5 @@ #include +#include "python/PyThingDiscoveryInfo.h" #include "pythonintegrationplugin.h" @@ -6,7 +7,8 @@ #include -PyObject* aview_write(PyObject* /*self*/, PyObject* args) +// Write to stdout/stderr +PyObject* nymea_write(PyObject* /*self*/, PyObject* args) { const char *what; if (!PyArg_ParseTuple(args, "s", &what)) @@ -15,35 +17,53 @@ PyObject* aview_write(PyObject* /*self*/, PyObject* args) return Py_BuildValue(""); } - -PyObject* aview_flush(PyObject* /*self*/, PyObject* /*args*/) +// Flush stdout/stderr +PyObject* nymea_flush(PyObject* /*self*/, PyObject* /*args*/) { + // Not really needed... qDebug() fluses already on its own return Py_BuildValue(""); } -static PyMethodDef aview_methods[] = +static PyMethodDef nymea_methods[] = { - {"write", aview_write, METH_VARARGS, "doc for write"}, - {"flush", aview_flush, METH_VARARGS, "doc for flush"}, + {"write", nymea_write, METH_VARARGS, "write to stdout through qDebug()"}, + {"flush", nymea_flush, METH_VARARGS, "flush stdout (no-op)"}, {nullptr, nullptr, 0, nullptr} // sentinel }; -static PyModuleDef aview_module = +static PyModuleDef nymea_module = { PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; - "aview", // const char* m_name; - "doc for aview", // const char* m_doc; + "nymea", // const char* m_name; + "nymea module for python based integration plugins", // const char* m_doc; -1, // Py_ssize_t m_size; - aview_methods, // PyMethodDef *m_methods + nymea_methods, // PyMethodDef *m_methods // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; nullptr, nullptr, nullptr, nullptr }; -PyMODINIT_FUNC PyInit_aview(void) +PyMODINIT_FUNC PyInit_nymea(void) { - PyObject* m = PyModule_Create(&aview_module); + PyObject* m = PyModule_Create(&nymea_module); + // Overrride stdout/stderr to use qDebug instead PySys_SetObject("stdout", m); PySys_SetObject("stderr", m); + + + PyThingDiscoveryInfoType.tp_new = PyType_GenericNew; + PyThingDiscoveryInfoType.tp_basicsize=sizeof(PyThingDiscoveryInfo); + PyThingDiscoveryInfoType.tp_dealloc=(destructor) PyThingDiscoveryInfo_dealloc; + PyThingDiscoveryInfoType.tp_flags=Py_TPFLAGS_DEFAULT; + PyThingDiscoveryInfoType.tp_doc="ThingDiscoveryInfo class"; + PyThingDiscoveryInfoType.tp_methods=PyThingDiscoveryInfo_methods; + //~ PyVoiceType.tp_members=Noddy_members; + PyThingDiscoveryInfoType.tp_init=(initproc)PyThingDiscoveryInfo_init; + + if (PyType_Ready(&PyThingDiscoveryInfoType) < 0) + return NULL; + + PyModule_AddObject(m, "ThingDiscoveryInfo", (PyObject *)&PyThingDiscoveryInfoType); // Add Voice object to the module + return m; } @@ -55,9 +75,9 @@ PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationP bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) { // TODO: Call this just once and call Py_Finalize() - PyImport_AppendInittab("aview", PyInit_aview); + PyImport_AppendInittab("nymea", PyInit_nymea); Py_Initialize(); - PyImport_ImportModule("aview"); + PyImport_ImportModule("nymea"); QFileInfo fi(scriptFile); qCDebug(dcThingManager()) << "Importing" << fi.absolutePath() << fi.fileName() << fi.baseName(); @@ -161,20 +181,36 @@ void PythonIntegrationPlugin::init() void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) { qCDebug(dcThingManager()) << "Python wrapper: discoverThings()" << info; - PyObject *pFunc = PyObject_GetAttrString(m_module, "init"); + PyObject *pFunc = PyObject_GetAttrString(m_module, "discoverThings"); if(!pFunc || !PyCallable_Check(pFunc)) { qCWarning(dcThingManager()) << "Python plugin does not implement \"discoverThings()\" method."; return; } -// PyObject* args = Py_BuildValue("(s)", ln.c_str()); -// if (!args) -// { -// PyErr_Print(); -// Py_DECREF(filterFunc); -// return "Error building args tuple"; -// } -// PyObject_CallObject(pFunc, nullptr); + PyThingDiscoveryInfo *pyInfo = (PyThingDiscoveryInfo*)_PyObject_New(&PyThingDiscoveryInfoType); + pyInfo->ptrObj = info; + + connect(info, &ThingDiscoveryInfo::finished, this, [=](){ + PyObject_Free(pyInfo); + }); + + PyObject_CallFunction(pFunc, "O", pyInfo); + + if (PyErr_Occurred()) { + PyObject *ptype, *pvalue, *ptraceback; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + if(pvalue) { + PyObject *pstr = PyObject_Str(pvalue); + if(pstr) { + const char* err_msg = PyUnicode_AsUTF8(pstr); + if(pstr) { + qWarning() << QString(err_msg); + } + + } + PyErr_Restore(ptype, pvalue, ptraceback); + } + } } diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index 8e149afe..03a9448d 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -20,6 +20,7 @@ RESOURCES += $$top_srcdir/icons.qrc \ HEADERS += nymeacore.h \ integrations/plugininfocache.h \ + integrations/python/PyThingDiscoveryInfo.h \ integrations/thingmanagerimplementation.h \ integrations/translator.h \ integrations/pythonintegrationplugin.h \ From 380e962bd18773d64a6c014e0eac108951675a5a Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Wed, 3 Jun 2020 18:26:31 +0200 Subject: [PATCH 03/46] some more work on python plugins --- libnymea-core/integrations/python/pything.h | 69 ++++++ .../python/pythingdiscoveryinfo.h | 233 ++++++++++++++++++ ...hingDiscoveryInfo.h => pythingsetupinfo.h} | 40 +-- .../integrations/pythonintegrationplugin.cpp | 230 ++++++++++------- .../integrations/pythonintegrationplugin.h | 12 + .../thingmanagerimplementation.cpp | 44 +++- libnymea-core/libnymea-core.pro | 4 +- 7 files changed, 513 insertions(+), 119 deletions(-) create mode 100644 libnymea-core/integrations/python/pything.h create mode 100644 libnymea-core/integrations/python/pythingdiscoveryinfo.h rename libnymea-core/integrations/python/{PyThingDiscoveryInfo.h => pythingsetupinfo.h} (69%) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h new file mode 100644 index 00000000..95c79876 --- /dev/null +++ b/libnymea-core/integrations/python/pything.h @@ -0,0 +1,69 @@ +#ifndef PYTHING_H +#define PYTHING_H + +#include +#include "structmember.h" + +#include "integrations/thing.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +typedef struct { + PyObject_HEAD + Thing* ptrObj; +} PyThing; + + +static int PyThing_init(PyThing */*self*/, PyObject */*args*/, PyObject */*kwds*/) +// initialize PyVoice Object +{ + return 0; +} + + +static void PyThing_dealloc(PyThing * self) +// destruct the object +{ + // FIXME: Why is this not called? Seems we're leaking... + Q_ASSERT(false); + Py_TYPE(self)->tp_free(self); +} + + +static PyMethodDef PyThing_methods[] = { +// { "addDescriptor", (PyCFunction)PyThing_addDescriptor, METH_VARARGS, "Add a new descriptor to the discovery" }, +// { "finish", (PyCFunction)PyThing_finish, METH_VARARGS, "finish a discovery" }, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyTypeObject PyThingDiscoveryInfoType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.Thing", /* tp_name */ + sizeof(PyThing), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Noddy objects", /* tp_doc */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + + +#pragma GCC diagnostic pop + +#endif // PYTHING_H diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h new file mode 100644 index 00000000..e1e56747 --- /dev/null +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -0,0 +1,233 @@ +#ifndef PYTHINGDISCOVERYINFO_H +#define PYTHINGDISCOVERYINFO_H + + +#include +#include "structmember.h" + +#include "integrations/thingdiscoveryinfo.h" + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" + + + +typedef struct { + PyObject_HEAD + PyObject* thingClassId; + PyObject* name; + PyObject* description; + ThingDescriptor descriptor; +} PyThingDescriptor; + +static PyMethodDef PyThingDescriptor_methods[] = { +// { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyMemberDef PyThingDescriptor_members[] = { + {"thingClassId", T_OBJECT_EX, offsetof(PyThingDescriptor, thingClassId), 0, "Descriptor thingClassId"}, + {"name", T_OBJECT_EX, offsetof(PyThingDescriptor, name), 0, "Descriptor name"}, + {"description", T_OBJECT_EX, offsetof(PyThingDescriptor, description), 0, "Descriptor description"}, + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +}; + + +static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"thingClassId", "name", "description", nullptr}; + PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr, *tmp = nullptr; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, &thingClassId, &name, &description)) + return -1; + + if (thingClassId) { + tmp = self->thingClassId; + Py_INCREF(thingClassId); + self->thingClassId = thingClassId; + Py_XDECREF(tmp); + } + if (name) { + tmp = self->name; + Py_INCREF(name); + self->name = name; + Py_XDECREF(tmp); + } + if (description) { + tmp = self->description; + Py_INCREF(description); + self->description = description; + Py_XDECREF(tmp); + } + return 0; +} + +//static PyGetSetDef PyThingDescriptor_getsetters[] = { +// {"name", (getter) PyThingDescriptor_getName, (setter) PyThingDescriptir_setName, +// "Descriptor name", NULL}, +// {"last", (getter) Custom_getlast, (setter) Custom_setlast, +// "last name", NULL}, +// {NULL} /* Sentinel */ +//}; + +static PyTypeObject PyThingDescriptorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.ThingDescriptor", /* tp_name */ + sizeof(PyThingDescriptor), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Noddy objects", /* tp_doc */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + + + + + + + + + + + + + + + + + + +typedef struct { + PyObject_HEAD + ThingDiscoveryInfo* ptrObj; +} PyThingDiscoveryInfo; + + +static int PyThingDiscoveryInfo_init(PyThingDiscoveryInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) +// initialize PyVoice Object +{ + return 0; +} + + +static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) +// destruct the object +{ + // FIXME: Why is this not called? Seems we're leaking... + Q_ASSERT(false); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * PyThingDiscoveryInfo_finish(PyThingDiscoveryInfo* self, PyObject* args) +{ + int status; + char *message; + + if (PyArg_ParseTuple(args, "is", &status, &message)) { + (self->ptrObj)->finish(static_cast(status), QString(message)); + return Py_BuildValue(""); + } + + if (PyArg_ParseTuple(args, "i", &status)) { + (self->ptrObj)->finish(static_cast(status)); + return Py_BuildValue(""); + } + + return nullptr; +} + +static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, PyObject* args) { + + PyObject *pyObj = nullptr; + + if (!PyArg_ParseTuple(args, "O", &pyObj)) { + PyErr_SetString(PyExc_ValueError, "Invalid argument. Not a ThingDescriptor."); + return nullptr; + } + if (pyObj->ob_type != &PyThingDescriptorType) { + PyErr_SetString(PyExc_ValueError, "Invalid argument. Not a ThingDescriptor."); + return nullptr; + } + PyThingDescriptor *pyDescriptor = (PyThingDescriptor*)pyObj; + + ThingClassId thingClassId; + if (pyDescriptor->thingClassId) { + thingClassId = ThingClassId(PyUnicode_AsUTF8(pyDescriptor->thingClassId)); + } + QString name; + if (pyDescriptor->name) { + name = QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->name)); + } + QString description; + if (pyDescriptor->description) { + description = QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->description)); + } + + ThingDescriptor descriptor(thingClassId, name, description); + + self->ptrObj->addThingDescriptor(descriptor); + + return Py_BuildValue(""); +} + +static PyMethodDef PyThingDiscoveryInfo_methods[] = { + { "addDescriptor", (PyCFunction)PyThingDiscoveryInfo_addDescriptor, METH_VARARGS, "Add a new descriptor to the discovery" }, + { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyTypeObject PyThingDiscoveryInfoType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.ThingDiscoveryInfo", /* tp_name */ + sizeof(PyThingDiscoveryInfo), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Noddy objects", /* tp_doc */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + + + + + + + + + + +#pragma GCC diagnostic pop + +#endif // PYTHINGDISCOVERYINFO_H diff --git a/libnymea-core/integrations/python/PyThingDiscoveryInfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h similarity index 69% rename from libnymea-core/integrations/python/PyThingDiscoveryInfo.h rename to libnymea-core/integrations/python/pythingsetupinfo.h index a98fcd0e..eff6f324 100644 --- a/libnymea-core/integrations/python/PyThingDiscoveryInfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -1,27 +1,31 @@ -#ifndef PYTHINGDISCOVERYINFO_H -#define PYTHINGDISCOVERYINFO_H - +#ifndef PYTHINGSETUPINFO_H +#define PYTHINGSETUPINFO_H #include +#include "structmember.h" -#include "integrations/thingdiscoveryinfo.h" +#include "pything.h" -#include +#include "integrations/thingsetupinfo.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" typedef struct { PyObject_HEAD - ThingDiscoveryInfo* ptrObj; -} PyThingDiscoveryInfo; + ThingSetupInfo* ptrObj; +} PyThingSetupInfo; -static int PyThingDiscoveryInfo_init(PyThingDiscoveryInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) +static int PyThingSetupInfo_init(PyThingSetupInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) // initialize PyVoice Object { return 0; } -static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) +static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) // destruct the object { // FIXME: Why is this not called? Seems we're leaking... @@ -29,7 +33,7 @@ static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) Py_TYPE(self)->tp_free(self); } -static PyObject * PyThingDiscoveryInfo_finish(PyThingDiscoveryInfo* self, PyObject* args) +static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args) { int status; char *message; @@ -44,18 +48,19 @@ static PyObject * PyThingDiscoveryInfo_finish(PyThingDiscoveryInfo* self, PyObje return Py_BuildValue(""); } - return Py_False; + return nullptr; } + static PyMethodDef PyThingDiscoveryInfo_methods[] = { { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, {nullptr, nullptr, 0, nullptr} // sentinel }; -static PyTypeObject PyThingDiscoveryInfoType = { +static PyTypeObject PyThingSetupInfoType = { PyVarObject_HEAD_INIT(NULL, 0) - "nymea.ThingDiscoveryInfo", /* tp_name */ - sizeof(PyThingDiscoveryInfo), /* tp_basicsize */ + "nymea.ThingSetupInfo", /* tp_name */ + sizeof(PyThingSetupInfo), /* tp_basicsize */ 0, /* tp_itemsize */ 0, /* tp_dealloc */ 0, /* tp_print */ @@ -76,4 +81,9 @@ static PyTypeObject PyThingDiscoveryInfoType = { "Noddy objects", /* tp_doc */ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; -#endif // PYTHINGDISCOVERYINFO_H + + + +#pragma GCC diagnostic pop + +#endif // PYTHINGSETUPINFO_H diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 9fe77af0..6db011d4 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -1,15 +1,20 @@ #include -#include "python/PyThingDiscoveryInfo.h" +#include "python/pythingdiscoveryinfo.h" #include "pythonintegrationplugin.h" #include "loggingcategories.h" #include +#include +#include + +QHash s_modules; // Write to stdout/stderr -PyObject* nymea_write(PyObject* /*self*/, PyObject* args) +PyObject* nymea_write(PyObject* self, PyObject* args) { + qWarning() << "write called" << self; const char *what; if (!PyArg_ParseTuple(args, "s", &what)) return nullptr; @@ -50,19 +55,40 @@ PyMODINIT_FUNC PyInit_nymea(void) PySys_SetObject("stderr", m); + QMetaEnum thingErrorEnum = QMetaEnum::fromType(); + for (int i = 0; i < thingErrorEnum.keyCount(); i++) { + PyModule_AddObject(m, thingErrorEnum.key(i), PyLong_FromLong(thingErrorEnum.value(i))); + } + + PyThingDiscoveryInfoType.tp_new = PyType_GenericNew; PyThingDiscoveryInfoType.tp_basicsize=sizeof(PyThingDiscoveryInfo); PyThingDiscoveryInfoType.tp_dealloc=(destructor) PyThingDiscoveryInfo_dealloc; PyThingDiscoveryInfoType.tp_flags=Py_TPFLAGS_DEFAULT; PyThingDiscoveryInfoType.tp_doc="ThingDiscoveryInfo class"; PyThingDiscoveryInfoType.tp_methods=PyThingDiscoveryInfo_methods; - //~ PyVoiceType.tp_members=Noddy_members; PyThingDiscoveryInfoType.tp_init=(initproc)PyThingDiscoveryInfo_init; - if (PyType_Ready(&PyThingDiscoveryInfoType) < 0) - return NULL; + if (PyType_Ready(&PyThingDiscoveryInfoType) < 0) { + qCWarning(dcThingManager()) << "Error creating PyThingDiscoveryInfo"; + return nullptr; + } + PyModule_AddObject(m, "ThingDiscoveryInfo", (PyObject *)&PyThingDiscoveryInfoType); + + PyThingDescriptorType.tp_new = PyType_GenericNew; + PyThingDescriptorType.tp_basicsize = sizeof(PyThingDescriptor); + PyThingDescriptorType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingDescriptorType.tp_doc = "ThingDescriptor class"; + PyThingDescriptorType.tp_methods = PyThingDescriptor_methods; + PyThingDescriptorType.tp_members = PyThingDescriptor_members; + PyThingDescriptorType.tp_init = (initproc)PyThingDescriptor_init; + + if (PyType_Ready(&PyThingDescriptorType) < 0) { + qCWarning(dcThingManager()) << "Error creating PyThingDescriptor"; + return nullptr; + } + PyModule_AddObject(m, "ThingDescriptor", (PyObject *)&PyThingDescriptorType); - PyModule_AddObject(m, "ThingDiscoveryInfo", (PyObject *)&PyThingDiscoveryInfoType); // Add Voice object to the module return m; } @@ -72,114 +98,87 @@ PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationP } -bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) +void PythonIntegrationPlugin::initPython() { // TODO: Call this just once and call Py_Finalize() PyImport_AppendInittab("nymea", PyInit_nymea); - Py_Initialize(); + Py_InitializeEx(0); + +} + +bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) +{ + QFileInfo fi(scriptFile); + + QFile metaData(fi.absolutePath() + "/" + fi.baseName() + ".json"); + if (!metaData.open(QFile::ReadOnly)) { + qCWarning(dcThingManager()) << "Error opening metadata file:" << metaData.fileName(); + return false; + } + QJsonParseError error; + QJsonDocument jsonDoc = QJsonDocument::fromJson(metaData.readAll(), &error); + if (error.error != QJsonParseError::NoError) { + qCWarning(dcThingManager()) << "Error parsing metadata file:" << error.errorString(); + return false; + } + m_metaData = jsonDoc.toVariant().toMap(); + + + // Create a new sub-interpreter for this plugin + m_interpreter = Py_NewInterpreter(); + PyThreadState_Swap(m_interpreter); + + + // Import nymea module into this interpreter PyImport_ImportModule("nymea"); - QFileInfo fi(scriptFile); - qCDebug(dcThingManager()) << "Importing" << fi.absolutePath() << fi.fileName() << fi.baseName(); - + // Add this plugin's locatioon to the import path PyObject* sysPath = PySys_GetObject("path"); PyList_Append(sysPath, PyUnicode_FromString(fi.absolutePath().toUtf8())); - - PyObject *pName = PyUnicode_FromString(fi.baseName().toUtf8()); - qCDebug(dcThingManager()) << "Importing python plugin from" << scriptFile; - m_module = PyImport_Import(pName); + // Finally, import the plugin + m_module = PyImport_ImportModule(fi.baseName().toUtf8()); if (!m_module) { - qCWarning(dcThingManager()) << "Error importing plugin"; + dumpError(); + qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath(); return false; } + qCDebug(dcThingManager()) << "Imported python plugin from" << fi.absoluteFilePath(); + + + exportIds(); + + + s_modules.insert(m_module, m_interpreter); + return true; } QJsonObject PythonIntegrationPlugin::metaData() const { - QVariantMap pluginMetaData; - pluginMetaData.insert("id", "ccc6dbc8-e352-48a1-8e87-3c89a4669fc2"); - pluginMetaData.insert("name", "CloudNotifications"); - pluginMetaData.insert("displayName", tr("Cloud Notifications")); - - QVariantList interfaces; -// interfaces.append("notifications"); - interfaces.append("connectable"); - - QVariantList createMethods; - createMethods.append("discovery"); - - QVariantMap testActionParamTitle; - testActionParamTitle.insert("id", "c9545e1c-55cd-42ca-a00f-43f21dfdf05a"); - testActionParamTitle.insert("name", "title"); - testActionParamTitle.insert("displayName", tr("Title")); - testActionParamTitle.insert("type", "QString"); - - QVariantList notifyActionParamTypes; - notifyActionParamTypes.append(testActionParamTitle); - - QVariantMap notifyAction; - notifyAction.insert("id", "cc6ad463-0a63-4570-ae13-956f50faa3a6"); - notifyAction.insert("name", "notify"); - notifyAction.insert("displayName", tr("Send notification")); - notifyAction.insert("paramTypes", notifyActionParamTypes); - - QVariantList actionTypes; - actionTypes.append(notifyAction); - - QVariantMap connectedState; - connectedState.insert("id", "292b5c5d-a6fc-43b6-a59e-f3e1a3ab42b4"); - connectedState.insert("name", "connected"); - connectedState.insert("displayName", tr("connected")); - connectedState.insert("type", "bool"); - connectedState.insert("displayNameEvent", tr("Connected changed")); - connectedState.insert("defaultValue", false); - - QVariantList stateTypes; - stateTypes.append(connectedState); - - - QVariantMap cloudNotificationsThingClass; - cloudNotificationsThingClass.insert("id", "0f1a441a-3793-4a1c-91fc-35d752443aff"); - cloudNotificationsThingClass.insert("name", "PyTest"); - cloudNotificationsThingClass.insert("displayName", tr("Python test")); - cloudNotificationsThingClass.insert("createMethods", createMethods); - cloudNotificationsThingClass.insert("interfaces", interfaces); - cloudNotificationsThingClass.insert("actionTypes", actionTypes); - cloudNotificationsThingClass.insert("stateTypes", stateTypes); - - QVariantList thingClasses; - thingClasses.append(cloudNotificationsThingClass); - - QVariantMap guhVendor; - guhVendor.insert("id", "2062d64d-3232-433c-88bc-0d33c0ba2ba6"); // nymea's id - guhVendor.insert("name", "nymea"); - guhVendor.insert("displayName", "nymea"); - guhVendor.insert("thingClasses", thingClasses); - - QVariantList vendors; - vendors.append(guhVendor); - pluginMetaData.insert("vendors", vendors); - - return QJsonObject::fromVariantMap(pluginMetaData); + return QJsonObject::fromVariantMap(m_metaData); } void PythonIntegrationPlugin::init() { - qCDebug(dcThingManager()) << "Python wrapper: init()"; + PyThreadState_Swap(m_interpreter); + + qCDebug(dcThingManager()) << "Python wrapper: init()" << m_interpreter; PyObject *pFunc = PyObject_GetAttrString(m_module, "init"); if(!pFunc || !PyCallable_Check(pFunc)) { qCDebug(dcThingManager()) << "Python plugin does not implement \"init()\" method."; return; } PyObject_CallObject(pFunc, nullptr); + dumpError(); } void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) { + PyThreadState_Swap(m_interpreter); + qCDebug(dcThingManager()) << "Python wrapper: discoverThings()" << info; PyObject *pFunc = PyObject_GetAttrString(m_module, "discoverThings"); if(!pFunc || !PyCallable_Check(pFunc)) { @@ -194,21 +193,64 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) PyObject_Free(pyInfo); }); - PyObject_CallFunction(pFunc, "O", pyInfo); - if (PyErr_Occurred()) { - PyObject *ptype, *pvalue, *ptraceback; - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - if(pvalue) { - PyObject *pstr = PyObject_Str(pvalue); - if(pstr) { - const char* err_msg = PyUnicode_AsUTF8(pstr); - if(pstr) { - qWarning() << QString(err_msg); - } +// PyObject *loop = PyObject_GetAttrString(m_module, "_loop"); +// dumpError(); +// qWarning() << "loop:" << loop; + + PyObject *future = PyObject_CallFunctionObjArgs(pFunc, pyInfo, nullptr); + dumpError(); + + qWarning() << "future:" << future; + + PyObject *asyncio = PyObject_GetAttrString(m_module, "asyncio"); + qWarning() << "ayncio:" << asyncio; + + PyObject *loop = PyObject_GetAttrString(m_module, "loop"); + PyObject *create_task = PyObject_GetAttrString(loop, "create_task"); + PyObject *task = PyObject_CallFunctionObjArgs(create_task, future, nullptr); + + dumpError(); + qWarning() << "Create task:" << task; + +} + +void PythonIntegrationPlugin::dumpError() +{ + if (!PyErr_Occurred()) { + return; + } + + PyObject *ptype, *pvalue, *ptraceback; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + if (pvalue) { + PyObject *pstr = PyObject_Str(pvalue); + if (pstr) { + const char* err_msg = PyUnicode_AsUTF8(pstr); + if (pstr) { + qCWarning(dcThingManager()) << QString(err_msg); } - PyErr_Restore(ptype, pvalue, ptraceback); + + } + PyErr_Restore(ptype, pvalue, ptraceback); + } +} + +void PythonIntegrationPlugin::exportIds() +{ + foreach (const QVariant &vendorVariant, m_metaData.value("vendors").toList()) { + QVariantMap vendor = vendorVariant.toMap(); + QString vendorIdName = vendor.value("name").toString() + "VendorId"; + QString vendorId = vendor.value("id").toString(); + qCDebug(dcThingManager()) << "Exporting:" << vendorIdName; + PyModule_AddStringConstant(m_module, vendorIdName.toUtf8(), vendorId.toUtf8()); + + foreach (const QVariant &thingClassVariant, vendor.value("thingClasses").toList()) { + QVariantMap thingClass = thingClassVariant.toMap(); + QString thingClassIdName = thingClass.value("name").toString() + "ThingClassId"; + QString thingClassId = thingClass.value("id").toString(); + PyModule_AddStringConstant(m_module, thingClassIdName.toUtf8(), thingClassId.toUtf8()); } } diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 562ec5a4..6034f16b 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -8,6 +8,7 @@ extern "C" { typedef struct _object PyObject; +typedef struct _ts PyThreadState; } @@ -17,6 +18,8 @@ class PythonIntegrationPlugin : public IntegrationPlugin public: explicit PythonIntegrationPlugin(QObject *parent = nullptr); + static void initPython(); + bool loadScript(const QString &scriptFile); QJsonObject metaData() const; @@ -27,7 +30,16 @@ public: private: + void dumpError(); + + void exportIds(); + +private: +// static QHash s_modules; + + QVariantMap m_metaData; PyObject *m_module = nullptr; + PyThreadState *m_interpreter = nullptr; }; diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index 9f8bfa66..7036863e 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -1343,17 +1343,43 @@ void ThingManagerImplementation::loadPlugins() } #endif - PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); - p->loadScript("/home/micha/Develop/nymea-plugin-pytest/pytest.py"); - PluginMetadata metaData(p->metaData()); - if (!metaData.isValid()) { - qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; - foreach (const QString &error, metaData.validationErrors()) { - qCWarning(dcThingManager()) << error; + PythonIntegrationPlugin::initPython(); + + { + PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); + bool ok = p->loadScript("/home/micha/Develop/nymea-plugin-pytest/integrationpluginpytest.py"); + if (!ok) { + qCWarning(dcThingManager()) << "Error loading plugin"; + return; } - return; + PluginMetadata metaData(p->metaData()); + if (!metaData.isValid()) { + qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; + foreach (const QString &error, metaData.validationErrors()) { + qCWarning(dcThingManager()) << error; + } + return; + } + loadPlugin(p, metaData); } - loadPlugin(p, metaData); +// { +// PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); +// bool ok = p->loadScript("/home/micha/Develop/nymea-plugin-pytest2/integrationpluginpytest2.py"); +// if (!ok) { +// qCWarning(dcThingManager()) << "Error loading plugin"; +// return; +// } +// PluginMetadata metaData(p->metaData()); +// if (!metaData.isValid()) { +// qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; +// foreach (const QString &error, metaData.validationErrors()) { +// qCWarning(dcThingManager()) << error; +// } +// return; +// } +// loadPlugin(p, metaData); + +// } } void ThingManagerImplementation::loadPlugin(IntegrationPlugin *pluginIface, const PluginMetadata &metaData) diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index 03a9448d..9e6dd20d 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -20,7 +20,9 @@ RESOURCES += $$top_srcdir/icons.qrc \ HEADERS += nymeacore.h \ integrations/plugininfocache.h \ - integrations/python/PyThingDiscoveryInfo.h \ + integrations/python/pything.h \ + integrations/python/pythingdiscoveryinfo.h \ + integrations/python/pythingsetupinfo.h \ integrations/thingmanagerimplementation.h \ integrations/translator.h \ integrations/pythonintegrationplugin.h \ From a2e6d9e185affad5c120fd707b4bbb0ef930456a Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Fri, 5 Jun 2020 19:40:15 +0200 Subject: [PATCH 04/46] Add an event loop --- libnymea-core/integrations/python/pything.h | 2 +- .../integrations/python/pythingsetupinfo.h | 4 +- .../integrations/pythonintegrationplugin.cpp | 68 +++++++++++++++---- .../integrations/pythonintegrationplugin.h | 1 + 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 95c79876..fd504d24 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -38,7 +38,7 @@ static PyMethodDef PyThing_methods[] = { {nullptr, nullptr, 0, nullptr} // sentinel }; -static PyTypeObject PyThingDiscoveryInfoType = { +static PyTypeObject PyThingType = { PyVarObject_HEAD_INIT(NULL, 0) "nymea.Thing", /* tp_name */ sizeof(PyThing), /* tp_basicsize */ diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index eff6f324..bd6f2ddb 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -52,8 +52,8 @@ static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args } -static PyMethodDef PyThingDiscoveryInfo_methods[] = { - { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, +static PyMethodDef PyThingSetupInfo_methods[] = { + { "finish", (PyCFunction)PyThingSetupInfo_finish, METH_VARARGS, "finish a setup" }, {nullptr, nullptr, 0, nullptr} // sentinel }; diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 6db011d4..b87e1a39 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -1,5 +1,6 @@ #include #include "python/pythingdiscoveryinfo.h" +#include "python/pythingsetupinfo.h" #include "pythonintegrationplugin.h" @@ -8,6 +9,7 @@ #include #include #include +#include QHash s_modules; @@ -152,6 +154,22 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) s_modules.insert(m_module, m_interpreter); + QtConcurrent::run([=](){ + PyObject *loop = PyObject_GetAttrString(m_module, "loop"); + dumpError(); +// PyObject *stop = PyObject_GetAttrString(loop, "stop"); +// PyObject *call_soon = PyObject_GetAttrString(loop, "call_soon"); + PyObject *run_forever = PyObject_GetAttrString(loop, "run_forever"); +// PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); + qCDebug(dcThingManager()) << "Starting python event loop"; + dumpError(); + PyObject_CallFunctionObjArgs(run_forever, nullptr); + qCDebug(dcThingManager()) << "stopped python event loop"; + dumpError(); + + }); + + return true; } @@ -177,8 +195,12 @@ void PythonIntegrationPlugin::init() void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) { + PyThreadState_Swap(m_interpreter); + PyGILState_STATE s = PyGILState_Ensure(); + + qCDebug(dcThingManager()) << "Python wrapper: discoverThings()" << info; PyObject *pFunc = PyObject_GetAttrString(m_module, "discoverThings"); if(!pFunc || !PyCallable_Check(pFunc)) { @@ -194,26 +216,48 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) }); -// PyObject *loop = PyObject_GetAttrString(m_module, "_loop"); -// dumpError(); -// qWarning() << "loop:" << loop; + PyObject *future = PyObject_CallFunctionObjArgs(pFunc, pyInfo, nullptr); + dumpError(); + + PyObject *asyncio = PyObject_GetAttrString(m_module, "asyncio"); + PyObject *loop = PyObject_GetAttrString(m_module, "loop"); + PyObject *run_coroutine_threadsafe = PyObject_GetAttrString(asyncio, "run_coroutine_threadsafe"); + PyObject_CallFunctionObjArgs(run_coroutine_threadsafe, future, loop, nullptr); + + PyGILState_Release(s); + +} + +void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) +{ + PyThreadState_Swap(m_interpreter); + + PyGILState_STATE s = PyGILState_Ensure(); + + qCDebug(dcThingManager()) << "Python wrapper: setupThing()" << info; + PyObject *pFunc = PyObject_GetAttrString(m_module, "setupThing"); + if(!pFunc || !PyCallable_Check(pFunc)) { + qCWarning(dcThingManager()) << "Python plugin does not implement \"setThing()\" method."; + return; + } + + PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)_PyObject_New(&PyThingSetupInfoType); + pyInfo->ptrObj = info; + + connect(info, &ThingSetupInfo::finished, this, [=](){ + PyObject_Free(pyInfo); + }); PyObject *future = PyObject_CallFunctionObjArgs(pFunc, pyInfo, nullptr); dumpError(); - qWarning() << "future:" << future; - PyObject *asyncio = PyObject_GetAttrString(m_module, "asyncio"); - qWarning() << "ayncio:" << asyncio; - PyObject *loop = PyObject_GetAttrString(m_module, "loop"); - PyObject *create_task = PyObject_GetAttrString(loop, "create_task"); - PyObject *task = PyObject_CallFunctionObjArgs(create_task, future, nullptr); - - dumpError(); - qWarning() << "Create task:" << task; + PyObject *run_coroutine_threadsafe = PyObject_GetAttrString(asyncio, "run_coroutine_threadsafe"); + PyObject_CallFunctionObjArgs(run_coroutine_threadsafe, future, loop, nullptr); + PyGILState_Release(s); } void PythonIntegrationPlugin::dumpError() diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 6034f16b..e4f8cc70 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -27,6 +27,7 @@ public: void init() override; void discoverThings(ThingDiscoveryInfo *info) override; + void setupThing(ThingSetupInfo *info) override; private: From 31cf425b794bb0a9c91e4c518644f2b66c52f1c2 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Fri, 5 Jun 2020 23:55:04 +0200 Subject: [PATCH 05/46] intermediate --- libnymea-core/integrations/python/pything.h | 17 +++ .../integrations/python/pythingdescriptor.h | 117 +++++++++++++++++ .../python/pythingdiscoveryinfo.h | 123 +++--------------- .../integrations/python/pythingsetupinfo.h | 23 +++- .../integrations/pythonintegrationplugin.cpp | 64 +++------ .../integrations/pythonintegrationplugin.h | 3 + libnymea-core/libnymea-core.pro | 1 + libnymea/loggingcategories.h | 3 + 8 files changed, 198 insertions(+), 153 deletions(-) create mode 100644 libnymea-core/integrations/python/pythingdescriptor.h diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index fd504d24..5fc9883e 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -63,6 +63,23 @@ static PyTypeObject PyThingType = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; +static void registerThingType(PyObject *module) +{ + PyThingType.tp_new = PyType_GenericNew; + PyThingType.tp_dealloc= reinterpret_cast(PyThing_dealloc); + PyThingType.tp_basicsize = sizeof(PyThing); + PyThingType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingType.tp_doc = "Thing class"; + PyThingType.tp_methods = PyThing_methods; +// PyThingType.tp_members = PyThingSetupInfo_members; + PyThingType.tp_init = reinterpret_cast(PyThing_init); + + if (PyType_Ready(&PyThingType) < 0) { + return; + } + PyModule_AddObject(module, "Thing", reinterpret_cast(&PyThingType)); +} + #pragma GCC diagnostic pop diff --git a/libnymea-core/integrations/python/pythingdescriptor.h b/libnymea-core/integrations/python/pythingdescriptor.h new file mode 100644 index 00000000..91fe2773 --- /dev/null +++ b/libnymea-core/integrations/python/pythingdescriptor.h @@ -0,0 +1,117 @@ +#ifndef PYTHINGDESCRIPTOR_H +#define PYTHINGDESCRIPTOR_H + +#include +#include "structmember.h" + +#include "integrations/thingdescriptor.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" + + +typedef struct { + PyObject_HEAD + PyObject* thingClassId; + PyObject* name; + PyObject* description; + ThingDescriptor descriptor; +} PyThingDescriptor; + +static PyMethodDef PyThingDescriptor_methods[] = { +// { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyMemberDef PyThingDescriptor_members[] = { + {"thingClassId", T_OBJECT_EX, offsetof(PyThingDescriptor, thingClassId), 0, "Descriptor thingClassId"}, + {"name", T_OBJECT_EX, offsetof(PyThingDescriptor, name), 0, "Descriptor name"}, + {"description", T_OBJECT_EX, offsetof(PyThingDescriptor, description), 0, "Descriptor description"}, + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +}; + + +static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"thingClassId", "name", "description", nullptr}; + PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr, *tmp = nullptr; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, &thingClassId, &name, &description)) + return -1; + + if (thingClassId) { + tmp = self->thingClassId; + Py_INCREF(thingClassId); + self->thingClassId = thingClassId; + Py_XDECREF(tmp); + } + if (name) { + tmp = self->name; + Py_INCREF(name); + self->name = name; + Py_XDECREF(tmp); + } + if (description) { + tmp = self->description; + Py_INCREF(description); + self->description = description; + Py_XDECREF(tmp); + } + return 0; +} + +//static PyGetSetDef PyThingDescriptor_getsetters[] = { +// {"name", (getter) PyThingDescriptor_getName, (setter) PyThingDescriptir_setName, +// "Descriptor name", NULL}, +// {"last", (getter) Custom_getlast, (setter) Custom_setlast, +// "last name", NULL}, +// {NULL} /* Sentinel */ +//}; + +static PyTypeObject PyThingDescriptorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.ThingDescriptor", /* tp_name */ + sizeof(PyThingDescriptor), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Noddy objects", /* tp_doc */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + + + +static void registerThingDescriptorType(PyObject *module) +{ + PyThingDescriptorType.tp_new = PyType_GenericNew; + PyThingDescriptorType.tp_basicsize = sizeof(PyThingDescriptor); + PyThingDescriptorType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingDescriptorType.tp_doc = "ThingDescriptor class"; + PyThingDescriptorType.tp_methods = PyThingDescriptor_methods; + PyThingDescriptorType.tp_members = PyThingDescriptor_members; + PyThingDescriptorType.tp_init = reinterpret_cast(PyThingDescriptor_init); + + if (PyType_Ready(&PyThingDescriptorType) < 0) { + return; + } + PyModule_AddObject(module, "ThingDescriptor", reinterpret_cast(&PyThingDescriptorType)); +} + +#pragma GCC diagnostic pop + +#endif // PYTHINGDESCRIPTOR_H diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index e1e56747..ceb955fb 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -5,116 +5,18 @@ #include #include "structmember.h" +#include "pythingdescriptor.h" + #include "integrations/thingdiscoveryinfo.h" #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" - -typedef struct { - PyObject_HEAD - PyObject* thingClassId; - PyObject* name; - PyObject* description; - ThingDescriptor descriptor; -} PyThingDescriptor; - -static PyMethodDef PyThingDescriptor_methods[] = { -// { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, - {nullptr, nullptr, 0, nullptr} // sentinel -}; - -static PyMemberDef PyThingDescriptor_members[] = { - {"thingClassId", T_OBJECT_EX, offsetof(PyThingDescriptor, thingClassId), 0, "Descriptor thingClassId"}, - {"name", T_OBJECT_EX, offsetof(PyThingDescriptor, name), 0, "Descriptor name"}, - {"description", T_OBJECT_EX, offsetof(PyThingDescriptor, description), 0, "Descriptor description"}, - {nullptr, 0, 0, 0, nullptr} /* Sentinel */ -}; - - -static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"thingClassId", "name", "description", nullptr}; - PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr, *tmp = nullptr; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, &thingClassId, &name, &description)) - return -1; - - if (thingClassId) { - tmp = self->thingClassId; - Py_INCREF(thingClassId); - self->thingClassId = thingClassId; - Py_XDECREF(tmp); - } - if (name) { - tmp = self->name; - Py_INCREF(name); - self->name = name; - Py_XDECREF(tmp); - } - if (description) { - tmp = self->description; - Py_INCREF(description); - self->description = description; - Py_XDECREF(tmp); - } - return 0; -} - -//static PyGetSetDef PyThingDescriptor_getsetters[] = { -// {"name", (getter) PyThingDescriptor_getName, (setter) PyThingDescriptir_setName, -// "Descriptor name", NULL}, -// {"last", (getter) Custom_getlast, (setter) Custom_setlast, -// "last name", NULL}, -// {NULL} /* Sentinel */ -//}; - -static PyTypeObject PyThingDescriptorType = { - PyVarObject_HEAD_INIT(NULL, 0) - "nymea.ThingDescriptor", /* tp_name */ - sizeof(PyThingDescriptor), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -}; - - - - - - - - - - - - - - - - - - typedef struct { PyObject_HEAD ThingDiscoveryInfo* ptrObj; @@ -221,9 +123,26 @@ static PyTypeObject PyThingDiscoveryInfoType = { +static void registerThingDiscoveryInfoType(PyObject *module) +{ + PyThingDiscoveryInfoType.tp_new = PyType_GenericNew; + PyThingDiscoveryInfoType.tp_basicsize = sizeof(PyThingDiscoveryInfo); + PyThingDiscoveryInfoType.tp_dealloc = reinterpret_cast(PyThingDiscoveryInfo_dealloc); + PyThingDiscoveryInfoType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingDiscoveryInfoType.tp_doc = "ThingDiscoveryInfo class"; + PyThingDiscoveryInfoType.tp_methods = PyThingDiscoveryInfo_methods; + PyThingDiscoveryInfoType.tp_init = reinterpret_cast(PyThingDiscoveryInfo_init); + if (PyType_Ready(&PyThingDiscoveryInfoType) < 0) { + return; + } + PyModule_AddObject(module, "ThingDiscoveryInfo", (PyObject *)&PyThingDiscoveryInfoType); - + QMetaEnum thingErrorEnum = QMetaEnum::fromType(); + for (int i = 0; i < thingErrorEnum.keyCount(); i++) { + PyModule_AddObject(module, thingErrorEnum.key(i), PyLong_FromLong(thingErrorEnum.value(i))); + } +} diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index bd6f2ddb..e62cba5b 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -63,10 +63,10 @@ static PyTypeObject PyThingSetupInfoType = { sizeof(PyThingSetupInfo), /* tp_basicsize */ 0, /* tp_itemsize */ 0, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ - 0, /* tp_reserved */ + 0, /* tp_as_async */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ @@ -78,10 +78,27 @@ static PyTypeObject PyThingSetupInfoType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ + "Noddy objects", /* tp_doc */ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; +static void registerThingSetupInfoType(PyObject *module) { + PyThingSetupInfoType.tp_new = PyType_GenericNew; + PyThingSetupInfoType.tp_dealloc=(destructor) PyThingSetupInfo_dealloc; + PyThingSetupInfoType.tp_basicsize = sizeof(PyThingSetupInfo); + PyThingSetupInfoType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingSetupInfoType.tp_doc = "ThingSetupInfo class"; + PyThingSetupInfoType.tp_methods = PyThingSetupInfo_methods; +// PyThingSetupInfoType.tp_members = PyThingSetupInfo_members; + PyThingSetupInfoType.tp_init = (initproc)PyThingSetupInfo_init; + + if (PyType_Ready(&PyThingSetupInfoType) < 0) { + return; + } + PyModule_AddObject(module, "ThingSetupInfo", (PyObject *)&PyThingSetupInfoType); +} + + #pragma GCC diagnostic pop diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index b87e1a39..58a6b1c9 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -1,4 +1,6 @@ #include + +#include "python/pything.h" #include "python/pythingdiscoveryinfo.h" #include "python/pythingsetupinfo.h" @@ -57,40 +59,10 @@ PyMODINIT_FUNC PyInit_nymea(void) PySys_SetObject("stderr", m); - QMetaEnum thingErrorEnum = QMetaEnum::fromType(); - for (int i = 0; i < thingErrorEnum.keyCount(); i++) { - PyModule_AddObject(m, thingErrorEnum.key(i), PyLong_FromLong(thingErrorEnum.value(i))); - } - - - PyThingDiscoveryInfoType.tp_new = PyType_GenericNew; - PyThingDiscoveryInfoType.tp_basicsize=sizeof(PyThingDiscoveryInfo); - PyThingDiscoveryInfoType.tp_dealloc=(destructor) PyThingDiscoveryInfo_dealloc; - PyThingDiscoveryInfoType.tp_flags=Py_TPFLAGS_DEFAULT; - PyThingDiscoveryInfoType.tp_doc="ThingDiscoveryInfo class"; - PyThingDiscoveryInfoType.tp_methods=PyThingDiscoveryInfo_methods; - PyThingDiscoveryInfoType.tp_init=(initproc)PyThingDiscoveryInfo_init; - - if (PyType_Ready(&PyThingDiscoveryInfoType) < 0) { - qCWarning(dcThingManager()) << "Error creating PyThingDiscoveryInfo"; - return nullptr; - } - PyModule_AddObject(m, "ThingDiscoveryInfo", (PyObject *)&PyThingDiscoveryInfoType); - - PyThingDescriptorType.tp_new = PyType_GenericNew; - PyThingDescriptorType.tp_basicsize = sizeof(PyThingDescriptor); - PyThingDescriptorType.tp_flags = Py_TPFLAGS_DEFAULT; - PyThingDescriptorType.tp_doc = "ThingDescriptor class"; - PyThingDescriptorType.tp_methods = PyThingDescriptor_methods; - PyThingDescriptorType.tp_members = PyThingDescriptor_members; - PyThingDescriptorType.tp_init = (initproc)PyThingDescriptor_init; - - if (PyType_Ready(&PyThingDescriptorType) < 0) { - qCWarning(dcThingManager()) << "Error creating PyThingDescriptor"; - return nullptr; - } - PyModule_AddObject(m, "ThingDescriptor", (PyObject *)&PyThingDescriptorType); - + registerThingType(m); + registerThingDescriptorType(m); + registerThingDiscoveryInfoType(m); + registerThingSetupInfoType(m); return m; } @@ -100,12 +72,16 @@ PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationP } +PythonIntegrationPlugin::~PythonIntegrationPlugin() +{ + m_eventLoop.cancel(); + m_eventLoop.waitForFinished(); +} + void PythonIntegrationPlugin::initPython() { - // TODO: Call this just once and call Py_Finalize() PyImport_AppendInittab("nymea", PyInit_nymea); Py_InitializeEx(0); - } bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) @@ -154,19 +130,14 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) s_modules.insert(m_module, m_interpreter); - QtConcurrent::run([=](){ + // Start an event loop for this plugin in its own thread + m_eventLoop = QtConcurrent::run([=](){ PyObject *loop = PyObject_GetAttrString(m_module, "loop"); dumpError(); -// PyObject *stop = PyObject_GetAttrString(loop, "stop"); -// PyObject *call_soon = PyObject_GetAttrString(loop, "call_soon"); PyObject *run_forever = PyObject_GetAttrString(loop, "run_forever"); -// PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); - qCDebug(dcThingManager()) << "Starting python event loop"; dumpError(); PyObject_CallFunctionObjArgs(run_forever, nullptr); - qCDebug(dcThingManager()) << "stopped python event loop"; dumpError(); - }); @@ -176,7 +147,6 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) QJsonObject PythonIntegrationPlugin::metaData() const { return QJsonObject::fromVariantMap(m_metaData); - } void PythonIntegrationPlugin::init() @@ -208,7 +178,7 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) return; } - PyThingDiscoveryInfo *pyInfo = (PyThingDiscoveryInfo*)_PyObject_New(&PyThingDiscoveryInfoType); + PyThingDiscoveryInfo *pyInfo = reinterpret_cast(_PyObject_New(&PyThingDiscoveryInfoType)); pyInfo->ptrObj = info; connect(info, &ThingDiscoveryInfo::finished, this, [=](){ @@ -241,7 +211,7 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) return; } - PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)_PyObject_New(&PyThingSetupInfoType); + PyThingSetupInfo *pyInfo = reinterpret_cast(_PyObject_New(&PyThingSetupInfoType)); pyInfo->ptrObj = info; connect(info, &ThingSetupInfo::finished, this, [=](){ @@ -287,7 +257,6 @@ void PythonIntegrationPlugin::exportIds() QVariantMap vendor = vendorVariant.toMap(); QString vendorIdName = vendor.value("name").toString() + "VendorId"; QString vendorId = vendor.value("id").toString(); - qCDebug(dcThingManager()) << "Exporting:" << vendorIdName; PyModule_AddStringConstant(m_module, vendorIdName.toUtf8(), vendorId.toUtf8()); foreach (const QVariant &thingClassVariant, vendor.value("thingClasses").toList()) { @@ -297,6 +266,5 @@ void PythonIntegrationPlugin::exportIds() PyModule_AddStringConstant(m_module, thingClassIdName.toUtf8(), thingClassId.toUtf8()); } } - } diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index e4f8cc70..852217fc 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -5,6 +5,7 @@ #include #include +#include extern "C" { typedef struct _object PyObject; @@ -17,6 +18,7 @@ class PythonIntegrationPlugin : public IntegrationPlugin Q_OBJECT public: explicit PythonIntegrationPlugin(QObject *parent = nullptr); + ~PythonIntegrationPlugin(); static void initPython(); @@ -41,6 +43,7 @@ private: QVariantMap m_metaData; PyObject *m_module = nullptr; PyThreadState *m_interpreter = nullptr; + QFuture m_eventLoop; }; diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index 9e6dd20d..dba852e4 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -21,6 +21,7 @@ RESOURCES += $$top_srcdir/icons.qrc \ HEADERS += nymeacore.h \ integrations/plugininfocache.h \ integrations/python/pything.h \ + integrations/python/pythingdescriptor.h \ integrations/python/pythingdiscoveryinfo.h \ integrations/python/pythingsetupinfo.h \ integrations/thingmanagerimplementation.h \ diff --git a/libnymea/loggingcategories.h b/libnymea/loggingcategories.h index b2f8849d..68557d79 100644 --- a/libnymea/loggingcategories.h +++ b/libnymea/loggingcategories.h @@ -89,6 +89,9 @@ Q_DECLARE_LOGGING_CATEGORY(dcMqtt) Q_DECLARE_LOGGING_CATEGORY(dcTranslations) Q_DECLARE_LOGGING_CATEGORY(dcCoap) Q_DECLARE_LOGGING_CATEGORY(dcI2C) +Q_DECLARE_LOGGING_CATEGORY(dcIntegrations) +Q_DECLARE_LOGGING_CATEGORY(dcJsIntegrations) +Q_DECLARE_LOGGING_CATEGORY(dcPythonIntegrations) /* From 13d10b8aa09fbd829863eaba1ed5fee3dc95716f Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 15 Jun 2020 17:56:54 +0200 Subject: [PATCH 06/46] some more python plugin work --- .../python/pynymealogginghandler.h | 112 ++++++++ libnymea-core/integrations/python/pything.h | 44 +++- .../python/pythingdiscoveryinfo.h | 5 +- .../integrations/python/pythingsetupinfo.h | 16 +- .../integrations/pythonintegrationplugin.cpp | 246 ++++++++++++------ .../integrations/pythonintegrationplugin.h | 15 +- .../thingmanagerimplementation.cpp | 34 +-- libnymea-core/libnymea-core.pro | 1 + 8 files changed, 361 insertions(+), 112 deletions(-) create mode 100644 libnymea-core/integrations/python/pynymealogginghandler.h diff --git a/libnymea-core/integrations/python/pynymealogginghandler.h b/libnymea-core/integrations/python/pynymealogginghandler.h new file mode 100644 index 00000000..69599a40 --- /dev/null +++ b/libnymea-core/integrations/python/pynymealogginghandler.h @@ -0,0 +1,112 @@ +#ifndef PYNYMEALOGGINGHANDLER_H +#define PYNYMEALOGGINGHANDLER_H + +#include +#include "structmember.h" + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +typedef struct { + PyObject_HEAD + char *category; +} PyNymeaLoggingHandler; + +static int PyNymeaLoggingHandler_init(PyNymeaLoggingHandler */*self*/, PyObject */*args*/, PyObject */*kwds*/) +{ + return 0; +} + +static void PyNymeaLoggingHandler_dealloc(PyNymeaLoggingHandler * self) +// destruct the object +{ + // FIXME: Why is this not called? Seems we're leaking... + Q_ASSERT(false); + Py_TYPE(self)->tp_free(self); +} + + +static PyObject * PyNymeaLoggingHandler_log(PyNymeaLoggingHandler* self, PyObject* args) +{ + QStringList strings; + for (int i = 0; i < PyTuple_GET_SIZE(args); i++) { + PyObject *obj = PyTuple_GET_ITEM(args, i); + PyObject* repr = PyObject_Repr(obj); + PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + const char *bytes = PyBytes_AS_STRING(str); + Py_XDECREF(repr); + Py_XDECREF(str); + strings.append(bytes); + } + qCDebug(QLoggingCategory(self->category)).noquote() << strings.join(' '); + Py_RETURN_NONE; +} + +static PyObject * PyNymeaLoggingHandler_warn(PyNymeaLoggingHandler* self, PyObject* args) +{ + QStringList strings; + for (int i = 0; i < PyTuple_GET_SIZE(args); i++) { + PyObject *obj = PyTuple_GET_ITEM(args, i); + PyObject* repr = PyObject_Repr(obj); + PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + const char *bytes = PyBytes_AS_STRING(str); + Py_XDECREF(repr); + Py_XDECREF(str); + strings.append(bytes); + } + qCWarning(QLoggingCategory(self->category)).noquote() << strings.join(' '); + Py_RETURN_NONE; +} + +static PyMethodDef PyNymeaLoggingHandler_methods[] = { + { "log", (PyCFunction)PyNymeaLoggingHandler_log, METH_VARARGS, "Add a new descriptor to the discovery" }, + { "warn", (PyCFunction)PyNymeaLoggingHandler_warn, METH_VARARGS, "Add a new descriptor to the discovery" }, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyTypeObject PyNymeaLoggingHandlerType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.NymeaLoggingHandler", /* tp_name */ + sizeof(PyNymeaLoggingHandler), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Logging handler for nymea", /* tp_doc */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + + +static void registerNymeaLoggingHandler(PyObject *module) +{ + + PyNymeaLoggingHandlerType.tp_new = PyType_GenericNew; + PyNymeaLoggingHandlerType.tp_dealloc = reinterpret_cast(PyNymeaLoggingHandler_dealloc); + PyNymeaLoggingHandlerType.tp_flags = Py_TPFLAGS_DEFAULT; + PyNymeaLoggingHandlerType.tp_doc = "NymeaLoggingHandler class"; + PyNymeaLoggingHandlerType.tp_methods = PyNymeaLoggingHandler_methods; + PyNymeaLoggingHandlerType.tp_init = reinterpret_cast(PyNymeaLoggingHandler_init); + if (PyType_Ready(&PyNymeaLoggingHandlerType) == 0) { + PyModule_AddObject(module, "NymeaLoggingHandler", (PyObject *)&PyNymeaLoggingHandlerType); + } +} + + +#endif // PYNYMEALOGGINGHANDLER_H diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 5fc9883e..58d83c8a 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -10,7 +10,7 @@ #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" -typedef struct { +typedef struct _thing { PyObject_HEAD Thing* ptrObj; } PyThing; @@ -31,9 +31,48 @@ static void PyThing_dealloc(PyThing * self) Py_TYPE(self)->tp_free(self); } +static PyObject *PyThing_getName(PyThing *self, void */*closure*/) +{ + if (!self->ptrObj) { + PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); + return nullptr; + } + PyObject *ret = PyUnicode_FromString(self->ptrObj->name().toUtf8().data()); + Py_INCREF(ret); + return ret; +} + +static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ + self->ptrObj->setName(QString(PyUnicode_AsUTF8(value))); + return 0; +} + +static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) +{ + char *stateTypeId; + int status; + + if (PyArg_ParseTuple(args, "ss", &stateTypeId, &message)) { + (self->ptrObj)->finish(static_cast(status), QString(message)); + Py_RETURN_NONE; + } + PyErr_Clear(); + + if (PyArg_ParseTuple(args, "i", &status)) { + (self->ptrObj)->finish(static_cast(status)); + Py_RETURN_NONE; + } + + Py_RETURN_NONE; +} + +static PyGetSetDef PyThing_getseters[] = { + {"name", (getter)PyThing_getName, (setter)PyThing_setName, "Thingname", nullptr}, + {nullptr , nullptr, nullptr, nullptr, nullptr} /* Sentinel */ +}; static PyMethodDef PyThing_methods[] = { -// { "addDescriptor", (PyCFunction)PyThing_addDescriptor, METH_VARARGS, "Add a new descriptor to the discovery" }, + { "setStateValue", (PyCFunction)PyThing_setStateValue, METH_VARARGS, "Set a things state value" }, // { "finish", (PyCFunction)PyThing_finish, METH_VARARGS, "finish a discovery" }, {nullptr, nullptr, 0, nullptr} // sentinel }; @@ -71,6 +110,7 @@ static void registerThingType(PyObject *module) PyThingType.tp_flags = Py_TPFLAGS_DEFAULT; PyThingType.tp_doc = "Thing class"; PyThingType.tp_methods = PyThing_methods; + PyThingType.tp_getset = PyThing_getseters; // PyThingType.tp_members = PyThingSetupInfo_members; PyThingType.tp_init = reinterpret_cast(PyThing_init); diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index ceb955fb..8038411b 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -45,14 +45,15 @@ static PyObject * PyThingDiscoveryInfo_finish(PyThingDiscoveryInfo* self, PyObje if (PyArg_ParseTuple(args, "is", &status, &message)) { (self->ptrObj)->finish(static_cast(status), QString(message)); - return Py_BuildValue(""); + Py_RETURN_NONE; } if (PyArg_ParseTuple(args, "i", &status)) { (self->ptrObj)->finish(static_cast(status)); - return Py_BuildValue(""); + Py_RETURN_NONE; } + PyErr_SetString(PyExc_TypeError, "Invalid arguments in finish call. Expected: finish(ThingError, message = \"\""); return nullptr; } diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index e62cba5b..3b63bb1a 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -15,18 +15,17 @@ typedef struct { PyObject_HEAD ThingSetupInfo* ptrObj; + PyThing *thing; } PyThingSetupInfo; static int PyThingSetupInfo_init(PyThingSetupInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) -// initialize PyVoice Object { return 0; } static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) -// destruct the object { // FIXME: Why is this not called? Seems we're leaking... Q_ASSERT(false); @@ -40,17 +39,22 @@ static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args if (PyArg_ParseTuple(args, "is", &status, &message)) { (self->ptrObj)->finish(static_cast(status), QString(message)); - return Py_BuildValue(""); + Py_RETURN_NONE; } + PyErr_Clear(); if (PyArg_ParseTuple(args, "i", &status)) { (self->ptrObj)->finish(static_cast(status)); - return Py_BuildValue(""); + Py_RETURN_NONE; } - return nullptr; + Py_RETURN_NONE; } +static PyMemberDef PyThingSetupInfo_members[] = { + {"thing", T_OBJECT_EX, offsetof(PyThingSetupInfo, thing), 0, "Thing being setup in this setup transaction"}, + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +}; static PyMethodDef PyThingSetupInfo_methods[] = { { "finish", (PyCFunction)PyThingSetupInfo_finish, METH_VARARGS, "finish a setup" }, @@ -89,7 +93,7 @@ static void registerThingSetupInfoType(PyObject *module) { PyThingSetupInfoType.tp_flags = Py_TPFLAGS_DEFAULT; PyThingSetupInfoType.tp_doc = "ThingSetupInfo class"; PyThingSetupInfoType.tp_methods = PyThingSetupInfo_methods; -// PyThingSetupInfoType.tp_members = PyThingSetupInfo_members; + PyThingSetupInfoType.tp_members = PyThingSetupInfo_members; PyThingSetupInfoType.tp_init = (initproc)PyThingSetupInfo_init; if (PyType_Ready(&PyThingSetupInfoType) < 0) { diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 58a6b1c9..6bd51878 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -1,5 +1,6 @@ #include +#include "python/pynymealogginghandler.h" #include "python/pything.h" #include "python/pythingdiscoveryinfo.h" #include "python/pythingsetupinfo.h" @@ -14,29 +15,66 @@ #include QHash s_modules; +PyThreadState* PythonIntegrationPlugin::s_mainThread = nullptr; +PyObject* PythonIntegrationPlugin::s_nymeaModule = nullptr; +PyObject* PythonIntegrationPlugin::s_asyncio = nullptr; + + + + // Write to stdout/stderr -PyObject* nymea_write(PyObject* self, PyObject* args) +PyObject* nymea_write(PyObject* /*self*/, PyObject* args) { - qWarning() << "write called" << self; const char *what; if (!PyArg_ParseTuple(args, "s", &what)) return nullptr; - qCDebug(dcThingManager()) << what; - return Py_BuildValue(""); + if (!QByteArray(what).trimmed().isEmpty()) { + qCDebug(dcThingManager()) << what; + } + Py_RETURN_NONE; } // Flush stdout/stderr PyObject* nymea_flush(PyObject* /*self*/, PyObject* /*args*/) { - // Not really needed... qDebug() fluses already on its own - return Py_BuildValue(""); + // Not really needed... qDebug() flushes already on its own + Py_RETURN_NONE; } +PyObject* task_done(PyObject* /*self*/, PyObject* args) +{ + + PyObject *result = nullptr; + + if (!PyArg_ParseTuple(args, "O", &result)) { + qCWarning(dcThingManager()) << "Cannot fetch result from coroutine callback."; + return nullptr; + } + + PyObject *exceptionMethod = PyObject_GetAttrString(result, "exception"); + + PyObject *exception = PyObject_CallFunctionObjArgs(exceptionMethod, nullptr); + if (exception != Py_None) { + PyObject* repr = PyObject_Repr(exception); + PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + const char *bytes = PyBytes_AS_STRING(str); + Py_XDECREF(repr); + Py_XDECREF(str); + + qCWarning(dcThingManager()) << "Exception:" << bytes; + + } + + Py_RETURN_NONE; +} + + static PyMethodDef nymea_methods[] = { {"write", nymea_write, METH_VARARGS, "write to stdout through qDebug()"}, {"flush", nymea_flush, METH_VARARGS, "flush stdout (no-op)"}, + {"task_done", task_done, METH_VARARGS, "callback to clean up after asyc coroutines"}, {nullptr, nullptr, 0, nullptr} // sentinel }; @@ -59,6 +97,7 @@ PyMODINIT_FUNC PyInit_nymea(void) PySys_SetObject("stderr", m); + registerNymeaLoggingHandler(m); registerThingType(m); registerThingDescriptorType(m); registerThingDiscoveryInfoType(m); @@ -82,6 +121,30 @@ void PythonIntegrationPlugin::initPython() { PyImport_AppendInittab("nymea", PyInit_nymea); Py_InitializeEx(0); + PyEval_InitThreads(); + + // Import nymea module into this interpreter + s_nymeaModule = PyImport_ImportModule("nymea"); + + + + + + // Spawn a event loop for python + s_asyncio = PyImport_ImportModule("asyncio"); + PyObject *get_event_loop = PyObject_GetAttrString(s_asyncio, "get_event_loop"); + PyObject *loop = PyObject_CallFunctionObjArgs(get_event_loop, nullptr); + PyObject *run_forever = PyObject_GetAttrString(loop, "run_forever"); + + // Need to release ths lock from the main thread before spawning the new thread + s_mainThread = PyEval_SaveThread(); + + QtConcurrent::run([=](){ + PyGILState_STATE s = PyGILState_Ensure(); + PyObject_CallFunctionObjArgs(run_forever, nullptr); + PyGILState_Release(s); + }); + } bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) @@ -102,45 +165,36 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) m_metaData = jsonDoc.toVariant().toMap(); - // Create a new sub-interpreter for this plugin - m_interpreter = Py_NewInterpreter(); - PyThreadState_Swap(m_interpreter); - - - // Import nymea module into this interpreter - PyImport_ImportModule("nymea"); - - // Add this plugin's locatioon to the import path - PyObject* sysPath = PySys_GetObject("path"); - PyList_Append(sysPath, PyUnicode_FromString(fi.absolutePath().toUtf8())); + PyGILState_STATE s = PyGILState_Ensure(); // Finally, import the plugin + PyObject* sysPath = PySys_GetObject("path"); + PyList_Append(sysPath, PyUnicode_FromString(fi.absolutePath().toUtf8())); m_module = PyImport_ImportModule(fi.baseName().toUtf8()); if (!m_module) { dumpError(); qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath(); + PyGILState_Release(s); return false; } qCDebug(dcThingManager()) << "Imported python plugin from" << fi.absoluteFilePath(); + // Set up logger with appropriate logging category + PyNymeaLoggingHandler *logger = reinterpret_cast(_PyObject_New(&PyNymeaLoggingHandlerType)); + QString category = m_metaData.value("name").toString(); + category = category.left(1).toUpper() + category.right(category.length() - 1); + logger->category = static_cast(malloc(category.length() + 1)); + memset(logger->category, '0', category.length() +1); + strcpy(logger->category, category.toUtf8().data()); + PyModule_AddObject(m_module, "logger", reinterpret_cast(logger)); + + + // Export metadata ids into module exportIds(); - - s_modules.insert(m_module, m_interpreter); - - // Start an event loop for this plugin in its own thread - m_eventLoop = QtConcurrent::run([=](){ - PyObject *loop = PyObject_GetAttrString(m_module, "loop"); - dumpError(); - PyObject *run_forever = PyObject_GetAttrString(loop, "run_forever"); - dumpError(); - PyObject_CallFunctionObjArgs(run_forever, nullptr); - dumpError(); - }); - - + PyGILState_Release(s); return true; } @@ -151,33 +205,11 @@ QJsonObject PythonIntegrationPlugin::metaData() const void PythonIntegrationPlugin::init() { - PyThreadState_Swap(m_interpreter); - - qCDebug(dcThingManager()) << "Python wrapper: init()" << m_interpreter; - PyObject *pFunc = PyObject_GetAttrString(m_module, "init"); - if(!pFunc || !PyCallable_Check(pFunc)) { - qCDebug(dcThingManager()) << "Python plugin does not implement \"init()\" method."; - return; - } - PyObject_CallObject(pFunc, nullptr); - dumpError(); + callPluginFunction("init", nullptr); } void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) { - - PyThreadState_Swap(m_interpreter); - - PyGILState_STATE s = PyGILState_Ensure(); - - - qCDebug(dcThingManager()) << "Python wrapper: discoverThings()" << info; - PyObject *pFunc = PyObject_GetAttrString(m_module, "discoverThings"); - if(!pFunc || !PyCallable_Check(pFunc)) { - qCWarning(dcThingManager()) << "Python plugin does not implement \"discoverThings()\" method."; - return; - } - PyThingDiscoveryInfo *pyInfo = reinterpret_cast(_PyObject_New(&PyThingDiscoveryInfoType)); pyInfo->ptrObj = info; @@ -185,49 +217,43 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) PyObject_Free(pyInfo); }); - - PyObject *future = PyObject_CallFunctionObjArgs(pFunc, pyInfo, nullptr); - dumpError(); - - PyObject *asyncio = PyObject_GetAttrString(m_module, "asyncio"); - PyObject *loop = PyObject_GetAttrString(m_module, "loop"); - PyObject *run_coroutine_threadsafe = PyObject_GetAttrString(asyncio, "run_coroutine_threadsafe"); - PyObject_CallFunctionObjArgs(run_coroutine_threadsafe, future, loop, nullptr); - - PyGILState_Release(s); - + callPluginFunction("discoverThings", reinterpret_cast(pyInfo)); } void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) { - PyThreadState_Swap(m_interpreter); - - PyGILState_STATE s = PyGILState_Ensure(); - - qCDebug(dcThingManager()) << "Python wrapper: setupThing()" << info; - PyObject *pFunc = PyObject_GetAttrString(m_module, "setupThing"); - if(!pFunc || !PyCallable_Check(pFunc)) { - qCWarning(dcThingManager()) << "Python plugin does not implement \"setThing()\" method."; - return; - } + PyThing *pyThing = reinterpret_cast(_PyObject_New(&PyThingType)); + pyThing->ptrObj = info->thing(); + m_things.insert(info->thing(), pyThing); + Py_INCREF(pyThing); PyThingSetupInfo *pyInfo = reinterpret_cast(_PyObject_New(&PyThingSetupInfoType)); pyInfo->ptrObj = info; + pyInfo->thing = pyThing; connect(info, &ThingSetupInfo::finished, this, [=](){ PyObject_Free(pyInfo); }); + callPluginFunction("setupThing", reinterpret_cast(pyInfo)); +} - PyObject *future = PyObject_CallFunctionObjArgs(pFunc, pyInfo, nullptr); - dumpError(); +void PythonIntegrationPlugin::postSetupThing(Thing *thing) +{ + PyThing* pyThing = m_things.value(thing); + callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); +} - PyObject *asyncio = PyObject_GetAttrString(m_module, "asyncio"); - PyObject *loop = PyObject_GetAttrString(m_module, "loop"); - PyObject *run_coroutine_threadsafe = PyObject_GetAttrString(asyncio, "run_coroutine_threadsafe"); - PyObject_CallFunctionObjArgs(run_coroutine_threadsafe, future, loop, nullptr); +void PythonIntegrationPlugin::thingRemoved(Thing *thing) +{ + PyThing *pyThing = m_things.value(thing); - PyGILState_Release(s); + callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); + + pyThing->ptrObj = nullptr; + Py_DECREF(pyThing); + + m_things.remove(thing); } void PythonIntegrationPlugin::dumpError() @@ -268,3 +294,59 @@ void PythonIntegrationPlugin::exportIds() } } +void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param) +{ + PyGILState_STATE s = PyGILState_Ensure(); + + qCDebug(dcThingManager()) << "Calling python plugin function" << function; + PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); + if(!pFunc || !PyCallable_Check(pFunc)) { + Py_XDECREF(pFunc); + qCWarning(dcThingManager()) << "Python plugin does not implement" << function; + return; + } + + + dumpError(); + + PyObject *future = PyObject_CallFunctionObjArgs(pFunc, param, nullptr); + + Py_XDECREF(pFunc); + + if (PyErr_Occurred()) { + qCWarning(dcThingManager()) << "Error calling python method:"; + dumpError(); + PyGILState_Release(s); + return; + } + + if (QByteArray(future->ob_type->tp_name) != "coroutine") { + PyGILState_Release(s); + return; + } + + PyObject *get_event_loop = PyObject_GetAttrString(s_asyncio, "get_event_loop"); + PyObject *loop = PyObject_CallFunctionObjArgs(get_event_loop, nullptr); + + PyObject *run_coroutine_threadsafe = PyObject_GetAttrString(s_asyncio, "run_coroutine_threadsafe"); + PyObject *task = PyObject_CallFunctionObjArgs(run_coroutine_threadsafe, future, loop, nullptr); + dumpError(); + + PyObject *add_done_callback = PyObject_GetAttrString(task, "add_done_callback"); + dumpError(); + + PyObject *task_done = PyObject_GetAttrString(s_nymeaModule, "task_done"); + PyObject *result = PyObject_CallFunctionObjArgs(add_done_callback, task_done, nullptr); + dumpError(); + + Py_DECREF(get_event_loop); + Py_DECREF(loop); + Py_DECREF(run_coroutine_threadsafe); + Py_DECREF(task); + Py_DECREF(add_done_callback); + Py_DECREF(get_event_loop); + Py_DECREF(result); + + PyGILState_Release(s); +} + diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 852217fc..92f75b51 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -10,6 +10,7 @@ extern "C" { typedef struct _object PyObject; typedef struct _ts PyThreadState; +typedef struct _thing PyThing; } @@ -30,21 +31,29 @@ public: void init() override; void discoverThings(ThingDiscoveryInfo *info) override; void setupThing(ThingSetupInfo *info) override; + void postSetupThing(Thing *thing) override; + void thingRemoved(Thing *thing) override; + static void dumpError(); private: - void dumpError(); - void exportIds(); + void callPluginFunction(const QString &function, PyObject *param); + private: // static QHash s_modules; + static PyThreadState* s_mainThread; + static PyObject *s_nymeaModule; + static PyObject *s_asyncio; + QVariantMap m_metaData; PyObject *m_module = nullptr; - PyThreadState *m_interpreter = nullptr; QFuture m_eventLoop; + QHash m_things; + }; #endif // PYTHONINTEGRATIONPLUGIN_H diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index 7036863e..f37ad9f6 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -1362,24 +1362,24 @@ void ThingManagerImplementation::loadPlugins() } loadPlugin(p, metaData); } -// { -// PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); -// bool ok = p->loadScript("/home/micha/Develop/nymea-plugin-pytest2/integrationpluginpytest2.py"); -// if (!ok) { -// qCWarning(dcThingManager()) << "Error loading plugin"; -// return; -// } -// PluginMetadata metaData(p->metaData()); -// if (!metaData.isValid()) { -// qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; -// foreach (const QString &error, metaData.validationErrors()) { -// qCWarning(dcThingManager()) << error; -// } -// return; -// } -// loadPlugin(p, metaData); + { + PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); + bool ok = p->loadScript("/home/micha/Develop/nymea-plugin-pytest2/integrationpluginpytest2.py"); + if (!ok) { + qCWarning(dcThingManager()) << "Error loading plugin"; + return; + } + PluginMetadata metaData(p->metaData()); + if (!metaData.isValid()) { + qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; + foreach (const QString &error, metaData.validationErrors()) { + qCWarning(dcThingManager()) << error; + } + return; + } + loadPlugin(p, metaData); -// } + } } void ThingManagerImplementation::loadPlugin(IntegrationPlugin *pluginIface, const PluginMetadata &metaData) diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index dba852e4..f2c5bd87 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -20,6 +20,7 @@ RESOURCES += $$top_srcdir/icons.qrc \ HEADERS += nymeacore.h \ integrations/plugininfocache.h \ + integrations/python/pynymealogginghandler.h \ integrations/python/pything.h \ integrations/python/pythingdescriptor.h \ integrations/python/pythingdiscoveryinfo.h \ From 125aee71530ae1d764e474ca44ff0a099b5254f8 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Tue, 16 Jun 2020 15:06:36 +0200 Subject: [PATCH 07/46] Some more python plugin stuff --- libnymea-core/integrations/python/pything.h | 35 +++-- .../integrations/pythonintegrationplugin.cpp | 133 +++++++++++++++--- .../integrations/pythonintegrationplugin.h | 13 +- .../thingmanagerimplementation.cpp | 9 +- libnymea/integrations/integrationplugin.cpp | 5 + libnymea/integrations/integrationplugin.h | 5 +- 6 files changed, 161 insertions(+), 39 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 58d83c8a..37504721 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -6,13 +6,15 @@ #include "integrations/thing.h" +#include + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" typedef struct _thing { PyObject_HEAD - Thing* ptrObj; + Thing *ptrObj; } PyThing; @@ -37,32 +39,45 @@ static PyObject *PyThing_getName(PyThing *self, void */*closure*/) PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); return nullptr; } + // FIXME: Needs blocking queued connection PyObject *ret = PyUnicode_FromString(self->ptrObj->name().toUtf8().data()); Py_INCREF(ret); return ret; } static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ + // FIXME: Needs queued connection self->ptrObj->setName(QString(PyUnicode_AsUTF8(value))); return 0; } static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) { - char *stateTypeId; - int status; + char *stateTypeIdStr; + PyObject *valueObj; - if (PyArg_ParseTuple(args, "ss", &stateTypeId, &message)) { - (self->ptrObj)->finish(static_cast(status), QString(message)); - Py_RETURN_NONE; + // FIXME: is there any better way to do this? Value is a variant + if (!PyArg_ParseTuple(args, "sO", &stateTypeIdStr, &valueObj)) { + qWarning() << "error parsing parameters"; + return nullptr; } - PyErr_Clear(); - if (PyArg_ParseTuple(args, "i", &status)) { - (self->ptrObj)->finish(static_cast(status)); - Py_RETURN_NONE; + StateTypeId stateTypeId = StateTypeId(stateTypeIdStr); + + PyObject* repr = PyObject_Repr(valueObj); + PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + const char *bytes = PyBytes_AS_STRING(str); + + qWarning() << "params:" << stateTypeId << bytes << self; + QVariant value(bytes); + + if (self->ptrObj != nullptr) { + QMetaObject::invokeMethod(self->ptrObj, "setStateValue", Qt::QueuedConnection, Q_ARG(StateTypeId, stateTypeId), Q_ARG(QVariant, value)); } + Py_XDECREF(repr); + Py_XDECREF(str); + Py_RETURN_NONE; } diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 6bd51878..f029c907 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -162,7 +162,7 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) qCWarning(dcThingManager()) << "Error parsing metadata file:" << error.errorString(); return false; } - m_metaData = jsonDoc.toVariant().toMap(); + m_metaData = PluginMetadata(jsonDoc.object()); PyGILState_STATE s = PyGILState_Ensure(); @@ -183,8 +183,8 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) // Set up logger with appropriate logging category PyNymeaLoggingHandler *logger = reinterpret_cast(_PyObject_New(&PyNymeaLoggingHandlerType)); - QString category = m_metaData.value("name").toString(); - category = category.left(1).toUpper() + category.right(category.length() - 1); + QString category = m_metaData.pluginName(); + category.replace(0, 1, category[0].toUpper()); logger->category = static_cast(malloc(category.length() + 1)); memset(logger->category, '0', category.length() +1); strcpy(logger->category, category.toUtf8().data()); @@ -198,11 +198,6 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) return true; } -QJsonObject PythonIntegrationPlugin::metaData() const -{ - return QJsonObject::fromVariantMap(m_metaData); -} - void PythonIntegrationPlugin::init() { callPluginFunction("init", nullptr); @@ -250,9 +245,14 @@ void PythonIntegrationPlugin::thingRemoved(Thing *thing) callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); + PyGILState_STATE s = PyGILState_Ensure(); + pyThing->ptrObj = nullptr; Py_DECREF(pyThing); + + PyGILState_Release(s); + m_things.remove(thing); } @@ -279,21 +279,115 @@ void PythonIntegrationPlugin::dumpError() void PythonIntegrationPlugin::exportIds() { - foreach (const QVariant &vendorVariant, m_metaData.value("vendors").toList()) { - QVariantMap vendor = vendorVariant.toMap(); - QString vendorIdName = vendor.value("name").toString() + "VendorId"; - QString vendorId = vendor.value("id").toString(); - PyModule_AddStringConstant(m_module, vendorIdName.toUtf8(), vendorId.toUtf8()); + qCDebug(dcThingManager()) << "Exporting plugin IDs:"; + QString pluginName = "pluginId"; + QString pluginId = m_metaData.pluginId().toString(); + qCDebug(dcThingManager()) << "- Plugin:" << pluginName << pluginId; + PyModule_AddStringConstant(m_module, pluginName.toUtf8(), pluginId.toUtf8()); - foreach (const QVariant &thingClassVariant, vendor.value("thingClasses").toList()) { - QVariantMap thingClass = thingClassVariant.toMap(); - QString thingClassIdName = thingClass.value("name").toString() + "ThingClassId"; - QString thingClassId = thingClass.value("id").toString(); - PyModule_AddStringConstant(m_module, thingClassIdName.toUtf8(), thingClassId.toUtf8()); - } + foreach (const ThingClass &thingClass, supportedThings()) { + exportThingClass(thingClass); } } +void PythonIntegrationPlugin::exportThingClass(const ThingClass &thingClass) +{ + QString variableName = QString("%1ThingClassId").arg(thingClass.name()); + if (m_variableNames.contains(variableName)) { + qWarning().nospace() << "Error: Duplicate name " << variableName << " for ThingClass " << thingClass.id() << ". Skipping entry."; + return; + } + m_variableNames.append(variableName); + + qCDebug(dcThingManager()) << "|- ThingClass:" << variableName << thingClass.id(); + PyModule_AddStringConstant(m_module, variableName.toUtf8(), thingClass.id().toString().toUtf8()); + + exportParamTypes(thingClass.paramTypes(), thingClass.name(), "", "thing"); + exportParamTypes(thingClass.settingsTypes(), thingClass.name(), "", "settings"); + exportParamTypes(thingClass.discoveryParamTypes(), thingClass.name(), "", "discovery"); + + exportStateTypes(thingClass.stateTypes(), thingClass.name()); + exportEventTypes(thingClass.eventTypes(), thingClass.name()); + exportActionTypes(thingClass.actionTypes(), thingClass.name()); + exportBrowserItemActionTypes(thingClass.browserItemActionTypes(), thingClass.name()); +} + +void PythonIntegrationPlugin::exportParamTypes(const ParamTypes ¶mTypes, const QString &thingClassName, const QString &typeClass, const QString &typeName) +{ + foreach (const ParamType ¶mType, paramTypes) { + QString variableName = QString("%1ParamTypeId").arg(thingClassName + typeName[0].toUpper() + typeName.right(typeName.length()-1) + typeClass + paramType.name()[0].toUpper() + paramType.name().right(paramType.name().length() -1 )); + if (m_variableNames.contains(variableName)) { + qWarning().nospace() << "Error: Duplicate name " << variableName << " for ParamTypeId " << paramType.id() << ". Skipping entry."; + continue; + } + m_variableNames.append(variableName); + + PyModule_AddStringConstant(m_module, variableName.toUtf8(), paramType.id().toString().toUtf8()); + } +} + +void PythonIntegrationPlugin::exportStateTypes(const StateTypes &stateTypes, const QString &thingClassName) +{ + foreach (const StateType &stateType, stateTypes) { + QString variableName = QString("%1%2StateTypeId").arg(thingClassName, stateType.name()[0].toUpper() + stateType.name().right(stateType.name().length() - 1)); + if (m_variableNames.contains(variableName)) { + qWarning().nospace() << "Error: Duplicate name " << variableName << " for StateType " << stateType.name() << " in ThingClass " << thingClassName << ". Skipping entry."; + return; + } + m_variableNames.append(variableName); + qCDebug(dcThingManager()) << "|- StateType:" << variableName << stateType.id(); + PyModule_AddStringConstant(m_module, variableName.toUtf8(), stateType.id().toString().toUtf8()); + } +} + +void PythonIntegrationPlugin::exportEventTypes(const EventTypes &eventTypes, const QString &thingClassName) +{ + foreach (const EventType &eventType, eventTypes) { + QString variableName = QString("%1%2EventTypeId").arg(thingClassName, eventType.name()[0].toUpper() + eventType.name().right(eventType.name().length() - 1)); + if (m_variableNames.contains(variableName)) { + qWarning().nospace() << "Error: Duplicate name " << variableName << " for EventType " << eventType.name() << " in ThingClass " << thingClassName << ". Skipping entry."; + return; + } + m_variableNames.append(variableName); + PyModule_AddStringConstant(m_module, variableName.toUtf8(), eventType.id().toString().toUtf8()); + + exportParamTypes(eventType.paramTypes(), thingClassName, "Event", eventType.name()); + } + +} + +void PythonIntegrationPlugin::exportActionTypes(const ActionTypes &actionTypes, const QString &thingClassName) +{ + foreach (const ActionType &actionType, actionTypes) { + QString variableName = QString("%1%2ActionTypeId").arg(thingClassName, actionType.name()[0].toUpper() + actionType.name().right(actionType.name().length() - 1)); + if (m_variableNames.contains(variableName)) { + qWarning().nospace() << "Error: Duplicate name " << variableName << " for ActionType " << actionType.name() << " in ThingClass " << thingClassName << ". Skipping entry."; + return; + } + m_variableNames.append(variableName); + PyModule_AddStringConstant(m_module, variableName.toUtf8(), actionType.id().toString().toUtf8()); + + exportParamTypes(actionType.paramTypes(), thingClassName, "Action", actionType.name()); + } +} + +void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &actionTypes, const QString &thingClassName) +{ + foreach (const ActionType &actionType, actionTypes) { + QString variableName = QString("%1%2BrowserItemActionTypeId").arg(thingClassName, actionType.name()[0].toUpper() + actionType.name().right(actionType.name().length() - 1)); + if (m_variableNames.contains(variableName)) { + qWarning().nospace() << "Error: Duplicate name " << variableName << " for Browser Item ActionType " << actionType.name() << " in ThingClass " << thingClassName << ". Skipping entry."; + return; + } + m_variableNames.append(variableName); + PyModule_AddStringConstant(m_module, variableName.toUtf8(), actionType.id().toString().toUtf8()); + + exportParamTypes(actionType.paramTypes(), thingClassName, "BrowserItemAction", actionType.name()); + } + +} + + void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param) { PyGILState_STATE s = PyGILState_Ensure(); @@ -303,6 +397,7 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje if(!pFunc || !PyCallable_Check(pFunc)) { Py_XDECREF(pFunc); qCWarning(dcThingManager()) << "Python plugin does not implement" << function; + PyGILState_Release(s); return; } diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 92f75b51..1e57686d 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -25,9 +25,6 @@ public: bool loadScript(const QString &scriptFile); - QJsonObject metaData() const; - - void init() override; void discoverThings(ThingDiscoveryInfo *info) override; void setupThing(ThingSetupInfo *info) override; @@ -38,6 +35,13 @@ public: static void dumpError(); private: void exportIds(); + void exportThingClass(const ThingClass &thingClass); + void exportParamTypes(const ParamTypes ¶mTypes, const QString &thingClassName, const QString &typeClass, const QString &typeName); + void exportStateTypes(const StateTypes &stateTypes, const QString &thingClassName); + void exportEventTypes(const EventTypes &eventTypes, const QString &thingClassName); + void exportActionTypes(const ActionTypes &actionTypes, const QString &thingClassName); + void exportBrowserItemActionTypes(const ActionTypes &actionTypes, const QString &thingClassName); + void callPluginFunction(const QString &function, PyObject *param); @@ -48,12 +52,13 @@ private: static PyObject *s_nymeaModule; static PyObject *s_asyncio; - QVariantMap m_metaData; PyObject *m_module = nullptr; QFuture m_eventLoop; QHash m_things; + QStringList m_variableNames; + }; #endif // PYTHONINTEGRATIONPLUGIN_H diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index f37ad9f6..c8b0f14f 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -1352,15 +1352,14 @@ void ThingManagerImplementation::loadPlugins() qCWarning(dcThingManager()) << "Error loading plugin"; return; } - PluginMetadata metaData(p->metaData()); - if (!metaData.isValid()) { + if (!p->metadata().isValid()) { qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; - foreach (const QString &error, metaData.validationErrors()) { + foreach (const QString &error, p->metadata().validationErrors()) { qCWarning(dcThingManager()) << error; } return; } - loadPlugin(p, metaData); + loadPlugin(p, p->metadata()); } { PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); @@ -1369,7 +1368,7 @@ void ThingManagerImplementation::loadPlugins() qCWarning(dcThingManager()) << "Error loading plugin"; return; } - PluginMetadata metaData(p->metaData()); + PluginMetadata metaData(p->metadata()); if (!metaData.isValid()) { qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; foreach (const QString &error, metaData.validationErrors()) { diff --git a/libnymea/integrations/integrationplugin.cpp b/libnymea/integrations/integrationplugin.cpp index 269a2b67..17faaf45 100644 --- a/libnymea/integrations/integrationplugin.cpp +++ b/libnymea/integrations/integrationplugin.cpp @@ -110,6 +110,11 @@ IntegrationPlugin::~IntegrationPlugin() } +PluginMetadata IntegrationPlugin::metadata() +{ + return m_metaData; +} + /*! Returns the name of this IntegrationPlugin. It returns the name value defined in the plugin's JSON file. */ QString IntegrationPlugin::pluginName() const { diff --git a/libnymea/integrations/integrationplugin.h b/libnymea/integrations/integrationplugin.h index c525fd62..afefb72d 100644 --- a/libnymea/integrations/integrationplugin.h +++ b/libnymea/integrations/integrationplugin.h @@ -79,6 +79,8 @@ public: IntegrationPlugin(QObject *parent = nullptr); virtual ~IntegrationPlugin(); + PluginMetadata metadata(); + virtual void init() {} PluginId pluginId() const; @@ -124,6 +126,8 @@ protected: HardwareManager *hardwareManager() const; QSettings *pluginStorage() const; + PluginMetadata m_metaData; + private: friend class ThingManager; friend class ThingManagerImplementation; @@ -145,7 +149,6 @@ private: HardwareManager *m_hardwareManager = nullptr; QSettings *m_storage = nullptr; - PluginMetadata m_metaData; ParamList m_config; }; Q_DECLARE_INTERFACE(IntegrationPlugin, "io.nymea.IntegrationPlugin") From 68e9c827582d7ef1b8bbb64c129e370489beed3c Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Wed, 17 Jun 2020 14:32:10 +0200 Subject: [PATCH 08/46] run a thread for each call --- .../integrations/pythonintegrationplugin.cpp | 89 ++++++++++++------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index f029c907..0eebdb09 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -128,23 +128,22 @@ void PythonIntegrationPlugin::initPython() - - - // Spawn a event loop for python +// // Spawn a event loop for python s_asyncio = PyImport_ImportModule("asyncio"); - PyObject *get_event_loop = PyObject_GetAttrString(s_asyncio, "get_event_loop"); - PyObject *loop = PyObject_CallFunctionObjArgs(get_event_loop, nullptr); - PyObject *run_forever = PyObject_GetAttrString(loop, "run_forever"); +// PyObject *get_event_loop = PyObject_GetAttrString(s_asyncio, "get_event_loop"); +// PyObject *loop = PyObject_CallFunctionObjArgs(get_event_loop, nullptr); +// PyObject *run_forever = PyObject_GetAttrString(loop, "run_forever"); + + +// QtConcurrent::run([=](){ +// PyGILState_STATE s = PyGILState_Ensure(); +// PyObject_CallFunctionObjArgs(run_forever, nullptr); +// PyGILState_Release(s); +// }); + // Need to release ths lock from the main thread before spawning the new thread s_mainThread = PyEval_SaveThread(); - - QtConcurrent::run([=](){ - PyGILState_STATE s = PyGILState_Ensure(); - PyObject_CallFunctionObjArgs(run_forever, nullptr); - PyGILState_Release(s); - }); - } bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) @@ -205,11 +204,14 @@ void PythonIntegrationPlugin::init() void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) { - PyThingDiscoveryInfo *pyInfo = reinterpret_cast(_PyObject_New(&PyThingDiscoveryInfoType)); + PyThingDiscoveryInfo *pyInfo = PyObject_New(PyThingDiscoveryInfo, &PyThingDiscoveryInfoType); pyInfo->ptrObj = info; - connect(info, &ThingDiscoveryInfo::finished, this, [=](){ - PyObject_Free(pyInfo); + connect(info, &ThingDiscoveryInfo::destroyed, this, [=](){ + PyGILState_STATE s = PyGILState_Ensure(); + pyInfo->ptrObj = nullptr; + PyObject_Del(pyInfo); + PyGILState_Release(s); }); callPluginFunction("discoverThings", reinterpret_cast(pyInfo)); @@ -217,18 +219,34 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) { - PyThing *pyThing = reinterpret_cast(_PyObject_New(&PyThingType)); + PyThing *pyThing = PyObject_New(PyThing, &PyThingType); pyThing->ptrObj = info->thing(); - m_things.insert(info->thing(), pyThing); - Py_INCREF(pyThing); - PyThingSetupInfo *pyInfo = reinterpret_cast(_PyObject_New(&PyThingSetupInfoType)); + PyThingSetupInfo *pyInfo = PyObject_New(PyThingSetupInfo, &PyThingSetupInfoType); pyInfo->ptrObj = info; pyInfo->thing = pyThing; + connect(info, &ThingSetupInfo::finished, this, [=](){ - PyObject_Free(pyInfo); + if (info->status() == Thing::ThingErrorNoError) { + m_things.insert(info->thing(), pyThing); + } else { + PyGILState_STATE s = PyGILState_Ensure(); + Py_DECREF(pyThing); + PyGILState_Release(s); + } }); + connect(info, &ThingSetupInfo::aborted, this, [=](){ + PyGILState_STATE s = PyGILState_Ensure(); + Py_DECREF(pyThing); + PyGILState_Release(s); + }); + connect(info, &ThingSetupInfo::destroyed, this, [=](){ + PyGILState_STATE s = PyGILState_Ensure(); + PyObject_Del(pyInfo); + PyGILState_Release(s); + }); + callPluginFunction("setupThing", reinterpret_cast(pyInfo)); } @@ -390,14 +408,14 @@ void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &ac void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param) { - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(s_mainThread); qCDebug(dcThingManager()) << "Calling python plugin function" << function; PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); if(!pFunc || !PyCallable_Check(pFunc)) { Py_XDECREF(pFunc); qCWarning(dcThingManager()) << "Python plugin does not implement" << function; - PyGILState_Release(s); + s_mainThread = PyEval_SaveThread(); return; } @@ -411,20 +429,21 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje if (PyErr_Occurred()) { qCWarning(dcThingManager()) << "Error calling python method:"; dumpError(); - PyGILState_Release(s); + s_mainThread = PyEval_SaveThread(); return; } if (QByteArray(future->ob_type->tp_name) != "coroutine") { - PyGILState_Release(s); + s_mainThread = PyEval_SaveThread(); return; } - PyObject *get_event_loop = PyObject_GetAttrString(s_asyncio, "get_event_loop"); - PyObject *loop = PyObject_CallFunctionObjArgs(get_event_loop, nullptr); + // Spawn a event loop for python + PyObject *new_event_loop = PyObject_GetAttrString(s_asyncio, "new_event_loop"); + PyObject *loop = PyObject_CallFunctionObjArgs(new_event_loop, nullptr); - PyObject *run_coroutine_threadsafe = PyObject_GetAttrString(s_asyncio, "run_coroutine_threadsafe"); - PyObject *task = PyObject_CallFunctionObjArgs(run_coroutine_threadsafe, future, loop, nullptr); + PyObject *run_coroutine_threadsafe = PyObject_GetAttrString(loop, "create_task"); + PyObject *task = PyObject_CallFunctionObjArgs(run_coroutine_threadsafe, future, nullptr); dumpError(); PyObject *add_done_callback = PyObject_GetAttrString(task, "add_done_callback"); @@ -434,14 +453,20 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje PyObject *result = PyObject_CallFunctionObjArgs(add_done_callback, task_done, nullptr); dumpError(); - Py_DECREF(get_event_loop); + PyObject *run_forever = PyObject_GetAttrString(loop, "run_until_complete"); + QtConcurrent::run([=](){ + PyGILState_STATE s = PyGILState_Ensure(); + PyObject_CallFunctionObjArgs(run_forever, task, nullptr); + PyGILState_Release(s); + }); + + Py_DECREF(new_event_loop); Py_DECREF(loop); Py_DECREF(run_coroutine_threadsafe); Py_DECREF(task); Py_DECREF(add_done_callback); - Py_DECREF(get_event_loop); Py_DECREF(result); - PyGILState_Release(s); + s_mainThread = PyEval_SaveThread(); } From 59c1c8e9f2a269787f7c41afcd072a45279d3348 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Wed, 17 Jun 2020 23:38:44 +0200 Subject: [PATCH 09/46] some more work --- libnymea-core/integrations/python/pything.h | 49 ++++++++++++++----- .../integrations/pythonintegrationplugin.cpp | 31 ++++-------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 37504721..092118db 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -5,6 +5,7 @@ #include "structmember.h" #include "integrations/thing.h" +#include "loggingcategories.h" #include @@ -18,16 +19,12 @@ typedef struct _thing { } PyThing; -static int PyThing_init(PyThing */*self*/, PyObject */*args*/, PyObject */*kwds*/) -// initialize PyVoice Object -{ +static int PyThing_init(PyThing */*self*/, PyObject */*args*/, PyObject */*kwds*/) { return 0; } -static void PyThing_dealloc(PyThing * self) -// destruct the object -{ +static void PyThing_dealloc(PyThing * self) { // FIXME: Why is this not called? Seems we're leaking... Q_ASSERT(false); Py_TYPE(self)->tp_free(self); @@ -53,12 +50,12 @@ static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) { - char *stateTypeIdStr; - PyObject *valueObj; + char *stateTypeIdStr = nullptr; + PyObject *valueObj = nullptr; // FIXME: is there any better way to do this? Value is a variant if (!PyArg_ParseTuple(args, "sO", &stateTypeIdStr, &valueObj)) { - qWarning() << "error parsing parameters"; + qCWarning(dcThingManager) << "Error parsing parameters"; return nullptr; } @@ -68,7 +65,6 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); const char *bytes = PyBytes_AS_STRING(str); - qWarning() << "params:" << stateTypeId << bytes << self; QVariant value(bytes); if (self->ptrObj != nullptr) { @@ -81,6 +77,37 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) Py_RETURN_NONE; } +static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) +{ + char *eventTypeIdStr = nullptr; + PyObject *valueObj = nullptr; + + if (!PyArg_ParseTuple(args, "s|O", &eventTypeIdStr, &valueObj)) { + qCWarning(dcThingManager) << "Error parsing parameters"; + return nullptr; + } + + EventTypeId eventTypeId = EventTypeId(eventTypeIdStr); + +// if (valueObj != nullptr) { +// PyObject* repr = PyObject_Repr(valueObj); +// PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); +// const char *bytes = PyBytes_AS_STRING(str); + +// QVariant value(bytes); +// } + + + if (self->ptrObj != nullptr) { + QMetaObject::invokeMethod(self->ptrObj, "emitEvent", Qt::QueuedConnection, Q_ARG(EventTypeId, eventTypeId)); + } + +// Py_XDECREF(repr); +// Py_XDECREF(str); + + Py_RETURN_NONE; +} + static PyGetSetDef PyThing_getseters[] = { {"name", (getter)PyThing_getName, (setter)PyThing_setName, "Thingname", nullptr}, {nullptr , nullptr, nullptr, nullptr, nullptr} /* Sentinel */ @@ -88,7 +115,7 @@ static PyGetSetDef PyThing_getseters[] = { static PyMethodDef PyThing_methods[] = { { "setStateValue", (PyCFunction)PyThing_setStateValue, METH_VARARGS, "Set a things state value" }, -// { "finish", (PyCFunction)PyThing_finish, METH_VARARGS, "finish a discovery" }, + { "emitEvent", (PyCFunction)PyThing_emitEvent, METH_VARARGS, "Emits an event" }, {nullptr, nullptr, 0, nullptr} // sentinel }; diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 0eebdb09..d007b616 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -126,23 +126,10 @@ void PythonIntegrationPlugin::initPython() // Import nymea module into this interpreter s_nymeaModule = PyImport_ImportModule("nymea"); - - -// // Spawn a event loop for python + // We'll be using asyncio everywhere, so let's import it right away s_asyncio = PyImport_ImportModule("asyncio"); -// PyObject *get_event_loop = PyObject_GetAttrString(s_asyncio, "get_event_loop"); -// PyObject *loop = PyObject_CallFunctionObjArgs(get_event_loop, nullptr); -// PyObject *run_forever = PyObject_GetAttrString(loop, "run_forever"); - -// QtConcurrent::run([=](){ -// PyGILState_STATE s = PyGILState_Ensure(); -// PyObject_CallFunctionObjArgs(run_forever, nullptr); -// PyGILState_Release(s); -// }); - - - // Need to release ths lock from the main thread before spawning the new thread + // Need to release ths lock from the main thread before spawning new threads s_mainThread = PyEval_SaveThread(); } @@ -408,14 +395,14 @@ void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &ac void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param) { - PyEval_RestoreThread(s_mainThread); + PyGILState_STATE s = PyGILState_Ensure(); qCDebug(dcThingManager()) << "Calling python plugin function" << function; PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); if(!pFunc || !PyCallable_Check(pFunc)) { Py_XDECREF(pFunc); qCWarning(dcThingManager()) << "Python plugin does not implement" << function; - s_mainThread = PyEval_SaveThread(); + PyGILState_Release(s); return; } @@ -429,12 +416,12 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje if (PyErr_Occurred()) { qCWarning(dcThingManager()) << "Error calling python method:"; dumpError(); - s_mainThread = PyEval_SaveThread(); + PyGILState_Release(s); return; } if (QByteArray(future->ob_type->tp_name) != "coroutine") { - s_mainThread = PyEval_SaveThread(); + PyGILState_Release(s); return; } @@ -453,10 +440,10 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje PyObject *result = PyObject_CallFunctionObjArgs(add_done_callback, task_done, nullptr); dumpError(); - PyObject *run_forever = PyObject_GetAttrString(loop, "run_until_complete"); + PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); QtConcurrent::run([=](){ PyGILState_STATE s = PyGILState_Ensure(); - PyObject_CallFunctionObjArgs(run_forever, task, nullptr); + PyObject_CallFunctionObjArgs(run_until_complete, task, nullptr); PyGILState_Release(s); }); @@ -467,6 +454,6 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje Py_DECREF(add_done_callback); Py_DECREF(result); - s_mainThread = PyEval_SaveThread(); + PyGILState_Release(s); } From f97e5bf4d0da0e1e7840114bbb1103545a2ec327 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Thu, 18 Jun 2020 11:38:35 +0200 Subject: [PATCH 10/46] more work --- libnymea-core/integrations/python/pyparam.h | 151 ++++++++++++++++++ libnymea-core/integrations/python/pything.h | 55 ++++--- .../integrations/python/pythingactioninfo.h | 111 +++++++++++++ .../integrations/python/pythingdescriptor.h | 30 ++-- .../python/pythingdiscoveryinfo.h | 46 +++--- .../integrations/python/pythingsetupinfo.h | 34 ++-- .../integrations/pythonintegrationplugin.cpp | 50 ++++-- .../integrations/pythonintegrationplugin.h | 1 + libnymea-core/libnymea-core.pro | 2 + 9 files changed, 389 insertions(+), 91 deletions(-) create mode 100644 libnymea-core/integrations/python/pyparam.h create mode 100644 libnymea-core/integrations/python/pythingactioninfo.h diff --git a/libnymea-core/integrations/python/pyparam.h b/libnymea-core/integrations/python/pyparam.h new file mode 100644 index 00000000..db6da3bd --- /dev/null +++ b/libnymea-core/integrations/python/pyparam.h @@ -0,0 +1,151 @@ +#ifndef PYPARAM_H +#define PYPARAM_H + +#include +#include "structmember.h" + +#include "types/param.h" + +#include "loggingcategories.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" + + +typedef struct _pyparam { + PyObject_HEAD + PyObject* pyParamTypeId = nullptr; + PyObject* pyValue = nullptr; +} PyParam; + +static void PyParam_dealloc(PyParam * self) { + // FIXME: Why is this not called? Seems we're leaking... + Q_ASSERT(false); + Py_XDECREF(self->pyParamTypeId); + Py_XDECREF(self->pyValue); + Py_TYPE(self)->tp_free(self); +} + +static PyMethodDef PyParam_methods[] = { + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyMemberDef PyParam_members[] = { + {"paramTypeId", T_OBJECT_EX, offsetof(PyParam, pyParamTypeId), 0, "Param type ID"}, + {"value", T_OBJECT_EX, offsetof(PyParam, pyValue), 0, "Param value"}, + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +}; + + +static int PyParam_init(PyParam *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"paramTypeId", "value", nullptr}; + PyObject *paramTypeId = nullptr, *value = nullptr; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwlist, ¶mTypeId, &value)) + return -1; + + if (paramTypeId) { + Py_XDECREF(self->pyParamTypeId); + Py_INCREF(paramTypeId); + self->pyParamTypeId = paramTypeId; + } + if (value) { + Py_XDECREF(self->pyValue); + Py_INCREF(value); + self->pyValue = value; + } + return 0; +} + +static PyTypeObject PyParamType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.Param", /* tp_name */ + sizeof(PyParam), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyParam_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +static PyParam* PyParam_fromParam(const Param ¶m) +{ + PyParam *pyParam = PyObject_New(PyParam, &PyParamType); + pyParam->pyParamTypeId = PyUnicode_FromString(param.paramTypeId().toString().toUtf8()); + + switch (param.value().type()) { + case QVariant::Bool: + pyParam->pyValue = PyBool_FromLong(*(long*)param.value().data()); + break; + case QVariant::Int: + case QVariant::UInt: + case QVariant::LongLong: + case QVariant::ULongLong: + pyParam->pyValue = PyLong_FromLong(*(long*)param.value().data()); + break; + case QVariant::String: + case QVariant::ByteArray: + pyParam->pyValue = PyUnicode_FromString(param.value().toString().toUtf8()); + break; + case QVariant::Double: + pyParam->pyValue = PyFloat_FromDouble(param.value().toDouble()); + break; + case QVariant::Invalid: + pyParam->pyValue = Py_None; + Py_INCREF(pyParam->pyValue); + break; + default: + qCWarning(dcThingManager) << "Unhandled data type in conversion from Param to PyParam!"; + pyParam->pyValue = Py_None; + Py_INCREF(pyParam->pyValue); + break; + } + return pyParam; +} + +static PyObject* PyParam_FromParamList(const ParamList ¶ms) +{ + PyObject* result = PyTuple_New(params.count()); + for (int i = 0; i < params.count(); i++) { + PyTuple_SET_ITEM(result, i, (PyObject*)PyParam_fromParam(params.at(i))); + } + return result; +} + + +static void registerParamType(PyObject *module) +{ + PyParamType.tp_new = PyType_GenericNew; + PyParamType.tp_basicsize = sizeof(PyParam); + PyParamType.tp_dealloc=(destructor) PyParam_dealloc; + PyParamType.tp_flags = Py_TPFLAGS_DEFAULT; + PyParamType.tp_doc = "Param class"; + PyParamType.tp_methods = PyParam_methods; + PyParamType.tp_members = PyParam_members; + PyParamType.tp_init = reinterpret_cast(PyParam_init); + + if (PyType_Ready(&PyParamType) < 0) { + return; + } + PyModule_AddObject(module, "Param", reinterpret_cast(&PyParamType)); +} + +#pragma GCC diagnostic pop + +#endif // PYPARAM_H diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 092118db..175b0177 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -4,6 +4,8 @@ #include #include "structmember.h" +#include "pyparam.h" + #include "integrations/thing.h" #include "loggingcategories.h" @@ -15,7 +17,7 @@ typedef struct _thing { PyObject_HEAD - Thing *ptrObj; + Thing *thing; } PyThing; @@ -25,26 +27,24 @@ static int PyThing_init(PyThing */*self*/, PyObject */*args*/, PyObject */*kwds* static void PyThing_dealloc(PyThing * self) { - // FIXME: Why is this not called? Seems we're leaking... - Q_ASSERT(false); Py_TYPE(self)->tp_free(self); } static PyObject *PyThing_getName(PyThing *self, void */*closure*/) { - if (!self->ptrObj) { + if (!self->thing) { PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); return nullptr; } // FIXME: Needs blocking queued connection - PyObject *ret = PyUnicode_FromString(self->ptrObj->name().toUtf8().data()); + PyObject *ret = PyUnicode_FromString(self->thing->name().toUtf8().data()); Py_INCREF(ret); return ret; } static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ // FIXME: Needs queued connection - self->ptrObj->setName(QString(PyUnicode_AsUTF8(value))); + self->thing->setName(QString(PyUnicode_AsUTF8(value))); return 0; } @@ -67,8 +67,8 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) QVariant value(bytes); - if (self->ptrObj != nullptr) { - QMetaObject::invokeMethod(self->ptrObj, "setStateValue", Qt::QueuedConnection, Q_ARG(StateTypeId, stateTypeId), Q_ARG(QVariant, value)); + if (self->thing != nullptr) { + QMetaObject::invokeMethod(self->thing, "setStateValue", Qt::QueuedConnection, Q_ARG(StateTypeId, stateTypeId), Q_ARG(QVariant, value)); } Py_XDECREF(repr); @@ -88,22 +88,39 @@ static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) } EventTypeId eventTypeId = EventTypeId(eventTypeIdStr); + ParamList params; -// if (valueObj != nullptr) { -// PyObject* repr = PyObject_Repr(valueObj); -// PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); -// const char *bytes = PyBytes_AS_STRING(str); + if (valueObj != nullptr) { + PyObject *iter = PyObject_GetIter(valueObj); -// QVariant value(bytes); -// } + while (iter) { + PyObject *next = PyIter_Next(iter); + if (!next) { + break; + } + if (next->ob_type != &PyParamType) { + qCWarning(dcThingManager()) << "Invalid parameter passed in param list"; + continue; + } + PyParam *pyParam = reinterpret_cast(next); + ParamTypeId paramTypeId = ParamTypeId(PyUnicode_AsUTF8(pyParam->pyParamTypeId)); - if (self->ptrObj != nullptr) { - QMetaObject::invokeMethod(self->ptrObj, "emitEvent", Qt::QueuedConnection, Q_ARG(EventTypeId, eventTypeId)); + // Is there a better way to convert a PyObject to a QVariant? + PyObject* repr = PyObject_Repr(pyParam->pyValue); + PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + const char *bytes = PyBytes_AS_STRING(str); + Py_XDECREF(repr); + Py_XDECREF(str); + + QVariant value(bytes); + params.append(Param(paramTypeId, value)); + } } -// Py_XDECREF(repr); -// Py_XDECREF(str); + if (self->thing != nullptr) { + QMetaObject::invokeMethod(self->thing, "emitEvent", Qt::QueuedConnection, Q_ARG(EventTypeId, eventTypeId), Q_ARG(ParamList, params)); + } Py_RETURN_NONE; } @@ -153,7 +170,7 @@ static void registerThingType(PyObject *module) PyThingType.tp_doc = "Thing class"; PyThingType.tp_methods = PyThing_methods; PyThingType.tp_getset = PyThing_getseters; -// PyThingType.tp_members = PyThingSetupInfo_members; + // PyThingType.tp_members = PyThingSetupInfo_members; PyThingType.tp_init = reinterpret_cast(PyThing_init); if (PyType_Ready(&PyThingType) < 0) { diff --git a/libnymea-core/integrations/python/pythingactioninfo.h b/libnymea-core/integrations/python/pythingactioninfo.h new file mode 100644 index 00000000..fbe98890 --- /dev/null +++ b/libnymea-core/integrations/python/pythingactioninfo.h @@ -0,0 +1,111 @@ +#ifndef PYTHINGACTIONINFO_H +#define PYTHINGACTIONINFO_H + +#include +#include "structmember.h" + +#include "pything.h" + +#include "integrations/thingactioninfo.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +typedef struct { + PyObject_HEAD + ThingActionInfo* info; + PyThing *pyThing; + PyObject *pyActionTypeId; + PyObject *pyParams; +} PyThingActionInfo; + + +static int PyThingActionInfo_init(PyThingActionInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) { + return 0; +} + + +static void PyThingActionInfo_dealloc(PyThingActionInfo * self) { + // FIXME: Why is this not called? Seems we're leaking... + Q_ASSERT(false); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * PyThingActionInfo_finish(PyThingActionInfo* self, PyObject* args) { + int status; + char *message = nullptr; + + if (!PyArg_ParseTuple(args, "i|s", &status, &message)) { + PyErr_SetString(PyExc_TypeError, "Invalid arguments in finish call. Expected: finish(ThingError, message = \"\""); + return nullptr; + } + + Thing::ThingError thingError = static_cast(status); + QString displayMessage = message != nullptr ? QString(message) : QString(); + + if (self->info) { + QMetaObject::invokeMethod(self->info, "finish", Qt::QueuedConnection, Q_ARG(Thing::ThingError, thingError), Q_ARG(QString, displayMessage)); + } + + Py_RETURN_NONE; +} + +static PyMemberDef PyThingActionInfo_members[] = { + {"thing", T_OBJECT_EX, offsetof(PyThingActionInfo, pyThing), 0, "Thing this action is for"}, + {"actionTypeId", T_OBJECT_EX, offsetof(PyThingActionInfo, pyActionTypeId), 0, "The action type id for this action"}, + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +}; + +static PyMethodDef PyThingActionInfo_methods[] = { + { "finish", (PyCFunction)PyThingActionInfo_finish, METH_VARARGS, "finish an action" }, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyTypeObject PyThingActionInfoType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.ThingActionInfo", /* tp_name */ + sizeof(PyThingActionInfo), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Noddy objects", /* tp_doc */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +static void registerThingActionInfoType(PyObject *module) { + PyThingActionInfoType.tp_new = PyType_GenericNew; + PyThingActionInfoType.tp_dealloc=(destructor) PyThingActionInfo_dealloc; + PyThingActionInfoType.tp_basicsize = sizeof(PyThingActionInfo); + PyThingActionInfoType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingActionInfoType.tp_doc = "ThingActionInfo class"; + PyThingActionInfoType.tp_methods = PyThingActionInfo_methods; + PyThingActionInfoType.tp_members = PyThingActionInfo_members; + PyThingActionInfoType.tp_init = (initproc)PyThingActionInfo_init; + + if (PyType_Ready(&PyThingActionInfoType) < 0) { + return; + } + PyModule_AddObject(module, "ThingActionInfo", (PyObject *)&PyThingActionInfoType); +} + + + + +#pragma GCC diagnostic pop + +#endif // PYTHINGACTIONINFO_H diff --git a/libnymea-core/integrations/python/pythingdescriptor.h b/libnymea-core/integrations/python/pythingdescriptor.h index 91fe2773..a57660b8 100644 --- a/libnymea-core/integrations/python/pythingdescriptor.h +++ b/libnymea-core/integrations/python/pythingdescriptor.h @@ -13,10 +13,9 @@ typedef struct { PyObject_HEAD - PyObject* thingClassId; - PyObject* name; - PyObject* description; - ThingDescriptor descriptor; + PyObject* pyThingClassId; + PyObject* pyName; + PyObject* pyDescription; } PyThingDescriptor; static PyMethodDef PyThingDescriptor_methods[] = { @@ -25,9 +24,9 @@ static PyMethodDef PyThingDescriptor_methods[] = { }; static PyMemberDef PyThingDescriptor_members[] = { - {"thingClassId", T_OBJECT_EX, offsetof(PyThingDescriptor, thingClassId), 0, "Descriptor thingClassId"}, - {"name", T_OBJECT_EX, offsetof(PyThingDescriptor, name), 0, "Descriptor name"}, - {"description", T_OBJECT_EX, offsetof(PyThingDescriptor, description), 0, "Descriptor description"}, + {"thingClassId", T_OBJECT_EX, offsetof(PyThingDescriptor, pyThingClassId), 0, "Descriptor thingClassId"}, + {"name", T_OBJECT_EX, offsetof(PyThingDescriptor, pyName), 0, "Descriptor name"}, + {"description", T_OBJECT_EX, offsetof(PyThingDescriptor, pyDescription), 0, "Descriptor description"}, {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; @@ -35,28 +34,25 @@ static PyMemberDef PyThingDescriptor_members[] = { static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"thingClassId", "name", "description", nullptr}; - PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr, *tmp = nullptr; + PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, &thingClassId, &name, &description)) return -1; if (thingClassId) { - tmp = self->thingClassId; + Py_XDECREF(self->pyThingClassId); Py_INCREF(thingClassId); - self->thingClassId = thingClassId; - Py_XDECREF(tmp); + self->pyThingClassId = thingClassId; } if (name) { - tmp = self->name; + Py_XDECREF(self->pyName); Py_INCREF(name); - self->name = name; - Py_XDECREF(tmp); + self->pyName = name; } if (description) { - tmp = self->description; + Py_XDECREF(self->pyDescription); Py_INCREF(description); - self->description = description; - Py_XDECREF(tmp); + self->pyDescription = description; } return 0; } diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index 8038411b..7037aaa6 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -19,20 +19,16 @@ typedef struct { PyObject_HEAD - ThingDiscoveryInfo* ptrObj; + ThingDiscoveryInfo* info; } PyThingDiscoveryInfo; -static int PyThingDiscoveryInfo_init(PyThingDiscoveryInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) -// initialize PyVoice Object -{ +static int PyThingDiscoveryInfo_init(PyThingDiscoveryInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) { return 0; } -static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) -// destruct the object -{ +static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) { // FIXME: Why is this not called? Seems we're leaking... Q_ASSERT(false); Py_TYPE(self)->tp_free(self); @@ -41,20 +37,20 @@ static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) static PyObject * PyThingDiscoveryInfo_finish(PyThingDiscoveryInfo* self, PyObject* args) { int status; - char *message; + char *message = nullptr; - if (PyArg_ParseTuple(args, "is", &status, &message)) { - (self->ptrObj)->finish(static_cast(status), QString(message)); - Py_RETURN_NONE; + if (!PyArg_ParseTuple(args, "i|s", &status, &message)) { + PyErr_SetString(PyExc_TypeError, "Invalid arguments in finish call. Expected: finish(ThingError, message = \"\""); + return nullptr; } - if (PyArg_ParseTuple(args, "i", &status)) { - (self->ptrObj)->finish(static_cast(status)); - Py_RETURN_NONE; - } + Thing::ThingError thingError = static_cast(status); + QString displayMessage = message != nullptr ? QString(message) : QString(); - PyErr_SetString(PyExc_TypeError, "Invalid arguments in finish call. Expected: finish(ThingError, message = \"\""); - return nullptr; + if (self->info) { + QMetaObject::invokeMethod(self->info, "finish", Qt::QueuedConnection, Q_ARG(Thing::ThingError, thingError), Q_ARG(QString, displayMessage)); + } + Py_RETURN_NONE; } static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, PyObject* args) { @@ -72,21 +68,23 @@ static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, PyThingDescriptor *pyDescriptor = (PyThingDescriptor*)pyObj; ThingClassId thingClassId; - if (pyDescriptor->thingClassId) { - thingClassId = ThingClassId(PyUnicode_AsUTF8(pyDescriptor->thingClassId)); + if (pyDescriptor->pyThingClassId) { + thingClassId = ThingClassId(PyUnicode_AsUTF8(pyDescriptor->pyThingClassId)); } QString name; - if (pyDescriptor->name) { - name = QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->name)); + if (pyDescriptor->pyName) { + name = QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->pyName)); } QString description; - if (pyDescriptor->description) { - description = QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->description)); + if (pyDescriptor->pyDescription) { + description = QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->pyDescription)); } ThingDescriptor descriptor(thingClassId, name, description); - self->ptrObj->addThingDescriptor(descriptor); + if (self->info) { + QMetaObject::invokeMethod(self->info, "addThingDescriptor", Qt::QueuedConnection, Q_ARG(ThingDescriptor, descriptor)); + } return Py_BuildValue(""); } diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index 3b63bb1a..1623d4aa 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -14,45 +14,41 @@ typedef struct { PyObject_HEAD - ThingSetupInfo* ptrObj; - PyThing *thing; + ThingSetupInfo* info; + PyThing *pyThing; } PyThingSetupInfo; -static int PyThingSetupInfo_init(PyThingSetupInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) -{ +static int PyThingSetupInfo_init(PyThingSetupInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) { return 0; } -static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) -{ - // FIXME: Why is this not called? Seems we're leaking... - Q_ASSERT(false); +static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) { Py_TYPE(self)->tp_free(self); } -static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args) -{ +static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args) { int status; - char *message; + char *message = nullptr; - if (PyArg_ParseTuple(args, "is", &status, &message)) { - (self->ptrObj)->finish(static_cast(status), QString(message)); - Py_RETURN_NONE; + if (!PyArg_ParseTuple(args, "i|s", &status, &message)) { + PyErr_SetString(PyExc_TypeError, "Invalid arguments in finish call. Expected: finish(ThingError, message = \"\""); + return nullptr; } - PyErr_Clear(); - if (PyArg_ParseTuple(args, "i", &status)) { - (self->ptrObj)->finish(static_cast(status)); - Py_RETURN_NONE; + Thing::ThingError thingError = static_cast(status); + QString displayMessage = message != nullptr ? QString(message) : QString(); + + if (self->info) { + QMetaObject::invokeMethod(self->info, "finish", Qt::QueuedConnection, Q_ARG(Thing::ThingError, thingError), Q_ARG(QString, displayMessage)); } Py_RETURN_NONE; } static PyMemberDef PyThingSetupInfo_members[] = { - {"thing", T_OBJECT_EX, offsetof(PyThingSetupInfo, thing), 0, "Thing being setup in this setup transaction"}, + {"thing", T_OBJECT_EX, offsetof(PyThingSetupInfo, pyThing), 0, "Thing being setup in this setup transaction"}, {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index d007b616..85717e38 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -4,6 +4,8 @@ #include "python/pything.h" #include "python/pythingdiscoveryinfo.h" #include "python/pythingsetupinfo.h" +#include "python/pyparam.h" +#include "python/pythingactioninfo.h" #include "pythonintegrationplugin.h" @@ -14,15 +16,11 @@ #include #include -QHash s_modules; PyThreadState* PythonIntegrationPlugin::s_mainThread = nullptr; PyObject* PythonIntegrationPlugin::s_nymeaModule = nullptr; PyObject* PythonIntegrationPlugin::s_asyncio = nullptr; - - - // Write to stdout/stderr PyObject* nymea_write(PyObject* /*self*/, PyObject* args) { @@ -98,10 +96,12 @@ PyMODINIT_FUNC PyInit_nymea(void) registerNymeaLoggingHandler(m); + registerParamType(m); registerThingType(m); registerThingDescriptorType(m); registerThingDiscoveryInfoType(m); registerThingSetupInfoType(m); + registerThingActionInfoType(m); return m; } @@ -192,11 +192,11 @@ void PythonIntegrationPlugin::init() void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) { PyThingDiscoveryInfo *pyInfo = PyObject_New(PyThingDiscoveryInfo, &PyThingDiscoveryInfoType); - pyInfo->ptrObj = info; + pyInfo->info = info; connect(info, &ThingDiscoveryInfo::destroyed, this, [=](){ PyGILState_STATE s = PyGILState_Ensure(); - pyInfo->ptrObj = nullptr; + pyInfo->info = nullptr; PyObject_Del(pyInfo); PyGILState_Release(s); }); @@ -207,11 +207,11 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) { PyThing *pyThing = PyObject_New(PyThing, &PyThingType); - pyThing->ptrObj = info->thing(); + pyThing->thing = info->thing(); PyThingSetupInfo *pyInfo = PyObject_New(PyThingSetupInfo, &PyThingSetupInfoType); - pyInfo->ptrObj = info; - pyInfo->thing = pyThing; + pyInfo->info = info; + pyInfo->pyThing = pyThing; connect(info, &ThingSetupInfo::finished, this, [=](){ @@ -230,7 +230,8 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) }); connect(info, &ThingSetupInfo::destroyed, this, [=](){ PyGILState_STATE s = PyGILState_Ensure(); - PyObject_Del(pyInfo); + pyInfo->info = nullptr; + Py_DECREF(pyInfo); PyGILState_Release(s); }); @@ -244,6 +245,32 @@ void PythonIntegrationPlugin::postSetupThing(Thing *thing) callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); } +void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) +{ + PyThing *pyThing = m_things.value(info->thing()); + + PyGILState_STATE s = PyGILState_Ensure(); + + PyThingActionInfo *pyInfo = PyObject_New(PyThingActionInfo, &PyThingActionInfoType); + pyInfo->info = info; + pyInfo->pyThing = pyThing; + pyInfo->pyActionTypeId = PyUnicode_FromString(info->action().actionTypeId().toString().toUtf8()); + pyInfo->pyParams = PyParam_FromParamList(info->action().params()); + + PyGILState_Release(s); + + connect(info, &ThingActionInfo::destroyed, this, [=](){ + PyGILState_STATE s = PyGILState_Ensure(); + pyInfo->pyActionTypeId = nullptr; + Py_DECREF(pyInfo->pyActionTypeId); + pyInfo->info = nullptr; + Py_DECREF(pyInfo); + PyGILState_Release(s); + }); + + callPluginFunction("executeAction", reinterpret_cast(pyInfo)); +} + void PythonIntegrationPlugin::thingRemoved(Thing *thing) { PyThing *pyThing = m_things.value(thing); @@ -252,10 +279,9 @@ void PythonIntegrationPlugin::thingRemoved(Thing *thing) PyGILState_STATE s = PyGILState_Ensure(); - pyThing->ptrObj = nullptr; + pyThing->thing = nullptr; Py_DECREF(pyThing); - PyGILState_Release(s); m_things.remove(thing); diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 1e57686d..a7cc47b2 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -29,6 +29,7 @@ public: void discoverThings(ThingDiscoveryInfo *info) override; void setupThing(ThingSetupInfo *info) override; void postSetupThing(Thing *thing) override; + void executeAction(ThingActionInfo *info) override; void thingRemoved(Thing *thing) override; diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index f2c5bd87..d40891df 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -21,7 +21,9 @@ RESOURCES += $$top_srcdir/icons.qrc \ HEADERS += nymeacore.h \ integrations/plugininfocache.h \ integrations/python/pynymealogginghandler.h \ + integrations/python/pyparam.h \ integrations/python/pything.h \ + integrations/python/pythingactioninfo.h \ integrations/python/pythingdescriptor.h \ integrations/python/pythingdiscoveryinfo.h \ integrations/python/pythingsetupinfo.h \ From 3c47e598119f201c8ae45bbfad5a41db65552d85 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Thu, 25 Jun 2020 12:23:33 +0200 Subject: [PATCH 11/46] some more work --- libnymea-core/integrations/python/pything.h | 48 +++++++++++++++++-- .../integrations/python/pythingactioninfo.h | 3 +- .../integrations/pythonintegrationplugin.cpp | 42 +++++++++++----- libnymea-core/jsonrpc/devicehandler.cpp | 1 - libnymea/integrations/thing.cpp | 2 + libnymea/integrations/thing.h | 14 +++--- 6 files changed, 84 insertions(+), 26 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 175b0177..500c8583 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -10,6 +10,7 @@ #include "loggingcategories.h" #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -36,15 +37,37 @@ static PyObject *PyThing_getName(PyThing *self, void */*closure*/) PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); return nullptr; } - // FIXME: Needs blocking queued connection - PyObject *ret = PyUnicode_FromString(self->thing->name().toUtf8().data()); + QString name; + // FIXME: Should not be a direct connection! + qWarning() << "name thread" << QThread::currentThread(); + QMetaObject::invokeMethod(self->thing, "name", Qt::DirectConnection, Q_RETURN_ARG(QString, name)); + PyObject *ret = PyUnicode_FromString(name.toUtf8().data()); Py_INCREF(ret); return ret; } static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ - // FIXME: Needs queued connection - self->thing->setName(QString(PyUnicode_AsUTF8(value))); + QString name = QString(PyUnicode_AsUTF8(value)); + QMetaObject::invokeMethod(self->thing, "setName", Qt::QueuedConnection, Q_ARG(QString, name)); + return 0; +} + +static PyObject *PyThing_getSettings(PyThing *self, void */*closure*/) +{ + if (!self->thing) { + PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); + return nullptr; + } + qWarning() << "setting thread" << QThread::currentThread(); + ParamList settings; + QMetaObject::invokeMethod(self->thing, "settings", Qt::BlockingQueuedConnection, Q_RETURN_ARG(ParamList, settings)); + PyObject *ret = PyParam_FromParamList(settings); + Py_INCREF(ret); + return ret; +} + +static int PyThing_setSettings(PyThing */*self*/, PyObject */*value*/, void */*closure*/){ + // self->thing->setName(QString(PyUnicode_AsUTF8(value))); return 0; } @@ -77,6 +100,19 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) Py_RETURN_NONE; } +static PyObject *PyThing_settingChanged(PyThing *self, void */*closure*/) +{ + if (!self->thing) { + PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); + return nullptr; + } + ParamList settings; + QMetaObject::invokeMethod(self->thing, "settings", Qt::BlockingQueuedConnection, Q_RETURN_ARG(ParamList, settings)); + PyObject *ret = PyParam_FromParamList(settings); + Py_INCREF(ret); + return ret; +} + static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) { char *eventTypeIdStr = nullptr; @@ -126,7 +162,9 @@ static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) } static PyGetSetDef PyThing_getseters[] = { - {"name", (getter)PyThing_getName, (setter)PyThing_setName, "Thingname", nullptr}, + {"name", (getter)PyThing_getName, (setter)PyThing_setName, "Thing name", nullptr}, + {"settings", (getter)PyThing_getSettings, (setter)PyThing_setSettings, "Thing settings", nullptr}, + {"settingChanged", (getter)PyThing_settingChanged, nullptr, "Signal for changed settings", nullptr}, {nullptr , nullptr, nullptr, nullptr, nullptr} /* Sentinel */ }; diff --git a/libnymea-core/integrations/python/pythingactioninfo.h b/libnymea-core/integrations/python/pythingactioninfo.h index fbe98890..08dc1898 100644 --- a/libnymea-core/integrations/python/pythingactioninfo.h +++ b/libnymea-core/integrations/python/pythingactioninfo.h @@ -27,8 +27,6 @@ static int PyThingActionInfo_init(PyThingActionInfo */*self*/, PyObject */*args* static void PyThingActionInfo_dealloc(PyThingActionInfo * self) { - // FIXME: Why is this not called? Seems we're leaking... - Q_ASSERT(false); Py_TYPE(self)->tp_free(self); } @@ -54,6 +52,7 @@ static PyObject * PyThingActionInfo_finish(PyThingActionInfo* self, PyObject* ar static PyMemberDef PyThingActionInfo_members[] = { {"thing", T_OBJECT_EX, offsetof(PyThingActionInfo, pyThing), 0, "Thing this action is for"}, {"actionTypeId", T_OBJECT_EX, offsetof(PyThingActionInfo, pyActionTypeId), 0, "The action type id for this action"}, + {"params", T_OBJECT_EX, offsetof(PyThingActionInfo, pyParams), 0, "The params for this action"}, {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 85717e38..45740c50 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -129,7 +129,7 @@ void PythonIntegrationPlugin::initPython() // We'll be using asyncio everywhere, so let's import it right away s_asyncio = PyImport_ImportModule("asyncio"); - // Need to release ths lock from the main thread before spawning new threads + // Need to release the lock from the main thread before spawning new threads s_mainThread = PyEval_SaveThread(); } @@ -150,6 +150,8 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) } m_metaData = PluginMetadata(jsonDoc.object()); + qWarning() << "main thread" << QThread::currentThread(); + PyGILState_STATE s = PyGILState_Ensure(); @@ -229,10 +231,13 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) PyGILState_Release(s); }); connect(info, &ThingSetupInfo::destroyed, this, [=](){ - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(s_mainThread); + +// PyGILState_STATE s = PyGILState_Ensure(); pyInfo->info = nullptr; Py_DECREF(pyInfo); - PyGILState_Release(s); +// PyGILState_Release(s); + s_mainThread = PyEval_SaveThread(); }); @@ -262,7 +267,7 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) connect(info, &ThingActionInfo::destroyed, this, [=](){ PyGILState_STATE s = PyGILState_Ensure(); pyInfo->pyActionTypeId = nullptr; - Py_DECREF(pyInfo->pyActionTypeId); + Py_XDECREF(pyInfo->pyActionTypeId); pyInfo->info = nullptr; Py_DECREF(pyInfo); PyGILState_Release(s); @@ -435,7 +440,7 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje dumpError(); - PyObject *future = PyObject_CallFunctionObjArgs(pFunc, param, nullptr); + PyObject *result = PyObject_CallFunctionObjArgs(pFunc, param, nullptr); Py_XDECREF(pFunc); @@ -446,36 +451,51 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje return; } - if (QByteArray(future->ob_type->tp_name) != "coroutine") { + if (QByteArray(result->ob_type->tp_name) != "coroutine") { + Py_DECREF(result); PyGILState_Release(s); return; } +// PyObject *coro = result; + +// PyObject *get_running_loop = PyObject_GetAttrString(s_asyncio, "get_event_loop"); +// PyObject *loop = PyObject_CallFunctionObjArgs(get_running_loop, nullptr); +// Py_DECREF(get_running_loop); +// PyObject *run_in_executor = PyObject_GetAttrString(loop, "run_in_executor"); +// result = PyObject_CallFunctionObjArgs(run_in_executor, Py_None, coro); + +// Py_DECREF(result); +// Py_DECREF(coro); // Spawn a event loop for python PyObject *new_event_loop = PyObject_GetAttrString(s_asyncio, "new_event_loop"); PyObject *loop = PyObject_CallFunctionObjArgs(new_event_loop, nullptr); - PyObject *run_coroutine_threadsafe = PyObject_GetAttrString(loop, "create_task"); - PyObject *task = PyObject_CallFunctionObjArgs(run_coroutine_threadsafe, future, nullptr); + Py_DECREF(new_event_loop); + + PyObject *create_task = PyObject_GetAttrString(loop, "create_task"); + PyObject *task = PyObject_CallFunctionObjArgs(create_task, result, nullptr); dumpError(); + Py_DECREF(result); + PyObject *add_done_callback = PyObject_GetAttrString(task, "add_done_callback"); dumpError(); PyObject *task_done = PyObject_GetAttrString(s_nymeaModule, "task_done"); - PyObject *result = PyObject_CallFunctionObjArgs(add_done_callback, task_done, nullptr); + result = PyObject_CallFunctionObjArgs(add_done_callback, task_done, nullptr); dumpError(); PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); QtConcurrent::run([=](){ + qWarning() << "new thread for func" << function << QThread::currentThread(); PyGILState_STATE s = PyGILState_Ensure(); PyObject_CallFunctionObjArgs(run_until_complete, task, nullptr); PyGILState_Release(s); }); - Py_DECREF(new_event_loop); Py_DECREF(loop); - Py_DECREF(run_coroutine_threadsafe); + Py_DECREF(create_task); Py_DECREF(task); Py_DECREF(add_done_callback); Py_DECREF(result); diff --git a/libnymea-core/jsonrpc/devicehandler.cpp b/libnymea-core/jsonrpc/devicehandler.cpp index 9e306d95..120992b1 100644 --- a/libnymea-core/jsonrpc/devicehandler.cpp +++ b/libnymea-core/jsonrpc/devicehandler.cpp @@ -865,7 +865,6 @@ JsonReply *DeviceHandler::ExecuteAction(const QVariantMap ¶ms, const JsonCon ThingActionInfo *info = NymeaCore::instance()->executeAction(action); connect(info, &ThingActionInfo::finished, jsonReply, [info, jsonReply, locale](){ - qWarning() << "finished..."; QVariantMap data; data.insert("deviceError", enumValueName(info->status()).replace("Thing", "Device")); if (!info->displayMessage().isEmpty()) { diff --git a/libnymea/integrations/thing.cpp b/libnymea/integrations/thing.cpp index 9f3bdbba..eab641fd 100644 --- a/libnymea/integrations/thing.cpp +++ b/libnymea/integrations/thing.cpp @@ -183,6 +183,7 @@ ThingClass Thing::thingClass() const /*! Returns the name of this Thing. This is visible to the user. */ QString Thing::name() const { + qWarning() << "thing name called"; return m_name; } @@ -233,6 +234,7 @@ void Thing::setParamValue(const ParamTypeId ¶mTypeId, const QVariant &value) ParamList Thing::settings() const { + qWarning() << "thing settings called"; return m_settings; } diff --git a/libnymea/integrations/thing.h b/libnymea/integrations/thing.h index e8f03ab3..2e739aa0 100644 --- a/libnymea/integrations/thing.h +++ b/libnymea/integrations/thing.h @@ -107,8 +107,8 @@ public: ThingClass thingClass() const; - QString name() const; - void setName(const QString &name); + Q_INVOKABLE QString name() const; + Q_INVOKABLE void setName(const QString &name); ParamList params() const; bool hasParam(const ParamTypeId ¶mTypeId) const; @@ -117,12 +117,12 @@ public: QVariant paramValue(const ParamTypeId ¶mTypeId) const; void setParamValue(const ParamTypeId ¶mName, const QVariant &value); - ParamList settings() const; - bool hasSetting(const ParamTypeId ¶mTypeId) const; - void setSettings(const ParamList &settings); + Q_INVOKABLE ParamList settings() const; + Q_INVOKABLE bool hasSetting(const ParamTypeId ¶mTypeId) const; + Q_INVOKABLE void setSettings(const ParamList &settings); - QVariant setting(const ParamTypeId ¶mTypeId) const; - void setSettingValue(const ParamTypeId ¶mTypeId, const QVariant &value); + Q_INVOKABLE QVariant setting(const ParamTypeId ¶mTypeId) const; + Q_INVOKABLE void setSettingValue(const ParamTypeId ¶mTypeId, const QVariant &value); States states() const; bool hasState(const StateTypeId &stateTypeId) const; From 5d0751ae276ebf6eee36e0374b26b6b46666acea Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Fri, 26 Jun 2020 10:23:53 +0200 Subject: [PATCH 12/46] fix thread syncronisation --- libnymea-core/integrations/python/pything.h | 104 +++++++++++------- .../integrations/python/pythingsetupinfo.h | 92 ++++++++++------ .../integrations/pythonintegrationplugin.cpp | 34 ++---- libnymea/integrations/thing.cpp | 2 - 4 files changed, 134 insertions(+), 98 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 500c8583..b396bc5d 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -11,6 +11,7 @@ #include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -19,28 +20,34 @@ typedef struct _thing { PyObject_HEAD Thing *thing; + QMutex *mutex; } PyThing; -static int PyThing_init(PyThing */*self*/, PyObject */*args*/, PyObject */*kwds*/) { - return 0; +static PyObject* PyThing_new(PyTypeObject *type, PyObject */*args*/, PyObject */*kwds*/) { + PyThing *self = (PyThing*)type->tp_alloc(type, 0); + if (self == NULL) { + return nullptr; + } + self->mutex = new QMutex(); + return (PyObject*)self; } static void PyThing_dealloc(PyThing * self) { Py_TYPE(self)->tp_free(self); + delete self->mutex; } static PyObject *PyThing_getName(PyThing *self, void */*closure*/) { + QMutexLocker(self->mutex); if (!self->thing) { PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); return nullptr; } QString name; - // FIXME: Should not be a direct connection! - qWarning() << "name thread" << QThread::currentThread(); - QMetaObject::invokeMethod(self->thing, "name", Qt::DirectConnection, Q_RETURN_ARG(QString, name)); + QMetaObject::invokeMethod(self->thing, "name", Qt::BlockingQueuedConnection, Q_RETURN_ARG(QString, name)); PyObject *ret = PyUnicode_FromString(name.toUtf8().data()); Py_INCREF(ret); return ret; @@ -48,12 +55,14 @@ static PyObject *PyThing_getName(PyThing *self, void */*closure*/) static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ QString name = QString(PyUnicode_AsUTF8(value)); + QMutexLocker(self->mutex); QMetaObject::invokeMethod(self->thing, "setName", Qt::QueuedConnection, Q_ARG(QString, name)); return 0; } static PyObject *PyThing_getSettings(PyThing *self, void */*closure*/) { + QMutexLocker(self->mutex); if (!self->thing) { PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); return nullptr; @@ -90,6 +99,7 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) QVariant value(bytes); + QMutexLocker(self->mutex); if (self->thing != nullptr) { QMetaObject::invokeMethod(self->thing, "setStateValue", Qt::QueuedConnection, Q_ARG(StateTypeId, stateTypeId), Q_ARG(QVariant, value)); } @@ -102,6 +112,7 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) static PyObject *PyThing_settingChanged(PyThing *self, void */*closure*/) { + QMutexLocker(self->mutex); if (!self->thing) { PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); return nullptr; @@ -154,6 +165,7 @@ static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) } } + QMutexLocker(self->mutex); if (self->thing != nullptr) { QMetaObject::invokeMethod(self->thing, "emitEvent", Qt::QueuedConnection, Q_ARG(EventTypeId, eventTypeId), Q_ARG(ParamList, params)); } @@ -161,7 +173,7 @@ static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) Py_RETURN_NONE; } -static PyGetSetDef PyThing_getseters[] = { +static PyGetSetDef PyThing_getset[] = { {"name", (getter)PyThing_getName, (setter)PyThing_setName, "Thing name", nullptr}, {"settings", (getter)PyThing_getSettings, (setter)PyThing_setSettings, "Thing settings", nullptr}, {"settingChanged", (getter)PyThing_settingChanged, nullptr, "Signal for changed settings", nullptr}, @@ -176,41 +188,59 @@ static PyMethodDef PyThing_methods[] = { static PyTypeObject PyThingType = { PyVarObject_HEAD_INIT(NULL, 0) - "nymea.Thing", /* tp_name */ - sizeof(PyThing), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + "nymea.Thing", /* tp_name */ + sizeof(PyThing), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyThing_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Thing", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyThing_methods, /* tp_methods */ + 0, /* tp_members */ + PyThing_getset, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + (newfunc)PyThing_new, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ + 0, /* tp_vectorcall */ + 0, /* tp_print DEPRECATED*/ }; static void registerThingType(PyObject *module) { - PyThingType.tp_new = PyType_GenericNew; - PyThingType.tp_dealloc= reinterpret_cast(PyThing_dealloc); - PyThingType.tp_basicsize = sizeof(PyThing); - PyThingType.tp_flags = Py_TPFLAGS_DEFAULT; - PyThingType.tp_doc = "Thing class"; - PyThingType.tp_methods = PyThing_methods; - PyThingType.tp_getset = PyThing_getseters; - // PyThingType.tp_members = PyThingSetupInfo_members; - PyThingType.tp_init = reinterpret_cast(PyThing_init); - if (PyType_Ready(&PyThingType) < 0) { return; } diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index 1623d4aa..fbb76d21 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -16,16 +16,22 @@ typedef struct { PyObject_HEAD ThingSetupInfo* info; PyThing *pyThing; + QMutex *mutex; } PyThingSetupInfo; -static int PyThingSetupInfo_init(PyThingSetupInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) { - return 0; +static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject */*args*/, PyObject */*kwds*/) { + PyThingSetupInfo *self = (PyThingSetupInfo*)type->tp_alloc(type, 0); + if (self == NULL) { + return nullptr; + } + self->mutex = new QMutex(); + return (PyObject*)self; } - static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) { Py_TYPE(self)->tp_free(self); + delete self->mutex; } static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args) { @@ -40,6 +46,7 @@ static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args Thing::ThingError thingError = static_cast(status); QString displayMessage = message != nullptr ? QString(message) : QString(); + QMutexLocker(self->mutex); if (self->info) { QMetaObject::invokeMethod(self->info, "finish", Qt::QueuedConnection, Q_ARG(Thing::ThingError, thingError), Q_ARG(QString, displayMessage)); } @@ -59,39 +66,58 @@ static PyMethodDef PyThingSetupInfo_methods[] = { static PyTypeObject PyThingSetupInfoType = { PyVarObject_HEAD_INIT(NULL, 0) - "nymea.ThingSetupInfo", /* tp_name */ - sizeof(PyThingSetupInfo), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + "nymea.ThingSetupInfo", /* tp_name */ + sizeof(PyThingSetupInfo), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyThingSetupInfo_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "ThingSetupInfo", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyThingSetupInfo_methods, /* tp_methods */ + PyThingSetupInfo_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + (newfunc)PyThingSetupInfo_new, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ + 0, /* tp_vectorcall */ + 0, /* tp_print DEPRECATED*/ }; static void registerThingSetupInfoType(PyObject *module) { - PyThingSetupInfoType.tp_new = PyType_GenericNew; - PyThingSetupInfoType.tp_dealloc=(destructor) PyThingSetupInfo_dealloc; - PyThingSetupInfoType.tp_basicsize = sizeof(PyThingSetupInfo); - PyThingSetupInfoType.tp_flags = Py_TPFLAGS_DEFAULT; - PyThingSetupInfoType.tp_doc = "ThingSetupInfo class"; - PyThingSetupInfoType.tp_methods = PyThingSetupInfo_methods; - PyThingSetupInfoType.tp_members = PyThingSetupInfo_members; - PyThingSetupInfoType.tp_init = (initproc)PyThingSetupInfo_init; - if (PyType_Ready(&PyThingSetupInfoType) < 0) { return; } diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 45740c50..a4551ed1 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -208,36 +208,32 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) { - PyThing *pyThing = PyObject_New(PyThing, &PyThingType); + PyGILState_STATE s = PyGILState_Ensure(); + PyThing *pyThing = (PyThing*)PyObject_CallObject((PyObject*)&PyThingType, NULL); + pyThing->thing = info->thing(); - PyThingSetupInfo *pyInfo = PyObject_New(PyThingSetupInfo, &PyThingSetupInfoType); + PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, NULL); pyInfo->info = info; pyInfo->pyThing = pyThing; + PyGILState_Release(s); + connect(info, &ThingSetupInfo::finished, this, [=](){ if (info->status() == Thing::ThingErrorNoError) { m_things.insert(info->thing(), pyThing); } else { - PyGILState_STATE s = PyGILState_Ensure(); Py_DECREF(pyThing); - PyGILState_Release(s); } }); connect(info, &ThingSetupInfo::aborted, this, [=](){ - PyGILState_STATE s = PyGILState_Ensure(); Py_DECREF(pyThing); - PyGILState_Release(s); }); connect(info, &ThingSetupInfo::destroyed, this, [=](){ - PyEval_RestoreThread(s_mainThread); - -// PyGILState_STATE s = PyGILState_Ensure(); + QMutexLocker(pyInfo->mutex); pyInfo->info = nullptr; Py_DECREF(pyInfo); -// PyGILState_Release(s); - s_mainThread = PyEval_SaveThread(); }); @@ -282,13 +278,10 @@ void PythonIntegrationPlugin::thingRemoved(Thing *thing) callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); - PyGILState_STATE s = PyGILState_Ensure(); - + QMutexLocker(pyThing->mutex); pyThing->thing = nullptr; Py_DECREF(pyThing); - PyGILState_Release(s); - m_things.remove(thing); } @@ -456,16 +449,6 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje PyGILState_Release(s); return; } -// PyObject *coro = result; - -// PyObject *get_running_loop = PyObject_GetAttrString(s_asyncio, "get_event_loop"); -// PyObject *loop = PyObject_CallFunctionObjArgs(get_running_loop, nullptr); -// Py_DECREF(get_running_loop); -// PyObject *run_in_executor = PyObject_GetAttrString(loop, "run_in_executor"); -// result = PyObject_CallFunctionObjArgs(run_in_executor, Py_None, coro); - -// Py_DECREF(result); -// Py_DECREF(coro); // Spawn a event loop for python PyObject *new_event_loop = PyObject_GetAttrString(s_asyncio, "new_event_loop"); @@ -488,7 +471,6 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); QtConcurrent::run([=](){ - qWarning() << "new thread for func" << function << QThread::currentThread(); PyGILState_STATE s = PyGILState_Ensure(); PyObject_CallFunctionObjArgs(run_until_complete, task, nullptr); PyGILState_Release(s); diff --git a/libnymea/integrations/thing.cpp b/libnymea/integrations/thing.cpp index eab641fd..9f3bdbba 100644 --- a/libnymea/integrations/thing.cpp +++ b/libnymea/integrations/thing.cpp @@ -183,7 +183,6 @@ ThingClass Thing::thingClass() const /*! Returns the name of this Thing. This is visible to the user. */ QString Thing::name() const { - qWarning() << "thing name called"; return m_name; } @@ -234,7 +233,6 @@ void Thing::setParamValue(const ParamTypeId ¶mTypeId, const QVariant &value) ParamList Thing::settings() const { - qWarning() << "thing settings called"; return m_settings; } From c7f957f2013f6afaa67b632ee2e931ba01a3397e Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 29 Jun 2020 14:55:42 +0200 Subject: [PATCH 13/46] some more python plugin work --- libnymea-core/cloud/cloudnotifications.cpp | 6 +- libnymea-core/integrations/python/pyparam.h | 69 ++-- libnymea-core/integrations/python/pything.h | 153 +++++---- .../integrations/python/pythingactioninfo.h | 96 ++++-- .../integrations/python/pythingdescriptor.h | 19 +- .../python/pythingdiscoveryinfo.h | 101 +++--- .../integrations/python/pythingsetupinfo.h | 5 +- libnymea-core/integrations/python/pyutils.h | 61 ++++ .../integrations/pythonintegrationplugin.cpp | 300 ++++++++++++++---- .../integrations/pythonintegrationplugin.h | 30 +- .../thingmanagerimplementation.cpp | 262 +++++++-------- .../integrations/thingmanagerimplementation.h | 6 +- libnymea-core/libnymea-core.pro | 1 + libnymea-core/nymeacore.cpp | 2 +- libnymea/integrations/integrationplugin.cpp | 3 +- libnymea/integrations/integrationplugin.h | 26 +- libnymea/integrations/pluginmetadata.cpp | 6 + libnymea/integrations/pluginmetadata.h | 4 + nymea.pro | 2 +- plugins/mock/mock.pro | 2 +- plugins/mock/plugininfo.h | 2 +- plugins/plugins.pro | 2 +- plugins/pymock/ids.py | 4 + plugins/pymock/integrationpluginpymock.json | 76 +++++ plugins/pymock/integrationpluginpymock.py | 17 + plugins/pymock/pymock.pro | 22 ++ 26 files changed, 840 insertions(+), 437 deletions(-) create mode 100644 libnymea-core/integrations/python/pyutils.h create mode 100644 plugins/pymock/ids.py create mode 100644 plugins/pymock/integrationpluginpymock.json create mode 100644 plugins/pymock/integrationpluginpymock.py create mode 100644 plugins/pymock/pymock.pro diff --git a/libnymea-core/cloud/cloudnotifications.cpp b/libnymea-core/cloud/cloudnotifications.cpp index 38a426e5..678fc4e1 100644 --- a/libnymea-core/cloud/cloudnotifications.cpp +++ b/libnymea-core/cloud/cloudnotifications.cpp @@ -53,10 +53,7 @@ CloudNotifications::CloudNotifications(AWSConnector* awsConnector, QObject *pare connect(m_awsConnector, &AWSConnector::pushNotificationEndpointsUpdated, this, &CloudNotifications::pushNotificationEndpointsUpdated); connect(m_awsConnector, &AWSConnector::pushNotificationEndpointAdded, this, &CloudNotifications::pushNotificationEndpointAdded); connect(m_awsConnector, &AWSConnector::pushNotificationSent, this, &CloudNotifications::pushNotificationSent); -} -PluginMetadata CloudNotifications::metaData() const -{ QVariantMap pluginMetaData; pluginMetaData.insert("id", "ccc6dbc8-e352-48a1-8e87-3c89a4669fc2"); pluginMetaData.insert("name", "CloudNotifications"); @@ -145,7 +142,8 @@ PluginMetadata CloudNotifications::metaData() const vendors.append(guhVendor); pluginMetaData.insert("vendors", vendors); - return PluginMetadata(QJsonObject::fromVariantMap(pluginMetaData), true); + setMetaData(PluginMetadata(QJsonObject::fromVariantMap(pluginMetaData), true)); + } void CloudNotifications::setupThing(ThingSetupInfo *info) diff --git a/libnymea-core/integrations/python/pyparam.h b/libnymea-core/integrations/python/pyparam.h index db6da3bd..cd84a4c2 100644 --- a/libnymea-core/integrations/python/pyparam.h +++ b/libnymea-core/integrations/python/pyparam.h @@ -2,7 +2,9 @@ #define PYPARAM_H #include -#include "structmember.h" +#include + +#include "pyutils.h" #include "types/param.h" @@ -20,8 +22,6 @@ typedef struct _pyparam { } PyParam; static void PyParam_dealloc(PyParam * self) { - // FIXME: Why is this not called? Seems we're leaking... - Q_ASSERT(false); Py_XDECREF(self->pyParamTypeId); Py_XDECREF(self->pyValue); Py_TYPE(self)->tp_free(self); @@ -88,38 +88,18 @@ static PyParam* PyParam_fromParam(const Param ¶m) { PyParam *pyParam = PyObject_New(PyParam, &PyParamType); pyParam->pyParamTypeId = PyUnicode_FromString(param.paramTypeId().toString().toUtf8()); - - switch (param.value().type()) { - case QVariant::Bool: - pyParam->pyValue = PyBool_FromLong(*(long*)param.value().data()); - break; - case QVariant::Int: - case QVariant::UInt: - case QVariant::LongLong: - case QVariant::ULongLong: - pyParam->pyValue = PyLong_FromLong(*(long*)param.value().data()); - break; - case QVariant::String: - case QVariant::ByteArray: - pyParam->pyValue = PyUnicode_FromString(param.value().toString().toUtf8()); - break; - case QVariant::Double: - pyParam->pyValue = PyFloat_FromDouble(param.value().toDouble()); - break; - case QVariant::Invalid: - pyParam->pyValue = Py_None; - Py_INCREF(pyParam->pyValue); - break; - default: - qCWarning(dcThingManager) << "Unhandled data type in conversion from Param to PyParam!"; - pyParam->pyValue = Py_None; - Py_INCREF(pyParam->pyValue); - break; - } + pyParam->pyValue = QVariantToPyObject(param.value()); return pyParam; } -static PyObject* PyParam_FromParamList(const ParamList ¶ms) +static Param PyParam_ToParam(PyParam *pyParam) +{ + ParamTypeId paramTypeId = ParamTypeId(PyUnicode_AsUTF8(pyParam->pyParamTypeId)); + QVariant value = PyObjectToQVariant(pyParam->pyValue); + return Param(paramTypeId, value); +} + +static PyObject* PyParams_FromParamList(const ParamList ¶ms) { PyObject* result = PyTuple_New(params.count()); for (int i = 0; i < params.count(); i++) { @@ -128,6 +108,31 @@ static PyObject* PyParam_FromParamList(const ParamList ¶ms) return result; } +static ParamList PyParams_ToParamList(PyObject *pyParams) +{ + ParamList params; + + if (pyParams != nullptr) { + return params; + } + + PyObject *iter = PyObject_GetIter(pyParams); + + while (iter) { + PyObject *next = PyIter_Next(iter); + if (!next) { + break; + } + if (next->ob_type != &PyParamType) { + qCWarning(dcThingManager()) << "Invalid parameter passed in param list"; + continue; + } + + PyParam *pyParam = reinterpret_cast(next); + params.append(PyParam_ToParam(pyParam)); + } + return params; +} static void registerParamType(PyObject *module) { diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index b396bc5d..0c742258 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -17,10 +17,22 @@ #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +/* Note: Thing is not threadsafe so we must never access thing directly in here. + * For writing access, invoking with QueuedConnections will decouple stuff + * For reading access, we keep a cache of the thing properties here and sync them + * over to the cache when they change. + * When using this, make sure to call PyThing_setThing() after constructing it. + */ + + typedef struct _thing { PyObject_HEAD - Thing *thing; - QMutex *mutex; + Thing *thing = nullptr; + PyObject *name = nullptr; + PyObject *params = nullptr; + PyObject *settings = nullptr; + PyObject *nameChangedHandler = nullptr; + QMutex *mutex = nullptr; } PyThing; @@ -30,13 +42,69 @@ static PyObject* PyThing_new(PyTypeObject *type, PyObject */*args*/, PyObject */ return nullptr; } self->mutex = new QMutex(); + return (PyObject*)self; } +static void PyThing_setThing(PyThing *self, Thing *thing) +{ + self->thing = thing; + + self->name = PyUnicode_FromString(self->thing->name().toUtf8().data()); + Py_INCREF(self->name); + + QObject::connect(thing, &Thing::nameChanged, [=](){ + self->mutex->lock(); + Py_XDECREF(self->name); + self->name = PyUnicode_FromString(self->thing->name().toUtf8().data()); + + if (!self->nameChangedHandler) { + self->mutex->unlock(); + return; + } + self->mutex->unlock(); + + PyGILState_STATE s = PyGILState_Ensure(); + PyObject_CallFunctionObjArgs(self->nameChangedHandler, self, nullptr); + + if (PyErr_Occurred()) { + PyObject *ptype, *pvalue, *ptraceback; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + if (pvalue) { + PyObject *pstr = PyObject_Str(pvalue); + if (pstr) { + const char* err_msg = PyUnicode_AsUTF8(pstr); + if (pstr) { + qCWarning(dcThingManager()) << QString(err_msg); + } + } + PyErr_Restore(ptype, pvalue, ptraceback); + } + } + + PyGILState_Release(s); + }); + + self->params = PyParams_FromParamList(self->thing->params()); + Py_INCREF(self->params); + + self->settings = PyParams_FromParamList(self->thing->settings()); + Py_INCREF(self->settings); + QObject::connect(thing, &Thing::settingChanged, [=](){ + QMutexLocker(self->mutex); + Py_XDECREF(self->settings); + self->settings = PyParams_FromParamList(self->thing->settings()); + }); +} + static void PyThing_dealloc(PyThing * self) { - Py_TYPE(self)->tp_free(self); + Py_XDECREF(self->name); + Py_XDECREF(self->params); + Py_XDECREF(self->settings); + Py_XDECREF(self->nameChangedHandler); delete self->mutex; + Py_TYPE(self)->tp_free(self); } static PyObject *PyThing_getName(PyThing *self, void */*closure*/) @@ -46,11 +114,9 @@ static PyObject *PyThing_getName(PyThing *self, void */*closure*/) PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); return nullptr; } - QString name; - QMetaObject::invokeMethod(self->thing, "name", Qt::BlockingQueuedConnection, Q_RETURN_ARG(QString, name)); - PyObject *ret = PyUnicode_FromString(name.toUtf8().data()); - Py_INCREF(ret); - return ret; + + Py_INCREF(self->name); + return self->name; } static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ @@ -67,12 +133,8 @@ static PyObject *PyThing_getSettings(PyThing *self, void */*closure*/) PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); return nullptr; } - qWarning() << "setting thread" << QThread::currentThread(); - ParamList settings; - QMetaObject::invokeMethod(self->thing, "settings", Qt::BlockingQueuedConnection, Q_RETURN_ARG(ParamList, settings)); - PyObject *ret = PyParam_FromParamList(settings); - Py_INCREF(ret); - return ret; + Py_INCREF(self->settings); + return self->settings; } static int PyThing_setSettings(PyThing */*self*/, PyObject */*value*/, void */*closure*/){ @@ -85,45 +147,22 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) char *stateTypeIdStr = nullptr; PyObject *valueObj = nullptr; - // FIXME: is there any better way to do this? Value is a variant if (!PyArg_ParseTuple(args, "sO", &stateTypeIdStr, &valueObj)) { qCWarning(dcThingManager) << "Error parsing parameters"; return nullptr; } StateTypeId stateTypeId = StateTypeId(stateTypeIdStr); - - PyObject* repr = PyObject_Repr(valueObj); - PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); - const char *bytes = PyBytes_AS_STRING(str); - - QVariant value(bytes); + QVariant value = PyObjectToQVariant(valueObj); QMutexLocker(self->mutex); if (self->thing != nullptr) { QMetaObject::invokeMethod(self->thing, "setStateValue", Qt::QueuedConnection, Q_ARG(StateTypeId, stateTypeId), Q_ARG(QVariant, value)); } - Py_XDECREF(repr); - Py_XDECREF(str); - Py_RETURN_NONE; } -static PyObject *PyThing_settingChanged(PyThing *self, void */*closure*/) -{ - QMutexLocker(self->mutex); - if (!self->thing) { - PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); - return nullptr; - } - ParamList settings; - QMetaObject::invokeMethod(self->thing, "settings", Qt::BlockingQueuedConnection, Q_RETURN_ARG(ParamList, settings)); - PyObject *ret = PyParam_FromParamList(settings); - Py_INCREF(ret); - return ret; -} - static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) { char *eventTypeIdStr = nullptr; @@ -135,35 +174,7 @@ static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) } EventTypeId eventTypeId = EventTypeId(eventTypeIdStr); - ParamList params; - - if (valueObj != nullptr) { - PyObject *iter = PyObject_GetIter(valueObj); - - while (iter) { - PyObject *next = PyIter_Next(iter); - if (!next) { - break; - } - if (next->ob_type != &PyParamType) { - qCWarning(dcThingManager()) << "Invalid parameter passed in param list"; - continue; - } - - PyParam *pyParam = reinterpret_cast(next); - ParamTypeId paramTypeId = ParamTypeId(PyUnicode_AsUTF8(pyParam->pyParamTypeId)); - - // Is there a better way to convert a PyObject to a QVariant? - PyObject* repr = PyObject_Repr(pyParam->pyValue); - PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); - const char *bytes = PyBytes_AS_STRING(str); - Py_XDECREF(repr); - Py_XDECREF(str); - - QVariant value(bytes); - params.append(Param(paramTypeId, value)); - } - } + ParamList params = PyParams_ToParamList(valueObj); QMutexLocker(self->mutex); if (self->thing != nullptr) { @@ -176,7 +187,6 @@ static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) static PyGetSetDef PyThing_getset[] = { {"name", (getter)PyThing_getName, (setter)PyThing_setName, "Thing name", nullptr}, {"settings", (getter)PyThing_getSettings, (setter)PyThing_setSettings, "Thing settings", nullptr}, - {"settingChanged", (getter)PyThing_settingChanged, nullptr, "Signal for changed settings", nullptr}, {nullptr , nullptr, nullptr, nullptr, nullptr} /* Sentinel */ }; @@ -186,6 +196,11 @@ static PyMethodDef PyThing_methods[] = { {nullptr, nullptr, 0, nullptr} // sentinel }; +static PyMemberDef PyThing_members[] = { + {"nameChangedHandler", T_OBJECT_EX, offsetof(PyThing, nameChangedHandler), 0, "Set a callback for when the thing name changes"}, + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +}; + static PyTypeObject PyThingType = { PyVarObject_HEAD_INIT(NULL, 0) "nymea.Thing", /* tp_name */ @@ -215,7 +230,7 @@ static PyTypeObject PyThingType = { 0, /* tp_iter */ 0, /* tp_iternext */ PyThing_methods, /* tp_methods */ - 0, /* tp_members */ + PyThing_members, /* tp_members */ PyThing_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ diff --git a/libnymea-core/integrations/python/pythingactioninfo.h b/libnymea-core/integrations/python/pythingactioninfo.h index 08dc1898..243dc9f6 100644 --- a/libnymea-core/integrations/python/pythingactioninfo.h +++ b/libnymea-core/integrations/python/pythingactioninfo.h @@ -18,15 +18,23 @@ typedef struct { PyThing *pyThing; PyObject *pyActionTypeId; PyObject *pyParams; + QMutex *mutex; } PyThingActionInfo; -static int PyThingActionInfo_init(PyThingActionInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) { - return 0; +static PyObject* PyThingActionInfo_new(PyTypeObject *type, PyObject */*args*/, PyObject */*kwds*/) { + PyThingActionInfo *self = (PyThingActionInfo*)type->tp_alloc(type, 0); + if (self == NULL) { + return nullptr; + } + self->mutex = new QMutex(); + return (PyObject*)self; } -static void PyThingActionInfo_dealloc(PyThingActionInfo * self) { +static void PyThingActionInfo_dealloc(PyThingActionInfo * self) +{ + delete self->mutex; Py_TYPE(self)->tp_free(self); } @@ -63,39 +71,59 @@ static PyMethodDef PyThingActionInfo_methods[] = { static PyTypeObject PyThingActionInfoType = { PyVarObject_HEAD_INIT(NULL, 0) - "nymea.ThingActionInfo", /* tp_name */ - sizeof(PyThingActionInfo), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + "nymea.ThingActionInfo", /* tp_name */ + sizeof(PyThingActionInfo), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyThingActionInfo_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "ThingActionInfo", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyThingActionInfo_methods, /* tp_methods */ + PyThingActionInfo_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + (newfunc)PyThingActionInfo_new, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ + 0, /* tp_vectorcall */ + 0, /* tp_print DEPRECATED*/ }; -static void registerThingActionInfoType(PyObject *module) { - PyThingActionInfoType.tp_new = PyType_GenericNew; - PyThingActionInfoType.tp_dealloc=(destructor) PyThingActionInfo_dealloc; - PyThingActionInfoType.tp_basicsize = sizeof(PyThingActionInfo); - PyThingActionInfoType.tp_flags = Py_TPFLAGS_DEFAULT; - PyThingActionInfoType.tp_doc = "ThingActionInfo class"; - PyThingActionInfoType.tp_methods = PyThingActionInfo_methods; - PyThingActionInfoType.tp_members = PyThingActionInfo_members; - PyThingActionInfoType.tp_init = (initproc)PyThingActionInfo_init; - +static void registerThingActionInfoType(PyObject *module) +{ if (PyType_Ready(&PyThingActionInfoType) < 0) { return; } diff --git a/libnymea-core/integrations/python/pythingdescriptor.h b/libnymea-core/integrations/python/pythingdescriptor.h index a57660b8..777d5b3f 100644 --- a/libnymea-core/integrations/python/pythingdescriptor.h +++ b/libnymea-core/integrations/python/pythingdescriptor.h @@ -18,10 +18,6 @@ typedef struct { PyObject* pyDescription; } PyThingDescriptor; -static PyMethodDef PyThingDescriptor_methods[] = { -// { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, - {nullptr, nullptr, 0, nullptr} // sentinel -}; static PyMemberDef PyThingDescriptor_members[] = { {"thingClassId", T_OBJECT_EX, offsetof(PyThingDescriptor, pyThingClassId), 0, "Descriptor thingClassId"}, @@ -30,7 +26,6 @@ static PyMemberDef PyThingDescriptor_members[] = { {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; - static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"thingClassId", "name", "description", nullptr}; @@ -57,14 +52,6 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj return 0; } -//static PyGetSetDef PyThingDescriptor_getsetters[] = { -// {"name", (getter) PyThingDescriptor_getName, (setter) PyThingDescriptir_setName, -// "Descriptor name", NULL}, -// {"last", (getter) Custom_getlast, (setter) Custom_setlast, -// "last name", NULL}, -// {NULL} /* Sentinel */ -//}; - static PyTypeObject PyThingDescriptorType = { PyVarObject_HEAD_INIT(NULL, 0) "nymea.ThingDescriptor", /* tp_name */ @@ -86,7 +73,7 @@ static PyTypeObject PyThingDescriptorType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ + "ThingDescriptor", /* tp_doc */ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; @@ -95,10 +82,6 @@ static PyTypeObject PyThingDescriptorType = { static void registerThingDescriptorType(PyObject *module) { PyThingDescriptorType.tp_new = PyType_GenericNew; - PyThingDescriptorType.tp_basicsize = sizeof(PyThingDescriptor); - PyThingDescriptorType.tp_flags = Py_TPFLAGS_DEFAULT; - PyThingDescriptorType.tp_doc = "ThingDescriptor class"; - PyThingDescriptorType.tp_methods = PyThingDescriptor_methods; PyThingDescriptorType.tp_members = PyThingDescriptor_members; PyThingDescriptorType.tp_init = reinterpret_cast(PyThingDescriptor_init); diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index 7037aaa6..af20d574 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -11,6 +11,7 @@ #include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -20,17 +21,24 @@ typedef struct { PyObject_HEAD ThingDiscoveryInfo* info; + QMutex *mutex; } PyThingDiscoveryInfo; - -static int PyThingDiscoveryInfo_init(PyThingDiscoveryInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) { - return 0; +static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/, PyObject */*kwds*/) +{ + PyThingDiscoveryInfo *self = (PyThingDiscoveryInfo*)type->tp_alloc(type, 0); + if (self == NULL) { + return nullptr; + } + self->mutex = new QMutex(); + return (PyObject*)self; } - -static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) { +static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) +{ // FIXME: Why is this not called? Seems we're leaking... Q_ASSERT(false); + delete self->mutex; Py_TYPE(self)->tp_free(self); } @@ -53,8 +61,8 @@ static PyObject * PyThingDiscoveryInfo_finish(PyThingDiscoveryInfo* self, PyObje Py_RETURN_NONE; } -static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, PyObject* args) { - +static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, PyObject* args) +{ PyObject *pyObj = nullptr; if (!PyArg_ParseTuple(args, "O", &pyObj)) { @@ -90,47 +98,68 @@ static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, } static PyMethodDef PyThingDiscoveryInfo_methods[] = { - { "addDescriptor", (PyCFunction)PyThingDiscoveryInfo_addDescriptor, METH_VARARGS, "Add a new descriptor to the discovery" }, - { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "finish a discovery" }, + { "addDescriptor", (PyCFunction)PyThingDiscoveryInfo_addDescriptor, METH_VARARGS, "Add a new descriptor to the discovery" }, + { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "Finish a discovery" }, {nullptr, nullptr, 0, nullptr} // sentinel }; static PyTypeObject PyThingDiscoveryInfoType = { PyVarObject_HEAD_INIT(NULL, 0) - "nymea.ThingDiscoveryInfo", /* tp_name */ + "nymea.ThingDiscoveryInfo", /* tp_name */ sizeof(PyThingDiscoveryInfo), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + 0, /* tp_itemsize */ + (destructor)PyThingDiscoveryInfo_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "ThingDiscoveryInfo", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyThingDiscoveryInfo_methods, /* tp_methods */ + 0, //PyThingDiscoveryInfo_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + (newfunc)PyThingDiscoveryInfo_new, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ + 0, /* tp_vectorcall */ + 0, /* tp_print DEPRECATED*/ }; static void registerThingDiscoveryInfoType(PyObject *module) { - PyThingDiscoveryInfoType.tp_new = PyType_GenericNew; - PyThingDiscoveryInfoType.tp_basicsize = sizeof(PyThingDiscoveryInfo); - PyThingDiscoveryInfoType.tp_dealloc = reinterpret_cast(PyThingDiscoveryInfo_dealloc); - PyThingDiscoveryInfoType.tp_flags = Py_TPFLAGS_DEFAULT; - PyThingDiscoveryInfoType.tp_doc = "ThingDiscoveryInfo class"; - PyThingDiscoveryInfoType.tp_methods = PyThingDiscoveryInfo_methods; - PyThingDiscoveryInfoType.tp_init = reinterpret_cast(PyThingDiscoveryInfo_init); if (PyType_Ready(&PyThingDiscoveryInfoType) < 0) { return; diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index fbb76d21..ab4a589f 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -30,8 +30,8 @@ static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject */*args*/, Py } static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) { - Py_TYPE(self)->tp_free(self); delete self->mutex; + Py_TYPE(self)->tp_free(self); } static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args) { @@ -117,7 +117,8 @@ static PyTypeObject PyThingSetupInfoType = { 0, /* tp_print DEPRECATED*/ }; -static void registerThingSetupInfoType(PyObject *module) { +static void registerThingSetupInfoType(PyObject *module) +{ if (PyType_Ready(&PyThingSetupInfoType) < 0) { return; } diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h new file mode 100644 index 00000000..ccce2c43 --- /dev/null +++ b/libnymea-core/integrations/python/pyutils.h @@ -0,0 +1,61 @@ +#ifndef PYUTILS_H +#define PYUTILS_H + +#include + +#include "loggingcategories.h" + +#include + +/* Returns a PyObject. RefCount will be 1 */ +PyObject *QVariantToPyObject(const QVariant &value) +{ + PyObject *pyValue = nullptr; + + switch (value.type()) { + case QVariant::Bool: + pyValue = PyBool_FromLong(*(long*)value.data()); + break; + case QVariant::Int: + case QVariant::UInt: + case QVariant::LongLong: + case QVariant::ULongLong: + pyValue = PyLong_FromLong(*(long*)value.data()); + break; + case QVariant::String: + case QVariant::ByteArray: + pyValue = PyUnicode_FromString(value.toString().toUtf8()); + break; + case QVariant::Double: + pyValue = PyFloat_FromDouble(value.toDouble()); + break; + case QVariant::Invalid: + pyValue = Py_None; + Py_INCREF(pyValue); + break; + default: + qCWarning(dcThingManager) << "Unhandled data type in conversion from Param to PyParam!"; + pyValue = Py_None; + Py_INCREF(pyValue); + break; + } + + return pyValue; +} + +QVariant PyObjectToQVariant(PyObject *pyObject) +{ + // FIXME: is there any better way to do this? + PyObject* repr = PyObject_Repr(pyObject); + PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + const char *bytes = PyBytes_AS_STRING(str); + + QVariant value(bytes); + + Py_XDECREF(repr); + Py_XDECREF(str); + + return value; +} + +#endif // PYUTILS_H diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index a4551ed1..3703cc4b 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -15,11 +15,14 @@ #include #include #include +#include +#include PyThreadState* PythonIntegrationPlugin::s_mainThread = nullptr; PyObject* PythonIntegrationPlugin::s_nymeaModule = nullptr; PyObject* PythonIntegrationPlugin::s_asyncio = nullptr; +QHash PythonIntegrationPlugin::s_plugins; // Write to stdout/stderr PyObject* nymea_write(PyObject* /*self*/, PyObject* args) @@ -106,6 +109,147 @@ PyMODINIT_FUNC PyInit_nymea(void) return m; } +PyObject* PythonIntegrationPlugin::pyConfiguration(PyObject* self, PyObject* /*args*/) +{ + PythonIntegrationPlugin *plugin = s_plugins.key(self); + if (!plugin) { + qCWarning(dcThingManager()) << "Cannot find plugin instance for this python module."; + return nullptr; + } + plugin->m_mutex.lock(); + PyObject *params = PyParams_FromParamList(plugin->configuration()); + plugin->m_mutex.unlock(); + return params; +} + +PyObject *PythonIntegrationPlugin::pyConfigValue(PyObject *self, PyObject *args) +{ + char *paramTypeIdStr = nullptr; + + if (!PyArg_ParseTuple(args, "s", ¶mTypeIdStr)) { + qCWarning(dcThingManager) << "Error parsing parameters"; + return nullptr; + } + + ParamTypeId paramTypeId = ParamTypeId(paramTypeIdStr); + + PythonIntegrationPlugin *plugin = s_plugins.key(self); + if (!plugin) { + qCWarning(dcThingManager()) << "Cannot find plugin instance for this python module."; + return nullptr; + } + + plugin->m_mutex.lock(); + QVariant value = plugin->m_pluginConfigCopy.paramValue(paramTypeId); + plugin->m_mutex.unlock(); + return QVariantToPyObject(value); +} + +PyObject *PythonIntegrationPlugin::pySetConfigValue(PyObject *self, PyObject *args) +{ + char *paramTypeIdStr = nullptr; + PyObject *valueObj = nullptr; + + if (!PyArg_ParseTuple(args, "sO", ¶mTypeIdStr, &valueObj)) { + qCWarning(dcThingManager) << "Error parsing parameters"; + return nullptr; + } + + ParamTypeId paramTypeId = EventTypeId(paramTypeIdStr); + QVariant value = PyObjectToQVariant(valueObj); + + PythonIntegrationPlugin *plugin = s_plugins.key(self); + if (!plugin) { + qCWarning(dcThingManager()) << "Cannot find plugin instance for this python module."; + return nullptr; + } + + QMetaObject::invokeMethod(plugin, "setConfigValue", Qt::QueuedConnection, Q_ARG(ParamTypeId, paramTypeId), Q_ARG(QVariant, value)); + + Py_RETURN_NONE; +} + +PyObject *PythonIntegrationPlugin::pyMyThings(PyObject *self, PyObject */*args*/) +{ + PythonIntegrationPlugin *plugin = s_plugins.key(self); + if (!plugin) { + qCWarning(dcThingManager()) << "Cannot find plugin instance for this python module."; + return nullptr; + } + + plugin->m_mutex.lock(); + PyObject* result = PyTuple_New(plugin->m_things.count()); + for (int i = 0; i < plugin->m_things.count(); i++) { + Thing *thing = plugin->m_things.keys().at(i); + PyTuple_SET_ITEM(result, i, (PyObject*)plugin->m_things.value(thing)); + } + plugin->m_mutex.unlock(); + return result; +} + +PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject *args) +{ + PyObject *pyParams; + + if (!PyArg_ParseTuple(args, "O", &pyParams)) { + qCWarning(dcThingManager()) << "Error parsing args. Not a param list"; + return nullptr; + } + + PyObject *iter = PyObject_GetIter(pyParams); + if (!iter) { + qCWarning(dcThingManager()) << "Error parsing args. Not a param list"; + return nullptr; + } + + ThingDescriptors descriptors; + + while (true) { + PyObject *next = PyIter_Next(iter); + if (!next) { + // nothing left in the iterator + break; + } + + if (next->ob_type != &PyThingDescriptorType) { + PyErr_SetString(PyExc_ValueError, "Invalid argument. Not a ThingDescriptor."); + return nullptr; + } + PyThingDescriptor *pyDescriptor = (PyThingDescriptor*)next; + + ThingClassId thingClassId; + if (pyDescriptor->pyThingClassId) { + thingClassId = ThingClassId(PyUnicode_AsUTF8(pyDescriptor->pyThingClassId)); + } + QString name; + if (pyDescriptor->pyName) { + name = QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->pyName)); + } + QString description; + if (pyDescriptor->pyDescription) { + description = QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->pyDescription)); + } + + ThingDescriptor descriptor(thingClassId, name, description); + descriptors.append(descriptor); + } + + PythonIntegrationPlugin *plugin = s_plugins.key(self); + QMetaObject::invokeMethod(plugin, "autoThingsAppeared", Qt::QueuedConnection, Q_ARG(ThingDescriptors, descriptors)); + + Py_RETURN_NONE; +} + +static PyMethodDef plugin_methods[] = +{ + {"configuration", PythonIntegrationPlugin::pyConfiguration, METH_VARARGS, "Get the plugin configuration."}, + {"configValue", PythonIntegrationPlugin::pyConfigValue, METH_VARARGS, "Get the plugin configuration value for a given config paramTypeId."}, + {"setConfigValue", PythonIntegrationPlugin::pySetConfigValue, METH_VARARGS, "Set the plugin configuration value for a given config paramTypeId."}, + {"myThings", PythonIntegrationPlugin::pyMyThings, METH_VARARGS, "Obtain a list of things owned by this plugin."}, + {"autoThingsAppeared", PythonIntegrationPlugin::pyAutoThingsAppeared, METH_VARARGS, "Inform the system about auto setup things having appeared."}, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationPlugin(parent) { @@ -113,8 +257,9 @@ PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationP PythonIntegrationPlugin::~PythonIntegrationPlugin() { - m_eventLoop.cancel(); - m_eventLoop.waitForFinished(); + PyGILState_STATE s = PyGILState_Ensure(); + Py_XDECREF(s_plugins.take(this)); + PyGILState_Release(s); } void PythonIntegrationPlugin::initPython() @@ -148,10 +293,11 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) qCWarning(dcThingManager()) << "Error parsing metadata file:" << error.errorString(); return false; } - m_metaData = PluginMetadata(jsonDoc.object()); - - qWarning() << "main thread" << QThread::currentThread(); - + setMetaData(PluginMetadata(jsonDoc.object())); + if (!metadata().isValid()) { + qCWarning(dcThingManager()) << "Plugin metadata not valid for plugin:" << scriptFile; + return false; + } PyGILState_STATE s = PyGILState_Ensure(); @@ -168,10 +314,11 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) } qCDebug(dcThingManager()) << "Imported python plugin from" << fi.absoluteFilePath(); + s_plugins.insert(this, m_module); // Set up logger with appropriate logging category PyNymeaLoggingHandler *logger = reinterpret_cast(_PyObject_New(&PyNymeaLoggingHandlerType)); - QString category = m_metaData.pluginName(); + QString category = metadata().pluginName(); category.replace(0, 1, category[0].toUpper()); logger->category = static_cast(malloc(category.length() + 1)); memset(logger->category, '0', category.length() +1); @@ -182,25 +329,55 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) // Export metadata ids into module exportIds(); + // Register config access methods + PyModule_AddFunctions(m_module, plugin_methods); + PyGILState_Release(s); + + // Set up connections to be forwareded into the plugin + connect(this, &PythonIntegrationPlugin::configValueChanged, this, [this](const ParamTypeId ¶mTypeId, const QVariant &value){ + // Sync changed value to the thread-safe copy + m_mutex.lock(); + m_pluginConfigCopy.setParamValue(paramTypeId, value); + m_mutex.unlock(); + + // And call the handler - if any + PyObject *pyParamTypeId = PyUnicode_FromString(paramTypeId.toString().toUtf8()); + PyObject *pyValue = QVariantToPyObject(value); + callPluginFunction("configValueChanged", pyParamTypeId, pyValue); + }); + return true; } void PythonIntegrationPlugin::init() { - callPluginFunction("init", nullptr); + m_mutex.lock(); + m_pluginConfigCopy = configuration(); + m_mutex.unlock(); + + callPluginFunction("init"); +} + +void PythonIntegrationPlugin::startMonitoringAutoThings() +{ + callPluginFunction("startMonitoringAutoThings"); } void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) { - PyThingDiscoveryInfo *pyInfo = PyObject_New(PyThingDiscoveryInfo, &PyThingDiscoveryInfoType); + PyGILState_STATE s = PyGILState_Ensure(); + + PyThingDiscoveryInfo *pyInfo = (PyThingDiscoveryInfo*)PyObject_CallObject((PyObject*)&PyThingDiscoveryInfoType, NULL); pyInfo->info = info; + PyGILState_Release(s); + + connect(info, &ThingDiscoveryInfo::destroyed, this, [=](){ - PyGILState_STATE s = PyGILState_Ensure(); + QMutexLocker(pyInfo->mutex); pyInfo->info = nullptr; - PyObject_Del(pyInfo); - PyGILState_Release(s); + Py_DECREF(pyInfo); }); callPluginFunction("discoverThings", reinterpret_cast(pyInfo)); @@ -209,9 +386,9 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) { PyGILState_STATE s = PyGILState_Ensure(); - PyThing *pyThing = (PyThing*)PyObject_CallObject((PyObject*)&PyThingType, NULL); - pyThing->thing = info->thing(); + PyThing *pyThing = (PyThing*)PyObject_CallObject((PyObject*)&PyThingType, NULL); + PyThing_setThing(pyThing, info->thing()); PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, NULL); pyInfo->info = info; @@ -222,13 +399,15 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) connect(info, &ThingSetupInfo::finished, this, [=](){ if (info->status() == Thing::ThingErrorNoError) { + m_mutex.lock(); m_things.insert(info->thing(), pyThing); + m_mutex.unlock(); } else { - Py_DECREF(pyThing); + cleanupPyThing(pyThing); } }); connect(info, &ThingSetupInfo::aborted, this, [=](){ - Py_DECREF(pyThing); + cleanupPyThing(pyThing); }); connect(info, &ThingSetupInfo::destroyed, this, [=](){ QMutexLocker(pyInfo->mutex); @@ -252,21 +431,20 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) PyGILState_STATE s = PyGILState_Ensure(); - PyThingActionInfo *pyInfo = PyObject_New(PyThingActionInfo, &PyThingActionInfoType); + PyThingActionInfo *pyInfo = (PyThingActionInfo*)PyObject_CallObject((PyObject*)&PyThingActionInfoType, NULL); pyInfo->info = info; pyInfo->pyThing = pyThing; pyInfo->pyActionTypeId = PyUnicode_FromString(info->action().actionTypeId().toString().toUtf8()); - pyInfo->pyParams = PyParam_FromParamList(info->action().params()); + pyInfo->pyParams = PyParams_FromParamList(info->action().params()); PyGILState_Release(s); - connect(info, &ThingActionInfo::destroyed, this, [=](){ - PyGILState_STATE s = PyGILState_Ensure(); + connect(info, &ThingActionInfo::destroyed, this, [=](){ + QMutexLocker(pyInfo->mutex); pyInfo->pyActionTypeId = nullptr; Py_XDECREF(pyInfo->pyActionTypeId); pyInfo->info = nullptr; Py_DECREF(pyInfo); - PyGILState_Release(s); }); callPluginFunction("executeAction", reinterpret_cast(pyInfo)); @@ -278,11 +456,11 @@ void PythonIntegrationPlugin::thingRemoved(Thing *thing) callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); - QMutexLocker(pyThing->mutex); - pyThing->thing = nullptr; - Py_DECREF(pyThing); + cleanupPyThing(pyThing); + m_mutex.lock(); m_things.remove(thing); + m_mutex.unlock(); } void PythonIntegrationPlugin::dumpError() @@ -309,10 +487,17 @@ void PythonIntegrationPlugin::dumpError() void PythonIntegrationPlugin::exportIds() { qCDebug(dcThingManager()) << "Exporting plugin IDs:"; - QString pluginName = "pluginId"; - QString pluginId = m_metaData.pluginId().toString(); + QString pluginName = metadata().pluginName(); + QString pluginId = metadata().pluginId().toString(); qCDebug(dcThingManager()) << "- Plugin:" << pluginName << pluginId; - PyModule_AddStringConstant(m_module, pluginName.toUtf8(), pluginId.toUtf8()); + PyModule_AddStringConstant(m_module, "pluginId", pluginId.toUtf8()); + + exportParamTypes(configurationDescription(), pluginName, "", "plugin"); + + foreach (const Vendor &vendor, supportedVendors()) { + qCDebug(dcThingManager()) << "|- Vendor:" << vendor.name() << vendor.id().toString(); + PyModule_AddStringConstant(m_module, vendor.name().toUtf8(), vendor.id().toString().toUtf8()); + } foreach (const ThingClass &thingClass, supportedThings()) { exportThingClass(thingClass); @@ -322,13 +507,8 @@ void PythonIntegrationPlugin::exportIds() void PythonIntegrationPlugin::exportThingClass(const ThingClass &thingClass) { QString variableName = QString("%1ThingClassId").arg(thingClass.name()); - if (m_variableNames.contains(variableName)) { - qWarning().nospace() << "Error: Duplicate name " << variableName << " for ThingClass " << thingClass.id() << ". Skipping entry."; - return; - } - m_variableNames.append(variableName); - qCDebug(dcThingManager()) << "|- ThingClass:" << variableName << thingClass.id(); + qCDebug(dcThingManager()) << "|- ThingClass:" << variableName << thingClass.id().toString(); PyModule_AddStringConstant(m_module, variableName.toUtf8(), thingClass.id().toString().toUtf8()); exportParamTypes(thingClass.paramTypes(), thingClass.name(), "", "thing"); @@ -345,12 +525,7 @@ void PythonIntegrationPlugin::exportParamTypes(const ParamTypes ¶mTypes, con { foreach (const ParamType ¶mType, paramTypes) { QString variableName = QString("%1ParamTypeId").arg(thingClassName + typeName[0].toUpper() + typeName.right(typeName.length()-1) + typeClass + paramType.name()[0].toUpper() + paramType.name().right(paramType.name().length() -1 )); - if (m_variableNames.contains(variableName)) { - qWarning().nospace() << "Error: Duplicate name " << variableName << " for ParamTypeId " << paramType.id() << ". Skipping entry."; - continue; - } - m_variableNames.append(variableName); - + qCDebug(dcThingManager()) << " |- ParamType:" << variableName << paramType.id().toString(); PyModule_AddStringConstant(m_module, variableName.toUtf8(), paramType.id().toString().toUtf8()); } } @@ -359,12 +534,7 @@ void PythonIntegrationPlugin::exportStateTypes(const StateTypes &stateTypes, con { foreach (const StateType &stateType, stateTypes) { QString variableName = QString("%1%2StateTypeId").arg(thingClassName, stateType.name()[0].toUpper() + stateType.name().right(stateType.name().length() - 1)); - if (m_variableNames.contains(variableName)) { - qWarning().nospace() << "Error: Duplicate name " << variableName << " for StateType " << stateType.name() << " in ThingClass " << thingClassName << ". Skipping entry."; - return; - } - m_variableNames.append(variableName); - qCDebug(dcThingManager()) << "|- StateType:" << variableName << stateType.id(); + qCDebug(dcThingManager()) << " |- StateType:" << variableName << stateType.id().toString(); PyModule_AddStringConstant(m_module, variableName.toUtf8(), stateType.id().toString().toUtf8()); } } @@ -373,13 +543,8 @@ void PythonIntegrationPlugin::exportEventTypes(const EventTypes &eventTypes, con { foreach (const EventType &eventType, eventTypes) { QString variableName = QString("%1%2EventTypeId").arg(thingClassName, eventType.name()[0].toUpper() + eventType.name().right(eventType.name().length() - 1)); - if (m_variableNames.contains(variableName)) { - qWarning().nospace() << "Error: Duplicate name " << variableName << " for EventType " << eventType.name() << " in ThingClass " << thingClassName << ". Skipping entry."; - return; - } - m_variableNames.append(variableName); + qCDebug(dcThingManager()) << " |- EventType:" << variableName << eventType.id().toString(); PyModule_AddStringConstant(m_module, variableName.toUtf8(), eventType.id().toString().toUtf8()); - exportParamTypes(eventType.paramTypes(), thingClassName, "Event", eventType.name()); } @@ -389,13 +554,8 @@ void PythonIntegrationPlugin::exportActionTypes(const ActionTypes &actionTypes, { foreach (const ActionType &actionType, actionTypes) { QString variableName = QString("%1%2ActionTypeId").arg(thingClassName, actionType.name()[0].toUpper() + actionType.name().right(actionType.name().length() - 1)); - if (m_variableNames.contains(variableName)) { - qWarning().nospace() << "Error: Duplicate name " << variableName << " for ActionType " << actionType.name() << " in ThingClass " << thingClassName << ". Skipping entry."; - return; - } - m_variableNames.append(variableName); + qCDebug(dcThingManager()) << " |- ActionType:" << variableName << actionType.id().toString(); PyModule_AddStringConstant(m_module, variableName.toUtf8(), actionType.id().toString().toUtf8()); - exportParamTypes(actionType.paramTypes(), thingClassName, "Action", actionType.name()); } } @@ -404,20 +564,15 @@ void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &ac { foreach (const ActionType &actionType, actionTypes) { QString variableName = QString("%1%2BrowserItemActionTypeId").arg(thingClassName, actionType.name()[0].toUpper() + actionType.name().right(actionType.name().length() - 1)); - if (m_variableNames.contains(variableName)) { - qWarning().nospace() << "Error: Duplicate name " << variableName << " for Browser Item ActionType " << actionType.name() << " in ThingClass " << thingClassName << ". Skipping entry."; - return; - } - m_variableNames.append(variableName); + qCDebug(dcThingManager()) << " |- BrowserActionType:" << variableName << actionType.id().toString(); PyModule_AddStringConstant(m_module, variableName.toUtf8(), actionType.id().toString().toUtf8()); - exportParamTypes(actionType.paramTypes(), thingClassName, "BrowserItemAction", actionType.name()); } } -void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param) +void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param1, PyObject *param2) { PyGILState_STATE s = PyGILState_Ensure(); @@ -433,7 +588,7 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje dumpError(); - PyObject *result = PyObject_CallFunctionObjArgs(pFunc, param, nullptr); + PyObject *result = PyObject_CallFunctionObjArgs(pFunc, param1, param2, nullptr); Py_XDECREF(pFunc); @@ -485,3 +640,18 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje PyGILState_Release(s); } +void PythonIntegrationPlugin::cleanupPyThing(PyThing *pyThing) +{ + // It could happen that the python thread is currently holding the mutex + // whike waiting on a blocking queued connection on the thing (e.g. PyThing_name). + // We'd deadlock if we wait for the mutex forever here. So let's process events + // while waiting for it... + while (!pyThing->mutex->tryLock()) { + qApp->processEvents(QEventLoop::EventLoopExec); + } + + pyThing->thing = nullptr; + pyThing->mutex->unlock(); + Py_DECREF(pyThing); +} + diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index a7cc47b2..f0145fe0 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -26,6 +26,7 @@ public: bool loadScript(const QString &scriptFile); void init() override; + void startMonitoringAutoThings() override; void discoverThings(ThingDiscoveryInfo *info) override; void setupThing(ThingSetupInfo *info) override; void postSetupThing(Thing *thing) override; @@ -34,6 +35,12 @@ public: static void dumpError(); + static PyObject* pyConfiguration(PyObject* self, PyObject* args); + static PyObject* pyConfigValue(PyObject* self, PyObject* args); + static PyObject* pySetConfigValue(PyObject* self, PyObject* args); + static PyObject* pyMyThings(PyObject *self, PyObject* args); + static PyObject* pyAutoThingsAppeared(PyObject *self, PyObject* args); + private: void exportIds(); void exportThingClass(const ThingClass &thingClass); @@ -44,22 +51,31 @@ private: void exportBrowserItemActionTypes(const ActionTypes &actionTypes, const QString &thingClassName); - void callPluginFunction(const QString &function, PyObject *param); + void callPluginFunction(const QString &function, PyObject *param1 = nullptr, PyObject *param2 = nullptr); + void cleanupPyThing(PyThing *pyThing); private: -// static QHash s_modules; - static PyThreadState* s_mainThread; + + // Modules imported into the global context static PyObject *s_nymeaModule; static PyObject *s_asyncio; - PyObject *m_module = nullptr; - QFuture m_eventLoop; + // A map of plugin instances to plugin python scripts/modules + // Make sure to hold the GIL when accessing this. + static QHash s_plugins; + // Used for guarding access from the python threads to the plugin instance + QMutex m_mutex; + + // The imported plugin module + PyObject *m_module = nullptr; + + // Things held by this plugin instance QHash m_things; - QStringList m_variableNames; - + // Need to keep a copy of plugin params and sync that in a thread-safe manner + ParamList m_pluginConfigCopy; }; #endif // PYTHONINTEGRATIONPLUGIN_H diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index c8b0f14f..d1a8a8d3 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -95,6 +95,8 @@ ThingManagerImplementation::ThingManagerImplementation(HardwareManager *hardware // Make sure this is always emitted after plugins and things are loaded QMetaObject::invokeMethod(this, "onLoaded", Qt::QueuedConnection); + + PythonIntegrationPlugin::initPython(); } ThingManagerImplementation::~ThingManagerImplementation() @@ -159,13 +161,13 @@ QList ThingManagerImplementation::pluginsMetadata() return pluginList; } -void ThingManagerImplementation::registerStaticPlugin(IntegrationPlugin *plugin, const PluginMetadata &metaData) +void ThingManagerImplementation::registerStaticPlugin(IntegrationPlugin *plugin) { - if (!metaData.isValid()) { + if (!plugin->metadata().isValid()) { qCWarning(dcThingManager()) << "Plugin metadata not valid. Not loading static plugin:" << plugin->pluginName(); return; } - loadPlugin(plugin, metaData); + loadPlugin(plugin); } IntegrationPlugins ThingManagerImplementation::plugins() const @@ -1230,161 +1232,73 @@ ThingActionInfo *ThingManagerImplementation::executeAction(const Action &action) } void ThingManagerImplementation::loadPlugins() -{ +{ + QStringList searchDirs; + // Add first level of subdirectories to the plugin search dirs so we can point to a collection of plugins foreach (const QString &path, pluginSearchDirs()) { + searchDirs.append(path); + QDir dir(path); + foreach (const QString &subdir, dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot)) { + searchDirs.append(path + '/' + subdir); + } + } + + foreach (const QString &path, searchDirs) { QDir dir(path); qCDebug(dcThingManager) << "Loading plugins from:" << dir.absolutePath(); - foreach (const QString &entry, dir.entryList()) { - QFileInfo fi; + foreach (const QString &entry, dir.entryList({"*.so", "*.js", "*.py"}, QDir::Files)) { + + IntegrationPlugin *plugin = nullptr; + + QFileInfo fi(path + '/' + entry); if (entry.startsWith("libnymea_integrationplugin") && entry.endsWith(".so")) { - fi.setFile(path + "/" + entry); - } else { - fi.setFile(path + "/" + entry + "/libnymea_integrationplugin" + entry + ".so"); - } - - if (!fi.exists()) - continue; - - // Check plugin API version compatibility - QLibrary lib(fi.absoluteFilePath()); - if (!lib.load()) { - qCWarning(dcThingManager()).nospace() << "Error loading plugin " << fi.absoluteFilePath() << ": " << lib.errorString(); - continue; - } - - QFunctionPointer versionFunc = lib.resolve("libnymea_api_version"); - if (!versionFunc) { - qCWarning(dcThingManager()).nospace() << "Unable to resolve version in plugin " << fi.absoluteFilePath() << ". Not loading plugin."; - lib.unload(); - continue; - } - - QString version = reinterpret_cast(versionFunc)(); - lib.unload(); - QStringList parts = version.split('.'); - QStringList coreParts = QString(LIBNYMEA_API_VERSION).split('.'); - if (parts.length() != 3 || parts.at(0).toInt() != coreParts.at(0).toInt() || parts.at(1).toInt() > coreParts.at(1).toInt()) { - qCWarning(dcThingManager()).nospace() << "Libnymea API mismatch for " << fi.absoluteFilePath() << ". Core API: " << LIBNYMEA_API_VERSION << ", Plugin API: " << version; - continue; - } - - // Version is ok. Now load the plugin - QPluginLoader loader; - loader.setFileName(fi.absoluteFilePath()); - loader.setLoadHints(QLibrary::ResolveAllSymbolsHint); - - qCDebug(dcThingManager()) << "Loading plugin from:" << fi.absoluteFilePath(); - if (!loader.load()) { - qCWarning(dcThingManager) << "Could not load plugin data of" << entry << "\n" << loader.errorString(); - continue; - } - - QJsonObject pluginInfo = loader.metaData().value("MetaData").toObject(); - PluginMetadata metaData(pluginInfo, false, false); - if (!metaData.isValid()) { - foreach (const QString &error, metaData.validationErrors()) { - qCWarning(dcThingManager()) << error; - } - loader.unload(); - continue; - } - - IntegrationPlugin *pluginIface = qobject_cast(loader.instance()); - if (!pluginIface) { - qCWarning(dcThingManager) << "Could not get plugin instance of" << entry; - loader.unload(); - continue; - } - if (m_integrationPlugins.contains(pluginIface->pluginId())) { - qCWarning(dcThingManager()) << "A plugin with this ID is already loaded. Not loading" << entry; - continue; - } - loadPlugin(pluginIface, metaData); - PluginInfoCache::cachePluginInfo(pluginInfo); - } - } + plugin = createCppIntegrationPlugin(fi.absoluteFilePath()); + } else if (entry.startsWith("integrationplugin") && entry.endsWith(".js")) { #if QT_VERSION >= QT_VERSION_CHECK(5,12,0) - foreach (const QString &path, pluginSearchDirs()) { - QDir dir(path); - qCDebug(dcThingManager) << "Loading JS plugins from:" << dir.absolutePath(); - foreach (const QString &entry, dir.entryList()) { - QFileInfo jsFi; - QFileInfo jsonFi; - - if (entry.endsWith(".js")) { - jsFi.setFile(path + "/" + entry); - } else { - jsFi.setFile(path + "/" + entry + "/" + entry + ".js"); - } - - if (!jsFi.exists()) { - continue; - } - - ScriptIntegrationPlugin *plugin = new ScriptIntegrationPlugin(this); - bool ret = plugin->loadScript(jsFi.absoluteFilePath()); - if (!ret) { - delete plugin; - qCWarning(dcThingManager()) << "JS plugin failed to load"; - continue; - } - PluginMetadata metaData(plugin->metaData()); - if (!metaData.isValid()) { - qCWarning(dcThingManager()) << "Not loading JS plugin. Invalid metadata."; - foreach (const QString &error, metaData.validationErrors()) { - qCWarning(dcThingManager()) << error; - delete plugin; - continue; + ScriptIntegrationPlugin *p = new ScriptIntegrationPlugin(this); + bool ok = p->loadScript(fi.absoluteFilePath()); + if (ok) { + plugin = p; + } else { + delete p; } - } - loadPlugin(plugin, metaData); - } - } +#else + qCWarning(dcThingManager()) << "Not loading JS plugin as JS plugin support is not included in this nymea instance." #endif - - PythonIntegrationPlugin::initPython(); - - { - PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); - bool ok = p->loadScript("/home/micha/Develop/nymea-plugin-pytest/integrationpluginpytest.py"); - if (!ok) { - qCWarning(dcThingManager()) << "Error loading plugin"; - return; - } - if (!p->metadata().isValid()) { - qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; - foreach (const QString &error, p->metadata().validationErrors()) { - qCWarning(dcThingManager()) << error; + } else if (entry.startsWith("integrationplugin") && entry.endsWith(".py")) { + PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); + bool ok = p->loadScript(fi.absoluteFilePath()); + if (ok) { + plugin = p; + } else { + delete p; + } + } else { + // Not a known plugin type + continue; } - return; - } - loadPlugin(p, p->metadata()); - } - { - PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); - bool ok = p->loadScript("/home/micha/Develop/nymea-plugin-pytest2/integrationpluginpytest2.py"); - if (!ok) { - qCWarning(dcThingManager()) << "Error loading plugin"; - return; - } - PluginMetadata metaData(p->metadata()); - if (!metaData.isValid()) { - qCWarning(dcThingManager()) << "Not loading Python plugin. Invalid metadata."; - foreach (const QString &error, metaData.validationErrors()) { - qCWarning(dcThingManager()) << error; - } - return; - } - loadPlugin(p, metaData); + if (!plugin) { + qCWarning(dcThingManager()) << "Error loading plugin:" << fi.absoluteFilePath(); + continue; + } + + if (m_integrationPlugins.contains(plugin->pluginId())) { + qCWarning(dcThingManager()) << "A plugin with this ID is already loaded. Not loading" << entry << plugin->pluginId(); + delete plugin; + continue; + } + loadPlugin(plugin); + PluginInfoCache::cachePluginInfo(plugin->metadata().jsonObject()); + } } } -void ThingManagerImplementation::loadPlugin(IntegrationPlugin *pluginIface, const PluginMetadata &metaData) +void ThingManagerImplementation::loadPlugin(IntegrationPlugin *pluginIface) { pluginIface->setParent(this); - pluginIface->initPlugin(metaData, this, m_hardwareManager); + pluginIface->initPlugin(this, m_hardwareManager); qCDebug(dcThingManager) << "**** Loaded plugin" << pluginIface->pluginName(); foreach (const Vendor &vendor, pluginIface->supportedVendors()) { @@ -2152,6 +2066,68 @@ void ThingManagerImplementation::registerThing(Thing *thing) connect(thing, &Thing::nameChanged, this, &ThingManagerImplementation::slotThingNameChanged); } +IntegrationPlugin *ThingManagerImplementation::createCppIntegrationPlugin(const QString &absoluteFilePath) +{ + // Check plugin API version compatibility + QLibrary lib(absoluteFilePath); + if (!lib.load()) { + qCWarning(dcThingManager()).nospace() << "Error loading plugin " << absoluteFilePath << ": " << lib.errorString(); + return nullptr; + } + + QFunctionPointer versionFunc = lib.resolve("libnymea_api_version"); + if (!versionFunc) { + qCWarning(dcThingManager()).nospace() << "Unable to resolve version in plugin " << absoluteFilePath << ". Not loading plugin."; + lib.unload(); + return nullptr; + } + + QString version = reinterpret_cast(versionFunc)(); + lib.unload(); + QStringList parts = version.split('.'); + QStringList coreParts = QString(LIBNYMEA_API_VERSION).split('.'); + if (parts.length() != 3 || parts.at(0).toInt() != coreParts.at(0).toInt() || parts.at(1).toInt() > coreParts.at(1).toInt()) { + qCWarning(dcThingManager()).nospace() << "Libnymea API mismatch for " << absoluteFilePath << ". Core API: " << LIBNYMEA_API_VERSION << ", Plugin API: " << version; + return nullptr; + } + + // Version is ok. Now load the plugin + QPluginLoader loader; + loader.setFileName(absoluteFilePath); + loader.setLoadHints(QLibrary::ResolveAllSymbolsHint); + + qCDebug(dcThingManager()) << "Loading plugin from:" << absoluteFilePath; + if (!loader.load()) { + qCWarning(dcThingManager) << "Could not load plugin data of" << absoluteFilePath << "\n" << loader.errorString(); + return nullptr; + } + + QJsonObject pluginInfo = loader.metaData().value("MetaData").toObject(); + PluginMetadata metaData(pluginInfo, false, false); + if (!metaData.isValid()) { + foreach (const QString &error, metaData.validationErrors()) { + qCWarning(dcThingManager()) << error; + } + loader.unload(); + return nullptr; + } + + QObject *p = loader.instance(); + if (!p) { + qCWarning(dcThingManager()) << "Error loading plugin:" << loader.errorString(); + return nullptr; + } + IntegrationPlugin *pluginIface = qobject_cast(p); + if (!pluginIface) { + qCWarning(dcThingManager) << "Could not get plugin instance of" << absoluteFilePath; + return nullptr; + } + + pluginIface->setMetaData(PluginMetadata(pluginInfo)); + + return pluginIface; +} + void ThingManagerImplementation::storeThingStates(Thing *thing) { ThingClass thingClass = m_supportedThings.value(thing->thingClassId()); diff --git a/libnymea-core/integrations/thingmanagerimplementation.h b/libnymea-core/integrations/thingmanagerimplementation.h index ed6c4060..c127eef7 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.h +++ b/libnymea-core/integrations/thingmanagerimplementation.h @@ -72,7 +72,7 @@ public: static QStringList pluginSearchDirs(); static QList pluginsMetadata(); - void registerStaticPlugin(IntegrationPlugin* plugin, const PluginMetadata &metaData); + void registerStaticPlugin(IntegrationPlugin* plugin); IntegrationPlugins plugins() const override; IntegrationPlugin *plugin(const PluginId &pluginId) const override; @@ -131,7 +131,7 @@ signals: private slots: void loadPlugins(); - void loadPlugin(IntegrationPlugin *pluginIface, const PluginMetadata &metaData); + void loadPlugin(IntegrationPlugin *pluginIface); void loadConfiguredThings(); void storeConfiguredThings(); void startMonitoringAutoThings(); @@ -165,6 +165,8 @@ private: void syncIOConnection(Thing *inputThing, const StateTypeId &stateTypeId); QVariant mapValue(const QVariant &value, const StateType &fromStateType, const StateType &toStateType, bool inverted) const; + IntegrationPlugin *createCppIntegrationPlugin(const QString &absoluteFilePath); + private: HardwareManager *m_hardwareManager; diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index d40891df..111f18d6 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -27,6 +27,7 @@ HEADERS += nymeacore.h \ integrations/python/pythingdescriptor.h \ integrations/python/pythingdiscoveryinfo.h \ integrations/python/pythingsetupinfo.h \ + integrations/python/pyutils.h \ integrations/thingmanagerimplementation.h \ integrations/translator.h \ integrations/pythonintegrationplugin.h \ diff --git a/libnymea-core/nymeacore.cpp b/libnymea-core/nymeacore.cpp index cd94309a..48c144c1 100644 --- a/libnymea-core/nymeacore.cpp +++ b/libnymea-core/nymeacore.cpp @@ -137,7 +137,7 @@ void NymeaCore::init() { CloudNotifications *cloudNotifications = m_cloudManager->createNotificationsPlugin(); - m_thingManager->registerStaticPlugin(cloudNotifications, cloudNotifications->metaData()); + m_thingManager->registerStaticPlugin(cloudNotifications); CloudTransport *cloudTransport = m_cloudManager->createTransportInterface(); m_serverManager->jsonServer()->registerTransportInterface(cloudTransport, false); diff --git a/libnymea/integrations/integrationplugin.cpp b/libnymea/integrations/integrationplugin.cpp index 17faaf45..3f15c8c9 100644 --- a/libnymea/integrations/integrationplugin.cpp +++ b/libnymea/integrations/integrationplugin.cpp @@ -374,9 +374,8 @@ ParamTypes IntegrationPlugin::configurationDescription() const return m_metaData.pluginSettings(); } -void IntegrationPlugin::initPlugin(const PluginMetadata &metadata, ThingManager *thingManager, HardwareManager *hardwareManager) +void IntegrationPlugin::initPlugin(ThingManager *thingManager, HardwareManager *hardwareManager) { - m_metaData = metadata; m_thingManager = thingManager; m_hardwareManager = hardwareManager; m_storage = new QSettings(NymeaSettings::settingsPath() + "/pluginconfig-" + pluginId().toString().remove(QRegExp("[{}]")) + ".conf", QSettings::IniFormat, this); diff --git a/libnymea/integrations/integrationplugin.h b/libnymea/integrations/integrationplugin.h index afefb72d..daeb4b06 100644 --- a/libnymea/integrations/integrationplugin.h +++ b/libnymea/integrations/integrationplugin.h @@ -107,11 +107,11 @@ public: virtual void executeBrowserItemAction(BrowserItemActionInfo *info); // Configuration - ParamTypes configurationDescription() const; - Thing::ThingError setConfiguration(const ParamList &configuration); - ParamList configuration() const; - QVariant configValue(const ParamTypeId ¶mTypeId) const; - Thing::ThingError setConfigValue(const ParamTypeId ¶mTypeId, const QVariant &value); + Q_INVOKABLE ParamTypes configurationDescription() const; + Q_INVOKABLE Thing::ThingError setConfiguration(const ParamList &configuration); + Q_INVOKABLE ParamList configuration() const; + Q_INVOKABLE QVariant configValue(const ParamTypeId ¶mTypeId) const; + Q_INVOKABLE Thing::ThingError setConfigValue(const ParamTypeId ¶mTypeId, const QVariant &value); bool isBuiltIn() const; @@ -126,25 +126,15 @@ protected: HardwareManager *hardwareManager() const; QSettings *pluginStorage() const; - PluginMetadata m_metaData; + void setMetaData(const PluginMetadata &metaData); private: friend class ThingManager; friend class ThingManagerImplementation; + void initPlugin(ThingManager *thingManager, HardwareManager *hardwareManager); - void setMetaData(const PluginMetadata &metaData); - void initPlugin(const PluginMetadata &metadata, ThingManager *thingManager, HardwareManager *hardwareManager); - - QPair > parseParamTypes(const QJsonArray &array) const; - - // Returns - QPair verifyFields(const QStringList &possibleFields, const QStringList &mandatoryFields, const QJsonObject &value) const; - - // load and verify enum values - QPair loadAndVerifyUnit(const QString &unitString) const; - QPair loadAndVerifyInputType(const QString &inputType) const; - + PluginMetadata m_metaData; ThingManager *m_thingManager = nullptr; HardwareManager *m_hardwareManager = nullptr; QSettings *m_storage = nullptr; diff --git a/libnymea/integrations/pluginmetadata.cpp b/libnymea/integrations/pluginmetadata.cpp index cd71154c..391993ea 100644 --- a/libnymea/integrations/pluginmetadata.cpp +++ b/libnymea/integrations/pluginmetadata.cpp @@ -47,6 +47,7 @@ PluginMetadata::PluginMetadata() } PluginMetadata::PluginMetadata(const QJsonObject &jsonObject, bool isBuiltIn, bool strict): + m_jsonObject(jsonObject), m_isBuiltIn(isBuiltIn), m_strictRun(strict) { @@ -98,6 +99,11 @@ ThingClasses PluginMetadata::thingClasses() const return m_thingClasses; } +QJsonObject PluginMetadata::jsonObject() const +{ + return m_jsonObject; +} + void PluginMetadata::parse(const QJsonObject &jsonObject) { bool hasError = false; diff --git a/libnymea/integrations/pluginmetadata.h b/libnymea/integrations/pluginmetadata.h index e95937e2..6fb5861b 100644 --- a/libnymea/integrations/pluginmetadata.h +++ b/libnymea/integrations/pluginmetadata.h @@ -34,6 +34,8 @@ #include "types/paramtype.h" #include "types/thingclass.h" +#include + class PluginMetadata { public: @@ -53,6 +55,7 @@ public: Vendors vendors() const; ThingClasses thingClasses() const; + QJsonObject jsonObject() const; private: void parse(const QJsonObject &jsonObject); @@ -64,6 +67,7 @@ private: bool verifyDuplicateUuid(const QUuid &uuid); private: + QJsonObject m_jsonObject; bool m_isValid = false; bool m_isBuiltIn = false; PluginId m_pluginId; diff --git a/nymea.pro b/nymea.pro index 1b2879eb..edb20dce 100644 --- a/nymea.pro +++ b/nymea.pro @@ -7,7 +7,7 @@ NYMEA_VERSION_STRING=$$system('dpkg-parsechangelog | sed -n -e "s/^Version: //p" JSON_PROTOCOL_VERSION_MAJOR=5 JSON_PROTOCOL_VERSION_MINOR=1 JSON_PROTOCOL_VERSION="$${JSON_PROTOCOL_VERSION_MAJOR}.$${JSON_PROTOCOL_VERSION_MINOR}" -LIBNYMEA_API_VERSION_MAJOR=6 +LIBNYMEA_API_VERSION_MAJOR=7 LIBNYMEA_API_VERSION_MINOR=0 LIBNYMEA_API_VERSION_PATCH=0 LIBNYMEA_API_VERSION="$${LIBNYMEA_API_VERSION_MAJOR}.$${LIBNYMEA_API_VERSION_MINOR}.$${LIBNYMEA_API_VERSION_PATCH}" diff --git a/plugins/mock/mock.pro b/plugins/mock/mock.pro index 975cc88e..63d494d4 100644 --- a/plugins/mock/mock.pro +++ b/plugins/mock/mock.pro @@ -4,7 +4,7 @@ QT+= network TARGET = $$qtLibraryTarget(nymea_integrationpluginmock) -OTHER_FILES += interationpluginmock.json +OTHER_FILES += integrationpluginmock.json SOURCES += \ integrationpluginmock.cpp \ diff --git a/plugins/mock/plugininfo.h b/plugins/mock/plugininfo.h index 4e4b891a..5b9e2da9 100644 --- a/plugins/mock/plugininfo.h +++ b/plugins/mock/plugininfo.h @@ -9,7 +9,7 @@ #include #include -extern "C" const QString libnymea_api_version() { return QString("6.0.0");} +extern "C" const QString libnymea_api_version() { return QString("7.0.0");} Q_DECLARE_LOGGING_CATEGORY(dcMock) Q_LOGGING_CATEGORY(dcMock, "Mock") diff --git a/plugins/plugins.pro b/plugins/plugins.pro index 71c20849..03817529 100644 --- a/plugins/plugins.pro +++ b/plugins/plugins.pro @@ -1,6 +1,6 @@ TEMPLATE = subdirs !disabletesting: { - SUBDIRS += mock + SUBDIRS += mock pymock } diff --git a/plugins/pymock/ids.py b/plugins/pymock/ids.py new file mode 100644 index 00000000..9941ae76 --- /dev/null +++ b/plugins/pymock/ids.py @@ -0,0 +1,4 @@ +import builtins + +builtins.pyMockPluginAutoThingCountParamTypeId = "{1d3422cb-fcdd-4ab5-ac6e-056288439343}" +builtins.foo = "bar" diff --git a/plugins/pymock/integrationpluginpymock.json b/plugins/pymock/integrationpluginpymock.json new file mode 100644 index 00000000..727a061b --- /dev/null +++ b/plugins/pymock/integrationpluginpymock.json @@ -0,0 +1,76 @@ +{ + "id": "9be90b21-778c-4080-93c9-84ae1ab60734", + "name": "pyMock", + "displayName": "Python mock plugin", + "paramTypes": [ + { + "id": "1d3422cb-fcdd-4ab5-ac6e-056288439343", + "name": "autoThingCount", + "displayName": "Number of auto things", + "type": "int", + "defaultValue": 0 + } + ], + "vendors": [ + { + "id": "2062d64d-3232-433c-88bc-0d33c0ba2ba6", + "name": "nymea", + "displayName": "nymea GmbH", + "thingClasses": [ + { + "id": "727d3e73-505d-4118-adf4-6408b46a5d48", + "name": "pyMockAuto", + "displayName": "Auto Python mock thing", + "createMethods": ["auto"], + "setupMethod": "justAdd", + "stateTypes": [ + { + "id": "12c82472-56b0-4229-b324-4aaa6850320e", + "name": "state1", + "displayName": "State 1", + "displayNameEvent": "State 1 changed", + "type": "bool", + "defaultValue": false + } + ] + }, + { + "id": "1761c256-99b1-41bd-988a-a76087f6a4f1", + "name": "pyMock", + "displayName": "Python mock thing", + "createMethods": ["user"], + "setupMethod": "justAdd", + "stateTypes": [ + { + "id": "24714828-93ec-41a7-875e-6a0b5b57d25c", + "name": "state1", + "displayName": "State 1", + "displayNameEvent": "State 1 changed", + "displayNameAction": "Set state 1", + "type": "int", + "defaultValue": 0 + } + ] + }, + { + "id": "248c5046-847b-44d0-ab7c-684ff79197dc", + "name": "pyMockDiscoveryPairing", + "displayName": "Python mock thing with discovery and pairing", + "createMethods": ["discovery"], + "setupMethod": "userAndPassword", + "stateTypes": [ + { + "id": "99d0af17-9e8c-42bb-bece-a5d114f051d3", + "name": "state1", + "displayName": "State 1", + "displayNameEvent": "State 1 changed", + "displayNameAction": "Set state 1", + "type": "int", + "defaultValue": 0 + } + ] + } + ] + } + ] +} diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py new file mode 100644 index 00000000..862b75de --- /dev/null +++ b/plugins/pymock/integrationpluginpymock.py @@ -0,0 +1,17 @@ +import nymea + +def init(): + logger.log("Python mock plugin init") + logger.log("Number of auto mocks", configValue(pyMockPluginAutoThingCountParamTypeId)) + + +def configValueChanged(paramTypeId, value): + logger.log("Plugin config value changed:", paramTypeId, value) + + +def startMonitoringAutoThings(): + logger.log("Start monitoring auto things. Already have", len(myThings())) + for i in range(configValue(pyMockPluginAutoThingCountParamTypeId), len(myThings())): + logger.log("auto thing") +# descriptor = nymea. +# autoThingsAppeared( diff --git a/plugins/pymock/pymock.pro b/plugins/pymock/pymock.pro new file mode 100644 index 00000000..bdf46f5d --- /dev/null +++ b/plugins/pymock/pymock.pro @@ -0,0 +1,22 @@ +TEMPLATE = aux + +OTHER_FILES = integrationpluginpymock.json \ + integrationpluginpymock.py + + +# Copy files to build dir as we've set plugin import paths to that +#mytarget.target = integrationpluginpymock.py +#mytarget.commands = cp $$PWD/$$mytarget.target $$mytarget.target +#mytarget.depends = mytarget2 + +#mytarget2.commands = cp $$PWD/integrationpluginpymock.json integrationpluginpymock.json + +#QMAKE_EXTRA_TARGETS += mytarget mytarget2 + + + +copydata.commands = $(COPY_DIR) $$PWD/integrationpluginpymock.json $$PWD/*.py $$OUT_PWD +first.depends = $(first) copydata +export(first.depends) +export(copydata.commands) +QMAKE_EXTRA_TARGETS += first copydata From 85c9d93ccd0a9b66032b261a90938798c0810eb4 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 29 Jun 2020 23:20:11 +0200 Subject: [PATCH 14/46] Somoe type fixes --- libnymea-core/integrations/python/pyutils.h | 4 ++-- plugins/pymock/integrationpluginpymock.json | 2 +- plugins/pymock/integrationpluginpymock.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index ccce2c43..18a220de 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -14,13 +14,13 @@ PyObject *QVariantToPyObject(const QVariant &value) switch (value.type()) { case QVariant::Bool: - pyValue = PyBool_FromLong(*(long*)value.data()); + pyValue = PyBool_FromLong(value.toBool()); break; case QVariant::Int: case QVariant::UInt: case QVariant::LongLong: case QVariant::ULongLong: - pyValue = PyLong_FromLong(*(long*)value.data()); + pyValue = PyLong_FromLong(value.toLongLong()); break; case QVariant::String: case QVariant::ByteArray: diff --git a/plugins/pymock/integrationpluginpymock.json b/plugins/pymock/integrationpluginpymock.json index 727a061b..2f2ba295 100644 --- a/plugins/pymock/integrationpluginpymock.json +++ b/plugins/pymock/integrationpluginpymock.json @@ -8,7 +8,7 @@ "name": "autoThingCount", "displayName": "Number of auto things", "type": "int", - "defaultValue": 0 + "defaultValue": "fds" } ], "vendors": [ diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 862b75de..e81c29a8 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -10,7 +10,7 @@ def configValueChanged(paramTypeId, value): def startMonitoringAutoThings(): - logger.log("Start monitoring auto things. Already have", len(myThings())) + logger.log("Start monitoring auto things. Already have", len(myThings()), configValue(pyMockPluginAutoThingCountParamTypeId)) for i in range(configValue(pyMockPluginAutoThingCountParamTypeId), len(myThings())): logger.log("auto thing") # descriptor = nymea. From 8aa91292fe1fafc9245407ce1d41bc8229c8a172 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Thu, 2 Jul 2020 11:04:37 +0200 Subject: [PATCH 15/46] more work --- libnymea-core/integrations/python/pything.h | 61 +++++++++++++------ .../python/pythingdiscoveryinfo.h | 5 -- .../integrations/pythonintegrationplugin.cpp | 38 ++++++------ .../integrations/pythonintegrationplugin.h | 2 +- plugins/pymock/integrationpluginpymock.py | 30 +++++++-- 5 files changed, 86 insertions(+), 50 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 0c742258..6fb48701 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -12,6 +12,7 @@ #include #include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -29,6 +30,8 @@ typedef struct _thing { PyObject_HEAD Thing *thing = nullptr; PyObject *name = nullptr; + PyObject *id = nullptr; + PyObject *thingClassId = nullptr; PyObject *params = nullptr; PyObject *settings = nullptr; PyObject *nameChangedHandler = nullptr; @@ -51,7 +54,8 @@ static void PyThing_setThing(PyThing *self, Thing *thing) self->thing = thing; self->name = PyUnicode_FromString(self->thing->name().toUtf8().data()); - Py_INCREF(self->name); + self->id = PyUnicode_FromString(self->thing->id().toString().toUtf8().data()); + self->thingClassId = PyUnicode_FromString(self->thing->thingClassId().toString().toUtf8().data()); QObject::connect(thing, &Thing::nameChanged, [=](){ self->mutex->lock(); @@ -67,29 +71,12 @@ static void PyThing_setThing(PyThing *self, Thing *thing) PyGILState_STATE s = PyGILState_Ensure(); PyObject_CallFunctionObjArgs(self->nameChangedHandler, self, nullptr); - if (PyErr_Occurred()) { - PyObject *ptype, *pvalue, *ptraceback; - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - if (pvalue) { - PyObject *pstr = PyObject_Str(pvalue); - if (pstr) { - const char* err_msg = PyUnicode_AsUTF8(pstr); - if (pstr) { - qCWarning(dcThingManager()) << QString(err_msg); - } - } - PyErr_Restore(ptype, pvalue, ptraceback); - } - } - PyGILState_Release(s); }); self->params = PyParams_FromParamList(self->thing->params()); - Py_INCREF(self->params); self->settings = PyParams_FromParamList(self->thing->settings()); - Py_INCREF(self->settings); QObject::connect(thing, &Thing::settingChanged, [=](){ QMutexLocker(self->mutex); Py_XDECREF(self->settings); @@ -99,6 +86,9 @@ static void PyThing_setThing(PyThing *self, Thing *thing) static void PyThing_dealloc(PyThing * self) { + Py_XDECREF(self->name); + Py_XDECREF(self->id); + Py_XDECREF(self->thingClassId); Py_XDECREF(self->name); Py_XDECREF(self->params); Py_XDECREF(self->settings); @@ -119,6 +109,30 @@ static PyObject *PyThing_getName(PyThing *self, void */*closure*/) return self->name; } +static PyObject *PyThing_getId(PyThing *self, void */*closure*/) +{ + QMutexLocker(self->mutex); + if (!self->thing) { + PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); + return nullptr; + } + + Py_INCREF(self->id); + return self->id; +} + +static PyObject *PyThing_getThingClassId(PyThing *self, void */*closure*/) +{ + QMutexLocker(self->mutex); + if (!self->thing) { + PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); + return nullptr; + } + + Py_INCREF(self->thingClassId); + return self->thingClassId; +} + static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ QString name = QString(PyUnicode_AsUTF8(value)); QMutexLocker(self->mutex); @@ -186,6 +200,8 @@ static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) static PyGetSetDef PyThing_getset[] = { {"name", (getter)PyThing_getName, (setter)PyThing_setName, "Thing name", nullptr}, + {"id", (getter)PyThing_getId, 0, "ThingId", nullptr}, + {"thingClassId", (getter)PyThing_getThingClassId, 0, "ThingClassId", nullptr}, {"settings", (getter)PyThing_getSettings, (setter)PyThing_setSettings, "Thing settings", nullptr}, {nullptr , nullptr, nullptr, nullptr, nullptr} /* Sentinel */ }; @@ -197,6 +213,7 @@ static PyMethodDef PyThing_methods[] = { }; static PyMemberDef PyThing_members[] = { + {"nameChangedHandler", T_OBJECT_EX, offsetof(PyThing, nameChangedHandler), READONLY, "Set a callback for when the thing name changes"}, {"nameChangedHandler", T_OBJECT_EX, offsetof(PyThing, nameChangedHandler), 0, "Set a callback for when the thing name changes"}, {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; @@ -238,7 +255,7 @@ static PyTypeObject PyThingType = { 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ - 0, /* tp_alloc */ + PyType_GenericAlloc, /* tp_alloc */ (newfunc)PyThing_new, /* tp_new */ 0, /* tp_free */ 0, /* tp_is_gc */ @@ -260,6 +277,12 @@ static void registerThingType(PyObject *module) return; } PyModule_AddObject(module, "Thing", reinterpret_cast(&PyThingType)); + + QMetaEnum thingErrorEnum = QMetaEnum::fromType(); + for (int i = 0; i < thingErrorEnum.keyCount(); i++) { + PyModule_AddObject(module, thingErrorEnum.key(i), PyLong_FromLong(thingErrorEnum.value(i))); + } + } diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index af20d574..9a13defe 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -165,11 +165,6 @@ static void registerThingDiscoveryInfoType(PyObject *module) return; } PyModule_AddObject(module, "ThingDiscoveryInfo", (PyObject *)&PyThingDiscoveryInfoType); - - QMetaEnum thingErrorEnum = QMetaEnum::fromType(); - for (int i = 0; i < thingErrorEnum.keyCount(); i++) { - PyModule_AddObject(module, thingErrorEnum.key(i), PyLong_FromLong(thingErrorEnum.value(i))); - } } diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 3703cc4b..027beeb0 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -308,6 +308,7 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) if (!m_module) { dumpError(); + PyErr_Clear(); qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath(); PyGILState_Release(s); return false; @@ -388,25 +389,18 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) PyGILState_STATE s = PyGILState_Ensure(); PyThing *pyThing = (PyThing*)PyObject_CallObject((PyObject*)&PyThingType, NULL); + dumpError(); PyThing_setThing(pyThing, info->thing()); PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, NULL); pyInfo->info = info; pyInfo->pyThing = pyThing; + m_things.insert(info->thing(), pyThing); + PyGILState_Release(s); - - connect(info, &ThingSetupInfo::finished, this, [=](){ - if (info->status() == Thing::ThingErrorNoError) { - m_mutex.lock(); - m_things.insert(info->thing(), pyThing); - m_mutex.unlock(); - } else { - cleanupPyThing(pyThing); - } - }); - connect(info, &ThingSetupInfo::aborted, this, [=](){ + connect(info->thing(), &Thing::destroyed, this, [=](){ cleanupPyThing(pyThing); }); connect(info, &ThingSetupInfo::destroyed, this, [=](){ @@ -416,7 +410,11 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) }); - callPluginFunction("setupThing", reinterpret_cast(pyInfo)); + bool result = callPluginFunction("setupThing", reinterpret_cast(pyInfo)); + if (!result) { + // The python code did not even start, so let's finish (fail) the setup right away + info->finish(Thing::ThingErrorSetupFailed); + } } void PythonIntegrationPlugin::postSetupThing(Thing *thing) @@ -456,8 +454,6 @@ void PythonIntegrationPlugin::thingRemoved(Thing *thing) callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); - cleanupPyThing(pyThing); - m_mutex.lock(); m_things.remove(thing); m_mutex.unlock(); @@ -496,7 +492,7 @@ void PythonIntegrationPlugin::exportIds() foreach (const Vendor &vendor, supportedVendors()) { qCDebug(dcThingManager()) << "|- Vendor:" << vendor.name() << vendor.id().toString(); - PyModule_AddStringConstant(m_module, vendor.name().toUtf8(), vendor.id().toString().toUtf8()); + PyModule_AddStringConstant(m_module, QString("%1VendorId").arg(vendor.name()).toUtf8(), vendor.id().toString().toUtf8()); } foreach (const ThingClass &thingClass, supportedThings()) { @@ -572,17 +568,18 @@ void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &ac } -void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param1, PyObject *param2) +bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param1, PyObject *param2) { PyGILState_STATE s = PyGILState_Ensure(); qCDebug(dcThingManager()) << "Calling python plugin function" << function; PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); if(!pFunc || !PyCallable_Check(pFunc)) { + PyErr_Clear(); Py_XDECREF(pFunc); qCWarning(dcThingManager()) << "Python plugin does not implement" << function; PyGILState_Release(s); - return; + return false; } @@ -595,14 +592,15 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje if (PyErr_Occurred()) { qCWarning(dcThingManager()) << "Error calling python method:"; dumpError(); + PyErr_Clear(); PyGILState_Release(s); - return; + return false; } if (QByteArray(result->ob_type->tp_name) != "coroutine") { Py_DECREF(result); PyGILState_Release(s); - return; + return true; } // Spawn a event loop for python @@ -638,6 +636,8 @@ void PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje Py_DECREF(result); PyGILState_Release(s); + + return true; } void PythonIntegrationPlugin::cleanupPyThing(PyThing *pyThing) diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index f0145fe0..6d4e906c 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -51,7 +51,7 @@ private: void exportBrowserItemActionTypes(const ActionTypes &actionTypes, const QString &thingClassName); - void callPluginFunction(const QString &function, PyObject *param1 = nullptr, PyObject *param2 = nullptr); + bool callPluginFunction(const QString &function, PyObject *param1 = nullptr, PyObject *param2 = nullptr); void cleanupPyThing(PyThing *pyThing); private: diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index e81c29a8..cd1495e9 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -1,17 +1,35 @@ import nymea +import asyncio def init(): logger.log("Python mock plugin init") - logger.log("Number of auto mocks", configValue(pyMockPluginAutoThingCountParamTypeId)) def configValueChanged(paramTypeId, value): logger.log("Plugin config value changed:", paramTypeId, value) + if paramTypeId == pyMockPluginAutoThingCountParamTypeId: + logger.log("Auto Thing Count plugin config changed:", value, "Currently there are:", len(autoThings()), "auto things") + for i in range(value, len(myThings())): + logger.log("Creating new auto thing") + descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") + autoThingsAppeared([descriptor]) def startMonitoringAutoThings(): - logger.log("Start monitoring auto things. Already have", len(myThings()), configValue(pyMockPluginAutoThingCountParamTypeId)) - for i in range(configValue(pyMockPluginAutoThingCountParamTypeId), len(myThings())): - logger.log("auto thing") -# descriptor = nymea. -# autoThingsAppeared( + logger.log("Start monitoring auto things. Have %i auto devices. Need %i." % (len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId))) + for i in range(len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId)): + logger.log("Creating new auto thing") + descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") + autoThingsAppeared([descriptor]) + + +async def setupThing(info): + info.finish(nymea.ThingErrorNoError) + + +def autoThings(): + autoThings = [] + for thing in myThings(): + if thing.thingClassId == pyMockAutoThingClassId: + autoThings.append(thing) + return autoThings From 1ce5d5980d0e3ff0e5e57a5aa7e80c6b1a147b39 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Thu, 2 Jul 2020 13:07:37 +0200 Subject: [PATCH 16/46] some fixes --- libnymea-core/integrations/python/pything.h | 3 ++- .../integrations/pythonintegrationplugin.cpp | 18 ++++++++++++------ .../integrations/pythonintegrationplugin.h | 4 ++++ plugins/pymock/integrationpluginpymock.py | 2 ++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 6fb48701..5118a44a 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -44,6 +44,7 @@ static PyObject* PyThing_new(PyTypeObject *type, PyObject */*args*/, PyObject */ if (self == NULL) { return nullptr; } + qWarning() << "*creating pyThing" << self; self->mutex = new QMutex(); return (PyObject*)self; @@ -86,6 +87,7 @@ static void PyThing_setThing(PyThing *self, Thing *thing) static void PyThing_dealloc(PyThing * self) { + qWarning() << "destryoing pything" << self; Py_XDECREF(self->name); Py_XDECREF(self->id); Py_XDECREF(self->thingClassId); @@ -213,7 +215,6 @@ static PyMethodDef PyThing_methods[] = { }; static PyMemberDef PyThing_members[] = { - {"nameChangedHandler", T_OBJECT_EX, offsetof(PyThing, nameChangedHandler), READONLY, "Set a callback for when the thing name changes"}, {"nameChangedHandler", T_OBJECT_EX, offsetof(PyThing, nameChangedHandler), 0, "Set a callback for when the thing name changes"}, {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 027beeb0..6e019619 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -43,8 +43,9 @@ PyObject* nymea_flush(PyObject* /*self*/, PyObject* /*args*/) Py_RETURN_NONE; } -PyObject* task_done(PyObject* /*self*/, PyObject* args) +PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) { + Q_UNUSED(self) PyObject *result = nullptr; @@ -63,8 +64,9 @@ PyObject* task_done(PyObject* /*self*/, PyObject* args) Py_XDECREF(repr); Py_XDECREF(str); - qCWarning(dcThingManager()) << "Exception:" << bytes; + qCWarning(dcThingManager()) << "Exception in plugin:" << bytes; + PyErr_Clear(); } Py_RETURN_NONE; @@ -75,7 +77,7 @@ static PyMethodDef nymea_methods[] = { {"write", nymea_write, METH_VARARGS, "write to stdout through qDebug()"}, {"flush", nymea_flush, METH_VARARGS, "flush stdout (no-op)"}, - {"task_done", task_done, METH_VARARGS, "callback to clean up after asyc coroutines"}, + {"task_done", PythonIntegrationPlugin::task_done, METH_VARARGS, "callback to clean up after asyc coroutines"}, {nullptr, nullptr, 0, nullptr} // sentinel }; @@ -181,7 +183,9 @@ PyObject *PythonIntegrationPlugin::pyMyThings(PyObject *self, PyObject */*args*/ PyObject* result = PyTuple_New(plugin->m_things.count()); for (int i = 0; i < plugin->m_things.count(); i++) { Thing *thing = plugin->m_things.keys().at(i); - PyTuple_SET_ITEM(result, i, (PyObject*)plugin->m_things.value(thing)); + PyThing *pyThing = plugin->m_things.value(thing); + Py_INCREF(pyThing); + PyTuple_SET_ITEM(result, i, (PyObject*)pyThing); } plugin->m_mutex.unlock(); return result; @@ -395,6 +399,7 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, NULL); pyInfo->info = info; pyInfo->pyThing = pyThing; + Py_INCREF(pyThing); m_things.insert(info->thing(), pyThing); @@ -572,7 +577,7 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje { PyGILState_STATE s = PyGILState_Ensure(); - qCDebug(dcThingManager()) << "Calling python plugin function" << function; + qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << s_plugins.key(m_module)->pluginName(); PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); if(!pFunc || !PyCallable_Check(pFunc)) { PyErr_Clear(); @@ -643,9 +648,10 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje void PythonIntegrationPlugin::cleanupPyThing(PyThing *pyThing) { // It could happen that the python thread is currently holding the mutex - // whike waiting on a blocking queued connection on the thing (e.g. PyThing_name). + // on the thing (e.g. PyThing_name). // We'd deadlock if we wait for the mutex forever here. So let's process events // while waiting for it... + qWarning() << "Locking cleanup"; while (!pyThing->mutex->tryLock()) { qApp->processEvents(QEventLoop::EventLoopExec); } diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 6d4e906c..2f2b2055 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -41,6 +41,10 @@ public: static PyObject* pyMyThings(PyObject *self, PyObject* args); static PyObject* pyAutoThingsAppeared(PyObject *self, PyObject* args); +public: + // python callbacks + static PyObject* task_done(PyObject* self, PyObject* args); + private: void exportIds(); void exportThingClass(const ThingClass &thingClass); diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index cd1495e9..c1872187 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -21,9 +21,11 @@ def startMonitoringAutoThings(): logger.log("Creating new auto thing") descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") autoThingsAppeared([descriptor]) + logger.log("Done start monitoring auto things") async def setupThing(info): + logger.log("setupThing for", info.thing.name) info.finish(nymea.ThingErrorNoError) From 43ed28334057af930dfe528b5d74d4c2c0d30be8 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Thu, 2 Jul 2020 22:54:00 +0200 Subject: [PATCH 17/46] intermediate commit --- plugins/pymock/integrationpluginpymock.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index c1872187..d96c4d75 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -9,11 +9,16 @@ def configValueChanged(paramTypeId, value): logger.log("Plugin config value changed:", paramTypeId, value) if paramTypeId == pyMockPluginAutoThingCountParamTypeId: logger.log("Auto Thing Count plugin config changed:", value, "Currently there are:", len(autoThings()), "auto things") - for i in range(value, len(myThings())): + things = autoThings(); + for i in range(len(things), value): logger.log("Creating new auto thing") descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") autoThingsAppeared([descriptor]) + for i in range(len(value), things): + logger.log("Removing auto thing") + autoThingDisappeared(things[i].id) + def startMonitoringAutoThings(): logger.log("Start monitoring auto things. Have %i auto devices. Need %i." % (len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId))) From a90841401c4241e738584abca31d8a3db3ee63a9 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Fri, 3 Jul 2020 15:27:19 +0200 Subject: [PATCH 18/46] more work --- .../integrations/python/pythingdescriptor.h | 10 ++- .../python/pythingdiscoveryinfo.h | 2 - .../integrations/python/pythingsetupinfo.h | 2 + libnymea-core/integrations/python/pyutils.h | 78 +++++++++++++++++++ .../integrations/pythonintegrationplugin.cpp | 72 +++++++++-------- .../integrations/pythonintegrationplugin.h | 1 + libnymea/integrations/thingdiscoveryinfo.h | 6 +- libnymea/loggingcategories.cpp | 1 + plugins/pymock/integrationpluginpymock.py | 23 +++++- 9 files changed, 152 insertions(+), 43 deletions(-) diff --git a/libnymea-core/integrations/python/pythingdescriptor.h b/libnymea-core/integrations/python/pythingdescriptor.h index 777d5b3f..0a7c1baf 100644 --- a/libnymea-core/integrations/python/pythingdescriptor.h +++ b/libnymea-core/integrations/python/pythingdescriptor.h @@ -5,6 +5,7 @@ #include "structmember.h" #include "integrations/thingdescriptor.h" +#include "loggingcategories.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -31,6 +32,7 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj static char *kwlist[] = {"thingClassId", "name", "description", nullptr}; PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr; + qWarning() << "++++ PyThingDescriptor"; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, &thingClassId, &name, &description)) return -1; @@ -52,12 +54,18 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj return 0; } +static void PyThingDescriptor_dealloc(PyThingDescriptor * self) +{ + qWarning() << "---- PyThingDescriptor"; + Py_TYPE(self)->tp_free(self); +} + static PyTypeObject PyThingDescriptorType = { PyVarObject_HEAD_INIT(NULL, 0) "nymea.ThingDescriptor", /* tp_name */ sizeof(PyThingDescriptor), /* tp_basicsize */ 0, /* tp_itemsize */ - 0, /* tp_dealloc */ + (destructor)PyThingDescriptor_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index 9a13defe..f699c220 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -36,8 +36,6 @@ static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/ static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) { - // FIXME: Why is this not called? Seems we're leaking... - Q_ASSERT(false); delete self->mutex; Py_TYPE(self)->tp_free(self); } diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index ab4a589f..29f29c62 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -25,11 +25,13 @@ static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject */*args*/, Py if (self == NULL) { return nullptr; } + qWarning() << "++++ PyThingSetupInfo"; self->mutex = new QMutex(); return (PyObject*)self; } static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) { + qWarning() << "--- PyThingSetupInfo"; delete self->mutex; Py_TYPE(self)->tp_free(self); } diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index 18a220de..d2d73f4e 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -58,4 +58,82 @@ QVariant PyObjectToQVariant(PyObject *pyObject) return value; } +void PyDumpError() +{ + if (!PyErr_Occurred()) { + return; + } + +} + +// Write to stdout +PyObject* pyLog_write(PyObject* /*self*/, PyObject* args) +{ + const char *what; + if (!PyArg_ParseTuple(args, "s", &what)) + return nullptr; + if (!QByteArray(what).trimmed().isEmpty()) { + qCDebug(dcThingManager()) << what; + } + Py_RETURN_NONE; +} +PyObject* pyLog_flush(PyObject* /*self*/, PyObject* /*args*/) +{ + // Not really needed... qDebug() flushes already on its own + Py_RETURN_NONE; +} + +static PyMethodDef pyLog_methods[] = +{ + {"write", pyLog_write, METH_VARARGS, "Writes to stdout through qDebug()"}, + {"flush", pyLog_flush, METH_VARARGS, "noop"}, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyModuleDef pyLog_module = +{ + PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; + "pyLog", // const char* m_name; + "pyLog stdout override",// const char* m_doc; + -1, // Py_ssize_t m_size; + pyLog_methods, // PyMethodDef *m_methods + // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; + nullptr, nullptr, nullptr, nullptr +}; + +// Write to stderr +PyObject* pyWarn_write(PyObject* /*self*/, PyObject* args) +{ + const char *what; + if (!PyArg_ParseTuple(args, "s", &what)) + return nullptr; + if (!QByteArray(what).trimmed().isEmpty()) { + qCWarning(dcThingManager()) << what; + } + Py_RETURN_NONE; +} +PyObject* pyWarn_flush(PyObject* /*self*/, PyObject* /*args*/) +{ + // Not really needed... qDebug() flushes already on its own + Py_RETURN_NONE; +} + +static PyMethodDef pyWarn_methods[] = +{ + {"write", pyWarn_write, METH_VARARGS, "Writes to stderr through qWarnging()"}, + {"flush", pyWarn_flush, METH_VARARGS, "noop"}, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyModuleDef pyWarn_module = +{ + PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; + "pyWarn", // const char* m_name; + "pyWarn stdout override",// const char* m_doc; + -1, // Py_ssize_t m_size; + pyWarn_methods, // PyMethodDef *m_methods + // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; + nullptr, nullptr, nullptr, nullptr +}; + #endif // PYUTILS_H diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 6e019619..2a81f08c 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -24,24 +24,6 @@ PyObject* PythonIntegrationPlugin::s_asyncio = nullptr; QHash PythonIntegrationPlugin::s_plugins; -// Write to stdout/stderr -PyObject* nymea_write(PyObject* /*self*/, PyObject* args) -{ - const char *what; - if (!PyArg_ParseTuple(args, "s", &what)) - return nullptr; - if (!QByteArray(what).trimmed().isEmpty()) { - qCDebug(dcThingManager()) << what; - } - Py_RETURN_NONE; -} - -// Flush stdout/stderr -PyObject* nymea_flush(PyObject* /*self*/, PyObject* /*args*/) -{ - // Not really needed... qDebug() flushes already on its own - Py_RETURN_NONE; -} PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) { @@ -75,8 +57,6 @@ PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) static PyMethodDef nymea_methods[] = { - {"write", nymea_write, METH_VARARGS, "write to stdout through qDebug()"}, - {"flush", nymea_flush, METH_VARARGS, "flush stdout (no-op)"}, {"task_done", PythonIntegrationPlugin::task_done, METH_VARARGS, "callback to clean up after asyc coroutines"}, {nullptr, nullptr, 0, nullptr} // sentinel }; @@ -94,12 +74,15 @@ static PyModuleDef nymea_module = PyMODINIT_FUNC PyInit_nymea(void) { - PyObject* m = PyModule_Create(&nymea_module); // Overrride stdout/stderr to use qDebug instead - PySys_SetObject("stdout", m); - PySys_SetObject("stderr", m); + PyObject* pyLog = PyModule_Create(&pyLog_module); + PySys_SetObject("stdout", pyLog); + PyObject* pyWarn = PyModule_Create(&pyWarn_module); + PySys_SetObject("stderr", pyWarn); + // Register nymea types + PyObject* m = PyModule_Create(&nymea_module); registerNymeaLoggingHandler(m); registerParamType(m); registerThingType(m); @@ -244,6 +227,21 @@ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject Py_RETURN_NONE; } +PyObject *PythonIntegrationPlugin::pyAutoThingDisappeared(PyObject *self, PyObject *args) +{ + char *thingIdStr = nullptr; + + if (!PyArg_ParseTuple(args, "s", &thingIdStr)) { + qCWarning(dcThingManager) << "Error parsing parameters"; + return nullptr; + } + ThingId thingId(thingIdStr); + PythonIntegrationPlugin *plugin = s_plugins.key(self); + QMetaObject::invokeMethod(plugin, "autoThingDisappeared", Qt::QueuedConnection, Q_ARG(ThingId, thingId)); + + Py_RETURN_NONE; +} + static PyMethodDef plugin_methods[] = { {"configuration", PythonIntegrationPlugin::pyConfiguration, METH_VARARGS, "Get the plugin configuration."}, @@ -251,6 +249,7 @@ static PyMethodDef plugin_methods[] = {"setConfigValue", PythonIntegrationPlugin::pySetConfigValue, METH_VARARGS, "Set the plugin configuration value for a given config paramTypeId."}, {"myThings", PythonIntegrationPlugin::pyMyThings, METH_VARARGS, "Obtain a list of things owned by this plugin."}, {"autoThingsAppeared", PythonIntegrationPlugin::pyAutoThingsAppeared, METH_VARARGS, "Inform the system about auto setup things having appeared."}, + {"autoThingDisappeared", PythonIntegrationPlugin::pyAutoThingDisappeared, METH_VARARGS, "Inform the system about auto setup things having disappeared."}, {nullptr, nullptr, 0, nullptr} // sentinel }; @@ -311,9 +310,9 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) m_module = PyImport_ImportModule(fi.baseName().toUtf8()); if (!m_module) { - dumpError(); - PyErr_Clear(); qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath(); + PyErr_Print(); + PyErr_Clear(); PyGILState_Release(s); return false; } @@ -399,7 +398,6 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, NULL); pyInfo->info = info; pyInfo->pyThing = pyThing; - Py_INCREF(pyThing); m_things.insert(info->thing(), pyThing); @@ -417,6 +415,7 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) bool result = callPluginFunction("setupThing", reinterpret_cast(pyInfo)); if (!result) { + Py_DECREF(pyThing); // The python code did not even start, so let's finish (fail) the setup right away info->finish(Thing::ThingErrorSetupFailed); } @@ -425,7 +424,11 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) void PythonIntegrationPlugin::postSetupThing(Thing *thing) { PyThing* pyThing = m_things.value(thing); - callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); + Py_INCREF(pyThing); + bool success = callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); + if (!success) { + Py_DECREF(pyThing); + } } void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) @@ -456,8 +459,12 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) void PythonIntegrationPlugin::thingRemoved(Thing *thing) { PyThing *pyThing = m_things.value(thing); + Py_INCREF(pyThing); - callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); + bool success = callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); + if (!success) { + Py_DECREF(pyThing); + } m_mutex.lock(); m_things.remove(thing); @@ -577,12 +584,12 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje { PyGILState_STATE s = PyGILState_Ensure(); - qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << s_plugins.key(m_module)->pluginName(); + qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << pluginName(); PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); if(!pFunc || !PyCallable_Check(pFunc)) { PyErr_Clear(); Py_XDECREF(pFunc); - qCWarning(dcThingManager()) << "Python plugin does not implement" << function; + qCWarning(dcThingManager()) << "Python plugin" << pluginName() << "does not implement" << function; PyGILState_Release(s); return false; } @@ -595,8 +602,8 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje Py_XDECREF(pFunc); if (PyErr_Occurred()) { - qCWarning(dcThingManager()) << "Error calling python method:"; - dumpError(); + qCWarning(dcThingManager()) << "Error calling python method:" << function << "on plugin" << pluginName(); + PyErr_Print(); PyErr_Clear(); PyGILState_Release(s); return false; @@ -651,7 +658,6 @@ void PythonIntegrationPlugin::cleanupPyThing(PyThing *pyThing) // on the thing (e.g. PyThing_name). // We'd deadlock if we wait for the mutex forever here. So let's process events // while waiting for it... - qWarning() << "Locking cleanup"; while (!pyThing->mutex->tryLock()) { qApp->processEvents(QEventLoop::EventLoopExec); } diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 2f2b2055..ada6c114 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -40,6 +40,7 @@ public: static PyObject* pySetConfigValue(PyObject* self, PyObject* args); static PyObject* pyMyThings(PyObject *self, PyObject* args); static PyObject* pyAutoThingsAppeared(PyObject *self, PyObject* args); + static PyObject* pyAutoThingDisappeared(PyObject *self, PyObject* args); public: // python callbacks diff --git a/libnymea/integrations/thingdiscoveryinfo.h b/libnymea/integrations/thingdiscoveryinfo.h index d261881a..48d03b45 100644 --- a/libnymea/integrations/thingdiscoveryinfo.h +++ b/libnymea/integrations/thingdiscoveryinfo.h @@ -53,15 +53,15 @@ public: Thing::ThingError status() const; - void addThingDescriptor(const ThingDescriptor &thingDescriptor); - void addThingDescriptors(const ThingDescriptors &thingDescriptors); - ThingDescriptors thingDescriptors() const; QString displayMessage() const; QString translatedDisplayMessage(const QLocale &locale); public slots: + void addThingDescriptor(const ThingDescriptor &thingDescriptor); + void addThingDescriptors(const ThingDescriptors &thingDescriptors); + void finish(Thing::ThingError status, const QString &displayMessage = QString()); signals: diff --git a/libnymea/loggingcategories.cpp b/libnymea/loggingcategories.cpp index 4d5070c7..d5d4cfd0 100644 --- a/libnymea/loggingcategories.cpp +++ b/libnymea/loggingcategories.cpp @@ -75,6 +75,7 @@ NYMEA_LOGGING_CATEGORY(dcBluetoothServerTraffic, "BluetoothServerTraffic") NYMEA_LOGGING_CATEGORY(dcMqtt, "Mqtt") NYMEA_LOGGING_CATEGORY(dcTranslations, "Translations") NYMEA_LOGGING_CATEGORY(dcI2C, "I2C") +NYMEA_LOGGING_CATEGORY(dcPythonIntegrations, "PythonIntegrations") static QFile s_logFile; diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index d96c4d75..42d4f56b 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -1,13 +1,15 @@ import nymea import asyncio +watchingAutoThings = False + def init(): logger.log("Python mock plugin init") def configValueChanged(paramTypeId, value): - logger.log("Plugin config value changed:", paramTypeId, value) - if paramTypeId == pyMockPluginAutoThingCountParamTypeId: + logger.log("Plugin config value changed:", paramTypeId, value, watchingAutoThings) + if watchingAutoThings and paramTypeId == pyMockPluginAutoThingCountParamTypeId: logger.log("Auto Thing Count plugin config changed:", value, "Currently there are:", len(autoThings()), "auto things") things = autoThings(); for i in range(len(things), value): @@ -15,20 +17,33 @@ def configValueChanged(paramTypeId, value): descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") autoThingsAppeared([descriptor]) - for i in range(len(value), things): + for i in range(value, len(things)): logger.log("Removing auto thing") autoThingDisappeared(things[i].id) def startMonitoringAutoThings(): + global watchingAutoThings + watchingAutoThings = True logger.log("Start monitoring auto things. Have %i auto devices. Need %i." % (len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId))) - for i in range(len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId)): + things = autoThings(); + for i in range(len(things), configValue(pyMockPluginAutoThingCountParamTypeId)): logger.log("Creating new auto thing") descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") autoThingsAppeared([descriptor]) + for i in range(configValue(pyMockPluginAutoThingCountParamTypeId), len(things)): + logger.log("Removing auto thing") + autoThingDisappeared(things[i].id) + logger.log("Done start monitoring auto things") +async def discoverThings(info): + await asyncio.sleep(1) + descriptor = nymea.ThingDescriptor(pyMockThingClassId, "Python mock thing") + info.addDescriptor(descriptor) + info.finish(nymea.ThingErrorNoError) + async def setupThing(info): logger.log("setupThing for", info.thing.name) info.finish(nymea.ThingErrorNoError) From 0cbd1ff5ecc7103c69f3398010c39d163371cd3a Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sun, 5 Jul 2020 00:14:44 +0200 Subject: [PATCH 19/46] Clenaup and polish types. --- libnymea-core/integrations/python/pyparam.h | 37 ++-- libnymea-core/integrations/python/pything.h | 203 +++++++++++------- .../integrations/python/pythingactioninfo.h | 34 ++- .../integrations/python/pythingdescriptor.h | 26 ++- .../python/pythingdiscoveryinfo.h | 54 ++++- .../integrations/python/pythingpairinginfo.h | 188 ++++++++++++++++ .../integrations/python/pythingsetupinfo.h | 38 +++- libnymea-core/integrations/python/pyutils.h | 14 +- .../integrations/pythonintegrationplugin.cpp | 152 +++++++++---- .../integrations/pythonintegrationplugin.h | 5 +- libnymea-core/libnymea-core.pro | 1 + libnymea/types/state.cpp | 10 + libnymea/types/state.h | 1 + plugins/pymock/integrationpluginpymock.json | 27 +++ plugins/pymock/integrationpluginpymock.py | 47 +++- 15 files changed, 665 insertions(+), 172 deletions(-) create mode 100644 libnymea-core/integrations/python/pythingpairinginfo.h diff --git a/libnymea-core/integrations/python/pyparam.h b/libnymea-core/integrations/python/pyparam.h index cd84a4c2..ab2b71a1 100644 --- a/libnymea-core/integrations/python/pyparam.h +++ b/libnymea-core/integrations/python/pyparam.h @@ -21,12 +21,6 @@ typedef struct _pyparam { PyObject* pyValue = nullptr; } PyParam; -static void PyParam_dealloc(PyParam * self) { - Py_XDECREF(self->pyParamTypeId); - Py_XDECREF(self->pyValue); - Py_TYPE(self)->tp_free(self); -} - static PyMethodDef PyParam_methods[] = { {nullptr, nullptr, 0, nullptr} // sentinel }; @@ -40,6 +34,7 @@ static PyMemberDef PyParam_members[] = { static int PyParam_init(PyParam *self, PyObject *args, PyObject *kwds) { + qWarning() << "++++ PyParam"; static char *kwlist[] = {"paramTypeId", "value", nullptr}; PyObject *paramTypeId = nullptr, *value = nullptr; @@ -47,18 +42,23 @@ static int PyParam_init(PyParam *self, PyObject *args, PyObject *kwds) return -1; if (paramTypeId) { - Py_XDECREF(self->pyParamTypeId); Py_INCREF(paramTypeId); self->pyParamTypeId = paramTypeId; } if (value) { - Py_XDECREF(self->pyValue); Py_INCREF(value); self->pyValue = value; } return 0; } +static void PyParam_dealloc(PyParam * self) { + qWarning() << "---- PyParam"; + Py_XDECREF(self->pyParamTypeId); + Py_XDECREF(self->pyValue); + Py_TYPE(self)->tp_free(self); +} + static PyTypeObject PyParamType = { PyVarObject_HEAD_INIT(NULL, 0) "nymea.Param", /* tp_name */ @@ -86,9 +86,14 @@ static PyTypeObject PyParamType = { static PyParam* PyParam_fromParam(const Param ¶m) { - PyParam *pyParam = PyObject_New(PyParam, &PyParamType); - pyParam->pyParamTypeId = PyUnicode_FromString(param.paramTypeId().toString().toUtf8()); - pyParam->pyValue = QVariantToPyObject(param.value()); + PyObject *pyParamValue = QVariantToPyObject(param.value()); + PyObject *args = Py_BuildValue("(sO)", param.paramTypeId().toString().toUtf8().data(), pyParamValue); + + PyParam *pyParam = (PyParam*)PyObject_CallObject((PyObject*)&PyParamType, args); + + Py_DECREF(pyParamValue); + Py_DECREF(args); + return pyParam; } @@ -101,9 +106,10 @@ static Param PyParam_ToParam(PyParam *pyParam) static PyObject* PyParams_FromParamList(const ParamList ¶ms) { - PyObject* result = PyTuple_New(params.count()); + PyObject* result = PyTuple_New(params.length()); for (int i = 0; i < params.count(); i++) { - PyTuple_SET_ITEM(result, i, (PyObject*)PyParam_fromParam(params.at(i))); + PyParam *pyParam = PyParam_fromParam(params.at(i)); + PyTuple_SetItem(result, i, (PyObject*)pyParam); } return result; } @@ -112,7 +118,7 @@ static ParamList PyParams_ToParamList(PyObject *pyParams) { ParamList params; - if (pyParams != nullptr) { + if (pyParams == nullptr) { return params; } @@ -130,7 +136,10 @@ static ParamList PyParams_ToParamList(PyObject *pyParams) PyParam *pyParam = reinterpret_cast(next); params.append(PyParam_ToParam(pyParam)); + Py_DECREF(next); } + + Py_DECREF(iter); return params; } diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 5118a44a..3074dd61 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -18,24 +18,34 @@ #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" -/* Note: Thing is not threadsafe so we must never access thing directly in here. - * For writing access, invoking with QueuedConnections will decouple stuff - * For reading access, we keep a cache of the thing properties here and sync them - * over to the cache when they change. - * When using this, make sure to call PyThing_setThing() after constructing it. +/* Note: + * When using this, make sure to call PyThing_setThing() while holding the GIL to initialize + * stuff after constructing it. + * + * The Thing class is not threadsafe and self->thing is owned by nymeas main thread. + * So we must never directly access anything of it in here. + * + * For writing to it, invoking methods with QueuedConnections will thread-decouple stuff. + * Make sure to lock the self->mutex while using the pointer to it for invoking stuff. + * + * For reading access, we keep copies of the thing properties here and sync them + * over to the according py* members when they change. + * */ typedef struct _thing { PyObject_HEAD - Thing *thing = nullptr; - PyObject *name = nullptr; - PyObject *id = nullptr; - PyObject *thingClassId = nullptr; - PyObject *params = nullptr; - PyObject *settings = nullptr; - PyObject *nameChangedHandler = nullptr; - QMutex *mutex = nullptr; + Thing *thing = nullptr; // the actual thing in nymea (not thread-safe!) + QMutex *mutex = nullptr; // The mutex for accessing the thing pointer + PyObject *pyId = nullptr; + PyObject *pyThingClassId = nullptr; + PyObject *pyName = nullptr; + PyObject *pyParams = nullptr; + PyObject *pySettings = nullptr; + PyObject *pyNameChangedHandler = nullptr; + PyObject *pySettingChangedHandler = nullptr; + PyObject *pyStates = nullptr; // A copy of the things states } PyThing; @@ -44,7 +54,7 @@ static PyObject* PyThing_new(PyTypeObject *type, PyObject */*args*/, PyObject */ if (self == NULL) { return nullptr; } - qWarning() << "*creating pyThing" << self; + qWarning() << "*++++ PyThing" << self; self->mutex = new QMutex(); return (PyObject*)self; @@ -54,103 +64,97 @@ static void PyThing_setThing(PyThing *self, Thing *thing) { self->thing = thing; - self->name = PyUnicode_FromString(self->thing->name().toUtf8().data()); - self->id = PyUnicode_FromString(self->thing->id().toString().toUtf8().data()); - self->thingClassId = PyUnicode_FromString(self->thing->thingClassId().toString().toUtf8().data()); + self->pyId = PyUnicode_FromString(self->thing->id().toString().toUtf8().data()); + self->pyThingClassId = PyUnicode_FromString(self->thing->thingClassId().toString().toUtf8().data()); + self->pyName = PyUnicode_FromString(self->thing->name().toUtf8().data()); + self->pyParams = PyParams_FromParamList(self->thing->params()); + self->pySettings = PyParams_FromParamList(self->thing->settings()); + self->pyStates = PyList_New(thing->states().count()); + for (int i = 0; i < thing->states().count(); i++) { + qWarning() << "i" << i; + State state = thing->states().at(i); + PyObject *pyState = Py_BuildValue("{s:s, s:O}", + "stateTypeId", state.stateTypeId().toString().toUtf8().data(), + "value", QVariantToPyObject(state.value())); + PyList_SetItem(self->pyStates, i, pyState); + } + + + // Connects signal handlers from the Thing to sync stuff over to the pyThing in a + // thread-safe manner. + + // Those lambdas Will be executed in the main thread context. This means we + // can access self->thing, but need to hold the GIL for interacting with python QObject::connect(thing, &Thing::nameChanged, [=](){ - self->mutex->lock(); - Py_XDECREF(self->name); - self->name = PyUnicode_FromString(self->thing->name().toUtf8().data()); - - if (!self->nameChangedHandler) { - self->mutex->unlock(); - return; - } - self->mutex->unlock(); - PyGILState_STATE s = PyGILState_Ensure(); - PyObject_CallFunctionObjArgs(self->nameChangedHandler, self, nullptr); - + Py_XDECREF(self->pyName); + self->pyName = PyUnicode_FromString(self->thing->name().toUtf8().data()); + if (self->pyNameChangedHandler) { + PyObject_CallFunctionObjArgs(self->pyNameChangedHandler, self, nullptr); + } PyGILState_Release(s); }); - self->params = PyParams_FromParamList(self->thing->params()); - self->settings = PyParams_FromParamList(self->thing->settings()); - QObject::connect(thing, &Thing::settingChanged, [=](){ - QMutexLocker(self->mutex); - Py_XDECREF(self->settings); - self->settings = PyParams_FromParamList(self->thing->settings()); + QObject::connect(thing, &Thing::settingChanged, [=](const ParamTypeId ¶mTypeId, const QVariant &value){ + PyGILState_STATE s = PyGILState_Ensure(); + Py_XDECREF(self->pySettings); + self->pySettings = PyParams_FromParamList(self->thing->settings()); + if (self->pySettingChangedHandler) { + PyObject_CallFunctionObjArgs(self->pySettingChangedHandler, self, PyUnicode_FromString(paramTypeId.toString().toUtf8().data()), QVariantToPyObject(value), nullptr); + } + PyGILState_Release(s); }); } static void PyThing_dealloc(PyThing * self) { - qWarning() << "destryoing pything" << self; - Py_XDECREF(self->name); - Py_XDECREF(self->id); - Py_XDECREF(self->thingClassId); - Py_XDECREF(self->name); - Py_XDECREF(self->params); - Py_XDECREF(self->settings); - Py_XDECREF(self->nameChangedHandler); + qWarning() << "----- PyThing" << self; + Py_XDECREF(self->pyId); + Py_XDECREF(self->pyThingClassId); + Py_XDECREF(self->pyName); + Py_XDECREF(self->pyParams); + Py_XDECREF(self->pySettings); + Py_XDECREF(self->pyStates); + Py_XDECREF(self->pyNameChangedHandler); + Py_XDECREF(self->pySettingChangedHandler); delete self->mutex; - Py_TYPE(self)->tp_free(self); + Py_TYPE(self)->tp_free(self); } static PyObject *PyThing_getName(PyThing *self, void */*closure*/) { - QMutexLocker(self->mutex); - if (!self->thing) { - PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); - return nullptr; - } - - Py_INCREF(self->name); - return self->name; + Py_INCREF(self->pyName); + return self->pyName; } static PyObject *PyThing_getId(PyThing *self, void */*closure*/) { - QMutexLocker(self->mutex); - if (!self->thing) { - PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); - return nullptr; - } - - Py_INCREF(self->id); - return self->id; + Py_INCREF(self->pyId); + return self->pyId; } static PyObject *PyThing_getThingClassId(PyThing *self, void */*closure*/) { - QMutexLocker(self->mutex); - if (!self->thing) { - PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); - return nullptr; - } - - Py_INCREF(self->thingClassId); - return self->thingClassId; + Py_INCREF(self->pyThingClassId); + return self->pyThingClassId; } static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ QString name = QString(PyUnicode_AsUTF8(value)); QMutexLocker(self->mutex); + if (!self->thing) { + return -1; + } QMetaObject::invokeMethod(self->thing, "setName", Qt::QueuedConnection, Q_ARG(QString, name)); return 0; } static PyObject *PyThing_getSettings(PyThing *self, void */*closure*/) { - QMutexLocker(self->mutex); - if (!self->thing) { - PyErr_SetString(PyExc_ValueError, "Thing has been removed from the system."); - return nullptr; - } - Py_INCREF(self->settings); - return self->settings; + Py_INCREF(self->pySettings); + return self->pySettings; } static int PyThing_setSettings(PyThing */*self*/, PyObject */*value*/, void */*closure*/){ @@ -158,6 +162,49 @@ static int PyThing_setSettings(PyThing */*self*/, PyObject */*value*/, void */*c return 0; } +static PyObject * PyThing_stateValue(PyThing* self, PyObject* args) +{ + char *stateTypeIdStr = nullptr; + + if (!PyArg_ParseTuple(args, "s", &stateTypeIdStr)) { + qCWarning(dcThingManager) << "Error parsing parameters"; + return nullptr; + } + + StateTypeId stateTypeId = StateTypeId(stateTypeIdStr); + PyObject *iterator = PyObject_GetIter(self->pyStates); + while (iterator) { + PyObject *pyState = PyIter_Next(iterator); + if (!pyState) { + break; + } + + PyObject *pyStateTypeId = PyDict_GetItemString(pyState, "stateTypeId"); + PyObject *tmp = PyUnicode_AsEncodedString(pyStateTypeId, "UTF-8", "strict"); + + StateTypeId stid = StateTypeId(PyBytes_AS_STRING(tmp)); + + Py_DECREF(pyStateTypeId); + Py_XDECREF(tmp); + + if (stid != stateTypeId) { + Py_DECREF(pyState); + continue; + } + + PyObject *pyStateValue = PyDict_GetItemString(pyState, "value"); + + Py_DECREF(pyState); + Py_DECREF(iterator); + + return pyStateValue; + } + + Py_DECREF(iterator); + qCWarning(dcPythonIntegrations()) << "No state for stateTypeId:" << stateTypeId; + Py_RETURN_NONE; +} + static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) { char *stateTypeIdStr = nullptr; @@ -209,13 +256,15 @@ static PyGetSetDef PyThing_getset[] = { }; static PyMethodDef PyThing_methods[] = { - { "setStateValue", (PyCFunction)PyThing_setStateValue, METH_VARARGS, "Set a things state value" }, + { "stateValue", (PyCFunction)PyThing_stateValue, METH_VARARGS, "Get a things state value by stateTypeId" }, + { "setStateValue", (PyCFunction)PyThing_setStateValue, METH_VARARGS, "Set a certain things state value by stateTypeIp" }, { "emitEvent", (PyCFunction)PyThing_emitEvent, METH_VARARGS, "Emits an event" }, {nullptr, nullptr, 0, nullptr} // sentinel }; static PyMemberDef PyThing_members[] = { - {"nameChangedHandler", T_OBJECT_EX, offsetof(PyThing, nameChangedHandler), 0, "Set a callback for when the thing name changes"}, + {"params", T_OBJECT_EX, offsetof(PyThing, pyParams), READONLY, "Thing params"}, + {"nameChangedHandler", T_OBJECT_EX, offsetof(PyThing, pyNameChangedHandler), 0, "Set a callback for when the thing name changes"}, {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; diff --git a/libnymea-core/integrations/python/pythingactioninfo.h b/libnymea-core/integrations/python/pythingactioninfo.h index 243dc9f6..05a1a35e 100644 --- a/libnymea-core/integrations/python/pythingactioninfo.h +++ b/libnymea-core/integrations/python/pythingactioninfo.h @@ -12,13 +12,29 @@ #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +/* Note: + * + * When using this, make sure to call PyThingActionInfo_setInfo() while holding the GIL to initialize + * stuff after constructing it. Also set info to nullptr while holding the GIL when the info object vanishes. + * + * The ThingActionInfo class is not threadsafe and self->info is owned by nymeas main thread. + * So we must never directly access anything of it in here. + * + * For writing to it, invoking methods with QueuedConnections will thread-decouple stuff. + * Make sure to check if the info object is still valid (it might not be if nymea finished + * the setup and destroyed it but the PyThingSetupInfo is not garbage collected yet. + * + * For reading access, we keep copies of the thing properties here and sync them + * over to the according py* members when they change. + * + */ + typedef struct { PyObject_HEAD ThingActionInfo* info; PyThing *pyThing; PyObject *pyActionTypeId; PyObject *pyParams; - QMutex *mutex; } PyThingActionInfo; @@ -27,14 +43,26 @@ static PyObject* PyThingActionInfo_new(PyTypeObject *type, PyObject */*args*/, P if (self == NULL) { return nullptr; } - self->mutex = new QMutex(); + qWarning() << "+++ PyThingActionInfo"; return (PyObject*)self; } +void PyThingActionInfo_setInfo(PyThingActionInfo *self, ThingActionInfo *info, PyThing *pyThing) +{ + self->info = info; + self->pyThing = pyThing; + Py_INCREF(pyThing); + self->pyActionTypeId = PyUnicode_FromString(info->action().actionTypeId().toString().toUtf8()); + self->pyParams = PyParams_FromParamList(info->action().params()); +} + static void PyThingActionInfo_dealloc(PyThingActionInfo * self) { - delete self->mutex; + qWarning() << "---- PyThingActionInfo"; + Py_DECREF(self->pyThing); + Py_DECREF(self->pyActionTypeId); + Py_DECREF(self->pyParams); Py_TYPE(self)->tp_free(self); } diff --git a/libnymea-core/integrations/python/pythingdescriptor.h b/libnymea-core/integrations/python/pythingdescriptor.h index 0a7c1baf..6ebc0e06 100644 --- a/libnymea-core/integrations/python/pythingdescriptor.h +++ b/libnymea-core/integrations/python/pythingdescriptor.h @@ -17,6 +17,8 @@ typedef struct { PyObject* pyThingClassId; PyObject* pyName; PyObject* pyDescription; + PyObject* pyThingId; + PyObject* pyParams; } PyThingDescriptor; @@ -24,39 +26,51 @@ static PyMemberDef PyThingDescriptor_members[] = { {"thingClassId", T_OBJECT_EX, offsetof(PyThingDescriptor, pyThingClassId), 0, "Descriptor thingClassId"}, {"name", T_OBJECT_EX, offsetof(PyThingDescriptor, pyName), 0, "Descriptor name"}, {"description", T_OBJECT_EX, offsetof(PyThingDescriptor, pyDescription), 0, "Descriptor description"}, + {"thingId", T_OBJECT_EX, offsetof(PyThingDescriptor, pyDescription), 0, "The thingId, if there exists a thing for this descriptor already."}, + {"params", T_OBJECT_EX, offsetof(PyThingDescriptor, pyParams), 0, "Params for the thing described by this descriptor."}, {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"thingClassId", "name", "description", nullptr}; - PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr; + static char *kwlist[] = {"thingClassId", "name", "description", "thingId", "params", nullptr}; + PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr, *thingId = nullptr, *params = nullptr; qWarning() << "++++ PyThingDescriptor"; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, &thingClassId, &name, &description)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOOO", kwlist, &thingClassId, &name, &description, &thingId, ¶ms)) return -1; if (thingClassId) { - Py_XDECREF(self->pyThingClassId); Py_INCREF(thingClassId); self->pyThingClassId = thingClassId; } if (name) { - Py_XDECREF(self->pyName); Py_INCREF(name); self->pyName = name; } if (description) { - Py_XDECREF(self->pyDescription); Py_INCREF(description); self->pyDescription = description; } + if (thingId) { + Py_INCREF(thingId); + self->pyThingId = thingId; + } + if (params) { + Py_INCREF(params); + self->pyParams = params; + } return 0; } static void PyThingDescriptor_dealloc(PyThingDescriptor * self) { qWarning() << "---- PyThingDescriptor"; + Py_XDECREF(self->pyThingClassId); + Py_XDECREF(self->pyName); + Py_XDECREF(self->pyDescription); + Py_XDECREF(self->pyThingId); + Py_XDECREF(self->pyParams); Py_TYPE(self)->tp_free(self); } diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index f699c220..d8fc77c4 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -6,6 +6,7 @@ #include "structmember.h" #include "pythingdescriptor.h" +#include "pyparam.h" #include "integrations/thingdiscoveryinfo.h" @@ -18,10 +19,28 @@ #pragma GCC diagnostic ignored "-Wwrite-strings" +/* Note: + * When using this, make sure to call PyThingDiscoveryInfo_setInfo() while holding the GIL to initialize + * stuff after constructing it. Also set info to nullptr while holding the GIL when the info object vanishes. + * + * The ThingDiscoveryInfo class is not threadsafe and self->info is owned by nymeas main thread. + * So we must never directly access anything of it in here. + * + * For writing to it, invoking methods with QueuedConnections will thread-decouple stuff. + * Make sure to check if the info object is still valid (it might not be if nymea finished + * the discovery and destroyed it but the PyThingDiscoveryInfo is not garbage collected yet. + * + * For reading access, we keep copies of the thing properties here and sync them + * over to the according py* members when they change. + * + */ + + typedef struct { PyObject_HEAD ThingDiscoveryInfo* info; - QMutex *mutex; +// PyObject* pyThingClassId = nullptr; +// PyObject *pyParams = nullptr; } PyThingDiscoveryInfo; static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/, PyObject */*kwds*/) @@ -30,13 +49,22 @@ static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/ if (self == NULL) { return nullptr; } - self->mutex = new QMutex(); + qWarning() << "++++ PyThingDiscoveryInfo"; return (PyObject*)self; } +void PyThingDiscoveryInfo_setInfo(PyThingDiscoveryInfo *self, ThingDiscoveryInfo *info) +{ + self->info = info; +// self->pyThingClassId = PyUnicode_FromString(info->thingClassId().toString().toUtf8().data()); +// self->pyParams = PyParams_FromParamList(info->params()); +} + static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) { - delete self->mutex; + qWarning() << "---- PyThingDiscoveryInfo"; +// Py_DECREF(self->pyThingClassId); +// Py_DECREF(self->pyParams); Py_TYPE(self)->tp_free(self); } @@ -64,11 +92,11 @@ static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, PyObject *pyObj = nullptr; if (!PyArg_ParseTuple(args, "O", &pyObj)) { - PyErr_SetString(PyExc_ValueError, "Invalid argument. Not a ThingDescriptor."); + PyErr_SetString(PyExc_ValueError, "Invalid argument to ThingDiscoveryInfo.addDescriptor(). Not a ThingDescriptor."); return nullptr; } if (pyObj->ob_type != &PyThingDescriptorType) { - PyErr_SetString(PyExc_ValueError, "Invalid argument. Not a ThingDescriptor."); + PyErr_SetString(PyExc_ValueError, "Invalid argument to ThingDiscoveryInfo.addDescriptor(). Not a ThingDescriptor."); return nullptr; } PyThingDescriptor *pyDescriptor = (PyThingDescriptor*)pyObj; @@ -87,14 +115,28 @@ static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, } ThingDescriptor descriptor(thingClassId, name, description); + if (pyDescriptor->pyThingId) { + descriptor.setThingId(ThingId(QString::fromUtf8(PyUnicode_AsUTF8(pyDescriptor->pyThingId)))); + } + + if (pyDescriptor->pyParams) { + descriptor.setParams(PyParams_ToParamList(pyDescriptor->pyParams)); + } if (self->info) { QMetaObject::invokeMethod(self->info, "addThingDescriptor", Qt::QueuedConnection, Q_ARG(ThingDescriptor, descriptor)); } - return Py_BuildValue(""); + Py_DECREF(pyDescriptor); + Py_RETURN_NONE; } +//static PyMemberDef PyThingDiscoveryInfo_members[] = { +// {"thingClassId", T_OBJECT_EX, offsetof(PyThingDiscoveryInfo, pyThingClassId), READONLY, "The ThingClassId this discovery is for."}, +//// {"params", T_OBJECT_EX, offsetof(PyThingDiscoveryInfo, pyParams), READONLY, "The params for this discovery"}, +// {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +//}; + static PyMethodDef PyThingDiscoveryInfo_methods[] = { { "addDescriptor", (PyCFunction)PyThingDiscoveryInfo_addDescriptor, METH_VARARGS, "Add a new descriptor to the discovery" }, { "finish", (PyCFunction)PyThingDiscoveryInfo_finish, METH_VARARGS, "Finish a discovery" }, diff --git a/libnymea-core/integrations/python/pythingpairinginfo.h b/libnymea-core/integrations/python/pythingpairinginfo.h new file mode 100644 index 00000000..8dce6565 --- /dev/null +++ b/libnymea-core/integrations/python/pythingpairinginfo.h @@ -0,0 +1,188 @@ +#ifndef PYTHINGPAIRINGINFO_H +#define PYTHINGPAIRINGINFO_H + + +#include +#include "structmember.h" + +#include "pyparam.h" + +#include "integrations/thingpairinginfo.h" + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +/* Note: + * When using this, make sure to call PyThingPairingInfo_setInfo() while holding the GIL to initialize + * stuff after constructing it. Also set info to nullptr while holding the GIL when the info object vanishes. + * + * The ThingPairingInfo class is not threadsafe and self->info is owned by nymeas main thread. + * So we must never directly access anything of it in here. + * + * For writing to it, invoking methods with QueuedConnections will thread-decouple stuff. + * Make sure to check if the info object is still valid (it might not be if nymea finished + * the pairing step and destroyed it but the PyThingPairingInfo is not garbage collected yet. + * + * For reading access, we keep copies of the thing properties here and sync them + * over to the according py* members when they change. + * + */ + + + +typedef struct { + PyObject_HEAD + ThingPairingInfo* info; + PyObject *pyTransactionId = nullptr; + PyObject *pyThingClassId = nullptr; + PyObject *pyThingId = nullptr; + PyObject *pyThingName = nullptr; + PyObject *pyParentId = nullptr; + PyObject *pyParams = nullptr; + PyObject *pyOAuthUrl = nullptr; +} PyThingPairingInfo; + +static PyMemberDef PyThingPairingInfo_members[] = { + {"transactionId", T_OBJECT_EX, offsetof(PyThingPairingInfo, pyTransactionId), READONLY, "The transaction id for this pairing procedure."}, + {"thingClassId", T_OBJECT_EX, offsetof(PyThingPairingInfo, pyThingClassId), READONLY, "The ThingClassId for the thing to be set up."}, + {"thingId", T_OBJECT_EX, offsetof(PyThingPairingInfo, pyThingId), READONLY, "The ThingId for the thing to be set up."}, + {"thingName", T_OBJECT_EX, offsetof(PyThingPairingInfo, pyThingName), READONLY, "The ThingId for the thing to be set up."}, + {"parentId", T_OBJECT_EX, offsetof(PyThingPairingInfo, pyParentId), READONLY, "The ThingId for the parent of the thing to be set up."}, + {"params", T_OBJECT_EX, offsetof(PyThingPairingInfo, pyParams), READONLY, "The params for the thing to be set up."}, + {"oAuthUrl", T_OBJECT_EX, offsetof(PyThingPairingInfo, pyOAuthUrl), 0, "An OAuth url if required for the pairing."}, + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +}; + +static int PyThingPairingInfo_init(PyThingPairingInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) +{ + qWarning() << "++++ ThingPairingInfo"; + return 0; +} + +void PyThingPairingInfo_setInfo(PyThingPairingInfo *self, ThingPairingInfo *info) +{ + self->info = info; + self->pyTransactionId = PyUnicode_FromString(info->transactionId().toString().toUtf8()); + self->pyThingClassId = PyUnicode_FromString(info->thingClassId().toString().toUtf8()); + self->pyThingId = PyUnicode_FromString(info->thingId().toString().toUtf8()); + self->pyThingName = PyUnicode_FromString(info->thingName().toUtf8()); + self->pyParentId = PyUnicode_FromString(info->parentId().toString().toUtf8()); + self->pyParams = PyParams_FromParamList(info->params()); +} + +static void PyThingPairingInfo_dealloc(PyThingPairingInfo * self) +{ + qWarning() << "---- ThingPairingInfo"; + Py_XDECREF(self->pyTransactionId); + Py_XDECREF(self->pyThingClassId); + Py_XDECREF(self->pyThingId); + Py_XDECREF(self->pyThingName); + Py_XDECREF(self->pyParentId); + Py_XDECREF(self->pyParams); + Py_XDECREF(self->pyOAuthUrl); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * PyThingPairingInfo_finish(PyThingPairingInfo* self, PyObject* args) +{ + int status; + char *message = nullptr; + + if (!PyArg_ParseTuple(args, "i|s", &status, &message)) { + PyErr_SetString(PyExc_TypeError, "Invalid arguments in finish call. Expected: finish(ThingError, message = \"\""); + return nullptr; + } + + Thing::ThingError thingError = static_cast(status); + QString displayMessage = message != nullptr ? QString(message) : QString(); + + if (self->info) { + if (self->pyOAuthUrl) { + QString oAuthUrl = QString::fromUtf8(PyUnicode_AsUTF8AndSize(self->pyOAuthUrl, nullptr)); + QMetaObject::invokeMethod(self->info, "setOAuthUrl", Qt::QueuedConnection, Q_ARG(QString, oAuthUrl)); + } + QMetaObject::invokeMethod(self->info, "finish", Qt::QueuedConnection, Q_ARG(Thing::ThingError, thingError), Q_ARG(QString, displayMessage)); + } + Py_RETURN_NONE; +} + +static PyMethodDef PyThingPairingInfo_methods[] = { + { "finish", (PyCFunction)PyThingPairingInfo_finish, METH_VARARGS, "Finish a discovery" }, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyTypeObject PyThingPairingInfoType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.ThingPairingInfo", /* tp_name */ + sizeof(PyThingPairingInfo), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyThingPairingInfo_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "ThingPairingInfo", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyThingPairingInfo_methods, /* tp_methods */ + PyThingPairingInfo_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PyThingPairingInfo_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + 0, /* tp_finalize */ + 0, /* tp_vectorcall */ + 0, /* tp_print DEPRECATED*/ +}; + + + +static void registerThingPairingInfoType(PyObject *module) +{ + + if (PyType_Ready(&PyThingPairingInfoType) < 0) { + return; + } + PyModule_AddObject(module, "ThingPairingInfo", (PyObject *)&PyThingPairingInfoType); +} + + + + +#pragma GCC diagnostic pop + +#endif // PYTHINGPAIRINGINFO_H diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index 29f29c62..19010c09 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -12,27 +12,56 @@ #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +/* Note: + * + * When using this, make sure to call PyThingSetupInfo_setInfo() while holding the GIL to initialize + * stuff after constructing it. Also set info to nullptr while holding the GIL when the info object vanishes. + * + * The ThingSetupInfo class is not threadsafe and self->info is owned by nymeas main thread. + * So we must never directly access anything of it in here. + * + * For writing to it, invoking methods with QueuedConnections will thread-decouple stuff. + * Make sure to check if the info object is still valid (it might not be if nymea finished + * the setup and destroyed it but the PyThingSetupInfo is not garbage collected yet. + * + * For reading access, we keep copies of the thing properties here and sync them + * over to the according py* members when they change. + * + */ + typedef struct { PyObject_HEAD ThingSetupInfo* info; PyThing *pyThing; - QMutex *mutex; } PyThingSetupInfo; -static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject */*args*/, PyObject */*kwds*/) { +static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyThingSetupInfo *self = (PyThingSetupInfo*)type->tp_alloc(type, 0); if (self == NULL) { return nullptr; } qWarning() << "++++ PyThingSetupInfo"; - self->mutex = new QMutex(); + + + static char *kwlist[] = {"thing", nullptr}; + PyObject *pyThing = nullptr; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &pyThing)) { + PyErr_Print(); + PyErr_SetString(PyExc_ValueError, "Invalid arguments."); + return nullptr; + } + + self->pyThing = (PyThing*)pyThing; + Py_INCREF(self->pyThing); + return (PyObject*)self; } static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) { qWarning() << "--- PyThingSetupInfo"; - delete self->mutex; + Py_DECREF(self->pyThing); Py_TYPE(self)->tp_free(self); } @@ -48,7 +77,6 @@ static PyObject * PyThingSetupInfo_finish(PyThingSetupInfo* self, PyObject* args Thing::ThingError thingError = static_cast(status); QString displayMessage = message != nullptr ? QString(message) : QString(); - QMutexLocker(self->mutex); if (self->info) { QMetaObject::invokeMethod(self->info, "finish", Qt::QueuedConnection, Q_ARG(Thing::ThingError, thingError), Q_ARG(QString, displayMessage)); } diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index d2d73f4e..a119b5e6 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -7,7 +7,7 @@ #include -/* Returns a PyObject. RefCount will be 1 */ +/* Returns a new reference to PyObject*. */ PyObject *QVariantToPyObject(const QVariant &value) { PyObject *pyValue = nullptr; @@ -20,7 +20,7 @@ PyObject *QVariantToPyObject(const QVariant &value) case QVariant::UInt: case QVariant::LongLong: case QVariant::ULongLong: - pyValue = PyLong_FromLong(value.toLongLong()); + pyValue = PyLong_FromLongLong(value.toLongLong()); break; case QVariant::String: case QVariant::ByteArray: @@ -34,7 +34,7 @@ PyObject *QVariantToPyObject(const QVariant &value) Py_INCREF(pyValue); break; default: - qCWarning(dcThingManager) << "Unhandled data type in conversion from Param to PyParam!"; + qCWarning(dcPythonIntegrations()) << "Unhandled data type in conversion from Param to PyParam!"; pyValue = Py_None; Py_INCREF(pyValue); break; @@ -46,6 +46,7 @@ PyObject *QVariantToPyObject(const QVariant &value) QVariant PyObjectToQVariant(PyObject *pyObject) { // FIXME: is there any better way to do this? + qWarning() << "Error:" << PyErr_CheckSignals(); PyObject* repr = PyObject_Repr(pyObject); PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); const char *bytes = PyBytes_AS_STRING(str); @@ -58,13 +59,6 @@ QVariant PyObjectToQVariant(PyObject *pyObject) return value; } -void PyDumpError() -{ - if (!PyErr_Occurred()) { - return; - } - -} // Write to stdout PyObject* pyLog_write(PyObject* /*self*/, PyObject* args) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 2a81f08c..86f960ff 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -6,6 +6,7 @@ #include "python/pythingsetupinfo.h" #include "python/pyparam.h" #include "python/pythingactioninfo.h" +#include "python/pythingpairinginfo.h" #include "pythonintegrationplugin.h" @@ -46,6 +47,8 @@ PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) Py_XDECREF(repr); Py_XDECREF(str); +// PyObject *traceback = PyObject_CallMethodObjArgs(exception, "__traceback__", nullptr); + qCWarning(dcThingManager()) << "Exception in plugin:" << bytes; PyErr_Clear(); @@ -88,6 +91,7 @@ PyMODINIT_FUNC PyInit_nymea(void) registerThingType(m); registerThingDescriptorType(m); registerThingDiscoveryInfoType(m); + registerThingPairingInfoType(m); registerThingSetupInfoType(m); registerThingActionInfoType(m); @@ -176,15 +180,16 @@ PyObject *PythonIntegrationPlugin::pyMyThings(PyObject *self, PyObject */*args*/ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject *args) { - PyObject *pyParams; + PyObject *pyDescriptors; - if (!PyArg_ParseTuple(args, "O", &pyParams)) { + if (!PyArg_ParseTuple(args, "O", &pyDescriptors)) { qCWarning(dcThingManager()) << "Error parsing args. Not a param list"; return nullptr; } - PyObject *iter = PyObject_GetIter(pyParams); + PyObject *iter = PyObject_GetIter(pyDescriptors); if (!iter) { + Py_DECREF(pyDescriptors); qCWarning(dcThingManager()) << "Error parsing args. Not a param list"; return nullptr; } @@ -200,7 +205,7 @@ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject if (next->ob_type != &PyThingDescriptorType) { PyErr_SetString(PyExc_ValueError, "Invalid argument. Not a ThingDescriptor."); - return nullptr; + continue; } PyThingDescriptor *pyDescriptor = (PyThingDescriptor*)next; @@ -218,12 +223,21 @@ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject } ThingDescriptor descriptor(thingClassId, name, description); + + if (pyDescriptor->pyParams) { + descriptor.setParams(PyParams_ToParamList(pyDescriptor->pyParams)); + } + descriptors.append(descriptor); + Py_DECREF(next); } PythonIntegrationPlugin *plugin = s_plugins.key(self); QMetaObject::invokeMethod(plugin, "autoThingsAppeared", Qt::QueuedConnection, Q_ARG(ThingDescriptors, descriptors)); + Py_DECREF(iter); + Py_DECREF(pyDescriptors); + Py_RETURN_NONE; } @@ -306,7 +320,9 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) // Finally, import the plugin PyObject* sysPath = PySys_GetObject("path"); - PyList_Append(sysPath, PyUnicode_FromString(fi.absolutePath().toUtf8())); + PyObject* importPath = PyUnicode_FromString(fi.absolutePath().toUtf8()); + PyList_Append(sysPath, importPath); + Py_DECREF(importPath); m_module = PyImport_ImportModule(fi.baseName().toUtf8()); if (!m_module) { @@ -349,6 +365,8 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) PyObject *pyParamTypeId = PyUnicode_FromString(paramTypeId.toString().toUtf8()); PyObject *pyValue = QVariantToPyObject(value); callPluginFunction("configValueChanged", pyParamTypeId, pyValue); + Py_DECREF(pyParamTypeId); + Py_DECREF(pyValue); }); return true; @@ -373,49 +391,110 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) PyGILState_STATE s = PyGILState_Ensure(); PyThingDiscoveryInfo *pyInfo = (PyThingDiscoveryInfo*)PyObject_CallObject((PyObject*)&PyThingDiscoveryInfoType, NULL); - pyInfo->info = info; + PyThingDiscoveryInfo_setInfo(pyInfo, info); + + PyGILState_Release(s); + + connect(info, &ThingDiscoveryInfo::destroyed, this, [=](){ + auto s = PyGILState_Ensure(); + pyInfo->info = nullptr; + Py_DECREF(pyInfo); + PyGILState_Release(s); + }); + + callPluginFunction("discoverThings", reinterpret_cast(pyInfo)); +} + +void PythonIntegrationPlugin::startPairing(ThingPairingInfo *info) +{ + PyGILState_STATE s = PyGILState_Ensure(); + + PyThingPairingInfo *pyInfo = (PyThingPairingInfo*)PyObject_CallObject((PyObject*)&PyThingPairingInfoType, nullptr); + PyThingPairingInfo_setInfo(pyInfo, info); PyGILState_Release(s); - connect(info, &ThingDiscoveryInfo::destroyed, this, [=](){ - QMutexLocker(pyInfo->mutex); + connect(info, &ThingPairingInfo::destroyed, this, [=](){ + auto s = PyGILState_Ensure(); pyInfo->info = nullptr; Py_DECREF(pyInfo); + PyGILState_Release(s); }); - callPluginFunction("discoverThings", reinterpret_cast(pyInfo)); + bool result = callPluginFunction("startPairing", reinterpret_cast(pyInfo)); + if (!result) { + info->finish(Thing::ThingErrorHardwareFailure, "Plugin error: " + pluginName()); + } +} + +void PythonIntegrationPlugin::confirmPairing(ThingPairingInfo *info, const QString &username, const QString &secret) +{ + PyGILState_STATE s = PyGILState_Ensure(); + + PyThingPairingInfo *pyInfo = (PyThingPairingInfo*)PyObject_CallObject((PyObject*)&PyThingPairingInfoType, nullptr); + PyThingPairingInfo_setInfo(pyInfo, info); + + PyGILState_Release(s); + + + connect(info, &ThingPairingInfo::destroyed, this, [=](){ + auto s = PyGILState_Ensure(); + pyInfo->info = nullptr; + Py_DECREF(pyInfo); + PyGILState_Release(s); + }); + + PyObject *pyUsername = PyUnicode_FromString(username.toUtf8().data()); + PyObject *pySecret = PyUnicode_FromString(secret.toUtf8().data()); + bool result = callPluginFunction("confirmPairing", reinterpret_cast(pyInfo), pyUsername, pySecret); + if (!result) { + info->finish(Thing::ThingErrorHardwareFailure, "Plugin error: " + pluginName()); + } + + Py_DECREF(pyUsername); + Py_DECREF(pySecret); } void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) { PyGILState_STATE s = PyGILState_Ensure(); - PyThing *pyThing = (PyThing*)PyObject_CallObject((PyObject*)&PyThingType, NULL); - dumpError(); - PyThing_setThing(pyThing, info->thing()); + PyThing *pyThing = nullptr; + if (m_things.contains(info->thing())) { + pyThing = m_things.value(info->thing()); + } else { + pyThing = (PyThing*)PyObject_CallObject((PyObject*)&PyThingType, NULL); + PyThing_setThing(pyThing, info->thing()); + m_things.insert(info->thing(), pyThing); + } + + PyObject *args = PyTuple_New(1); + PyTuple_SetItem(args, 0, (PyObject*)pyThing); + + PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, args); + Py_DECREF(args); - PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, NULL); pyInfo->info = info; - pyInfo->pyThing = pyThing; - - m_things.insert(info->thing(), pyThing); PyGILState_Release(s); connect(info->thing(), &Thing::destroyed, this, [=](){ - cleanupPyThing(pyThing); + auto s = PyGILState_Ensure(); + pyThing->thing = nullptr; + Py_DECREF(pyThing); + PyGILState_Release(s); }); connect(info, &ThingSetupInfo::destroyed, this, [=](){ - QMutexLocker(pyInfo->mutex); + auto s = PyGILState_Ensure(); pyInfo->info = nullptr; Py_DECREF(pyInfo); + PyGILState_Release(s); }); bool result = callPluginFunction("setupThing", reinterpret_cast(pyInfo)); if (!result) { - Py_DECREF(pyThing); // The python code did not even start, so let's finish (fail) the setup right away info->finish(Thing::ThingErrorSetupFailed); } @@ -425,10 +504,12 @@ void PythonIntegrationPlugin::postSetupThing(Thing *thing) { PyThing* pyThing = m_things.value(thing); Py_INCREF(pyThing); + bool success = callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); if (!success) { Py_DECREF(pyThing); } + } void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) @@ -438,19 +519,15 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) PyGILState_STATE s = PyGILState_Ensure(); PyThingActionInfo *pyInfo = (PyThingActionInfo*)PyObject_CallObject((PyObject*)&PyThingActionInfoType, NULL); - pyInfo->info = info; - pyInfo->pyThing = pyThing; - pyInfo->pyActionTypeId = PyUnicode_FromString(info->action().actionTypeId().toString().toUtf8()); - pyInfo->pyParams = PyParams_FromParamList(info->action().params()); + PyThingActionInfo_setInfo(pyInfo, info, pyThing); PyGILState_Release(s); connect(info, &ThingActionInfo::destroyed, this, [=](){ - QMutexLocker(pyInfo->mutex); - pyInfo->pyActionTypeId = nullptr; - Py_XDECREF(pyInfo->pyActionTypeId); + auto s = PyGILState_Ensure(); pyInfo->info = nullptr; Py_DECREF(pyInfo); + PyGILState_Release(s); }); callPluginFunction("executeAction", reinterpret_cast(pyInfo)); @@ -580,7 +657,7 @@ void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &ac } -bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param1, PyObject *param2) +bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param1, PyObject *param2, PyObject *param3) { PyGILState_STATE s = PyGILState_Ensure(); @@ -595,9 +672,7 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje } - dumpError(); - - PyObject *result = PyObject_CallFunctionObjArgs(pFunc, param1, param2, nullptr); + PyObject *result = PyObject_CallFunctionObjArgs(pFunc, param1, param2, param3, nullptr); Py_XDECREF(pFunc); @@ -615,7 +690,7 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje return true; } - // Spawn a event loop for python + // Spawn a event loop for the thread PyObject *new_event_loop = PyObject_GetAttrString(s_asyncio, "new_event_loop"); PyObject *loop = PyObject_CallFunctionObjArgs(new_event_loop, nullptr); @@ -652,18 +727,3 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje return true; } -void PythonIntegrationPlugin::cleanupPyThing(PyThing *pyThing) -{ - // It could happen that the python thread is currently holding the mutex - // on the thing (e.g. PyThing_name). - // We'd deadlock if we wait for the mutex forever here. So let's process events - // while waiting for it... - while (!pyThing->mutex->tryLock()) { - qApp->processEvents(QEventLoop::EventLoopExec); - } - - pyThing->thing = nullptr; - pyThing->mutex->unlock(); - Py_DECREF(pyThing); -} - diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index ada6c114..fbe2093e 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -28,6 +28,8 @@ public: void init() override; void startMonitoringAutoThings() override; void discoverThings(ThingDiscoveryInfo *info) override; + void startPairing(ThingPairingInfo *info) override; + void confirmPairing(ThingPairingInfo *info, const QString &username, const QString &secret) override; void setupThing(ThingSetupInfo *info) override; void postSetupThing(Thing *thing) override; void executeAction(ThingActionInfo *info) override; @@ -56,8 +58,7 @@ private: void exportBrowserItemActionTypes(const ActionTypes &actionTypes, const QString &thingClassName); - bool callPluginFunction(const QString &function, PyObject *param1 = nullptr, PyObject *param2 = nullptr); - void cleanupPyThing(PyThing *pyThing); + bool callPluginFunction(const QString &function, PyObject *param1 = nullptr, PyObject *param2 = nullptr, PyObject *param3 = nullptr); private: static PyThreadState* s_mainThread; diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index 111f18d6..a1afcd38 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -26,6 +26,7 @@ HEADERS += nymeacore.h \ integrations/python/pythingactioninfo.h \ integrations/python/pythingdescriptor.h \ integrations/python/pythingdiscoveryinfo.h \ + integrations/python/pythingpairinginfo.h \ integrations/python/pythingsetupinfo.h \ integrations/python/pyutils.h \ integrations/thingmanagerimplementation.h \ diff --git a/libnymea/types/state.cpp b/libnymea/types/state.cpp index 8366511f..8df85f32 100644 --- a/libnymea/types/state.cpp +++ b/libnymea/types/state.cpp @@ -117,3 +117,13 @@ void States::put(const QVariant &variant) { append(variant.value()); } + +QVariant States::stateValue(const StateTypeId &stateTypeId) +{ + foreach (const State & state, *this) { + if (state.stateTypeId() == stateTypeId) { + return state.value(); + } + } + return QVariant(); +} diff --git a/libnymea/types/state.h b/libnymea/types/state.h index 165c2fb9..f6b43659 100644 --- a/libnymea/types/state.h +++ b/libnymea/types/state.h @@ -69,6 +69,7 @@ public: States(const QList &other); Q_INVOKABLE QVariant get(int index) const; Q_INVOKABLE void put(const QVariant &variant); + Q_INVOKABLE QVariant stateValue(const StateTypeId &stateTypeId); }; Q_DECLARE_METATYPE(States) diff --git a/plugins/pymock/integrationpluginpymock.json b/plugins/pymock/integrationpluginpymock.json index 2f2ba295..4797d60f 100644 --- a/plugins/pymock/integrationpluginpymock.json +++ b/plugins/pymock/integrationpluginpymock.json @@ -23,6 +23,15 @@ "displayName": "Auto Python mock thing", "createMethods": ["auto"], "setupMethod": "justAdd", + "paramTypes": [ + { + "id": "8733557e-c599-4169-bcfa-5cc033c17b85", + "name": "param1", + "displayName": "Param 1", + "type": "bool", + "defaultValue": false + } + ], "stateTypes": [ { "id": "12c82472-56b0-4229-b324-4aaa6850320e", @@ -58,6 +67,24 @@ "displayName": "Python mock thing with discovery and pairing", "createMethods": ["discovery"], "setupMethod": "userAndPassword", + "discoveryParamTypes": [ + { + "id": "ef5f6b90-e9d8-4e77-a14d-6725cfb07116", + "name": "resultCount", + "displayName": "Result count", + "type": "int", + "defaultValue": 2 + } + ], + "paramTypes": [ + { + "id": "69328949-13ad-4bf4-b664-e7ee409ee51d", + "name": "param1", + "displayName": "Param 1", + "type": "int", + "defaultValue": 0 + } + ], "stateTypes": [ { "id": "99d0af17-9e8c-42bb-bece-a5d114f051d3", diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 42d4f56b..14b1009c 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -15,6 +15,7 @@ def configValueChanged(paramTypeId, value): for i in range(len(things), value): logger.log("Creating new auto thing") descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") + descriptor.params = [nymea.Param(pyMockAutoThingParam1ParamTypeId, True)] autoThingsAppeared([descriptor]) for i in range(value, len(things)): @@ -30,6 +31,7 @@ def startMonitoringAutoThings(): for i in range(len(things), configValue(pyMockPluginAutoThingCountParamTypeId)): logger.log("Creating new auto thing") descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") + descriptor.params = [nymea.Param(pyMockAutoThingParam1ParamTypeId, True)] autoThingsAppeared([descriptor]) for i in range(configValue(pyMockPluginAutoThingCountParamTypeId), len(things)): logger.log("Removing auto thing") @@ -39,16 +41,55 @@ def startMonitoringAutoThings(): async def discoverThings(info): - await asyncio.sleep(1) - descriptor = nymea.ThingDescriptor(pyMockThingClassId, "Python mock thing") - info.addDescriptor(descriptor) + logger.log("Discovery started for") #, info.thingClassId, "with result count:") #, info.params[0].value) + logger.log("a") + await asyncio.sleep(1) # Some delay for giving a feeling of a discovery + # Add 2 new discovery results + logger.log("b") + info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, "Python mock thing 1")) + info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, "Python mock thing 2")) + logger.log("c") + # Also add existing ones again so reconfiguration is possible + for thing in myThings(): + logger.log("d") + if thing.thingClassId == pyMockDiscoveryPairingThingClassId: + logger.log("e") + info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, thing.name, thingId=thing.id)) + logger.log("f") + + logger.log("g") info.finish(nymea.ThingErrorNoError) + logger.log("h") + + +async def startPairing(info): + logger.log("startPairing for", info.thingName, info.thingId, info.params) + info.finish(nymea.ThingErrorNoError, "Log in as user \"john\" with password \"smith\".") + + +async def confirmPairing(info, username, secret): + logger.log("confirming pairing for", info.thingName, username, secret) + await asyncio.sleep(1) + if username == "john" and secret == "smith": + info.finish(nymea.ThingErrorNoError) + else: + info.finish(nymea.ThingErrorAuthenticationFailure, "Error logging in here!") + async def setupThing(info): +# logger.log("setupThing for", info.thing.name, info.thing.params) logger.log("setupThing for", info.thing.name) info.finish(nymea.ThingErrorNoError) +def postSetupThing(thing): + logger.log("postSetupThing for", thing.name, thing.params[0].value) + thing.nameChangedHandler = lambda thing : logger.log("Thing name changed", thing.name) + + if thing.thingClassId == pyMockAutoThingClassId: + logger.log("State 1 value:", thing.stateValue(pyMockAutoState1StateTypeId)) + + def autoThings(): autoThings = [] for thing in myThings(): From f132c6b006395ef6cb77e35969b520d7aaf50b6f Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sun, 5 Jul 2020 12:28:25 +0200 Subject: [PATCH 20/46] some more work --- .../python/pythingdiscoveryinfo.h | 25 ++++++++-------- .../integrations/pythonintegrationplugin.cpp | 29 +++++++++++++++---- plugins/pymock/integrationpluginpymock.json | 9 ++++++ plugins/pymock/integrationpluginpymock.py | 17 ++++------- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index d8fc77c4..9b8e07b7 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -39,8 +39,8 @@ typedef struct { PyObject_HEAD ThingDiscoveryInfo* info; -// PyObject* pyThingClassId = nullptr; -// PyObject *pyParams = nullptr; + PyObject* pyThingClassId = nullptr; + PyObject *pyParams = nullptr; } PyThingDiscoveryInfo; static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/, PyObject */*kwds*/) @@ -56,15 +56,15 @@ static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/ void PyThingDiscoveryInfo_setInfo(PyThingDiscoveryInfo *self, ThingDiscoveryInfo *info) { self->info = info; -// self->pyThingClassId = PyUnicode_FromString(info->thingClassId().toString().toUtf8().data()); -// self->pyParams = PyParams_FromParamList(info->params()); + self->pyThingClassId = PyUnicode_FromString(info->thingClassId().toString().toUtf8().data()); + self->pyParams = PyParams_FromParamList(info->params()); } static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) { qWarning() << "---- PyThingDiscoveryInfo"; -// Py_DECREF(self->pyThingClassId); -// Py_DECREF(self->pyParams); + Py_DECREF(self->pyThingClassId); + Py_DECREF(self->pyParams); Py_TYPE(self)->tp_free(self); } @@ -127,15 +127,14 @@ static PyObject * PyThingDiscoveryInfo_addDescriptor(PyThingDiscoveryInfo* self, QMetaObject::invokeMethod(self->info, "addThingDescriptor", Qt::QueuedConnection, Q_ARG(ThingDescriptor, descriptor)); } - Py_DECREF(pyDescriptor); Py_RETURN_NONE; } -//static PyMemberDef PyThingDiscoveryInfo_members[] = { -// {"thingClassId", T_OBJECT_EX, offsetof(PyThingDiscoveryInfo, pyThingClassId), READONLY, "The ThingClassId this discovery is for."}, -//// {"params", T_OBJECT_EX, offsetof(PyThingDiscoveryInfo, pyParams), READONLY, "The params for this discovery"}, -// {nullptr, 0, 0, 0, nullptr} /* Sentinel */ -//}; +static PyMemberDef PyThingDiscoveryInfo_members[] = { + {"thingClassId", T_OBJECT_EX, offsetof(PyThingDiscoveryInfo, pyThingClassId), READONLY, "The ThingClassId this discovery is for."}, + {"params", T_OBJECT_EX, offsetof(PyThingDiscoveryInfo, pyParams), READONLY, "The params for this discovery"}, + {nullptr, 0, 0, 0, nullptr} /* Sentinel */ +}; static PyMethodDef PyThingDiscoveryInfo_methods[] = { { "addDescriptor", (PyCFunction)PyThingDiscoveryInfo_addDescriptor, METH_VARARGS, "Add a new descriptor to the discovery" }, @@ -172,7 +171,7 @@ static PyTypeObject PyThingDiscoveryInfoType = { 0, /* tp_iter */ 0, /* tp_iternext */ PyThingDiscoveryInfo_methods, /* tp_methods */ - 0, //PyThingDiscoveryInfo_members, /* tp_members */ + PyThingDiscoveryInfo_members, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 86f960ff..8fe0aafb 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -316,6 +316,10 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) return false; } +// PyThreadState *m_thread = Py_NewInterpreter(); +// PyInterpreterState *m_interpreter = PyInterpreterState_New(); + +// PyEval_RestoreThread(s_mainThread); PyGILState_STATE s = PyGILState_Ensure(); // Finally, import the plugin @@ -329,6 +333,7 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath(); PyErr_Print(); PyErr_Clear(); +// PyEval_SaveThread(); PyGILState_Release(s); return false; } @@ -352,6 +357,7 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) // Register config access methods PyModule_AddFunctions(m_module, plugin_methods); +// PyEval_SaveThread(); PyGILState_Release(s); // Set up connections to be forwareded into the plugin @@ -660,6 +666,7 @@ void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &ac bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param1, PyObject *param2, PyObject *param3) { PyGILState_STATE s = PyGILState_Ensure(); +// PyEval_RestoreThread(s_mainThread); qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << pluginName(); PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); @@ -668,6 +675,7 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje Py_XDECREF(pFunc); qCWarning(dcThingManager()) << "Python plugin" << pluginName() << "does not implement" << function; PyGILState_Release(s); +// PyEval_SaveThread(); return false; } @@ -680,12 +688,14 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje qCWarning(dcThingManager()) << "Error calling python method:" << function << "on plugin" << pluginName(); PyErr_Print(); PyErr_Clear(); +// PyEval_SaveThread(); PyGILState_Release(s); return false; } if (QByteArray(result->ob_type->tp_name) != "coroutine") { Py_DECREF(result); +// PyEval_SaveThread(); PyGILState_Release(s); return true; } @@ -710,19 +720,26 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje dumpError(); PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); - QtConcurrent::run([=](){ - PyGILState_STATE s = PyGILState_Ensure(); + QtConcurrent::run([run_until_complete, task, loop, result](){ + PyGILState_STATE g = PyGILState_Ensure(); +// auto s = PyThreadState_New(PyInterpreterState_Main()); +// PyThreadState *previousThreadState = PyThreadState_Swap(s); PyObject_CallFunctionObjArgs(run_until_complete, task, nullptr); - PyGILState_Release(s); + Py_DECREF(loop); + Py_DECREF(task); + Py_DECREF(run_until_complete); + Py_DECREF(result); +// PyThreadState_Swap(previousThreadState); +// PyThreadState_Clear(s); +// PyThreadState_Delete(s); + PyGILState_Release(g); }); - Py_DECREF(loop); Py_DECREF(create_task); - Py_DECREF(task); Py_DECREF(add_done_callback); - Py_DECREF(result); PyGILState_Release(s); +// PyEval_SaveThread(); return true; } diff --git a/plugins/pymock/integrationpluginpymock.json b/plugins/pymock/integrationpluginpymock.json index 4797d60f..01b89a3e 100644 --- a/plugins/pymock/integrationpluginpymock.json +++ b/plugins/pymock/integrationpluginpymock.json @@ -85,6 +85,15 @@ "defaultValue": 0 } ], + "settingsTypes": [ + { + "id": "d2312d00-22f5-4e2c-a6cc-8e4cade2d58b", + "name": "setting1", + "displayName": "Setting 1", + "type": "QString", + "defaultValue": "hello" + } + ], "stateTypes": [ { "id": "99d0af17-9e8c-42bb-bece-a5d114f051d3", diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 14b1009c..0aa6b620 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -41,25 +41,17 @@ def startMonitoringAutoThings(): async def discoverThings(info): - logger.log("Discovery started for") #, info.thingClassId, "with result count:") #, info.params[0].value) - logger.log("a") + logger.log("Discovery started for", info.thingClassId, "with result count:", info.params[0].value) await asyncio.sleep(1) # Some delay for giving a feeling of a discovery # Add 2 new discovery results - logger.log("b") - info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, "Python mock thing 1")) - info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, "Python mock thing 2")) - logger.log("c") + for i in range(0, info.params[0].value): + info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, "Python mock thing %i" % i)) # Also add existing ones again so reconfiguration is possible for thing in myThings(): - logger.log("d") if thing.thingClassId == pyMockDiscoveryPairingThingClassId: - logger.log("e") info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, thing.name, thingId=thing.id)) - logger.log("f") - logger.log("g") info.finish(nymea.ThingErrorNoError) - logger.log("h") async def startPairing(info): @@ -89,6 +81,9 @@ def postSetupThing(thing): if thing.thingClassId == pyMockAutoThingClassId: logger.log("State 1 value:", thing.stateValue(pyMockAutoState1StateTypeId)) + if thing.thingClassId == pyMockDiscoveryPairingThingClassId: + logger.log("Setting 1 value:", thing.settingsValue(pyMockDiscoveryPairingSettingsSetting1ParamTypeId)) + def autoThings(): autoThings = [] From 3296d4b417c314f7f0b83e511e614c80565b823e Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 6 Jul 2020 00:30:30 +0200 Subject: [PATCH 21/46] more work --- libnymea-core/integrations/python/pyparam.h | 3 +- libnymea-core/integrations/python/pything.h | 166 +++++++++++++++----- libnymea-core/integrations/python/pyutils.h | 21 +-- libnymea/types/paramtype.cpp | 4 +- libnymea/types/paramtype.h | 4 +- libnymea/types/statetype.h | 2 +- plugins/pymock/integrationpluginpymock.json | 17 +- plugins/pymock/integrationpluginpymock.py | 18 ++- 8 files changed, 175 insertions(+), 60 deletions(-) diff --git a/libnymea-core/integrations/python/pyparam.h b/libnymea-core/integrations/python/pyparam.h index ab2b71a1..387b446f 100644 --- a/libnymea-core/integrations/python/pyparam.h +++ b/libnymea-core/integrations/python/pyparam.h @@ -7,6 +7,7 @@ #include "pyutils.h" #include "types/param.h" +#include "types/paramtype.h" #include "loggingcategories.h" @@ -99,7 +100,7 @@ static PyParam* PyParam_fromParam(const Param ¶m) static Param PyParam_ToParam(PyParam *pyParam) { - ParamTypeId paramTypeId = ParamTypeId(PyUnicode_AsUTF8(pyParam->pyParamTypeId)); + ParamTypeId paramTypeId = ParamTypeId(PyUnicode_AsUTF8AndSize(pyParam->pyParamTypeId, nullptr)); QVariant value = PyObjectToQVariant(pyParam->pyValue); return Param(paramTypeId, value); } diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 3074dd61..647a4998 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -11,7 +11,6 @@ #include #include -#include #include #pragma GCC diagnostic push @@ -26,7 +25,7 @@ * So we must never directly access anything of it in here. * * For writing to it, invoking methods with QueuedConnections will thread-decouple stuff. - * Make sure to lock the self->mutex while using the pointer to it for invoking stuff. + * Make sure to hold the GIL whenver accessing the pointer value for invoking stuff. * * For reading access, we keep copies of the thing properties here and sync them * over to the according py* members when they change. @@ -37,7 +36,7 @@ typedef struct _thing { PyObject_HEAD Thing *thing = nullptr; // the actual thing in nymea (not thread-safe!) - QMutex *mutex = nullptr; // The mutex for accessing the thing pointer + ThingClass *thingClass = nullptr; // A copy of the thing class. This is owned by the python thread PyObject *pyId = nullptr; PyObject *pyThingClassId = nullptr; PyObject *pyName = nullptr; @@ -55,7 +54,6 @@ static PyObject* PyThing_new(PyTypeObject *type, PyObject */*args*/, PyObject */ return nullptr; } qWarning() << "*++++ PyThing" << self; - self->mutex = new QMutex(); return (PyObject*)self; } @@ -64,6 +62,9 @@ static void PyThing_setThing(PyThing *self, Thing *thing) { self->thing = thing; + // Creating a copy because we cannot access the actual thing from the python thread + self->thingClass = new ThingClass(thing->thingClass()); + self->pyId = PyUnicode_FromString(self->thing->id().toString().toUtf8().data()); self->pyThingClassId = PyUnicode_FromString(self->thing->thingClassId().toString().toUtf8().data()); self->pyName = PyUnicode_FromString(self->thing->name().toUtf8().data()); @@ -106,6 +107,24 @@ static void PyThing_setThing(PyThing *self, Thing *thing) } PyGILState_Release(s); }); + + QObject::connect(thing, &Thing::stateValueChanged, [=](const StateTypeId &stateTypeId, const QVariant &value){ + PyGILState_STATE s = PyGILState_Ensure(); + for (int i = 0; i < PyList_Size(self->pyStates); i++) { + PyObject *pyState = PyList_GetItem(self->pyStates, i); + PyObject *pyStateTypeId = PyDict_GetItemString(pyState, "stateTypeId"); + StateTypeId stid = StateTypeId(PyUnicode_AsUTF8AndSize(pyStateTypeId, nullptr)); + if (stid == stateTypeId) { + qWarning() << "Updating state" << stateTypeId << value; + pyState = Py_BuildValue("{s:s, s:O}", + "stateTypeId", stateTypeId.toString().toUtf8().data(), + "value", QVariantToPyObject(value)); + PyList_SetItem(self->pyStates, i, pyState); + break; + } + } + PyGILState_Release(s); + }); } @@ -119,8 +138,8 @@ static void PyThing_dealloc(PyThing * self) { Py_XDECREF(self->pyStates); Py_XDECREF(self->pyNameChangedHandler); Py_XDECREF(self->pySettingChangedHandler); - delete self->mutex; - Py_TYPE(self)->tp_free(self); + delete self->thingClass; + Py_TYPE(self)->tp_free(self); } static PyObject *PyThing_getName(PyThing *self, void */*closure*/) @@ -143,7 +162,6 @@ static PyObject *PyThing_getThingClassId(PyThing *self, void */*closure*/) static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ QString name = QString(PyUnicode_AsUTF8(value)); - QMutexLocker(self->mutex); if (!self->thing) { return -1; } @@ -151,6 +169,74 @@ static int PyThing_setName(PyThing *self, PyObject *value, void */*closure*/){ return 0; } +static PyObject * PyThing_paramValue(PyThing* self, PyObject* args) +{ + char *paramTypeIdStr = nullptr; + + if (!PyArg_ParseTuple(args, "s", ¶mTypeIdStr)) { + qCWarning(dcThingManager) << "Error parsing parameters"; + return nullptr; + } + + ParamTypeId paramTypeId = ParamTypeId(paramTypeIdStr); + PyObject *iterator = PyObject_GetIter(self->pyParams); + while (iterator) { + PyObject *pyParam = PyIter_Next(iterator); + if (!pyParam) { + break; + } + + Param param = PyParam_ToParam((PyParam*)pyParam); + Py_DECREF(pyParam); + + if (param.paramTypeId() != paramTypeId) { + continue; + } + + Py_DECREF(iterator); + + return QVariantToPyObject(param.value()); + } + + Py_DECREF(iterator); + qCWarning(dcPythonIntegrations()) << "No param for paramTypeId:" << paramTypeId; + Py_RETURN_NONE; +} + +static PyObject * PyThing_setting(PyThing* self, PyObject* args) +{ + char *paramTypeIdStr = nullptr; + + if (!PyArg_ParseTuple(args, "s", ¶mTypeIdStr)) { + qCWarning(dcThingManager) << "Error parsing parameters"; + return nullptr; + } + + ParamTypeId paramTypeId = ParamTypeId(paramTypeIdStr); + PyObject *iterator = PyObject_GetIter(self->pySettings); + while (iterator) { + PyObject *pyParam = PyIter_Next(iterator); + if (!pyParam) { + break; + } + + Param param = PyParam_ToParam((PyParam*)pyParam); + Py_DECREF(pyParam); + + if (param.paramTypeId() != paramTypeId) { + continue; + } + + Py_DECREF(iterator); + + return QVariantToPyObject(param.value()); + } + + Py_DECREF(iterator); + qCWarning(dcPythonIntegrations()) << "No setting for paramTypeId:" << paramTypeId; + Py_RETURN_NONE; +} + static PyObject *PyThing_getSettings(PyThing *self, void */*closure*/) { Py_INCREF(self->pySettings); @@ -167,42 +253,25 @@ static PyObject * PyThing_stateValue(PyThing* self, PyObject* args) char *stateTypeIdStr = nullptr; if (!PyArg_ParseTuple(args, "s", &stateTypeIdStr)) { - qCWarning(dcThingManager) << "Error parsing parameters"; + PyErr_SetString(PyExc_ValueError, "Error parsing arguments. Signature is 's'"); return nullptr; } StateTypeId stateTypeId = StateTypeId(stateTypeIdStr); - PyObject *iterator = PyObject_GetIter(self->pyStates); - while (iterator) { - PyObject *pyState = PyIter_Next(iterator); - if (!pyState) { - break; - } + for (int i = 0; i < PyList_Size(self->pyStates); i++) { + PyObject *pyState = PyList_GetItem(self->pyStates, i); PyObject *pyStateTypeId = PyDict_GetItemString(pyState, "stateTypeId"); - PyObject *tmp = PyUnicode_AsEncodedString(pyStateTypeId, "UTF-8", "strict"); - - StateTypeId stid = StateTypeId(PyBytes_AS_STRING(tmp)); - - Py_DECREF(pyStateTypeId); - Py_XDECREF(tmp); - - if (stid != stateTypeId) { - Py_DECREF(pyState); - continue; + StateTypeId stid = StateTypeId(PyUnicode_AsUTF8AndSize(pyStateTypeId, nullptr)); + if (stid == stateTypeId) { + PyObject *value = PyDict_GetItemString(pyState, "value"); + Py_INCREF(value); + return value; } - - PyObject *pyStateValue = PyDict_GetItemString(pyState, "value"); - - Py_DECREF(pyState); - Py_DECREF(iterator); - - return pyStateValue; } - Py_DECREF(iterator); - qCWarning(dcPythonIntegrations()) << "No state for stateTypeId:" << stateTypeId; - Py_RETURN_NONE; + PyErr_SetString(PyExc_ValueError, QString("No state type %1 in thing class %2").arg(stateTypeId.toString()).arg(self->thingClass->name()).toUtf8()); + return nullptr; } static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) @@ -211,14 +280,18 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) PyObject *valueObj = nullptr; if (!PyArg_ParseTuple(args, "sO", &stateTypeIdStr, &valueObj)) { - qCWarning(dcThingManager) << "Error parsing parameters"; + PyErr_SetString(PyExc_ValueError, "Error parsing arguments. Signature is 'sO'"); return nullptr; } StateTypeId stateTypeId = StateTypeId(stateTypeIdStr); + StateType stateType = self->thingClass->stateTypes().findById(stateTypeId); + if (!stateType.isValid()) { + PyErr_SetString(PyExc_ValueError, QString("No state type %1 in thing class %2").arg(stateTypeId.toString()).arg(self->thingClass->name()).toUtf8()); + return nullptr; + } QVariant value = PyObjectToQVariant(valueObj); - QMutexLocker(self->mutex); if (self->thing != nullptr) { QMetaObject::invokeMethod(self->thing, "setStateValue", Qt::QueuedConnection, Q_ARG(StateTypeId, stateTypeId), Q_ARG(QVariant, value)); } @@ -232,14 +305,23 @@ static PyObject * PyThing_emitEvent(PyThing* self, PyObject* args) PyObject *valueObj = nullptr; if (!PyArg_ParseTuple(args, "s|O", &eventTypeIdStr, &valueObj)) { - qCWarning(dcThingManager) << "Error parsing parameters"; + PyErr_SetString(PyExc_TypeError, "Supplied arguments for emitEvent must be a ParamList"); + return nullptr; + } + if (qstrcmp(valueObj->ob_type->tp_name, "list") != 0) { + PyErr_SetString(PyExc_TypeError, "Supplied arguments for emitEvent must be a ParamList"); return nullptr; } EventTypeId eventTypeId = EventTypeId(eventTypeIdStr); + EventType eventType = self->thingClass->eventTypes().findById(eventTypeId); + if (!eventType.isValid()) { + PyErr_SetString(PyExc_ValueError, QString("No event type %1 in thing class %2").arg(eventTypeId.toString()).arg(self->thingClass->name()).toUtf8()); + return nullptr; + } + ParamList params = PyParams_ToParamList(valueObj); - QMutexLocker(self->mutex); if (self->thing != nullptr) { QMetaObject::invokeMethod(self->thing, "emitEvent", Qt::QueuedConnection, Q_ARG(EventTypeId, eventTypeId), Q_ARG(ParamList, params)); } @@ -256,9 +338,11 @@ static PyGetSetDef PyThing_getset[] = { }; static PyMethodDef PyThing_methods[] = { - { "stateValue", (PyCFunction)PyThing_stateValue, METH_VARARGS, "Get a things state value by stateTypeId" }, - { "setStateValue", (PyCFunction)PyThing_setStateValue, METH_VARARGS, "Set a certain things state value by stateTypeIp" }, - { "emitEvent", (PyCFunction)PyThing_emitEvent, METH_VARARGS, "Emits an event" }, + { "paramValue", (PyCFunction)PyThing_paramValue, METH_VARARGS, "Get a things param value by paramTypeId" }, + { "setting", (PyCFunction)PyThing_setting, METH_VARARGS, "Get a things setting value by paramTypeId" }, + { "stateValue", (PyCFunction)PyThing_stateValue, METH_VARARGS, "Get a things state value by stateTypeId" }, + { "setStateValue", (PyCFunction)PyThing_setStateValue, METH_VARARGS, "Set a certain things state value by stateTypeIp" }, + { "emitEvent", (PyCFunction)PyThing_emitEvent, METH_VARARGS, "Emits an event" }, {nullptr, nullptr, 0, nullptr} // sentinel }; diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index a119b5e6..0e0ef81a 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -45,18 +45,21 @@ PyObject *QVariantToPyObject(const QVariant &value) QVariant PyObjectToQVariant(PyObject *pyObject) { - // FIXME: is there any better way to do this? - qWarning() << "Error:" << PyErr_CheckSignals(); - PyObject* repr = PyObject_Repr(pyObject); - PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); - const char *bytes = PyBytes_AS_STRING(str); + qWarning() << "**************** type" << pyObject->ob_type->tp_name; - QVariant value(bytes); + if (qstrcmp(pyObject->ob_type->tp_name, "int") == 0) { + return QVariant(PyLong_AsLongLong(pyObject)); + } - Py_XDECREF(repr); - Py_XDECREF(str); + if (qstrcmp(pyObject->ob_type->tp_name, "str") == 0) { + return QVariant(PyUnicode_AsUTF8AndSize(pyObject, nullptr)); + } - return value; + if (qstrcmp(pyObject->ob_type->tp_name, "double") == 0) { + return QVariant(PyFloat_AsDouble(pyObject)); + } + Q_ASSERT_X(false, "pyutils.h", "Unhandled data type in conversion from Param to PyParam!"); + return QVariant(); } diff --git a/libnymea/types/paramtype.cpp b/libnymea/types/paramtype.cpp index 13a6ca85..fd0758c4 100644 --- a/libnymea/types/paramtype.cpp +++ b/libnymea/types/paramtype.cpp @@ -272,7 +272,7 @@ void ParamTypes::put(const QVariant &variant) append(variant.value()); } -ParamType ParamTypes::findByName(const QString &name) +ParamType ParamTypes::findByName(const QString &name) const { foreach (const ParamType ¶mType, *this) { if (paramType.name() == name) { @@ -282,7 +282,7 @@ ParamType ParamTypes::findByName(const QString &name) return ParamType(); } -ParamType ParamTypes::findById(const ParamTypeId &id) +ParamType ParamTypes::findById(const ParamTypeId &id) const { foreach (const ParamType ¶mType, *this) { if (paramType.id() == id) { diff --git a/libnymea/types/paramtype.h b/libnymea/types/paramtype.h index ea4df364..8f8cede2 100644 --- a/libnymea/types/paramtype.h +++ b/libnymea/types/paramtype.h @@ -124,8 +124,8 @@ public: ParamTypes(const QList &other); Q_INVOKABLE QVariant get(int index) const; Q_INVOKABLE void put(const QVariant &variant); - ParamType findByName(const QString &name); - ParamType findById(const ParamTypeId &id); + ParamType findByName(const QString &name) const; + ParamType findById(const ParamTypeId &id) const; }; Q_DECLARE_METATYPE(QList) Q_DECLARE_METATYPE(ParamTypes) diff --git a/libnymea/types/statetype.h b/libnymea/types/statetype.h index f87e91eb..1532a434 100644 --- a/libnymea/types/statetype.h +++ b/libnymea/types/statetype.h @@ -103,7 +103,7 @@ private: QString m_name; QString m_displayName; int m_index = 0; - QVariant::Type m_type; + QVariant::Type m_type = QVariant::Invalid; QVariant m_defaultValue; QVariant m_minValue; QVariant m_maxValue; diff --git a/plugins/pymock/integrationpluginpymock.json b/plugins/pymock/integrationpluginpymock.json index 01b89a3e..39636d8a 100644 --- a/plugins/pymock/integrationpluginpymock.json +++ b/plugins/pymock/integrationpluginpymock.json @@ -8,7 +8,7 @@ "name": "autoThingCount", "displayName": "Number of auto things", "type": "int", - "defaultValue": "fds" + "defaultValue": 0 } ], "vendors": [ @@ -94,6 +94,21 @@ "defaultValue": "hello" } ], + "eventTypes": [ + { + "id": "e6b98ef6-7922-48e6-b508-238d178b86ca", + "name": "event1", + "displayName": "Event 1", + "paramTypes": [ + { + "id": "7c265a6a-f0ae-4822-a14f-e6a090f5a310", + "name": "param1", + "displayName": "Event param 1", + "type": "QString" + } + ] + } + ], "stateTypes": [ { "id": "99d0af17-9e8c-42bb-bece-a5d114f051d3", diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 0aa6b620..3c4b2cc0 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -3,9 +3,19 @@ import asyncio watchingAutoThings = False -def init(): +async def init(): logger.log("Python mock plugin init") + while True: + await asyncio.sleep(2); + logger.log("Updating stuff") + for thing in myThings(): + if thing.thingClassId == pyMockDiscoveryPairingThingClassId: + logger.log("Emitting event 1 for", thing.name) +# thing.emitEvent(pyMockDiscoveryPairingEvent1EventTypeId, [nymea.Param(pyMockDiscoveryPairingEvent1EventParam1ParamTypeId, "Im an event")]) + logger.log("Setting state 1 for", thing.name, "Old value is:", thing.stateValue(pyMockDiscoveryPairingState1StateTypeId)) + thing.setStateValue(pyMockDiscoveryPairingState1StateTypeId, thing.stateValue(pyMockDiscoveryPairingState1StateTypeId) + 1) + def configValueChanged(paramTypeId, value): logger.log("Plugin config value changed:", paramTypeId, value, watchingAutoThings) @@ -74,7 +84,7 @@ async def setupThing(info): info.finish(nymea.ThingErrorNoError) -def postSetupThing(thing): +async def postSetupThing(thing): logger.log("postSetupThing for", thing.name, thing.params[0].value) thing.nameChangedHandler = lambda thing : logger.log("Thing name changed", thing.name) @@ -82,7 +92,9 @@ def postSetupThing(thing): logger.log("State 1 value:", thing.stateValue(pyMockAutoState1StateTypeId)) if thing.thingClassId == pyMockDiscoveryPairingThingClassId: - logger.log("Setting 1 value:", thing.settingsValue(pyMockDiscoveryPairingSettingsSetting1ParamTypeId)) + logger.log("Param 1 value:", thing.paramValue(pyMockDiscoveryPairingThingParam1ParamTypeId)) + logger.log("Setting 1 value:", thing.setting(pyMockDiscoveryPairingSettingsSetting1ParamTypeId)) + def autoThings(): From dadffcb7844b16683255c4de99aa5c18d756898a Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 6 Jul 2020 14:17:52 +0200 Subject: [PATCH 22/46] some fixes --- libnymea-core/integrations/python/pything.h | 5 ----- plugins/pymock/integrationpluginpymock.json | 8 +++++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 647a4998..18688eef 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -285,11 +285,6 @@ static PyObject * PyThing_setStateValue(PyThing* self, PyObject* args) } StateTypeId stateTypeId = StateTypeId(stateTypeIdStr); - StateType stateType = self->thingClass->stateTypes().findById(stateTypeId); - if (!stateType.isValid()) { - PyErr_SetString(PyExc_ValueError, QString("No state type %1 in thing class %2").arg(stateTypeId.toString()).arg(self->thingClass->name()).toUtf8()); - return nullptr; - } QVariant value = PyObjectToQVariant(valueObj); if (self->thing != nullptr) { diff --git a/plugins/pymock/integrationpluginpymock.json b/plugins/pymock/integrationpluginpymock.json index 39636d8a..52e7702e 100644 --- a/plugins/pymock/integrationpluginpymock.json +++ b/plugins/pymock/integrationpluginpymock.json @@ -65,15 +65,17 @@ "id": "248c5046-847b-44d0-ab7c-684ff79197dc", "name": "pyMockDiscoveryPairing", "displayName": "Python mock thing with discovery and pairing", - "createMethods": ["discovery"], + "createMethods": ["discovery", "user"], "setupMethod": "userAndPassword", "discoveryParamTypes": [ { "id": "ef5f6b90-e9d8-4e77-a14d-6725cfb07116", "name": "resultCount", "displayName": "Result count", - "type": "int", - "defaultValue": 2 + "type": "uint", + "defaultValue": 2, + "minValue": 0, + "maxValue": 20 } ], "paramTypes": [ From b0b480a5395254ea77a18aeb15ba3233171a5459 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Thu, 9 Jul 2020 12:48:02 +0200 Subject: [PATCH 23/46] drop some debugs --- libnymea-core/integrations/python/pything.h | 1 - libnymea-core/integrations/python/pyutils.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 18688eef..96bec099 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -115,7 +115,6 @@ static void PyThing_setThing(PyThing *self, Thing *thing) PyObject *pyStateTypeId = PyDict_GetItemString(pyState, "stateTypeId"); StateTypeId stid = StateTypeId(PyUnicode_AsUTF8AndSize(pyStateTypeId, nullptr)); if (stid == stateTypeId) { - qWarning() << "Updating state" << stateTypeId << value; pyState = Py_BuildValue("{s:s, s:O}", "stateTypeId", stateTypeId.toString().toUtf8().data(), "value", QVariantToPyObject(value)); diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index 0e0ef81a..103e9d9b 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -45,8 +45,6 @@ PyObject *QVariantToPyObject(const QVariant &value) QVariant PyObjectToQVariant(PyObject *pyObject) { - qWarning() << "**************** type" << pyObject->ob_type->tp_name; - if (qstrcmp(pyObject->ob_type->tp_name, "int") == 0) { return QVariant(PyLong_AsLongLong(pyObject)); } From 746f3e41213e33854c7296658a0a28b66cdda502 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Fri, 10 Jul 2020 23:02:06 +0200 Subject: [PATCH 24/46] fix actions cleanup --- .../integrations/pythonintegrationplugin.cpp | 5 +- plugins/pymock/integrationpluginpymock.json | 47 +++++++++++++++++++ plugins/pymock/integrationpluginpymock.py | 12 ++++- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 8fe0aafb..ee26620c 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -536,7 +536,10 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) PyGILState_Release(s); }); - callPluginFunction("executeAction", reinterpret_cast(pyInfo)); + bool success = callPluginFunction("executeAction", reinterpret_cast(pyInfo)); + if (!success) { + info->finish(Thing::ThingErrorUnsupportedFeature); + } } void PythonIntegrationPlugin::thingRemoved(Thing *thing) diff --git a/plugins/pymock/integrationpluginpymock.json b/plugins/pymock/integrationpluginpymock.json index 52e7702e..9ec7c8e9 100644 --- a/plugins/pymock/integrationpluginpymock.json +++ b/plugins/pymock/integrationpluginpymock.json @@ -49,6 +49,21 @@ "displayName": "Python mock thing", "createMethods": ["user"], "setupMethod": "justAdd", + "eventTypes": [ + { + "id": "de6c2425-0dee-413f-8f4c-bb0929e83c0d", + "name": "event1", + "displayName": "Event 1", + "paramTypes": [ + { + "id": "9f6aef52-dcde-4ee1-8ae3-4823594bf153", + "name": "param1", + "displayName": "Event param 1", + "type": "QString" + } + ] + } + ], "stateTypes": [ { "id": "24714828-93ec-41a7-875e-6a0b5b57d25c", @@ -59,6 +74,22 @@ "type": "int", "defaultValue": 0 } + ], + "actionTypes": [ + { + "id": "a504933a-2b86-41c1-b188-8998d445adf8", + "name": "action1", + "displayName": "Action 1", + "paramTypes": [ + { + "id": "5a824a21-3e97-49a5-8b4d-2a5bc3ea99ef", + "name": "param1", + "displayName": "Action param 1", + "type": "QString", + "defaultValue": "hello" + } + ] + } ] }, { @@ -121,6 +152,22 @@ "type": "int", "defaultValue": 0 } + ], + "actionTypes": [ + { + "id": "9bcc17ee-52a4-48ef-9a76-b2df4433dac5", + "name": "action1", + "displayName": "Action 1", + "paramTypes": [ + { + "id": "2e86cb76-e1a9-4afd-8fd3-e900cbb738f5", + "name": "param1", + "displayName": "Action param 1", + "type": "QString", + "defaultValue": "hello" + } + ] + } ] } ] diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 3c4b2cc0..28abe6f1 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -10,9 +10,14 @@ async def init(): await asyncio.sleep(2); logger.log("Updating stuff") for thing in myThings(): + if thing.thingClassId == pyMockThingClassId: + logger.log("Emitting event 1 for", thing.name, "eventTypeId", pyMockEvent1EventTypeId) + thing.emitEvent(pyMockEvent1EventTypeId, [nymea.Param(pyMockEvent1EventParam1ParamTypeId, "Im an event")]) + logger.log("Setting state 1 for", thing.name, "to", thing.stateValue(pyMockState1StateTypeId) + 1) + thing.setStateValue(pyMockState1StateTypeId, thing.stateValue(pyMockState1StateTypeId) + 1) if thing.thingClassId == pyMockDiscoveryPairingThingClassId: logger.log("Emitting event 1 for", thing.name) -# thing.emitEvent(pyMockDiscoveryPairingEvent1EventTypeId, [nymea.Param(pyMockDiscoveryPairingEvent1EventParam1ParamTypeId, "Im an event")]) + thing.emitEvent(pyMockDiscoveryPairingEvent1EventTypeId, [nymea.Param(pyMockDiscoveryPairingEvent1EventParam1ParamTypeId, "Im an event")]) logger.log("Setting state 1 for", thing.name, "Old value is:", thing.stateValue(pyMockDiscoveryPairingState1StateTypeId)) thing.setStateValue(pyMockDiscoveryPairingState1StateTypeId, thing.stateValue(pyMockDiscoveryPairingState1StateTypeId) + 1) @@ -79,7 +84,6 @@ async def confirmPairing(info, username, secret): async def setupThing(info): -# logger.log("setupThing for", info.thing.name, info.thing.params) logger.log("setupThing for", info.thing.name) info.finish(nymea.ThingErrorNoError) @@ -96,6 +100,10 @@ async def postSetupThing(thing): logger.log("Setting 1 value:", thing.setting(pyMockDiscoveryPairingSettingsSetting1ParamTypeId)) +async def executeAction(info): + logger.log("executeAction for", info.thing.name, info.actionTypeId, "with params", info.params[0].value) + info.finish(nymea.ThingErrorNoError) + def autoThings(): autoThings = [] From b87014060892e9d5e3a1d33c86fb16a969c592f1 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sat, 11 Jul 2020 01:05:24 +0200 Subject: [PATCH 25/46] Add tests, fix shutdown --- .../python/pynymealogginghandler.h | 2 - libnymea-core/integrations/python/pything.h | 1 - .../integrations/pythonintegrationplugin.cpp | 17 ++- .../integrations/pythonintegrationplugin.h | 2 + plugins/pymock/integrationpluginpymock.py | 7 +- tests/auto/auto.pro | 1 + tests/auto/pythonplugins/pythonplugins.pro | 6 + .../auto/pythonplugins/testpythonplugins.cpp | 140 ++++++++++++++++++ 8 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 tests/auto/pythonplugins/pythonplugins.pro create mode 100644 tests/auto/pythonplugins/testpythonplugins.cpp diff --git a/libnymea-core/integrations/python/pynymealogginghandler.h b/libnymea-core/integrations/python/pynymealogginghandler.h index 69599a40..2fa5d3c9 100644 --- a/libnymea-core/integrations/python/pynymealogginghandler.h +++ b/libnymea-core/integrations/python/pynymealogginghandler.h @@ -24,8 +24,6 @@ static int PyNymeaLoggingHandler_init(PyNymeaLoggingHandler */*self*/, PyObject static void PyNymeaLoggingHandler_dealloc(PyNymeaLoggingHandler * self) // destruct the object { - // FIXME: Why is this not called? Seems we're leaking... - Q_ASSERT(false); Py_TYPE(self)->tp_free(self); } diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 96bec099..5ab70716 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -73,7 +73,6 @@ static void PyThing_setThing(PyThing *self, Thing *thing) self->pyStates = PyList_New(thing->states().count()); for (int i = 0; i < thing->states().count(); i++) { - qWarning() << "i" << i; State state = thing->states().at(i); PyObject *pyState = Py_BuildValue("{s:s, s:O}", "stateTypeId", state.stateTypeId().toString().toUtf8().data(), diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index ee26620c..9ae6f5de 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -18,6 +18,7 @@ #include #include #include +#include PyThreadState* PythonIntegrationPlugin::s_mainThread = nullptr; PyObject* PythonIntegrationPlugin::s_nymeaModule = nullptr; @@ -71,7 +72,6 @@ static PyModuleDef nymea_module = "nymea module for python based integration plugins", // const char* m_doc; -1, // Py_ssize_t m_size; nymea_methods, // PyMethodDef *m_methods - // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; nullptr, nullptr, nullptr, nullptr }; @@ -274,9 +274,16 @@ PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationP PythonIntegrationPlugin::~PythonIntegrationPlugin() { - PyGILState_STATE s = PyGILState_Ensure(); + PyGILState_Ensure(); + + while (!m_runningThreads.isEmpty()) { + PyObject *loop = m_runningThreads.keys().first(); + PyObject *stop = PyObject_GetAttrString(loop, "stop"); + PyObject_CallFunctionObjArgs(stop, nullptr); + } + Py_XDECREF(s_plugins.take(this)); - PyGILState_Release(s); + Py_FinalizeEx(); } void PythonIntegrationPlugin::initPython() @@ -723,7 +730,7 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje dumpError(); PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); - QtConcurrent::run([run_until_complete, task, loop, result](){ + QFuture future = QtConcurrent::run([this, run_until_complete, task, loop, result](){ PyGILState_STATE g = PyGILState_Ensure(); // auto s = PyThreadState_New(PyInterpreterState_Main()); // PyThreadState *previousThreadState = PyThreadState_Swap(s); @@ -732,11 +739,13 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje Py_DECREF(task); Py_DECREF(run_until_complete); Py_DECREF(result); + m_runningThreads.remove(loop); // PyThreadState_Swap(previousThreadState); // PyThreadState_Clear(s); // PyThreadState_Delete(s); PyGILState_Release(g); }); + m_runningThreads.insert(loop, future); Py_DECREF(create_task); Py_DECREF(add_done_callback); diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index fbe2093e..9a0e5af4 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -82,6 +82,8 @@ private: // Need to keep a copy of plugin params and sync that in a thread-safe manner ParamList m_pluginConfigCopy; + + QHash> m_runningThreads; }; #endif // PYTHONINTEGRATIONPLUGIN_H diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 28abe6f1..63b2220e 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -89,7 +89,7 @@ async def setupThing(info): async def postSetupThing(thing): - logger.log("postSetupThing for", thing.name, thing.params[0].value) + logger.log("postSetupThing for", thing.name) thing.nameChangedHandler = lambda thing : logger.log("Thing name changed", thing.name) if thing.thingClassId == pyMockAutoThingClassId: @@ -111,3 +111,8 @@ def autoThings(): if thing.thingClassId == pyMockAutoThingClassId: autoThings.append(thing) return autoThings + + +# Intentionally commented out to also have a test case for unimplmented functions +# def thingRemoved(thing): +# logger.log("thingRemoved for", thing.name) diff --git a/tests/auto/auto.pro b/tests/auto/auto.pro index c6431395..db10112b 100644 --- a/tests/auto/auto.pro +++ b/tests/auto/auto.pro @@ -13,6 +13,7 @@ SUBDIRS = \ loggingloading \ mqttbroker \ plugins \ + pythonplugins \ rules \ scripts \ states \ diff --git a/tests/auto/pythonplugins/pythonplugins.pro b/tests/auto/pythonplugins/pythonplugins.pro new file mode 100644 index 00000000..438e619e --- /dev/null +++ b/tests/auto/pythonplugins/pythonplugins.pro @@ -0,0 +1,6 @@ +TARGET = testactions + +include(../../../nymea.pri) +include(../autotests.pri) + +SOURCES += testpythonplugins.cpp diff --git a/tests/auto/pythonplugins/testpythonplugins.cpp b/tests/auto/pythonplugins/testpythonplugins.cpp new file mode 100644 index 00000000..e3b08b4e --- /dev/null +++ b/tests/auto/pythonplugins/testpythonplugins.cpp @@ -0,0 +1,140 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* +* Copyright 2013 - 2020, nymea GmbH +* Contact: contact@nymea.io +* +* This file is part of nymea. +* This project including source code and documentation is protected by +* copyright law, and remains the property of nymea GmbH. All rights, including +* reproduction, publication, editing and translation, are reserved. The use of +* this project is subject to the terms of a license agreement to be concluded +* with nymea GmbH in accordance with the terms of use of nymea GmbH, available +* under https://nymea.io/license +* +* GNU General Public License Usage +* Alternatively, this project may be redistributed and/or modified under the +* terms of the GNU General Public License as published by the Free Software +* Foundation, GNU version 3. This project is distributed in the hope that it +* will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +* Public License for more details. +* +* You should have received a copy of the GNU General Public License along with +* this project. If not, see . +* +* For any further details and any questions please contact us under +* contact@nymea.io or see our FAQ/Licensing Information on +* https://nymea.io/license/faq +* +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "nymeatestbase.h" + +#include "integrations/thing.h" + +ThingClassId pyMockThingClassId = ThingClassId("1761c256-99b1-41bd-988a-a76087f6a4f1"); +ThingClassId pyMockDiscoveryPairingThingClassId = ThingClassId("248c5046-847b-44d0-ab7c-684ff79197dc"); +ParamTypeId pyMockDiscoveryPairingResultCountDiscoveryParamTypeID = ParamTypeId("ef5f6b90-e9d8-4e77-a14d-6725cfb07116"); + +using namespace nymeaserver; + +class TestPythonPlugins: public NymeaTestBase +{ + Q_OBJECT + +private: + inline void verifyThingError(const QVariant &response, Thing::ThingError error = Thing::ThingErrorNoError) { + verifyError(response, "thingError", enumValueName(error)); + } + +private slots: + + void initTestCase(); + + void setupAndRemoveThing(); + void testDiscoverPairAndRemoveThing(); + + +}; + + +void TestPythonPlugins::initTestCase() +{ + NymeaTestBase::initTestCase(); + QLoggingCategory::setFilterRules("*.debug=false\n" + "Tests.debug=true\n" + "PyMock.debug=true\n" + ); +} + +void TestPythonPlugins::setupAndRemoveThing() +{ + QVariantMap resultCountParam; + resultCountParam.insert("paramTypeId", pyMockDiscoveryPairingResultCountDiscoveryParamTypeID); + resultCountParam.insert("value", 2); + + QVariantList discoveryParams; + discoveryParams.append(resultCountParam); + + QVariantMap params; + params.insert("thingClassId", pyMockThingClassId); + params.insert("name", "Py test thing"); + QVariant response = injectAndWait("Integrations.AddThing", params); + + verifyThingError(response, Thing::ThingErrorNoError); + ThingId thingId = response.toMap().value("params").toMap().value("thingId").toUuid(); + qCDebug(dcTests()) << "New thing id" << thingId; + + params.clear(); + params.insert("thingId", thingId); + injectAndWait("Integrations.RemoveThing", params); + verifyThingError(response, Thing::ThingErrorNoError); +} + +void TestPythonPlugins::testDiscoverPairAndRemoveThing() +{ + // Discover + QVariantMap resultCountParam; + resultCountParam.insert("paramTypeId", pyMockDiscoveryPairingResultCountDiscoveryParamTypeID); + resultCountParam.insert("value", 2); + + QVariantList discoveryParams; + discoveryParams.append(resultCountParam); + + QVariantMap params; + params.insert("thingClassId", pyMockDiscoveryPairingThingClassId); + params.insert("discoveryParams", discoveryParams); + QVariant response = injectAndWait("Integrations.DiscoverThings", params); + + verifyThingError(response, Thing::ThingErrorNoError); + QCOMPARE(response.toMap().value("params").toMap().value("thingDescriptors").toList().count(), 2); + + ThingDescriptorId descriptorId = response.toMap().value("params").toMap().value("thingDescriptors").toList().first().toMap().value("id").toUuid(); + + // Pair + params.clear(); + params.insert("thingDescriptorId", descriptorId); + response = injectAndWait("Integrations.PairThing", params); + verifyThingError(response, Thing::ThingErrorNoError); + + qWarning() << "respo" << response.toMap().value("params").toMap(); + PairingTransactionId transactionId = response.toMap().value("params").toMap().value("pairingTransactionId").toUuid(); + qWarning() << "transactionId" << transactionId; + + params.clear(); + params.insert("pairingTransactionId", transactionId); + params.insert("username", "john"); + params.insert("secret", "smith"); + response = injectAndWait("Integrations.ConfirmPairing", params); + verifyThingError(response, Thing::ThingErrorNoError); + ThingId thingId = response.toMap().value("params").toMap().value("thingId").toUuid(); + + // Remove + params.clear(); + params.insert("thingId", thingId); + response = injectAndWait("Integrations.RemoveThing", params); + verifyThingError(response, Thing::ThingErrorNoError); +} + +#include "testpythonplugins.moc" +QTEST_MAIN(TestPythonPlugins) From 78e37f0d87a6e2073c1b0790f923421bd8fda285 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Wed, 15 Jul 2020 00:06:34 +0200 Subject: [PATCH 26/46] Fix shutdown --- .../integrations/pythonintegrationplugin.cpp | 11 +++++++++-- libnymea-core/integrations/pythonintegrationplugin.h | 1 + .../integrations/thingmanagerimplementation.cpp | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 9ae6f5de..d4a2f33a 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -274,7 +274,7 @@ PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationP PythonIntegrationPlugin::~PythonIntegrationPlugin() { - PyGILState_Ensure(); + PyGILState_STATE s = PyGILState_Ensure(); while (!m_runningThreads.isEmpty()) { PyObject *loop = m_runningThreads.keys().first(); @@ -283,7 +283,7 @@ PythonIntegrationPlugin::~PythonIntegrationPlugin() } Py_XDECREF(s_plugins.take(this)); - Py_FinalizeEx(); + PyGILState_Release(s); } void PythonIntegrationPlugin::initPython() @@ -302,6 +302,13 @@ void PythonIntegrationPlugin::initPython() s_mainThread = PyEval_SaveThread(); } +void PythonIntegrationPlugin::deinitPython() +{ + PyEval_RestoreThread(s_mainThread); + + Py_FinalizeEx(); +} + bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) { QFileInfo fi(scriptFile); diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 9a0e5af4..e34835e9 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -22,6 +22,7 @@ public: ~PythonIntegrationPlugin(); static void initPython(); + static void deinitPython(); bool loadScript(const QString &scriptFile); diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index d1a8a8d3..e8e9e653 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -101,6 +101,7 @@ ThingManagerImplementation::ThingManagerImplementation(HardwareManager *hardware ThingManagerImplementation::~ThingManagerImplementation() { + delete m_translator; foreach (Thing *thing, m_configuredThings) { @@ -116,6 +117,8 @@ ThingManagerImplementation::~ThingManagerImplementation() qCDebug(dcThingManager()) << "Not deleting plugin" << plugin->pluginName(); } } + + PythonIntegrationPlugin::deinitPython(); } QStringList ThingManagerImplementation::pluginSearchDirs() @@ -1951,7 +1954,7 @@ void ThingManagerImplementation::loadThingStates(Thing *thing) ThingClass thingClass = m_supportedThings.value(thing->thingClassId()); foreach (const StateType &stateType, thingClass.stateTypes()) { if (stateType.cached()) { - QVariant value(stateType.defaultValue()); + QVariant value = stateType.defaultValue(); if (settings.contains(stateType.id().toString())) { value = settings.value(stateType.id().toString()); @@ -1963,6 +1966,7 @@ void ThingManagerImplementation::loadThingStates(Thing *thing) } value.convert(stateType.type()); thing->setStateValue(stateType.id(), value); + qWarning() << "**** loaded state" << stateType.name() << value; } else { thing->setStateValue(stateType.id(), stateType.defaultValue()); } From 1018083052f08e6e6173a99bdca7c69c7e54ab80 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Thu, 16 Jul 2020 23:48:29 +0200 Subject: [PATCH 27/46] add 2 missing data type conversions --- libnymea-core/integrations/python/pyutils.h | 11 ++++++++++- .../integrations/thingmanagerimplementation.cpp | 1 - plugins/pymock/integrationpluginpymock.py | 3 +-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index 103e9d9b..62b3cc7b 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -56,7 +56,16 @@ QVariant PyObjectToQVariant(PyObject *pyObject) if (qstrcmp(pyObject->ob_type->tp_name, "double") == 0) { return QVariant(PyFloat_AsDouble(pyObject)); } - Q_ASSERT_X(false, "pyutils.h", "Unhandled data type in conversion from Param to PyParam!"); + + if (qstrcmp(pyObject->ob_type->tp_name, "float") == 0) { + return QVariant(PyFloat_AsDouble(pyObject)); + } + + if (qstrcmp(pyObject->ob_type->tp_name, "bool") == 0) { + return QVariant(PyObject_IsTrue(pyObject)); + } + + Q_ASSERT_X(false, "pyutils.h", "Unhandled data type in conversion PyObject to QVariant!"); return QVariant(); } diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index e8e9e653..e21e99f1 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -1966,7 +1966,6 @@ void ThingManagerImplementation::loadThingStates(Thing *thing) } value.convert(stateType.type()); thing->setStateValue(stateType.id(), value); - qWarning() << "**** loaded state" << stateType.name() << value; } else { thing->setStateValue(stateType.id(), stateType.defaultValue()); } diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 63b2220e..285a0071 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -7,8 +7,7 @@ async def init(): logger.log("Python mock plugin init") while True: - await asyncio.sleep(2); - logger.log("Updating stuff") + await asyncio.sleep(5); for thing in myThings(): if thing.thingClassId == pyMockThingClassId: logger.log("Emitting event 1 for", thing.name, "eventTypeId", pyMockEvent1EventTypeId) From 64e7d2784c5006580bada920d10198d5ec107777 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sat, 18 Jul 2020 19:24:25 +0200 Subject: [PATCH 28/46] checkpoint asyncio --- .../integrations/pythonintegrationplugin.cpp | 249 ++++++++++-------- .../integrations/pythonintegrationplugin.h | 24 +- plugins/pymock/integrationpluginpymock.py | 3 +- 3 files changed, 162 insertions(+), 114 deletions(-) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index d4a2f33a..0ba9f215 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -19,10 +19,10 @@ #include #include #include +#include -PyThreadState* PythonIntegrationPlugin::s_mainThread = nullptr; -PyObject* PythonIntegrationPlugin::s_nymeaModule = nullptr; -PyObject* PythonIntegrationPlugin::s_asyncio = nullptr; +PyThreadState* PythonIntegrationPlugin::s_mainThreadState = nullptr; +QThreadPool* PythonIntegrationPlugin::s_threadPool = nullptr; QHash PythonIntegrationPlugin::s_plugins; @@ -34,7 +34,7 @@ PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) PyObject *result = nullptr; if (!PyArg_ParseTuple(args, "O", &result)) { - qCWarning(dcThingManager()) << "Cannot fetch result from coroutine callback."; + qCWarning(dcPythonIntegrations()) << "Cannot fetch result from coroutine callback."; return nullptr; } @@ -50,7 +50,7 @@ PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) // PyObject *traceback = PyObject_CallMethodObjArgs(exception, "__traceback__", nullptr); - qCWarning(dcThingManager()) << "Exception in plugin:" << bytes; + qCWarning(dcPythonIntegrations()) << "Exception in plugin:" << bytes; PyErr_Clear(); } @@ -274,39 +274,47 @@ PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationP PythonIntegrationPlugin::~PythonIntegrationPlugin() { - PyGILState_STATE s = PyGILState_Ensure(); + // Acquire GIL for this plugin's interpreter + PyEval_RestoreThread(m_threadState); - while (!m_runningThreads.isEmpty()) { - PyObject *loop = m_runningThreads.keys().first(); - PyObject *stop = PyObject_GetAttrString(loop, "stop"); - PyObject_CallFunctionObjArgs(stop, nullptr); + // Cancel all the thread in here + while (!m_runningTasks.isEmpty()) { + QFutureWatcher *watcher = m_runningTasks.values().first(); + watcher->cancel(); + watcher->waitForFinished(); } Py_XDECREF(s_plugins.take(this)); - PyGILState_Release(s); + + Py_EndInterpreter(m_threadState); + + PyThreadState_Swap(s_mainThreadState); + PyEval_ReleaseThread(s_mainThreadState); } void PythonIntegrationPlugin::initPython() { + Q_ASSERT_X(s_mainThreadState == nullptr, "PythonIntegrationPlugin::initPython()", "initPython() must be called exactly once."); + PyImport_AppendInittab("nymea", PyInit_nymea); Py_InitializeEx(0); PyEval_InitThreads(); + // Store the main thread state and release the GIL + s_mainThreadState = PyEval_SaveThread(); - // Import nymea module into this interpreter - s_nymeaModule = PyImport_ImportModule("nymea"); - - // We'll be using asyncio everywhere, so let's import it right away - s_asyncio = PyImport_ImportModule("asyncio"); - - // Need to release the lock from the main thread before spawning new threads - s_mainThread = PyEval_SaveThread(); + // Allocate a shared thread pool for the plugins + s_threadPool = new QThreadPool(); + qCDebug(dcPythonIntegrations()) << "Created a thread pool with a maximum of" << s_threadPool->maxThreadCount() << "threads for python plugins."; } void PythonIntegrationPlugin::deinitPython() { - PyEval_RestoreThread(s_mainThread); + PyEval_RestoreThread(s_mainThreadState); Py_FinalizeEx(); + + s_threadPool->deleteLater(); + s_threadPool = nullptr; } bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) @@ -330,31 +338,52 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) return false; } -// PyThreadState *m_thread = Py_NewInterpreter(); -// PyInterpreterState *m_interpreter = PyInterpreterState_New(); + // Grab the main thread context and GIL + PyEval_RestoreThread(s_mainThreadState); -// PyEval_RestoreThread(s_mainThread); - PyGILState_STATE s = PyGILState_Ensure(); + // Create a new interpreter + m_threadState = Py_NewInterpreter(); - // Finally, import the plugin + // Switch to the new interpreter thread state + PyThreadState_Swap(m_threadState); + + // Import nymea module into this interpreter + m_nymeaModule = PyImport_ImportModule("nymea"); + + // Set up import paths for the plugin PyObject* sysPath = PySys_GetObject("path"); - PyObject* importPath = PyUnicode_FromString(fi.absolutePath().toUtf8()); - PyList_Append(sysPath, importPath); - Py_DECREF(importPath); + PyObject* pluginImportPath = PyUnicode_FromString(fi.absolutePath().toUtf8()); + PyList_Append(sysPath, pluginImportPath); + Py_DECREF(pluginImportPath); + + PyObject* pluginModulesImportPath = PyUnicode_FromString(QString("%1/modules/").arg(fi.absolutePath()).toUtf8()); + PyList_Append(sysPath, pluginModulesImportPath); + Py_DECREF(pluginModulesImportPath); + m_module = PyImport_ImportModule(fi.baseName().toUtf8()); if (!m_module) { qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath(); PyErr_Print(); PyErr_Clear(); -// PyEval_SaveThread(); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); return false; } qCDebug(dcThingManager()) << "Imported python plugin from" << fi.absoluteFilePath(); s_plugins.insert(this, m_module); + // We'll be using asyncio everywhere, so let's import it right away + // IMPORTANT: The asyncio module is a bit special in a sense that it actually shares + // stuff between interpreters. See https://docs.python.org/3/c-api/init.html#c.Py_NewInterpreter + // for the listed bugs and caveats. + // If we destroy the first interpreter that imports asyncio, things will become crashy as the + // interpreter will tear down the module, leaving shallow copies of the modules dict in other + // interpreters. So let's import the module down here after we're sure this module compiles and + // won't be unloaded any more. + // Note: This becomes a problem when we'll be supporting to unload plugins at runtime. + m_asyncio = PyImport_ImportModule("asyncio"); + // Set up logger with appropriate logging category PyNymeaLoggingHandler *logger = reinterpret_cast(_PyObject_New(&PyNymeaLoggingHandlerType)); QString category = metadata().pluginName(); @@ -371,8 +400,7 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) // Register config access methods PyModule_AddFunctions(m_module, plugin_methods); -// PyEval_SaveThread(); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); // Set up connections to be forwareded into the plugin connect(this, &PythonIntegrationPlugin::configValueChanged, this, [this](const ParamTypeId ¶mTypeId, const QVariant &value){ @@ -408,18 +436,18 @@ void PythonIntegrationPlugin::startMonitoringAutoThings() void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) { - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); PyThingDiscoveryInfo *pyInfo = (PyThingDiscoveryInfo*)PyObject_CallObject((PyObject*)&PyThingDiscoveryInfoType, NULL); PyThingDiscoveryInfo_setInfo(pyInfo, info); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); connect(info, &ThingDiscoveryInfo::destroyed, this, [=](){ - auto s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); pyInfo->info = nullptr; Py_DECREF(pyInfo); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); }); callPluginFunction("discoverThings", reinterpret_cast(pyInfo)); @@ -427,19 +455,18 @@ void PythonIntegrationPlugin::discoverThings(ThingDiscoveryInfo *info) void PythonIntegrationPlugin::startPairing(ThingPairingInfo *info) { - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); PyThingPairingInfo *pyInfo = (PyThingPairingInfo*)PyObject_CallObject((PyObject*)&PyThingPairingInfoType, nullptr); PyThingPairingInfo_setInfo(pyInfo, info); - PyGILState_Release(s); - + PyEval_ReleaseThread(m_threadState); connect(info, &ThingPairingInfo::destroyed, this, [=](){ - auto s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); pyInfo->info = nullptr; Py_DECREF(pyInfo); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); }); bool result = callPluginFunction("startPairing", reinterpret_cast(pyInfo)); @@ -450,19 +477,18 @@ void PythonIntegrationPlugin::startPairing(ThingPairingInfo *info) void PythonIntegrationPlugin::confirmPairing(ThingPairingInfo *info, const QString &username, const QString &secret) { - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); PyThingPairingInfo *pyInfo = (PyThingPairingInfo*)PyObject_CallObject((PyObject*)&PyThingPairingInfoType, nullptr); PyThingPairingInfo_setInfo(pyInfo, info); - PyGILState_Release(s); - + PyEval_ReleaseThread(m_threadState); connect(info, &ThingPairingInfo::destroyed, this, [=](){ - auto s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); pyInfo->info = nullptr; Py_DECREF(pyInfo); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); }); PyObject *pyUsername = PyUnicode_FromString(username.toUtf8().data()); @@ -478,7 +504,7 @@ void PythonIntegrationPlugin::confirmPairing(ThingPairingInfo *info, const QStri void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) { - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); PyThing *pyThing = nullptr; if (m_things.contains(info->thing())) { @@ -497,19 +523,19 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) pyInfo->info = info; - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); connect(info->thing(), &Thing::destroyed, this, [=](){ - auto s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); pyThing->thing = nullptr; Py_DECREF(pyThing); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); }); connect(info, &ThingSetupInfo::destroyed, this, [=](){ - auto s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); pyInfo->info = nullptr; Py_DECREF(pyInfo); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); }); @@ -529,25 +555,24 @@ void PythonIntegrationPlugin::postSetupThing(Thing *thing) if (!success) { Py_DECREF(pyThing); } - } void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) { PyThing *pyThing = m_things.value(info->thing()); - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(m_threadState); PyThingActionInfo *pyInfo = (PyThingActionInfo*)PyObject_CallObject((PyObject*)&PyThingActionInfoType, NULL); PyThingActionInfo_setInfo(pyInfo, info, pyThing); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); - connect(info, &ThingActionInfo::destroyed, this, [=](){ - auto s = PyGILState_Ensure(); + connect(info, &ThingActionInfo::destroyed, this, [=](){ + PyEval_RestoreThread(m_threadState); pyInfo->info = nullptr; Py_DECREF(pyInfo); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); }); bool success = callPluginFunction("executeAction", reinterpret_cast(pyInfo)); @@ -682,84 +707,98 @@ void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &ac bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param1, PyObject *param2, PyObject *param3) { - PyGILState_STATE s = PyGILState_Ensure(); -// PyEval_RestoreThread(s_mainThread); + PyEval_RestoreThread(m_threadState); qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << pluginName(); - PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); - if(!pFunc || !PyCallable_Check(pFunc)) { + PyObject *pluginFunction = PyObject_GetAttrString(m_module, function.toUtf8()); + if(!pluginFunction || !PyCallable_Check(pluginFunction)) { PyErr_Clear(); - Py_XDECREF(pFunc); - qCWarning(dcThingManager()) << "Python plugin" << pluginName() << "does not implement" << function; - PyGILState_Release(s); -// PyEval_SaveThread(); + Py_XDECREF(pluginFunction); + qCDebug(dcThingManager()) << "Python plugin" << pluginName() << "does not implement" << function; + PyEval_ReleaseThread(m_threadState); return false; } - PyObject *result = PyObject_CallFunctionObjArgs(pFunc, param1, param2, param3, nullptr); + PyObject *pluginFunctionResult = PyObject_CallFunctionObjArgs(pluginFunction, param1, param2, param3, nullptr); - Py_XDECREF(pFunc); + Py_XDECREF(pluginFunction); if (PyErr_Occurred()) { qCWarning(dcThingManager()) << "Error calling python method:" << function << "on plugin" << pluginName(); PyErr_Print(); PyErr_Clear(); -// PyEval_SaveThread(); - PyGILState_Release(s); + PyEval_ReleaseThread(m_threadState); return false; } - if (QByteArray(result->ob_type->tp_name) != "coroutine") { - Py_DECREF(result); -// PyEval_SaveThread(); - PyGILState_Release(s); + if (QByteArray(pluginFunctionResult->ob_type->tp_name) != "coroutine") { + Py_DECREF(pluginFunctionResult); + PyEval_ReleaseThread(m_threadState); return true; } - // Spawn a event loop for the thread - PyObject *new_event_loop = PyObject_GetAttrString(s_asyncio, "new_event_loop"); - PyObject *loop = PyObject_CallFunctionObjArgs(new_event_loop, nullptr); + // Spawn a new event loop for the thread + QFuture future = QtConcurrent::run([this, pluginFunctionResult, function](){ + qCDebug(dcPythonIntegrations()) << "Spawning thread for" << function << "in plugin" << metadata().pluginName(); - Py_DECREF(new_event_loop); + // Register this new thread in the interpreter + PyThreadState *thread = PyThreadState_New(m_threadState->interp); - PyObject *create_task = PyObject_GetAttrString(loop, "create_task"); - PyObject *task = PyObject_CallFunctionObjArgs(create_task, result, nullptr); - dumpError(); + // Acquire GIL and make the new thread state the current one + PyEval_RestoreThread(thread); - Py_DECREF(result); + PyObject *new_event_loop = PyObject_GetAttrString(m_asyncio, "new_event_loop"); + PyObject *loop = PyObject_CallFunctionObjArgs(new_event_loop, nullptr); - PyObject *add_done_callback = PyObject_GetAttrString(task, "add_done_callback"); - dumpError(); + Py_DECREF(new_event_loop); - PyObject *task_done = PyObject_GetAttrString(s_nymeaModule, "task_done"); - result = PyObject_CallFunctionObjArgs(add_done_callback, task_done, nullptr); - dumpError(); + PyObject *create_task = PyObject_GetAttrString(loop, "create_task"); + PyObject *task = PyObject_CallFunctionObjArgs(create_task, pluginFunctionResult, nullptr); + dumpError(); + + Py_DECREF(create_task); + Py_DECREF(pluginFunctionResult); + + PyObject *add_done_callback = PyObject_GetAttrString(task, "add_done_callback"); + dumpError(); + + PyObject *task_done = PyObject_GetAttrString(m_nymeaModule, "task_done"); + PyObject *coroutineResult = PyObject_CallFunctionObjArgs(add_done_callback, task_done, nullptr); + dumpError(); + + Py_DECREF(coroutineResult); + Py_DECREF(add_done_callback); + Py_DECREF(task_done); + + PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); - PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); - QFuture future = QtConcurrent::run([this, run_until_complete, task, loop, result](){ - PyGILState_STATE g = PyGILState_Ensure(); -// auto s = PyThreadState_New(PyInterpreterState_Main()); -// PyThreadState *previousThreadState = PyThreadState_Swap(s); - PyObject_CallFunctionObjArgs(run_until_complete, task, nullptr); Py_DECREF(loop); - Py_DECREF(task); + + PyObject *taskResult = PyObject_CallFunctionObjArgs(run_until_complete, task, nullptr); + dumpError(); + + Py_XDECREF(taskResult); Py_DECREF(run_until_complete); - Py_DECREF(result); - m_runningThreads.remove(loop); -// PyThreadState_Swap(previousThreadState); -// PyThreadState_Clear(s); -// PyThreadState_Delete(s); - PyGILState_Release(g); + Py_DECREF(task); + + // Destroy the thread and release the GIL + PyThreadState_Clear(thread); + PyThreadState_DeleteCurrent(); + qCDebug(dcPythonIntegrations()) << "Thread for" << function << "in plugin" << metadata().pluginName() << "ended"; }); - m_runningThreads.insert(loop, future); - Py_DECREF(create_task); - Py_DECREF(add_done_callback); + QFutureWatcher *watcher = new QFutureWatcher(this); + watcher->setFuture(future); + m_runningTasks.insert(watcher); - PyGILState_Release(s); -// PyEval_SaveThread(); + connect(watcher, &QFutureWatcher::finished, this, [this, watcher](){ + m_runningTasks.remove(watcher); + delete watcher; + }); + + PyEval_ReleaseThread(m_threadState); return true; } diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index e34835e9..927cd314 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -6,11 +6,13 @@ #include #include #include +#include extern "C" { typedef struct _object PyObject; typedef struct _ts PyThreadState; typedef struct _thing PyThing; +typedef struct _is PyInterpreterState; } @@ -62,11 +64,19 @@ private: bool callPluginFunction(const QString &function, PyObject *param1 = nullptr, PyObject *param2 = nullptr, PyObject *param3 = nullptr); private: - static PyThreadState* s_mainThread; + // The main thread state in which we create an interpreter per plugin + static PyThreadState* s_mainThreadState; + static QThreadPool *s_threadPool; - // Modules imported into the global context - static PyObject *s_nymeaModule; - static PyObject *s_asyncio; + // A per plugin thread state and interpreter + PyThreadState *m_threadState = nullptr; + + // Modules imported into the interpreter + PyObject *m_nymeaModule; + PyObject *m_asyncio; + + // The imported plugin module (the plugin.py) + PyObject *m_module = nullptr; // A map of plugin instances to plugin python scripts/modules // Make sure to hold the GIL when accessing this. @@ -75,16 +85,14 @@ private: // Used for guarding access from the python threads to the plugin instance QMutex m_mutex; - // The imported plugin module - PyObject *m_module = nullptr; - // Things held by this plugin instance QHash m_things; // Need to keep a copy of plugin params and sync that in a thread-safe manner ParamList m_pluginConfigCopy; - QHash> m_runningThreads; + // Running concurrent tasks in this plugin + QSet*> m_runningTasks; }; #endif // PYTHONINTEGRATIONPLUGIN_H diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 285a0071..2107a51e 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -1,5 +1,6 @@ import nymea import asyncio +#from fastdotcom import fast_com watchingAutoThings = False @@ -56,7 +57,7 @@ def startMonitoringAutoThings(): async def discoverThings(info): logger.log("Discovery started for", info.thingClassId, "with result count:", info.params[0].value) - await asyncio.sleep(1) # Some delay for giving a feeling of a discovery + await asyncio.sleep(10) # Some delay for giving a feeling of a discovery # Add 2 new discovery results for i in range(0, info.params[0].value): info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, "Python mock thing %i" % i)) From 3436e9b998380519e39fa238d728e378e8474207 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sun, 19 Jul 2020 23:18:18 +0200 Subject: [PATCH 29/46] drop usage of asyncio and run everything regularly threaded --- .../python/pynymealogginghandler.h | 16 +- .../integrations/python/pynymeamodule.h | 62 ++++ libnymea-core/integrations/python/pyparam.h | 4 +- libnymea-core/integrations/python/pything.h | 4 +- .../integrations/python/pythingactioninfo.h | 4 +- .../integrations/python/pythingsetupinfo.h | 4 +- libnymea-core/integrations/python/pyutils.h | 2 +- .../integrations/pythonintegrationplugin.cpp | 319 ++++++------------ .../integrations/pythonintegrationplugin.h | 21 +- libnymea-core/libnymea-core.pro | 1 + plugins/pymock/integrationpluginpymock.py | 32 +- 11 files changed, 218 insertions(+), 251 deletions(-) create mode 100644 libnymea-core/integrations/python/pynymeamodule.h diff --git a/libnymea-core/integrations/python/pynymealogginghandler.h b/libnymea-core/integrations/python/pynymealogginghandler.h index 2fa5d3c9..1a5b3f5f 100644 --- a/libnymea-core/integrations/python/pynymealogginghandler.h +++ b/libnymea-core/integrations/python/pynymealogginghandler.h @@ -7,6 +7,8 @@ #include #include +Q_DECLARE_LOGGING_CATEGORY(dcPythonIntegrations) + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -16,14 +18,23 @@ typedef struct { char *category; } PyNymeaLoggingHandler; -static int PyNymeaLoggingHandler_init(PyNymeaLoggingHandler */*self*/, PyObject */*args*/, PyObject */*kwds*/) +static int PyNymeaLoggingHandler_init(PyNymeaLoggingHandler *self, PyObject *args, PyObject */*kwds*/) { + char *category = nullptr; + if (!PyArg_ParseTuple(args, "s", &category)) { + qCWarning(dcPythonIntegrations()) << "PyNymeaLoggingHandler: Error parsing parameters"; + return -1; + } + + self->category = (char*)malloc(qstrlen(category)); + qstrcpy(self->category, category); + return 0; } static void PyNymeaLoggingHandler_dealloc(PyNymeaLoggingHandler * self) -// destruct the object { + free(self->category); Py_TYPE(self)->tp_free(self); } @@ -106,5 +117,6 @@ static void registerNymeaLoggingHandler(PyObject *module) } } +#pragma GCC diagnostic pop #endif // PYNYMEALOGGINGHANDLER_H diff --git a/libnymea-core/integrations/python/pynymeamodule.h b/libnymea-core/integrations/python/pynymeamodule.h new file mode 100644 index 00000000..e750e650 --- /dev/null +++ b/libnymea-core/integrations/python/pynymeamodule.h @@ -0,0 +1,62 @@ +#ifndef PYNYMEAMODULE_H +#define PYNYMEAMODULE_H + +#include + +#include "pynymealogginghandler.h" +#include "pything.h" +#include "pythingdiscoveryinfo.h" +#include "pythingsetupinfo.h" +#include "pyparam.h" +#include "pythingactioninfo.h" +#include "pythingpairinginfo.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +static int nymea_exec(PyObject *m) { + + // Override stdout/stderr to use qDebug instead + PyObject* pyLog = PyModule_Create(&pyLog_module); + PySys_SetObject("stdout", pyLog); + PyObject* pyWarn = PyModule_Create(&pyWarn_module); + PySys_SetObject("stderr", pyWarn); + + registerNymeaLoggingHandler(m); + registerParamType(m); + registerThingType(m); + registerThingDescriptorType(m); + registerThingDiscoveryInfoType(m); + registerThingPairingInfoType(m); + registerThingSetupInfoType(m); + registerThingActionInfoType(m); + + return 0; +} + +static struct PyModuleDef_Slot nymea_slots[] = { + {Py_mod_exec, (void*)nymea_exec}, + {0, NULL}, +}; + +static struct PyModuleDef nymea_module = { + PyModuleDef_HEAD_INIT, + "nymea", + "The nymea module. Provdes types used in the nymea plugin API.", + 0, + nullptr, // methods + nymea_slots, // slots + nullptr, + nullptr, + nullptr +}; + +PyMODINIT_FUNC PyInit_nymea(void) +{ + return PyModuleDef_Init(&nymea_module); +} + +#pragma GCC diagnostic pop + +#endif // PYNYMEAMODULE_H diff --git a/libnymea-core/integrations/python/pyparam.h b/libnymea-core/integrations/python/pyparam.h index 387b446f..87ae8063 100644 --- a/libnymea-core/integrations/python/pyparam.h +++ b/libnymea-core/integrations/python/pyparam.h @@ -35,7 +35,7 @@ static PyMemberDef PyParam_members[] = { static int PyParam_init(PyParam *self, PyObject *args, PyObject *kwds) { - qWarning() << "++++ PyParam"; + qCDebug(dcPythonIntegrations()) << "+++ PyParam"; static char *kwlist[] = {"paramTypeId", "value", nullptr}; PyObject *paramTypeId = nullptr, *value = nullptr; @@ -54,7 +54,7 @@ static int PyParam_init(PyParam *self, PyObject *args, PyObject *kwds) } static void PyParam_dealloc(PyParam * self) { - qWarning() << "---- PyParam"; + qCDebug(dcPythonIntegrations()) << "--- PyParam"; Py_XDECREF(self->pyParamTypeId); Py_XDECREF(self->pyValue); Py_TYPE(self)->tp_free(self); diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 5ab70716..28e9092b 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -53,7 +53,7 @@ static PyObject* PyThing_new(PyTypeObject *type, PyObject */*args*/, PyObject */ if (self == NULL) { return nullptr; } - qWarning() << "*++++ PyThing" << self; + qCDebug(dcPythonIntegrations()) << "+++ PyThing" << self; return (PyObject*)self; } @@ -127,7 +127,7 @@ static void PyThing_setThing(PyThing *self, Thing *thing) static void PyThing_dealloc(PyThing * self) { - qWarning() << "----- PyThing" << self; + qCDebug(dcPythonIntegrations()) << "--- PyThing" << self; Py_XDECREF(self->pyId); Py_XDECREF(self->pyThingClassId); Py_XDECREF(self->pyName); diff --git a/libnymea-core/integrations/python/pythingactioninfo.h b/libnymea-core/integrations/python/pythingactioninfo.h index 05a1a35e..d29c4e4e 100644 --- a/libnymea-core/integrations/python/pythingactioninfo.h +++ b/libnymea-core/integrations/python/pythingactioninfo.h @@ -43,7 +43,7 @@ static PyObject* PyThingActionInfo_new(PyTypeObject *type, PyObject */*args*/, P if (self == NULL) { return nullptr; } - qWarning() << "+++ PyThingActionInfo"; + qCDebug(dcPythonIntegrations()) << "+++ PyThingActionInfo"; return (PyObject*)self; } @@ -59,7 +59,7 @@ void PyThingActionInfo_setInfo(PyThingActionInfo *self, ThingActionInfo *info, P static void PyThingActionInfo_dealloc(PyThingActionInfo * self) { - qWarning() << "---- PyThingActionInfo"; + qCDebug(dcPythonIntegrations()) << "--- PyThingActionInfo"; Py_DECREF(self->pyThing); Py_DECREF(self->pyActionTypeId); Py_DECREF(self->pyParams); diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index 19010c09..83199546 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -41,7 +41,7 @@ static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject *args, PyObje if (self == NULL) { return nullptr; } - qWarning() << "++++ PyThingSetupInfo"; + qCDebug(dcPythonIntegrations()) << "+++ PyThingSetupInfo"; static char *kwlist[] = {"thing", nullptr}; @@ -60,7 +60,7 @@ static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject *args, PyObje } static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) { - qWarning() << "--- PyThingSetupInfo"; + qCDebug(dcPythonIntegrations()) << "--- PyThingSetupInfo"; Py_DECREF(self->pyThing); Py_TYPE(self)->tp_free(self); } diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index 62b3cc7b..2e00da05 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -65,7 +65,7 @@ QVariant PyObjectToQVariant(PyObject *pyObject) return QVariant(PyObject_IsTrue(pyObject)); } - Q_ASSERT_X(false, "pyutils.h", "Unhandled data type in conversion PyObject to QVariant!"); + Q_ASSERT_X(false, "pyutils.h", QString("Unhandled data type in conversion PyObject to QVariant: %1").arg(pyObject->ob_type->tp_name).toUtf8()); return QVariant(); } diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 0ba9f215..8cabf7b3 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -1,14 +1,7 @@ #include -#include "python/pynymealogginghandler.h" -#include "python/pything.h" -#include "python/pythingdiscoveryinfo.h" -#include "python/pythingsetupinfo.h" -#include "python/pyparam.h" -#include "python/pythingactioninfo.h" -#include "python/pythingpairinginfo.h" - #include "pythonintegrationplugin.h" +#include "python/pynymeamodule.h" #include "loggingcategories.h" @@ -22,82 +15,8 @@ #include PyThreadState* PythonIntegrationPlugin::s_mainThreadState = nullptr; -QThreadPool* PythonIntegrationPlugin::s_threadPool = nullptr; - QHash PythonIntegrationPlugin::s_plugins; - -PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) -{ - Q_UNUSED(self) - - PyObject *result = nullptr; - - if (!PyArg_ParseTuple(args, "O", &result)) { - qCWarning(dcPythonIntegrations()) << "Cannot fetch result from coroutine callback."; - return nullptr; - } - - PyObject *exceptionMethod = PyObject_GetAttrString(result, "exception"); - - PyObject *exception = PyObject_CallFunctionObjArgs(exceptionMethod, nullptr); - if (exception != Py_None) { - PyObject* repr = PyObject_Repr(exception); - PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); - const char *bytes = PyBytes_AS_STRING(str); - Py_XDECREF(repr); - Py_XDECREF(str); - -// PyObject *traceback = PyObject_CallMethodObjArgs(exception, "__traceback__", nullptr); - - qCWarning(dcPythonIntegrations()) << "Exception in plugin:" << bytes; - - PyErr_Clear(); - } - - Py_RETURN_NONE; -} - - -static PyMethodDef nymea_methods[] = -{ - {"task_done", PythonIntegrationPlugin::task_done, METH_VARARGS, "callback to clean up after asyc coroutines"}, - {nullptr, nullptr, 0, nullptr} // sentinel -}; - -static PyModuleDef nymea_module = -{ - PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; - "nymea", // const char* m_name; - "nymea module for python based integration plugins", // const char* m_doc; - -1, // Py_ssize_t m_size; - nymea_methods, // PyMethodDef *m_methods - nullptr, nullptr, nullptr, nullptr -}; - -PyMODINIT_FUNC PyInit_nymea(void) -{ - // Overrride stdout/stderr to use qDebug instead - PyObject* pyLog = PyModule_Create(&pyLog_module); - PySys_SetObject("stdout", pyLog); - PyObject* pyWarn = PyModule_Create(&pyWarn_module); - PySys_SetObject("stderr", pyWarn); - - - // Register nymea types - PyObject* m = PyModule_Create(&nymea_module); - registerNymeaLoggingHandler(m); - registerParamType(m); - registerThingType(m); - registerThingDescriptorType(m); - registerThingDiscoveryInfoType(m); - registerThingPairingInfoType(m); - registerThingSetupInfoType(m); - registerThingActionInfoType(m); - - return m; -} - PyObject* PythonIntegrationPlugin::pyConfiguration(PyObject* self, PyObject* /*args*/) { PythonIntegrationPlugin *plugin = s_plugins.key(self); @@ -274,17 +193,27 @@ PythonIntegrationPlugin::PythonIntegrationPlugin(QObject *parent) : IntegrationP PythonIntegrationPlugin::~PythonIntegrationPlugin() { + if (m_pluginModule) { + callPluginFunction("deinit"); + } + // Acquire GIL for this plugin's interpreter PyEval_RestoreThread(m_threadState); - // Cancel all the thread in here while (!m_runningTasks.isEmpty()) { - QFutureWatcher *watcher = m_runningTasks.values().first(); - watcher->cancel(); + QFutureWatcher *watcher = m_runningTasks.keys().first(); + QString function = m_runningTasks.value(watcher); + + Py_BEGIN_ALLOW_THREADS + qCDebug(dcPythonIntegrations()) << "Waiting for" << metadata().pluginName() << "to finish" << function; watcher->waitForFinished(); + Py_END_ALLOW_THREADS } - Py_XDECREF(s_plugins.take(this)); + s_plugins.take(this); + Py_XDECREF(m_pluginModule); + Py_XDECREF(m_logger); + Py_DECREF(m_nymeaModule); Py_EndInterpreter(m_threadState); @@ -301,33 +230,25 @@ void PythonIntegrationPlugin::initPython() PyEval_InitThreads(); // Store the main thread state and release the GIL s_mainThreadState = PyEval_SaveThread(); - - // Allocate a shared thread pool for the plugins - s_threadPool = new QThreadPool(); - qCDebug(dcPythonIntegrations()) << "Created a thread pool with a maximum of" << s_threadPool->maxThreadCount() << "threads for python plugins."; } void PythonIntegrationPlugin::deinitPython() { PyEval_RestoreThread(s_mainThreadState); - Py_FinalizeEx(); - - s_threadPool->deleteLater(); - s_threadPool = nullptr; } bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) { QFileInfo fi(scriptFile); - QFile metaData(fi.absolutePath() + "/" + fi.baseName() + ".json"); - if (!metaData.open(QFile::ReadOnly)) { - qCWarning(dcThingManager()) << "Error opening metadata file:" << metaData.fileName(); + QFile metaDataFile(fi.absolutePath() + "/" + fi.baseName() + ".json"); + if (!metaDataFile.open(QFile::ReadOnly)) { + qCWarning(dcThingManager()) << "Error opening metadata file:" << metaDataFile.fileName(); return false; } QJsonParseError error; - QJsonDocument jsonDoc = QJsonDocument::fromJson(metaData.readAll(), &error); + QJsonDocument jsonDoc = QJsonDocument::fromJson(metaDataFile.readAll(), &error); if (error.error != QJsonParseError::NoError) { qCWarning(dcThingManager()) << "Error parsing metadata file:" << error.errorString(); return false; @@ -350,19 +271,21 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) // Import nymea module into this interpreter m_nymeaModule = PyImport_ImportModule("nymea"); - // Set up import paths for the plugin + // Set up import path for the plugin directory PyObject* sysPath = PySys_GetObject("path"); PyObject* pluginImportPath = PyUnicode_FromString(fi.absolutePath().toUtf8()); PyList_Append(sysPath, pluginImportPath); Py_DECREF(pluginImportPath); + // Set up import path for the "modules" subdir in the plugin directory PyObject* pluginModulesImportPath = PyUnicode_FromString(QString("%1/modules/").arg(fi.absolutePath()).toUtf8()); PyList_Append(sysPath, pluginModulesImportPath); Py_DECREF(pluginModulesImportPath); - m_module = PyImport_ImportModule(fi.baseName().toUtf8()); + // Import the plugin + m_pluginModule = PyImport_ImportModule(fi.baseName().toUtf8()); - if (!m_module) { + if (!m_pluginModule) { qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath(); PyErr_Print(); PyErr_Clear(); @@ -371,34 +294,37 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) } qCDebug(dcThingManager()) << "Imported python plugin from" << fi.absoluteFilePath(); - s_plugins.insert(this, m_module); - - // We'll be using asyncio everywhere, so let's import it right away - // IMPORTANT: The asyncio module is a bit special in a sense that it actually shares - // stuff between interpreters. See https://docs.python.org/3/c-api/init.html#c.Py_NewInterpreter - // for the listed bugs and caveats. - // If we destroy the first interpreter that imports asyncio, things will become crashy as the - // interpreter will tear down the module, leaving shallow copies of the modules dict in other - // interpreters. So let's import the module down here after we're sure this module compiles and - // won't be unloaded any more. - // Note: This becomes a problem when we'll be supporting to unload plugins at runtime. - m_asyncio = PyImport_ImportModule("asyncio"); + s_plugins.insert(this, m_pluginModule); // Set up logger with appropriate logging category - PyNymeaLoggingHandler *logger = reinterpret_cast(_PyObject_New(&PyNymeaLoggingHandlerType)); QString category = metadata().pluginName(); category.replace(0, 1, category[0].toUpper()); - logger->category = static_cast(malloc(category.length() + 1)); - memset(logger->category, '0', category.length() +1); - strcpy(logger->category, category.toUtf8().data()); - PyModule_AddObject(m_module, "logger", reinterpret_cast(logger)); - + PyObject *args = Py_BuildValue("(s)", category.toUtf8().data()); + PyNymeaLoggingHandler *logger = reinterpret_cast(PyObject_CallObject((PyObject*)&PyNymeaLoggingHandlerType, args)); + Py_DECREF(args); + PyModule_AddObject(m_pluginModule, "logger", reinterpret_cast(logger)); + m_logger = (PyObject*)logger; // Export metadata ids into module exportIds(); - // Register config access methods - PyModule_AddFunctions(m_module, plugin_methods); + // Register plugin api methods (plugin params etc) + PyModule_AddFunctions(m_pluginModule, plugin_methods); + + // As python does not have an event loop by default and uses blocking code a lot, we'll + // call every plugin method in a threaded way to prevent blocking the core while still not + // forcing every plugin developer to deal with threading in the plugin. + // In oder to not create and destroy a thread for each plugin api call, we'll be using a + // thread pool. + // The maximum number of threads in a plugin will be amount of things it manages + 2. + // This would allow for e.g. running an event loop using init(), performing something on a thing + // and still allow the user to perform a discovery at the same time. On the other hand, this is + // strict enough to not encourage the plugin developer to block forever in ever api call but use + // proper task processing means (timers, event loops etc) instead. + // Plugins can still spawn more threads on their own if the need to but have to manage them on their own. + m_threadPool = new QThreadPool(this); + m_threadPool->setMaxThreadCount(2); + qCDebug(dcPythonIntegrations()) << "Created a thread pool with a maximum of" << m_threadPool->maxThreadCount() << "threads for python plugin" << metadata().pluginName(); PyEval_ReleaseThread(m_threadState); @@ -517,18 +443,24 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) PyObject *args = PyTuple_New(1); PyTuple_SetItem(args, 0, (PyObject*)pyThing); + Py_INCREF(pyThing); PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, args); Py_DECREF(args); pyInfo->info = info; + m_threadPool->setMaxThreadCount(m_threadPool->maxThreadCount() + 1); + qCDebug(dcPythonIntegrations()) << "Expanded thread pool for plugin" << metadata().pluginName() << "to" << m_threadPool->maxThreadCount(); + PyEval_ReleaseThread(m_threadState); connect(info->thing(), &Thing::destroyed, this, [=](){ PyEval_RestoreThread(m_threadState); pyThing->thing = nullptr; Py_DECREF(pyThing); + m_threadPool->setMaxThreadCount(m_threadPool->maxThreadCount() - 1); + qCDebug(dcPythonIntegrations()) << "Shrunk thread pool for plugin" << metadata().pluginName() << "to" << m_threadPool->maxThreadCount(); PyEval_ReleaseThread(m_threadState); }); connect(info, &ThingSetupInfo::destroyed, this, [=](){ @@ -549,12 +481,7 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) void PythonIntegrationPlugin::postSetupThing(Thing *thing) { PyThing* pyThing = m_things.value(thing); - Py_INCREF(pyThing); - - bool success = callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); - if (!success) { - Py_DECREF(pyThing); - } + callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); } void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) @@ -569,10 +496,13 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) PyEval_ReleaseThread(m_threadState); connect(info, &ThingActionInfo::destroyed, this, [=](){ + qCDebug(dcPythonIntegrations()) << "Info destroyed"; PyEval_RestoreThread(m_threadState); + qCDebug(dcPythonIntegrations()) << "Info destroyed2"; pyInfo->info = nullptr; Py_DECREF(pyInfo); PyEval_ReleaseThread(m_threadState); + qCDebug(dcPythonIntegrations()) << "Info destroyed3"; }); bool success = callPluginFunction("executeAction", reinterpret_cast(pyInfo)); @@ -584,12 +514,7 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) void PythonIntegrationPlugin::thingRemoved(Thing *thing) { PyThing *pyThing = m_things.value(thing); - Py_INCREF(pyThing); - - bool success = callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); - if (!success) { - Py_DECREF(pyThing); - } + callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); m_mutex.lock(); m_things.remove(thing); @@ -623,13 +548,13 @@ void PythonIntegrationPlugin::exportIds() QString pluginName = metadata().pluginName(); QString pluginId = metadata().pluginId().toString(); qCDebug(dcThingManager()) << "- Plugin:" << pluginName << pluginId; - PyModule_AddStringConstant(m_module, "pluginId", pluginId.toUtf8()); + PyModule_AddStringConstant(m_pluginModule, "pluginId", pluginId.toUtf8()); exportParamTypes(configurationDescription(), pluginName, "", "plugin"); foreach (const Vendor &vendor, supportedVendors()) { qCDebug(dcThingManager()) << "|- Vendor:" << vendor.name() << vendor.id().toString(); - PyModule_AddStringConstant(m_module, QString("%1VendorId").arg(vendor.name()).toUtf8(), vendor.id().toString().toUtf8()); + PyModule_AddStringConstant(m_pluginModule, QString("%1VendorId").arg(vendor.name()).toUtf8(), vendor.id().toString().toUtf8()); } foreach (const ThingClass &thingClass, supportedThings()) { @@ -642,7 +567,7 @@ void PythonIntegrationPlugin::exportThingClass(const ThingClass &thingClass) QString variableName = QString("%1ThingClassId").arg(thingClass.name()); qCDebug(dcThingManager()) << "|- ThingClass:" << variableName << thingClass.id().toString(); - PyModule_AddStringConstant(m_module, variableName.toUtf8(), thingClass.id().toString().toUtf8()); + PyModule_AddStringConstant(m_pluginModule, variableName.toUtf8(), thingClass.id().toString().toUtf8()); exportParamTypes(thingClass.paramTypes(), thingClass.name(), "", "thing"); exportParamTypes(thingClass.settingsTypes(), thingClass.name(), "", "settings"); @@ -659,7 +584,7 @@ void PythonIntegrationPlugin::exportParamTypes(const ParamTypes ¶mTypes, con foreach (const ParamType ¶mType, paramTypes) { QString variableName = QString("%1ParamTypeId").arg(thingClassName + typeName[0].toUpper() + typeName.right(typeName.length()-1) + typeClass + paramType.name()[0].toUpper() + paramType.name().right(paramType.name().length() -1 )); qCDebug(dcThingManager()) << " |- ParamType:" << variableName << paramType.id().toString(); - PyModule_AddStringConstant(m_module, variableName.toUtf8(), paramType.id().toString().toUtf8()); + PyModule_AddStringConstant(m_pluginModule, variableName.toUtf8(), paramType.id().toString().toUtf8()); } } @@ -668,7 +593,7 @@ void PythonIntegrationPlugin::exportStateTypes(const StateTypes &stateTypes, con foreach (const StateType &stateType, stateTypes) { QString variableName = QString("%1%2StateTypeId").arg(thingClassName, stateType.name()[0].toUpper() + stateType.name().right(stateType.name().length() - 1)); qCDebug(dcThingManager()) << " |- StateType:" << variableName << stateType.id().toString(); - PyModule_AddStringConstant(m_module, variableName.toUtf8(), stateType.id().toString().toUtf8()); + PyModule_AddStringConstant(m_pluginModule, variableName.toUtf8(), stateType.id().toString().toUtf8()); } } @@ -677,10 +602,9 @@ void PythonIntegrationPlugin::exportEventTypes(const EventTypes &eventTypes, con foreach (const EventType &eventType, eventTypes) { QString variableName = QString("%1%2EventTypeId").arg(thingClassName, eventType.name()[0].toUpper() + eventType.name().right(eventType.name().length() - 1)); qCDebug(dcThingManager()) << " |- EventType:" << variableName << eventType.id().toString(); - PyModule_AddStringConstant(m_module, variableName.toUtf8(), eventType.id().toString().toUtf8()); + PyModule_AddStringConstant(m_pluginModule, variableName.toUtf8(), eventType.id().toString().toUtf8()); exportParamTypes(eventType.paramTypes(), thingClassName, "Event", eventType.name()); } - } void PythonIntegrationPlugin::exportActionTypes(const ActionTypes &actionTypes, const QString &thingClassName) @@ -688,7 +612,7 @@ void PythonIntegrationPlugin::exportActionTypes(const ActionTypes &actionTypes, foreach (const ActionType &actionType, actionTypes) { QString variableName = QString("%1%2ActionTypeId").arg(thingClassName, actionType.name()[0].toUpper() + actionType.name().right(actionType.name().length() - 1)); qCDebug(dcThingManager()) << " |- ActionType:" << variableName << actionType.id().toString(); - PyModule_AddStringConstant(m_module, variableName.toUtf8(), actionType.id().toString().toUtf8()); + PyModule_AddStringConstant(m_pluginModule, variableName.toUtf8(), actionType.id().toString().toUtf8()); exportParamTypes(actionType.paramTypes(), thingClassName, "Action", actionType.name()); } } @@ -698,19 +622,17 @@ void PythonIntegrationPlugin::exportBrowserItemActionTypes(const ActionTypes &ac foreach (const ActionType &actionType, actionTypes) { QString variableName = QString("%1%2BrowserItemActionTypeId").arg(thingClassName, actionType.name()[0].toUpper() + actionType.name().right(actionType.name().length() - 1)); qCDebug(dcThingManager()) << " |- BrowserActionType:" << variableName << actionType.id().toString(); - PyModule_AddStringConstant(m_module, variableName.toUtf8(), actionType.id().toString().toUtf8()); + PyModule_AddStringConstant(m_pluginModule, variableName.toUtf8(), actionType.id().toString().toUtf8()); exportParamTypes(actionType.paramTypes(), thingClassName, "BrowserItemAction", actionType.name()); } - } - bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObject *param1, PyObject *param2, PyObject *param3) { PyEval_RestoreThread(m_threadState); qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << pluginName(); - PyObject *pluginFunction = PyObject_GetAttrString(m_module, function.toUtf8()); + PyObject *pluginFunction = PyObject_GetAttrString(m_pluginModule, function.toUtf8()); if(!pluginFunction || !PyCallable_Check(pluginFunction)) { PyErr_Clear(); Py_XDECREF(pluginFunction); @@ -719,84 +641,45 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje return false; } - - PyObject *pluginFunctionResult = PyObject_CallFunctionObjArgs(pluginFunction, param1, param2, param3, nullptr); - - Py_XDECREF(pluginFunction); - - if (PyErr_Occurred()) { - qCWarning(dcThingManager()) << "Error calling python method:" << function << "on plugin" << pluginName(); - PyErr_Print(); - PyErr_Clear(); - PyEval_ReleaseThread(m_threadState); - return false; - } - - if (QByteArray(pluginFunctionResult->ob_type->tp_name) != "coroutine") { - Py_DECREF(pluginFunctionResult); - PyEval_ReleaseThread(m_threadState); - return true; - } - - // Spawn a new event loop for the thread - QFuture future = QtConcurrent::run([this, pluginFunctionResult, function](){ - qCDebug(dcPythonIntegrations()) << "Spawning thread for" << function << "in plugin" << metadata().pluginName(); - - // Register this new thread in the interpreter - PyThreadState *thread = PyThreadState_New(m_threadState->interp); - - // Acquire GIL and make the new thread state the current one - PyEval_RestoreThread(thread); - - PyObject *new_event_loop = PyObject_GetAttrString(m_asyncio, "new_event_loop"); - PyObject *loop = PyObject_CallFunctionObjArgs(new_event_loop, nullptr); - - Py_DECREF(new_event_loop); - - PyObject *create_task = PyObject_GetAttrString(loop, "create_task"); - PyObject *task = PyObject_CallFunctionObjArgs(create_task, pluginFunctionResult, nullptr); - dumpError(); - - Py_DECREF(create_task); - Py_DECREF(pluginFunctionResult); - - PyObject *add_done_callback = PyObject_GetAttrString(task, "add_done_callback"); - dumpError(); - - PyObject *task_done = PyObject_GetAttrString(m_nymeaModule, "task_done"); - PyObject *coroutineResult = PyObject_CallFunctionObjArgs(add_done_callback, task_done, nullptr); - dumpError(); - - Py_DECREF(coroutineResult); - Py_DECREF(add_done_callback); - Py_DECREF(task_done); - - PyObject *run_until_complete = PyObject_GetAttrString(loop, "run_until_complete"); - - Py_DECREF(loop); - - PyObject *taskResult = PyObject_CallFunctionObjArgs(run_until_complete, task, nullptr); - dumpError(); - - Py_XDECREF(taskResult); - Py_DECREF(run_until_complete); - Py_DECREF(task); - - // Destroy the thread and release the GIL - PyThreadState_Clear(thread); - PyThreadState_DeleteCurrent(); - qCDebug(dcPythonIntegrations()) << "Thread for" << function << "in plugin" << metadata().pluginName() << "ended"; - }); + Py_XINCREF(param1); + Py_XINCREF(param2); + Py_XINCREF(param3); QFutureWatcher *watcher = new QFutureWatcher(this); - watcher->setFuture(future); - m_runningTasks.insert(watcher); - connect(watcher, &QFutureWatcher::finished, this, [this, watcher](){ + // Run the plugin function in the thread pool + QFuture future = QtConcurrent::run(m_threadPool, [=](){ + qCDebug(dcPythonIntegrations()) << "+++ Thread for" << function << "in plugin" << metadata().pluginName(); + + // Register this new thread in the interpreter + PyThreadState *threadState = PyThreadState_New(m_threadState->interp); + + // Acquire GIL and make the new thread state the current one + PyEval_RestoreThread(threadState); + + PyObject *pluginFunctionResult = PyObject_CallFunctionObjArgs(pluginFunction, param1, param2, param3, nullptr); + dumpError(); + + if (PyErr_Occurred()) { + qCWarning(dcThingManager()) << "Error calling python method:" << function << "on plugin" << pluginName(); + } + + Py_DECREF(pluginFunction); + Py_XDECREF(pluginFunctionResult); + Py_XDECREF(param1); + Py_XDECREF(param2); + Py_XDECREF(param3); + m_runningTasks.remove(watcher); - delete watcher; - }); + // Destroy the thread and release the GIL + PyThreadState_Clear(threadState); + PyEval_ReleaseThread(threadState); + PyThreadState_Delete(threadState); + qCDebug(dcPythonIntegrations()) << "--- Thread for" << function << "in plugin" << metadata().pluginName(); + }); + watcher->setFuture(future); + m_runningTasks.insert(watcher, function); PyEval_ReleaseThread(m_threadState); return true; diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 927cd314..733777da 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -47,10 +47,6 @@ public: static PyObject* pyAutoThingsAppeared(PyObject *self, PyObject* args); static PyObject* pyAutoThingDisappeared(PyObject *self, PyObject* args); -public: - // python callbacks - static PyObject* task_done(PyObject* self, PyObject* args); - private: void exportIds(); void exportThingClass(const ThingClass &thingClass); @@ -66,17 +62,22 @@ private: private: // The main thread state in which we create an interpreter per plugin static PyThreadState* s_mainThreadState; - static QThreadPool *s_threadPool; // A per plugin thread state and interpreter PyThreadState *m_threadState = nullptr; - // Modules imported into the interpreter - PyObject *m_nymeaModule; - PyObject *m_asyncio; + // A per plugin thread pool + QThreadPool *m_threadPool = nullptr; + // Running concurrent tasks in this plugins thread pool + QHash*, QString> m_runningTasks; + + // The nymea module we import into the interpreter + PyObject *m_nymeaModule = nullptr; // The imported plugin module (the plugin.py) - PyObject *m_module = nullptr; + PyObject *m_pluginModule = nullptr; + + PyObject *m_logger = nullptr; // A map of plugin instances to plugin python scripts/modules // Make sure to hold the GIL when accessing this. @@ -91,8 +92,6 @@ private: // Need to keep a copy of plugin params and sync that in a thread-safe manner ParamList m_pluginConfigCopy; - // Running concurrent tasks in this plugin - QSet*> m_runningTasks; }; #endif // PYTHONINTEGRATIONPLUGIN_H diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index a1afcd38..c7f488cc 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -21,6 +21,7 @@ RESOURCES += $$top_srcdir/icons.qrc \ HEADERS += nymeacore.h \ integrations/plugininfocache.h \ integrations/python/pynymealogginghandler.h \ + integrations/python/pynymeamodule.h \ integrations/python/pyparam.h \ integrations/python/pything.h \ integrations/python/pythingactioninfo.h \ diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 2107a51e..42eaa846 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -1,14 +1,17 @@ import nymea -import asyncio +import time #from fastdotcom import fast_com watchingAutoThings = False +loopRunning = False -async def init(): +def init(): logger.log("Python mock plugin init") + global loopRunning + loopRunning = True - while True: - await asyncio.sleep(5); + while loopRunning: + time.sleep(5); for thing in myThings(): if thing.thingClassId == pyMockThingClassId: logger.log("Emitting event 1 for", thing.name, "eventTypeId", pyMockEvent1EventTypeId) @@ -20,6 +23,13 @@ async def init(): thing.emitEvent(pyMockDiscoveryPairingEvent1EventTypeId, [nymea.Param(pyMockDiscoveryPairingEvent1EventParam1ParamTypeId, "Im an event")]) logger.log("Setting state 1 for", thing.name, "Old value is:", thing.stateValue(pyMockDiscoveryPairingState1StateTypeId)) thing.setStateValue(pyMockDiscoveryPairingState1StateTypeId, thing.stateValue(pyMockDiscoveryPairingState1StateTypeId) + 1) + logger.log("Bye bye") + + +def deinit(): + logger.log("shutting down") + global loopRunning + loopRunning = False def configValueChanged(paramTypeId, value): @@ -55,9 +65,9 @@ def startMonitoringAutoThings(): logger.log("Done start monitoring auto things") -async def discoverThings(info): +def discoverThings(info): logger.log("Discovery started for", info.thingClassId, "with result count:", info.params[0].value) - await asyncio.sleep(10) # Some delay for giving a feeling of a discovery + time.sleep(10) # Some delay for giving a feeling of a discovery # Add 2 new discovery results for i in range(0, info.params[0].value): info.addDescriptor(nymea.ThingDescriptor(pyMockDiscoveryPairingThingClassId, "Python mock thing %i" % i)) @@ -69,26 +79,26 @@ async def discoverThings(info): info.finish(nymea.ThingErrorNoError) -async def startPairing(info): +def startPairing(info): logger.log("startPairing for", info.thingName, info.thingId, info.params) info.finish(nymea.ThingErrorNoError, "Log in as user \"john\" with password \"smith\".") -async def confirmPairing(info, username, secret): +def confirmPairing(info, username, secret): logger.log("confirming pairing for", info.thingName, username, secret) - await asyncio.sleep(1) + time.sleep(1) if username == "john" and secret == "smith": info.finish(nymea.ThingErrorNoError) else: info.finish(nymea.ThingErrorAuthenticationFailure, "Error logging in here!") -async def setupThing(info): +def setupThing(info): logger.log("setupThing for", info.thing.name) info.finish(nymea.ThingErrorNoError) -async def postSetupThing(thing): +def postSetupThing(thing): logger.log("postSetupThing for", thing.name) thing.nameChangedHandler = lambda thing : logger.log("Thing name changed", thing.name) From 2543191639736001e1934f00a2fdd9d3d43353dc Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 20 Jul 2020 11:32:21 +0200 Subject: [PATCH 30/46] Add a todo file for stuff still missing in python plugins --- python-todos.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 python-todos.txt diff --git a/python-todos.txt b/python-todos.txt new file mode 100644 index 00000000..bb95d61c --- /dev/null +++ b/python-todos.txt @@ -0,0 +1,3 @@ + +* thing.settingChangedHandler missing +* pluginStorage missing From 828250b85acf86e3ca42312f12916879d42885be Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 20 Jul 2020 13:57:39 +0200 Subject: [PATCH 31/46] Add dependency on python3 --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 43764632..13688d65 100644 --- a/debian/control +++ b/debian/control @@ -12,6 +12,7 @@ Build-Depends: debhelper (>= 9.0.0), libnymea-mqtt-dev (>= 0.1.2), libnymea-networkmanager-dev (>= 0.4.0), libnymea-remoteproxyclient-dev, + libpython3-dev, libqt5websockets5-dev, libqt5bluetooth5, libqt5sql5-sqlite, From 75f4877f82a2a73d98d5b45f9a706a2e58cee5cd Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Tue, 21 Jul 2020 00:45:46 +0200 Subject: [PATCH 32/46] add settingChangedHandler, fix some issues, cleanup some warnings --- libnymea-core/integrations/python/pything.h | 30 ++++++++----- .../integrations/python/pythingdescriptor.h | 4 +- .../python/pythingdiscoveryinfo.h | 4 +- .../integrations/python/pythingpairinginfo.h | 4 +- libnymea-core/integrations/python/pyutils.h | 1 - .../integrations/pythonintegrationplugin.cpp | 42 ++++--------------- .../integrations/pythonintegrationplugin.h | 1 - plugins/pymock/integrationpluginpymock.py | 10 +++++ python-todos.txt | 1 - 9 files changed, 45 insertions(+), 52 deletions(-) diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 28e9092b..94325dd6 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -45,6 +45,7 @@ typedef struct _thing { PyObject *pyNameChangedHandler = nullptr; PyObject *pySettingChangedHandler = nullptr; PyObject *pyStates = nullptr; // A copy of the things states + PyThreadState *threadState = nullptr; // The python threadstate this thing belongs to } PyThing; @@ -58,9 +59,10 @@ static PyObject* PyThing_new(PyTypeObject *type, PyObject */*args*/, PyObject */ return (PyObject*)self; } -static void PyThing_setThing(PyThing *self, Thing *thing) +static void PyThing_setThing(PyThing *self, Thing *thing, PyThreadState *threadState) { self->thing = thing; + self->threadState = threadState; // Creating a copy because we cannot access the actual thing from the python thread self->thingClass = new ThingClass(thing->thingClass()); @@ -87,28 +89,36 @@ static void PyThing_setThing(PyThing *self, Thing *thing) // Those lambdas Will be executed in the main thread context. This means we // can access self->thing, but need to hold the GIL for interacting with python QObject::connect(thing, &Thing::nameChanged, [=](){ - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(self->threadState); Py_XDECREF(self->pyName); self->pyName = PyUnicode_FromString(self->thing->name().toUtf8().data()); if (self->pyNameChangedHandler) { - PyObject_CallFunctionObjArgs(self->pyNameChangedHandler, self, nullptr); + PyObject *ret = PyObject_CallFunctionObjArgs(self->pyNameChangedHandler, self, nullptr); + if (PyErr_Occurred()) { + PyErr_Print(); + } + Py_XDECREF(ret); } - PyGILState_Release(s); + PyEval_ReleaseThread(self->threadState); }); QObject::connect(thing, &Thing::settingChanged, [=](const ParamTypeId ¶mTypeId, const QVariant &value){ - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(self->threadState); Py_XDECREF(self->pySettings); self->pySettings = PyParams_FromParamList(self->thing->settings()); if (self->pySettingChangedHandler) { - PyObject_CallFunctionObjArgs(self->pySettingChangedHandler, self, PyUnicode_FromString(paramTypeId.toString().toUtf8().data()), QVariantToPyObject(value), nullptr); + PyObject * ret = PyObject_CallFunctionObjArgs(self->pySettingChangedHandler, self, PyUnicode_FromString(paramTypeId.toString().toUtf8().data()), QVariantToPyObject(value), nullptr); + if (PyErr_Occurred()) { + PyErr_Print(); + } + Py_XDECREF(ret); } - PyGILState_Release(s); + PyEval_ReleaseThread(self->threadState); }); QObject::connect(thing, &Thing::stateValueChanged, [=](const StateTypeId &stateTypeId, const QVariant &value){ - PyGILState_STATE s = PyGILState_Ensure(); + PyEval_RestoreThread(self->threadState); for (int i = 0; i < PyList_Size(self->pyStates); i++) { PyObject *pyState = PyList_GetItem(self->pyStates, i); PyObject *pyStateTypeId = PyDict_GetItemString(pyState, "stateTypeId"); @@ -121,7 +131,7 @@ static void PyThing_setThing(PyThing *self, Thing *thing) break; } } - PyGILState_Release(s); + PyEval_ReleaseThread(self->threadState); }); } @@ -342,6 +352,7 @@ static PyMethodDef PyThing_methods[] = { static PyMemberDef PyThing_members[] = { {"params", T_OBJECT_EX, offsetof(PyThing, pyParams), READONLY, "Thing params"}, {"nameChangedHandler", T_OBJECT_EX, offsetof(PyThing, pyNameChangedHandler), 0, "Set a callback for when the thing name changes"}, + {"settingChangedHandler", T_OBJECT_EX, offsetof(PyThing, pySettingChangedHandler), 0, "Set a callback for when a thing setting changes"}, {nullptr, 0, 0, 0, nullptr} /* Sentinel */ }; @@ -409,7 +420,6 @@ static void registerThingType(PyObject *module) for (int i = 0; i < thingErrorEnum.keyCount(); i++) { PyModule_AddObject(module, thingErrorEnum.key(i), PyLong_FromLong(thingErrorEnum.value(i))); } - } diff --git a/libnymea-core/integrations/python/pythingdescriptor.h b/libnymea-core/integrations/python/pythingdescriptor.h index 6ebc0e06..f1d0ba6e 100644 --- a/libnymea-core/integrations/python/pythingdescriptor.h +++ b/libnymea-core/integrations/python/pythingdescriptor.h @@ -36,7 +36,7 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj static char *kwlist[] = {"thingClassId", "name", "description", "thingId", "params", nullptr}; PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr, *thingId = nullptr, *params = nullptr; - qWarning() << "++++ PyThingDescriptor"; + qCDebug(dcPythonIntegrations()) << "+++ PyThingDescriptor"; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOOO", kwlist, &thingClassId, &name, &description, &thingId, ¶ms)) return -1; @@ -65,7 +65,7 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj static void PyThingDescriptor_dealloc(PyThingDescriptor * self) { - qWarning() << "---- PyThingDescriptor"; + qCDebug(dcPythonIntegrations()) << "--- PyThingDescriptor"; Py_XDECREF(self->pyThingClassId); Py_XDECREF(self->pyName); Py_XDECREF(self->pyDescription); diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index 9b8e07b7..cfacfe53 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -49,7 +49,7 @@ static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/ if (self == NULL) { return nullptr; } - qWarning() << "++++ PyThingDiscoveryInfo"; + qCDebug(dcPythonIntegrations()) << "+++ PyThingDiscoveryInfo"; return (PyObject*)self; } @@ -62,7 +62,7 @@ void PyThingDiscoveryInfo_setInfo(PyThingDiscoveryInfo *self, ThingDiscoveryInfo static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) { - qWarning() << "---- PyThingDiscoveryInfo"; + qCDebug(dcPythonIntegrations()) << "--- PyThingDiscoveryInfo"; Py_DECREF(self->pyThingClassId); Py_DECREF(self->pyParams); Py_TYPE(self)->tp_free(self); diff --git a/libnymea-core/integrations/python/pythingpairinginfo.h b/libnymea-core/integrations/python/pythingpairinginfo.h index 8dce6565..6194a4ea 100644 --- a/libnymea-core/integrations/python/pythingpairinginfo.h +++ b/libnymea-core/integrations/python/pythingpairinginfo.h @@ -60,7 +60,7 @@ static PyMemberDef PyThingPairingInfo_members[] = { static int PyThingPairingInfo_init(PyThingPairingInfo */*self*/, PyObject */*args*/, PyObject */*kwds*/) { - qWarning() << "++++ ThingPairingInfo"; + qCDebug(dcPythonIntegrations()) << "+++ ThingPairingInfo"; return 0; } @@ -77,7 +77,7 @@ void PyThingPairingInfo_setInfo(PyThingPairingInfo *self, ThingPairingInfo *info static void PyThingPairingInfo_dealloc(PyThingPairingInfo * self) { - qWarning() << "---- ThingPairingInfo"; + qCDebug(dcPythonIntegrations()) << "--- ThingPairingInfo"; Py_XDECREF(self->pyTransactionId); Py_XDECREF(self->pyThingClassId); Py_XDECREF(self->pyThingId); diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index 2e00da05..65f653ff 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -69,7 +69,6 @@ QVariant PyObjectToQVariant(PyObject *pyObject) return QVariant(); } - // Write to stdout PyObject* pyLog_write(PyObject* /*self*/, PyObject* args) { diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 8cabf7b3..c588eeb8 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -108,7 +108,6 @@ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject PyObject *iter = PyObject_GetIter(pyDescriptors); if (!iter) { - Py_DECREF(pyDescriptors); qCWarning(dcThingManager()) << "Error parsing args. Not a param list"; return nullptr; } @@ -124,6 +123,7 @@ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject if (next->ob_type != &PyThingDescriptorType) { PyErr_SetString(PyExc_ValueError, "Invalid argument. Not a ThingDescriptor."); + Py_DECREF(next); continue; } PyThingDescriptor *pyDescriptor = (PyThingDescriptor*)next; @@ -155,8 +155,6 @@ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject QMetaObject::invokeMethod(plugin, "autoThingsAppeared", Qt::QueuedConnection, Q_ARG(ThingDescriptors, descriptors)); Py_DECREF(iter); - Py_DECREF(pyDescriptors); - Py_RETURN_NONE; } @@ -432,13 +430,15 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) { PyEval_RestoreThread(m_threadState); + Thing *thing = info->thing(); + PyThing *pyThing = nullptr; - if (m_things.contains(info->thing())) { - pyThing = m_things.value(info->thing()); + if (m_things.contains(thing)) { + pyThing = m_things.value(thing); } else { pyThing = (PyThing*)PyObject_CallObject((PyObject*)&PyThingType, NULL); - PyThing_setThing(pyThing, info->thing()); - m_things.insert(info->thing(), pyThing); + PyThing_setThing(pyThing, thing, m_threadState); + m_things.insert(thing, pyThing); } PyObject *args = PyTuple_New(1); @@ -457,6 +457,7 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) connect(info->thing(), &Thing::destroyed, this, [=](){ PyEval_RestoreThread(m_threadState); + m_things.remove(thing); pyThing->thing = nullptr; Py_DECREF(pyThing); m_threadPool->setMaxThreadCount(m_threadPool->maxThreadCount() - 1); @@ -515,31 +516,6 @@ void PythonIntegrationPlugin::thingRemoved(Thing *thing) { PyThing *pyThing = m_things.value(thing); callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); - - m_mutex.lock(); - m_things.remove(thing); - m_mutex.unlock(); -} - -void PythonIntegrationPlugin::dumpError() -{ - if (!PyErr_Occurred()) { - return; - } - - PyObject *ptype, *pvalue, *ptraceback; - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - if (pvalue) { - PyObject *pstr = PyObject_Str(pvalue); - if (pstr) { - const char* err_msg = PyUnicode_AsUTF8(pstr); - if (pstr) { - qCWarning(dcThingManager()) << QString(err_msg); - } - - } - PyErr_Restore(ptype, pvalue, ptraceback); - } } void PythonIntegrationPlugin::exportIds() @@ -658,10 +634,10 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje PyEval_RestoreThread(threadState); PyObject *pluginFunctionResult = PyObject_CallFunctionObjArgs(pluginFunction, param1, param2, param3, nullptr); - dumpError(); if (PyErr_Occurred()) { qCWarning(dcThingManager()) << "Error calling python method:" << function << "on plugin" << pluginName(); + PyErr_Print(); } Py_DECREF(pluginFunction); diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 733777da..1539202f 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -39,7 +39,6 @@ public: void thingRemoved(Thing *thing) override; - static void dumpError(); static PyObject* pyConfiguration(PyObject* self, PyObject* args); static PyObject* pyConfigValue(PyObject* self, PyObject* args); static PyObject* pySetConfigValue(PyObject* self, PyObject* args); diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 42eaa846..31ef8c7c 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -96,6 +96,8 @@ def confirmPairing(info, username, secret): def setupThing(info): logger.log("setupThing for", info.thing.name) info.finish(nymea.ThingErrorNoError) + info.thing.nameChangedHandler = thingNameChanged + info.thing.settingChangedHandler = thingSettingChanged def postSetupThing(thing): @@ -123,6 +125,14 @@ def autoThings(): return autoThings +def thingNameChanged(thing, name): + logger.log("Thing name changed:", thing.name) + + +def thingSettingChanged(thing, paramTypeId, value): + logger.log("Thing setting changed:", thing.name, paramTypeId, value) + + # Intentionally commented out to also have a test case for unimplmented functions # def thingRemoved(thing): # logger.log("thingRemoved for", thing.name) diff --git a/python-todos.txt b/python-todos.txt index bb95d61c..59fce1f5 100644 --- a/python-todos.txt +++ b/python-todos.txt @@ -1,3 +1,2 @@ -* thing.settingChangedHandler missing * pluginStorage missing From be01d4c802f494d62662ba9693c6f5b8c7e7227d Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Thu, 23 Jul 2020 17:29:17 +0200 Subject: [PATCH 33/46] Restrict import paths to not use random python libs in the system --- .../integrations/pythonintegrationplugin.cpp | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index c588eeb8..b559f3fb 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -270,15 +270,27 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) m_nymeaModule = PyImport_ImportModule("nymea"); // Set up import path for the plugin directory + // We intentionally strip site-packages and dist-packages because + // that's too unpredictive in distribution. Instead all dependencies + // should be installed into the plugins "modules" subdir. PyObject* sysPath = PySys_GetObject("path"); - PyObject* pluginImportPath = PyUnicode_FromString(fi.absolutePath().toUtf8()); - PyList_Append(sysPath, pluginImportPath); - Py_DECREF(pluginImportPath); + QStringList importPaths; + for (int i = 0; i < PyList_Size(sysPath); i++) { + QString path = QString::fromUtf8(PyUnicode_AsUTF8(PyList_GetItem(sysPath, i))); + if (!path.contains("site-packages") && !path.contains("dist-packages")) { + importPaths.append(path); + } + } + importPaths.append(fi.absolutePath()); + importPaths.append(QString("%1/modules/").arg(fi.absolutePath())); - // Set up import path for the "modules" subdir in the plugin directory - PyObject* pluginModulesImportPath = PyUnicode_FromString(QString("%1/modules/").arg(fi.absolutePath()).toUtf8()); - PyList_Append(sysPath, pluginModulesImportPath); - Py_DECREF(pluginModulesImportPath); + PyObject* pluginPaths = PyList_New(importPaths.length()); + for (int i = 0; i < importPaths.length(); i++) { + const QString &path = importPaths.at(i); + PyObject *pyPath = PyUnicode_FromString(path.toUtf8()); + PyList_SetItem(pluginPaths, i, pyPath); + } + PySys_SetObject("path", pluginPaths); // Import the plugin m_pluginModule = PyImport_ImportModule(fi.baseName().toUtf8()); From fdbdb02c16aa7a24248f367d0d172c1860d5a1f4 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Fri, 24 Jul 2020 00:09:43 +0200 Subject: [PATCH 34/46] Make it compatible with python 3.5 --- .../python/pynymealogginghandler.h | 28 +++------- libnymea-core/integrations/python/pyparam.h | 25 ++------- libnymea-core/integrations/python/pything.h | 53 +++--------------- .../integrations/python/pythingactioninfo.h | 54 +++---------------- .../integrations/python/pythingdescriptor.h | 20 ++----- .../python/pythingdiscoveryinfo.h | 51 +++--------------- .../integrations/python/pythingpairinginfo.h | 54 +++---------------- .../integrations/python/pythingsetupinfo.h | 52 +++--------------- .../integrations/pythonintegrationplugin.cpp | 2 +- .../thingmanagerimplementation.cpp | 2 +- libnymea-core/libnymea-core.pro | 10 +++- 11 files changed, 59 insertions(+), 292 deletions(-) diff --git a/libnymea-core/integrations/python/pynymealogginghandler.h b/libnymea-core/integrations/python/pynymealogginghandler.h index 1a5b3f5f..a31d8213 100644 --- a/libnymea-core/integrations/python/pynymealogginghandler.h +++ b/libnymea-core/integrations/python/pynymealogginghandler.h @@ -12,6 +12,7 @@ Q_DECLARE_LOGGING_CATEGORY(dcPythonIntegrations) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" typedef struct { PyObject_HEAD @@ -82,24 +83,7 @@ static PyTypeObject PyNymeaLoggingHandlerType = { "nymea.NymeaLoggingHandler", /* tp_name */ sizeof(PyNymeaLoggingHandler), /* tp_basicsize */ 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Logging handler for nymea", /* tp_doc */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + (destructor)PyNymeaLoggingHandler_dealloc,/* tp_dealloc */ }; @@ -107,11 +91,11 @@ static void registerNymeaLoggingHandler(PyObject *module) { PyNymeaLoggingHandlerType.tp_new = PyType_GenericNew; - PyNymeaLoggingHandlerType.tp_dealloc = reinterpret_cast(PyNymeaLoggingHandler_dealloc); - PyNymeaLoggingHandlerType.tp_flags = Py_TPFLAGS_DEFAULT; - PyNymeaLoggingHandlerType.tp_doc = "NymeaLoggingHandler class"; - PyNymeaLoggingHandlerType.tp_methods = PyNymeaLoggingHandler_methods; PyNymeaLoggingHandlerType.tp_init = reinterpret_cast(PyNymeaLoggingHandler_init); + PyNymeaLoggingHandlerType.tp_flags = Py_TPFLAGS_DEFAULT; + PyNymeaLoggingHandlerType.tp_methods = PyNymeaLoggingHandler_methods; + PyNymeaLoggingHandlerType.tp_doc = "Logging handler for nymea."; + if (PyType_Ready(&PyNymeaLoggingHandlerType) == 0) { PyModule_AddObject(module, "NymeaLoggingHandler", (PyObject *)&PyNymeaLoggingHandlerType); } diff --git a/libnymea-core/integrations/python/pyparam.h b/libnymea-core/integrations/python/pyparam.h index 87ae8063..8278c343 100644 --- a/libnymea-core/integrations/python/pyparam.h +++ b/libnymea-core/integrations/python/pyparam.h @@ -14,6 +14,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" typedef struct _pyparam { @@ -66,23 +67,7 @@ static PyTypeObject PyParamType = { sizeof(PyParam), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)PyParam_dealloc,/* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + }; static PyParam* PyParam_fromParam(const Param ¶m) @@ -147,13 +132,11 @@ static ParamList PyParams_ToParamList(PyObject *pyParams) static void registerParamType(PyObject *module) { PyParamType.tp_new = PyType_GenericNew; - PyParamType.tp_basicsize = sizeof(PyParam); - PyParamType.tp_dealloc=(destructor) PyParam_dealloc; + PyParamType.tp_init = reinterpret_cast(PyParam_init); PyParamType.tp_flags = Py_TPFLAGS_DEFAULT; - PyParamType.tp_doc = "Param class"; PyParamType.tp_methods = PyParam_methods; PyParamType.tp_members = PyParam_members; - PyParamType.tp_init = reinterpret_cast(PyParam_init); + PyParamType.tp_doc = "Param class"; if (PyType_Ready(&PyParamType) < 0) { return; diff --git a/libnymea-core/integrations/python/pything.h b/libnymea-core/integrations/python/pything.h index 94325dd6..1678e993 100644 --- a/libnymea-core/integrations/python/pything.h +++ b/libnymea-core/integrations/python/pything.h @@ -16,6 +16,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* Note: * When using this, make sure to call PyThing_setThing() while holding the GIL to initialize @@ -362,55 +363,17 @@ static PyTypeObject PyThingType = { sizeof(PyThing), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)PyThing_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Thing", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyThing_methods, /* tp_methods */ - PyThing_members, /* tp_members */ - PyThing_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - (newfunc)PyThing_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ - 0, /* tp_vectorcall */ - 0, /* tp_print DEPRECATED*/ }; static void registerThingType(PyObject *module) { + PyThingType.tp_new = (newfunc)PyThing_new; + PyThingType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingType.tp_methods = PyThing_methods; + PyThingType.tp_members = PyThing_members; + PyThingType.tp_getset = PyThing_getset; + PyThingType.tp_doc = "The Thing class represents a thing in nymea."; + if (PyType_Ready(&PyThingType) < 0) { return; } diff --git a/libnymea-core/integrations/python/pythingactioninfo.h b/libnymea-core/integrations/python/pythingactioninfo.h index d29c4e4e..7c62a421 100644 --- a/libnymea-core/integrations/python/pythingactioninfo.h +++ b/libnymea-core/integrations/python/pythingactioninfo.h @@ -11,6 +11,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* Note: * @@ -103,55 +104,16 @@ static PyTypeObject PyThingActionInfoType = { sizeof(PyThingActionInfo), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)PyThingActionInfo_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "ThingActionInfo", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyThingActionInfo_methods, /* tp_methods */ - PyThingActionInfo_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - (newfunc)PyThingActionInfo_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ - 0, /* tp_vectorcall */ - 0, /* tp_print DEPRECATED*/ }; static void registerThingActionInfoType(PyObject *module) { + PyThingActionInfoType.tp_new = (newfunc)PyThingActionInfo_new; + PyThingActionInfoType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingActionInfoType.tp_methods = PyThingActionInfo_methods; + PyThingActionInfoType.tp_members = PyThingActionInfo_members; + PyThingActionInfoType.tp_doc = "The ThingActionInfo is used to dispatch actions to things"; + if (PyType_Ready(&PyThingActionInfoType) < 0) { return; } @@ -159,8 +121,6 @@ static void registerThingActionInfoType(PyObject *module) } - - #pragma GCC diagnostic pop #endif // PYTHINGACTIONINFO_H diff --git a/libnymea-core/integrations/python/pythingdescriptor.h b/libnymea-core/integrations/python/pythingdescriptor.h index f1d0ba6e..e27d0440 100644 --- a/libnymea-core/integrations/python/pythingdescriptor.h +++ b/libnymea-core/integrations/python/pythingdescriptor.h @@ -10,6 +10,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" typedef struct { @@ -80,23 +81,6 @@ static PyTypeObject PyThingDescriptorType = { sizeof(PyThingDescriptor), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)PyThingDescriptor_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "ThingDescriptor", /* tp_doc */ - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 }; @@ -106,6 +90,8 @@ static void registerThingDescriptorType(PyObject *module) PyThingDescriptorType.tp_new = PyType_GenericNew; PyThingDescriptorType.tp_members = PyThingDescriptor_members; PyThingDescriptorType.tp_init = reinterpret_cast(PyThingDescriptor_init); + PyThingDescriptorType.tp_doc = "ThingDescriptors are used to inform the system about things that may be added."; + PyThingDescriptorType.tp_flags = Py_TPFLAGS_DEFAULT; if (PyType_Ready(&PyThingDescriptorType) < 0) { return; diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index cfacfe53..2b6a4e56 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -17,6 +17,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* Note: @@ -148,57 +149,17 @@ static PyTypeObject PyThingDiscoveryInfoType = { sizeof(PyThingDiscoveryInfo), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)PyThingDiscoveryInfo_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "ThingDiscoveryInfo", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyThingDiscoveryInfo_methods, /* tp_methods */ - PyThingDiscoveryInfo_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - (newfunc)PyThingDiscoveryInfo_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ - 0, /* tp_vectorcall */ - 0, /* tp_print DEPRECATED*/ }; static void registerThingDiscoveryInfoType(PyObject *module) { + PyThingDiscoveryInfoType.tp_new = (newfunc)PyThingDiscoveryInfo_new; + PyThingDiscoveryInfoType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingDiscoveryInfoType.tp_methods = PyThingDiscoveryInfo_methods; + PyThingDiscoveryInfoType.tp_members = PyThingDiscoveryInfo_members; + PyThingDiscoveryInfoType.tp_doc = "The ThingDiscoveryInfo is used to perform discoveries of things."; if (PyType_Ready(&PyThingDiscoveryInfoType) < 0) { return; diff --git a/libnymea-core/integrations/python/pythingpairinginfo.h b/libnymea-core/integrations/python/pythingpairinginfo.h index 6194a4ea..7a9a8071 100644 --- a/libnymea-core/integrations/python/pythingpairinginfo.h +++ b/libnymea-core/integrations/python/pythingpairinginfo.h @@ -16,6 +16,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* Note: * When using this, make sure to call PyThingPairingInfo_setInfo() while holding the GIL to initialize @@ -122,57 +123,16 @@ static PyTypeObject PyThingPairingInfoType = { sizeof(PyThingPairingInfo), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)PyThingPairingInfo_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "ThingPairingInfo", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyThingPairingInfo_methods, /* tp_methods */ - PyThingPairingInfo_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)PyThingPairingInfo_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ - 0, /* tp_vectorcall */ - 0, /* tp_print DEPRECATED*/ }; - - static void registerThingPairingInfoType(PyObject *module) { + PyThingPairingInfoType.tp_new = PyType_GenericNew; + PyThingPairingInfoType.tp_init = (initproc)PyThingPairingInfo_init; + PyThingPairingInfoType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingPairingInfoType.tp_methods = PyThingPairingInfo_methods; + PyThingPairingInfoType.tp_members = PyThingPairingInfo_members; + PyThingPairingInfoType.tp_doc = "The ThingPairingInfo is used to aithenticate with a thing."; if (PyType_Ready(&PyThingPairingInfoType) < 0) { return; diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index 83199546..bad30f43 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -11,6 +11,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" #pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* Note: * @@ -100,55 +101,16 @@ static PyTypeObject PyThingSetupInfoType = { sizeof(PyThingSetupInfo), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)PyThingSetupInfo_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "ThingSetupInfo", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyThingSetupInfo_methods, /* tp_methods */ - PyThingSetupInfo_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - (newfunc)PyThingSetupInfo_new, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - 0, /* tp_finalize */ - 0, /* tp_vectorcall */ - 0, /* tp_print DEPRECATED*/ }; static void registerThingSetupInfoType(PyObject *module) { + PyThingSetupInfoType.tp_new = (newfunc)PyThingSetupInfo_new; + PyThingSetupInfoType.tp_flags = Py_TPFLAGS_DEFAULT; + PyThingSetupInfoType.tp_methods = PyThingSetupInfo_methods; + PyThingSetupInfoType.tp_members = PyThingSetupInfo_members; + PyThingSetupInfoType.tp_doc = "The ThingSetupInfo is used to set up a thing."; + if (PyType_Ready(&PyThingSetupInfoType) < 0) { return; } diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index b559f3fb..abe434e0 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -233,7 +233,7 @@ void PythonIntegrationPlugin::initPython() void PythonIntegrationPlugin::deinitPython() { PyEval_RestoreThread(s_mainThreadState); - Py_FinalizeEx(); + Py_Finalize(); } bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index e21e99f1..bd4b5a65 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -1267,7 +1267,7 @@ void ThingManagerImplementation::loadPlugins() delete p; } #else - qCWarning(dcThingManager()) << "Not loading JS plugin as JS plugin support is not included in this nymea instance." + qCWarning(dcThingManager()) << "Not loading JS plugin as JS plugin support is not included in this nymea instance."; #endif } else if (entry.startsWith("integrationplugin") && entry.endsWith(".py")) { PythonIntegrationPlugin *p = new PythonIntegrationPlugin(this); diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index c7f488cc..fc380368 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -8,7 +8,15 @@ INCLUDEPATH += $$top_srcdir/libnymea $$top_builddir LIBS += -L$$top_builddir/libnymea/ -lnymea -lssl -lcrypto CONFIG += link_pkgconfig -PKGCONFIG += nymea-mqtt nymea-networkmanager python3-embed +PKGCONFIG += nymea-mqtt nymea-networkmanager + +packagesExist(python3-embed) { + PKGCONFIG += python3-embed +} else:packagesExist(python-3.5) { + PKGCONFIG += python-3.5 +} else { + error("Python development package not found.") +} target.path = $$[QT_INSTALL_LIBS] INSTALLS += target From d4889b5b7855a28839d122d7c2a0db6f51150933 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sat, 29 Aug 2020 15:47:16 +0200 Subject: [PATCH 35/46] Make it build with 0.23 --- libnymea-core/integrations/python/pyutils.h | 9 +++++---- libnymea-core/integrations/pythonintegrationplugin.cpp | 2 ++ libnymea/loggingcategories.cpp | 1 - libnymea/loggingcategories.h | 1 - 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index 65f653ff..f18bbd19 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -3,10 +3,11 @@ #include -#include "loggingcategories.h" - +#include #include +Q_DECLARE_LOGGING_CATEGORY(dcPythonIntegrations) + /* Returns a new reference to PyObject*. */ PyObject *QVariantToPyObject(const QVariant &value) { @@ -76,7 +77,7 @@ PyObject* pyLog_write(PyObject* /*self*/, PyObject* args) if (!PyArg_ParseTuple(args, "s", &what)) return nullptr; if (!QByteArray(what).trimmed().isEmpty()) { - qCDebug(dcThingManager()) << what; + qCDebug(dcPythonIntegrations()) << what; } Py_RETURN_NONE; } @@ -111,7 +112,7 @@ PyObject* pyWarn_write(PyObject* /*self*/, PyObject* args) if (!PyArg_ParseTuple(args, "s", &what)) return nullptr; if (!QByteArray(what).trimmed().isEmpty()) { - qCWarning(dcThingManager()) << what; + qCWarning(dcPythonIntegrations()) << what; } Py_RETURN_NONE; } diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index abe434e0..6b09d409 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -14,6 +14,8 @@ #include #include +NYMEA_LOGGING_CATEGORY(dcPythonIntegrations, "PythonIntegrations") + PyThreadState* PythonIntegrationPlugin::s_mainThreadState = nullptr; QHash PythonIntegrationPlugin::s_plugins; diff --git a/libnymea/loggingcategories.cpp b/libnymea/loggingcategories.cpp index d5d4cfd0..4d5070c7 100644 --- a/libnymea/loggingcategories.cpp +++ b/libnymea/loggingcategories.cpp @@ -75,7 +75,6 @@ NYMEA_LOGGING_CATEGORY(dcBluetoothServerTraffic, "BluetoothServerTraffic") NYMEA_LOGGING_CATEGORY(dcMqtt, "Mqtt") NYMEA_LOGGING_CATEGORY(dcTranslations, "Translations") NYMEA_LOGGING_CATEGORY(dcI2C, "I2C") -NYMEA_LOGGING_CATEGORY(dcPythonIntegrations, "PythonIntegrations") static QFile s_logFile; diff --git a/libnymea/loggingcategories.h b/libnymea/loggingcategories.h index 68557d79..dfb88212 100644 --- a/libnymea/loggingcategories.h +++ b/libnymea/loggingcategories.h @@ -91,7 +91,6 @@ Q_DECLARE_LOGGING_CATEGORY(dcCoap) Q_DECLARE_LOGGING_CATEGORY(dcI2C) Q_DECLARE_LOGGING_CATEGORY(dcIntegrations) Q_DECLARE_LOGGING_CATEGORY(dcJsIntegrations) -Q_DECLARE_LOGGING_CATEGORY(dcPythonIntegrations) /* From 16c4fa334591404dac77c7c690819587f2cd1d43 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 31 Aug 2020 19:29:35 +0200 Subject: [PATCH 36/46] fix issues in teardown. --- .../python/pynymealogginghandler.h | 3 +- libnymea-core/integrations/python/pyparam.h | 1 - .../integrations/pythonintegrationplugin.cpp | 34 ++++++++++++++----- .../integrations/pythonintegrationplugin.h | 2 -- tests/auto/integrations/testintegrations.cpp | 1 + tests/testlib/nymeatestbase.cpp | 2 ++ 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/libnymea-core/integrations/python/pynymealogginghandler.h b/libnymea-core/integrations/python/pynymealogginghandler.h index a31d8213..2bcce1d4 100644 --- a/libnymea-core/integrations/python/pynymealogginghandler.h +++ b/libnymea-core/integrations/python/pynymealogginghandler.h @@ -21,6 +21,7 @@ typedef struct { static int PyNymeaLoggingHandler_init(PyNymeaLoggingHandler *self, PyObject *args, PyObject */*kwds*/) { + qCDebug(dcPythonIntegrations()) << "+++ PyNymeaLoggingHandler"; char *category = nullptr; if (!PyArg_ParseTuple(args, "s", &category)) { qCWarning(dcPythonIntegrations()) << "PyNymeaLoggingHandler: Error parsing parameters"; @@ -35,11 +36,11 @@ static int PyNymeaLoggingHandler_init(PyNymeaLoggingHandler *self, PyObject *arg static void PyNymeaLoggingHandler_dealloc(PyNymeaLoggingHandler * self) { + qCDebug(dcPythonIntegrations()) << "--- PyNymeaLoggingHandler"; free(self->category); Py_TYPE(self)->tp_free(self); } - static PyObject * PyNymeaLoggingHandler_log(PyNymeaLoggingHandler* self, PyObject* args) { QStringList strings; diff --git a/libnymea-core/integrations/python/pyparam.h b/libnymea-core/integrations/python/pyparam.h index 8278c343..ac37583e 100644 --- a/libnymea-core/integrations/python/pyparam.h +++ b/libnymea-core/integrations/python/pyparam.h @@ -67,7 +67,6 @@ static PyTypeObject PyParamType = { sizeof(PyParam), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)PyParam_dealloc,/* tp_dealloc */ - }; static PyParam* PyParam_fromParam(const Param ¶m) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 6b09d409..dae5f754 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -212,7 +212,6 @@ PythonIntegrationPlugin::~PythonIntegrationPlugin() s_plugins.take(this); Py_XDECREF(m_pluginModule); - Py_XDECREF(m_logger); Py_DECREF(m_nymeaModule); Py_EndInterpreter(m_threadState); @@ -223,19 +222,33 @@ PythonIntegrationPlugin::~PythonIntegrationPlugin() void PythonIntegrationPlugin::initPython() { - Q_ASSERT_X(s_mainThreadState == nullptr, "PythonIntegrationPlugin::initPython()", "initPython() must be called exactly once."); + Q_ASSERT_X(s_mainThreadState == nullptr, "PythonIntegrationPlugin::initPython()", "initPython() must be called exactly once before calling deinitPython()."); - PyImport_AppendInittab("nymea", PyInit_nymea); + // Only modify the init tab once (initPython() might be called again after calling deinitPython()) + static bool initTabPrepared = false; + if (!initTabPrepared) { + PyImport_AppendInittab("nymea", PyInit_nymea); + initTabPrepared = true; + } + + // Initialize the python engine and fire up threading support Py_InitializeEx(0); PyEval_InitThreads(); + // Store the main thread state and release the GIL s_mainThreadState = PyEval_SaveThread(); } void PythonIntegrationPlugin::deinitPython() { + // Restore our main thread state PyEval_RestoreThread(s_mainThreadState); - Py_Finalize(); + + // Tear down the python engine + Py_FinalizeEx(); + + // Our main thread state is destroyed now + s_mainThreadState = nullptr; } bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) @@ -244,18 +257,18 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) QFile metaDataFile(fi.absolutePath() + "/" + fi.baseName() + ".json"); if (!metaDataFile.open(QFile::ReadOnly)) { - qCWarning(dcThingManager()) << "Error opening metadata file:" << metaDataFile.fileName(); + qCWarning(dcPythonIntegrations()) << "Error opening metadata file:" << metaDataFile.fileName(); return false; } QJsonParseError error; QJsonDocument jsonDoc = QJsonDocument::fromJson(metaDataFile.readAll(), &error); if (error.error != QJsonParseError::NoError) { - qCWarning(dcThingManager()) << "Error parsing metadata file:" << error.errorString(); + qCWarning(dcPythonIntegrations()) << "Error parsing metadata file:" << error.errorString(); return false; } setMetaData(PluginMetadata(jsonDoc.object())); if (!metadata().isValid()) { - qCWarning(dcThingManager()) << "Plugin metadata not valid for plugin:" << scriptFile; + qCWarning(dcPythonIntegrations()) << "Plugin metadata not valid for plugin:" << scriptFile; return false; } @@ -314,8 +327,11 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) PyObject *args = Py_BuildValue("(s)", category.toUtf8().data()); PyNymeaLoggingHandler *logger = reinterpret_cast(PyObject_CallObject((PyObject*)&PyNymeaLoggingHandlerType, args)); Py_DECREF(args); - PyModule_AddObject(m_pluginModule, "logger", reinterpret_cast(logger)); - m_logger = (PyObject*)logger; + int loggerAdded = PyModule_AddObject(m_pluginModule, "logger", reinterpret_cast(logger)); + if (loggerAdded != 0) { + qCWarning(dcPythonIntegrations()) << "Failed to add the logger object"; + Py_DECREF(logger); + } // Export metadata ids into module exportIds(); diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 1539202f..37bcaf28 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -76,8 +76,6 @@ private: // The imported plugin module (the plugin.py) PyObject *m_pluginModule = nullptr; - PyObject *m_logger = nullptr; - // A map of plugin instances to plugin python scripts/modules // Make sure to hold the GIL when accessing this. static QHash s_plugins; diff --git a/tests/auto/integrations/testintegrations.cpp b/tests/auto/integrations/testintegrations.cpp index b3c6e373..5b5d8039 100644 --- a/tests/auto/integrations/testintegrations.cpp +++ b/tests/auto/integrations/testintegrations.cpp @@ -161,6 +161,7 @@ void TestIntegrations::initTestCase() "Tests.debug=true\n" "Mock.debug=true\n" "Translations.debug=true\n" + "PythonIntegrations.debug=true\n" ); // Adding an async mock to be used in tests below diff --git a/tests/testlib/nymeatestbase.cpp b/tests/testlib/nymeatestbase.cpp index d124b865..cdc687f6 100644 --- a/tests/testlib/nymeatestbase.cpp +++ b/tests/testlib/nymeatestbase.cpp @@ -414,7 +414,9 @@ void NymeaTestBase::waitForDBSync() void NymeaTestBase::restartServer() { // Destroy and recreate the core instance... + qCDebug(dcTests()) << "Tearing down server instance"; NymeaCore::instance()->destroy(); + qCDebug(dcTests()) << "Restarting server instance"; NymeaCore::instance()->init(); QSignalSpy coreSpy(NymeaCore::instance(), SIGNAL(initialized())); coreSpy.wait(); From 9f49187dc1ee579a9bd5777b8d706645da3d84e9 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 31 Aug 2020 20:32:24 +0200 Subject: [PATCH 37/46] Fix tests and xenial build --- libnymea-core/integrations/pythonintegrationplugin.cpp | 2 +- tests/auto/integrations/testintegrations.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index dae5f754..3891a84a 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -245,7 +245,7 @@ void PythonIntegrationPlugin::deinitPython() PyEval_RestoreThread(s_mainThreadState); // Tear down the python engine - Py_FinalizeEx(); + Py_Finalize(); // Our main thread state is destroyed now s_mainThreadState = nullptr; diff --git a/tests/auto/integrations/testintegrations.cpp b/tests/auto/integrations/testintegrations.cpp index 5b5d8039..bfacf23b 100644 --- a/tests/auto/integrations/testintegrations.cpp +++ b/tests/auto/integrations/testintegrations.cpp @@ -291,8 +291,8 @@ void TestIntegrations::getThingClasses_data() QTest::addColumn("vendorId"); QTest::addColumn("resultCount"); - QTest::newRow("vendor nymea") << nymeaVendorId << 14; - QTest::newRow("no filter") << VendorId() << 14; + QTest::newRow("vendor nymea") << nymeaVendorId << 17; + QTest::newRow("no filter") << VendorId() << 17; QTest::newRow("invalid vendor") << VendorId("93e7d361-8025-4354-b17e-b68406c800bc") << 0; } From e50eece45c3290d93c9f1d8fe066162c02aefefe Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 31 Aug 2020 21:52:15 +0200 Subject: [PATCH 38/46] Fix metatype registering for old Qt versions --- .../integrations/thingmanagerimplementation.cpp | 3 --- libnymea/integrations/thingmanager.cpp | 17 +++++++++++++++++ tests/auto/pythonplugins/testpythonplugins.cpp | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index bd4b5a65..3dcd4576 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -71,9 +71,6 @@ ThingManagerImplementation::ThingManagerImplementation(HardwareManager *hardware m_locale(locale), m_translator(new Translator(this)) { - qRegisterMetaType(); - qRegisterMetaType(); - foreach (const Interface &interface, ThingUtils::allInterfaces()) { m_supportedInterfaces.insert(interface.name(), interface); } diff --git a/libnymea/integrations/thingmanager.cpp b/libnymea/integrations/thingmanager.cpp index ae02ed1a..5053d861 100644 --- a/libnymea/integrations/thingmanager.cpp +++ b/libnymea/integrations/thingmanager.cpp @@ -47,8 +47,25 @@ ThingManager::ThingManager(QObject *parent) : QObject(parent) { qRegisterMetaType(); qRegisterMetaType(); + qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); } /*! Connect two states. diff --git a/tests/auto/pythonplugins/testpythonplugins.cpp b/tests/auto/pythonplugins/testpythonplugins.cpp index e3b08b4e..c15dd764 100644 --- a/tests/auto/pythonplugins/testpythonplugins.cpp +++ b/tests/auto/pythonplugins/testpythonplugins.cpp @@ -64,6 +64,7 @@ void TestPythonPlugins::initTestCase() QLoggingCategory::setFilterRules("*.debug=false\n" "Tests.debug=true\n" "PyMock.debug=true\n" + "PythonIntegrations.debug=true\n" ); } From bc04e455ac4482265687437915c8acfa673b44be Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 31 Aug 2020 23:32:45 +0200 Subject: [PATCH 39/46] don't fail build on in-source builds --- plugins/pymock/pymock.pro | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/plugins/pymock/pymock.pro b/plugins/pymock/pymock.pro index bdf46f5d..4eb1428c 100644 --- a/plugins/pymock/pymock.pro +++ b/plugins/pymock/pymock.pro @@ -5,17 +5,7 @@ OTHER_FILES = integrationpluginpymock.json \ # Copy files to build dir as we've set plugin import paths to that -#mytarget.target = integrationpluginpymock.py -#mytarget.commands = cp $$PWD/$$mytarget.target $$mytarget.target -#mytarget.depends = mytarget2 - -#mytarget2.commands = cp $$PWD/integrationpluginpymock.json integrationpluginpymock.json - -#QMAKE_EXTRA_TARGETS += mytarget mytarget2 - - - -copydata.commands = $(COPY_DIR) $$PWD/integrationpluginpymock.json $$PWD/*.py $$OUT_PWD +copydata.commands = $(COPY_DIR) $$PWD/integrationpluginpymock.json $$PWD/*.py $$OUT_PWD || true first.depends = $(first) copydata export(first.depends) export(copydata.commands) From a9ae885cfa2cb73bca8de9922fdd31f89d4d3495 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Tue, 1 Sep 2020 16:10:59 +0200 Subject: [PATCH 40/46] Also accept python 3.7 --- libnymea-core/libnymea-core.pro | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index fc380368..500c14de 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -11,9 +11,14 @@ CONFIG += link_pkgconfig PKGCONFIG += nymea-mqtt nymea-networkmanager packagesExist(python3-embed) { + # Ubuntu focal PKGCONFIG += python3-embed } else:packagesExist(python-3.5) { + # Ubuntu xenial PKGCONFIG += python-3.5 +} else:packagesExist(python-3.7) { + # Debian buster + PKGCONFIG += python-3.7 } else { error("Python development package not found.") } From dbe5fab122e1bd8c14ea18efb1155c98ef0caabf Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Tue, 1 Sep 2020 16:58:39 +0200 Subject: [PATCH 41/46] also build on bionic --- libnymea-core/libnymea-core.pro | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index 500c14de..b4da93f8 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -10,14 +10,18 @@ LIBS += -L$$top_builddir/libnymea/ -lnymea -lssl -lcrypto CONFIG += link_pkgconfig PKGCONFIG += nymea-mqtt nymea-networkmanager +# As of Ubuntu focal, there's a commonly named python3-embed pointing to the distro version of python +# For everything below python 3.8 we need to manually select one packagesExist(python3-embed) { - # Ubuntu focal PKGCONFIG += python3-embed } else:packagesExist(python-3.5) { - # Ubuntu xenial + # xenial, stretch PKGCONFIG += python-3.5 +} else:packagesExist(python-3.6) { + # bionic + PKGCONFIG += python-3.6 } else:packagesExist(python-3.7) { - # Debian buster + # buster, eoan PKGCONFIG += python-3.7 } else { error("Python development package not found.") From 85f742a38d314c93758746197ae8a2f4c4dcac29 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Fri, 4 Sep 2020 22:41:51 +0200 Subject: [PATCH 42/46] Add ThingActionInfo.paramValue() method --- .../integrations/python/pythingactioninfo.h | 24 ++++++++++++++++++- libnymea-core/integrations/python/pyutils.h | 3 ++- plugins/pymock/integrationpluginpymock.py | 7 ++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/libnymea-core/integrations/python/pythingactioninfo.h b/libnymea-core/integrations/python/pythingactioninfo.h index 7c62a421..6dd90aa8 100644 --- a/libnymea-core/integrations/python/pythingactioninfo.h +++ b/libnymea-core/integrations/python/pythingactioninfo.h @@ -72,7 +72,7 @@ static PyObject * PyThingActionInfo_finish(PyThingActionInfo* self, PyObject* ar char *message = nullptr; if (!PyArg_ParseTuple(args, "i|s", &status, &message)) { - PyErr_SetString(PyExc_TypeError, "Invalid arguments in finish call. Expected: finish(ThingError, message = \"\""); + PyErr_SetString(PyExc_TypeError, "Invalid arguments in finish call. Expected: finish(ThingError, message = \"\")"); return nullptr; } @@ -86,6 +86,27 @@ static PyObject * PyThingActionInfo_finish(PyThingActionInfo* self, PyObject* ar Py_RETURN_NONE; } +static PyObject * PyThingActionInfo_paramValue(PyThingActionInfo* self, PyObject* args) { + char *paramTypeIdStr = nullptr; + if (!PyArg_ParseTuple(args, "s", ¶mTypeIdStr)) { + PyErr_SetString(PyExc_TypeError, "Invalid arguments in paramValue call. Expected: paramValue(paramTypeId)"); + return nullptr; + } + + ParamTypeId paramTypeId = ParamTypeId(paramTypeIdStr); + for (int i = 0; i < PyTuple_Size(self->pyParams); i++) { + PyParam *pyParam = reinterpret_cast(PyTuple_GetItem(self->pyParams, i)); + // We're intentionally converting both ids to QUuid here in order to be more flexible with different UUID notations + ParamTypeId ptid = StateTypeId(PyUnicode_AsUTF8AndSize(pyParam->pyParamTypeId, nullptr)); + if (ptid == paramTypeId) { + Py_INCREF(pyParam->pyValue); + return pyParam->pyValue; + } + } + qCWarning(dcPythonIntegrations()) << "No such ParamTypeId in action params" << paramTypeId; + Py_RETURN_NONE; +}; + static PyMemberDef PyThingActionInfo_members[] = { {"thing", T_OBJECT_EX, offsetof(PyThingActionInfo, pyThing), 0, "Thing this action is for"}, {"actionTypeId", T_OBJECT_EX, offsetof(PyThingActionInfo, pyActionTypeId), 0, "The action type id for this action"}, @@ -95,6 +116,7 @@ static PyMemberDef PyThingActionInfo_members[] = { static PyMethodDef PyThingActionInfo_methods[] = { { "finish", (PyCFunction)PyThingActionInfo_finish, METH_VARARGS, "finish an action" }, + { "paramValue", (PyCFunction)PyThingActionInfo_paramValue, METH_VARARGS, "Get an actions param value"}, {nullptr, nullptr, 0, nullptr} // sentinel }; diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index f18bbd19..5e24a232 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -66,7 +66,8 @@ QVariant PyObjectToQVariant(PyObject *pyObject) return QVariant(PyObject_IsTrue(pyObject)); } - Q_ASSERT_X(false, "pyutils.h", QString("Unhandled data type in conversion PyObject to QVariant: %1").arg(pyObject->ob_type->tp_name).toUtf8()); + Q_ASSERT_X(false, "pyutils.h", QString("Unhandled data type in converting PyObject to QVariant: %1").arg(pyObject->ob_type->tp_name).toUtf8()); + qCWarning(dcPythonIntegrations()) << QString("Unhandled data type in converting PyObject to QVariant: %1").arg(pyObject->ob_type->tp_name).toUtf8(); return QVariant(); } diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index 31ef8c7c..e16199c2 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -112,8 +112,11 @@ def postSetupThing(thing): logger.log("Setting 1 value:", thing.setting(pyMockDiscoveryPairingSettingsSetting1ParamTypeId)) -async def executeAction(info): - logger.log("executeAction for", info.thing.name, info.actionTypeId, "with params", info.params[0].value) +def executeAction(info): + logger.log("executeAction for", info.thing.name, info.actionTypeId, "with params", info.params) + paramValueByIndex = info.params[0].value + paramValueById = info.paramValue(pyMockAction1ActionParam1ParamTypeId) + logger.log("Param by index:", paramValueByIndex, "by ID:", paramValueById) info.finish(nymea.ThingErrorNoError) From 15eead0976b71238e5be2540aedaf96d49053acc Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Sun, 6 Sep 2020 23:43:02 +0200 Subject: [PATCH 43/46] Properly catch stdout and stderr and redirect it to qmessagelogger --- .../python/pynymealogginghandler.h | 25 +++- .../integrations/python/pynymeamodule.h | 8 +- .../integrations/python/pystdouthandler.h | 108 ++++++++++++++++++ libnymea-core/integrations/python/pyutils.h | 69 ----------- .../integrations/pythonintegrationplugin.cpp | 13 +++ libnymea-core/libnymea-core.pro | 1 + plugins/pymock/integrationpluginpymock.py | 5 +- 7 files changed, 150 insertions(+), 79 deletions(-) create mode 100644 libnymea-core/integrations/python/pystdouthandler.h diff --git a/libnymea-core/integrations/python/pynymealogginghandler.h b/libnymea-core/integrations/python/pynymealogginghandler.h index 2bcce1d4..a34ca452 100644 --- a/libnymea-core/integrations/python/pynymealogginghandler.h +++ b/libnymea-core/integrations/python/pynymealogginghandler.h @@ -41,7 +41,23 @@ static void PyNymeaLoggingHandler_dealloc(PyNymeaLoggingHandler * self) Py_TYPE(self)->tp_free(self); } -static PyObject * PyNymeaLoggingHandler_log(PyNymeaLoggingHandler* self, PyObject* args) +static PyObject * PyNymeaLoggingHandler_info(PyNymeaLoggingHandler* self, PyObject* args) +{ + QStringList strings; + for (int i = 0; i < PyTuple_GET_SIZE(args); i++) { + PyObject *obj = PyTuple_GET_ITEM(args, i); + PyObject* repr = PyObject_Repr(obj); + PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + const char *bytes = PyBytes_AS_STRING(str); + Py_XDECREF(repr); + Py_XDECREF(str); + strings.append(bytes); + } + qCInfo(QLoggingCategory(self->category)).noquote() << strings.join(' '); + Py_RETURN_NONE; +} + +static PyObject * PyNymeaLoggingHandler_debug(PyNymeaLoggingHandler* self, PyObject* args) { QStringList strings; for (int i = 0; i < PyTuple_GET_SIZE(args); i++) { @@ -73,9 +89,12 @@ static PyObject * PyNymeaLoggingHandler_warn(PyNymeaLoggingHandler* self, PyObje Py_RETURN_NONE; } + static PyMethodDef PyNymeaLoggingHandler_methods[] = { - { "log", (PyCFunction)PyNymeaLoggingHandler_log, METH_VARARGS, "Add a new descriptor to the discovery" }, - { "warn", (PyCFunction)PyNymeaLoggingHandler_warn, METH_VARARGS, "Add a new descriptor to the discovery" }, + { "log", (PyCFunction)PyNymeaLoggingHandler_info, METH_VARARGS, "Log an info message to the nymea log. Same as info()." }, + { "info", (PyCFunction)PyNymeaLoggingHandler_info, METH_VARARGS, "Log an info message to the nymea log." }, + { "debug", (PyCFunction)PyNymeaLoggingHandler_debug, METH_VARARGS, "Log a debug message to the nymea log." }, + { "warn", (PyCFunction)PyNymeaLoggingHandler_warn, METH_VARARGS, "Log a warning message to the nymea log." }, {nullptr, nullptr, 0, nullptr} // sentinel }; diff --git a/libnymea-core/integrations/python/pynymeamodule.h b/libnymea-core/integrations/python/pynymeamodule.h index e750e650..b9f01844 100644 --- a/libnymea-core/integrations/python/pynymeamodule.h +++ b/libnymea-core/integrations/python/pynymeamodule.h @@ -3,6 +3,7 @@ #include +#include "pystdouthandler.h" #include "pynymealogginghandler.h" #include "pything.h" #include "pythingdiscoveryinfo.h" @@ -17,12 +18,7 @@ static int nymea_exec(PyObject *m) { - // Override stdout/stderr to use qDebug instead - PyObject* pyLog = PyModule_Create(&pyLog_module); - PySys_SetObject("stdout", pyLog); - PyObject* pyWarn = PyModule_Create(&pyWarn_module); - PySys_SetObject("stderr", pyWarn); - + registerStdOutHandler(m); registerNymeaLoggingHandler(m); registerParamType(m); registerThingType(m); diff --git a/libnymea-core/integrations/python/pystdouthandler.h b/libnymea-core/integrations/python/pystdouthandler.h new file mode 100644 index 00000000..24d3fdba --- /dev/null +++ b/libnymea-core/integrations/python/pystdouthandler.h @@ -0,0 +1,108 @@ +#ifndef PYSTDOUTHANDLER_H +#define PYSTDOUTHANDLER_H + +#include +#include "structmember.h" + +#include +#include + +Q_DECLARE_LOGGING_CATEGORY(dcPythonIntegrations) + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" + +typedef struct { + PyObject_HEAD + char *category; + QtMsgType msgType; +} PyStdOutHandler; + +static int PyStdOutHandler_init(PyStdOutHandler *self, PyObject *args, PyObject */*kwds*/) +{ + qCDebug(dcPythonIntegrations()) << "+++ PyStdOutHandler"; + char *category = nullptr; + QtMsgType msgType; + if (!PyArg_ParseTuple(args, "si", &category, &msgType)) { + qCWarning(dcPythonIntegrations()) << "PyStdOutHandler: Error parsing parameters"; + return -1; + } + + self->category = (char*)malloc(qstrlen(category)); + self->msgType = msgType; + qstrcpy(self->category, category); + + return 0; +} + +static void PyStdOutHandler_dealloc(PyStdOutHandler * self) +{ + qCDebug(dcPythonIntegrations()) << "--- PyStdOutHandler"; + free(self->category); + Py_TYPE(self)->tp_free(self); +} + +static PyObject* PyStdOutHandler_write(PyStdOutHandler* self, PyObject* args) +{ + const char *what; + if (!PyArg_ParseTuple(args, "s", &what)) + return nullptr; + if (!QByteArray(what).trimmed().isEmpty()) { + switch (self->msgType) { + case QtMsgType::QtInfoMsg: + qCInfo(QLoggingCategory(self->category)) << what; + break; + case QtMsgType::QtDebugMsg: + qCDebug(QLoggingCategory(self->category)) << what; + break; + case QtMsgType::QtWarningMsg: + qCWarning(QLoggingCategory(self->category)) << what; + break; + case QtMsgType::QtCriticalMsg: + qCCritical(QLoggingCategory(self->category)) << what; + break; + default: + qCDebug(QLoggingCategory(self->category)) << what; + break; + } + } + Py_RETURN_NONE; +} + +static PyObject* PyStdOutHandler_flush(PyObject* /*self*/, PyObject* /*args*/) +{ + // Not really needed... QDebug flushes already on its own + Py_RETURN_NONE; +} + +static PyMethodDef PyStdOutHandler_methods[] = { + { "write", (PyCFunction)PyStdOutHandler_write, METH_VARARGS, "Writes to stdout through qDebug()"}, + { "flush", (PyCFunction)PyStdOutHandler_flush, METH_VARARGS, "no-op"}, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyTypeObject PyStdOutHandlerType = { + PyVarObject_HEAD_INIT(NULL, 0) + "nymea.StdOutHandler", /* tp_name */ + sizeof(PyStdOutHandler), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PyStdOutHandler_dealloc,/* tp_dealloc */ +}; + +static void registerStdOutHandler(PyObject *module) +{ + + PyStdOutHandlerType.tp_new = PyType_GenericNew; + PyStdOutHandlerType.tp_init = reinterpret_cast(PyStdOutHandler_init); + PyStdOutHandlerType.tp_flags = Py_TPFLAGS_DEFAULT; + PyStdOutHandlerType.tp_methods = PyStdOutHandler_methods; + PyStdOutHandlerType.tp_doc = "Logging handler for nymea."; + + if (PyType_Ready(&PyStdOutHandlerType) == 0) { + PyModule_AddObject(module, "NymeaLoggingHandler", (PyObject *)&PyStdOutHandlerType); + } +} + +#endif // PYSTDOUTHANDLER_H diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index 5e24a232..996183c7 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -71,74 +71,5 @@ QVariant PyObjectToQVariant(PyObject *pyObject) return QVariant(); } -// Write to stdout -PyObject* pyLog_write(PyObject* /*self*/, PyObject* args) -{ - const char *what; - if (!PyArg_ParseTuple(args, "s", &what)) - return nullptr; - if (!QByteArray(what).trimmed().isEmpty()) { - qCDebug(dcPythonIntegrations()) << what; - } - Py_RETURN_NONE; -} -PyObject* pyLog_flush(PyObject* /*self*/, PyObject* /*args*/) -{ - // Not really needed... qDebug() flushes already on its own - Py_RETURN_NONE; -} - -static PyMethodDef pyLog_methods[] = -{ - {"write", pyLog_write, METH_VARARGS, "Writes to stdout through qDebug()"}, - {"flush", pyLog_flush, METH_VARARGS, "noop"}, - {nullptr, nullptr, 0, nullptr} // sentinel -}; - -static PyModuleDef pyLog_module = -{ - PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; - "pyLog", // const char* m_name; - "pyLog stdout override",// const char* m_doc; - -1, // Py_ssize_t m_size; - pyLog_methods, // PyMethodDef *m_methods - // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; - nullptr, nullptr, nullptr, nullptr -}; - -// Write to stderr -PyObject* pyWarn_write(PyObject* /*self*/, PyObject* args) -{ - const char *what; - if (!PyArg_ParseTuple(args, "s", &what)) - return nullptr; - if (!QByteArray(what).trimmed().isEmpty()) { - qCWarning(dcPythonIntegrations()) << what; - } - Py_RETURN_NONE; -} -PyObject* pyWarn_flush(PyObject* /*self*/, PyObject* /*args*/) -{ - // Not really needed... qDebug() flushes already on its own - Py_RETURN_NONE; -} - -static PyMethodDef pyWarn_methods[] = -{ - {"write", pyWarn_write, METH_VARARGS, "Writes to stderr through qWarnging()"}, - {"flush", pyWarn_flush, METH_VARARGS, "noop"}, - {nullptr, nullptr, 0, nullptr} // sentinel -}; - -static PyModuleDef pyWarn_module = -{ - PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; - "pyWarn", // const char* m_name; - "pyWarn stdout override",// const char* m_doc; - -1, // Py_ssize_t m_size; - pyWarn_methods, // PyMethodDef *m_methods - // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; - nullptr, nullptr, nullptr, nullptr -}; #endif // PYUTILS_H diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 3891a84a..47ffac42 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -2,6 +2,7 @@ #include "pythonintegrationplugin.h" #include "python/pynymeamodule.h" +#include "python/pystdouthandler.h" #include "loggingcategories.h" @@ -327,6 +328,18 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) PyObject *args = Py_BuildValue("(s)", category.toUtf8().data()); PyNymeaLoggingHandler *logger = reinterpret_cast(PyObject_CallObject((PyObject*)&PyNymeaLoggingHandlerType, args)); Py_DECREF(args); + + // Override stdout and stderr + + args = Py_BuildValue("(si)", category.toUtf8().data(), QtMsgType::QtInfoMsg); + PyStdOutHandler*stdOutHandler = reinterpret_cast(PyObject_CallObject((PyObject*)&PyStdOutHandlerType, args)); + Py_DECREF(args); + PySys_SetObject("stdout", (PyObject*)stdOutHandler); + args = Py_BuildValue("(si)", category.toUtf8().data(), QtMsgType::QtWarningMsg); + PyStdOutHandler*stdErrHandler = reinterpret_cast(PyObject_CallObject((PyObject*)&PyStdOutHandlerType, args)); + PySys_SetObject("stderr", (PyObject*)stdErrHandler); + Py_DECREF(args); + int loggerAdded = PyModule_AddObject(m_pluginModule, "logger", reinterpret_cast(logger)); if (loggerAdded != 0) { qCWarning(dcPythonIntegrations()) << "Failed to add the logger object"; diff --git a/libnymea-core/libnymea-core.pro b/libnymea-core/libnymea-core.pro index b4da93f8..f16a69a2 100644 --- a/libnymea-core/libnymea-core.pro +++ b/libnymea-core/libnymea-core.pro @@ -40,6 +40,7 @@ HEADERS += nymeacore.h \ integrations/python/pynymealogginghandler.h \ integrations/python/pynymeamodule.h \ integrations/python/pyparam.h \ + integrations/python/pystdouthandler.h \ integrations/python/pything.h \ integrations/python/pythingactioninfo.h \ integrations/python/pythingdescriptor.h \ diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index e16199c2..fea164a3 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -6,10 +6,13 @@ watchingAutoThings = False loopRunning = False def init(): - logger.log("Python mock plugin init") global loopRunning loopRunning = True + logger.log("Python mock plugin init") + logger.warn("Python mock warning") + print("python stdout") + while loopRunning: time.sleep(5); for thing in myThings(): From 8aa2feb7c6acbb1b37fadff6bd119ca1b561b7c5 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 7 Sep 2020 00:08:08 +0200 Subject: [PATCH 44/46] Make python plugins appear in the logging categories list --- .../integrations/pythonintegrationplugin.cpp | 1 - .../thingmanagerimplementation.cpp | 45 ++++++++++++++----- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 47ffac42..799b050e 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -330,7 +330,6 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) Py_DECREF(args); // Override stdout and stderr - args = Py_BuildValue("(si)", category.toUtf8().data(), QtMsgType::QtInfoMsg); PyStdOutHandler*stdOutHandler = reinterpret_cast(PyObject_CallObject((PyObject*)&PyStdOutHandlerType, args)); Py_DECREF(args); diff --git a/libnymea-core/integrations/thingmanagerimplementation.cpp b/libnymea-core/integrations/thingmanagerimplementation.cpp index 3dcd4576..e9edb9f0 100644 --- a/libnymea-core/integrations/thingmanagerimplementation.cpp +++ b/libnymea-core/integrations/thingmanagerimplementation.cpp @@ -64,6 +64,7 @@ #include #include #include +#include ThingManagerImplementation::ThingManagerImplementation(HardwareManager *hardwareManager, const QLocale &locale, QObject *parent) : ThingManager(parent), @@ -142,20 +143,44 @@ QStringList ThingManagerImplementation::pluginSearchDirs() QList ThingManagerImplementation::pluginsMetadata() { QList pluginList; + QStringList searchDirs; + // Add first level of subdirectories to the plugin search dirs so we can point to a collection of plugins foreach (const QString &path, pluginSearchDirs()) { + searchDirs.append(path); QDir dir(path); - foreach (const QString &entry, dir.entryList()) { - QFileInfo fi; + foreach (const QString &subdir, dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot)) { + searchDirs.append(path + '/' + subdir); + } + } + + foreach (const QString &path, searchDirs) { + QDir dir(path); + qCDebug(dcThingManager) << "Loading plugins from:" << dir.absolutePath(); + foreach (const QString &entry, dir.entryList({"*.so", "*.js", "*.py"}, QDir::Files)) { + + QFileInfo fi(path + '/' + entry); if (entry.startsWith("libnymea_integrationplugin") && entry.endsWith(".so")) { - fi.setFile(path + "/" + entry); - } else { - fi.setFile(path + "/" + entry + "/libnymea_integrationplugin" + entry + ".so"); + QPluginLoader loader(fi.absoluteFilePath()); + pluginList.append(loader.metaData().value("MetaData").toObject()); +#if QT_VERSION >= QT_VERSION_CHECK(5,12,0) + } else if (entry.startsWith("integrationplugin") && entry.endsWith(".js")) { + QFile jsonFile(fi.absolutePath() + "/" + fi.baseName() + ".json"); + if (!jsonFile.open(QFile::ReadOnly)) { + qCDebug(dcThingManager()) << "Failed to open json file for:" << entry; + continue; + } + QJsonDocument jsonDoc = QJsonDocument::fromJson(jsonFile.readAll()); + pluginList.append(jsonDoc.object()); +#endif + } else if (entry.startsWith("integrationplugin") && entry.endsWith(".py")) { + QFile jsonFile(fi.absolutePath() + "/" + fi.baseName() + ".json"); + if (!jsonFile.open(QFile::ReadOnly)) { + qCDebug(dcThingManager()) << "Failed to open json file for:" << entry; + continue; + } + QJsonDocument jsonDoc = QJsonDocument::fromJson(jsonFile.readAll()); + pluginList.append(jsonDoc.object()); } - if (!fi.exists()) { - continue; - } - QPluginLoader loader(fi.absoluteFilePath()); - pluginList.append(loader.metaData().value("MetaData").toObject()); } } return pluginList; From befda2ec00fa1563e0893803845ac3400641b21b Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Mon, 7 Sep 2020 19:44:48 +0200 Subject: [PATCH 45/46] Revert back to qDebug, we're not ready for qInfo yet --- libnymea-core/integrations/python/pynymealogginghandler.h | 4 +++- libnymea-core/integrations/pythonintegrationplugin.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libnymea-core/integrations/python/pynymealogginghandler.h b/libnymea-core/integrations/python/pynymealogginghandler.h index a34ca452..fe2e8970 100644 --- a/libnymea-core/integrations/python/pynymealogginghandler.h +++ b/libnymea-core/integrations/python/pynymealogginghandler.h @@ -53,7 +53,9 @@ static PyObject * PyNymeaLoggingHandler_info(PyNymeaLoggingHandler* self, PyObje Py_XDECREF(str); strings.append(bytes); } - qCInfo(QLoggingCategory(self->category)).noquote() << strings.join(' '); + // FIXME: We'll want to use qCInfo() here but the system can't really deal with that yet + // Move from qCDebug() to qCInfo() when we support controlling that + qCDebug(QLoggingCategory(self->category)).noquote() << strings.join(' '); Py_RETURN_NONE; } diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 799b050e..4bcec22a 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -330,7 +330,7 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) Py_DECREF(args); // Override stdout and stderr - args = Py_BuildValue("(si)", category.toUtf8().data(), QtMsgType::QtInfoMsg); + args = Py_BuildValue("(si)", category.toUtf8().data(), QtMsgType::QtDebugMsg); PyStdOutHandler*stdOutHandler = reinterpret_cast(PyObject_CallObject((PyObject*)&PyStdOutHandlerType, args)); Py_DECREF(args); PySys_SetObject("stdout", (PyObject*)stdOutHandler); From 4c1361d10da4150771f15e71360156fe0957906b Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Wed, 28 Oct 2020 17:38:10 +0100 Subject: [PATCH 46/46] Fix test target name --- tests/auto/pythonplugins/pythonplugins.pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/auto/pythonplugins/pythonplugins.pro b/tests/auto/pythonplugins/pythonplugins.pro index 438e619e..36d4b792 100644 --- a/tests/auto/pythonplugins/pythonplugins.pro +++ b/tests/auto/pythonplugins/pythonplugins.pro @@ -1,4 +1,4 @@ -TARGET = testactions +TARGET = testpythonplugins include(../../../nymea.pri) include(../autotests.pri)