From a90841401c4241e738584abca31d8a3db3ee63a9 Mon Sep 17 00:00:00 2001 From: Michael Zanetti Date: Fri, 3 Jul 2020 15:27:19 +0200 Subject: [PATCH] more work --- .../integrations/python/pythingdescriptor.h | 10 ++- .../python/pythingdiscoveryinfo.h | 2 - .../integrations/python/pythingsetupinfo.h | 2 + libnymea-core/integrations/python/pyutils.h | 78 +++++++++++++++++++ .../integrations/pythonintegrationplugin.cpp | 72 +++++++++-------- .../integrations/pythonintegrationplugin.h | 1 + libnymea/integrations/thingdiscoveryinfo.h | 6 +- libnymea/loggingcategories.cpp | 1 + plugins/pymock/integrationpluginpymock.py | 23 +++++- 9 files changed, 152 insertions(+), 43 deletions(-) diff --git a/libnymea-core/integrations/python/pythingdescriptor.h b/libnymea-core/integrations/python/pythingdescriptor.h index 777d5b3f..0a7c1baf 100644 --- a/libnymea-core/integrations/python/pythingdescriptor.h +++ b/libnymea-core/integrations/python/pythingdescriptor.h @@ -5,6 +5,7 @@ #include "structmember.h" #include "integrations/thingdescriptor.h" +#include "loggingcategories.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -31,6 +32,7 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj static char *kwlist[] = {"thingClassId", "name", "description", nullptr}; PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr; + qWarning() << "++++ PyThingDescriptor"; if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, &thingClassId, &name, &description)) return -1; @@ -52,12 +54,18 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj return 0; } +static void PyThingDescriptor_dealloc(PyThingDescriptor * self) +{ + qWarning() << "---- PyThingDescriptor"; + Py_TYPE(self)->tp_free(self); +} + static PyTypeObject PyThingDescriptorType = { PyVarObject_HEAD_INIT(NULL, 0) "nymea.ThingDescriptor", /* tp_name */ sizeof(PyThingDescriptor), /* tp_basicsize */ 0, /* tp_itemsize */ - 0, /* tp_dealloc */ + (destructor)PyThingDescriptor_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ diff --git a/libnymea-core/integrations/python/pythingdiscoveryinfo.h b/libnymea-core/integrations/python/pythingdiscoveryinfo.h index 9a13defe..f699c220 100644 --- a/libnymea-core/integrations/python/pythingdiscoveryinfo.h +++ b/libnymea-core/integrations/python/pythingdiscoveryinfo.h @@ -36,8 +36,6 @@ static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/ static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self) { - // FIXME: Why is this not called? Seems we're leaking... - Q_ASSERT(false); delete self->mutex; Py_TYPE(self)->tp_free(self); } diff --git a/libnymea-core/integrations/python/pythingsetupinfo.h b/libnymea-core/integrations/python/pythingsetupinfo.h index ab4a589f..29f29c62 100644 --- a/libnymea-core/integrations/python/pythingsetupinfo.h +++ b/libnymea-core/integrations/python/pythingsetupinfo.h @@ -25,11 +25,13 @@ static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject */*args*/, Py if (self == NULL) { return nullptr; } + qWarning() << "++++ PyThingSetupInfo"; self->mutex = new QMutex(); return (PyObject*)self; } static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) { + qWarning() << "--- PyThingSetupInfo"; delete self->mutex; Py_TYPE(self)->tp_free(self); } diff --git a/libnymea-core/integrations/python/pyutils.h b/libnymea-core/integrations/python/pyutils.h index 18a220de..d2d73f4e 100644 --- a/libnymea-core/integrations/python/pyutils.h +++ b/libnymea-core/integrations/python/pyutils.h @@ -58,4 +58,82 @@ QVariant PyObjectToQVariant(PyObject *pyObject) return value; } +void PyDumpError() +{ + if (!PyErr_Occurred()) { + return; + } + +} + +// Write to stdout +PyObject* pyLog_write(PyObject* /*self*/, PyObject* args) +{ + const char *what; + if (!PyArg_ParseTuple(args, "s", &what)) + return nullptr; + if (!QByteArray(what).trimmed().isEmpty()) { + qCDebug(dcThingManager()) << what; + } + Py_RETURN_NONE; +} +PyObject* pyLog_flush(PyObject* /*self*/, PyObject* /*args*/) +{ + // Not really needed... qDebug() flushes already on its own + Py_RETURN_NONE; +} + +static PyMethodDef pyLog_methods[] = +{ + {"write", pyLog_write, METH_VARARGS, "Writes to stdout through qDebug()"}, + {"flush", pyLog_flush, METH_VARARGS, "noop"}, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyModuleDef pyLog_module = +{ + PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; + "pyLog", // const char* m_name; + "pyLog stdout override",// const char* m_doc; + -1, // Py_ssize_t m_size; + pyLog_methods, // PyMethodDef *m_methods + // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; + nullptr, nullptr, nullptr, nullptr +}; + +// Write to stderr +PyObject* pyWarn_write(PyObject* /*self*/, PyObject* args) +{ + const char *what; + if (!PyArg_ParseTuple(args, "s", &what)) + return nullptr; + if (!QByteArray(what).trimmed().isEmpty()) { + qCWarning(dcThingManager()) << what; + } + Py_RETURN_NONE; +} +PyObject* pyWarn_flush(PyObject* /*self*/, PyObject* /*args*/) +{ + // Not really needed... qDebug() flushes already on its own + Py_RETURN_NONE; +} + +static PyMethodDef pyWarn_methods[] = +{ + {"write", pyWarn_write, METH_VARARGS, "Writes to stderr through qWarnging()"}, + {"flush", pyWarn_flush, METH_VARARGS, "noop"}, + {nullptr, nullptr, 0, nullptr} // sentinel +}; + +static PyModuleDef pyWarn_module = +{ + PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base; + "pyWarn", // const char* m_name; + "pyWarn stdout override",// const char* m_doc; + -1, // Py_ssize_t m_size; + pyWarn_methods, // PyMethodDef *m_methods + // inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; + nullptr, nullptr, nullptr, nullptr +}; + #endif // PYUTILS_H diff --git a/libnymea-core/integrations/pythonintegrationplugin.cpp b/libnymea-core/integrations/pythonintegrationplugin.cpp index 6e019619..2a81f08c 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.cpp +++ b/libnymea-core/integrations/pythonintegrationplugin.cpp @@ -24,24 +24,6 @@ PyObject* PythonIntegrationPlugin::s_asyncio = nullptr; QHash PythonIntegrationPlugin::s_plugins; -// Write to stdout/stderr -PyObject* nymea_write(PyObject* /*self*/, PyObject* args) -{ - const char *what; - if (!PyArg_ParseTuple(args, "s", &what)) - return nullptr; - if (!QByteArray(what).trimmed().isEmpty()) { - qCDebug(dcThingManager()) << what; - } - Py_RETURN_NONE; -} - -// Flush stdout/stderr -PyObject* nymea_flush(PyObject* /*self*/, PyObject* /*args*/) -{ - // Not really needed... qDebug() flushes already on its own - Py_RETURN_NONE; -} PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) { @@ -75,8 +57,6 @@ PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args) static PyMethodDef nymea_methods[] = { - {"write", nymea_write, METH_VARARGS, "write to stdout through qDebug()"}, - {"flush", nymea_flush, METH_VARARGS, "flush stdout (no-op)"}, {"task_done", PythonIntegrationPlugin::task_done, METH_VARARGS, "callback to clean up after asyc coroutines"}, {nullptr, nullptr, 0, nullptr} // sentinel }; @@ -94,12 +74,15 @@ static PyModuleDef nymea_module = PyMODINIT_FUNC PyInit_nymea(void) { - PyObject* m = PyModule_Create(&nymea_module); // Overrride stdout/stderr to use qDebug instead - PySys_SetObject("stdout", m); - PySys_SetObject("stderr", m); + PyObject* pyLog = PyModule_Create(&pyLog_module); + PySys_SetObject("stdout", pyLog); + PyObject* pyWarn = PyModule_Create(&pyWarn_module); + PySys_SetObject("stderr", pyWarn); + // Register nymea types + PyObject* m = PyModule_Create(&nymea_module); registerNymeaLoggingHandler(m); registerParamType(m); registerThingType(m); @@ -244,6 +227,21 @@ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject Py_RETURN_NONE; } +PyObject *PythonIntegrationPlugin::pyAutoThingDisappeared(PyObject *self, PyObject *args) +{ + char *thingIdStr = nullptr; + + if (!PyArg_ParseTuple(args, "s", &thingIdStr)) { + qCWarning(dcThingManager) << "Error parsing parameters"; + return nullptr; + } + ThingId thingId(thingIdStr); + PythonIntegrationPlugin *plugin = s_plugins.key(self); + QMetaObject::invokeMethod(plugin, "autoThingDisappeared", Qt::QueuedConnection, Q_ARG(ThingId, thingId)); + + Py_RETURN_NONE; +} + static PyMethodDef plugin_methods[] = { {"configuration", PythonIntegrationPlugin::pyConfiguration, METH_VARARGS, "Get the plugin configuration."}, @@ -251,6 +249,7 @@ static PyMethodDef plugin_methods[] = {"setConfigValue", PythonIntegrationPlugin::pySetConfigValue, METH_VARARGS, "Set the plugin configuration value for a given config paramTypeId."}, {"myThings", PythonIntegrationPlugin::pyMyThings, METH_VARARGS, "Obtain a list of things owned by this plugin."}, {"autoThingsAppeared", PythonIntegrationPlugin::pyAutoThingsAppeared, METH_VARARGS, "Inform the system about auto setup things having appeared."}, + {"autoThingDisappeared", PythonIntegrationPlugin::pyAutoThingDisappeared, METH_VARARGS, "Inform the system about auto setup things having disappeared."}, {nullptr, nullptr, 0, nullptr} // sentinel }; @@ -311,9 +310,9 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile) m_module = PyImport_ImportModule(fi.baseName().toUtf8()); if (!m_module) { - dumpError(); - PyErr_Clear(); qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath(); + PyErr_Print(); + PyErr_Clear(); PyGILState_Release(s); return false; } @@ -399,7 +398,6 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, NULL); pyInfo->info = info; pyInfo->pyThing = pyThing; - Py_INCREF(pyThing); m_things.insert(info->thing(), pyThing); @@ -417,6 +415,7 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) bool result = callPluginFunction("setupThing", reinterpret_cast(pyInfo)); if (!result) { + Py_DECREF(pyThing); // The python code did not even start, so let's finish (fail) the setup right away info->finish(Thing::ThingErrorSetupFailed); } @@ -425,7 +424,11 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info) void PythonIntegrationPlugin::postSetupThing(Thing *thing) { PyThing* pyThing = m_things.value(thing); - callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); + Py_INCREF(pyThing); + bool success = callPluginFunction("postSetupThing", reinterpret_cast(pyThing)); + if (!success) { + Py_DECREF(pyThing); + } } void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) @@ -456,8 +459,12 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info) void PythonIntegrationPlugin::thingRemoved(Thing *thing) { PyThing *pyThing = m_things.value(thing); + Py_INCREF(pyThing); - callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); + bool success = callPluginFunction("thingRemoved", reinterpret_cast(pyThing)); + if (!success) { + Py_DECREF(pyThing); + } m_mutex.lock(); m_things.remove(thing); @@ -577,12 +584,12 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje { PyGILState_STATE s = PyGILState_Ensure(); - qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << s_plugins.key(m_module)->pluginName(); + qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << pluginName(); PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8()); if(!pFunc || !PyCallable_Check(pFunc)) { PyErr_Clear(); Py_XDECREF(pFunc); - qCWarning(dcThingManager()) << "Python plugin does not implement" << function; + qCWarning(dcThingManager()) << "Python plugin" << pluginName() << "does not implement" << function; PyGILState_Release(s); return false; } @@ -595,8 +602,8 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje Py_XDECREF(pFunc); if (PyErr_Occurred()) { - qCWarning(dcThingManager()) << "Error calling python method:"; - dumpError(); + qCWarning(dcThingManager()) << "Error calling python method:" << function << "on plugin" << pluginName(); + PyErr_Print(); PyErr_Clear(); PyGILState_Release(s); return false; @@ -651,7 +658,6 @@ void PythonIntegrationPlugin::cleanupPyThing(PyThing *pyThing) // on the thing (e.g. PyThing_name). // We'd deadlock if we wait for the mutex forever here. So let's process events // while waiting for it... - qWarning() << "Locking cleanup"; while (!pyThing->mutex->tryLock()) { qApp->processEvents(QEventLoop::EventLoopExec); } diff --git a/libnymea-core/integrations/pythonintegrationplugin.h b/libnymea-core/integrations/pythonintegrationplugin.h index 2f2b2055..ada6c114 100644 --- a/libnymea-core/integrations/pythonintegrationplugin.h +++ b/libnymea-core/integrations/pythonintegrationplugin.h @@ -40,6 +40,7 @@ public: static PyObject* pySetConfigValue(PyObject* self, PyObject* args); static PyObject* pyMyThings(PyObject *self, PyObject* args); static PyObject* pyAutoThingsAppeared(PyObject *self, PyObject* args); + static PyObject* pyAutoThingDisappeared(PyObject *self, PyObject* args); public: // python callbacks diff --git a/libnymea/integrations/thingdiscoveryinfo.h b/libnymea/integrations/thingdiscoveryinfo.h index d261881a..48d03b45 100644 --- a/libnymea/integrations/thingdiscoveryinfo.h +++ b/libnymea/integrations/thingdiscoveryinfo.h @@ -53,15 +53,15 @@ public: Thing::ThingError status() const; - void addThingDescriptor(const ThingDescriptor &thingDescriptor); - void addThingDescriptors(const ThingDescriptors &thingDescriptors); - ThingDescriptors thingDescriptors() const; QString displayMessage() const; QString translatedDisplayMessage(const QLocale &locale); public slots: + void addThingDescriptor(const ThingDescriptor &thingDescriptor); + void addThingDescriptors(const ThingDescriptors &thingDescriptors); + void finish(Thing::ThingError status, const QString &displayMessage = QString()); signals: diff --git a/libnymea/loggingcategories.cpp b/libnymea/loggingcategories.cpp index 4d5070c7..d5d4cfd0 100644 --- a/libnymea/loggingcategories.cpp +++ b/libnymea/loggingcategories.cpp @@ -75,6 +75,7 @@ NYMEA_LOGGING_CATEGORY(dcBluetoothServerTraffic, "BluetoothServerTraffic") NYMEA_LOGGING_CATEGORY(dcMqtt, "Mqtt") NYMEA_LOGGING_CATEGORY(dcTranslations, "Translations") NYMEA_LOGGING_CATEGORY(dcI2C, "I2C") +NYMEA_LOGGING_CATEGORY(dcPythonIntegrations, "PythonIntegrations") static QFile s_logFile; diff --git a/plugins/pymock/integrationpluginpymock.py b/plugins/pymock/integrationpluginpymock.py index d96c4d75..42d4f56b 100644 --- a/plugins/pymock/integrationpluginpymock.py +++ b/plugins/pymock/integrationpluginpymock.py @@ -1,13 +1,15 @@ import nymea import asyncio +watchingAutoThings = False + def init(): logger.log("Python mock plugin init") def configValueChanged(paramTypeId, value): - logger.log("Plugin config value changed:", paramTypeId, value) - if paramTypeId == pyMockPluginAutoThingCountParamTypeId: + logger.log("Plugin config value changed:", paramTypeId, value, watchingAutoThings) + if watchingAutoThings and paramTypeId == pyMockPluginAutoThingCountParamTypeId: logger.log("Auto Thing Count plugin config changed:", value, "Currently there are:", len(autoThings()), "auto things") things = autoThings(); for i in range(len(things), value): @@ -15,20 +17,33 @@ def configValueChanged(paramTypeId, value): descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") autoThingsAppeared([descriptor]) - for i in range(len(value), things): + for i in range(value, len(things)): logger.log("Removing auto thing") autoThingDisappeared(things[i].id) def startMonitoringAutoThings(): + global watchingAutoThings + watchingAutoThings = True logger.log("Start monitoring auto things. Have %i auto devices. Need %i." % (len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId))) - for i in range(len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId)): + things = autoThings(); + for i in range(len(things), configValue(pyMockPluginAutoThingCountParamTypeId)): logger.log("Creating new auto thing") descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing") autoThingsAppeared([descriptor]) + for i in range(configValue(pyMockPluginAutoThingCountParamTypeId), len(things)): + logger.log("Removing auto thing") + autoThingDisappeared(things[i].id) + logger.log("Done start monitoring auto things") +async def discoverThings(info): + await asyncio.sleep(1) + descriptor = nymea.ThingDescriptor(pyMockThingClassId, "Python mock thing") + info.addDescriptor(descriptor) + info.finish(nymea.ThingErrorNoError) + async def setupThing(info): logger.log("setupThing for", info.thing.name) info.finish(nymea.ThingErrorNoError)