more work

libnymea-core/integrations/python/pythingdescriptor.h

@@ -5,6 +5,7 @@
 #include "structmember.h"
 
 #include "integrations/thingdescriptor.h"
+#include "loggingcategories.h"
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Winvalid-offsetof"
@@ -31,6 +32,7 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj
     static char *kwlist[] = {"thingClassId", "name", "description", nullptr};
     PyObject *thingClassId = nullptr, *name = nullptr, *description = nullptr;
 
+    qWarning() << "++++ PyThingDescriptor";
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOO", kwlist, &thingClassId, &name, &description))
         return -1;
 
@@ -52,12 +54,18 @@ static int PyThingDescriptor_init(PyThingDescriptor *self, PyObject *args, PyObj
     return 0;
 }
 
+static void PyThingDescriptor_dealloc(PyThingDescriptor * self)
+{
+    qWarning() << "---- PyThingDescriptor";
+    Py_TYPE(self)->tp_free(self);
+}
+
 static PyTypeObject PyThingDescriptorType = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "nymea.ThingDescriptor",   /* tp_name */
     sizeof(PyThingDescriptor), /* tp_basicsize */
     0,                         /* tp_itemsize */
-    0,                         /* tp_dealloc */
+    (destructor)PyThingDescriptor_dealloc, /* tp_dealloc */
     0,                         /* tp_print */
     0,                         /* tp_getattr */
     0,                         /* tp_setattr */

libnymea-core/integrations/python/pythingdiscoveryinfo.h

@@ -36,8 +36,6 @@ static PyObject* PyThingDiscoveryInfo_new(PyTypeObject *type, PyObject */*args*/
 
 static void PyThingDiscoveryInfo_dealloc(PyThingDiscoveryInfo * self)
 {
-    // FIXME: Why is this not called? Seems we're leaking...
-    Q_ASSERT(false);
     delete self->mutex;
     Py_TYPE(self)->tp_free(self);
 }

libnymea-core/integrations/python/pythingsetupinfo.h

@@ -25,11 +25,13 @@ static PyObject* PyThingSetupInfo_new(PyTypeObject *type, PyObject */*args*/, Py
     if (self == NULL) {
         return nullptr;
     }
+    qWarning() << "++++ PyThingSetupInfo";
     self->mutex = new QMutex();
     return (PyObject*)self;
 }
 
 static void PyThingSetupInfo_dealloc(PyThingSetupInfo * self) {
+    qWarning() << "--- PyThingSetupInfo";
     delete self->mutex;
     Py_TYPE(self)->tp_free(self);
 }

libnymea-core/integrations/python/pyutils.h

@@ -58,4 +58,82 @@ QVariant PyObjectToQVariant(PyObject *pyObject)
     return value;
 }
 
+void PyDumpError()
+{
+    if (!PyErr_Occurred()) {
+        return;
+    }
+
+}
+
+// Write to stdout
+PyObject* pyLog_write(PyObject* /*self*/, PyObject* args)
+{
+    const char *what;
+    if (!PyArg_ParseTuple(args, "s", &what))
+        return nullptr;
+    if (!QByteArray(what).trimmed().isEmpty()) {
+        qCDebug(dcThingManager()) << what;
+    }
+    Py_RETURN_NONE;
+}
+PyObject* pyLog_flush(PyObject* /*self*/, PyObject* /*args*/)
+{
+    // Not really needed... qDebug() flushes already on its own
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef pyLog_methods[] =
+{
+    {"write", pyLog_write, METH_VARARGS, "Writes to stdout through qDebug()"},
+    {"flush", pyLog_flush, METH_VARARGS, "noop"},
+    {nullptr, nullptr, 0, nullptr} // sentinel
+};
+
+static PyModuleDef pyLog_module =
+{
+    PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base;
+    "pyLog",               // const char* m_name;
+    "pyLog stdout override",// const char* m_doc;
+    -1,                    // Py_ssize_t m_size;
+    pyLog_methods,        // PyMethodDef *m_methods
+    //  inquiry m_reload;  traverseproc m_traverse;  inquiry m_clear;  freefunc m_free;
+    nullptr, nullptr, nullptr, nullptr
+};
+
+// Write to stderr
+PyObject* pyWarn_write(PyObject* /*self*/, PyObject* args)
+{
+    const char *what;
+    if (!PyArg_ParseTuple(args, "s", &what))
+        return nullptr;
+    if (!QByteArray(what).trimmed().isEmpty()) {
+        qCWarning(dcThingManager()) << what;
+    }
+    Py_RETURN_NONE;
+}
+PyObject* pyWarn_flush(PyObject* /*self*/, PyObject* /*args*/)
+{
+    // Not really needed... qDebug() flushes already on its own
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef pyWarn_methods[] =
+{
+    {"write", pyWarn_write, METH_VARARGS, "Writes to stderr through qWarnging()"},
+    {"flush", pyWarn_flush, METH_VARARGS, "noop"},
+    {nullptr, nullptr, 0, nullptr} // sentinel
+};
+
+static PyModuleDef pyWarn_module =
+{
+    PyModuleDef_HEAD_INIT, // PyModuleDef_Base m_base;
+    "pyWarn",               // const char* m_name;
+    "pyWarn stdout override",// const char* m_doc;
+    -1,                    // Py_ssize_t m_size;
+    pyWarn_methods,        // PyMethodDef *m_methods
+    //  inquiry m_reload;  traverseproc m_traverse;  inquiry m_clear;  freefunc m_free;
+    nullptr, nullptr, nullptr, nullptr
+};
+
 #endif // PYUTILS_H

libnymea-core/integrations/pythonintegrationplugin.cpp

@@ -24,24 +24,6 @@ PyObject* PythonIntegrationPlugin::s_asyncio = nullptr;
 
 QHash<PythonIntegrationPlugin*, PyObject*> PythonIntegrationPlugin::s_plugins;
 
-// Write to stdout/stderr
-PyObject* nymea_write(PyObject* /*self*/, PyObject* args)
-{
-    const char *what;
-    if (!PyArg_ParseTuple(args, "s", &what))
-        return nullptr;
-    if (!QByteArray(what).trimmed().isEmpty()) {
-        qCDebug(dcThingManager()) << what;
-    }
-    Py_RETURN_NONE;
-}
-
-// Flush stdout/stderr
-PyObject* nymea_flush(PyObject* /*self*/, PyObject* /*args*/)
-{
-    // Not really needed... qDebug() flushes already on its own
-    Py_RETURN_NONE;
-}
 
 PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args)
 {
@@ -75,8 +57,6 @@ PyObject* PythonIntegrationPlugin::task_done(PyObject* self, PyObject* args)
 
 static PyMethodDef nymea_methods[] =
 {
-    {"write", nymea_write, METH_VARARGS, "write to stdout through qDebug()"},
-    {"flush", nymea_flush, METH_VARARGS, "flush stdout (no-op)"},
     {"task_done", PythonIntegrationPlugin::task_done, METH_VARARGS, "callback to clean up after asyc coroutines"},
     {nullptr, nullptr, 0, nullptr} // sentinel
 };
@@ -94,12 +74,15 @@ static PyModuleDef nymea_module =
 
 PyMODINIT_FUNC PyInit_nymea(void)
 {
-    PyObject* m = PyModule_Create(&nymea_module);
     // Overrride stdout/stderr to use qDebug instead
-    PySys_SetObject("stdout", m);
+    PyObject* pyLog = PyModule_Create(&pyLog_module);
-    PySys_SetObject("stderr", m);
+    PySys_SetObject("stdout", pyLog);
+    PyObject* pyWarn = PyModule_Create(&pyWarn_module);
+    PySys_SetObject("stderr", pyWarn);
 
 
+    // Register nymea types
+    PyObject* m = PyModule_Create(&nymea_module);
     registerNymeaLoggingHandler(m);
     registerParamType(m);
     registerThingType(m);
@@ -244,6 +227,21 @@ PyObject *PythonIntegrationPlugin::pyAutoThingsAppeared(PyObject *self, PyObject
     Py_RETURN_NONE;
 }
 
+PyObject *PythonIntegrationPlugin::pyAutoThingDisappeared(PyObject *self, PyObject *args)
+{
+    char *thingIdStr = nullptr;
+
+    if (!PyArg_ParseTuple(args, "s", &thingIdStr)) {
+        qCWarning(dcThingManager) << "Error parsing parameters";
+        return nullptr;
+    }
+    ThingId thingId(thingIdStr);
+    PythonIntegrationPlugin *plugin = s_plugins.key(self);
+    QMetaObject::invokeMethod(plugin, "autoThingDisappeared", Qt::QueuedConnection, Q_ARG(ThingId, thingId));
+
+    Py_RETURN_NONE;
+}
+
 static PyMethodDef plugin_methods[] =
 {
     {"configuration", PythonIntegrationPlugin::pyConfiguration, METH_VARARGS, "Get the plugin configuration."},
@@ -251,6 +249,7 @@ static PyMethodDef plugin_methods[] =
     {"setConfigValue", PythonIntegrationPlugin::pySetConfigValue, METH_VARARGS, "Set the plugin configuration value for a given config paramTypeId."},
     {"myThings", PythonIntegrationPlugin::pyMyThings, METH_VARARGS, "Obtain a list of things owned by this plugin."},
     {"autoThingsAppeared", PythonIntegrationPlugin::pyAutoThingsAppeared, METH_VARARGS, "Inform the system about auto setup things having appeared."},
+    {"autoThingDisappeared", PythonIntegrationPlugin::pyAutoThingDisappeared, METH_VARARGS, "Inform the system about auto setup things having disappeared."},
     {nullptr, nullptr, 0, nullptr} // sentinel
 };
 
@@ -311,9 +310,9 @@ bool PythonIntegrationPlugin::loadScript(const QString &scriptFile)
     m_module = PyImport_ImportModule(fi.baseName().toUtf8());
 
     if (!m_module) {
-        dumpError();
-        PyErr_Clear();
         qCWarning(dcThingManager()) << "Error importing python plugin from:" << fi.absoluteFilePath();
+        PyErr_Print();
+        PyErr_Clear();
         PyGILState_Release(s);
         return false;
     }
@@ -399,7 +398,6 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info)
     PyThingSetupInfo *pyInfo = (PyThingSetupInfo*)PyObject_CallObject((PyObject*)&PyThingSetupInfoType, NULL);
     pyInfo->info = info;
     pyInfo->pyThing = pyThing;
-    Py_INCREF(pyThing);
 
     m_things.insert(info->thing(), pyThing);
 
@@ -417,6 +415,7 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info)
 
     bool result = callPluginFunction("setupThing", reinterpret_cast<PyObject*>(pyInfo));
     if (!result) {
+        Py_DECREF(pyThing);
         // The python code did not even start, so let's finish (fail) the setup right away
         info->finish(Thing::ThingErrorSetupFailed);
     }
@@ -425,7 +424,11 @@ void PythonIntegrationPlugin::setupThing(ThingSetupInfo *info)
 void PythonIntegrationPlugin::postSetupThing(Thing *thing)
 {
     PyThing* pyThing = m_things.value(thing);
-    callPluginFunction("postSetupThing", reinterpret_cast<PyObject*>(pyThing));
+    Py_INCREF(pyThing);
+    bool success = callPluginFunction("postSetupThing", reinterpret_cast<PyObject*>(pyThing));
+    if (!success) {
+        Py_DECREF(pyThing);
+    }
 }
 
 void PythonIntegrationPlugin::executeAction(ThingActionInfo *info)
@@ -456,8 +459,12 @@ void PythonIntegrationPlugin::executeAction(ThingActionInfo *info)
 void PythonIntegrationPlugin::thingRemoved(Thing *thing)
 {
     PyThing *pyThing = m_things.value(thing);
+    Py_INCREF(pyThing);
 
-    callPluginFunction("thingRemoved", reinterpret_cast<PyObject*>(pyThing));
+    bool success = callPluginFunction("thingRemoved", reinterpret_cast<PyObject*>(pyThing));
+    if (!success) {
+        Py_DECREF(pyThing);
+    }
 
     m_mutex.lock();
     m_things.remove(thing);
@@ -577,12 +584,12 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje
 {
     PyGILState_STATE s = PyGILState_Ensure();
 
-    qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << s_plugins.key(m_module)->pluginName();
+    qCDebug(dcThingManager()) << "Calling python plugin function" << function << "on plugin" << pluginName();
     PyObject *pFunc = PyObject_GetAttrString(m_module, function.toUtf8());
     if(!pFunc || !PyCallable_Check(pFunc)) {
         PyErr_Clear();
         Py_XDECREF(pFunc);
-        qCWarning(dcThingManager()) << "Python plugin does not implement" << function;
+        qCWarning(dcThingManager()) << "Python plugin" << pluginName() << "does not implement" << function;
         PyGILState_Release(s);
         return false;
     }
@@ -595,8 +602,8 @@ bool PythonIntegrationPlugin::callPluginFunction(const QString &function, PyObje
     Py_XDECREF(pFunc);
 
     if (PyErr_Occurred()) {
-        qCWarning(dcThingManager()) << "Error calling python method:";
+        qCWarning(dcThingManager()) << "Error calling python method:" << function << "on plugin" << pluginName();
-        dumpError();
+        PyErr_Print();
         PyErr_Clear();
         PyGILState_Release(s);
         return false;
@@ -651,7 +658,6 @@ void PythonIntegrationPlugin::cleanupPyThing(PyThing *pyThing)
     // on the thing (e.g. PyThing_name).
     // We'd deadlock if we wait for the mutex forever here. So let's process events
     // while waiting for it...
-    qWarning() << "Locking cleanup";
     while (!pyThing->mutex->tryLock()) {
         qApp->processEvents(QEventLoop::EventLoopExec);
     }

libnymea-core/integrations/pythonintegrationplugin.h

@@ -40,6 +40,7 @@ public:
     static PyObject* pySetConfigValue(PyObject* self, PyObject* args);
     static PyObject* pyMyThings(PyObject *self, PyObject* args);
     static PyObject* pyAutoThingsAppeared(PyObject *self, PyObject* args);
+    static PyObject* pyAutoThingDisappeared(PyObject *self, PyObject* args);
 
 public:
     // python callbacks

libnymea/integrations/thingdiscoveryinfo.h

@@ -53,15 +53,15 @@ public:
 
     Thing::ThingError status() const;
 
-    void addThingDescriptor(const ThingDescriptor &thingDescriptor);
-    void addThingDescriptors(const ThingDescriptors &thingDescriptors);
-
     ThingDescriptors thingDescriptors() const;
 
     QString displayMessage() const;
     QString translatedDisplayMessage(const QLocale &locale);
 
 public slots:
+    void addThingDescriptor(const ThingDescriptor &thingDescriptor);
+    void addThingDescriptors(const ThingDescriptors &thingDescriptors);
+
     void finish(Thing::ThingError status,  const QString &displayMessage = QString());
 
 signals:

libnymea/loggingcategories.cpp

@@ -75,6 +75,7 @@ NYMEA_LOGGING_CATEGORY(dcBluetoothServerTraffic, "BluetoothServerTraffic")
 NYMEA_LOGGING_CATEGORY(dcMqtt, "Mqtt")
 NYMEA_LOGGING_CATEGORY(dcTranslations, "Translations")
 NYMEA_LOGGING_CATEGORY(dcI2C, "I2C")
+NYMEA_LOGGING_CATEGORY(dcPythonIntegrations, "PythonIntegrations")
 
 
 static QFile s_logFile;

plugins/pymock/integrationpluginpymock.py

@@ -1,13 +1,15 @@
 import nymea
 import asyncio
 
+watchingAutoThings = False
+
 def init():
     logger.log("Python mock plugin init")
 
 
 def configValueChanged(paramTypeId, value):
-    logger.log("Plugin config value changed:", paramTypeId, value)
+    logger.log("Plugin config value changed:", paramTypeId, value, watchingAutoThings)
-    if paramTypeId == pyMockPluginAutoThingCountParamTypeId:
+    if watchingAutoThings and paramTypeId == pyMockPluginAutoThingCountParamTypeId:
         logger.log("Auto Thing Count plugin config changed:", value, "Currently there are:", len(autoThings()), "auto things")
         things = autoThings();
         for i in range(len(things), value):
@@ -15,20 +17,33 @@ def configValueChanged(paramTypeId, value):
             descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing")
             autoThingsAppeared([descriptor])
 
-        for i in range(len(value), things):
+        for i in range(value, len(things)):
             logger.log("Removing auto thing")
             autoThingDisappeared(things[i].id)
 
 
 def startMonitoringAutoThings():
+    global watchingAutoThings
+    watchingAutoThings = True
     logger.log("Start monitoring auto things. Have %i auto devices. Need %i." % (len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId)))
-    for i in range(len(autoThings()), configValue(pyMockPluginAutoThingCountParamTypeId)):
+    things = autoThings();
+    for i in range(len(things), configValue(pyMockPluginAutoThingCountParamTypeId)):
         logger.log("Creating new auto thing")
         descriptor = nymea.ThingDescriptor(pyMockAutoThingClassId, "Python Mock auto thing")
         autoThingsAppeared([descriptor])
+    for i in range(configValue(pyMockPluginAutoThingCountParamTypeId), len(things)):
+        logger.log("Removing auto thing")
+        autoThingDisappeared(things[i].id)
+
     logger.log("Done start monitoring auto things")
 
 
+async def discoverThings(info):
+    await asyncio.sleep(1)
+    descriptor = nymea.ThingDescriptor(pyMockThingClassId, "Python mock thing")
+    info.addDescriptor(descriptor)
+    info.finish(nymea.ThingErrorNoError)
+
 async def setupThing(info):
     logger.log("setupThing for", info.thing.name)
     info.finish(nymea.ThingErrorNoError)