Merge branch 'main' into fix-burst-in-ws

CMakeLists.txt

@@ -172,6 +172,7 @@ else()
     if(CSP_BUILD_NO_CXX_ABI)
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
     endif()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
     if (COVERAGE)
         # TODO windows
         add_compile_options(--coverage)

Makefile

@@ -26,6 +26,9 @@ build-debug:  ## build the library ( DEBUG ) - May need a make clean when switch
 build-conda:  ## build the library in Conda
 	python setup.py build build_ext --csp-no-vcpkg --inplace
 
+build-conda-debug:  ## build the library ( DEBUG ) - in Conda
+	SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --csp-no-vcpkg --inplace
+
 install:  ## install library
 	python -m pip install .
 

conda/dev-environment-unix.yml

@@ -45,7 +45,6 @@ dependencies:
  - ruamel.yaml
  - ruff>=0.3,<0.4
  - scikit-build
- - slack-sdk
  - sqlalchemy
  - tar
  - threadpoolctl

conda/dev-environment-win.yml

@@ -44,7 +44,6 @@ dependencies:
  - ruamel.yaml
  - ruff>=0.3,<0.4
  - scikit-build
- - slack-sdk
  - sqlalchemy
  - threadpoolctl
  - tornado

cpp/csp/adapters/kafka/KafkaAdapterManager.h

@@ -47,7 +47,7 @@ struct KafkaStatusMessageTypeTraits
 using KafkaStatusMessageType = csp::Enum<KafkaStatusMessageTypeTraits>;
 
 //Top level AdapterManager object for all kafka adapters in the engine
-class KafkaAdapterManager final : public csp::AdapterManager
+class CSP_PUBLIC KafkaAdapterManager final : public csp::AdapterManager
 {
 public:
     KafkaAdapterManager( csp::Engine * engine, const Dictionary & properties );

cpp/csp/adapters/parquet/DialectGenericListReaderInterface.h

@@ -25,7 +25,7 @@ class DialectGenericListReaderInterface
 };
 
 template< typename T >
-class TypedDialectGenericListReaderInterface : public DialectGenericListReaderInterface
+class CSP_PUBLIC TypedDialectGenericListReaderInterface : public DialectGenericListReaderInterface
 {
 public:
     using Ptr = std::shared_ptr<TypedDialectGenericListReaderInterface<T>>;
@@ -45,4 +45,4 @@ class TypedDialectGenericListReaderInterface : public DialectGenericListReaderIn
 
 }
 
-#endif
+#endif

cpp/csp/adapters/parquet/ParquetInputAdapterManager.h

@@ -18,7 +18,7 @@ namespace csp::adapters::parquet
 
 
 //Top level AdapterManager object for all parquet adapters in the engine
-class ParquetInputAdapterManager final : public csp::AdapterManager
+class CSP_PUBLIC ParquetInputAdapterManager final : public csp::AdapterManager
 {
 public:
     using GeneratorPtr = csp::Generator<std::string, csp::DateTime, csp::DateTime>::Ptr;

cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h

@@ -21,7 +21,7 @@ class ParquetOutputFilenameAdapter;
 class ParquetDictBasketOutputWriter;
 
 //Top level AdapterManager object for all parquet adapters in the engine
-class ParquetOutputAdapterManager final : public csp::AdapterManager
+class CSP_PUBLIC ParquetOutputAdapterManager final : public csp::AdapterManager
 {
 public:
     using FileVisitorCallback = std::function<void(const std::string &)>;

cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp

@@ -734,7 +734,7 @@ void ListColumnAdapter<ValueArrayType, ValueType>::readCurValue()
     if( this -> m_curChunkArray -> IsValid( curRow ) )
     {
         auto values      = this -> m_curChunkArray -> value_slice( curRow );
-        auto typedValues = std::dynamic_pointer_cast<ValueArrayType>( values );
+        auto typedValues = std::static_pointer_cast<ValueArrayType>( values );
 
         auto arrayValue     = m_listReader -> create( typedValues -> length() );
         auto* internalBuffer = m_listReader -> getRawDataBuffer( arrayValue );

cpp/csp/adapters/websocket/ClientAdapterManager.h

@@ -40,10 +40,8 @@ struct WebsocketClientStatusTypeTraits
 
 using ClientStatusType = Enum<WebsocketClientStatusTypeTraits>;
 
-class ClientAdapterManager final : public AdapterManager
+class CSP_PUBLIC ClientAdapterManager final : public AdapterManager
 {
-
-
 public:
     ClientAdapterManager(
         Engine * engine,
@@ -78,4 +76,4 @@ class ClientAdapterManager final : public AdapterManager
 
 }
 
-#endif
+#endif

cpp/csp/core/Exception.h

@@ -10,7 +10,7 @@
 namespace csp
 {
 
-class Exception : public std::exception
+class CSP_PUBLIC Exception : public std::exception
 {
 public:
     Exception( const char * exType, const std::string & description, const char * file, const char * func, int line ) :
@@ -59,7 +59,7 @@ class Exception : public std::exception
 };
 
 #define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__)
-#define CSP_DECLARE_EXCEPTION( DerivedException, BaseException ) class DerivedException : public BaseException { public: DerivedException( const char * exType, const std::string &r, const char * file, const char * func, int line ) : BaseException( exType, r, file, func, line ) {} };
+#define CSP_DECLARE_EXCEPTION( DerivedException, BaseException ) class CSP_PUBLIC DerivedException : public BaseException { public: DerivedException( const char * exType, const std::string &r, const char * file, const char * func, int line ) : BaseException( exType, r, file, func, line ) {} };
 
 CSP_DECLARE_EXCEPTION( AssertionError,     Exception )
 CSP_DECLARE_EXCEPTION( RuntimeException,   Exception )

cpp/csp/core/Platform.h

@@ -14,7 +14,8 @@
 #undef ERROR
 #undef GetMessage
 
-#define DLL_LOCAL
+#define CSP_LOCAL
+#define CSP_PUBLIC __declspec(dllexport)
 
 #ifdef CSPTYPESIMPL_EXPORTS
 #define CSPTYPESIMPL_EXPORT __declspec(dllexport)
@@ -90,10 +91,11 @@ inline uint8_t ffs(uint64_t n)
 
 #else
 
-#define CSPIMPL_EXPORT
-#define CSPTYPESIMPL_EXPORT
+#define CSPIMPL_EXPORT      __attribute__ ((visibility ("default")))
+#define CSPTYPESIMPL_EXPORT __attribute__ ((visibility ("default")))
 
-#define DLL_LOCAL __attribute__ ((visibility ("hidden")))
+#define CSP_LOCAL  __attribute__ ((visibility ("hidden")))
+#define CSP_PUBLIC __attribute__ ((visibility ("default")))
 
 #define START_PACKED
 #define END_PACKED __attribute__((packed))

cpp/csp/engine/AdapterManager.h

@@ -93,7 +93,7 @@ bool ManagedSimInputAdapter::pushNullTick()
     return true;
 }
 
-class AdapterManager : public EngineOwned
+class CSP_PUBLIC AdapterManager : public EngineOwned
 {
 public:
     AdapterManager( csp::Engine * );

cpp/csp/engine/Feedback.h

@@ -28,7 +28,7 @@ class FeedbackOutputAdapter final : public OutputAdapter
 };
 
 template<typename T>
-class FeedbackInputAdapter final : public InputAdapter
+class CSP_PUBLIC FeedbackInputAdapter final : public InputAdapter
 {
 public:
     using InputAdapter::InputAdapter;

cpp/csp/engine/Struct.h

@@ -756,13 +756,13 @@ class Struct
     void decref()
     {
 //Work around GCC12 bug mis-identifying this code as use-after-free
-#ifdef __linux__
+#if defined(__linux__) && !defined(__clang__)
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wuse-after-free"
 #endif
         if( --hidden() -> refcount == 0 )
             delete this;
-#ifdef __linux__
+#if defined(__linux__) && !defined(__clang__)
 #pragma GCC diagnostic pop
 #endif
     }

cpp/csp/python/Exception.h

@@ -8,17 +8,28 @@
 namespace csp::python
 {
 
-class PythonPassthrough : public csp::Exception
+class CSP_PUBLIC PythonPassthrough : public csp::Exception
 {
 public:
     PythonPassthrough( const char * exType, const std::string &r, const char * file,
                        const char * func, int line ) :
         csp::Exception( exType, r, file, func, line )
     {
         //Fetch the current error to clear out the error indicator while the stack gets unwound
+        //We own the references to all the members assigned in PyErr_Fetch
+        //We need to hold the reference since PyErr_Restore takes back a reference to each of its arguments
         PyErr_Fetch( &m_type, &m_value, &m_traceback );
     }
 
+    PythonPassthrough( PyObject * pyException ) :
+        csp::Exception( "", "" )
+    {
+        // Note: all of these methods return strong references, so we own them like in the other constructor
+        m_type = PyObject_Type( pyException );
+        m_value = PyObject_Str( pyException );
+        m_traceback = PyException_GetTraceback( pyException );
+    }
+
     void restore()
     {
         if( !description().empty() )
@@ -39,7 +50,6 @@ class PythonPassthrough : public csp::Exception
     PyObject * m_type;
     PyObject * m_value;
     PyObject * m_traceback;
-
 };
 
 CSP_DECLARE_EXCEPTION( AttributeError, ::csp::Exception );

cpp/csp/python/InitHelper.h

@@ -10,7 +10,7 @@
 namespace csp::python
 {
 
-class DLL_LOCAL InitHelper
+class CSP_LOCAL InitHelper
 {
 public:
     ~InitHelper() {}
@@ -111,4 +111,21 @@ inline bool InitHelper::execute( PyObject * module )
 }
 
 }
+
+//PyMODINIT_FUNC in Python <3.9 doesn't export the function/make visible
+//this is required since we build with hidden visibility by default
+//the below macro code can be removed once 3.8 support is dropped
+//
+//see similar issues:
+//https://github.com/scipy/scipy/issues/15996
+//https://github.com/mesonbuild/meson/pull/10369
+
+#if PY_VERSION_HEX < 0x03090000
+#ifdef PyMODINIT_FUNC
+#undef PyMODINIT_FUNC
+#endif
+
+#define PyMODINIT_FUNC extern "C" CSP_PUBLIC PyObject*
+#endif
+
 #endif

cpp/csp/python/PyAdapterManager.cpp

@@ -68,6 +68,15 @@ class PyAdapterManager : public AdapterManager
 static PyObject * PyAdapterManager_PyObject_starttime( PyAdapterManager_PyObject * self ) { return toPython( self -> manager -> starttime() ); }
 static PyObject * PyAdapterManager_PyObject_endtime( PyAdapterManager_PyObject * self )   { return toPython( self -> manager -> endtime() ); }
 
+static PyObject * PyAdapterManager_PyObject_shutdown_engine( PyAdapterManager_PyObject * self, PyObject * pyException )
+{
+    CSP_BEGIN_METHOD;
+
+    self -> manager -> rootEngine() -> shutdown( PyEngine_shutdown_make_exception( pyException ) );
+
+    CSP_RETURN_NONE;
+}
+
 static int PyAdapterManager_init( PyAdapterManager_PyObject *self, PyObject *args, PyObject *kwds )
 {
     CSP_BEGIN_METHOD;
@@ -83,8 +92,9 @@ static int PyAdapterManager_init( PyAdapterManager_PyObject *self, PyObject *arg
 }
 
 static PyMethodDef PyAdapterManager_methods[] = {
-    { "starttime", (PyCFunction) PyAdapterManager_PyObject_starttime, METH_NOARGS, "starttime" },
-    { "endtime",   (PyCFunction) PyAdapterManager_PyObject_endtime,   METH_NOARGS, "endtime" },
+    { "starttime",          (PyCFunction) PyAdapterManager_PyObject_starttime, METH_NOARGS, "starttime" },
+    { "endtime",            (PyCFunction) PyAdapterManager_PyObject_endtime,    METH_NOARGS, "endtime" },
+    { "shutdown_engine",    (PyCFunction) PyAdapterManager_PyObject_shutdown_engine,  METH_O, "shutdown_engine" },
     {NULL}
 };
 

cpp/csp/python/PyCspEnum.h

@@ -27,7 +27,7 @@ struct CSPTYPESIMPL_EXPORT PyCspEnumMeta : public PyHeapTypeObject
     static PyTypeObject PyType;
 };
 
-//TODO Windows - need to figure out why adding DLL_PUBLIC to this class leads to weird compilation errors on CspEnumMeta's unordered_map...
+//TODO Windows - need to figure out why adding CSP_PUBLIC to this class leads to weird compilation errors on CspEnumMeta's unordered_map...
 
 //This is an extension of csp::CspEnumMeta for python dialect, we need it in order to 
 //keep a reference to the python enum type from conversion to/from csp::CspEnumMeta <-> PyObject properly

cpp/csp/python/PyEngine.h

@@ -2,6 +2,7 @@
 #define _IN_CSP_PYTHON_PYENGINE_H
 
 #include <csp/core/Time.h>
+#include <csp/python/Exception.h>
 #include <csp/engine/RootEngine.h>
 #include <csp/python/PyObjectPtr.h>
 #include <Python.h>
@@ -54,6 +55,21 @@ class CSPIMPL_EXPORT PyEngine final: public PyObject
     Engine    * m_engine;
 };
 
+inline std::exception_ptr PyEngine_shutdown_make_exception( PyObject * pyException )
+{
+    if( !PyExceptionInstance_Check( pyException ) )
+    {
+        PyObjectPtr pyExceptionStr = PyObjectPtr::own( PyObject_Str( pyException ) );
+        if( !pyExceptionStr.ptr() )
+            CSP_THROW( PythonPassthrough, "" );
+        std::string pyExceptionString = PyUnicode_AsUTF8( pyExceptionStr.ptr() );
+        std::string desc = "Expected Exception object as argument for shutdown_engine: got " + pyExceptionString + " of type " + Py_TYPE( pyException ) -> tp_name;
+        return std::make_exception_ptr( csp::Exception( "TypeError", desc ) );
+    }
+    else
+        return std::make_exception_ptr( PythonPassthrough( pyException ) );
+}
+
 };
 
 #endif

cpp/csp/python/PyPushInputAdapter.cpp

@@ -198,11 +198,21 @@ struct PyPushInputAdapter_PyObject
         CSP_RETURN_NONE;
     }
 
+    static PyObject * shutdown_engine( PyPushInputAdapter_PyObject * self, PyObject * pyException )
+    {
+        CSP_BEGIN_METHOD;
+
+        self -> adapter -> rootEngine() -> shutdown( PyEngine_shutdown_make_exception( pyException ) );
+
+        CSP_RETURN_NONE;
+    }
+
     static PyTypeObject PyType;
 };
 
 static PyMethodDef PyPushInputAdapter_PyObject_methods[] = {
-    { "push_tick", (PyCFunction) PyPushInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" },
+    { "push_tick",          (PyCFunction) PyPushInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" },
+    { "shutdown_engine",    (PyCFunction) PyPushInputAdapter_PyObject::shutdown_engine, METH_O, "shutdown_engine" },
     {NULL}
 };
 

cpp/csp/python/PyPushPullInputAdapter.cpp

@@ -119,12 +119,22 @@ struct PyPushPullInputAdapter_PyObject
         CSP_RETURN_NONE;
     }
 
+    static PyObject * shutdown_engine( PyPushPullInputAdapter_PyObject * self, PyObject * pyException )
+    {
+        CSP_BEGIN_METHOD;
+
+        self -> adapter -> rootEngine() -> shutdown( PyEngine_shutdown_make_exception( pyException ) );
+
+        CSP_RETURN_NONE;
+    }
+
     static PyTypeObject PyType;
 };
 
 static PyMethodDef PyPushPullInputAdapter_PyObject_methods[] = {
-    { "push_tick", (PyCFunction) PyPushPullInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" },
-    { "flag_replay_complete", (PyCFunction) PyPushPullInputAdapter_PyObject::flagReplayComplete, METH_VARARGS, "finish replay ticks" },
+    { "push_tick",              (PyCFunction) PyPushPullInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" },
+    { "flag_replay_complete",   (PyCFunction) PyPushPullInputAdapter_PyObject::flagReplayComplete, METH_VARARGS, "finish replay ticks" },
+    { "shutdown_engine",        (PyCFunction) PyPushPullInputAdapter_PyObject::shutdown_engine, METH_O, "shutdown engine" },
     {NULL}
 };
 

cpp/csp/python/adapters/CMakeLists.txt

@@ -35,7 +35,7 @@ if(CSP_BUILD_PARQUET_ADAPTER)
     endif()
     target_link_libraries(parquetadapterimpl csp_core csp_engine cspimpl csp_parquet_adapter)
     target_include_directories(parquetadapterimpl PUBLIC ${ARROW_INCLUDE_DIR} ${PARQUET_INCLUDE_DIR} "${VENDORED_PYARROW_ROOT}")
-    target_compile_definitions(parquetadapterimpl PUBLIC ARROW_PYTHON_STATIC)
+    target_compile_definitions(parquetadapterimpl PUBLIC ARROW_PYTHON_STATIC -DARROW_PYTHON_EXPORT=)
     install(TARGETS parquetadapterimpl RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR} )
 endif()