From 5deccad943fe196496303054078a268f79a128d1 Mon Sep 17 00:00:00 2001 From: Arham Chopra Date: Thu, 13 Jun 2024 12:19:12 -0400 Subject: [PATCH 01/17] Add postprocess_to_dict hook for to_dict method in structs Signed-off-by: Arham Chopra --- cpp/csp/python/PyStructToDict.cpp | 8 ++++ csp/impl/struct.py | 10 ++++ csp/tests/impl/test_struct.py | 78 +++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/cpp/csp/python/PyStructToDict.cpp b/cpp/csp/python/PyStructToDict.cpp index 83ea191a4..9f5212765 100644 --- a/cpp/csp/python/PyStructToDict.cpp +++ b/cpp/csp/python/PyStructToDict.cpp @@ -108,6 +108,14 @@ PyObjectPtr parseStructToDictRecursive( const StructPtr& self, PyObject * callab } ); PyDict_SetItemString( new_dict.get(), key.c_str(), py_obj.get() ); } + + // Optional postprocess hook in python to allow caller to customize to_dict behavior for struct + PyObject * py_type = ( PyObject * ) meta -> pyType(); + if( PyObject_HasAttrString( py_type, "postprocess_to_dict" ) ) + { + auto postprocess_dict_callable = PyObjectPtr::own( PyObject_GetAttrString( py_type, "postprocess_to_dict" ) ); + new_dict = PyObjectPtr::check( PyObject_CallFunction( postprocess_dict_callable.get(), "(O)", new_dict.get() ) ); + } return new_dict; } diff --git a/csp/impl/struct.py b/csp/impl/struct.py index bcdab8454..88910edfd 100644 --- a/csp/impl/struct.py +++ b/csp/impl/struct.py @@ -165,6 +165,16 @@ def to_dict_depr(self): res = self._obj_to_python(self) return res + @classmethod + def postprocess_to_dict(self, obj): + """Postprocess hook for to_dict method + + This method is invoked by to_dict after converting a struct to a dict + as an additional hook for users to modify the dict before it is returned + by the to_dict method + """ + return obj + def to_dict(self, callback=None): """Create a dictionary representation of the struct diff --git a/csp/tests/impl/test_struct.py b/csp/tests/impl/test_struct.py index db0fd852b..fe5bd288a 100644 --- a/csp/tests/impl/test_struct.py +++ b/csp/tests/impl/test_struct.py @@ -1376,6 +1376,84 @@ class A(csp.Struct): r = repr(a) self.assertTrue(repr(raw) in r) + def test_to_dict_recursion(self): + class MyStruct(csp.Struct): + l1: list + l2: list + d1: dict + d2: dict + t1: tuple + t2: tuple + + test_struct = MyStruct(l1=[1], l2=[2]) + result_dict = {"l1": [1], "l2": [2]} + self.assertEqual(test_struct.to_dict(), result_dict) + + test_struct = MyStruct(l1=[1], l2=[2]) + test_struct.l1.append(test_struct.l2) + test_struct.l2.append(test_struct.l1) + with self.assertRaises(RecursionError): + test_struct.to_dict() + + test_struct = MyStruct(l1=[1]) + test_struct.l1.append(test_struct.l1) + with self.assertRaises(RecursionError): + test_struct.to_dict() + + test_struct = MyStruct(l1=[1]) + test_struct.l1.append(test_struct) + with self.assertRaises(RecursionError): + test_struct.to_dict() + + test_struct = MyStruct(d1={1: 1}, d2={2: 2}) + result_dict = {"d1": {1: 1}, "d2": {2: 2}} + self.assertEqual(test_struct.to_dict(), result_dict) + + test_struct = MyStruct(d1={1: 1}, d2={2: 2}) + test_struct.d1["d2"] = test_struct.d2 + test_struct.d2["d1"] = test_struct.d1 + with self.assertRaises(RecursionError): + test_struct.to_dict() + + test_struct = MyStruct(d1={1: 1}, d2={2: 2}) + test_struct.d1["d1"] = test_struct.d1 + with self.assertRaises(RecursionError): + test_struct.to_dict() + + test_struct = MyStruct(d1={1: 1}, d2={2: 2}) + test_struct.d1["d1"] = test_struct + with self.assertRaises(RecursionError): + test_struct.to_dict() + + test_struct = MyStruct(t1=(1, 1), t2=(2, 2)) + result_dict = {"t1": (1, 1), "t2": (2, 2)} + self.assertEqual(test_struct.to_dict(), result_dict) + + test_struct = MyStruct(t1=(1, 1)) + test_struct.t1 = (1, 2, test_struct) + with self.assertRaises(RecursionError): + test_struct.to_dict() + + def test_to_dict_postprocess(self): + class MySubStruct(csp.Struct): + i: int = 0 + + def postprocess_to_dict(obj): + obj["postprocess_called"] = True + return obj + + class MyStruct(csp.Struct): + i: int = 1 + mss: MySubStruct = MySubStruct() + + def postprocess_to_dict(obj): + obj["postprocess_called"] = True + return obj + + test_struct = MyStruct() + result_dict = {"i": 1, "postprocess_called": True, "mss": {"i": 0, "postprocess_called": True}} + self.assertEqual(test_struct.to_dict(), result_dict) + def test_to_json_primitives(self): class MyStruct(csp.Struct): b: bool = True From d625e4da422c602cb005484093e54b97b76852dc Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Mon, 17 Jun 2024 09:55:45 -0400 Subject: [PATCH 02/17] Remove note about gnu toolchain from docs Signed-off-by: Tim Paine --- docs/wiki/dev-guides/Build-CSP-from-Source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wiki/dev-guides/Build-CSP-from-Source.md b/docs/wiki/dev-guides/Build-CSP-from-Source.md index fc480feae..df2ae0efb 100644 --- a/docs/wiki/dev-guides/Build-CSP-from-Source.md +++ b/docs/wiki/dev-guides/Build-CSP-from-Source.md @@ -45,7 +45,7 @@ test run the tests ## Prerequisites -CSP has a few system-level dependencies which you can install from your machine package manager. Other package managers like `conda`, `nix`, etc, should also work fine. Currently, CSP relies on the `GNU` compiler toolchain only. +CSP has a few system-level dependencies which you can install from your machine package manager. Other package managers like `conda`, `nix`, etc, should also work fine. ## Building with Conda on Linux From 8fa4e977ef7271a03f9a764912b12dc0432e7921 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Mon, 17 Jun 2024 10:03:36 -0400 Subject: [PATCH 03/17] fix listing/linting typo Signed-off-by: Tim Paine --- docs/wiki/dev-guides/Build-CSP-from-Source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wiki/dev-guides/Build-CSP-from-Source.md b/docs/wiki/dev-guides/Build-CSP-from-Source.md index df2ae0efb..4210e0823 100644 --- a/docs/wiki/dev-guides/Build-CSP-from-Source.md +++ b/docs/wiki/dev-guides/Build-CSP-from-Source.md @@ -196,7 +196,7 @@ By default, we pull and build dependencies with [vcpkg](https://vcpkg.io/en/). W ## Lint and Autoformat -CSP has listing and auto formatting. +CSP has linting and auto formatting. | Language | Linter | Autoformatter | Description | | :------- | :------------- | :------------- | :---------- | From 489922d7de8d0b23d684b7b0d424c486deb6f9ad Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Mon, 17 Jun 2024 10:06:09 -0400 Subject: [PATCH 04/17] add markdown linters to wiki Signed-off-by: Tim Paine --- docs/wiki/dev-guides/Build-CSP-from-Source.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/wiki/dev-guides/Build-CSP-from-Source.md b/docs/wiki/dev-guides/Build-CSP-from-Source.md index 4210e0823..c24299f08 100644 --- a/docs/wiki/dev-guides/Build-CSP-from-Source.md +++ b/docs/wiki/dev-guides/Build-CSP-from-Source.md @@ -203,6 +203,8 @@ CSP has linting and auto formatting. | C++ | `clang-format` | `clang-format` | Style | | Python | `ruff` | `ruff` | Style | | Python | `isort` | `isort` | Imports | +| Markdown | `mdformat` | `mdformat` | Style | +| Markdown | `codespell` | | Spelling | **C++ Linting** From 95846180fc12bc885209b2b8c9b3a8b648e92acb Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Mon, 17 Jun 2024 15:18:08 -0400 Subject: [PATCH 05/17] Force numpy<2 for now Signed-off-by: Tim Paine --- conda/dev-environment-unix.yml | 2 +- conda/dev-environment-win.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/dev-environment-unix.yml b/conda/dev-environment-unix.yml index 385370523..65d72f4f5 100644 --- a/conda/dev-environment-unix.yml +++ b/conda/dev-environment-unix.yml @@ -26,7 +26,7 @@ dependencies: - mamba - mdformat>=0.7.17,<0.8 - ninja - - numpy + - numpy<2 - pandas - pillow - polars diff --git a/conda/dev-environment-win.yml b/conda/dev-environment-win.yml index 1e46c7892..5610ce3f7 100644 --- a/conda/dev-environment-win.yml +++ b/conda/dev-environment-win.yml @@ -24,7 +24,7 @@ dependencies: - mamba - mdformat>=0.7.17,<0.8 - ninja - - numpy + - numpy<2 - pandas - pillow - polars diff --git a/pyproject.toml b/pyproject.toml index 55ebed0f4..cfef783a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ requires-python = ">=3.8" dependencies = [ "backports.zoneinfo; python_version<'3.9'", - "numpy", + "numpy<2", "packaging", "pandas", "psutil", From 225bd062de0fce253a21bf5a66c5dbb72d2866f4 Mon Sep 17 00:00:00 2001 From: Will Rieger Date: Tue, 4 Jun 2024 16:28:55 -0400 Subject: [PATCH 06/17] auto resolve port based on protocol Signed-off-by: Will Rieger --- csp/adapters/websocket.py | 10 ++++++++-- csp/tests/adapters/test_websocket.py | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/csp/adapters/websocket.py b/csp/adapters/websocket.py index e418088b9..d7f9cf86e 100644 --- a/csp/adapters/websocket.py +++ b/csp/adapters/websocket.py @@ -407,13 +407,19 @@ def __init__( resp = urllib.parse.urlparse(uri) self._properties = dict( host=resp.hostname, - port=str(resp.port) or "80", - route=resp.path, + # if no port is explicitly present in the uri, the resp.port is None + port=self._sanitize_port(uri, resp.port), + route=resp.path or "/", # resource shouldn't be empty string use_ssl=uri.startswith("wss"), reconnect_interval=reconnect_interval, headers=headers if headers else {}, ) + def _sanitize_port(self, uri: str, port): + if port: + return str(port) + return "443" if uri.startswith("wss") else "80" + def subscribe( self, ts_type: type, diff --git a/csp/tests/adapters/test_websocket.py b/csp/tests/adapters/test_websocket.py index 414ef0341..bd2c6b6ba 100644 --- a/csp/tests/adapters/test_websocket.py +++ b/csp/tests/adapters/test_websocket.py @@ -117,3 +117,12 @@ def g(n: int): msgs = csp.run(g, n, starttime=datetime.now(pytz.UTC), realtime=True) assert len(msgs["recv"]) == n assert msgs["recv"][0][1] != msgs["recv"][-1][1] + + def test_unkown_host_graceful_shutdown(self): + @csp.graph + def g(): + ws = WebsocketAdapterManager("wss://localhost/") + assert ws._properties["port"] == "443" + csp.stop_engine(ws.status()) + + csp.run(g, starttime=datetime.now(pytz.UTC), realtime=True) From c7a20881c19514c5ac227c2063589a7d34cfce20 Mon Sep 17 00:00:00 2001 From: Petr Ogarok Date: Tue, 18 Jun 2024 17:57:20 +0200 Subject: [PATCH 07/17] Implement pickling for PyStructList and FastList (#285) * Attempt to pinpoint PicklingError: unable to import _cspimpl Signed-off-by: Petr Ogarok * Remove note about gnu toolchain from docs Signed-off-by: Tim Paine Signed-off-by: Petr Ogarok * fix listing/linting typo Signed-off-by: Tim Paine Signed-off-by: Petr Ogarok * add markdown linters to wiki Signed-off-by: Tim Paine Signed-off-by: Petr Ogarok * Force numpy<2 for now Signed-off-by: Tim Paine Signed-off-by: Petr Ogarok * Implement pickling for PyStructList and FastList Signed-off-by: Petr Ogarok * Clean up code changes for pickling impl Signed-off-by: Petr Ogarok * Fix linting Signed-off-by: Petr Ogarok * Remove Py_NewRef as not available in PY3.8 Signed-off-by: Petr Ogarok * Address code review comments Signed-off-by: Petr Ogarok --------- Signed-off-by: Petr Ogarok Signed-off-by: Tim Paine Co-authored-by: Tim Paine --- cpp/csp/python/PyStructFastList_impl.h | 14 ++++++++++ cpp/csp/python/PyStructList_impl.h | 30 ++++++++++++++------ csp/tests/impl/test_struct.py | 38 ++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/cpp/csp/python/PyStructFastList_impl.h b/cpp/csp/python/PyStructFastList_impl.h index 17dc96533..666c42253 100644 --- a/cpp/csp/python/PyStructFastList_impl.h +++ b/cpp/csp/python/PyStructFastList_impl.h @@ -511,6 +511,19 @@ static PyObject * PyStructFastList_Reversed( PyStructFastList * self, CSP_RETURN_NULL; } +template +static PyObject * PyStructFastList_reduce( PyStructFastList * self, PyObject * Py_UNUSED( ignored) ) +{ + CSP_BEGIN_METHOD; + + typename VectorWrapper::IndexType sz = self -> vector.size(); + PyObjectPtr list = PyObjectPtr::own( toPython( self -> vector.getVector(), self -> arrayType ) ); + PyObject * result = Py_BuildValue( "O(O)", &PyList_Type, list.ptr() ); + return result; + + CSP_RETURN_NULL; +} + template static PyMethodDef PyStructFastList_methods[] = { { "__getitem__", ( PyCFunction ) py_struct_fast_list_subscript, METH_VARARGS, NULL }, @@ -527,6 +540,7 @@ static PyMethodDef PyStructFastList_methods[] = { { "extend", ( PyCFunction ) PyStructFastList_Extend, METH_VARARGS, NULL }, { "remove", ( PyCFunction ) PyStructFastList_Remove, METH_VARARGS, NULL }, { "clear", ( PyCFunction ) PyStructFastList_Clear, METH_NOARGS, NULL }, + {"__reduce__", ( PyCFunction ) PyStructFastList_reduce, METH_NOARGS, NULL }, { NULL}, }; diff --git a/cpp/csp/python/PyStructList_impl.h b/cpp/csp/python/PyStructList_impl.h index b488894ab..b80c0f23a 100644 --- a/cpp/csp/python/PyStructList_impl.h +++ b/cpp/csp/python/PyStructList_impl.h @@ -297,16 +297,30 @@ static PyObject * py_struct_list_inplace_repeat( PyObject * sself, Py_ssize_t n CSP_RETURN_NULL; } +template +static PyObject * PyStructList_reduce( PyStructList * self, PyObject * Py_UNUSED( ignored ) ) +{ + CSP_BEGIN_METHOD; + + typename VectorWrapper::IndexType sz = self -> vector.size(); + PyObjectPtr list = PyObjectPtr::own( toPython( self -> vector.getVector(), self -> arrayType ) ); + PyObject * result = Py_BuildValue( "O(O)", &PyList_Type, list.ptr() ); + return result; + + CSP_RETURN_NULL; +} + template static PyMethodDef PyStructList_methods[] = { - { "append", ( PyCFunction ) PyStructList_Append, METH_VARARGS, NULL }, - { "insert", ( PyCFunction ) PyStructList_Insert, METH_VARARGS, NULL }, - { "pop", ( PyCFunction ) PyStructList_Pop, METH_VARARGS, NULL }, - { "reverse", ( PyCFunction ) PyStructList_Reverse, METH_NOARGS, NULL }, - { "sort", ( PyCFunction ) PyStructList_Sort, METH_VARARGS | METH_KEYWORDS, NULL }, - { "extend", ( PyCFunction ) PyStructList_Extend, METH_VARARGS, NULL }, - { "remove", ( PyCFunction ) PyStructList_Remove, METH_VARARGS, NULL }, - { "clear", ( PyCFunction ) PyStructList_Clear, METH_NOARGS, NULL }, + { "append", ( PyCFunction ) PyStructList_Append, METH_VARARGS, NULL }, + { "insert", ( PyCFunction ) PyStructList_Insert, METH_VARARGS, NULL }, + { "pop", ( PyCFunction ) PyStructList_Pop, METH_VARARGS, NULL }, + { "reverse", ( PyCFunction ) PyStructList_Reverse, METH_NOARGS, NULL }, + { "sort", ( PyCFunction ) PyStructList_Sort, METH_VARARGS | METH_KEYWORDS, NULL }, + { "extend", ( PyCFunction ) PyStructList_Extend, METH_VARARGS, NULL }, + { "remove", ( PyCFunction ) PyStructList_Remove, METH_VARARGS, NULL }, + { "clear", ( PyCFunction ) PyStructList_Clear, METH_NOARGS, NULL }, + {"__reduce__", ( PyCFunction ) PyStructList_reduce, METH_NOARGS, NULL }, { NULL}, }; diff --git a/csp/tests/impl/test_struct.py b/csp/tests/impl/test_struct.py index db0fd852b..7ff5b89da 100644 --- a/csp/tests/impl/test_struct.py +++ b/csp/tests/impl/test_struct.py @@ -1,5 +1,6 @@ import json import numpy as np +import pickle import pytz import typing import unittest @@ -169,6 +170,14 @@ def __init__(self, x: int): self.x = x +class SimpleStructForPickleList(csp.Struct): + a: typing.List[int] + + +class SimpleStructForPickleFastList(csp.Struct): + a: FastList[int] + + # Common set of values for Struct list field tests # For each type: # items[:-2] are normal values of the given type that should be handled, @@ -2745,6 +2754,35 @@ class B(csp.Struct): p = csp.unroll(csp.const(A(a=[1, 2, 3])).a) q = csp.unroll(csp.const(B(a=[1, 2, 3])).a) + def test_list_field_pickle(self): + """Was a BUG when the struct with list field was not recognizing changes made to this field in python""" + # Not using pystruct_list_test_values, as pickling tests are of different semantics (picklability of struct fields matters). + v = [1, 5, 2] + + s = SimpleStructForPickleList(a=[v[0], v[1], v[2]]) + + t = pickle.loads(pickle.dumps(s)) + + self.assertEqual(t.a, s.a) + self.assertEqual(type(t.a), type(s.a)) + + b = pickle.loads(pickle.dumps(s.a)) + + self.assertEqual(b, s.a) + self.assertEqual(type(b), list) + + s = SimpleStructForPickleFastList(a=[v[0], v[1], v[2]]) + + t = pickle.loads(pickle.dumps(s)) + + self.assertEqual(t.a, s.a) + self.assertEqual(type(t.a), type(s.a)) + + b = pickle.loads(pickle.dumps(s.a)) + + self.assertEqual(b, s.a) + self.assertEqual(type(b), list) + if __name__ == "__main__": unittest.main() From ade60e1575b119192f3fb16deea829473d7b54ec Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Thu, 13 Jun 2024 16:04:00 -0400 Subject: [PATCH 08/17] Fix two bugs in unadjusted, tick-based EMA when NaNs are present in the data Signed-off-by: Adam Glustein --- cpp/csp/cppnodes/statsimpl.h | 14 +++++++++++++- csp/tests/test_stats.py | 11 +++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cpp/csp/cppnodes/statsimpl.h b/cpp/csp/cppnodes/statsimpl.h index 43bdee451..50f2d7492 100644 --- a/cpp/csp/cppnodes/statsimpl.h +++ b/cpp/csp/cppnodes/statsimpl.h @@ -1,3 +1,6 @@ +#ifndef _IN_CSP_CPPNODES_STATSIMPL_H +#define _IN_CSP_CPPNODES_STATSIMPL_H + #include #include @@ -1582,8 +1585,9 @@ class EMA } else { - m_ema = ( pow( m_ema * ( 1 - m_alpha ), m_offset ) + m_alpha * x ) / + m_ema = ( m_ema * pow( ( 1 - m_alpha ), m_offset ) + m_alpha * x ) / ( pow( 1 - m_alpha, m_offset ) + m_alpha ); + m_offset = 1; } } } @@ -1730,6 +1734,12 @@ class AlphaDebiasEMA w0 = 1 - m_decay; m_sqsum += pow( w0, 2 ); m_wsum += w0; + if( !m_adjust ) + { + double correction = decay_factor + w0; + m_wsum /= correction; + m_sqsum /= ( correction * correction ); + } } else { @@ -2358,3 +2368,5 @@ DECLARE_CPPNODE( _generic_cross_sectional ) }; } + +#endif // _IN_CSP_CPPNODES_STATSIMPL_H \ No newline at end of file diff --git a/csp/tests/test_stats.py b/csp/tests/test_stats.py index 4d8e2c79b..3a98498e5 100644 --- a/csp/tests/test_stats.py +++ b/csp/tests/test_stats.py @@ -465,6 +465,11 @@ def graph(): def test_ema(self): dvalues = np.random.uniform(low=-100, high=100, size=(1000,)) + for i in range(1000): + p = np.random.rand() + if p < 0.1: + dvalues[i] = np.nan + st = datetime(2020, 1, 1) @csp.graph @@ -473,10 +478,12 @@ def graph(): ema = csp.stats.ema(x, alpha=0.1, adjust=False) ema_var = csp.stats.ema_var(x, min_periods=3, span=20, adjust=True, bias=True) ema_std = csp.stats.ema_std(x, min_periods=3, span=20, adjust=True, bias=False) + ema_std2 = csp.stats.ema_std(x, min_periods=3, span=20, adjust=False, ignore_na=False, bias=False) csp.add_graph_output("ema", ema) csp.add_graph_output("ema_v", ema_var) csp.add_graph_output("ema_s", ema_std) + csp.add_graph_output("ema_s2", ema_std2) values = pd.Series(dvalues) pd_alpha = values.ewm(alpha=0.1, adjust=False).mean() @@ -484,12 +491,16 @@ def graph(): pd_var = pd_span.var(bias=True) pd_std = pd_span.std(bias=False) + pd_span2 = values.ewm(span=20, adjust=False, ignore_na=False) + pd_std2 = pd_span2.std(bias=False) + results = csp.run(graph, starttime=st, endtime=st + timedelta(milliseconds=1000)) # floats, ensure accurate to 1e-6 np.testing.assert_almost_equal(np.array(pd_alpha), np.array(results["ema"])[:, 1], decimal=7) np.testing.assert_almost_equal(np.array(pd_var)[2:], np.array(results["ema_v"])[:, 1], decimal=7) np.testing.assert_almost_equal(np.array(pd_std)[2:], np.array(results["ema_s"])[:, 1], decimal=7) + np.testing.assert_almost_equal(np.array(pd_std2)[2:], np.array(results["ema_s2"])[:, 1], decimal=7) def test_triggers(self): dvalues = [i + 1 for i in range(20)] From fe9762eb0a137ad17f4e00330ebec464f5460a9e Mon Sep 17 00:00:00 2001 From: Sorin Vatasoiu Date: Tue, 18 Jun 2024 17:29:51 -0400 Subject: [PATCH 09/17] fix nan handling when first values are nan Signed-off-by: Sorin Vatasoiu --- cpp/csp/cppnodes/statsimpl.h | 10 +++++----- csp/tests/test_stats.py | 28 +++++++++++++++++++--------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/cpp/csp/cppnodes/statsimpl.h b/cpp/csp/cppnodes/statsimpl.h index 50f2d7492..00b59aec1 100644 --- a/cpp/csp/cppnodes/statsimpl.h +++ b/cpp/csp/cppnodes/statsimpl.h @@ -1572,7 +1572,7 @@ class EMA m_ema = x; m_first = false; } - else if( unlikely( isnan( x ) ) && !m_ignore_na ) + else if( unlikely( isnan( x ) ) && !m_ignore_na && likely( !m_first ) ) { m_offset++; } @@ -1603,7 +1603,7 @@ class EMA double compute() const { - return m_ema; + return unlikely( m_first ) ? std::numeric_limits::quiet_NaN() : m_ema; } private: @@ -1714,7 +1714,7 @@ class AlphaDebiasEMA void add( double x ) { - if( m_first ) + if( m_first && likely( !isnan( x ) ) ) { m_wsum = 1; m_sqsum = 1; @@ -1741,7 +1741,7 @@ class AlphaDebiasEMA m_sqsum /= ( correction * correction ); } } - else + else if ( likely( !m_first ) ) { m_offset++; m_nan_count++; @@ -2369,4 +2369,4 @@ DECLARE_CPPNODE( _generic_cross_sectional ) } -#endif // _IN_CSP_CPPNODES_STATSIMPL_H \ No newline at end of file +#endif // _IN_CSP_CPPNODES_STATSIMPL_H diff --git a/csp/tests/test_stats.py b/csp/tests/test_stats.py index 3a98498e5..3e765dd75 100644 --- a/csp/tests/test_stats.py +++ b/csp/tests/test_stats.py @@ -464,17 +464,19 @@ def graph(): np.testing.assert_almost_equal(excess_kurtosis, kurtosis - 3, decimal=7) def test_ema(self): - dvalues = np.random.uniform(low=-100, high=100, size=(1000,)) - for i in range(1000): + N = 1000 + dvalues = np.random.uniform(low=-100, high=100, size=(N,)) + dvalues[0] = np.nan # this forces edge cases around first value being nan + for i in range(N): p = np.random.rand() - if p < 0.1: + if p < 0.2: dvalues[i] = np.nan st = datetime(2020, 1, 1) @csp.graph def graph(): - x = csp.curve(typ=float, data=[(st + timedelta(milliseconds=i + 1), dvalues[i]) for i in range(1000)]) + x = csp.curve(typ=float, data=[(st + timedelta(milliseconds=i + 1), dvalues[i]) for i in range(N)]) ema = csp.stats.ema(x, alpha=0.1, adjust=False) ema_var = csp.stats.ema_var(x, min_periods=3, span=20, adjust=True, bias=True) ema_std = csp.stats.ema_std(x, min_periods=3, span=20, adjust=True, bias=False) @@ -496,11 +498,19 @@ def graph(): results = csp.run(graph, starttime=st, endtime=st + timedelta(milliseconds=1000)) - # floats, ensure accurate to 1e-6 - np.testing.assert_almost_equal(np.array(pd_alpha), np.array(results["ema"])[:, 1], decimal=7) - np.testing.assert_almost_equal(np.array(pd_var)[2:], np.array(results["ema_v"])[:, 1], decimal=7) - np.testing.assert_almost_equal(np.array(pd_std)[2:], np.array(results["ema_s"])[:, 1], decimal=7) - np.testing.assert_almost_equal(np.array(pd_std2)[2:], np.array(results["ema_s2"])[:, 1], decimal=7) + # floats, ensure accurate to 1.5e-7 + np.testing.assert_allclose( + np.array(pd_alpha), np.array(results["ema"])[:, 1].astype(np.float64), atol=1.5e-7, equal_nan=True + ) + np.testing.assert_allclose( + np.array(pd_var)[2:], np.array(results["ema_v"])[:, 1].astype(np.float64), atol=1.5e-7, equal_nan=True + ) + np.testing.assert_allclose( + np.array(pd_std)[2:], np.array(results["ema_s"])[:, 1].astype(np.float64), atol=1.5e-7, equal_nan=True + ) + np.testing.assert_allclose( + np.array(pd_std2)[2:], np.array(results["ema_s2"])[:, 1].astype(np.float64), atol=1.5e-7, equal_nan=True + ) def test_triggers(self): dvalues = [i + 1 for i in range(20)] From 7b38c875685733569cda4e82fbd47801f12aa910 Mon Sep 17 00:00:00 2001 From: Sorin Vatasoiu Date: Thu, 20 Jun 2024 11:52:45 -0400 Subject: [PATCH 10/17] optimize halflife-based EMA's Signed-off-by: Sorin Vatasoiu --- cpp/csp/cppnodes/statsimpl.h | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/cpp/csp/cppnodes/statsimpl.h b/cpp/csp/cppnodes/statsimpl.h index 00b59aec1..ee5cf2679 100644 --- a/cpp/csp/cppnodes/statsimpl.h +++ b/cpp/csp/cppnodes/statsimpl.h @@ -1732,7 +1732,7 @@ class AlphaDebiasEMA w0 = 1.0; else w0 = 1 - m_decay; - m_sqsum += pow( w0, 2 ); + m_sqsum += w0 * w0; m_wsum += w0; if( !m_adjust ) { @@ -1756,7 +1756,7 @@ class AlphaDebiasEMA double wh = pow( m_decay, lookback ); if( !m_adjust ) wh *= ( 1- m_decay ); - m_sqsum -= pow( wh, 2 ); + m_sqsum -= wh * wh; m_wsum -= wh; if( m_wsum < EPSILON || m_sqsum < EPSILON ) { @@ -1777,7 +1777,7 @@ class AlphaDebiasEMA double compute() const { - double wsum_sq = pow( m_wsum, 2 ); + double wsum_sq = m_wsum * m_wsum; if( abs( wsum_sq - m_sqsum ) > EPSILON ) return wsum_sq / ( wsum_sq - m_sqsum ); else @@ -1804,7 +1804,7 @@ class HalflifeEMA HalflifeEMA( TimeDelta halflife, DateTime start ) { - m_halflife = halflife; + m_decay_factor = log( 0.5 ) / halflife.asNanoseconds(); m_last_tick = start; reset(); } @@ -1818,12 +1818,9 @@ class HalflifeEMA if( likely( !isnan( x ) ) ) { TimeDelta delta_t = now - m_last_tick; - double decay_duration = ( double )( delta_t.asNanoseconds() ) / ( m_halflife.asNanoseconds() ); - double decay = exp( -( decay_duration ) * log( 2 ) ); - m_ema *= decay; - m_norm *= decay; - m_ema += x; - m_norm++; + double decay = exp( m_decay_factor * delta_t.asNanoseconds() ); + m_ema = decay * m_ema + x; + m_norm = decay * m_norm + 1.0; m_last_tick = now; } } @@ -1842,7 +1839,7 @@ class HalflifeEMA double m_ema; double m_norm; - TimeDelta m_halflife; + double m_decay_factor; DateTime m_last_tick; }; @@ -1854,7 +1851,7 @@ class HalflifeDebiasEMA HalflifeDebiasEMA( TimeDelta halflife, DateTime start ) { - m_halflife = halflife; + m_decay_factor = log( 0.5 ) / halflife.asNanoseconds(); m_last_tick = start; reset(); } @@ -1868,11 +1865,9 @@ class HalflifeDebiasEMA if( likely( !isnan( x ) ) ) { TimeDelta delta_t = now - m_last_tick; - double decay = exp( -( delta_t/m_halflife ) * log( 2 ) ); - m_sqsum *= pow( decay, 2 ); - m_wsum *= decay; - m_sqsum++; - m_wsum++; + double decay = exp( m_decay_factor * delta_t.asNanoseconds() ); + m_sqsum = decay * decay * m_sqsum + 1.0; + m_wsum = decay * m_wsum + 1.0; m_last_tick = now; } } @@ -1884,7 +1879,7 @@ class HalflifeDebiasEMA double compute() const { - double wsum_sq = pow( m_wsum, 2 ); + double wsum_sq = m_wsum * m_wsum; if( wsum_sq != m_sqsum ) return wsum_sq / ( wsum_sq - m_sqsum ); else @@ -1895,7 +1890,7 @@ class HalflifeDebiasEMA double m_wsum; double m_sqsum; - TimeDelta m_halflife; + double m_decay_factor; DateTime m_last_tick; }; From 3eed9eea981a7b5595eed46f6e3fb9971f3a2b2c Mon Sep 17 00:00:00 2001 From: Rob Ambalu Date: Wed, 26 Jun 2024 00:26:58 -0400 Subject: [PATCH 11/17] websocket - raise a clear error is hostname isnt extracted from URI ( crashes otherwise ). Add Usage to example Signed-off-by: Rob Ambalu --- csp/adapters/websocket.py | 3 +++ examples/03_using_adapters/websocket/e1_websocket_client.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/csp/adapters/websocket.py b/csp/adapters/websocket.py index d7f9cf86e..e65232a13 100644 --- a/csp/adapters/websocket.py +++ b/csp/adapters/websocket.py @@ -405,6 +405,9 @@ def __init__( """ assert reconnect_interval >= timedelta(seconds=1) resp = urllib.parse.urlparse(uri) + if resp.hostname is None: + raise ValueError(f"Failed to parse host from URI: {uri}") + self._properties = dict( host=resp.hostname, # if no port is explicitly present in the uri, the resp.port is None diff --git a/examples/03_using_adapters/websocket/e1_websocket_client.py b/examples/03_using_adapters/websocket/e1_websocket_client.py index 124a3169e..eec4eaa34 100644 --- a/examples/03_using_adapters/websocket/e1_websocket_client.py +++ b/examples/03_using_adapters/websocket/e1_websocket_client.py @@ -24,6 +24,10 @@ def g(uri: str): if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: e1_websocket_client ") + sys.exit(1) + csp.run( g, starttime=datetime.utcnow(), From c7ec61a9291cc464587483f90ba9a6cdadeb8bfb Mon Sep 17 00:00:00 2001 From: Rob Ambalu Date: Wed, 26 Jun 2024 10:38:52 -0400 Subject: [PATCH 12/17] release v0.0.5 Signed-off-by: Rob Ambalu --- .bumpversion.cfg | 5 +---- CMakeLists.txt | 2 +- csp/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2d69a06d8..264c7ef13 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,10 +1,7 @@ [bumpversion] -current_version = 0.0.4 +current_version = 0.0.5 commit = False tag = False -# We don't commit right now, but -# if we do in the future, this will -# ensure commits are signed commit_args = -s [bumpversion:file:csp/__init__.py] diff --git a/CMakeLists.txt b/CMakeLists.txt index 14350c702..d2df6650b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ # Project Configuration # ######################### cmake_minimum_required(VERSION 3.20.0) -project(csp VERSION "0.0.4") +project(csp VERSION "0.0.5") set(CMAKE_CXX_STANDARD 20) ################################################################################################################################################### diff --git a/csp/__init__.py b/csp/__init__.py index 48cea50b5..a223dc7aa 100644 --- a/csp/__init__.py +++ b/csp/__init__.py @@ -31,7 +31,7 @@ from . import stats -__version__ = "0.0.4" +__version__ = "0.0.5" def get_include_path(): diff --git a/pyproject.toml b/pyproject.toml index cfef783a2..7a19ea3b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ name = "csp" authors = [{name = "the csp authors", email = "CSPOpenSource@point72.com"}] description="csp is a high performance reactive stream processing library, written in C++ and Python" readme = "README.md" -version = "0.0.4" +version = "0.0.5" requires-python = ">=3.8" dependencies = [ diff --git a/setup.py b/setup.py index edc709e38..ea2a1fa68 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ setup( name="csp", - version="0.0.4", + version="0.0.5", packages=["csp"], cmake_install_dir="csp", cmake_args=cmake_args, From be5f17fb74e33f3cdfb84b68ab7ab0fce9009047 Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 29 Jun 2024 12:41:11 -0400 Subject: [PATCH 13/17] Move symphony adapter to separate package Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- csp/adapters/symphony.py | 538 +----------------- csp/tests/adapters/test_symphony.py | 273 --------- .../Input-Output-Adapters-API.md | 5 - examples/03_using_adapters/slack/README.md | 1 - pyproject.toml | 3 +- 5 files changed, 5 insertions(+), 815 deletions(-) delete mode 100644 csp/tests/adapters/test_symphony.py delete mode 100644 examples/03_using_adapters/slack/README.md diff --git a/csp/adapters/symphony.py b/csp/adapters/symphony.py index 782d1008e..34e7099ce 100644 --- a/csp/adapters/symphony.py +++ b/csp/adapters/symphony.py @@ -1,534 +1,4 @@ -import http.client -import json -import requests -import ssl -import threading -from logging import getLogger -from queue import Queue -from tempfile import NamedTemporaryFile -from typing import Dict, Optional - -import csp -from csp import ts -from csp.impl.enum import Enum -from csp.impl.pushadapter import PushInputAdapter -from csp.impl.wiring import py_push_adapter_def - -__all__ = ["SymphonyAdapter", "SymphonyMessage"] - -log = getLogger(__file__) - - -def _sync_create_data_feed(datafeed_create_url: str, header: Dict[str, str]) -> str: - r = requests.post( - url=datafeed_create_url, - headers=header, - ) - datafeed_id = r.json()["id"] - log.info(f"created symphony datafeed with id={datafeed_id}") - return r, datafeed_id - - -def _client_cert_post(host: str, request_url: str, cert_file: str, key_file: str) -> str: - request_headers = {"Content-Type": "application/json"} - request_body_dict = {} - - # Define the client certificate settings for https connection - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.load_cert_chain(certfile=cert_file, keyfile=key_file) - - # Create a connection to submit HTTP requests - connection = http.client.HTTPSConnection(host, port=443, context=context) - - # Use connection to submit a HTTP POST request - connection.request(method="POST", url=request_url, headers=request_headers, body=json.dumps(request_body_dict)) - - # Print the HTTP response from the IOT service endpoint - response = connection.getresponse() - - if response.status != 200: - raise Exception( - f"Cannot connect for symphony handshake to https://{host}{request_url}: {response.status}:{response.reason}" - ) - data = response.read().decode("utf-8") - return json.loads(data) - - -def _symphony_session( - auth_host: str, - session_auth_path: str, - key_auth_path: str, - cert_string: str, - key_string: str, -) -> Dict[str, str]: - """Setup symphony session and return the header - - Args: - auth_host (str): authentication host, like `company-api.symphony.com` - session_auth_path (str): path to authenticate session, like `/sessionauth/v1/authenticate` - key_auth_path (str): path to authenticate key, like `/keyauth/v1/authenticate` - cert_string (str): pem format string of client certificate - key_string (str): pem format string of client private key - Returns: - Dict[str, str]: headers from authentication - """ - with NamedTemporaryFile(mode="wt", delete=False) as cert_file: - with NamedTemporaryFile(mode="wt", delete=False) as key_file: - cert_file.write(cert_string) - key_file.write(key_string) - - data = _client_cert_post(auth_host, session_auth_path, cert_file.name, key_file.name) - session_token = data["token"] - - data = _client_cert_post(auth_host, key_auth_path, cert_file.name, key_file.name) - key_manager_token = data["token"] - - headers = { - "sessionToken": session_token, - "keyManagerToken": key_manager_token, - "Accept": "application/json", - } - return headers - - -class Presence(csp.Enum): - AVAILABLE = Enum.auto() - AWAY = Enum.auto() - - -def send_symphony_message(msg: str, room_id: str, message_create_url: str, header: Dict[str, str]): - """Wrap message string and send it to symphony""" - out_json = { - "message": f""" - - {msg} - - """ - } - url = message_create_url.format(sid=room_id) - return requests.post( - url=url, - json=out_json, - headers=header, - ) - - -def _get_room_id(room_name: str, room_search_url: str, header: Dict[str, str]): - """Given a room name, find its room ID""" - query = {"query": room_name} - res = requests.post( - url=room_search_url, - json=query, - headers=header, - ) - if res and res.status_code == 200: - res_json = res.json() - for room in res_json["rooms"]: - # in theory there could be a room whose name is a subset of another, and so the search could return multiple - # go through search results to find room with name exactly as given - name = room.get("roomAttributes", {}).get("name") - if name and name == room_name: - id = room.get("roomSystemInfo", {}).get("id") - if id: - return id - return None # actually no exact matches, or malformed content from symphony - else: - log.error(f"ERROR looking up Symphony room_id for room {room_name}: status {res.status_code} text {res.text}") - - -def _get_room_name(room_id: str, room_info_url: str, header: Dict[str, str]): - """Given a room ID, find its name""" - url = room_info_url.format(room_id=room_id) - res = requests.get( - url, - headers=header, - ) - if res and res.status_code == 200: - res_json = res.json() - name = res_json.get("roomAttributes", {}).get("name") - if name: - return name - log.error( - f"ERROR: malformed response from Symphony room info call to get name from id {room_id} via url {url}: {res_json}" - ) - else: - log.error( - f"ERROR: failed to query Symphony for room name from id {room_id} via url {url}: code {res.status_code} text {res.text}" - ) - - -def _get_user_mentions(payload): - # try to extract user mentions - user_mentions = [] - try: - payload_data = json.loads(payload.get("data", "{}")) - for value in payload_data.values(): - if value["type"] == "com.symphony.user.mention": - # if its a user mention (the only supported one for now), - # then grab the payload - user_id = str(value["id"][0]["value"]) - user_mentions.append(user_id) - finally: - return user_mentions - - -class SymphonyRoomMapper: - def __init__(self, room_search_url: str, room_info_url: str, header: Dict[str, str]): - self._name_to_id = {} - self._id_to_name = {} - self._room_search_url = room_search_url - self._room_info_url = room_info_url - self._header = header - self._lock = threading.Lock() - - def get_room_id(self, room_name): - with self._lock: - if room_name in self._name_to_id: - return self._name_to_id[room_name] - else: - room_id = _get_room_id(room_name, self._room_search_url, self._header) - self._name_to_id[room_name] = room_id - self._id_to_name[room_id] = room_name - return room_id - - def get_room_name(self, room_id): - with self._lock: - if room_id in self._id_to_name: - return self._id_to_name[room_id] - else: - room_name = _get_room_name(room_id, self._room_info_url, self._header) - self._name_to_id[room_name] = room_id - self._id_to_name[room_id] = room_name - return room_name - - def set_im_id(self, user, id): - with self._lock: - self._id_to_name[id] = user - self._name_to_id[user] = id - - -def mention_user(email_or_userid: str = ""): - if email_or_userid: - if "@" in str(email_or_userid): - return f'' - else: - return f'' - return "" - - -class SymphonyMessage(csp.Struct): - user: str - user_email: str # email of the author, for mentions - user_id: str # uid of the author, for mentions - tags: [str] # list of user ids in message, for mentions - room: str - msg: str - form_id: str - form_values: dict - - -class SymphonyReaderPushAdapterImpl(PushInputAdapter): - def __init__( - self, - datafeed_create_url: str, - datafeed_delete_url: str, - datafeed_read_url: str, - header: Dict[str, str], - rooms: set, - exit_msg: str = "", - room_mapper: Optional[SymphonyRoomMapper] = None, - ): - """Setup Symphony Reader - - Args: - datafeed_create_url (str): string path to create datafeed, like `https://SYMPHONY_HOST/agent/v5/datafeeds` - datafeed_delete_url (str): format-string path to create datafeed, like `https://SYMPHONY_HOST/agent/v5/datafeeds/{{datafeed_id}}` - datafeed_read_url (str): format-string path to create datafeed, like `https://SYMPHONY_HOST/agent/v5/datafeeds/{{datafeed_id}}/read` - - room_search_url (str): format-string path to create datafeed, like `https://SYMPHONY_HOST/pod/v3/room/search` - room_info_url (str): format-string path to create datafeed, like `https://SYMPHONY_HOST/pod/v3/room/{{room_id}}/info` - - header (Dict[str, str]): authentication headers - - rooms (set): set of initial rooms for the bot to enter - exit_msg (str): message to send on shutdown - room_mapper (SymphonyRoomMapper): convenience object to map rooms that bot dynamically discovers - """ - self._thread = None - self._running = False - - # message and datafeed - self._datafeed_create_url = datafeed_create_url - self._datafeed_delete_url = datafeed_delete_url - self._datafeed_read_url = datafeed_read_url - - # auth - self._header = header - - # rooms to enter by default - self._rooms = rooms - self._room_ids = set() - self._exit_msg = exit_msg - self._room_mapper = room_mapper - - def start(self, starttime, endtime): - # get symphony session - resp, datafeed_id = _sync_create_data_feed(self._datafeed_create_url, self._header) - if resp.status_code not in (200, 201, 204): - raise Exception( - f"ERROR: bad status ({resp.status_code}) from _sync_create_data_feed, cannot start Symphony reader" - ) - else: - self._url = self._datafeed_read_url.format(datafeed_id=datafeed_id) - self._datafeed_delete_url = self._datafeed_delete_url.format(datafeed_id=datafeed_id) - - for room in self._rooms: - room_id = self._room_mapper.get_room_id(room) - if not room_id: - raise Exception(f"ERROR: unable to find Symphony room named {room}") - self._room_ids.add(room_id) - - # start reader thread - self._thread = threading.Thread(target=self._run, daemon=True) - self._running = True - self._thread.start() - - def stop(self): - if self._running: - # in order to unblock current requests.get, send a message to one of the rooms we are listening on - self._running = False - if self._datafeed_delete_url is not None: - resp = requests.delete(url=self._datafeed_delete_url, headers=self._header) - log.info(f"Deleted datafeed with url={self._datafeed_delete_url}: resp={resp}") - if self._exit_msg: - send_symphony_message(self._exit_msg, next(iter(self._room_ids)), self._header) - self._thread.join() - - def _run(self): - ack_id = "" - while self._running: - resp = requests.post(url=self._url, headers=self._header, json={"ackId": ack_id}) - ret = [] - if resp.status_code == 200: - msg_json = resp.json() - if "ackId" in msg_json: - ack_id = msg_json["ackId"] - events = msg_json.get("events", []) - for m in events: - if "type" in m and "payload" in m: - if m["type"] == "MESSAGESENT": - payload = m.get("payload", {}).get("messageSent", {}).get("message") - if payload: - payload_stream_id = payload.get("stream", {}).get("streamId") - if payload_stream_id and (not self._room_ids or payload_stream_id in self._room_ids): - user = payload.get("user", {}).get("displayName", "USER_ERROR") - user_email = payload.get("user", {}).get("email", "USER_ERROR") - user_id = str(payload.get("user", {}).get("userId", "USER_ERROR")) - user_mentions = _get_user_mentions(payload) - msg = payload.get("message", "MSG_ERROR") - - # room name or "IM" for direct message - room_type = payload.get("stream", {}).get("streamType", "ROOM") - if room_type == "ROOM": - room_name = self._room_mapper.get_room_name(payload_stream_id) - elif room_type == "IM": - # register the room name for the user so bot can respond - self._room_mapper.set_im_id(user, payload_stream_id) - room_name = "IM" - else: - room_name = "" - - if room_name: - ret.append( - SymphonyMessage( - user=user, - user_email=user_email, - user_id=user_id, - tags=user_mentions, - room=room_name, - msg=msg, - ) - ) - elif m["type"] == "SYMPHONYELEMENTSACTION": - payload = m.get("payload").get("symphonyElementsAction", {}) - payload_stream_id = payload.get("stream", {}).get("streamId") - - if not payload_stream_id: - continue - - if self._room_ids and payload_stream_id not in self._room_ids: - continue - - user = m.get("initiator", {}).get("user", {}).get("displayName", "USER_ERROR") - user_email = m.get("initiator", {}).get("user", {}).get("email", "USER_ERROR") - user_id = str(m.get("initiator", {}).get("user", {}).get("userId", "USER_ERROR")) - user_mentions = _get_user_mentions(m.get("initiator", {})) - form_id = payload.get("formId", "FORM_ID_ERROR") - form_values = payload.get("formValues", {}) - - # room name or "IM" for direct message - room_type = payload.get("stream", {}).get("streamType", "ROOM") - if room_type == "ROOM": - room_name = self._room_mapper.get_room_name(payload_stream_id) - elif room_type == "IM": - # register the room name for the user so bot can respond - self._room_mapper.set_im_id(user, payload_stream_id) - room_name = "IM" - else: - room_name = "" - - if room_name: - ret.append( - SymphonyMessage( - user=user, - user_email=user_email, - user_id=user_id, - tags=user_mentions, - room=room_name, - form_id=form_id, - form_values=form_values, - ) - ) - - if ret: - self.push_tick(ret) - - -SymphonyReaderPushAdapter = py_push_adapter_def( - "SymphonyReaderPushAdapter", - SymphonyReaderPushAdapterImpl, - ts[[SymphonyMessage]], - datafeed_create_url=str, - datafeed_delete_url=str, - datafeed_read_url=str, - header=Dict[str, str], - rooms=set, - exit_msg=str, - room_mapper=object, -) - - -def _send_messages( - msg_queue: Queue, - header: Dict[str, str], - room_mapper: SymphonyRoomMapper, - message_create_url: str, -): - """read messages from msg_queue and write to symphony. msg_queue to contain instances of SymphonyMessage, or None to shut down""" - while True: - msg = msg_queue.get() - msg_queue.task_done() - if not msg: # send None to kill - break - - room_id = room_mapper.get_room_id(msg.room) - if not room_id: - log.error(f"cannot find id for symphony room {msg.room} found in SymphonyMessage") - else: - r = send_symphony_message(msg.msg, room_id, message_create_url, header) - if r.status_code != 200: - log.error(f"Cannot send message - symphony server response: {r.status_code} {r.text}") - - -class SymphonyAdapter: - def __init__( - self, - auth_host: str, - session_auth_path: str, - key_auth_path: str, - message_create_url: str, - presence_url: str, - datafeed_create_url: str, - datafeed_delete_url: str, - datafeed_read_url: str, - room_search_url: str, - room_info_url: str, - cert_string: str, - key_string: str, - ): - """Setup Symphony Reader - - Args: - auth_host (str): authentication host, like `company-api.symphony.com` - - session_auth_path (str): path to authenticate session, like `/sessionauth/v1/authenticate` - key_auth_path (str): path to authenticate key, like `/keyauth/v1/authenticate` - - message_create_url (str): string path to create a message, like `https://SYMPHONY_HOST/agent/v4/stream/{{sid}}/message/create` - presence_url (str): string path to create a message, like `https://SYMPHONY_HOST/pod/v2/user/presence` - datafeed_create_url (str): string path to create datafeed, like `https://SYMPHONY_HOST/agent/v5/datafeeds` - datafeed_delete_url (str): format-string path to create datafeed, like `https://SYMPHONY_HOST/agent/v5/datafeeds/{{datafeed_id}}` - datafeed_read_url (str): format-string path to create datafeed, like `https://SYMPHONY_HOST/agent/v5/datafeeds/{{datafeed_id}}/read` - - room_search_url (str): format-string path to create datafeed, like `https://SYMPHONY_HOST/pod/v3/room/search` - room_info_url (str): format-string path to create datafeed, like `https://SYMPHONY_HOST/pod/v3/room/{{room_id}}/info` - - cert_string (str): pem format string of client certificate - key_string (str): pem format string of client private key - rooms (set): set of initial rooms for the bot to enter - """ - self._auth_host = auth_host - self._session_auth_path = session_auth_path - self._key_auth_path = key_auth_path - self._message_create_url = message_create_url - self._presence_url = presence_url - self._datafeed_create_url = datafeed_create_url - self._datafeed_delete_url = datafeed_delete_url - self._datafeed_read_url = datafeed_read_url - self._room_search_url = room_search_url - self._room_info_url = room_info_url - self._cert_string = cert_string - self._key_string = key_string - self._header = _symphony_session( - self._auth_host, self._session_auth_path, self._key_auth_path, self._cert_string, self._key_string - ) - self._room_mapper = SymphonyRoomMapper(self._room_search_url, self._room_info_url, self._header) - - @csp.graph - def subscribe(self, rooms: set = set(), exit_msg: str = "") -> ts[[SymphonyMessage]]: - return SymphonyReaderPushAdapter( - datafeed_create_url=self._datafeed_create_url, - datafeed_delete_url=self._datafeed_delete_url, - datafeed_read_url=self._datafeed_read_url, - header=self._header, - rooms=rooms, - exit_msg=exit_msg, - room_mapper=self._room_mapper, - ) - - # take in SymphonyMessage and send to symphony on separate thread - @csp.node - def _symphony_write(self, msg: ts[SymphonyMessage]): - with csp.state(): - s_thread = None - s_queue = None - - with csp.start(): - s_queue = Queue(maxsize=0) - s_thread = threading.Thread( - target=_send_messages, args=(s_queue, self._header, self._room_mapper, self._message_create_url) - ) - s_thread.start() - - with csp.stop(): - if s_thread: - s_queue.put(None) # send a None to tell the writer thread to exit - s_queue.join() # wait till the writer thread is done with the queue - s_thread.join() # then join with the thread - - if csp.ticked(msg): - s_queue.put(msg) - - @csp.node - def _set_presense(self, presence: ts[Presence]): - ret = requests.post(url=self._presence_url, json={"category": presence.name}, headers=self._header) - if ret.status_code != 200: - log.error(f"Cannot set presence - symphony server response: {ret.status_code} {ret.text}") - - @csp.graph - def publish_presence(self, presence: ts[Presence]): - self._set_presense(presence=presence) - - @csp.graph - def publish(self, msg: ts[SymphonyMessage]): - self._symphony_write(msg=msg) +try: + from csp_adapter_symphony import * # noqa: F403 +except ImportError: + raise ModuleNotFoundError("Install `csp-adapter-symphony` to use csp's Symphony adapter") diff --git a/csp/tests/adapters/test_symphony.py b/csp/tests/adapters/test_symphony.py deleted file mode 100644 index a21964af6..000000000 --- a/csp/tests/adapters/test_symphony.py +++ /dev/null @@ -1,273 +0,0 @@ -from time import sleep -from unittest.mock import MagicMock, call, patch - -import csp -from csp import ts -from csp.adapters.symphony import ( - SymphonyAdapter, - SymphonyMessage, - SymphonyRoomMapper, - mention_user, - send_symphony_message, -) - -SAMPLE_EVENTS = [ - { - "type": "MESSAGESENT", - "payload": { - "messageSent": { - "message": { - "stream": {"streamId": "a-stream-id", "streamType": "ROOM"}, - "user": {"displayName": "Sender User", "email": "sender@user.blerg", "userId": "sender-user-id"}, - "data": '{"key": {"type": "com.symphony.user.mention", "id": [{"value":"a-mentioned-user-id"}] } }', - "message": "a test message @a-mentioned-user-name", - }, - }, - }, - }, - # TODO - # { - # "type": "SYMPHONYELEMENTSACTION", - # }, -] - - -@csp.node -def hello(msg: ts[SymphonyMessage]) -> ts[SymphonyMessage]: - if csp.ticked(msg): - text = f"Hello <@{msg.user_id}>!" - return SymphonyMessage( - room="another sample room", - msg=text, - ) - - -class TestSymphony: - def test_send_symphony_message(self): - msg = "test_msg" - room_id = "test_room_id" - message_create_url = "message/create/url" - header = {"Authorization": "Bearer Blerg"} - with patch("requests.post") as requests_mock: - send_symphony_message(msg, room_id, message_create_url, header) - assert requests_mock.call_args_list == [ - call( - url="message/create/url", - json={"message": "\n \n test_msg\n \n "}, - headers={"Authorization": "Bearer Blerg"}, - ) - ] - - def test_room_mapper(self): - room_mapper = SymphonyRoomMapper("room/search/url", "room/info/url", {"authorization": "bearer blerg"}) - - with patch("requests.get") as requests_get_mock, patch("requests.post") as requests_post_mock: - requests_get_mock.return_value.status_code = 200 - requests_get_mock.return_value.json.return_value = {"roomAttributes": {"name": "a sample room"}} - requests_post_mock.return_value.status_code = 200 - requests_post_mock.return_value.json.return_value = { - "rooms": [{"roomAttributes": {"name": "another sample room"}, "roomSystemInfo": {"id": "an id"}}] - } - - # call twice for both paths - assert room_mapper.get_room_name("anything") == "a sample room" - assert room_mapper.get_room_name("anything") == "a sample room" - # call twice for both paths - assert room_mapper.get_room_id("another sample room") == "an id" - assert room_mapper.get_room_id("another sample room") == "an id" - - room_mapper.set_im_id("username", "id") - assert room_mapper.get_room_id("username") == "id" - - def test_mention_user(self): - assert mention_user("blerg@blerg.com") == '' - assert mention_user("blergid") == '' - - def test_symphony_instantiation(self): - with patch("requests.get") as requests_get_mock, patch("requests.post") as requests_post_mock, patch( - "requests.delete" - ) as requests_delete_mock, patch("ssl.SSLContext") as ssl_context_mock, patch( - "http.client.HTTPSConnection" - ) as https_client_connection_mock, patch( - "csp.adapters.symphony.NamedTemporaryFile" - ) as named_temporary_file_mock: - # mock https connection - https_connection_mock = MagicMock() - https_client_connection_mock.return_value = https_connection_mock - https_connection_mock.getresponse.return_value.status = 200 - https_connection_mock.getresponse.return_value.read.return_value = b'{"token": "a-fake-token"}' - - # mock temporary file creation for cert / key - named_temporary_file_mock.return_value.__enter__.return_value.name = "a_temp_file" - - # mock get request response based on url - def get_request(url, headers, json=None): - assert url in ("https://symphony.host/pod/v3/room/{room_id}/info",) - resp_mock = MagicMock() - resp_mock.status_code = 200 - if url == "https://symphony.host/pod/v3/room/{room_id}/info": - resp_mock.json.return_value = {"roomAttributes": {"name": "a sample room"}} - return resp_mock - - requests_get_mock.side_effect = get_request - - # mock post request response based on url - def post_request(url, headers, json=None): - assert url in ( - # create datafeed - "https://symphony.host/agent/v5/datafeeds", - # read messages in room - "https://symphony.host/agent/v5/datafeeds/{datafeed_id}/read", - # room lookup - "https://symphony.host/pod/v3/room/search", - # send message - "https://symphony.host/agent/v4/stream/{sid}/message/create", - ) - resp_mock = MagicMock() - resp_mock.status_code = 200 - if url == "https://symphony.host/agent/v5/datafeeds": - # create datafeed - resp_mock.json.return_value = {"id": "an id"} - elif url == "https://symphony.host/agent/v5/datafeeds/{datafeed_id}/read": - # read messages in room - resp_mock.json.return_value = {"id": "an id", "events": SAMPLE_EVENTS * 3} - elif url == "https://symphony.host/pod/v3/room/search": - # room lookup - resp_mock.json.return_value = { - "rooms": [ - {"roomAttributes": {"name": "another sample room"}, "roomSystemInfo": {"id": "an id"}} - ] - } - elif url == "https://symphony.host/agent/v4/stream/{sid}/message/create": - # send message - ... - sleep(0.1) - return resp_mock - - requests_post_mock.side_effect = post_request - - # instantiate - adapter = SymphonyAdapter( - "auth.host", - "/sessionauth/v1/authenticate", - "/keyauth/v1/authenticate", - "https://symphony.host/agent/v4/stream/{{sid}}/message/create", - "https://symphony.host/pod/v2/user/presence", - "https://symphony.host/agent/v5/datafeeds", - "https://symphony.host/agent/v5/datafeeds/{{datafeed_id}}", - "https://symphony.host/agent/v5/datafeeds/{{datafeed_id}}/read", - "https://symphony.host/pod/v3/room/search", - "https://symphony.host/pod/v3/room/{{room_id}}/info", - "my_cert_string", - "my_key_string", - ) - - # assert auth worked properly to get token - assert named_temporary_file_mock.return_value.__enter__.return_value.write.call_args_list == [ - call("my_cert_string"), - call("my_key_string"), - ] - assert ssl_context_mock.return_value.load_cert_chain.call_args_list == [ - # session token - call(certfile="a_temp_file", keyfile="a_temp_file"), - # key manager token - call(certfile="a_temp_file", keyfile="a_temp_file"), - ] - - @csp.graph - def graph(): - # send a fake slack message to the app - # stop = send_fake_message(clientmock, reqmock, am) - - # send a response - resp = hello(csp.unroll(adapter.subscribe())) - adapter.publish(resp) - - csp.add_graph_output("response", resp) - - # stop after first messages - done_flag = csp.count(resp) == 2 - done_flag = csp.filter(done_flag, done_flag) - csp.stop_engine(done_flag) - - # run the graph - resp = csp.run(graph, realtime=True) - - assert len(resp["response"]) == 2 - assert resp["response"][0][1] == SymphonyMessage( - room="another sample room", - msg="Hello <@sender-user-id>!", - ) - - assert requests_get_mock.call_count == 1 - assert requests_get_mock.call_args_list == [ - call( - "https://symphony.host/pod/v3/room/{room_id}/info", - headers={ - "sessionToken": "a-fake-token", - "keyManagerToken": "a-fake-token", - "Accept": "application/json", - }, - ) - ] - assert requests_post_mock.call_count >= 5 - assert ( - call( - url="https://symphony.host/agent/v5/datafeeds", - headers={ - "sessionToken": "a-fake-token", - "keyManagerToken": "a-fake-token", - "Accept": "application/json", - }, - ) - in requests_post_mock.call_args_list - ) - assert ( - call( - url="https://symphony.host/agent/v5/datafeeds/{datafeed_id}/read", - headers={ - "sessionToken": "a-fake-token", - "keyManagerToken": "a-fake-token", - "Accept": "application/json", - }, - json={"ackId": ""}, - ) - in requests_post_mock.call_args_list - ) - assert ( - call( - url="https://symphony.host/pod/v3/room/search", - json={"query": "another sample room"}, - headers={ - "sessionToken": "a-fake-token", - "keyManagerToken": "a-fake-token", - "Accept": "application/json", - }, - ) - in requests_post_mock.call_args_list - ) - assert ( - call( - url="https://symphony.host/agent/v4/stream/{sid}/message/create", - json={ - "message": "\n \n Hello <@sender-user-id>!\n \n " - }, - headers={ - "sessionToken": "a-fake-token", - "keyManagerToken": "a-fake-token", - "Accept": "application/json", - }, - ) - in requests_post_mock.call_args_list - ) - assert requests_delete_mock.call_count == 1 - assert requests_delete_mock.call_args_list == [ - call( - url="https://symphony.host/agent/v5/datafeeds/{datafeed_id}", - headers={ - "sessionToken": "a-fake-token", - "keyManagerToken": "a-fake-token", - "Accept": "application/json", - }, - ) - ] diff --git a/docs/wiki/api-references/Input-Output-Adapters-API.md b/docs/wiki/api-references/Input-Output-Adapters-API.md index 7dcd70d75..7ed5976e7 100644 --- a/docs/wiki/api-references/Input-Output-Adapters-API.md +++ b/docs/wiki/api-references/Input-Output-Adapters-API.md @@ -15,7 +15,6 @@ - [Publishing](#publishing) - [DBReader](#dbreader) - [TimeAccessor](#timeaccessor) -- [Symphony](#symphony) - [Slack](#slack) ## Kafka @@ -351,10 +350,6 @@ Both of these calls expect `typ` to be a `csp.Struct` type. `subscribe_all` is used to retrieve all the data resulting from the request as a single timeseries. -## Symphony - -The Symphony adapter allows for reading and writing of messages from the [Symphony](https://symphony.com/) message platform using [`requests`](https://requests.readthedocs.io/en/latest/) and the [Symphony SDK](https://docs.developers.symphony.com/). - ## Slack The Slack adapter allows for reading and writing of messages from the [Slack](https://slack.com) message platform using the [Slack Python SDK](https://slack.dev/python-slack-sdk/). diff --git a/examples/03_using_adapters/slack/README.md b/examples/03_using_adapters/slack/README.md deleted file mode 100644 index be657ba7a..000000000 --- a/examples/03_using_adapters/slack/README.md +++ /dev/null @@ -1 +0,0 @@ -# Slack Adapter diff --git a/pyproject.toml b/pyproject.toml index 7a19ea3b9..48d48d1ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,7 +79,6 @@ develop = [ "psutil", # test_engine/test_history "slack-sdk>=3", # slack "sqlalchemy", # db - "requests", # symphony "threadpoolctl", # test_random "tornado", # profiler, perspective, websocket ] @@ -104,7 +103,7 @@ test = [ "tornado", ] symphony = [ - "requests", + "csp-adapter-symphony", ] slack = [ "slack-sdk>=3", From 14dc4cdc55399396b83e330d2578605b97374871 Mon Sep 17 00:00:00 2001 From: Stephen Markacs Date: Mon, 1 Jul 2024 08:40:06 -0400 Subject: [PATCH 14/17] tiny fix to README Signed-off-by: Stephen Markacs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b827ddd5..2821d3289 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The high level goal of `csp` is to make writing realtime code simple and performant. Write event driven code once, test it in simulation, then deploy as realtime without any code changes. -Here is a very simple example of a small `csp` program to calculate a [bid-ask spread](https://en.wikipedia.org/wiki/Bid%E2%80%93ask_spread). In this example, we use a constant bid and ask, but in the real world you might pipe these directly into your live streaming data source, or into your historical data source, without modifications to your core logic. +Here is a very simple example of a small `csp` program to calculate a [bid-ask spread](https://en.wikipedia.org/wiki/Bid%E2%80%93ask_spread). In this example, we use a constant bid and ask, but in the real world you might pipe these directly in from your live streaming data source, or in from your historical data source, without modifications to your core logic. ```python import csp From 3264c3d554b95f7946642461cdb761d742fafa3c Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:55:22 -0400 Subject: [PATCH 15/17] Update vcpkg baseline for bugfixes Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- NOTICE | 237 +++++++++++++++++++++++++++++++++-------------------- vcpkg | 2 +- vcpkg.json | 2 +- 3 files changed, 148 insertions(+), 93 deletions(-) diff --git a/NOTICE b/NOTICE index 0dbfd65ba..4f42bf946 100644 --- a/NOTICE +++ b/NOTICE @@ -2,7 +2,7 @@ CSP - Copyright 2024 Point72, L.P. This project contains software with the below copyrights -vcpkg_installed/arm64-osx/share/abseil/copyright +vcpkg_installed/x64-linux/share/abseil/copyright @@ -210,7 +210,7 @@ vcpkg_installed/arm64-osx/share/abseil/copyright -vcpkg_installed/arm64-osx/share/arrow/copyright +vcpkg_installed/x64-linux/share/arrow/copyright @@ -2476,7 +2476,7 @@ These file are derived from code from Netty, which is made available under the Apache License 2.0. -vcpkg_installed/arm64-osx/share/boost-algorithm/copyright +vcpkg_installed/x64-linux/share/boost-algorithm/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2504,7 +2504,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-align/copyright +vcpkg_installed/x64-linux/share/boost-align/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2532,7 +2532,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-array/copyright +vcpkg_installed/x64-linux/share/boost-array/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2560,7 +2560,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-asio/copyright +vcpkg_installed/x64-linux/share/boost-asio/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2588,7 +2588,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-assert/copyright +vcpkg_installed/x64-linux/share/boost-assert/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2616,7 +2616,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-atomic/copyright +vcpkg_installed/x64-linux/share/boost-atomic/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2644,7 +2644,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-beast/copyright +vcpkg_installed/x64-linux/share/boost-beast/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2672,7 +2672,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-bind/copyright +vcpkg_installed/x64-linux/share/boost-bind/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2700,7 +2700,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-chrono/copyright +vcpkg_installed/x64-linux/share/boost-chrono/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2728,7 +2728,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-cmake/copyright +vcpkg_installed/x64-linux/share/boost-cmake/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2756,7 +2756,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-concept-check/copyright +vcpkg_installed/x64-linux/share/boost-concept-check/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2784,7 +2784,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-config/copyright +vcpkg_installed/x64-linux/share/boost-config/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2812,7 +2812,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-container-hash/copyright +vcpkg_installed/x64-linux/share/boost-container/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2840,7 +2840,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-container/copyright +vcpkg_installed/x64-linux/share/boost-container-hash/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2868,7 +2868,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-context/copyright +vcpkg_installed/x64-linux/share/boost-context/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2896,7 +2896,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-conversion/copyright +vcpkg_installed/x64-linux/share/boost-conversion/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2924,7 +2924,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-core/copyright +vcpkg_installed/x64-linux/share/boost-core/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2952,7 +2952,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-coroutine/copyright +vcpkg_installed/x64-linux/share/boost-coroutine/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -2980,7 +2980,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-date-time/copyright +vcpkg_installed/x64-linux/share/boost-date-time/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3008,7 +3008,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-describe/copyright +vcpkg_installed/x64-linux/share/boost-describe/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3036,7 +3036,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-detail/copyright +vcpkg_installed/x64-linux/share/boost-detail/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3064,7 +3064,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-dynamic-bitset/copyright +vcpkg_installed/x64-linux/share/boost-dynamic-bitset/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3092,7 +3092,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-endian/copyright +vcpkg_installed/x64-linux/share/boost-endian/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3120,7 +3120,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-exception/copyright +vcpkg_installed/x64-linux/share/boost-exception/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3148,7 +3148,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-filesystem/copyright +vcpkg_installed/x64-linux/share/boost-filesystem/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3176,7 +3176,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-function-types/copyright +vcpkg_installed/x64-linux/share/boost-functional/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3204,7 +3204,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-function/copyright +vcpkg_installed/x64-linux/share/boost-function/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3232,7 +3232,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-functional/copyright +vcpkg_installed/x64-linux/share/boost-function-types/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3260,7 +3260,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-fusion/copyright +vcpkg_installed/x64-linux/share/boost-fusion/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3288,7 +3288,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-headers/copyright +vcpkg_installed/x64-linux/share/boost-headers/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3316,7 +3316,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-integer/copyright +vcpkg_installed/x64-linux/share/boost-integer/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3344,7 +3344,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-intrusive/copyright +vcpkg_installed/x64-linux/share/boost-intrusive/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3372,7 +3372,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-io/copyright +vcpkg_installed/x64-linux/share/boost-io/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3400,7 +3400,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-iterator/copyright +vcpkg_installed/x64-linux/share/boost-iterator/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3428,7 +3428,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-lexical-cast/copyright +vcpkg_installed/x64-linux/share/boost-lexical-cast/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3456,7 +3456,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-locale/copyright +vcpkg_installed/x64-linux/share/boost-locale/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3484,7 +3484,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-logic/copyright +vcpkg_installed/x64-linux/share/boost-logic/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3512,7 +3512,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-math/copyright +vcpkg_installed/x64-linux/share/boost-math/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3540,7 +3540,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-move/copyright +vcpkg_installed/x64-linux/share/boost-move/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3568,7 +3568,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-mp11/copyright +vcpkg_installed/x64-linux/share/boost-mp11/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3596,7 +3596,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-mpl/copyright +vcpkg_installed/x64-linux/share/boost-mpl/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3624,7 +3624,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-multiprecision/copyright +vcpkg_installed/x64-linux/share/boost-multiprecision/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3652,7 +3652,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-numeric-conversion/copyright +vcpkg_installed/x64-linux/share/boost-numeric-conversion/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3680,7 +3680,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-optional/copyright +vcpkg_installed/x64-linux/share/boost-optional/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3708,7 +3708,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-pool/copyright +vcpkg_installed/x64-linux/share/boost-pool/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3736,7 +3736,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-predef/copyright +vcpkg_installed/x64-linux/share/boost-predef/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3764,7 +3764,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-preprocessor/copyright +vcpkg_installed/x64-linux/share/boost-preprocessor/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3792,7 +3792,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-random/copyright +vcpkg_installed/x64-linux/share/boost-random/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3820,7 +3820,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-range/copyright +vcpkg_installed/x64-linux/share/boost-range/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3848,7 +3848,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-ratio/copyright +vcpkg_installed/x64-linux/share/boost-ratio/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3876,7 +3876,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-regex/copyright +vcpkg_installed/x64-linux/share/boost-regex/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3904,7 +3904,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-scope-exit/copyright +vcpkg_installed/x64-linux/share/boost-scope/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3932,7 +3932,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-smart-ptr/copyright +vcpkg_installed/x64-linux/share/boost-scope-exit/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3960,7 +3960,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-static-assert/copyright +vcpkg_installed/x64-linux/share/boost-smart-ptr/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -3988,7 +3988,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-static-string/copyright +vcpkg_installed/x64-linux/share/boost-static-assert/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4016,7 +4016,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-system/copyright +vcpkg_installed/x64-linux/share/boost-static-string/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4044,7 +4044,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-thread/copyright +vcpkg_installed/x64-linux/share/boost-system/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4072,7 +4072,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-throw-exception/copyright +vcpkg_installed/x64-linux/share/boost-thread/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4100,7 +4100,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-tokenizer/copyright +vcpkg_installed/x64-linux/share/boost-throw-exception/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4128,7 +4128,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-tuple/copyright +vcpkg_installed/x64-linux/share/boost-tokenizer/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4156,7 +4156,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-type-index/copyright +vcpkg_installed/x64-linux/share/boost-tuple/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4184,7 +4184,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-type-traits/copyright +vcpkg_installed/x64-linux/share/boost-type-index/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4212,7 +4212,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-typeof/copyright +vcpkg_installed/x64-linux/share/boost-typeof/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4240,7 +4240,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-unordered/copyright +vcpkg_installed/x64-linux/share/boost-type-traits/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4268,7 +4268,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-utility/copyright +vcpkg_installed/x64-linux/share/boost-unordered/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4296,7 +4296,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-variant2/copyright +vcpkg_installed/x64-linux/share/boost-utility/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4324,7 +4324,7 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/boost-winapi/copyright +vcpkg_installed/x64-linux/share/boost-variant2/copyright Boost Software License - Version 1.0 - August 17th, 2003 @@ -4352,7 +4352,35 @@ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/brotli/copyright +vcpkg_installed/x64-linux/share/boost-winapi/copyright + + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +vcpkg_installed/x64-linux/share/brotli/copyright Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. @@ -4376,7 +4404,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/bzip2/copyright +vcpkg_installed/x64-linux/share/bzip2/copyright @@ -4423,7 +4451,7 @@ bzip2/libbzip2 version 1.0.8 of 13 July 2019 -------------------------------------------------------------------------- -vcpkg_installed/arm64-osx/share/exprtk/copyright +vcpkg_installed/x64-linux/share/exprtk/copyright Copyright 1999-2022 Arash Partow @@ -4436,7 +4464,7 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/gflags/copyright +vcpkg_installed/x64-linux/share/gflags/copyright Copyright (c) 2006, Google Inc. @@ -4469,7 +4497,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -vcpkg_installed/arm64-osx/share/gtest/copyright +vcpkg_installed/x64-linux/share/gtest/copyright Copyright 2008, Google Inc. @@ -4502,7 +4530,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -vcpkg_installed/arm64-osx/share/libevent/copyright +vcpkg_installed/x64-linux/share/libevent/copyright Libevent is available for use under the following license, commonly known @@ -4658,7 +4686,7 @@ limitations under the License. This file is part of mbed TLS (https://tls.mbed.org) -vcpkg_installed/arm64-osx/share/librdkafka/copyright +vcpkg_installed/x64-linux/share/librdkafka/copyright LICENSE @@ -5056,7 +5084,7 @@ For the files wingetopt.c wingetopt.h downloaded from https://github.com/alex85k -vcpkg_installed/arm64-osx/share/lz4/copyright +vcpkg_installed/x64-linux/share/lz4/copyright LZ4 Library @@ -5085,7 +5113,7 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -vcpkg_installed/arm64-osx/share/openssl/copyright +vcpkg_installed/x64-linux/share/openssl/copyright @@ -5267,7 +5295,7 @@ vcpkg_installed/arm64-osx/share/openssl/copyright END OF TERMS AND CONDITIONS -vcpkg_installed/arm64-osx/share/protobuf/copyright +vcpkg_installed/x64-linux/share/protobuf/copyright Copyright 2008 Google Inc. All rights reserved. @@ -5304,7 +5332,7 @@ standalone and requires a support library to be linked with it. This support library is itself covered by the above license. -vcpkg_installed/arm64-osx/share/rapidjson/copyright +vcpkg_installed/x64-linux/share/rapidjson/copyright Tencent is pleased to support the open source community by making RapidJSON available. @@ -5366,7 +5394,7 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/re2/copyright +vcpkg_installed/x64-linux/share/re2/copyright // Copyright (c) 2009 The RE2 Authors. All rights reserved. @@ -5398,7 +5426,7 @@ vcpkg_installed/arm64-osx/share/re2/copyright // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -vcpkg_installed/arm64-osx/share/snappy/copyright +vcpkg_installed/x64-linux/share/snappy/copyright Copyright 2011, Google Inc. @@ -5457,7 +5485,7 @@ Some of the benchmark data in testdata/ is licensed differently: (http://www.gutenberg.org/ebooks/53). -vcpkg_installed/arm64-osx/share/thrift/copyright +vcpkg_installed/x64-linux/share/thrift/copyright @@ -5768,7 +5796,7 @@ For t_cl_generator.cc --------------------------------------------------- -vcpkg_installed/arm64-osx/share/utf8proc/copyright +vcpkg_installed/x64-linux/share/utf8proc/copyright ## utf8proc license ## @@ -5866,7 +5894,34 @@ registered in some jurisdictions. All other trademarks and registered trademarks mentioned herein are the property of their respective owners. -vcpkg_installed/arm64-osx/share/vcpkg-boost/copyright +vcpkg_installed/x64-linux/share/utf8-range/copyright + + +MIT License + +Copyright (c) 2019 Yibo Cai +Copyright 2022 Google LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +vcpkg_installed/x64-linux/share/vcpkg-boost/copyright MIT License @@ -5891,7 +5946,7 @@ CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFT OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/vcpkg-cmake-config/copyright +vcpkg_installed/x64-linux/share/vcpkg-cmake-config/copyright Copyright (c) Microsoft Corporation @@ -5919,7 +5974,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/vcpkg-cmake-get-vars/copyright +vcpkg_installed/x64-linux/share/vcpkg-cmake/copyright MIT License @@ -5944,7 +5999,7 @@ CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFT OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/vcpkg-cmake/copyright +vcpkg_installed/x64-linux/share/vcpkg-cmake-get-vars/copyright MIT License @@ -5969,7 +6024,7 @@ CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFT OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -vcpkg_installed/arm64-osx/share/xsimd/copyright +vcpkg_installed/x64-linux/share/xsimd/copyright Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and Martin Renou @@ -6003,7 +6058,7 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -vcpkg_installed/arm64-osx/share/zlib/copyright +vcpkg_installed/x64-linux/share/zlib/copyright Copyright notice: @@ -6030,7 +6085,7 @@ Copyright notice: jloup@gzip.org madler@alumni.caltech.edu -vcpkg_installed/arm64-osx/share/zstd/copyright +vcpkg_installed/x64-linux/share/zstd/copyright ZSTD is dual licensed under BSD and GPLv2. diff --git a/vcpkg b/vcpkg index 04b0cf2b3..1b5f73466 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 04b0cf2b3fd1752d3c3db969cbc10ba0a4613cee +Subproject commit 1b5f7346612cd63910567df714d867f5b3fa8e3b diff --git a/vcpkg.json b/vcpkg.json index 94d513e4d..324ae5045 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -29,5 +29,5 @@ "port-version" : 1 } ], - "builtin-baseline": "04b0cf2b3fd1752d3c3db969cbc10ba0a4613cee" + "builtin-baseline": "1b5f7346612cd63910567df714d867f5b3fa8e3b" } From 147133c1479454c5f51b3b80d508d89265733fdc Mon Sep 17 00:00:00 2001 From: Adam Glustein Date: Mon, 8 Jul 2024 16:07:32 -0400 Subject: [PATCH 16/17] Expose engine shutdown method for Python push-type adapters Co-authored-by: Artur Gajowniczek Signed-off-by: Adam Glustein Signed-off-by: Gajowniczek, Artur --- cpp/csp/python/Exception.h | 11 +++++ cpp/csp/python/PyAdapterManager.cpp | 18 ++++++- cpp/csp/python/PyEngine.h | 16 +++++++ cpp/csp/python/PyPushInputAdapter.cpp | 3 +- cpp/csp/python/PyPushPullInputAdapter.cpp | 5 +- csp/impl/adaptermanager.py | 4 ++ csp/impl/error_handling.py | 11 +++++ csp/impl/pushadapter.py | 4 ++ csp/impl/pushpulladapter.py | 4 ++ csp/tests/impl/test_pushadapter.py | 45 ++++++++++++++++++ csp/tests/impl/test_pushpulladapter.py | 58 +++++++++++++++++++++++ csp/tests/test_engine.py | 42 ++++++++++++++++ 12 files changed, 216 insertions(+), 5 deletions(-) diff --git a/cpp/csp/python/Exception.h b/cpp/csp/python/Exception.h index 104a3509a..1355c6f67 100644 --- a/cpp/csp/python/Exception.h +++ b/cpp/csp/python/Exception.h @@ -8,6 +8,17 @@ namespace csp::python { +class TracebackStringException : public std::exception +{ +public: + TracebackStringException( const char * message ) : m_message( message ) {} + + const char * what() const noexcept override { return m_message; } + +private: + const char * m_message; +}; + class PythonPassthrough : public csp::Exception { public: diff --git a/cpp/csp/python/PyAdapterManager.cpp b/cpp/csp/python/PyAdapterManager.cpp index 3a9ec7211..c412a0ee0 100644 --- a/cpp/csp/python/PyAdapterManager.cpp +++ b/cpp/csp/python/PyAdapterManager.cpp @@ -68,6 +68,19 @@ class PyAdapterManager : public AdapterManager static PyObject * PyAdapterManager_PyObject_starttime( PyAdapterManager_PyObject * self ) { return toPython( self -> manager -> starttime() ); } static PyObject * PyAdapterManager_PyObject_endtime( PyAdapterManager_PyObject * self ) { return toPython( self -> manager -> endtime() ); } +static PyObject * PyAdapterManager_PyObject__engine_shutdown( PyAdapterManager_PyObject * self, PyObject * args, PyObject * kwargs) +{ + CSP_BEGIN_METHOD; + + const char * msg; + if( !PyArg_ParseTuple( args, "s", &msg ) ) + return NULL; + + self -> manager -> rootEngine() -> shutdown( std::make_exception_ptr( TracebackStringException( msg ) ) ); + + CSP_RETURN_NONE; +} + static int PyAdapterManager_init( PyAdapterManager_PyObject *self, PyObject *args, PyObject *kwds ) { CSP_BEGIN_METHOD; @@ -83,8 +96,9 @@ static int PyAdapterManager_init( PyAdapterManager_PyObject *self, PyObject *arg } static PyMethodDef PyAdapterManager_methods[] = { - { "starttime", (PyCFunction) PyAdapterManager_PyObject_starttime, METH_NOARGS, "starttime" }, - { "endtime", (PyCFunction) PyAdapterManager_PyObject_endtime, METH_NOARGS, "endtime" }, + { "starttime", (PyCFunction) PyAdapterManager_PyObject_starttime, METH_NOARGS, "starttime" }, + { "endtime", (PyCFunction) PyAdapterManager_PyObject_endtime, METH_NOARGS, "endtime" }, + { "_engine_shutdown", (PyCFunction) PyAdapterManager_PyObject__engine_shutdown, METH_VARARGS, "_engine_shutdown" }, {NULL} }; diff --git a/cpp/csp/python/PyEngine.h b/cpp/csp/python/PyEngine.h index 4a694c0c6..e059e81c7 100644 --- a/cpp/csp/python/PyEngine.h +++ b/cpp/csp/python/PyEngine.h @@ -2,6 +2,7 @@ #define _IN_CSP_PYTHON_PYENGINE_H #include +#include #include #include #include @@ -54,6 +55,21 @@ class CSPIMPL_EXPORT PyEngine final: public PyObject Engine * m_engine; }; +// Generic engine shutdown function for push-type adapters +template +PyObject * PyEngine_shutdown( T * self, PyObject * args, PyObject * kwargs ) +{ + CSP_BEGIN_METHOD; + + const char * msg; + if( !PyArg_ParseTuple( args, "s", &msg ) ) + return NULL; + + self -> adapter -> rootEngine() -> shutdown( std::make_exception_ptr( TracebackStringException( msg ) ) ); + + CSP_RETURN_NONE; +} + }; #endif diff --git a/cpp/csp/python/PyPushInputAdapter.cpp b/cpp/csp/python/PyPushInputAdapter.cpp index 3b5f91b07..013e78db9 100644 --- a/cpp/csp/python/PyPushInputAdapter.cpp +++ b/cpp/csp/python/PyPushInputAdapter.cpp @@ -202,7 +202,8 @@ struct PyPushInputAdapter_PyObject }; static PyMethodDef PyPushInputAdapter_PyObject_methods[] = { - { "push_tick", (PyCFunction) PyPushInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, + { "push_tick", (PyCFunction) PyPushInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, + { "_engine_shutdown", (PyCFunction) PyEngine_shutdown, METH_VARARGS, "engine shutdown" }, {NULL} }; diff --git a/cpp/csp/python/PyPushPullInputAdapter.cpp b/cpp/csp/python/PyPushPullInputAdapter.cpp index d53ed972d..971999078 100644 --- a/cpp/csp/python/PyPushPullInputAdapter.cpp +++ b/cpp/csp/python/PyPushPullInputAdapter.cpp @@ -123,8 +123,9 @@ struct PyPushPullInputAdapter_PyObject }; static PyMethodDef PyPushPullInputAdapter_PyObject_methods[] = { - { "push_tick", (PyCFunction) PyPushPullInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, - { "flag_replay_complete", (PyCFunction) PyPushPullInputAdapter_PyObject::flagReplayComplete, METH_VARARGS, "finish replay ticks" }, + { "push_tick", (PyCFunction) PyPushPullInputAdapter_PyObject::pushTick, METH_VARARGS, "push new tick" }, + { "flag_replay_complete", (PyCFunction) PyPushPullInputAdapter_PyObject::flagReplayComplete, METH_VARARGS, "finish replay ticks" }, + { "_engine_shutdown", (PyCFunction) PyEngine_shutdown, METH_VARARGS, "engine shutdown" }, {NULL} }; diff --git a/csp/impl/adaptermanager.py b/csp/impl/adaptermanager.py index 9ec42d0e0..b53df8338 100644 --- a/csp/impl/adaptermanager.py +++ b/csp/impl/adaptermanager.py @@ -2,6 +2,7 @@ import csp from csp.impl.__cspimpl import _cspimpl +from csp.impl.error_handling import format_engine_shutdown_stack class AdapterManagerImpl(_cspimpl.PyAdapterManager): @@ -24,6 +25,9 @@ def process_next_sim_timeslice(self, now): """ return None + def engine_shutdown(self, msg): + self._engine_shutdown(format_engine_shutdown_stack(msg)) + class ManagedSimInputAdapter(_cspimpl.PyManagedSimInputAdapter): def __init__(self, typ, field_map): diff --git a/csp/impl/error_handling.py b/csp/impl/error_handling.py index e662b7eb8..b4a0303b2 100644 --- a/csp/impl/error_handling.py +++ b/csp/impl/error_handling.py @@ -1,5 +1,6 @@ import ast import os +import traceback import csp.impl from csp.impl.__cspimpl import _cspimpl @@ -48,3 +49,13 @@ def set_print_full_exception_stack(new_value: bool): res = ExceptionContext.PRINT_EXCEPTION_FULL_STACK ExceptionContext.PRINT_EXCEPTION_FULL_STACK = new_value return res + + +def format_engine_shutdown_stack(msg: str): + tb = traceback.format_stack() + tb = tb[:-2] # remove traceback call and internal engine shutdown method + if len(tb) > 1: + tb = tb[-2:] # only keep the current function and the caller (adapter manager) + tb.append(msg) # add the user's error message + tb = "".join(tb) # format into a single string + return tb diff --git a/csp/impl/pushadapter.py b/csp/impl/pushadapter.py index b64356738..e9fe48fcd 100644 --- a/csp/impl/pushadapter.py +++ b/csp/impl/pushadapter.py @@ -1,4 +1,5 @@ from csp.impl.__cspimpl import _cspimpl +from csp.impl.error_handling import format_engine_shutdown_stack PushGroup = _cspimpl.PushGroup PushBatch = _cspimpl.PushBatch @@ -11,5 +12,8 @@ def start(self, starttime, endtime): def stop(self): pass + def engine_shutdown(self, msg): + self._engine_shutdown(format_engine_shutdown_stack(msg)) + # base class # def push_tick( self, value ) diff --git a/csp/impl/pushpulladapter.py b/csp/impl/pushpulladapter.py index f2beccd3a..c92f85411 100644 --- a/csp/impl/pushpulladapter.py +++ b/csp/impl/pushpulladapter.py @@ -1,4 +1,5 @@ from csp.impl.__cspimpl import _cspimpl +from csp.impl.error_handling import format_engine_shutdown_stack PushGroup = _cspimpl.PushGroup PushBatch = _cspimpl.PushBatch @@ -11,5 +12,8 @@ def start(self, starttime, endtime): def stop(self): pass + def engine_shutdown(self, msg): + self._engine_shutdown(format_engine_shutdown_stack(msg)) + # base class # def push_tick( self, time, value ) diff --git a/csp/tests/impl/test_pushadapter.py b/csp/tests/impl/test_pushadapter.py index 1aa7c0566..b77f38f65 100644 --- a/csp/tests/impl/test_pushadapter.py +++ b/csp/tests/impl/test_pushadapter.py @@ -240,6 +240,51 @@ def graph(): result = list(x[1] for x in result) self.assertEqual(result, expected) + def test_adapter_engine_shutdown(self): + class MyPushAdapterImpl(PushInputAdapter): + def __init__(self): + self._thread = None + self._running = False + + def start(self, starttime, endtime): + self._running = True + self._thread = threading.Thread(target=self._run) + self._thread.start() + + def stop(self): + if self._running: + self._running = False + self._thread.join() + + def _run(self): + pushed = False + while self._running: + if pushed: + time.sleep(0.1) + self.engine_shutdown("Dummy exception message") + else: + self.push_tick(0) + pushed = True + + MyPushAdapter = py_push_adapter_def("MyPushAdapter", MyPushAdapterImpl, ts[int]) + + status = {"count": 0} + + @csp.node + def node(x: ts[object]): + if csp.ticked(x): + status["count"] += 1 + + @csp.graph + def graph(): + adapter = MyPushAdapter() + node(adapter) + csp.print("adapter", adapter) + + with self.assertRaisesRegex(Exception, "Dummy exception message"): + csp.run(graph, starttime=datetime.utcnow(), realtime=True) + self.assertEqual(status["count"], 1) + if __name__ == "__main__": unittest.main() diff --git a/csp/tests/impl/test_pushpulladapter.py b/csp/tests/impl/test_pushpulladapter.py index 74faff89f..92557a2a8 100644 --- a/csp/tests/impl/test_pushpulladapter.py +++ b/csp/tests/impl/test_pushpulladapter.py @@ -143,6 +143,64 @@ def graph(): result = [out[1] for out in graph_out[0]] self.assertEqual(result, [1, 2, 3]) + def test_adapter_engine_shutdown(self): + class MyPushPullAdapterImpl(PushPullInputAdapter): + def __init__(self, typ, data, shutdown_before_live): + self._data = data + self._thread = None + self._running = False + self._shutdown_before_live = shutdown_before_live + + def start(self, starttime, endtime): + self._running = True + self._thread = threading.Thread(target=self._run) + self._thread.start() + + def stop(self): + if self._running: + self._running = False + self._thread.join() + + def _run(self): + idx = 0 + while self._running and idx < len(self._data): + if idx and self._shutdown_before_live: + time.sleep(0.1) + self.engine_shutdown("Dummy exception message") + t, v = self._data[idx] + self.push_tick(False, t, v) + idx += 1 + self.flag_replay_complete() + + idx = 0 + while self._running: + self.push_tick(True, datetime.utcnow(), len(self._data) + 1) + if idx and not self._shutdown_before_live: + time.sleep(0.1) + self.engine_shutdown("Dummy exception message") + idx += 1 + + MyPushPullAdapter = py_pushpull_adapter_def( + "MyPushPullAdapter", MyPushPullAdapterImpl, ts["T"], typ="T", data=list, shutdown_before_live=bool + ) + + @csp.graph + def graph(shutdown_before_live: bool): + data = [(datetime(2020, 1, 1, 2), 1), (datetime(2020, 1, 1, 3), 2)] + adapter = MyPushPullAdapter(int, data, shutdown_before_live) + csp.print("adapter", adapter) + + with self.assertRaisesRegex(Exception, "Dummy exception message"): + csp.run(graph, True, starttime=datetime(2020, 1, 1, 1)) + with self.assertRaisesRegex(Exception, "Dummy exception message"): + csp.run( + graph, + False, + starttime=datetime(2020, 1, 1, 1), + endtime=datetime.utcnow() + timedelta(seconds=2), + realtime=True, + ) + if __name__ == "__main__": unittest.main() diff --git a/csp/tests/test_engine.py b/csp/tests/test_engine.py index adb67d372..9b0698333 100644 --- a/csp/tests/test_engine.py +++ b/csp/tests/test_engine.py @@ -799,6 +799,48 @@ def graph(): b[t].append(v) self.assertEqual(results["b"], list(b.items())) + def test_adapter_manager_engine_shutdown(self): + from csp.impl.adaptermanager import AdapterManagerImpl, ManagedSimInputAdapter + from csp.impl.wiring import py_managed_adapter_def + + class TestAdapterManager: + def __init__(self): + self._impl = None + + def subscribe(self): + return TestAdapter(self) + + def _create(self, engine, memo): + self._impl = TestAdapterManagerImpl(engine) + return self._impl + + class TestAdapterManagerImpl(AdapterManagerImpl): + def __init__(self, engine): + super().__init__(engine) + + def start(self, starttime, endtime): + pass + + def stop(self): + pass + + def process_next_sim_timeslice(self, now): + self.engine_shutdown("Dummy exception message") + + class TestAdapterImpl(ManagedSimInputAdapter): + def __init__(self, manager_impl): + pass + + TestAdapter = py_managed_adapter_def("TestAdapter", TestAdapterImpl, ts[int], TestAdapterManager) + + def graph(): + adapter = TestAdapterManager() + nc = adapter.subscribe() + csp.add_graph_output("nc", nc) + + with self.assertRaisesRegex(Exception, "Dummy exception message"): + csp.run(graph, starttime=datetime(2020, 1, 1), endtime=timedelta(seconds=1)) + def test_feedback(self): # Dummy example class Request(csp.Struct): From 84cc40dd5f440abe68290d423be4e575270333ab Mon Sep 17 00:00:00 2001 From: "Gajowniczek, Artur" Date: Thu, 11 Jul 2024 09:12:12 -0400 Subject: [PATCH 17/17] passing exception to engine shutdown wip Signed-off-by: Gajowniczek, Artur --- cpp/csp/python/Exception.h | 13 +++++++++++++ cpp/csp/python/PyAdapterManager.cpp | 12 +++++++++--- csp/impl/adaptermanager.py | 2 +- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/cpp/csp/python/Exception.h b/cpp/csp/python/Exception.h index 1355c6f67..6249d2ccc 100644 --- a/cpp/csp/python/Exception.h +++ b/cpp/csp/python/Exception.h @@ -30,6 +30,18 @@ class PythonPassthrough : public csp::Exception PyErr_Fetch( &m_type, &m_value, &m_traceback ); } + PythonPassthrough( PyObject * pyException ) : + m_pyException( pyException ), + m_type( PyObject_Type( pyException ) ), + m_value( PyObject_Str( pyException ) ), + m_traceback( PyException_GetTraceback( pyException ) ), + csp::Exception( PyUnicode_AsUTF8( m_type ), std::string( PyUnicode_AsUTF8( m_value ) ) ) + { + Py_INCREF( pyException ); + } + + ~PythonPassthrough() { Py_DECREF( m_pyException ); } + void restore() { if( !description().empty() ) @@ -50,6 +62,7 @@ class PythonPassthrough : public csp::Exception PyObject * m_type; PyObject * m_value; PyObject * m_traceback; + PyObject * m_pyException; }; diff --git a/cpp/csp/python/PyAdapterManager.cpp b/cpp/csp/python/PyAdapterManager.cpp index c412a0ee0..1ecd04b37 100644 --- a/cpp/csp/python/PyAdapterManager.cpp +++ b/cpp/csp/python/PyAdapterManager.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace csp::python { @@ -72,11 +73,16 @@ static PyObject * PyAdapterManager_PyObject__engine_shutdown( PyAdapterManager_P { CSP_BEGIN_METHOD; - const char * msg; - if( !PyArg_ParseTuple( args, "s", &msg ) ) + PyObject * pyException = nullptr; + + if( !PyArg_ParseTuple( args, "O", &pyException ) ) return NULL; - self -> manager -> rootEngine() -> shutdown( std::make_exception_ptr( TracebackStringException( msg ) ) ); + PyObject * type = PyObject_Type( pyException ); + PyObject * val = PyObject_Str( pyException ); + PyObject * traceback = PyException_GetTraceback( pyException ); + + self -> manager -> rootEngine() -> shutdown( std::make_exception_ptr( PythonPassthrough( pyException ) ) ); CSP_RETURN_NONE; } diff --git a/csp/impl/adaptermanager.py b/csp/impl/adaptermanager.py index b53df8338..10c57bc2e 100644 --- a/csp/impl/adaptermanager.py +++ b/csp/impl/adaptermanager.py @@ -26,7 +26,7 @@ def process_next_sim_timeslice(self, now): return None def engine_shutdown(self, msg): - self._engine_shutdown(format_engine_shutdown_stack(msg)) + self._engine_shutdown(Exception('test')) # temporary for testing purposes class ManagedSimInputAdapter(_cspimpl.PyManagedSimInputAdapter):