Merge pull request #108 from bluescarni/pickle_experiment

Cloudpickle

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.2)
 
-project(pagmo VERSION 2.2)
+project(pagmo VERSION 2.3)
 
 enable_testing()
 

appveyor.yml

@@ -27,12 +27,12 @@ install:
 
 - if [%BUILD_TYPE%]==[MSVC_64_Python35] set PATH=C:\Miniconda35-x64\Scripts;%PATH%
 - if [%BUILD_TYPE%]==[MSVC_64_Python35] conda config --add channels conda-forge --force
-- if [%BUILD_TYPE%]==[MSVC_64_Python35] conda create -y --name pagmo python=3.5 cmake boost eigen nlopt numpy dill ipyparallel
+- if [%BUILD_TYPE%]==[MSVC_64_Python35] conda create -y --name pagmo python=3.5 cmake boost eigen nlopt numpy cloudpickle ipyparallel
 - if [%BUILD_TYPE%]==[MSVC_64_Python35] call activate pagmo
 
 - if [%BUILD_TYPE%]==[MSVC_64_Python36] set PATH=C:\Miniconda36-x64\Scripts;%PATH%
 - if [%BUILD_TYPE%]==[MSVC_64_Python36] conda config --add channels conda-forge --force
-- if [%BUILD_TYPE%]==[MSVC_64_Python36] conda create -y --name pagmo python=3.6 cmake boost eigen nlopt numpy dill ipyparallel
+- if [%BUILD_TYPE%]==[MSVC_64_Python36] conda create -y --name pagmo python=3.6 cmake boost eigen nlopt numpy cloudpickle ipyparallel
 - if [%BUILD_TYPE%]==[MSVC_64_Python36] call activate pagmo
 
 # Rename sh.exe as sh.exe in PATH interferes with MinGW.

doc/sphinx/changelog.rst

@@ -1,6 +1,21 @@
 Changelog
 =========
 
+2.3 (2017-05-19)
+----------------
+
+Changes
+~~~~~~~
+
+- Move from dill to cloudpickle as a serialization backend. This fixes various serialization issues reported in
+  `#106 <https://github.com/esa/pagmo2/issues/106>`_.
+
+Fix
+~~~
+
+- Various documentation fixes and improvements (`#103 <https://github.com/esa/pagmo2/issues/103>`_,
+  `#104 <https://github.com/esa/pagmo2/issues/104>`_, `#107 <https://github.com/esa/pagmo2/issues/107>`_).
+
 2.2 (2017-05-12)
 ----------------
 

doc/sphinx/docs/python/tutorials/coding_udi.rst

@@ -30,7 +30,7 @@ on some pygmo classes. The above UDI can then be used to construct a :class:`~py
 
     >>> isl = pg.island(algo = pg.de(100), prob = pg.ackley(5), udi = my_isl(), size = 20)
     >>> print(isl) # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
-    Island name: Its my island!
+    Island name: It's my island!
         Status: idle
     <BLANKLINE>
     Algorithm: Differential Evolution

pygmo/_py_islands.py

@@ -275,10 +275,10 @@ def _shutdown_pool():
                 mp_island._pool.join()
 
 
-# Make sure we use dill for serialization, if ipyparallel is available.
+# Make sure we use cloudpickle for serialization, if ipyparallel is available.
 try:
-    from ipyparallel import use_dill as _use_dill
-    _use_dill()
+    from ipyparallel import use_cloudpickle as _use_cloudpickle
+    _use_cloudpickle()
 except ImportError:
     pass
 

pygmo/core.cpp

@@ -318,13 +318,13 @@ BOOST_PYTHON_MODULE(core)
     }
     wrap_import_array();
 
-    // Check that dill is available.
+    // Check that cloudpickle is available.
     try {
-        bp::import("dill");
+        bp::import("cloudpickle");
     } catch (...) {
         pygmo::builtin().attr("print")(
-            u8"\033[91m====ERROR====\nThe dill module could not be imported. "
-            u8"Please make sure that dill has been correctly installed.\n====ERROR====\033[0m");
+            u8"\033[91m====ERROR====\nThe cloudpickle module could not be imported. "
+            u8"Please make sure that cloudpickle has been correctly installed.\n====ERROR====\033[0m");
         pygmo_throw(PyExc_ImportError, "");
     }
 

pygmo/object_serialization.hpp

@@ -53,12 +53,12 @@ inline void save(Archive &archive, const boost::python::object &o)
     // http://stackoverflow.com/questions/27518554/c-cereal-serialize-c-style-array
     using namespace boost::python;
     // This will dump to a bytes object.
-    object tmp = import("dill").attr("dumps")(o);
+    object tmp = import("cloudpickle").attr("dumps")(o);
     // This gives a null-terminated char * to the internal
     // content of the bytes object.
     auto ptr = PyBytes_AsString(tmp.ptr());
     if (!ptr) {
-        pygmo_throw(PyExc_TypeError, "dill's dumps() function did not return a bytes object");
+        pygmo_throw(PyExc_TypeError, "cloudpickle's dumps() function did not return a bytes object");
     }
     // NOTE: this will be the length of the bytes object *without* the terminator.
     const auto size = len(tmp);
@@ -76,7 +76,7 @@ inline void load(Archive &archive, boost::python::object &o)
     std::vector<char> v;
     archive(v);
     auto b = pygmo::make_bytes(v.data(), boost::numeric_cast<Py_ssize_t>(v.size()));
-    o = import("dill").attr("loads")(b);
+    o = import("cloudpickle").attr("loads")(b);
 }
 }
 

tools/install_appveyor_mingw.py

@@ -109,8 +109,8 @@ def run_command(raw_command, directory=None, verbose=True):
     run_command(pinterp + ' get-pip.py --force-reinstall')
     # NOTE: at the moment we have troubles installing ipyparallel.
     # Just skip it.
-    # run_command(pip + ' install numpy dill ipyparallel')
-    run_command(pip + ' install numpy dill')
+    # run_command(pip + ' install numpy cloudpickle ipyparallel')
+    run_command(pip + ' install numpy cloudpickle')
     if is_release_build:
         run_command(pip + ' install twine')
 

tools/install_deps.sh

@@ -17,9 +17,9 @@ if [[ "${PAGMO_BUILD}" != manylinux* ]]; then
     conda_pkgs="boost>=1.55 cmake>=3.2 eigen nlopt ipopt"
 
     if [[ "${PAGMO_BUILD}" == "Python36" || "${PAGMO_BUILD}" == "OSXPython36" ]]; then
-        conda_pkgs="$conda_pkgs python=3.6 numpy dill ipyparallel numba"
+        conda_pkgs="$conda_pkgs python=3.6 numpy cloudpickle ipyparallel numba"
     elif [[ "${PAGMO_BUILD}" == "Python27" || "${PAGMO_BUILD}" == "OSXPython27" ]]; then
-        conda_pkgs="$conda_pkgs python=2.7 numpy dill ipyparallel numba"
+        conda_pkgs="$conda_pkgs python=2.7 numpy cloudpickle ipyparallel numba"
     fi
 
     if [[ "${PAGMO_BUILD}" == Python* ]]; then

tools/install_docker.sh

@@ -74,7 +74,7 @@ make -j2 install > /dev/null
 cd ..
 
 # Python deps
-/opt/python/${PYTHON_DIR}/bin/pip install dill numpy ipyparallel
+/opt/python/${PYTHON_DIR}/bin/pip install cloudpickle numpy ipyparallel
 /opt/python/${PYTHON_DIR}/bin/ipcluster start --daemonize=True
 sleep 20
 

tools/install_travis.sh

@@ -53,7 +53,7 @@ elif [[ "${PAGMO_BUILD}" == Python* ]]; then
     sleep 20;
     python -c "import pygmo; pygmo.test.run_test_suite(1)"
     # At the moment conda has these packages only for Python 3.4. Install via pip instead.
-    pip install sphinx breathe requests[security] sphinx-bootstrap-theme;
+    pip install 'sphinx<1.6' breathe requests[security] sphinx-bootstrap-theme;
     # Run doxygen and check the output.
     cd ../doc/doxygen;
     export DOXYGEN_OUTPUT=`doxygen 2>&1 >/dev/null`;

tools/wheel_setup.py

@@ -31,7 +31,7 @@
     'Programming Language :: Python :: 3'
 ]
 KEYWORDS = 'science math physics optimization ai evolutionary-computing parallel-computing metaheuristics'
-INSTALL_REQUIRES = ['numpy', 'dill']
+INSTALL_REQUIRES = ['numpy', 'cloudpickle']
 PLATFORMS = ['Unix', 'Windows', 'OSX']