-
-
Notifications
You must be signed in to change notification settings - Fork 30.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
gh-84016: Implement missing PyObject_CopyToObject
in PEP3118
#124747
Changes from all commits
5124891
686673e
2eff13d
4569791
a91c5ea
df16015
50bfb42
6498381
3c0436e
0e7de15
1d797ef
3388eff
6e9bfd8
e83402a
981b6c5
7bc039e
edd093d
66101e0
611ad22
eb1a891
1153275
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -517,6 +517,16 @@ Buffer-related functions | |||||
|
||||||
``0`` is returned on success, ``-1`` on error. | ||||||
|
||||||
.. c:function:: int PyObject_CopyToObject(PyObject *obj, void *buf, Py_ssize_t len, char fort) | ||||||
|
||||||
Copy data from *buf* to *obj* buffer. Can convert between C-style and | ||||||
or Fortran-style buffers (C-style if *fort* is ``'C'`` or Fortran-style | ||||||
if *fort* is ``'F'``, ``'A'`` or other for defalut quick copy). | ||||||
|
||||||
``0`` is returned on success, ``-1`` on error. | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
|
||||||
.. versionadded:: 3.14 | ||||||
|
||||||
.. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) | ||||||
|
||||||
Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if | ||||||
|
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -339,6 +339,20 @@ def test_object_delattrstring(self): | |
# CRASHES delattrstring(obj, NULL) | ||
# CRASHES delattrstring(NULL, b'a') | ||
|
||
def test_copy_to_object(self): | ||
copy_to_object = _testcapi.object_copy_to_object | ||
s1 = copy_to_object(bytearray(3), 'abc', b'C') | ||
s2 = copy_to_object(bytearray(3), 'abc', b'F') | ||
s3 = copy_to_object(bytearray(3), 'abc', b'A') | ||
self.assertEqual(s1, s2) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would prefer to compare it to a known bytearray, such as: |
||
self.assertEqual(s2, s3) | ||
self.assertEqual(s1, s3) | ||
self.assertRaises(BufferError, copy_to_object, bytearray(2), 'abc', b'C') | ||
self.assertRaises(BufferError, copy_to_object, bytearray(2), 'abc', b'F') | ||
self.assertRaises(BufferError, copy_to_object, bytearray(2), 'abc', b'A') | ||
self.assertRaises(TypeError, copy_to_object, list(), 'abc', b'C') | ||
self.assertRaises(TypeError, copy_to_object, list(), 'abc', b'F') | ||
self.assertRaises(TypeError, copy_to_object, list(), 'abc', b'A') | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You should test read-only buffer. |
||
|
||
def test_mapping_check(self): | ||
check = _testlimitedcapi.mapping_check | ||
|
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
Added :c:func:`PyObject_CopyToObject` to copy a C-style or Fortran-style | ||
buffer to a Python object. |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2283,6 +2283,8 @@ | |
added = '3.11' | ||
[function.PyObject_CopyData] | ||
added = '3.11' | ||
[function.PyObject_CopyToObject] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Don't add this function directly to the stable ABI: remove this entry. |
||
added = '3.14' | ||
[function.PyBuffer_IsContiguous] | ||
added = '3.11' | ||
[function.PyBuffer_FillContiguousStrides] | ||
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -78,6 +78,25 @@ object_hasattrstringwitherror(PyObject *self, PyObject *args) | |||||
RETURN_INT(PyObject_HasAttrStringWithError(obj, attr_name)); | ||||||
} | ||||||
|
||||||
static PyObject * | ||||||
object_copy_to_object(PyObject *self, PyObject *args) | ||||||
{ | ||||||
PyObject *obj; | ||||||
Py_ssize_t len; | ||||||
int result; | ||||||
char *buf, fort; | ||||||
|
||||||
if (!PyArg_ParseTuple(args, "Os#c", &obj, &buf, &len, &fort)) { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You should not expect Unicode, but raw bytes:
Suggested change
|
||||||
return NULL; | ||||||
} | ||||||
result = PyObject_CopyToObject(obj, buf, len, fort); | ||||||
if (result < 0) { | ||||||
return NULL; | ||||||
} | ||||||
Py_INCREF(obj); | ||||||
return obj; | ||||||
} | ||||||
|
||||||
static PyObject * | ||||||
mapping_getoptionalitemstring(PyObject *self, PyObject *args) | ||||||
{ | ||||||
|
@@ -162,6 +181,8 @@ static PyMethodDef test_methods[] = { | |||||
{"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, | ||||||
{"object_hasattrwitherror", object_hasattrwitherror, METH_VARARGS}, | ||||||
{"object_hasattrstringwitherror", object_hasattrstringwitherror, METH_VARARGS}, | ||||||
{"object_copy_to_object", object_copy_to_object, METH_VARARGS}, | ||||||
|
||||||
{"mapping_getoptionalitem", mapping_getoptionalitem, METH_VARARGS}, | ||||||
{"mapping_getoptionalitemstring", mapping_getoptionalitemstring, METH_VARARGS}, | ||||||
|
||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -663,7 +663,8 @@ PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, | |
return 0; | ||
} | ||
|
||
int PyObject_CopyData(PyObject *dest, PyObject *src) | ||
int | ||
PyObject_CopyData(PyObject *dest, PyObject *src) | ||
{ | ||
Py_buffer view_dest, view_src; | ||
int k; | ||
|
@@ -733,6 +734,59 @@ int PyObject_CopyData(PyObject *dest, PyObject *src) | |
return 0; | ||
} | ||
|
||
int | ||
PyObject_CopyToObject(PyObject *obj, void *buf, Py_ssize_t len, char fort) | ||
{ | ||
char *tmp_obj, *tmp_buf = buf; | ||
Py_buffer view_obj; | ||
Py_ssize_t *indices, elem_num = 1; | ||
|
||
assert(buf != NULL); | ||
if (!PyObject_CheckBuffer(obj)) { | ||
PyErr_SetString(PyExc_TypeError, | ||
"object supporting the buffer protocol required"); | ||
return -1; | ||
} | ||
|
||
if (PyObject_GetBuffer(obj, &view_obj, PyBUF_FULL)) { | ||
return -1; | ||
} | ||
|
||
if (view_obj.len < len) { | ||
PyErr_SetString(PyExc_BufferError, | ||
"destination is too small to receive data from buf"); | ||
PyBuffer_Release(&view_obj); | ||
return -1; | ||
} | ||
|
||
/* just copy it directly through memcpy */ | ||
if (fort == 'C' || fort == 'F') { | ||
memcpy(view_obj.buf, buf, len); | ||
PyBuffer_Release(&view_obj); | ||
return 0; | ||
} | ||
Comment on lines
+762
to
+767
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This case is still incorrect; There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When the target is multi-dimensional, their memory is still continuous. Refer to PyObject_CopyData for details. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm sorry but maybe there is some misunderstanding. I suggest reading this post to understand the difference between a C and a F-style ordering (namely row vs column-major order): https://manik.cc/2021/02/25/memory-order.html. Note that PyObject_CopyData only uses memcpy when the memory is contiguous, and this is checked via |
||
|
||
/* quick copy implementation */ | ||
indices = (Py_ssize_t *)PyMem_Calloc(view_obj.ndim, sizeof(Py_ssize_t)); | ||
if (!indices) { | ||
PyErr_NoMemory(); | ||
PyBuffer_Release(&view_obj); | ||
return -1; | ||
} | ||
for (int i = 0; i < view_obj.ndim; i++) { | ||
/* XXX(nnorwitz): can this overflow? */ | ||
elem_num *= view_obj.shape[i]; | ||
} | ||
while (elem_num--) { | ||
tmp_obj = PyBuffer_GetPointer(&view_obj, indices); | ||
memcpy(tmp_obj, tmp_buf++, view_obj.itemsize); | ||
_Py_add_one_to_index_C(view_obj.ndim, indices, view_obj.shape); | ||
} | ||
PyMem_Free(indices); | ||
PyBuffer_Release(&view_obj); | ||
return 0; | ||
} | ||
|
||
void | ||
PyBuffer_FillContiguousStrides(int nd, Py_ssize_t *shape, | ||
Py_ssize_t *strides, int itemsize, | ||
|
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.