From 9706b4090c233f726d76c78f5898379d1d401bf7 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Wed, 3 Sep 2025 12:50:17 +0700 Subject: [PATCH 01/20] Turn rapidjson.Decoder into heap type Each static type is uniq and global for all subinterpreters and can't access cached PyObjects. Decoder do use cached PyObjecs and must access cache stored in module rather than in static variables. Static type it can't access "proper module" due to its "per process singleton nature". The recomended way is to turn static types into heap types in order to use cached PyObjects. --- rapidjson.cpp | 123 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 78 insertions(+), 45 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 1cb2e1f..9a426e2 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -26,6 +27,53 @@ using namespace rapidjson; +#if PY_VERSION_HEX < 0x030A0000 +// Fallback for Py_TPFLAGS_IMMUTABLETYPE which is added in 3.10 +#define Py_TPFLAGS_IMMUTABLETYPE 0 +#endif + + +static uint32_t +py_version_hex() noexcept { +#if PY_VERSION_HEX < 0x030B0000 + static const uint32_t once_fetched_ver = [] { + constexpr uint32_t ver_fallback = 0x03000000; + + auto* ver_tuple = PySys_GetObject("version_info"); + if (!ver_tuple) + return ver_fallback; + + long major, minor, micro, serial; + const char* releaselevel; + if (!PyArg_ParseTuple(ver_tuple, "lllsl", &major, &minor, µ, &releaselevel, &serial)) + return ver_fallback; + + return uint32_t(((major & 0xFF) << 24) | ((minor & 0xFF) << 16) | ((micro & 0xFF) << 8)); + }(); + return once_fetched_ver; +#else + return Py_Version; +#endif +} + + +struct PyDerefer { + void operator() (PyObject* obj) const noexcept {Py_DecRef(obj);} +}; +using PyStrongRef = std::unique_ptr; + + +static PyStrongRef +inline from_module_and_spec(PyObject& module, PyType_Spec& spec) noexcept { +#if PY_VERSION_HEX >= 0x03090000 + PyStrongRef type{PyType_FromModuleAndSpec(&module, &spec, NULL)}; +#else + PyStrongRef type{PyType_FromSpec(&spec)}; +#endif + return type; +} + + /* On some MacOS combo, using Py_IS_XXX() macros does not work (see https://github.com/python-rapidjson/python-rapidjson/issues/78). OTOH, MSVC < 2015 does not have std::isxxx() (see @@ -206,6 +254,7 @@ static PyObject* do_decode(PyObject* decoder, unsigned uuidMode, unsigned parseMode); static PyObject* decoder_call(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs); +static int decoder_traverse(PyObject *op, visitproc visit, void *arg); static PyObject* do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, @@ -1882,46 +1931,22 @@ static PyMemberDef decoder_members[] = { }; -static PyTypeObject Decoder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "rapidjson.Decoder", /* tp_name */ - sizeof(DecoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc) decoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - decoder_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - decoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - decoder_new, /* tp_new */ - PyObject_Del, /* tp_free */ +static PyType_Slot Decoder_Type_Slot[] = { + {Py_tp_doc, const_cast(decoder_doc)}, + {Py_tp_call, reinterpret_cast(decoder_call)}, + {Py_tp_members, decoder_members}, + {Py_tp_new, reinterpret_cast(decoder_new)}, + {Py_tp_traverse, reinterpret_cast(decoder_traverse)}, + {0, NULL} +}; + + +static PyType_Spec Decoder_Type_Spec = { + "rapidjson.Decoder", /* name */ + sizeof(DecoderObject), /* basicsize */ + 0, /* itemsize */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, /* flags */ + Decoder_Type_Slot /* slots */ }; @@ -2287,6 +2312,16 @@ decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) } +static int +decoder_traverse(PyObject *op, visitproc visit, void *arg) +{ + if (py_version_hex() >= 0x03090000) + Py_VISIT(Py_TYPE(op)); + return 0; +} + + + ///////////// // Encoder // ///////////// @@ -3868,7 +3903,8 @@ module_exec(PyObject* m) PyObject* decimalModule; PyObject* uuidModule; - if (PyType_Ready(&Decoder_Type) < 0) + auto decoder_type = from_module_and_spec(*m, Decoder_Type_Spec); + if (!decoder_type) return -1; if (PyType_Ready(&Encoder_Type) < 0) @@ -4051,11 +4087,8 @@ module_exec(PyObject* m) ) return -1; - Py_INCREF(&Decoder_Type); - if (PyModule_AddObject(m, "Decoder", (PyObject*) &Decoder_Type) < 0) { - Py_DECREF(&Decoder_Type); + if (PyModule_AddObject(m, "Decoder", decoder_type.get()) < 0) return -1; - } Py_INCREF(&Encoder_Type); if (PyModule_AddObject(m, "Encoder", (PyObject*) &Encoder_Type) < 0) { From d8a3f836d1a5403fdb230c786679da4293d91a24 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Mon, 24 Aug 2026 07:52:29 +0200 Subject: [PATCH 02/20] Properly release the stolen StrongRef See https://github.com/python-rapidjson/python-rapidjson/pull/227#pullrequestreview-4980034622 for an explanation. --- rapidjson.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/rapidjson.cpp b/rapidjson.cpp index 9a426e2..00bf912 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -4089,6 +4089,7 @@ module_exec(PyObject* m) if (PyModule_AddObject(m, "Decoder", decoder_type.get()) < 0) return -1; + decoder_type.release(); Py_INCREF(&Encoder_Type); if (PyModule_AddObject(m, "Encoder", (PyObject*) &Encoder_Type) < 0) { From c3c5581db3e3fa4ad710d4329b67b65ec8fde36c Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Mon, 24 Aug 2026 08:43:08 +0200 Subject: [PATCH 03/20] Simplify, assuming Python >= 3.10 --- rapidjson.cpp | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 00bf912..f2d5ff1 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -27,36 +27,6 @@ using namespace rapidjson; -#if PY_VERSION_HEX < 0x030A0000 -// Fallback for Py_TPFLAGS_IMMUTABLETYPE which is added in 3.10 -#define Py_TPFLAGS_IMMUTABLETYPE 0 -#endif - - -static uint32_t -py_version_hex() noexcept { -#if PY_VERSION_HEX < 0x030B0000 - static const uint32_t once_fetched_ver = [] { - constexpr uint32_t ver_fallback = 0x03000000; - - auto* ver_tuple = PySys_GetObject("version_info"); - if (!ver_tuple) - return ver_fallback; - - long major, minor, micro, serial; - const char* releaselevel; - if (!PyArg_ParseTuple(ver_tuple, "lllsl", &major, &minor, µ, &releaselevel, &serial)) - return ver_fallback; - - return uint32_t(((major & 0xFF) << 24) | ((minor & 0xFF) << 16) | ((micro & 0xFF) << 8)); - }(); - return once_fetched_ver; -#else - return Py_Version; -#endif -} - - struct PyDerefer { void operator() (PyObject* obj) const noexcept {Py_DecRef(obj);} }; @@ -65,11 +35,7 @@ using PyStrongRef = std::unique_ptr; static PyStrongRef inline from_module_and_spec(PyObject& module, PyType_Spec& spec) noexcept { -#if PY_VERSION_HEX >= 0x03090000 PyStrongRef type{PyType_FromModuleAndSpec(&module, &spec, NULL)}; -#else - PyStrongRef type{PyType_FromSpec(&spec)}; -#endif return type; } @@ -2315,8 +2281,7 @@ decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) static int decoder_traverse(PyObject *op, visitproc visit, void *arg) { - if (py_version_hex() >= 0x03090000) - Py_VISIT(Py_TYPE(op)); + Py_VISIT(Py_TYPE(op)); return 0; } From e027049cc589b6627e81c9c63fea90c547b0bbfd Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:19:51 +0200 Subject: [PATCH 04/20] Turn rapidjson.Encoder into heap type --- rapidjson.cpp | 85 ++++++++++++++++++--------------------------------- 1 file changed, 29 insertions(+), 56 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index f2d5ff1..550471a 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -40,6 +40,14 @@ inline from_module_and_spec(PyObject& module, PyType_Spec& spec) noexcept { } +static int +heap_type_traverse(PyObject *op, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(op)); + return 0; +} + + /* On some MacOS combo, using Py_IS_XXX() macros does not work (see https://github.com/python-rapidjson/python-rapidjson/issues/78). OTOH, MSVC < 2015 does not have std::isxxx() (see @@ -220,7 +228,6 @@ static PyObject* do_decode(PyObject* decoder, unsigned uuidMode, unsigned parseMode); static PyObject* decoder_call(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs); -static int decoder_traverse(PyObject *op, visitproc visit, void *arg); static PyObject* do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, @@ -1902,7 +1909,7 @@ static PyType_Slot Decoder_Type_Slot[] = { {Py_tp_call, reinterpret_cast(decoder_call)}, {Py_tp_members, decoder_members}, {Py_tp_new, reinterpret_cast(decoder_new)}, - {Py_tp_traverse, reinterpret_cast(decoder_traverse)}, + {Py_tp_traverse, reinterpret_cast(heap_type_traverse)}, {0, NULL} }; @@ -2278,15 +2285,6 @@ decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) } -static int -decoder_traverse(PyObject *op, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(op)); - return 0; -} - - - ///////////// // Encoder // ///////////// @@ -3331,46 +3329,22 @@ static PyGetSetDef encoder_props[] = { {NULL} }; -static PyTypeObject Encoder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "rapidjson.Encoder", /* tp_name */ - sizeof(EncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc) encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - encoder_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - encoder_props, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - PyObject_Del, /* tp_free */ +static PyType_Slot Encoder_Type_Slot[] = { + {Py_tp_doc, const_cast(encoder_doc)}, + {Py_tp_call, reinterpret_cast(encoder_call)}, + {Py_tp_members, encoder_members}, + {Py_tp_getset, encoder_props}, + {Py_tp_new, reinterpret_cast(encoder_new)}, + {Py_tp_traverse, reinterpret_cast(heap_type_traverse)}, + {0, NULL} +}; + +static PyType_Spec Encoder_Type_Spec = { + "rapidjson.Encoder", /* name */ + sizeof(EncoderObject), /* basicsize */ + 0, /* itemsize */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, /* flags */ + Encoder_Type_Slot /* slots */ }; @@ -3872,7 +3846,8 @@ module_exec(PyObject* m) if (!decoder_type) return -1; - if (PyType_Ready(&Encoder_Type) < 0) + auto encoder_type = from_module_and_spec(*m, Encoder_Type_Spec); + if (!encoder_type) return -1; if (PyType_Ready(&Validator_Type) < 0) @@ -4056,11 +4031,9 @@ module_exec(PyObject* m) return -1; decoder_type.release(); - Py_INCREF(&Encoder_Type); - if (PyModule_AddObject(m, "Encoder", (PyObject*) &Encoder_Type) < 0) { - Py_DECREF(&Encoder_Type); + if (PyModule_AddObject(m, "Encoder", encoder_type.get()) < 0) return -1; - } + encoder_type.release(); Py_INCREF(&Validator_Type); if (PyModule_AddObject(m, "Validator", (PyObject*) &Validator_Type) < 0) { From 0e8c7c069c29ede6a0a97106f43c200d951d6ceb Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:24:57 +0200 Subject: [PATCH 05/20] Turn rapidjson.Validator into heap type --- rapidjson.cpp | 65 ++++++++++++++++----------------------------------- 1 file changed, 20 insertions(+), 45 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 550471a..13da2b4 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -3629,46 +3629,22 @@ PyDoc_STRVAR(validator_doc, " string."); -static PyTypeObject Validator_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "rapidjson.Validator", /* tp_name */ - sizeof(ValidatorObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) validator_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc) validator_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - validator_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - validator_new, /* tp_new */ - PyObject_Del, /* tp_free */ +static PyType_Slot Validator_Type_Slot[] = { + {Py_tp_doc, const_cast(validator_doc)}, + {Py_tp_call, reinterpret_cast(validator_call)}, + {Py_tp_new, reinterpret_cast(validator_new)}, + {Py_tp_traverse, reinterpret_cast(heap_type_traverse)}, + {Py_tp_dealloc, reinterpret_cast(validator_dealloc)}, + {0, NULL} +}; + + +static PyType_Spec Validator_Type_Spec = { + "rapidjson.Validator", /* name */ + sizeof(ValidatorObject), /* basicsize */ + 0, /* itemsize */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, /* flags */ + Validator_Type_Slot /* slots */ }; @@ -3850,7 +3826,8 @@ module_exec(PyObject* m) if (!encoder_type) return -1; - if (PyType_Ready(&Validator_Type) < 0) + auto validator_type = from_module_and_spec(*m, Validator_Type_Spec); + if (!validator_type) return -1; if (PyType_Ready(&RawJSON_Type) < 0) @@ -4035,11 +4012,9 @@ module_exec(PyObject* m) return -1; encoder_type.release(); - Py_INCREF(&Validator_Type); - if (PyModule_AddObject(m, "Validator", (PyObject*) &Validator_Type) < 0) { - Py_DECREF(&Validator_Type); + if (PyModule_AddObject(m, "Validator", validator_type.get()) < 0) return -1; - } + validator_type.release(); Py_INCREF(&RawJSON_Type); if (PyModule_AddObject(m, "RawJSON", (PyObject*) &RawJSON_Type) < 0) { From 7606903eb39e6e61ecc703882c37587e171ebffc Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:29:36 +0200 Subject: [PATCH 06/20] Turn rapidjson.RawJSON into heap type --- rapidjson.cpp | 68 +++++++++++++++++---------------------------------- 1 file changed, 23 insertions(+), 45 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 13da2b4..a69b884 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -67,6 +67,7 @@ static PyObject* decimal_type = NULL; static PyObject* timezone_type = NULL; static PyObject* timezone_utc = NULL; static PyObject* uuid_type = NULL; +static PyObject* rawjson_type = NULL; static PyObject* validation_error = NULL; static PyObject* decode_error = NULL; @@ -526,45 +527,22 @@ PyDoc_STRVAR(rawjson_doc, "'{\"already\": \"serialized\"}'"); -static PyTypeObject RawJSON_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "rapidjson.RawJSON", /* tp_name */ - sizeof(RawJSON), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor) RawJSON_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - rawjson_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - RawJSON_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - RawJSON_new, /* tp_new */ +static PyType_Slot RawJSON_Type_Slot[] = { + {Py_tp_dealloc, reinterpret_cast(RawJSON_dealloc)}, + {Py_tp_doc, const_cast(rawjson_doc)}, + {Py_tp_members, reinterpret_cast(RawJSON_members)}, + {Py_tp_new, reinterpret_cast(RawJSON_new)}, + {Py_tp_traverse, reinterpret_cast(heap_type_traverse)}, + {0, NULL} +}; + + +static PyType_Spec RawJSON_Type_Spec = { + "rapidjson.RawJSON", /* name */ + sizeof(RawJSON), /* basicsize */ + 0, /* itemsize */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, /* flags */ + RawJSON_Type_Slot /* slots */ }; @@ -2960,7 +2938,7 @@ dumps_internal( return false; writer->EndArray(); - } else if (PyObject_TypeCheck(object, &RawJSON_Type)) { + } else if (PyObject_TypeCheck(object, (PyTypeObject*) rawjson_type)) { const char* jsonStr; Py_ssize_t l; jsonStr = PyUnicode_AsUTF8AndSize(((RawJSON*) object)->value, &l); @@ -3830,8 +3808,10 @@ module_exec(PyObject* m) if (!validator_type) return -1; - if (PyType_Ready(&RawJSON_Type) < 0) + auto rawjson_type_local_ref = from_module_and_spec(*m, RawJSON_Type_Spec); + if (!rawjson_type_local_ref) return -1; + rawjson_type = rawjson_type_local_ref.get(); PyDateTime_IMPORT; if(!PyDateTimeAPI) @@ -4016,11 +3996,9 @@ module_exec(PyObject* m) return -1; validator_type.release(); - Py_INCREF(&RawJSON_Type); - if (PyModule_AddObject(m, "RawJSON", (PyObject*) &RawJSON_Type) < 0) { - Py_DECREF(&RawJSON_Type); + if (PyModule_AddObject(m, "RawJSON", rawjson_type_local_ref.get()) < 0) return -1; - } + rawjson_type_local_ref.release(); validation_error = PyErr_NewException("rapidjson.ValidationError", PyExc_ValueError, NULL); From aa0257265a271a05f7e235766fc8107a1368d656 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:34:01 +0200 Subject: [PATCH 07/20] Move PyObject caches into a single variable of struct type. This is a first step in the direction of subinterpreter compatibility. Is's recommended to keep such caches in the module state rather then process wide globals: https://docs.python.org/3/howto/isolating-extensions.html#managing-per-module-state Dedicated struct was created which can later on be stored in the module state. It can't be done right now since those members are accessed from static types initialization code which doesn't have access to the module. Turning such static types into heap types is also subinterpreter compatibility prerequisite: https://docs.python.org/3/howto/isolating-extensions.html#heap-types --- rapidjson.cpp | 298 +++++++++++++++++++++++++++----------------------- 1 file changed, 161 insertions(+), 137 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index a69b884..603a512 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -62,14 +62,19 @@ heap_type_traverse(PyObject *op, visitproc visit, void *arg) #define IS_INF(x) std::isinf(x) #endif +struct Errors { + PyObject* validation = NULL; + PyObject* decode = NULL; +}; + -static PyObject* decimal_type = NULL; -static PyObject* timezone_type = NULL; -static PyObject* timezone_utc = NULL; -static PyObject* uuid_type = NULL; -static PyObject* rawjson_type = NULL; -static PyObject* validation_error = NULL; -static PyObject* decode_error = NULL; +struct Types { + PyObject* decimal = NULL; + PyObject* timezone = NULL; + PyObject* uuid = NULL; + PyObject* rawjson = NULL; + Errors errors; +}; /* These are the names of often used methods or literal values, interned in the module @@ -79,25 +84,44 @@ static PyObject* decode_error = NULL; We cannot use _Py_IDENTIFIER() because that upsets the GNU C++ compiler in -pedantic mode. */ -static PyObject* astimezone_name = NULL; -static PyObject* hex_name = NULL; -static PyObject* timestamp_name = NULL; -static PyObject* total_seconds_name = NULL; -static PyObject* utcoffset_name = NULL; -static PyObject* is_infinite_name = NULL; -static PyObject* is_nan_name = NULL; -static PyObject* start_object_name = NULL; -static PyObject* end_object_name = NULL; -static PyObject* default_name = NULL; -static PyObject* end_array_name = NULL; -static PyObject* string_name = NULL; -static PyObject* read_name = NULL; -static PyObject* write_name = NULL; -static PyObject* encoding_name = NULL; - -static PyObject* minus_inf_string_value = NULL; -static PyObject* nan_string_value = NULL; -static PyObject* plus_inf_string_value = NULL; +struct Names { + PyObject* astimezone = NULL; + PyObject* hex = NULL; + PyObject* timestamp = NULL; + PyObject* total_seconds = NULL; + PyObject* utcoffset = NULL; + PyObject* is_infinite = NULL; + PyObject* is_nan = NULL; + PyObject* start_object = NULL; + PyObject* end_object = NULL; + PyObject* default_name = NULL; + PyObject* end_array = NULL; + PyObject* string = NULL; + PyObject* read = NULL; + PyObject* write = NULL; + PyObject* encoding = NULL; +}; + + +struct Strings { + PyObject* minus_inf = NULL; + PyObject* nan = NULL; + PyObject* plus_inf = NULL; +}; + + +struct Consts { + PyObject* timezone_utc = NULL; + Names names; + Strings strings; +}; + + +struct Cache { + Types types; + Consts consts; +}; +static Cache cache; struct HandlerContext { @@ -323,7 +347,7 @@ class PyReadStreamWrapper { void Read() { Py_CLEAR(chunk); - chunk = PyObject_CallMethodObjArgs(stream, read_name, chunkSize, NULL); + chunk = PyObject_CallMethodObjArgs(stream, cache.consts.names.read, chunkSize, NULL); if (chunk == NULL) { eof = true; @@ -373,7 +397,7 @@ class PyWriteStreamWrapper { bufferEnd = buffer + size; cursor = buffer; multiByteChar = NULL; - isBinary = !PyObject_HasAttr(stream, encoding_name); + isBinary = !PyObject_HasAttr(stream, cache.consts.names.encoding); } ~PyWriteStreamWrapper() { @@ -420,7 +444,7 @@ class PyWriteStreamWrapper { if (c == NULL) { // Propagate the error state, it will be caught by dumps_internal() } else { - PyObject* res = PyObject_CallMethodObjArgs(stream, write_name, c, NULL); + PyObject* res = PyObject_CallMethodObjArgs(stream, cache.consts.names.write, c, NULL); if (res == NULL) { // Likewise } else { @@ -834,17 +858,17 @@ struct PyHandler { stack.reserve(128); if (decoder != NULL) { assert(!objectHook); - if (PyObject_HasAttr(decoder, start_object_name)) { - decoderStartObject = PyObject_GetAttr(decoder, start_object_name); + if (PyObject_HasAttr(decoder, cache.consts.names.start_object)) { + decoderStartObject = PyObject_GetAttr(decoder, cache.consts.names.start_object); } - if (PyObject_HasAttr(decoder, end_object_name)) { - decoderEndObject = PyObject_GetAttr(decoder, end_object_name); + if (PyObject_HasAttr(decoder, cache.consts.names.end_object)) { + decoderEndObject = PyObject_GetAttr(decoder, cache.consts.names.end_object); } - if (PyObject_HasAttr(decoder, end_array_name)) { - decoderEndArray = PyObject_GetAttr(decoder, end_array_name); + if (PyObject_HasAttr(decoder, cache.consts.names.end_array)) { + decoderEndArray = PyObject_GetAttr(decoder, cache.consts.names.end_array); } - if (PyObject_HasAttr(decoder, string_name)) { - decoderString = PyObject_GetAttr(decoder, string_name); + if (PyObject_HasAttr(decoder, cache.consts.names.string)) { + decoderString = PyObject_GetAttr(decoder, cache.consts.names.string); } } sharedKeys = PyDict_New(); @@ -1206,9 +1230,9 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(decimal_type, nan_string_value, NULL); + value = PyObject_CallFunctionObjArgs(cache.types.decimal, cache.consts.strings.nan, NULL); } else { - value = PyFloat_FromString(nan_string_value); + value = PyFloat_FromString(cache.consts.strings.nan); } if (value == NULL) @@ -1226,14 +1250,14 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(decimal_type, + value = PyObject_CallFunctionObjArgs(cache.types.decimal, minus - ? minus_inf_string_value - : plus_inf_string_value, NULL); + ? cache.consts.strings.minus_inf + : cache.consts.strings.plus_inf, NULL); } else { value = PyFloat_FromString(minus - ? minus_inf_string_value - : plus_inf_string_value); + ? cache.consts.strings.minus_inf + : cache.consts.strings.plus_inf); } if (value == NULL) @@ -1300,7 +1324,7 @@ struct PyHandler { PyObject* pystr = PyUnicode_FromStringAndSize(str, length); if (pystr == NULL) return false; - value = PyObject_CallFunctionObjArgs(decimal_type, pystr, NULL); + value = PyObject_CallFunctionObjArgs(cache.types.decimal, pystr, NULL); Py_DECREF(pystr); } else { std::string zstr(str, length); @@ -1455,11 +1479,11 @@ struct PyHandler { if ((datetimeMode & DM_NAIVE_IS_UTC || isZ) && !hasOffset) { if (hasDate) { value = PyDateTimeAPI->DateTime_FromDateAndTime( - year, month, day, hours, mins, secs, usecs, timezone_utc, + year, month, day, hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->DateTimeType); } else { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->TimeType); } } else if (datetimeMode & DM_IGNORE_TZ || (!hasOffset && !isZ)) { if (hasDate) { @@ -1474,14 +1498,14 @@ struct PyHandler { value = NULL; } else if (!hasDate && datetimeMode & DM_SHIFT_TO_UTC) { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->TimeType); } else { PyObject* offset = PyDateTimeAPI->Delta_FromDelta(0, tzoff, 0, 1, PyDateTimeAPI->DeltaType); if (offset == NULL) { value = NULL; } else { - PyObject* tz = PyObject_CallFunctionObjArgs(timezone_type, offset, NULL); + PyObject* tz = PyObject_CallFunctionObjArgs(cache.types.timezone, offset, NULL); Py_DECREF(offset); if (tz == NULL) { value = NULL; @@ -1492,7 +1516,7 @@ struct PyHandler { PyDateTimeAPI->DateTimeType); if (value != NULL && datetimeMode & DM_SHIFT_TO_UTC) { PyObject* asUTC = PyObject_CallMethodObjArgs( - value, astimezone_name, timezone_utc, NULL); + value, cache.consts.names.astimezone, cache.consts.timezone_utc, NULL); Py_DECREF(value); if (asUTC == NULL) { value = NULL; @@ -1539,7 +1563,7 @@ struct PyHandler { if (pystr == NULL) return false; - PyObject* value = PyObject_CallFunctionObjArgs(uuid_type, pystr, NULL); + PyObject* value = PyObject_CallFunctionObjArgs(cache.types.uuid, pystr, NULL); Py_DECREF(pystr); if (value == NULL) @@ -1757,7 +1781,7 @@ load(PyObject* self, PyObject* args, PyObject* kwargs) &allowNan)) return NULL; - if (!PyObject_HasAttr(jsonObject, read_name)) { + if (!PyObject_HasAttr(jsonObject, cache.consts.names.read)) { PyErr_SetString(PyExc_TypeError, "Expected file-like object"); return NULL; } @@ -2067,7 +2091,7 @@ do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen, PyErr_Restore(etype, evalue, etraceback); } else - PyErr_Format(decode_error, "Parse error at offset %zu: %s", + PyErr_Format(cache.types.errors.decode, "Parse error at offset %zu: %s", offset, GetParseError_En(result.Code())); Py_XDECREF(handler.root); @@ -2134,7 +2158,7 @@ decoder_call(PyObject* self, PyObject* args, PyObject* kwargs) Py_DECREF(asUnicode); return NULL; } - } else if (PyObject_HasAttr(jsonObject, read_name)) { + } else if (PyObject_HasAttr(jsonObject, cache.consts.names.read)) { jsonStr = NULL; jsonStrLen = 0; } else { @@ -2327,14 +2351,14 @@ dumps_internal( } else if (PyBool_Check(object)) { writer->Bool(object == Py_True); } else if (numberMode & NM_DECIMAL - && (is_decimal = PyObject_IsInstance(object, decimal_type))) { + && (is_decimal = PyObject_IsInstance(object, cache.types.decimal))) { if (is_decimal == -1) { return false; } if (!(numberMode & NM_NAN)) { bool is_inf_or_nan; - PyObject* is_inf = PyObject_CallMethodObjArgs(object, is_infinite_name, + PyObject* is_inf = PyObject_CallMethodObjArgs(object, cache.consts.names.is_infinite, NULL); if (is_inf == NULL) { @@ -2344,7 +2368,7 @@ dumps_internal( Py_DECREF(is_inf); if (!is_inf_or_nan) { - PyObject* is_nan = PyObject_CallMethodObjArgs(object, is_nan_name, + PyObject* is_nan = PyObject_CallMethodObjArgs(object, cache.consts.names.is_nan, NULL); if (is_nan == NULL) { @@ -2631,9 +2655,9 @@ dumps_internal( char timeZone[TIMEZONE_LEN] = { 0 }; if (!(datetimeMode & DM_IGNORE_TZ) - && PyObject_HasAttr(object, utcoffset_name)) { + && PyObject_HasAttr(object, cache.consts.names.utcoffset)) { PyObject* utcOffset = PyObject_CallMethodObjArgs(object, - utcoffset_name, + cache.consts.names.utcoffset, NULL); if (utcOffset == NULL) @@ -2653,7 +2677,7 @@ dumps_internal( asUTC = PyDateTimeAPI->DateTime_FromDateAndTime( year, month, day, hour, min, sec, microsec, - timezone_utc, PyDateTimeAPI->DateTimeType); + cache.consts.timezone_utc, PyDateTimeAPI->DateTimeType); } else { hour = PyDateTime_TIME_GET_HOUR(dtObject); min = PyDateTime_TIME_GET_MINUTE(dtObject); @@ -2661,7 +2685,7 @@ dumps_internal( microsec = PyDateTime_TIME_GET_MICROSECOND(dtObject); asUTC = PyDateTimeAPI->Time_FromTime( hour, min, sec, microsec, - timezone_utc, PyDateTimeAPI->TimeType); + cache.consts.timezone_utc, PyDateTimeAPI->TimeType); } if (asUTC == NULL) { @@ -2679,8 +2703,8 @@ dumps_internal( if (datetimeMode & DM_SHIFT_TO_UTC) { // If it's not already in UTC, shift the value if (PyObject_IsTrue(utcOffset)) { - asUTC = PyObject_CallMethodObjArgs(object, astimezone_name, - timezone_utc, NULL); + asUTC = PyObject_CallMethodObjArgs(object, cache.consts.names.astimezone, + cache.consts.timezone_utc, NULL); if (asUTC == NULL) { Py_DECREF(utcOffset); @@ -2697,7 +2721,7 @@ dumps_internal( if (PyObject_IsTrue(utcOffset)) { PyObject* tsObj = PyObject_CallMethodObjArgs(utcOffset, - total_seconds_name, + cache.consts.names.total_seconds, NULL); if (tsObj == NULL) { @@ -2777,7 +2801,7 @@ dumps_internal( } else /* if (datetimeMode & DM_UNIX_TIME) */ { if (PyDateTime_Check(dtObject)) { PyObject* timestampObj = PyObject_CallMethodObjArgs(dtObject, - timestamp_name, + cache.consts.names.timestamp, NULL); if (timestampObj == NULL) { @@ -2842,7 +2866,7 @@ dumps_internal( if (datetimeMode & (DM_SHIFT_TO_UTC | DM_NAIVE_IS_UTC)) midnightObj = PyDateTimeAPI->DateTime_FromDateAndTime( year, month, day, 0, 0, 0, 0, - timezone_utc, PyDateTimeAPI->DateTimeType); + cache.consts.timezone_utc, PyDateTimeAPI->DateTimeType); else midnightObj = PyDateTime_FromDateAndTime(year, month, day, 0, 0, 0, 0); @@ -2851,7 +2875,7 @@ dumps_internal( return false; } - timestampObj = PyObject_CallMethodObjArgs(midnightObj, timestamp_name, + timestampObj = PyObject_CallMethodObjArgs(midnightObj, cache.consts.names.timestamp, NULL); Py_DECREF(midnightObj); @@ -2880,12 +2904,12 @@ dumps_internal( } } } else if (uuidMode != UM_NONE - && PyObject_TypeCheck(object, (PyTypeObject*) uuid_type)) { + && PyObject_TypeCheck(object, (PyTypeObject*) cache.types.uuid)) { PyObject* hexval; if (uuidMode == UM_CANONICAL) hexval = PyObject_Str(object); else - hexval = PyObject_GetAttr(object, hex_name); + hexval = PyObject_GetAttr(object, cache.consts.names.hex); if (hexval == NULL) return false; @@ -2938,7 +2962,7 @@ dumps_internal( return false; writer->EndArray(); - } else if (PyObject_TypeCheck(object, (PyTypeObject*) rawjson_type)) { + } else if (PyObject_TypeCheck(object, (PyTypeObject*) cache.types.rawjson)) { const char* jsonStr; Py_ssize_t l; jsonStr = PyUnicode_AsUTF8AndSize(((RawJSON*) object)->value, &l); @@ -3453,7 +3477,7 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) EncoderObject* e = (EncoderObject*) self; if (stream != NULL && stream != Py_None) { - if (!PyObject_HasAttr(stream, write_name)) { + if (!PyObject_HasAttr(stream, cache.consts.names.write)) { PyErr_SetString(PyExc_TypeError, "Expected a writable stream"); return NULL; } @@ -3461,8 +3485,8 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) if (!accept_chunk_size_arg(chunkSizeObj, chunkSize)) return NULL; - if (PyObject_HasAttr(self, default_name)) { - defaultFn = PyObject_GetAttr(self, default_name); + if (PyObject_HasAttr(self, cache.consts.names.default_name)) { + defaultFn = PyObject_GetAttr(self, cache.consts.names.default_name); } result = do_stream_encode(value, stream, chunkSize, defaultFn, e->ensureAscii, @@ -3470,8 +3494,8 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) e->numberMode, e->datetimeMode, e->uuidMode, e->bytesMode, e->iterableMode, e->mappingMode); } else { - if (PyObject_HasAttr(self, default_name)) { - defaultFn = PyObject_GetAttr(self, default_name); + if (PyObject_HasAttr(self, cache.consts.names.default_name)) { + defaultFn = PyObject_GetAttr(self, cache.consts.names.default_name); } result = do_encode(value, defaultFn, e->ensureAscii, e->writeMode, e->indentChar, @@ -3667,7 +3691,7 @@ static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs if (error) { if (asUnicode != NULL) Py_DECREF(asUnicode); - PyErr_SetString(decode_error, "Invalid JSON"); + PyErr_SetString(cache.types.errors.decode, "Invalid JSON"); return NULL; } @@ -3692,7 +3716,7 @@ static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs PyObject* error = Py_BuildValue("sss", validator.GetInvalidSchemaKeyword(), sptr.GetString(), dptr.GetString()); - PyErr_SetObject(validation_error, error); + PyErr_SetObject(cache.types.errors.validation, error); if (error != NULL) Py_DECREF(error); @@ -3757,7 +3781,7 @@ static PyObject* validator_new(PyTypeObject* type, PyObject* args, PyObject* kwa Py_DECREF(asUnicode); if (error) { - PyErr_SetString(decode_error, "Invalid JSON"); + PyErr_SetString(cache.types.errors.decode, "Invalid JSON"); return NULL; } @@ -3808,10 +3832,10 @@ module_exec(PyObject* m) if (!validator_type) return -1; - auto rawjson_type_local_ref = from_module_and_spec(*m, RawJSON_Type_Spec); - if (!rawjson_type_local_ref) + auto rawjson_type = from_module_and_spec(*m, RawJSON_Type_Spec); + if (!rawjson_type) return -1; - rawjson_type = rawjson_type_local_ref.get(); + cache.types.rawjson = rawjson_type.get(); PyDateTime_IMPORT; if(!PyDateTimeAPI) @@ -3825,102 +3849,102 @@ module_exec(PyObject* m) if (decimalModule == NULL) return -1; - decimal_type = PyObject_GetAttrString(decimalModule, "Decimal"); + cache.types.decimal = PyObject_GetAttrString(decimalModule, "Decimal"); Py_DECREF(decimalModule); - if (decimal_type == NULL) + if (cache.types.decimal == NULL) return -1; - timezone_type = PyObject_GetAttrString(datetimeModule, "timezone"); + cache.types.timezone = PyObject_GetAttrString(datetimeModule, "timezone"); Py_DECREF(datetimeModule); - if (timezone_type == NULL) + if (cache.types.timezone == NULL) return -1; - timezone_utc = PyObject_GetAttrString(timezone_type, "utc"); - if (timezone_utc == NULL) + cache.consts.timezone_utc = PyObject_GetAttrString(cache.types.timezone, "utc"); + if (cache.consts.timezone_utc == NULL) return -1; uuidModule = PyImport_ImportModule("uuid"); if (uuidModule == NULL) return -1; - uuid_type = PyObject_GetAttrString(uuidModule, "UUID"); + cache.types.uuid = PyObject_GetAttrString(uuidModule, "UUID"); Py_DECREF(uuidModule); - if (uuid_type == NULL) + if (cache.types.uuid == NULL) return -1; - astimezone_name = PyUnicode_InternFromString("astimezone"); - if (astimezone_name == NULL) + cache.consts.names.astimezone = PyUnicode_InternFromString("astimezone"); + if (cache.consts.names.astimezone == NULL) return -1; - hex_name = PyUnicode_InternFromString("hex"); - if (hex_name == NULL) + cache.consts.names.hex = PyUnicode_InternFromString("hex"); + if (cache.consts.names.hex == NULL) return -1; - timestamp_name = PyUnicode_InternFromString("timestamp"); - if (timestamp_name == NULL) + cache.consts.names.timestamp = PyUnicode_InternFromString("timestamp"); + if (cache.consts.names.timestamp == NULL) return -1; - total_seconds_name = PyUnicode_InternFromString("total_seconds"); - if (total_seconds_name == NULL) + cache.consts.names.total_seconds = PyUnicode_InternFromString("total_seconds"); + if (cache.consts.names.total_seconds == NULL) return -1; - utcoffset_name = PyUnicode_InternFromString("utcoffset"); - if (utcoffset_name == NULL) + cache.consts.names.utcoffset = PyUnicode_InternFromString("utcoffset"); + if (cache.consts.names.utcoffset == NULL) return -1; - is_infinite_name = PyUnicode_InternFromString("is_infinite"); - if (is_infinite_name == NULL) + cache.consts.names.is_infinite = PyUnicode_InternFromString("is_infinite"); + if (cache.consts.names.is_infinite == NULL) return -1; - is_nan_name = PyUnicode_InternFromString("is_nan"); - if (is_nan_name == NULL) + cache.consts.names.is_nan = PyUnicode_InternFromString("is_nan"); + if (cache.consts.names.is_nan == NULL) return -1; - minus_inf_string_value = PyUnicode_InternFromString("-Infinity"); - if (minus_inf_string_value == NULL) + cache.consts.strings.minus_inf = PyUnicode_InternFromString("-Infinity"); + if (cache.consts.strings.minus_inf == NULL) return -1; - nan_string_value = PyUnicode_InternFromString("nan"); - if (nan_string_value == NULL) + cache.consts.strings.nan = PyUnicode_InternFromString("nan"); + if (cache.consts.strings.nan == NULL) return -1; - plus_inf_string_value = PyUnicode_InternFromString("+Infinity"); - if (plus_inf_string_value == NULL) + cache.consts.strings.plus_inf = PyUnicode_InternFromString("+Infinity"); + if (cache.consts.strings.plus_inf == NULL) return -1; - start_object_name = PyUnicode_InternFromString("start_object"); - if (start_object_name == NULL) + cache.consts.names.start_object = PyUnicode_InternFromString("start_object"); + if (cache.consts.names.start_object == NULL) return -1; - end_object_name = PyUnicode_InternFromString("end_object"); - if (end_object_name == NULL) + cache.consts.names.end_object = PyUnicode_InternFromString("end_object"); + if (cache.consts.names.end_object == NULL) return -1; - default_name = PyUnicode_InternFromString("default"); - if (default_name == NULL) + cache.consts.names.default_name = PyUnicode_InternFromString("default"); + if (cache.consts.names.default_name == NULL) return -1; - end_array_name = PyUnicode_InternFromString("end_array"); - if (end_array_name == NULL) + cache.consts.names.end_array = PyUnicode_InternFromString("end_array"); + if (cache.consts.names.end_array == NULL) return -1; - string_name = PyUnicode_InternFromString("string"); - if (string_name == NULL) + cache.consts.names.string = PyUnicode_InternFromString("string"); + if (cache.consts.names.string == NULL) return -1; - read_name = PyUnicode_InternFromString("read"); - if (read_name == NULL) + cache.consts.names.read = PyUnicode_InternFromString("read"); + if (cache.consts.names.read == NULL) return -1; - write_name = PyUnicode_InternFromString("write"); - if (write_name == NULL) + cache.consts.names.write = PyUnicode_InternFromString("write"); + if (cache.consts.names.write == NULL) return -1; - encoding_name = PyUnicode_InternFromString("encoding"); - if (encoding_name == NULL) + cache.consts.names.encoding = PyUnicode_InternFromString("encoding"); + if (cache.consts.names.encoding == NULL) return -1; #define STRINGIFY(x) XSTRINGIFY(x) @@ -3996,27 +4020,27 @@ module_exec(PyObject* m) return -1; validator_type.release(); - if (PyModule_AddObject(m, "RawJSON", rawjson_type_local_ref.get()) < 0) + if (PyModule_AddObject(m, "RawJSON", rawjson_type.get()) < 0) return -1; - rawjson_type_local_ref.release(); + rawjson_type.release(); - validation_error = PyErr_NewException("rapidjson.ValidationError", + cache.types.errors.validation = PyErr_NewException("rapidjson.ValidationError", PyExc_ValueError, NULL); - if (validation_error == NULL) + if (cache.types.errors.validation == NULL) return -1; - Py_INCREF(validation_error); - if (PyModule_AddObject(m, "ValidationError", validation_error) < 0) { - Py_DECREF(validation_error); + Py_INCREF(cache.types.errors.validation); + if (PyModule_AddObject(m, "ValidationError", cache.types.errors.validation) < 0) { + Py_DECREF(cache.types.errors.validation); return -1; } - decode_error = PyErr_NewException("rapidjson.JSONDecodeError", + cache.types.errors.decode = PyErr_NewException("rapidjson.JSONDecodeError", PyExc_ValueError, NULL); - if (decode_error == NULL) + if (cache.types.errors.decode == NULL) return -1; - Py_INCREF(decode_error); - if (PyModule_AddObject(m, "JSONDecodeError", decode_error) < 0) { - Py_DECREF(decode_error); + Py_INCREF(cache.types.errors.decode); + if (PyModule_AddObject(m, "JSONDecodeError", cache.types.errors.decode) < 0) { + Py_DECREF(cache.types.errors.decode); return -1; } From 86e3fceb85ac2d176ec2db24e9ba275a12dbf58c Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:39:42 +0200 Subject: [PATCH 08/20] Reduce usage of cache global var by passing required parts as args --- rapidjson.cpp | 116 +++++++++++++++++++++++++++----------------------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 603a512..0ba21c3 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -245,7 +245,8 @@ enum MappingMode { ////////////////////////// -static PyObject* do_decode(PyObject* decoder, +static PyObject* do_decode(const Consts& consts, const Types& types, + PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrlen, PyObject* jsonStream, size_t chunkSize, PyObject* objectHook, @@ -260,7 +261,7 @@ static PyObject* do_encode(PyObject* value, PyObject* defaultFn, bool ensureAsci unsigned numberMode, unsigned datetimeMode, unsigned uuidMode, unsigned bytesMode, unsigned iterableMode, unsigned mappingMode); -static PyObject* do_stream_encode(PyObject* value, PyObject* stream, size_t chunkSize, +static PyObject* do_stream_encode(const Names& names, PyObject* value, PyObject* stream, size_t chunkSize, PyObject* defaultFn, bool ensureAscii, unsigned writeMode, char indentChar, unsigned indentCount, unsigned numberMode, @@ -285,8 +286,8 @@ class PyReadStreamWrapper { public: typedef char Ch; - PyReadStreamWrapper(PyObject* stream, size_t size) - : stream(stream) { + PyReadStreamWrapper(const Names& names, PyObject* stream, size_t size) + : names(&names), stream(stream) { Py_INCREF(stream); chunkSize = PyLong_FromUnsignedLong(size); buffer = NULL; @@ -347,7 +348,7 @@ class PyReadStreamWrapper { void Read() { Py_CLEAR(chunk); - chunk = PyObject_CallMethodObjArgs(stream, cache.consts.names.read, chunkSize, NULL); + chunk = PyObject_CallMethodObjArgs(stream, names->read, chunkSize, NULL); if (chunk == NULL) { eof = true; @@ -374,6 +375,7 @@ class PyReadStreamWrapper { } } + const Names* names; PyObject* stream; PyObject* chunkSize; PyObject* chunk; @@ -389,15 +391,15 @@ class PyWriteStreamWrapper { public: typedef char Ch; - PyWriteStreamWrapper(PyObject* stream, size_t size) - : stream(stream) { + PyWriteStreamWrapper(const Names& names, PyObject* stream, size_t size) + : names(&names), stream(stream) { Py_INCREF(stream); buffer = (char*) PyMem_Malloc(size); assert(buffer); bufferEnd = buffer + size; cursor = buffer; multiByteChar = NULL; - isBinary = !PyObject_HasAttr(stream, cache.consts.names.encoding); + isBinary = !PyObject_HasAttr(stream, names.encoding); } ~PyWriteStreamWrapper() { @@ -444,7 +446,7 @@ class PyWriteStreamWrapper { if (c == NULL) { // Propagate the error state, it will be caught by dumps_internal() } else { - PyObject* res = PyObject_CallMethodObjArgs(stream, cache.consts.names.write, c, NULL); + PyObject* res = PyObject_CallMethodObjArgs(stream, names->write, c, NULL); if (res == NULL) { // Likewise } else { @@ -478,6 +480,7 @@ class PyWriteStreamWrapper { } private: + const Names* names; PyObject* stream; Ch* buffer; Ch* bufferEnd; @@ -827,6 +830,8 @@ float_from_string(const char* s, Py_ssize_t len) struct PyHandler { + const Consts* consts; + const Types* types; PyObject* decoderStartObject; PyObject* decoderEndObject; PyObject* decoderEndArray; @@ -840,12 +845,16 @@ struct PyHandler { unsigned recursionLimit; std::vector stack; - PyHandler(PyObject* decoder, + PyHandler(const Consts& consts, + const Types& types, + PyObject* decoder, PyObject* hook, unsigned dm, unsigned um, unsigned nm) - : decoderStartObject(NULL), + : consts(&consts), + types(&types), + decoderStartObject(NULL), decoderEndObject(NULL), decoderEndArray(NULL), decoderString(NULL), @@ -858,17 +867,17 @@ struct PyHandler { stack.reserve(128); if (decoder != NULL) { assert(!objectHook); - if (PyObject_HasAttr(decoder, cache.consts.names.start_object)) { - decoderStartObject = PyObject_GetAttr(decoder, cache.consts.names.start_object); + if (PyObject_HasAttr(decoder, consts.names.start_object)) { + decoderStartObject = PyObject_GetAttr(decoder, consts.names.start_object); } - if (PyObject_HasAttr(decoder, cache.consts.names.end_object)) { - decoderEndObject = PyObject_GetAttr(decoder, cache.consts.names.end_object); + if (PyObject_HasAttr(decoder, consts.names.end_object)) { + decoderEndObject = PyObject_GetAttr(decoder, consts.names.end_object); } - if (PyObject_HasAttr(decoder, cache.consts.names.end_array)) { - decoderEndArray = PyObject_GetAttr(decoder, cache.consts.names.end_array); + if (PyObject_HasAttr(decoder, consts.names.end_array)) { + decoderEndArray = PyObject_GetAttr(decoder, consts.names.end_array); } - if (PyObject_HasAttr(decoder, cache.consts.names.string)) { - decoderString = PyObject_GetAttr(decoder, cache.consts.names.string); + if (PyObject_HasAttr(decoder, consts.names.string)) { + decoderString = PyObject_GetAttr(decoder, consts.names.string); } } sharedKeys = PyDict_New(); @@ -1230,9 +1239,9 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(cache.types.decimal, cache.consts.strings.nan, NULL); + value = PyObject_CallFunctionObjArgs(types->decimal, consts->strings.nan, NULL); } else { - value = PyFloat_FromString(cache.consts.strings.nan); + value = PyFloat_FromString(consts->strings.nan); } if (value == NULL) @@ -1250,14 +1259,14 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(cache.types.decimal, + value = PyObject_CallFunctionObjArgs(types->decimal, minus - ? cache.consts.strings.minus_inf - : cache.consts.strings.plus_inf, NULL); + ? consts->strings.minus_inf + : consts->strings.plus_inf, NULL); } else { value = PyFloat_FromString(minus - ? cache.consts.strings.minus_inf - : cache.consts.strings.plus_inf); + ? consts->strings.minus_inf + : consts->strings.plus_inf); } if (value == NULL) @@ -1324,7 +1333,7 @@ struct PyHandler { PyObject* pystr = PyUnicode_FromStringAndSize(str, length); if (pystr == NULL) return false; - value = PyObject_CallFunctionObjArgs(cache.types.decimal, pystr, NULL); + value = PyObject_CallFunctionObjArgs(types->decimal, pystr, NULL); Py_DECREF(pystr); } else { std::string zstr(str, length); @@ -1479,11 +1488,11 @@ struct PyHandler { if ((datetimeMode & DM_NAIVE_IS_UTC || isZ) && !hasOffset) { if (hasDate) { value = PyDateTimeAPI->DateTime_FromDateAndTime( - year, month, day, hours, mins, secs, usecs, cache.consts.timezone_utc, + year, month, day, hours, mins, secs, usecs, consts->timezone_utc, PyDateTimeAPI->DateTimeType); } else { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, consts->timezone_utc, PyDateTimeAPI->TimeType); } } else if (datetimeMode & DM_IGNORE_TZ || (!hasOffset && !isZ)) { if (hasDate) { @@ -1498,14 +1507,14 @@ struct PyHandler { value = NULL; } else if (!hasDate && datetimeMode & DM_SHIFT_TO_UTC) { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, cache.consts.timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, consts->timezone_utc, PyDateTimeAPI->TimeType); } else { PyObject* offset = PyDateTimeAPI->Delta_FromDelta(0, tzoff, 0, 1, PyDateTimeAPI->DeltaType); if (offset == NULL) { value = NULL; } else { - PyObject* tz = PyObject_CallFunctionObjArgs(cache.types.timezone, offset, NULL); + PyObject* tz = PyObject_CallFunctionObjArgs(types->timezone, offset, NULL); Py_DECREF(offset); if (tz == NULL) { value = NULL; @@ -1516,7 +1525,7 @@ struct PyHandler { PyDateTimeAPI->DateTimeType); if (value != NULL && datetimeMode & DM_SHIFT_TO_UTC) { PyObject* asUTC = PyObject_CallMethodObjArgs( - value, cache.consts.names.astimezone, cache.consts.timezone_utc, NULL); + value, consts->names.astimezone, consts->timezone_utc, NULL); Py_DECREF(value); if (asUTC == NULL) { value = NULL; @@ -1563,7 +1572,7 @@ struct PyHandler { if (pystr == NULL) return false; - PyObject* value = PyObject_CallFunctionObjArgs(cache.types.uuid, pystr, NULL); + PyObject* value = PyObject_CallFunctionObjArgs(types->uuid, pystr, NULL); Py_DECREF(pystr); if (value == NULL) @@ -1719,7 +1728,7 @@ loads(PyObject* self, PyObject* args, PyObject* kwargs) return NULL; } - PyObject* result = do_decode(NULL, jsonStr, jsonStrLen, NULL, 0, objectHook, + PyObject* result = do_decode(cache.consts, cache.types, NULL, jsonStr, jsonStrLen, NULL, 0, objectHook, numberMode, datetimeMode, uuidMode, parseMode); if (asUnicode != NULL) @@ -1877,7 +1886,7 @@ load(PyObject* self, PyObject* args, PyObject* kwargs) } } - return do_decode(NULL, NULL, 0, jsonObject, chunkSize, objectHook, + return do_decode(cache.consts, cache.types, NULL, NULL, 0, jsonObject, chunkSize, objectHook, numberMode, datetimeMode, uuidMode, parseMode); } @@ -2033,12 +2042,13 @@ static PyType_Spec Decoder_Type_Spec = { static PyObject* -do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen, - PyObject* jsonStream, size_t chunkSize, PyObject* objectHook, - unsigned numberMode, unsigned datetimeMode, unsigned uuidMode, - unsigned parseMode) +do_decode(const Consts& consts, const Types& types, PyObject* decoder, + const char* jsonStr, Py_ssize_t jsonStrLen, PyObject* jsonStream, + size_t chunkSize, PyObject* objectHook, unsigned numberMode, + unsigned datetimeMode, unsigned uuidMode, unsigned parseMode) { - PyHandler handler(decoder, objectHook, datetimeMode, uuidMode, numberMode); + PyHandler handler(consts, types, decoder, objectHook, + datetimeMode, uuidMode, numberMode); Reader reader; ParseResult result; @@ -2060,7 +2070,7 @@ do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen, PyMem_Free(jsonStrCopy); } else { - PyReadStreamWrapper sw(jsonStream, chunkSize); + PyReadStreamWrapper sw(consts.names, jsonStream, chunkSize); DECODE(reader, kParseNoFlags, sw, handler); @@ -2091,7 +2101,7 @@ do_decode(PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrLen, PyErr_Restore(etype, evalue, etraceback); } else - PyErr_Format(cache.types.errors.decode, "Parse error at offset %zu: %s", + PyErr_Format(types.errors.decode, "Parse error at offset %zu: %s", offset, GetParseError_En(result.Code())); Py_XDECREF(handler.root); @@ -2170,9 +2180,9 @@ decoder_call(PyObject* self, PyObject* args, PyObject* kwargs) DecoderObject* d = (DecoderObject*) self; - PyObject* result = do_decode(self, jsonStr, jsonStrLen, jsonObject, chunkSize, NULL, - d->numberMode, d->datetimeMode, d->uuidMode, - d->parseMode); + PyObject* result = do_decode(cache.consts, cache.types, self, jsonStr, jsonStrLen, + jsonObject, chunkSize, NULL, d->numberMode, d->datetimeMode, + d->uuidMode, d->parseMode); if (asUnicode != NULL) Py_DECREF(asUnicode); @@ -3259,7 +3269,7 @@ dump(PyObject* self, PyObject* args, PyObject* kwargs) if (sortKeys) mappingMode |= MM_SORT_KEYS; - return do_stream_encode(value, stream, chunkSize, defaultFn, + return do_stream_encode(cache.consts.names, value, stream, chunkSize, defaultFn, ensureAscii ? true : false, writeMode, indentChar, indentCount, numberMode, datetimeMode, uuidMode, bytesMode, iterableMode, mappingMode); @@ -3417,13 +3427,13 @@ do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, unsigned write static PyObject* -do_stream_encode(PyObject* value, PyObject* stream, size_t chunkSize, PyObject* defaultFn, - bool ensureAscii, unsigned writeMode, char indentChar, - unsigned indentCount, unsigned numberMode, unsigned datetimeMode, - unsigned uuidMode, unsigned bytesMode, unsigned iterableMode, - unsigned mappingMode) +do_stream_encode(const Names& names, PyObject* value, PyObject* stream, size_t chunkSize, + PyObject* defaultFn, bool ensureAscii, unsigned writeMode, + char indentChar, unsigned indentCount, unsigned numberMode, + unsigned datetimeMode, unsigned uuidMode, unsigned bytesMode, + unsigned iterableMode, unsigned mappingMode) { - PyWriteStreamWrapper os(stream, chunkSize); + PyWriteStreamWrapper os(names, stream, chunkSize); if (writeMode == WM_COMPACT) { if (ensureAscii) { @@ -3489,7 +3499,7 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) defaultFn = PyObject_GetAttr(self, cache.consts.names.default_name); } - result = do_stream_encode(value, stream, chunkSize, defaultFn, e->ensureAscii, + result = do_stream_encode(cache.consts.names, value, stream, chunkSize, defaultFn, e->ensureAscii, e->writeMode, e->indentChar, e->indentCount, e->numberMode, e->datetimeMode, e->uuidMode, e->bytesMode, e->iterableMode, e->mappingMode); From 2d6d650ca2b5b584a2dad3818000172d43e86995 Mon Sep 17 00:00:00 2001 From: Sergey Vidyuk Date: Sun, 30 Aug 2026 10:41:47 +0200 Subject: [PATCH 09/20] Enable subinterpreters on Python >= 3.12 --- rapidjson.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rapidjson.cpp b/rapidjson.cpp index 0ba21c3..95bdc74 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -4060,6 +4060,9 @@ module_exec(PyObject* m) static struct PyModuleDef_Slot slots[] = { {Py_mod_exec, (void*) module_exec}, +#if PY_VERSION_HEX >= 0x030C0000 + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, +#endif {0, NULL} }; From 5475457f2eb34f56025e5e4d0ef5951e36167236 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Thu, 3 Sep 2026 07:30:30 +0200 Subject: [PATCH 10/20] Attempt at replacing static data with module state --- rapidjson.cpp | 530 +++++++++++++++++++++++++++++++------------------- 1 file changed, 326 insertions(+), 204 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 95bdc74..c1bfe70 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -62,6 +62,7 @@ heap_type_traverse(PyObject *op, visitproc visit, void *arg) #define IS_INF(x) std::isinf(x) #endif + struct Errors { PyObject* validation = NULL; PyObject* decode = NULL; @@ -117,11 +118,100 @@ struct Consts { }; -struct Cache { +typedef struct { Types types; Consts consts; -}; -static Cache cache; +} rj_state; + + +static inline rj_state* +get_rj_state(PyObject* module) +{ + void* state = PyModule_GetState(module); + assert(state != NULL); + return (rj_state*) state; +} + + +static int +rj_clear(PyObject* self) +{ + rj_state* state = get_rj_state(self); + + Py_CLEAR(state->types.decimal); + Py_CLEAR(state->types.timezone); + Py_CLEAR(state->types.uuid); + Py_CLEAR(state->types.rawjson); + Py_CLEAR(state->types.errors.validation); + Py_CLEAR(state->types.errors.decode); + + Py_CLEAR(state->consts.timezone_utc); + Py_CLEAR(state->consts.names.astimezone); + Py_CLEAR(state->consts.names.hex); + Py_CLEAR(state->consts.names.timestamp); + Py_CLEAR(state->consts.names.total_seconds); + Py_CLEAR(state->consts.names.utcoffset); + Py_CLEAR(state->consts.names.is_infinite); + Py_CLEAR(state->consts.names.is_nan); + Py_CLEAR(state->consts.names.start_object); + Py_CLEAR(state->consts.names.end_object); + Py_CLEAR(state->consts.names.default_name); + Py_CLEAR(state->consts.names.end_array); + Py_CLEAR(state->consts.names.string); + Py_CLEAR(state->consts.names.read); + Py_CLEAR(state->consts.names.write); + Py_CLEAR(state->consts.names.encoding); + + Py_CLEAR(state->consts.strings.minus_inf); + Py_CLEAR(state->consts.strings.nan); + Py_CLEAR(state->consts.strings.plus_inf); + + return 0; +} + + +static int +rj_traverse(PyObject* self, visitproc visit, void* arg) +{ + rj_state* state = get_rj_state(self); + + Py_VISIT(state->types.decimal); + Py_VISIT(state->types.timezone); + Py_VISIT(state->types.uuid); + Py_VISIT(state->types.rawjson); + Py_VISIT(state->types.errors.validation); + Py_VISIT(state->types.errors.decode); + + Py_VISIT(state->consts.timezone_utc); + Py_VISIT(state->consts.names.astimezone); + Py_VISIT(state->consts.names.hex); + Py_VISIT(state->consts.names.timestamp); + Py_VISIT(state->consts.names.total_seconds); + Py_VISIT(state->consts.names.utcoffset); + Py_VISIT(state->consts.names.is_infinite); + Py_VISIT(state->consts.names.is_nan); + Py_VISIT(state->consts.names.start_object); + Py_VISIT(state->consts.names.end_object); + Py_VISIT(state->consts.names.default_name); + Py_VISIT(state->consts.names.end_array); + Py_VISIT(state->consts.names.string); + Py_VISIT(state->consts.names.read); + Py_VISIT(state->consts.names.write); + Py_VISIT(state->consts.names.encoding); + + Py_VISIT(state->consts.strings.minus_inf); + Py_VISIT(state->consts.strings.nan); + Py_VISIT(state->consts.strings.plus_inf); + + return 0; +} + + +static void +rj_free(void* self) +{ + (void) rj_clear((PyObject*) self); +} struct HandlerContext { @@ -245,8 +335,7 @@ enum MappingMode { ////////////////////////// -static PyObject* do_decode(const Consts& consts, const Types& types, - PyObject* decoder, +static PyObject* do_decode(const rj_state* state, PyObject* decoder, const char* jsonStr, Py_ssize_t jsonStrlen, PyObject* jsonStream, size_t chunkSize, PyObject* objectHook, @@ -256,12 +345,12 @@ static PyObject* decoder_call(PyObject* self, PyObject* args, PyObject* kwargs); static PyObject* decoder_new(PyTypeObject* type, PyObject* args, PyObject* kwargs); -static PyObject* do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, +static PyObject* do_encode(const rj_state* state, PyObject* value, PyObject* defaultFn, bool ensureAscii, unsigned writeMode, char indentChar, unsigned indentCount, unsigned numberMode, unsigned datetimeMode, unsigned uuidMode, unsigned bytesMode, unsigned iterableMode, unsigned mappingMode); -static PyObject* do_stream_encode(const Names& names, PyObject* value, PyObject* stream, size_t chunkSize, +static PyObject* do_stream_encode(const rj_state* state, PyObject* value, PyObject* stream, size_t chunkSize, PyObject* defaultFn, bool ensureAscii, unsigned writeMode, char indentChar, unsigned indentCount, unsigned numberMode, @@ -388,18 +477,19 @@ class PyReadStreamWrapper { class PyWriteStreamWrapper { + const rj_state* state; public: typedef char Ch; - PyWriteStreamWrapper(const Names& names, PyObject* stream, size_t size) - : names(&names), stream(stream) { + PyWriteStreamWrapper(const rj_state* state, PyObject* stream, size_t size) + : state(state), stream(stream) { Py_INCREF(stream); buffer = (char*) PyMem_Malloc(size); assert(buffer); bufferEnd = buffer + size; cursor = buffer; multiByteChar = NULL; - isBinary = !PyObject_HasAttr(stream, names.encoding); + isBinary = !PyObject_HasAttr(stream, state->consts.names.encoding); } ~PyWriteStreamWrapper() { @@ -446,7 +536,8 @@ class PyWriteStreamWrapper { if (c == NULL) { // Propagate the error state, it will be caught by dumps_internal() } else { - PyObject* res = PyObject_CallMethodObjArgs(stream, names->write, c, NULL); + PyObject* res = PyObject_CallMethodObjArgs(stream, state->consts.names.write, + c, NULL); if (res == NULL) { // Likewise } else { @@ -830,8 +921,7 @@ float_from_string(const char* s, Py_ssize_t len) struct PyHandler { - const Consts* consts; - const Types* types; + const rj_state* state; PyObject* decoderStartObject; PyObject* decoderEndObject; PyObject* decoderEndArray; @@ -845,15 +935,13 @@ struct PyHandler { unsigned recursionLimit; std::vector stack; - PyHandler(const Consts& consts, - const Types& types, + PyHandler(const rj_state* state, PyObject* decoder, PyObject* hook, unsigned dm, unsigned um, unsigned nm) - : consts(&consts), - types(&types), + : state(state), decoderStartObject(NULL), decoderEndObject(NULL), decoderEndArray(NULL), @@ -867,17 +955,21 @@ struct PyHandler { stack.reserve(128); if (decoder != NULL) { assert(!objectHook); - if (PyObject_HasAttr(decoder, consts.names.start_object)) { - decoderStartObject = PyObject_GetAttr(decoder, consts.names.start_object); + if (PyObject_HasAttr(decoder, state->consts.names.start_object)) { + decoderStartObject = PyObject_GetAttr(decoder, + state->consts.names.start_object); } - if (PyObject_HasAttr(decoder, consts.names.end_object)) { - decoderEndObject = PyObject_GetAttr(decoder, consts.names.end_object); + if (PyObject_HasAttr(decoder, state->consts.names.end_object)) { + decoderEndObject = PyObject_GetAttr(decoder, + state->consts.names.end_object); } - if (PyObject_HasAttr(decoder, consts.names.end_array)) { - decoderEndArray = PyObject_GetAttr(decoder, consts.names.end_array); + if (PyObject_HasAttr(decoder, state->consts.names.end_array)) { + decoderEndArray = PyObject_GetAttr(decoder, + state->consts.names.end_array); } - if (PyObject_HasAttr(decoder, consts.names.string)) { - decoderString = PyObject_GetAttr(decoder, consts.names.string); + if (PyObject_HasAttr(decoder, state->consts.names.string)) { + decoderString = PyObject_GetAttr(decoder, + state->consts.names.string); } } sharedKeys = PyDict_New(); @@ -1239,9 +1331,10 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(types->decimal, consts->strings.nan, NULL); + value = PyObject_CallFunctionObjArgs(state->types.decimal, + state->consts.strings.nan, NULL); } else { - value = PyFloat_FromString(consts->strings.nan); + value = PyFloat_FromString(state->consts.strings.nan); } if (value == NULL) @@ -1259,14 +1352,14 @@ struct PyHandler { PyObject* value; if (numberMode & NM_DECIMAL) { - value = PyObject_CallFunctionObjArgs(types->decimal, + value = PyObject_CallFunctionObjArgs(state->types.decimal, minus - ? consts->strings.minus_inf - : consts->strings.plus_inf, NULL); + ? state->consts.strings.minus_inf + : state->consts.strings.plus_inf, NULL); } else { value = PyFloat_FromString(minus - ? consts->strings.minus_inf - : consts->strings.plus_inf); + ? state->consts.strings.minus_inf + : state->consts.strings.plus_inf); } if (value == NULL) @@ -1333,7 +1426,7 @@ struct PyHandler { PyObject* pystr = PyUnicode_FromStringAndSize(str, length); if (pystr == NULL) return false; - value = PyObject_CallFunctionObjArgs(types->decimal, pystr, NULL); + value = PyObject_CallFunctionObjArgs(state->types.decimal, pystr, NULL); Py_DECREF(pystr); } else { std::string zstr(str, length); @@ -1488,11 +1581,12 @@ struct PyHandler { if ((datetimeMode & DM_NAIVE_IS_UTC || isZ) && !hasOffset) { if (hasDate) { value = PyDateTimeAPI->DateTime_FromDateAndTime( - year, month, day, hours, mins, secs, usecs, consts->timezone_utc, + year, month, day, hours, mins, secs, usecs, state->consts.timezone_utc, PyDateTimeAPI->DateTimeType); } else { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, consts->timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, state->consts.timezone_utc, + PyDateTimeAPI->TimeType); } } else if (datetimeMode & DM_IGNORE_TZ || (!hasOffset && !isZ)) { if (hasDate) { @@ -1507,14 +1601,15 @@ struct PyHandler { value = NULL; } else if (!hasDate && datetimeMode & DM_SHIFT_TO_UTC) { value = PyDateTimeAPI->Time_FromTime( - hours, mins, secs, usecs, consts->timezone_utc, PyDateTimeAPI->TimeType); + hours, mins, secs, usecs, state->consts.timezone_utc, + PyDateTimeAPI->TimeType); } else { PyObject* offset = PyDateTimeAPI->Delta_FromDelta(0, tzoff, 0, 1, PyDateTimeAPI->DeltaType); if (offset == NULL) { value = NULL; } else { - PyObject* tz = PyObject_CallFunctionObjArgs(types->timezone, offset, NULL); + PyObject* tz = PyObject_CallFunctionObjArgs(state->types.timezone, offset, NULL); Py_DECREF(offset); if (tz == NULL) { value = NULL; @@ -1525,7 +1620,8 @@ struct PyHandler { PyDateTimeAPI->DateTimeType); if (value != NULL && datetimeMode & DM_SHIFT_TO_UTC) { PyObject* asUTC = PyObject_CallMethodObjArgs( - value, consts->names.astimezone, consts->timezone_utc, NULL); + value, state->consts.names.astimezone, + state->consts.timezone_utc, NULL); Py_DECREF(value); if (asUTC == NULL) { value = NULL; @@ -1572,7 +1668,7 @@ struct PyHandler { if (pystr == NULL) return false; - PyObject* value = PyObject_CallFunctionObjArgs(types->uuid, pystr, NULL); + PyObject* value = PyObject_CallFunctionObjArgs(state->types.uuid, pystr, NULL); Py_DECREF(pystr); if (value == NULL) @@ -1659,6 +1755,7 @@ loads(PyObject* self, PyObject* args, PyObject* kwargs) PyObject* parseModeObj = NULL; unsigned parseMode = PM_NONE; int allowNan = -1; + rj_state* state = get_rj_state(self); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|$OOOOOp:rapidjson.loads", (char**) kwlist, @@ -1728,7 +1825,7 @@ loads(PyObject* self, PyObject* args, PyObject* kwargs) return NULL; } - PyObject* result = do_decode(cache.consts, cache.types, NULL, jsonStr, jsonStrLen, NULL, 0, objectHook, + PyObject* result = do_decode(state, NULL, jsonStr, jsonStrLen, NULL, 0, objectHook, numberMode, datetimeMode, uuidMode, parseMode); if (asUnicode != NULL) @@ -1777,6 +1874,7 @@ load(PyObject* self, PyObject* args, PyObject* kwargs) PyObject* chunkSizeObj = NULL; size_t chunkSize = 65536; int allowNan = -1; + rj_state* state = get_rj_state(self); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|$OOOOOOp:rapidjson.load", (char**) kwlist, @@ -1790,7 +1888,7 @@ load(PyObject* self, PyObject* args, PyObject* kwargs) &allowNan)) return NULL; - if (!PyObject_HasAttr(jsonObject, cache.consts.names.read)) { + if (!PyObject_HasAttr(jsonObject, state->consts.names.read)) { PyErr_SetString(PyExc_TypeError, "Expected file-like object"); return NULL; } @@ -1886,8 +1984,8 @@ load(PyObject* self, PyObject* args, PyObject* kwargs) } } - return do_decode(cache.consts, cache.types, NULL, NULL, 0, jsonObject, chunkSize, objectHook, - numberMode, datetimeMode, uuidMode, parseMode); + return do_decode(state, NULL, NULL, 0, jsonObject, chunkSize, objectHook, numberMode, + datetimeMode, uuidMode, parseMode); } @@ -2042,13 +2140,12 @@ static PyType_Spec Decoder_Type_Spec = { static PyObject* -do_decode(const Consts& consts, const Types& types, PyObject* decoder, - const char* jsonStr, Py_ssize_t jsonStrLen, PyObject* jsonStream, - size_t chunkSize, PyObject* objectHook, unsigned numberMode, - unsigned datetimeMode, unsigned uuidMode, unsigned parseMode) +do_decode(const rj_state* state, PyObject* decoder, const char* jsonStr, + Py_ssize_t jsonStrLen, PyObject* jsonStream, size_t chunkSize, + PyObject* objectHook, unsigned numberMode, unsigned datetimeMode, + unsigned uuidMode, unsigned parseMode) { - PyHandler handler(consts, types, decoder, objectHook, - datetimeMode, uuidMode, numberMode); + PyHandler handler(state, decoder, objectHook, datetimeMode, uuidMode, numberMode); Reader reader; ParseResult result; @@ -2070,7 +2167,7 @@ do_decode(const Consts& consts, const Types& types, PyObject* decoder, PyMem_Free(jsonStrCopy); } else { - PyReadStreamWrapper sw(consts.names, jsonStream, chunkSize); + PyReadStreamWrapper sw(state->consts.names, jsonStream, chunkSize); DECODE(reader, kParseNoFlags, sw, handler); @@ -2101,7 +2198,7 @@ do_decode(const Consts& consts, const Types& types, PyObject* decoder, PyErr_Restore(etype, evalue, etraceback); } else - PyErr_Format(types.errors.decode, "Parse error at offset %zu: %s", + PyErr_Format(state->types.errors.decode, "Parse error at offset %zu: %s", offset, GetParseError_En(result.Code())); Py_XDECREF(handler.root); @@ -2127,6 +2224,7 @@ decoder_call(PyObject* self, PyObject* args, PyObject* kwargs) PyObject* jsonObject; PyObject* chunkSizeObj = NULL; size_t chunkSize = 65536; + rj_state* state = get_rj_state(self); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|$O", (char**) kwlist, @@ -2168,7 +2266,7 @@ decoder_call(PyObject* self, PyObject* args, PyObject* kwargs) Py_DECREF(asUnicode); return NULL; } - } else if (PyObject_HasAttr(jsonObject, cache.consts.names.read)) { + } else if (PyObject_HasAttr(jsonObject, state->consts.names.read)) { jsonStr = NULL; jsonStrLen = 0; } else { @@ -2180,9 +2278,9 @@ decoder_call(PyObject* self, PyObject* args, PyObject* kwargs) DecoderObject* d = (DecoderObject*) self; - PyObject* result = do_decode(cache.consts, cache.types, self, jsonStr, jsonStrLen, - jsonObject, chunkSize, NULL, d->numberMode, d->datetimeMode, - d->uuidMode, d->parseMode); + PyObject* result = do_decode(state, self, jsonStr, jsonStrLen, jsonObject, chunkSize, + NULL, d->numberMode, d->datetimeMode, d->uuidMode, + d->parseMode); if (asUnicode != NULL) Py_DECREF(asUnicode); @@ -2333,6 +2431,7 @@ all_keys_are_string(PyObject* dict) { template static bool dumps_internal( + const rj_state* state, WriterT* writer, PyObject* object, PyObject* defaultFn, @@ -2345,7 +2444,7 @@ dumps_internal( { int is_decimal; -#define RECURSE(v) dumps_internal(writer, v, defaultFn, \ +#define RECURSE(v) dumps_internal(state, writer, v, defaultFn, \ numberMode, datetimeMode, uuidMode, \ bytesMode, iterableMode, mappingMode) @@ -2361,14 +2460,15 @@ dumps_internal( } else if (PyBool_Check(object)) { writer->Bool(object == Py_True); } else if (numberMode & NM_DECIMAL - && (is_decimal = PyObject_IsInstance(object, cache.types.decimal))) { + && (is_decimal = PyObject_IsInstance(object, state->types.decimal))) { if (is_decimal == -1) { return false; } if (!(numberMode & NM_NAN)) { bool is_inf_or_nan; - PyObject* is_inf = PyObject_CallMethodObjArgs(object, cache.consts.names.is_infinite, + PyObject* is_inf = PyObject_CallMethodObjArgs(object, + state->consts.names.is_infinite, NULL); if (is_inf == NULL) { @@ -2378,7 +2478,8 @@ dumps_internal( Py_DECREF(is_inf); if (!is_inf_or_nan) { - PyObject* is_nan = PyObject_CallMethodObjArgs(object, cache.consts.names.is_nan, + PyObject* is_nan = PyObject_CallMethodObjArgs(object, + state->consts.names.is_nan, NULL); if (is_nan == NULL) { @@ -2665,9 +2766,9 @@ dumps_internal( char timeZone[TIMEZONE_LEN] = { 0 }; if (!(datetimeMode & DM_IGNORE_TZ) - && PyObject_HasAttr(object, cache.consts.names.utcoffset)) { + && PyObject_HasAttr(object, state->consts.names.utcoffset)) { PyObject* utcOffset = PyObject_CallMethodObjArgs(object, - cache.consts.names.utcoffset, + state->consts.names.utcoffset, NULL); if (utcOffset == NULL) @@ -2687,7 +2788,7 @@ dumps_internal( asUTC = PyDateTimeAPI->DateTime_FromDateAndTime( year, month, day, hour, min, sec, microsec, - cache.consts.timezone_utc, PyDateTimeAPI->DateTimeType); + state->consts.timezone_utc, PyDateTimeAPI->DateTimeType); } else { hour = PyDateTime_TIME_GET_HOUR(dtObject); min = PyDateTime_TIME_GET_MINUTE(dtObject); @@ -2695,7 +2796,7 @@ dumps_internal( microsec = PyDateTime_TIME_GET_MICROSECOND(dtObject); asUTC = PyDateTimeAPI->Time_FromTime( hour, min, sec, microsec, - cache.consts.timezone_utc, PyDateTimeAPI->TimeType); + state->consts.timezone_utc, PyDateTimeAPI->TimeType); } if (asUTC == NULL) { @@ -2713,8 +2814,9 @@ dumps_internal( if (datetimeMode & DM_SHIFT_TO_UTC) { // If it's not already in UTC, shift the value if (PyObject_IsTrue(utcOffset)) { - asUTC = PyObject_CallMethodObjArgs(object, cache.consts.names.astimezone, - cache.consts.timezone_utc, NULL); + asUTC = PyObject_CallMethodObjArgs(object, state->consts.names.astimezone, + state->consts.timezone_utc, + NULL); if (asUTC == NULL) { Py_DECREF(utcOffset); @@ -2731,7 +2833,7 @@ dumps_internal( if (PyObject_IsTrue(utcOffset)) { PyObject* tsObj = PyObject_CallMethodObjArgs(utcOffset, - cache.consts.names.total_seconds, + state->consts.names.total_seconds, NULL); if (tsObj == NULL) { @@ -2811,7 +2913,7 @@ dumps_internal( } else /* if (datetimeMode & DM_UNIX_TIME) */ { if (PyDateTime_Check(dtObject)) { PyObject* timestampObj = PyObject_CallMethodObjArgs(dtObject, - cache.consts.names.timestamp, + state->consts.names.timestamp, NULL); if (timestampObj == NULL) { @@ -2876,7 +2978,7 @@ dumps_internal( if (datetimeMode & (DM_SHIFT_TO_UTC | DM_NAIVE_IS_UTC)) midnightObj = PyDateTimeAPI->DateTime_FromDateAndTime( year, month, day, 0, 0, 0, 0, - cache.consts.timezone_utc, PyDateTimeAPI->DateTimeType); + state->consts.timezone_utc, PyDateTimeAPI->DateTimeType); else midnightObj = PyDateTime_FromDateAndTime(year, month, day, 0, 0, 0, 0); @@ -2885,7 +2987,8 @@ dumps_internal( return false; } - timestampObj = PyObject_CallMethodObjArgs(midnightObj, cache.consts.names.timestamp, + timestampObj = PyObject_CallMethodObjArgs(midnightObj, + state->consts.names.timestamp, NULL); Py_DECREF(midnightObj); @@ -2914,12 +3017,12 @@ dumps_internal( } } } else if (uuidMode != UM_NONE - && PyObject_TypeCheck(object, (PyTypeObject*) cache.types.uuid)) { + && PyObject_TypeCheck(object, (PyTypeObject*) state->types.uuid)) { PyObject* hexval; if (uuidMode == UM_CANONICAL) hexval = PyObject_Str(object); else - hexval = PyObject_GetAttr(object, cache.consts.names.hex); + hexval = PyObject_GetAttr(object, state->consts.names.hex); if (hexval == NULL) return false; @@ -2972,7 +3075,7 @@ dumps_internal( return false; writer->EndArray(); - } else if (PyObject_TypeCheck(object, (PyTypeObject*) cache.types.rawjson)) { + } else if (PyObject_TypeCheck(object, (PyTypeObject*) state->types.rawjson)) { const char* jsonStr; Py_ssize_t l; jsonStr = PyUnicode_AsUTF8AndSize(((RawJSON*) object)->value, &l); @@ -3079,6 +3182,7 @@ dumps(PyObject* self, PyObject* args, PyObject* kwargs) int skipKeys = false; int sortKeys = false; int allowNan = -1; + rj_state* state = get_rj_state(self); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|$ppOOpOOOOOOOp:rapidjson.dumps", (char**) kwlist, @@ -3137,9 +3241,9 @@ dumps(PyObject* self, PyObject* args, PyObject* kwargs) if (sortKeys) mappingMode |= MM_SORT_KEYS; - return do_encode(value, defaultFn, ensureAscii ? true : false, writeMode, indentChar, - indentCount, numberMode, datetimeMode, uuidMode, bytesMode, - iterableMode, mappingMode); + return do_encode(state, value, defaultFn, ensureAscii ? true : false, writeMode, + indentChar, indentCount, numberMode, datetimeMode, uuidMode, + bytesMode, iterableMode, mappingMode); } @@ -3206,6 +3310,7 @@ dump(PyObject* self, PyObject* args, PyObject* kwargs) }; int skipKeys = false; int sortKeys = false; + rj_state* state = get_rj_state(self); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|$ppOOpOOOOOOOOp:rapidjson.dump", (char**) kwlist, @@ -3269,7 +3374,7 @@ dump(PyObject* self, PyObject* args, PyObject* kwargs) if (sortKeys) mappingMode |= MM_SORT_KEYS; - return do_stream_encode(cache.consts.names, value, stream, chunkSize, defaultFn, + return do_stream_encode(state, value, stream, chunkSize, defaultFn, ensureAscii ? true : false, writeMode, indentChar, indentCount, numberMode, datetimeMode, uuidMode, bytesMode, iterableMode, mappingMode); @@ -3365,7 +3470,8 @@ static PyType_Spec Encoder_Type_Spec = { #define DUMPS_INTERNAL_CALL \ - (dumps_internal(&writer, \ + (dumps_internal(state, \ + &writer, \ value, \ defaultFn, \ numberMode, \ @@ -3378,8 +3484,8 @@ static PyType_Spec Encoder_Type_Spec = { static PyObject* -do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, unsigned writeMode, - char indentChar, unsigned indentCount, unsigned numberMode, +do_encode(const rj_state* state, PyObject* value, PyObject* defaultFn, bool ensureAscii, + unsigned writeMode, char indentChar, unsigned indentCount, unsigned numberMode, unsigned datetimeMode, unsigned uuidMode, unsigned bytesMode, unsigned iterableMode, unsigned mappingMode) { @@ -3414,7 +3520,8 @@ do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, unsigned write #define DUMP_INTERNAL_CALL \ - (dumps_internal(&writer, \ + (dumps_internal(state, \ + &writer, \ value, \ defaultFn, \ numberMode, \ @@ -3427,13 +3534,13 @@ do_encode(PyObject* value, PyObject* defaultFn, bool ensureAscii, unsigned write static PyObject* -do_stream_encode(const Names& names, PyObject* value, PyObject* stream, size_t chunkSize, - PyObject* defaultFn, bool ensureAscii, unsigned writeMode, - char indentChar, unsigned indentCount, unsigned numberMode, - unsigned datetimeMode, unsigned uuidMode, unsigned bytesMode, - unsigned iterableMode, unsigned mappingMode) +do_stream_encode(const rj_state* state, PyObject* value, PyObject* stream, + size_t chunkSize, PyObject* defaultFn, bool ensureAscii, + unsigned writeMode, char indentChar, unsigned indentCount, + unsigned numberMode, unsigned datetimeMode, unsigned uuidMode, + unsigned bytesMode, unsigned iterableMode, unsigned mappingMode) { - PyWriteStreamWrapper os(names, stream, chunkSize); + PyWriteStreamWrapper os(state, stream, chunkSize); if (writeMode == WM_COMPACT) { if (ensureAscii) { @@ -3476,6 +3583,7 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) size_t chunkSize = 65536; PyObject* defaultFn = NULL; PyObject* result; + rj_state* state = get_rj_state(self); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O$O", (char**) kwlist, @@ -3487,7 +3595,7 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) EncoderObject* e = (EncoderObject*) self; if (stream != NULL && stream != Py_None) { - if (!PyObject_HasAttr(stream, cache.consts.names.write)) { + if (!PyObject_HasAttr(stream, state->consts.names.write)) { PyErr_SetString(PyExc_TypeError, "Expected a writable stream"); return NULL; } @@ -3495,22 +3603,23 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) if (!accept_chunk_size_arg(chunkSizeObj, chunkSize)) return NULL; - if (PyObject_HasAttr(self, cache.consts.names.default_name)) { - defaultFn = PyObject_GetAttr(self, cache.consts.names.default_name); + if (PyObject_HasAttr(self, state->consts.names.default_name)) { + defaultFn = PyObject_GetAttr(self, state->consts.names.default_name); } - result = do_stream_encode(cache.consts.names, value, stream, chunkSize, defaultFn, e->ensureAscii, - e->writeMode, e->indentChar, e->indentCount, - e->numberMode, e->datetimeMode, e->uuidMode, - e->bytesMode, e->iterableMode, e->mappingMode); + result = do_stream_encode(state, value, stream, chunkSize, defaultFn, + e->ensureAscii, e->writeMode, e->indentChar, + e->indentCount, e->numberMode, e->datetimeMode, + e->uuidMode, e->bytesMode, e->iterableMode, + e->mappingMode); } else { - if (PyObject_HasAttr(self, cache.consts.names.default_name)) { - defaultFn = PyObject_GetAttr(self, cache.consts.names.default_name); + if (PyObject_HasAttr(self, state->consts.names.default_name)) { + defaultFn = PyObject_GetAttr(self, state->consts.names.default_name); } - result = do_encode(value, defaultFn, e->ensureAscii, e->writeMode, e->indentChar, - e->indentCount, e->numberMode, e->datetimeMode, e->uuidMode, - e->bytesMode, e->iterableMode, e->mappingMode); + result = do_encode(state, value, defaultFn, e->ensureAscii, e->writeMode, + e->indentChar, e->indentCount, e->numberMode, e->datetimeMode, + e->uuidMode, e->bytesMode, e->iterableMode, e->mappingMode); } if (defaultFn != NULL) @@ -3663,6 +3772,7 @@ static PyType_Spec Validator_Type_Spec = { static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs) { PyObject* jsonObject; + rj_state* state = get_rj_state(self); if (!PyArg_ParseTuple(args, "O", &jsonObject)) return NULL; @@ -3701,7 +3811,7 @@ static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs if (error) { if (asUnicode != NULL) Py_DECREF(asUnicode); - PyErr_SetString(cache.types.errors.decode, "Invalid JSON"); + PyErr_SetString(state->types.errors.decode, "Invalid JSON"); return NULL; } @@ -3726,7 +3836,7 @@ static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs PyObject* error = Py_BuildValue("sss", validator.GetInvalidSchemaKeyword(), sptr.GetString(), dptr.GetString()); - PyErr_SetObject(cache.types.errors.validation, error); + PyErr_SetObject(state->types.errors.validation, error); if (error != NULL) Py_DECREF(error); @@ -3752,6 +3862,16 @@ static void validator_dealloc(PyObject* self) static PyObject* validator_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) { PyObject* jsonObject; + PyObject* self = PyType_GetModuleByDef(type, &module); + if (self == NULL) { + return NULL; + } + rj_state* state = (rj_state*) PyModule_GetState(self); + if (state == NULL) { + PyErr_Format(PyExc_SystemError, + "rapidjson.Validator: No rapidjson module state found"); + return NULL; + } if (!PyArg_ParseTuple(args, "O", &jsonObject)) return NULL; @@ -3791,7 +3911,7 @@ static PyObject* validator_new(PyTypeObject* type, PyObject* args, PyObject* kwa Py_DECREF(asUnicode); if (error) { - PyErr_SetString(cache.types.errors.decode, "Invalid JSON"); + PyErr_SetString(state->types.errors.decode, "Invalid JSON"); return NULL; } @@ -3824,28 +3944,29 @@ static PyMethodDef functions[] = { static int -module_exec(PyObject* m) +module_exec(PyObject* self) { PyObject* datetimeModule; PyObject* decimalModule; PyObject* uuidModule; + rj_state* state = get_rj_state(self); - auto decoder_type = from_module_and_spec(*m, Decoder_Type_Spec); + auto decoder_type = from_module_and_spec(*self, Decoder_Type_Spec); if (!decoder_type) return -1; - auto encoder_type = from_module_and_spec(*m, Encoder_Type_Spec); + auto encoder_type = from_module_and_spec(*self, Encoder_Type_Spec); if (!encoder_type) return -1; - auto validator_type = from_module_and_spec(*m, Validator_Type_Spec); + auto validator_type = from_module_and_spec(*self, Validator_Type_Spec); if (!validator_type) return -1; - auto rawjson_type = from_module_and_spec(*m, RawJSON_Type_Spec); + auto rawjson_type = from_module_and_spec(*self, RawJSON_Type_Spec); if (!rawjson_type) return -1; - cache.types.rawjson = rawjson_type.get(); + state->types.rawjson = rawjson_type.get(); PyDateTime_IMPORT; if(!PyDateTimeAPI) @@ -3859,153 +3980,154 @@ module_exec(PyObject* m) if (decimalModule == NULL) return -1; - cache.types.decimal = PyObject_GetAttrString(decimalModule, "Decimal"); + state->types.decimal = PyObject_GetAttrString(decimalModule, "Decimal"); Py_DECREF(decimalModule); - if (cache.types.decimal == NULL) + if (state->types.decimal == NULL) return -1; - cache.types.timezone = PyObject_GetAttrString(datetimeModule, "timezone"); + state->types.timezone = PyObject_GetAttrString(datetimeModule, "timezone"); Py_DECREF(datetimeModule); - if (cache.types.timezone == NULL) + if (state->types.timezone == NULL) return -1; - cache.consts.timezone_utc = PyObject_GetAttrString(cache.types.timezone, "utc"); - if (cache.consts.timezone_utc == NULL) + state->consts.timezone_utc = PyObject_GetAttrString(state->types.timezone, "utc"); + if (state->consts.timezone_utc == NULL) return -1; uuidModule = PyImport_ImportModule("uuid"); if (uuidModule == NULL) return -1; - cache.types.uuid = PyObject_GetAttrString(uuidModule, "UUID"); + state->types.uuid = PyObject_GetAttrString(uuidModule, "UUID"); Py_DECREF(uuidModule); - if (cache.types.uuid == NULL) + if (state->types.uuid == NULL) return -1; - cache.consts.names.astimezone = PyUnicode_InternFromString("astimezone"); - if (cache.consts.names.astimezone == NULL) + state->consts.names.astimezone = PyUnicode_InternFromString("astimezone"); + if (state->consts.names.astimezone == NULL) return -1; - cache.consts.names.hex = PyUnicode_InternFromString("hex"); - if (cache.consts.names.hex == NULL) + state->consts.names.hex = PyUnicode_InternFromString("hex"); + if (state->consts.names.hex == NULL) return -1; - cache.consts.names.timestamp = PyUnicode_InternFromString("timestamp"); - if (cache.consts.names.timestamp == NULL) + state->consts.names.timestamp = PyUnicode_InternFromString("timestamp"); + if (state->consts.names.timestamp == NULL) return -1; - cache.consts.names.total_seconds = PyUnicode_InternFromString("total_seconds"); - if (cache.consts.names.total_seconds == NULL) + state->consts.names.total_seconds = PyUnicode_InternFromString("total_seconds"); + if (state->consts.names.total_seconds == NULL) return -1; - cache.consts.names.utcoffset = PyUnicode_InternFromString("utcoffset"); - if (cache.consts.names.utcoffset == NULL) + state->consts.names.utcoffset = PyUnicode_InternFromString("utcoffset"); + if (state->consts.names.utcoffset == NULL) return -1; - cache.consts.names.is_infinite = PyUnicode_InternFromString("is_infinite"); - if (cache.consts.names.is_infinite == NULL) + state->consts.names.is_infinite = PyUnicode_InternFromString("is_infinite"); + if (state->consts.names.is_infinite == NULL) return -1; - cache.consts.names.is_nan = PyUnicode_InternFromString("is_nan"); - if (cache.consts.names.is_nan == NULL) + state->consts.names.is_nan = PyUnicode_InternFromString("is_nan"); + if (state->consts.names.is_nan == NULL) return -1; - cache.consts.strings.minus_inf = PyUnicode_InternFromString("-Infinity"); - if (cache.consts.strings.minus_inf == NULL) + state->consts.strings.minus_inf = PyUnicode_InternFromString("-Infinity"); + if (state->consts.strings.minus_inf == NULL) return -1; - cache.consts.strings.nan = PyUnicode_InternFromString("nan"); - if (cache.consts.strings.nan == NULL) + state->consts.strings.nan = PyUnicode_InternFromString("nan"); + if (state->consts.strings.nan == NULL) return -1; - cache.consts.strings.plus_inf = PyUnicode_InternFromString("+Infinity"); - if (cache.consts.strings.plus_inf == NULL) + state->consts.strings.plus_inf = PyUnicode_InternFromString("+Infinity"); + if (state->consts.strings.plus_inf == NULL) return -1; - cache.consts.names.start_object = PyUnicode_InternFromString("start_object"); - if (cache.consts.names.start_object == NULL) + state->consts.names.start_object = PyUnicode_InternFromString("start_object"); + if (state->consts.names.start_object == NULL) return -1; - cache.consts.names.end_object = PyUnicode_InternFromString("end_object"); - if (cache.consts.names.end_object == NULL) + state->consts.names.end_object = PyUnicode_InternFromString("end_object"); + if (state->consts.names.end_object == NULL) return -1; - cache.consts.names.default_name = PyUnicode_InternFromString("default"); - if (cache.consts.names.default_name == NULL) + state->consts.names.default_name = PyUnicode_InternFromString("default"); + if (state->consts.names.default_name == NULL) return -1; - cache.consts.names.end_array = PyUnicode_InternFromString("end_array"); - if (cache.consts.names.end_array == NULL) + state->consts.names.end_array = PyUnicode_InternFromString("end_array"); + if (state->consts.names.end_array == NULL) return -1; - cache.consts.names.string = PyUnicode_InternFromString("string"); - if (cache.consts.names.string == NULL) + state->consts.names.string = PyUnicode_InternFromString("string"); + if (state->consts.names.string == NULL) return -1; - cache.consts.names.read = PyUnicode_InternFromString("read"); - if (cache.consts.names.read == NULL) + state->consts.names.read = PyUnicode_InternFromString("read"); + if (state->consts.names.read == NULL) return -1; - cache.consts.names.write = PyUnicode_InternFromString("write"); - if (cache.consts.names.write == NULL) + state->consts.names.write = PyUnicode_InternFromString("write"); + if (state->consts.names.write == NULL) return -1; - cache.consts.names.encoding = PyUnicode_InternFromString("encoding"); - if (cache.consts.names.encoding == NULL) + state->consts.names.encoding = PyUnicode_InternFromString("encoding"); + if (state->consts.names.encoding == NULL) return -1; #define STRINGIFY(x) XSTRINGIFY(x) #define XSTRINGIFY(x) #x - if (PyModule_AddIntConstant(m, "DM_NONE", DM_NONE) - || PyModule_AddIntConstant(m, "DM_ISO8601", DM_ISO8601) - || PyModule_AddIntConstant(m, "DM_UNIX_TIME", DM_UNIX_TIME) - || PyModule_AddIntConstant(m, "DM_ONLY_SECONDS", DM_ONLY_SECONDS) - || PyModule_AddIntConstant(m, "DM_IGNORE_TZ", DM_IGNORE_TZ) - || PyModule_AddIntConstant(m, "DM_NAIVE_IS_UTC", DM_NAIVE_IS_UTC) - || PyModule_AddIntConstant(m, "DM_SHIFT_TO_UTC", DM_SHIFT_TO_UTC) + if (PyModule_AddIntConstant(self, "DM_NONE", DM_NONE) + || PyModule_AddIntConstant(self, "DM_ISO8601", DM_ISO8601) + || PyModule_AddIntConstant(self, "DM_UNIX_TIME", DM_UNIX_TIME) + || PyModule_AddIntConstant(self, "DM_ONLY_SECONDS", DM_ONLY_SECONDS) + || PyModule_AddIntConstant(self, "DM_IGNORE_TZ", DM_IGNORE_TZ) + || PyModule_AddIntConstant(self, "DM_NAIVE_IS_UTC", DM_NAIVE_IS_UTC) + || PyModule_AddIntConstant(self, "DM_SHIFT_TO_UTC", DM_SHIFT_TO_UTC) - || PyModule_AddIntConstant(m, "UM_NONE", UM_NONE) - || PyModule_AddIntConstant(m, "UM_HEX", UM_HEX) - || PyModule_AddIntConstant(m, "UM_CANONICAL", UM_CANONICAL) + || PyModule_AddIntConstant(self, "UM_NONE", UM_NONE) + || PyModule_AddIntConstant(self, "UM_HEX", UM_HEX) + || PyModule_AddIntConstant(self, "UM_CANONICAL", UM_CANONICAL) - || PyModule_AddIntConstant(m, "NM_NONE", NM_NONE) - || PyModule_AddIntConstant(m, "NM_NAN", NM_NAN) - || PyModule_AddIntConstant(m, "NM_DECIMAL", NM_DECIMAL) - || PyModule_AddIntConstant(m, "NM_NATIVE", NM_NATIVE) + || PyModule_AddIntConstant(self, "NM_NONE", NM_NONE) + || PyModule_AddIntConstant(self, "NM_NAN", NM_NAN) + || PyModule_AddIntConstant(self, "NM_DECIMAL", NM_DECIMAL) + || PyModule_AddIntConstant(self, "NM_NATIVE", NM_NATIVE) - || PyModule_AddIntConstant(m, "PM_NONE", PM_NONE) - || PyModule_AddIntConstant(m, "PM_COMMENTS", PM_COMMENTS) - || PyModule_AddIntConstant(m, "PM_TRAILING_COMMAS", PM_TRAILING_COMMAS) + || PyModule_AddIntConstant(self, "PM_NONE", PM_NONE) + || PyModule_AddIntConstant(self, "PM_COMMENTS", PM_COMMENTS) + || PyModule_AddIntConstant(self, "PM_TRAILING_COMMAS", PM_TRAILING_COMMAS) - || PyModule_AddIntConstant(m, "BM_NONE", BM_NONE) - || PyModule_AddIntConstant(m, "BM_UTF8", BM_UTF8) + || PyModule_AddIntConstant(self, "BM_NONE", BM_NONE) + || PyModule_AddIntConstant(self, "BM_UTF8", BM_UTF8) - || PyModule_AddIntConstant(m, "WM_COMPACT", WM_COMPACT) - || PyModule_AddIntConstant(m, "WM_PRETTY", WM_PRETTY) - || PyModule_AddIntConstant(m, "WM_SINGLE_LINE_ARRAY", WM_SINGLE_LINE_ARRAY) + || PyModule_AddIntConstant(self, "WM_COMPACT", WM_COMPACT) + || PyModule_AddIntConstant(self, "WM_PRETTY", WM_PRETTY) + || PyModule_AddIntConstant(self, "WM_SINGLE_LINE_ARRAY", WM_SINGLE_LINE_ARRAY) - || PyModule_AddIntConstant(m, "IM_ANY_ITERABLE", IM_ANY_ITERABLE) - || PyModule_AddIntConstant(m, "IM_ONLY_LISTS", IM_ONLY_LISTS) + || PyModule_AddIntConstant(self, "IM_ANY_ITERABLE", IM_ANY_ITERABLE) + || PyModule_AddIntConstant(self, "IM_ONLY_LISTS", IM_ONLY_LISTS) - || PyModule_AddIntConstant(m, "MM_ANY_MAPPING", MM_ANY_MAPPING) - || PyModule_AddIntConstant(m, "MM_ONLY_DICTS", MM_ONLY_DICTS) - || PyModule_AddIntConstant(m, "MM_COERCE_KEYS_TO_STRINGS", + || PyModule_AddIntConstant(self, "MM_ANY_MAPPING", MM_ANY_MAPPING) + || PyModule_AddIntConstant(self, "MM_ONLY_DICTS", MM_ONLY_DICTS) + || PyModule_AddIntConstant(self, "MM_COERCE_KEYS_TO_STRINGS", MM_COERCE_KEYS_TO_STRINGS) - || PyModule_AddIntConstant(m, "MM_SKIP_NON_STRING_KEYS", MM_SKIP_NON_STRING_KEYS) - || PyModule_AddIntConstant(m, "MM_SORT_KEYS", MM_SORT_KEYS) + || PyModule_AddIntConstant(self, "MM_SKIP_NON_STRING_KEYS", + MM_SKIP_NON_STRING_KEYS) + || PyModule_AddIntConstant(self, "MM_SORT_KEYS", MM_SORT_KEYS) - || PyModule_AddStringConstant(m, "__version__", + || PyModule_AddStringConstant(self, "__version__", STRINGIFY(PYTHON_RAPIDJSON_VERSION)) - || PyModule_AddStringConstant(m, "__author__", + || PyModule_AddStringConstant(self, "__author__", "Ken Robbins " ", Lele Gaifax ") - || PyModule_AddStringConstant(m, "__rapidjson_version__", + || PyModule_AddStringConstant(self, "__rapidjson_version__", RAPIDJSON_VERSION_STRING) - || PyModule_AddStringConstant(m, "__rapidjson_exact_version__", + || PyModule_AddStringConstant(self, "__rapidjson_exact_version__", #ifdef RAPIDJSON_EXACT_VERSION STRINGIFY(RAPIDJSON_EXACT_VERSION) #else @@ -4018,39 +4140,39 @@ module_exec(PyObject* m) ) return -1; - if (PyModule_AddObject(m, "Decoder", decoder_type.get()) < 0) + if (PyModule_AddObject(self, "Decoder", decoder_type.get()) < 0) return -1; decoder_type.release(); - if (PyModule_AddObject(m, "Encoder", encoder_type.get()) < 0) + if (PyModule_AddObject(self, "Encoder", encoder_type.get()) < 0) return -1; encoder_type.release(); - if (PyModule_AddObject(m, "Validator", validator_type.get()) < 0) + if (PyModule_AddObject(self, "Validator", validator_type.get()) < 0) return -1; validator_type.release(); - if (PyModule_AddObject(m, "RawJSON", rawjson_type.get()) < 0) + if (PyModule_AddObject(self, "RawJSON", rawjson_type.get()) < 0) return -1; rawjson_type.release(); - cache.types.errors.validation = PyErr_NewException("rapidjson.ValidationError", - PyExc_ValueError, NULL); - if (cache.types.errors.validation == NULL) + state->types.errors.validation = PyErr_NewException("rapidjson.ValidationError", + PyExc_ValueError, NULL); + if (state->types.errors.validation == NULL) return -1; - Py_INCREF(cache.types.errors.validation); - if (PyModule_AddObject(m, "ValidationError", cache.types.errors.validation) < 0) { - Py_DECREF(cache.types.errors.validation); + Py_INCREF(state->types.errors.validation); + if (PyModule_AddObject(self, "ValidationError", state->types.errors.validation) < 0) { + Py_DECREF(state->types.errors.validation); return -1; } - cache.types.errors.decode = PyErr_NewException("rapidjson.JSONDecodeError", - PyExc_ValueError, NULL); - if (cache.types.errors.decode == NULL) + state->types.errors.decode = PyErr_NewException("rapidjson.JSONDecodeError", + PyExc_ValueError, NULL); + if (state->types.errors.decode == NULL) return -1; - Py_INCREF(cache.types.errors.decode); - if (PyModule_AddObject(m, "JSONDecodeError", cache.types.errors.decode) < 0) { - Py_DECREF(cache.types.errors.decode); + Py_INCREF(state->types.errors.decode); + if (PyModule_AddObject(self, "JSONDecodeError", state->types.errors.decode) < 0) { + Py_DECREF(state->types.errors.decode); return -1; } @@ -4071,12 +4193,12 @@ static PyModuleDef module = { PyModuleDef_HEAD_INIT, /* m_base */ "rapidjson", /* m_name */ PyDoc_STR("Fast, simple JSON encoder and decoder. Based on RapidJSON C++ library."), - 0, /* m_size */ + sizeof(rj_state), /* m_size */ functions, /* m_methods */ slots, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ + rj_traverse, /* m_traverse */ + rj_clear, /* m_clear */ + rj_free /* m_free */ }; From d1836e9b42256494efd3272818e27939eb111319 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Fri, 4 Sep 2026 08:05:16 +0200 Subject: [PATCH 11/20] Try with a anonymous namespace --- rapidjson.cpp | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index c1bfe70..550b62f 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -3859,6 +3859,8 @@ static void validator_dealloc(PyObject* self) } +namespace { extern PyModuleDef module; } + static PyObject* validator_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) { PyObject* jsonObject; @@ -4188,19 +4190,21 @@ static struct PyModuleDef_Slot slots[] = { {0, NULL} }; +namespace { + + PyModuleDef module = { + PyModuleDef_HEAD_INIT, /* m_base */ + "rapidjson", /* m_name */ + PyDoc_STR("Fast, simple JSON encoder and decoder. Based on RapidJSON C++ library."), + sizeof(rj_state), /* m_size */ + functions, /* m_methods */ + slots, /* m_slots */ + rj_traverse, /* m_traverse */ + rj_clear, /* m_clear */ + rj_free /* m_free */ + }; -static PyModuleDef module = { - PyModuleDef_HEAD_INIT, /* m_base */ - "rapidjson", /* m_name */ - PyDoc_STR("Fast, simple JSON encoder and decoder. Based on RapidJSON C++ library."), - sizeof(rj_state), /* m_size */ - functions, /* m_methods */ - slots, /* m_slots */ - rj_traverse, /* m_traverse */ - rj_clear, /* m_clear */ - rj_free /* m_free */ -}; - +} PyMODINIT_FUNC PyInit_rapidjson() From e797ed89f912a1ed3ef57fde2ab22989539b5e15 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sun, 1 Nov 2026 09:02:53 +0100 Subject: [PATCH 12/20] Adapt code introduced in PR #255 --- rapidjson.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 550b62f..2d6d7e9 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -2497,7 +2497,8 @@ dumps_internal( } if (numberMode & NM_NAN) { - PyObject* is_nan = PyObject_CallMethodObjArgs(object, is_nan_name, + PyObject* is_nan = PyObject_CallMethodObjArgs(object, + state->consts.names.is_nan, NULL); if (is_nan == NULL) return false; From 0991cf36b749a0ec735df1e632732b77c03022c9 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sun, 1 Nov 2026 11:49:27 +0100 Subject: [PATCH 13/20] Add missing INCREF on the rawjson reference in the state --- rapidjson.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/rapidjson.cpp b/rapidjson.cpp index 2d6d7e9..f6b96c2 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -3970,6 +3970,7 @@ module_exec(PyObject* self) if (!rawjson_type) return -1; state->types.rawjson = rawjson_type.get(); + Py_INCREF(state->types.rawjson); PyDateTime_IMPORT; if(!PyDateTimeAPI) From ee29bf562cfccc2d0a128eb49ee6ed3ed28f1838 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sun, 1 Nov 2026 11:55:33 +0100 Subject: [PATCH 14/20] Fix access to the module's state from methods and slots --- rapidjson.cpp | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index f6b96c2..6f8be1f 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -124,6 +124,9 @@ typedef struct { } rj_state; +namespace { extern PyModuleDef module_def; } + + static inline rj_state* get_rj_state(PyObject* module) { @@ -133,6 +136,19 @@ get_rj_state(PyObject* module) } +/* Module level functions receive the module itself as their first argument, but methods + and the tp_call slot of the heap types receive an instance: in that case the module + owning the type (possibly a superclass of it) must be looked up through its MRO. */ +static inline rj_state* +get_type_state(PyTypeObject* type) +{ + PyObject* module = PyType_GetModuleByDef(type, &module_def); + if (module == NULL) + return NULL; + return get_rj_state(module); +} + + static int rj_clear(PyObject* self) { @@ -2224,7 +2240,9 @@ decoder_call(PyObject* self, PyObject* args, PyObject* kwargs) PyObject* jsonObject; PyObject* chunkSizeObj = NULL; size_t chunkSize = 65536; - rj_state* state = get_rj_state(self); + rj_state* state = get_type_state(Py_TYPE(self)); + if (state == NULL) + return NULL; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|$O", (char**) kwlist, @@ -3584,7 +3602,9 @@ encoder_call(PyObject* self, PyObject* args, PyObject* kwargs) size_t chunkSize = 65536; PyObject* defaultFn = NULL; PyObject* result; - rj_state* state = get_rj_state(self); + rj_state* state = get_type_state(Py_TYPE(self)); + if (state == NULL) + return NULL; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O$O", (char**) kwlist, @@ -3773,7 +3793,9 @@ static PyType_Spec Validator_Type_Spec = { static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs) { PyObject* jsonObject; - rj_state* state = get_rj_state(self); + rj_state* state = get_type_state(Py_TYPE(self)); + if (state == NULL) + return NULL; if (!PyArg_ParseTuple(args, "O", &jsonObject)) return NULL; @@ -3860,16 +3882,10 @@ static void validator_dealloc(PyObject* self) } -namespace { extern PyModuleDef module; } - static PyObject* validator_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) { PyObject* jsonObject; - PyObject* self = PyType_GetModuleByDef(type, &module); - if (self == NULL) { - return NULL; - } - rj_state* state = (rj_state*) PyModule_GetState(self); + rj_state* state = get_type_state(type); if (state == NULL) { PyErr_Format(PyExc_SystemError, "rapidjson.Validator: No rapidjson module state found"); @@ -4194,7 +4210,7 @@ static struct PyModuleDef_Slot slots[] = { namespace { - PyModuleDef module = { + PyModuleDef module_def = { PyModuleDef_HEAD_INIT, /* m_base */ "rapidjson", /* m_name */ PyDoc_STR("Fast, simple JSON encoder and decoder. Based on RapidJSON C++ library."), @@ -4211,5 +4227,5 @@ namespace { PyMODINIT_FUNC PyInit_rapidjson() { - return PyModuleDef_Init(&module); + return PyModuleDef_Init(&module_def); } From 4a2fc182e324ef00af6a78682fba97a6f32cc33d Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sun, 1 Nov 2026 14:48:17 +0100 Subject: [PATCH 15/20] Move also the datetime API into the module's state I'm not really sure this is required, because AFAIU all the API is actually static, and apparently shared by all interpreters... Anyway, this seems clearer. --- rapidjson.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 6f8be1f..d4e22af 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -7,7 +7,15 @@ // #include + +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif #include +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif #include #include @@ -27,6 +35,10 @@ using namespace rapidjson; +/* Redirect references to the DT API through the module state. */ +#define PyDateTimeAPI (state->apis.datetime) + + struct PyDerefer { void operator() (PyObject* obj) const noexcept {Py_DecRef(obj);} }; @@ -118,9 +130,15 @@ struct Consts { }; +struct APIs { + PyDateTime_CAPI* datetime = NULL; +}; + + typedef struct { Types types; Consts consts; + APIs apis; } rj_state; @@ -3988,8 +4006,8 @@ module_exec(PyObject* self) state->types.rawjson = rawjson_type.get(); Py_INCREF(state->types.rawjson); - PyDateTime_IMPORT; - if(!PyDateTimeAPI) + state->apis.datetime = (PyDateTime_CAPI*) PyCapsule_Import(PyDateTime_CAPSULE_NAME, 0); + if (state->apis.datetime == NULL) return -1; datetimeModule = PyImport_ImportModule("datetime"); From 86a8ea804bbee1a0954d63f6d59c52863d67d2c9 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sun, 1 Nov 2026 15:21:08 +0100 Subject: [PATCH 16/20] Add some simple tests to exercize sub-interpreters --- tests/test_subinterpreters.py | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tests/test_subinterpreters.py diff --git a/tests/test_subinterpreters.py b/tests/test_subinterpreters.py new file mode 100644 index 0000000..054aee2 --- /dev/null +++ b/tests/test_subinterpreters.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# :Project: python-rapidjson -- Subinterpreters tests +# :Author: Lele Gaifax +# :License: MIT License +# :Copyright: © 2026 Lele Gaifax +# + +import contextlib +import json +import os + +import pytest + +import rapidjson as rj + +interpreters = pytest.importorskip('_interpreters', + reason='this Python has no subinterpreters support') + +# Ensure that the directory where the binary module lives is present on the sys.path: +# apparently a newly spawned interpreter may not have it while developing, when usually +# the module is built in-place. +MODULE_DIR = os.path.dirname(os.path.abspath(rj.__file__)) + +PRELUDE = f"""import sys +sys.path.insert(0, {MODULE_DIR!r}) +import rapidjson as rj +""" + + +@contextlib.contextmanager +def interpreter(): + sub = interpreters.create() + try: + interpreters.run_string(sub, PRELUDE) + yield sub + finally: + interpreters.destroy(sub) + + +def test_full_api(): + with interpreter() as sub: + interpreters.run_string(sub, """ +import datetime +assert rj.dumps({'a': [1, 2.5, None, True, 'x']}) == '{"a":[1,2.5,null,true,"x"]}' +assert rj.loads('{"a": [1, 2]}') == {'a': [1, 2]} +assert rj.Encoder(sort_keys=True)({'b': 1, 'a': 2}) == '{"a":2,"b":1}' +assert rj.Decoder()('[1, 2]') == [1, 2] +assert rj.Validator('{"type": "integer"}')('42') is None +assert rj.dumps({'a': rj.RawJSON('[1,2]')}) == '{"a":[1,2]}' +when = datetime.datetime(2020, 1, 2, 3, 4, 5) +assert rj.loads(rj.dumps(when, datetime_mode=rj.DM_ISO8601), + datetime_mode=rj.DM_ISO8601) == when +""") + + +def test_independent_state(tmp_path): + seen = tmp_path / 'ids.json' + with interpreter() as first, interpreter() as second: + for sub in (first, second): + interpreters.run_string(sub, f""" +import json +with open({str(seen)!r}, 'a') as f: + print(json.dumps([id(rj.Decoder), id(rj.ValidationError), id(rj.RawJSON)]), file=f) +""") + # The second interpreter executed the module while the first was alive: the + # first must still be using the types cached in its own state. + interpreters.run_string(first, """ +assert rj.dumps({'a': rj.RawJSON('[1,2]')}) == '{"a":[1,2]}' +""") + + ids = [json.loads(line) for line in seen.read_text().splitlines()] + assert len(ids) == 2 + assert set(ids[0]).isdisjoint(ids[1]) + + +def test_repeated_teardown(): + # Exercises module_exec/module_clear/module_free several times in the same process + for _ in range(20): + with interpreter() as sub: + interpreters.run_string(sub, """ +assert rj.loads(rj.dumps([1, 'x'])) == [1, 'x'] +""") From 82b1453ea1af5429d11dd724f8901f83b30a45d9 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Sun, 1 Nov 2026 16:06:05 +0100 Subject: [PATCH 17/20] Add backward compatibility with Python 3.10 --- rapidjson.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rapidjson.cpp b/rapidjson.cpp index d4e22af..381200a 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -160,7 +160,17 @@ get_rj_state(PyObject* module) static inline rj_state* get_type_state(PyTypeObject* type) { +#if PY_VERSION_HEX >= 0x030B0000 PyObject* module = PyType_GetModuleByDef(type, &module_def); +#else + /* Backward compatibility for Python 3.10, until we support that: + PyType_GetModuleByDef() has been introduced in 3.11 and before 3.12 (pre PEP-684 + that is) the module is the same in the whole process. */ + (void) type; + PyObject* module = PyState_FindModule(&module_def); + if (module == NULL) + PyErr_SetString(PyExc_SystemError, "rapidjson: module state not found"); +#endif if (module == NULL) return NULL; return get_rj_state(module); From d0bcde3ca8d6a5c9d5d0c1c59b96a457a053ce51 Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Mon, 2 Nov 2026 08:33:34 +0100 Subject: [PATCH 18/20] Fix refcount leak in the dealloc of RawJSON and Validator Both are heap types now: their instances hold a reference to the type and, being GC-tracked, must be untracked before being freed. Their dealloc was only calling tp_free(), leaking one reference to the type per instance and, on debug interpreters, emitting a ResourceWarning. --- rapidjson.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rapidjson.cpp b/rapidjson.cpp index 381200a..b7ce6ae 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -644,8 +644,12 @@ typedef struct { static void RawJSON_dealloc(RawJSON* self) { + PyTypeObject* tp = Py_TYPE(self); + + PyObject_GC_UnTrack(self); Py_XDECREF(self->value); - Py_TYPE(self)->tp_free((PyObject*) self); + tp->tp_free((PyObject*) self); + Py_DECREF(tp); } @@ -3904,9 +3908,13 @@ static PyObject* validator_call(PyObject* self, PyObject* args, PyObject* kwargs static void validator_dealloc(PyObject* self) { + PyTypeObject* tp = Py_TYPE(self); ValidatorObject* s = (ValidatorObject*) self; + + PyObject_GC_UnTrack(self); delete s->schema; - Py_TYPE(self)->tp_free(self); + tp->tp_free(self); + Py_DECREF(tp); } From 5a33e6b0fe69abcb6a1074895f253fdd316da84f Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Mon, 2 Nov 2026 09:00:10 +0100 Subject: [PATCH 19/20] Drop support for Python 3.10 Python 3.10 is EOL since 2026-10-01, and its not easy for me to even test for it locally. This also reverts commit 82b1453ea1af5429d11dd724f8901f83b30a45d9. --- .github/workflows/main.yml | 2 +- rapidjson.cpp | 10 ---------- setup.py | 7 +++---- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03fc6b1..a7093c2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -128,7 +128,7 @@ jobs: CIBW_ARCHS_MACOS: ${{ matrix.arch }} CIBW_TEST_REQUIRES: "pytest pytest-benchmark pytz" CIBW_TEST_COMMAND: "pytest {project}/tests" - CIBW_SKIP: "cp39*" + CIBW_SKIP: "cp39* cp310*" # Skip tests on emulated hardware, take too long or not supported CIBW_TEST_SKIP: "*-*linux_{aarch64,ppc64le,riscv64} *-macosx_*:arm64" CIBW_ENABLE: cpython-prerelease diff --git a/rapidjson.cpp b/rapidjson.cpp index b7ce6ae..f8cf8df 100644 --- a/rapidjson.cpp +++ b/rapidjson.cpp @@ -160,17 +160,7 @@ get_rj_state(PyObject* module) static inline rj_state* get_type_state(PyTypeObject* type) { -#if PY_VERSION_HEX >= 0x030B0000 PyObject* module = PyType_GetModuleByDef(type, &module_def); -#else - /* Backward compatibility for Python 3.10, until we support that: - PyType_GetModuleByDef() has been introduced in 3.11 and before 3.12 (pre PEP-684 - that is) the module is the same in the whole process. */ - (void) type; - PyObject* module = PyState_FindModule(&module_def); - if (module == NULL) - PyErr_SetString(PyExc_SystemError, "rapidjson: module state not found"); -#endif if (module == NULL) return NULL; return get_rj_state(module); diff --git a/setup.py b/setup.py index deaed9b..0c59842 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ from distutils import sysconfig -if sys.version_info < (3, 10): - raise NotImplementedError("Only Python 3.10+ is supported.") +if sys.version_info < (3, 11): + raise NotImplementedError("Only Python 3.11+ is supported.") ROOT_PATH = os.path.abspath(os.path.dirname(__file__)) @@ -90,7 +90,6 @@ 'Programming Language :: C++', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', @@ -98,7 +97,7 @@ 'Programming Language :: Python :: 3.15', 'Programming Language :: Python', ], - python_requires=">=3.10", + python_requires=">=3.11", ext_modules=[Extension('rapidjson', **extension_options)], package_dir={"rapidjson-stubs": "typings/rapidjson"}, packages=["rapidjson-stubs"], From cc85e9adc1cd14558bd407b0f78b0f6f2d0af52b Mon Sep 17 00:00:00 2001 From: Lele Gaifax Date: Mon, 2 Nov 2026 09:04:12 +0100 Subject: [PATCH 20/20] Drop tox Its configuration still referenced Python 3.9 and 3.10, a good evidence that it was abandoned lot of time ago. --- Makefile | 2 +- setup.cfg | 2 +- tox.ini | 27 --------------------------- 3 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 tox.ini diff --git a/Makefile b/Makefile index 3232d4c..cabae88 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ help:: .PHONY: distclean distclean:: clean - rm -rf .tox build dist rapidjson_exact_version.txt + rm -rf build dist rapidjson_exact_version.txt git submodule deinit --all help:: diff --git a/setup.cfg b/setup.cfg index a8eccbf..ac56110 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [tool:pytest] -norecursedirs = *.egg .tox +norecursedirs = *.egg addopts = # Shows a line for every test # You probably want to turn this off if you use pytest-sugar. diff --git a/tox.ini b/tox.ini deleted file mode 100644 index c440842..0000000 --- a/tox.ini +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# :Project: python-rapidjson -- Tox configuration -# :Author: Lele Gaifax -# :License: MIT License -# :Copyright: © 2015, 2016, 2017, 2018, 2021 Lele Gaifax -# - -[tox] -envlist = py39, py39-debug, py310, docs - -[testenv] -deps = -rrequirements-test.txt -changedir = tests -commands = py.test {posargs} - -# The following uses a debug-enabled compilation of the Python interpreter: on -# Debian it's packaged as python3.9-dbg; the test_refs_count module depends on -# functionality available only in that variant -[testenv:py39-debug] -basepython = python3.9-dbg - -[testenv:docs] -basepython = python3 -changedir = docs -deps = sphinx -usedevelop = True -commands = sphinx-build -b doctest -d {envtmpdir}/doctrees . {envtmpdir}/doctest