Where Online Learning is simpler!
The C and C++ Include Header Files
cat -n /usr/include/python3.14/cpython/unicodeobject.h
1 #ifndef Py_CPYTHON_UNICODEOBJECT_H 2 # error "this header file must not be included directly" 3 #endif 4 5 /* Py_UNICODE was the native Unicode storage format (code unit) used by 6 Python and represents a single Unicode element in the Unicode type. 7 With PEP 393, Py_UNICODE is deprecated and replaced with a 8 typedef to wchar_t. */ 9 Py_DEPRECATED(3.13) typedef wchar_t PY_UNICODE_TYPE; 10 Py_DEPRECATED(3.13) typedef wchar_t Py_UNICODE; 11 12 13 /* --- Internal Unicode Operations ---------------------------------------- */ 14 15 // Static inline functions to work with surrogates 16 static inline int Py_UNICODE_IS_SURROGATE(Py_UCS4 ch) { 17 return (0xD800 <= ch && ch <= 0xDFFF); 18 } 19 static inline int Py_UNICODE_IS_HIGH_SURROGATE(Py_UCS4 ch) { 20 return (0xD800 <= ch && ch <= 0xDBFF); 21 } 22 static inline int Py_UNICODE_IS_LOW_SURROGATE(Py_UCS4 ch) { 23 return (0xDC00 <= ch && ch <= 0xDFFF); 24 } 25 26 // Join two surrogate characters and return a single Py_UCS4 value. 27 static inline Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) { 28 assert(Py_UNICODE_IS_HIGH_SURROGATE(high)); 29 assert(Py_UNICODE_IS_LOW_SURROGATE(low)); 30 return 0x10000 + (((high & 0x03FF) << 10) | (low & 0x03FF)); 31 } 32 33 // High surrogate = top 10 bits added to 0xD800. 34 // The character must be in the range [U+10000; U+10ffff]. 35 static inline Py_UCS4 Py_UNICODE_HIGH_SURROGATE(Py_UCS4 ch) { 36 assert(0x10000 <= ch && ch <= 0x10ffff); 37 return (0xD800 - (0x10000 >> 10) + (ch >> 10)); 38 } 39 40 // Low surrogate = bottom 10 bits added to 0xDC00. 41 // The character must be in the range [U+10000; U+10ffff]. 42 static inline Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) { 43 assert(0x10000 <= ch && ch <= 0x10ffff); 44 return (0xDC00 + (ch & 0x3FF)); 45 } 46 47 48 /* --- Unicode Type ------------------------------------------------------- */ 49 50 struct _PyUnicodeObject_state { 51 /* If interned is non-zero, the two references from the 52 dictionary to this object are *not* counted in ob_refcnt. 53 The possible values here are: 54 0: Not Interned 55 1: Interned 56 2: Interned and Immortal 57 3: Interned, Immortal, and Static 58 This categorization allows the runtime to determine the right 59 cleanup mechanism at runtime shutdown. */ 60 #ifdef Py_GIL_DISABLED 61 // Needs to be accessed atomically, so can't be a bit field. 62 unsigned char interned; 63 #else 64 unsigned int interned:2; 65 #endif 66 /* Character size: 67 68 - PyUnicode_1BYTE_KIND (1): 69 70 * character type = Py_UCS1 (8 bits, unsigned) 71 * all characters are in the range U+0000-U+00FF (latin1) 72 * if ascii is set, all characters are in the range U+0000-U+007F 73 (ASCII), otherwise at least one character is in the range 74 U+0080-U+00FF 75 76 - PyUnicode_2BYTE_KIND (2): 77 78 * character type = Py_UCS2 (16 bits, unsigned) 79 * all characters are in the range U+0000-U+FFFF (BMP) 80 * at least one character is in the range U+0100-U+FFFF 81 82 - PyUnicode_4BYTE_KIND (4): 83 84 * character type = Py_UCS4 (32 bits, unsigned) 85 * all characters are in the range U+0000-U+10FFFF 86 * at least one character is in the range U+10000-U+10FFFF 87 */ 88 unsigned int kind:3; 89 /* Compact is with respect to the allocation scheme. Compact unicode 90 objects only require one memory block while non-compact objects use 91 one block for the PyUnicodeObject struct and another for its data 92 buffer. */ 93 unsigned int compact:1; 94 /* The string only contains characters in the range U+0000-U+007F (ASCII) 95 and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is 96 set, use the PyASCIIObject structure. */ 97 unsigned int ascii:1; 98 /* The object is statically allocated. */ 99 unsigned int statically_allocated:1; 100 #ifndef Py_GIL_DISABLED 101 /* Historical: padding to ensure that PyUnicode_DATA() is always aligned to 102 4 bytes (see issue gh-63736 on m68k) */ 103 unsigned int :24; 104 #endif 105 }; 106 107 /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject 108 structure. state.ascii and state.compact are set, and the data 109 immediately follow the structure. utf8_length can be found 110 in the length field; the utf8 pointer is equal to the data pointer. */ 111 typedef struct { 112 /* There are 4 forms of Unicode strings: 113 114 - compact ascii: 115 116 * structure = PyASCIIObject 117 * test: PyUnicode_IS_COMPACT_ASCII(op) 118 * kind = PyUnicode_1BYTE_KIND 119 * compact = 1 120 * ascii = 1 121 * (length is the length of the utf8) 122 * (data starts just after the structure) 123 * (since ASCII is decoded from UTF-8, the utf8 string are the data) 124 125 - compact: 126 127 * structure = PyCompactUnicodeObject 128 * test: PyUnicode_IS_COMPACT(op) && !PyUnicode_IS_ASCII(op) 129 * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or 130 PyUnicode_4BYTE_KIND 131 * compact = 1 132 * ascii = 0 133 * utf8 is not shared with data 134 * utf8_length = 0 if utf8 is NULL 135 * (data starts just after the structure) 136 137 - legacy string: 138 139 * structure = PyUnicodeObject structure 140 * test: !PyUnicode_IS_COMPACT(op) 141 * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or 142 PyUnicode_4BYTE_KIND 143 * compact = 0 144 * data.any is not NULL 145 * utf8 is shared and utf8_length = length with data.any if ascii = 1 146 * utf8_length = 0 if utf8 is NULL 147 148 Compact strings use only one memory block (structure + characters), 149 whereas legacy strings use one block for the structure and one block 150 for characters. 151 152 Legacy strings are created by subclasses of Unicode. 153 154 See also _PyUnicode_CheckConsistency(). 155 */ 156 PyObject_HEAD 157 Py_ssize_t length; /* Number of code points in the string */ 158 Py_hash_t hash; /* Hash value; -1 if not set */ 159 /* Ensure 4 byte alignment for PyUnicode_DATA(), see gh-63736 on m68k. */ 160 _Py_ALIGNED_DEF(4, struct _PyUnicodeObject_state) state; 161 } PyASCIIObject; 162 163 /* Non-ASCII strings allocated through PyUnicode_New use the 164 PyCompactUnicodeObject structure. state.compact is set, and the data 165 immediately follow the structure. */ 166 typedef struct { 167 PyASCIIObject _base; 168 Py_ssize_t utf8_length; /* Number of bytes in utf8, excluding the 169 * terminating \0. */ 170 char *utf8; /* UTF-8 representation (null-terminated) */ 171 } PyCompactUnicodeObject; 172 173 /* Object format for Unicode subclasses. */ 174 typedef struct { 175 PyCompactUnicodeObject _base; 176 union { 177 void *any; 178 Py_UCS1 *latin1; 179 Py_UCS2 *ucs2; 180 Py_UCS4 *ucs4; 181 } data; /* Canonical, smallest-form Unicode buffer */ 182 } PyUnicodeObject; 183 184 185 #define _PyASCIIObject_CAST(op) \ 186 (assert(PyUnicode_Check(op)), \ 187 _Py_CAST(PyASCIIObject*, (op))) 188 #define _PyCompactUnicodeObject_CAST(op) \ 189 (assert(PyUnicode_Check(op)), \ 190 _Py_CAST(PyCompactUnicodeObject*, (op))) 191 #define _PyUnicodeObject_CAST(op) \ 192 (assert(PyUnicode_Check(op)), \ 193 _Py_CAST(PyUnicodeObject*, (op))) 194 195 196 /* --- Flexible String Representation Helper Macros (PEP 393) -------------- */ 197 198 /* Values for PyASCIIObject.state: */ 199 200 /* Interning state. */ 201 #define SSTATE_NOT_INTERNED 0 202 #define SSTATE_INTERNED_MORTAL 1 203 #define SSTATE_INTERNED_IMMORTAL 2 204 #define SSTATE_INTERNED_IMMORTAL_STATIC 3 205 206 /* Use only if you know it's a string */ 207 static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) { 208 #ifdef Py_GIL_DISABLED 209 return _Py_atomic_load_uint8_relaxed(&_PyASCIIObject_CAST(op)->state.interned); 210 #else 211 return _PyASCIIObject_CAST(op)->state.interned; 212 #endif 213 } 214 #define PyUnicode_CHECK_INTERNED(op) PyUnicode_CHECK_INTERNED(_PyObject_CAST(op)) 215 216 /* For backward compatibility. Soft-deprecated. */ 217 static inline unsigned int PyUnicode_IS_READY(PyObject* Py_UNUSED(op)) { 218 return 1; 219 } 220 #define PyUnicode_IS_READY(op) PyUnicode_IS_READY(_PyObject_CAST(op)) 221 222 /* Return true if the string contains only ASCII characters, or 0 if not. The 223 string may be compact (PyUnicode_IS_COMPACT_ASCII) or not. */ 224 static inline unsigned int PyUnicode_IS_ASCII(PyObject *op) { 225 return _PyASCIIObject_CAST(op)->state.ascii; 226 } 227 #define PyUnicode_IS_ASCII(op) PyUnicode_IS_ASCII(_PyObject_CAST(op)) 228 229 /* Return true if the string is compact or 0 if not. 230 No type checks are performed. */ 231 static inline unsigned int PyUnicode_IS_COMPACT(PyObject *op) { 232 return _PyASCIIObject_CAST(op)->state.compact; 233 } 234 #define PyUnicode_IS_COMPACT(op) PyUnicode_IS_COMPACT(_PyObject_CAST(op)) 235 236 /* Return true if the string is a compact ASCII string (use PyASCIIObject 237 structure), or 0 if not. No type checks are performed. */ 238 static inline int PyUnicode_IS_COMPACT_ASCII(PyObject *op) { 239 return (_PyASCIIObject_CAST(op)->state.ascii && PyUnicode_IS_COMPACT(op)); 240 } 241 #define PyUnicode_IS_COMPACT_ASCII(op) PyUnicode_IS_COMPACT_ASCII(_PyObject_CAST(op)) 242 243 enum PyUnicode_Kind { 244 /* Return values of the PyUnicode_KIND() function: */ 245 PyUnicode_1BYTE_KIND = 1, 246 PyUnicode_2BYTE_KIND = 2, 247 PyUnicode_4BYTE_KIND = 4 248 }; 249 250 PyAPI_FUNC(int) PyUnicode_KIND(PyObject *op); 251 252 // PyUnicode_KIND(): Return one of the PyUnicode_*_KIND values defined above. 253 // 254 // gh-89653: Converting this macro to a static inline function would introduce 255 // new compiler warnings on "kind < PyUnicode_KIND(str)" (compare signed and 256 // unsigned numbers) where kind type is an int or on 257 // "unsigned int kind = PyUnicode_KIND(str)" (cast signed to unsigned). 258 #define PyUnicode_KIND(op) _Py_RVALUE(_PyASCIIObject_CAST(op)->state.kind) 259 260 /* Return a void pointer to the raw unicode buffer. */ 261 static inline void* _PyUnicode_COMPACT_DATA(PyObject *op) { 262 if (PyUnicode_IS_ASCII(op)) { 263 return _Py_STATIC_CAST(void*, (_PyASCIIObject_CAST(op) + 1)); 264 } 265 return _Py_STATIC_CAST(void*, (_PyCompactUnicodeObject_CAST(op) + 1)); 266 } 267 268 static inline void* _PyUnicode_NONCOMPACT_DATA(PyObject *op) { 269 void *data; 270 assert(!PyUnicode_IS_COMPACT(op)); 271 data = _PyUnicodeObject_CAST(op)->data.any; 272 assert(data != NULL); 273 return data; 274 } 275 276 PyAPI_FUNC(void*) PyUnicode_DATA(PyObject *op); 277 278 static inline void* _PyUnicode_DATA(PyObject *op) { 279 if (PyUnicode_IS_COMPACT(op)) { 280 return _PyUnicode_COMPACT_DATA(op); 281 } 282 return _PyUnicode_NONCOMPACT_DATA(op); 283 } 284 #define PyUnicode_DATA(op) _PyUnicode_DATA(_PyObject_CAST(op)) 285 286 /* Return pointers to the canonical representation cast to unsigned char, 287 Py_UCS2, or Py_UCS4 for direct character access. 288 No checks are performed, use PyUnicode_KIND() before to ensure 289 these will work correctly. */ 290 291 #define PyUnicode_1BYTE_DATA(op) _Py_STATIC_CAST(Py_UCS1*, PyUnicode_DATA(op)) 292 #define PyUnicode_2BYTE_DATA(op) _Py_STATIC_CAST(Py_UCS2*, PyUnicode_DATA(op)) 293 #define PyUnicode_4BYTE_DATA(op) _Py_STATIC_CAST(Py_UCS4*, PyUnicode_DATA(op)) 294 295 /* Returns the length of the unicode string. */ 296 static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { 297 return _PyASCIIObject_CAST(op)->length; 298 } 299 #define PyUnicode_GET_LENGTH(op) PyUnicode_GET_LENGTH(_PyObject_CAST(op)) 300 301 /* Write into the canonical representation, this function does not do any sanity 302 checks and is intended for usage in loops. The caller should cache the 303 kind and data pointers obtained from other function calls. 304 index is the index in the string (starts at 0) and value is the new 305 code point value which should be written to that location. */ 306 static inline void PyUnicode_WRITE(int kind, void *data, 307 Py_ssize_t index, Py_UCS4 value) 308 { 309 assert(index >= 0); 310 if (kind == PyUnicode_1BYTE_KIND) { 311 assert(value <= 0xffU); 312 _Py_STATIC_CAST(Py_UCS1*, data)[index] = _Py_STATIC_CAST(Py_UCS1, value); 313 } 314 else if (kind == PyUnicode_2BYTE_KIND) { 315 assert(value <= 0xffffU); 316 _Py_STATIC_CAST(Py_UCS2*, data)[index] = _Py_STATIC_CAST(Py_UCS2, value); 317 } 318 else { 319 assert(kind == PyUnicode_4BYTE_KIND); 320 assert(value <= 0x10ffffU); 321 _Py_STATIC_CAST(Py_UCS4*, data)[index] = value; 322 } 323 } 324 #define PyUnicode_WRITE(kind, data, index, value) \ 325 PyUnicode_WRITE(_Py_STATIC_CAST(int, kind), _Py_CAST(void*, data), \ 326 (index), _Py_STATIC_CAST(Py_UCS4, value)) 327 328 /* Read a code point from the string's canonical representation. No checks 329 are performed. */ 330 static inline Py_UCS4 PyUnicode_READ(int kind, 331 const void *data, Py_ssize_t index) 332 { 333 assert(index >= 0); 334 if (kind == PyUnicode_1BYTE_KIND) { 335 return _Py_STATIC_CAST(const Py_UCS1*, data)[index]; 336 } 337 if (kind == PyUnicode_2BYTE_KIND) { 338 return _Py_STATIC_CAST(const Py_UCS2*, data)[index]; 339 } 340 assert(kind == PyUnicode_4BYTE_KIND); 341 return _Py_STATIC_CAST(const Py_UCS4*, data)[index]; 342 } 343 #define PyUnicode_READ(kind, data, index) \ 344 PyUnicode_READ(_Py_STATIC_CAST(int, kind), \ 345 _Py_STATIC_CAST(const void*, data), \ 346 (index)) 347 348 /* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it 349 calls PyUnicode_KIND() and might call it twice. For single reads, use 350 PyUnicode_READ_CHAR, for multiple consecutive reads callers should 351 cache kind and use PyUnicode_READ instead. */ 352 static inline Py_UCS4 PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) 353 { 354 int kind; 355 356 assert(index >= 0); 357 // Tolerate reading the NUL character at str[len(str)] 358 assert(index <= PyUnicode_GET_LENGTH(unicode)); 359 360 kind = PyUnicode_KIND(unicode); 361 if (kind == PyUnicode_1BYTE_KIND) { 362 return PyUnicode_1BYTE_DATA(unicode)[index]; 363 } 364 if (kind == PyUnicode_2BYTE_KIND) { 365 return PyUnicode_2BYTE_DATA(unicode)[index]; 366 } 367 assert(kind == PyUnicode_4BYTE_KIND); 368 return PyUnicode_4BYTE_DATA(unicode)[index]; 369 } 370 #define PyUnicode_READ_CHAR(unicode, index) \ 371 PyUnicode_READ_CHAR(_PyObject_CAST(unicode), (index)) 372 373 /* Return a maximum character value which is suitable for creating another 374 string based on op. This is always an approximation but more efficient 375 than iterating over the string. */ 376 static inline Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *op) 377 { 378 int kind; 379 380 if (PyUnicode_IS_ASCII(op)) { 381 return 0x7fU; 382 } 383 384 kind = PyUnicode_KIND(op); 385 if (kind == PyUnicode_1BYTE_KIND) { 386 return 0xffU; 387 } 388 if (kind == PyUnicode_2BYTE_KIND) { 389 return 0xffffU; 390 } 391 assert(kind == PyUnicode_4BYTE_KIND); 392 return 0x10ffffU; 393 } 394 #define PyUnicode_MAX_CHAR_VALUE(op) \ 395 PyUnicode_MAX_CHAR_VALUE(_PyObject_CAST(op)) 396 397 398 /* === Public API ========================================================= */ 399 400 /* With PEP 393, this is the recommended way to allocate a new unicode object. 401 This function will allocate the object and its buffer in a single memory 402 block. Objects created using this function are not resizable. */ 403 PyAPI_FUNC(PyObject*) PyUnicode_New( 404 Py_ssize_t size, /* Number of code points in the new string */ 405 Py_UCS4 maxchar /* maximum code point value in the string */ 406 ); 407 408 /* For backward compatibility. Soft-deprecated. */ 409 static inline int PyUnicode_READY(PyObject* Py_UNUSED(op)) 410 { 411 return 0; 412 } 413 #define PyUnicode_READY(op) PyUnicode_READY(_PyObject_CAST(op)) 414 415 /* Copy character from one unicode object into another, this function performs 416 character conversion when necessary and falls back to memcpy() if possible. 417 418 Fail if to is too small (smaller than *how_many* or smaller than 419 len(from)-from_start), or if kind(from[from_start:from_start+how_many]) > 420 kind(to), or if *to* has more than 1 reference. 421 422 Return the number of written character, or return -1 and raise an exception 423 on error. 424 425 Pseudo-code: 426 427 how_many = min(how_many, len(from) - from_start) 428 to[to_start:to_start+how_many] = from[from_start:from_start+how_many] 429 return how_many 430 431 Note: The function doesn't write a terminating null character. 432 */ 433 PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters( 434 PyObject *to, 435 Py_ssize_t to_start, 436 PyObject *from, 437 Py_ssize_t from_start, 438 Py_ssize_t how_many 439 ); 440 441 /* Fill a string with a character: write fill_char into 442 unicode[start:start+length]. 443 444 Fail if fill_char is bigger than the string maximum character, or if the 445 string has more than 1 reference. 446 447 Return the number of written character, or return -1 and raise an exception 448 on error. */ 449 PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill( 450 PyObject *unicode, 451 Py_ssize_t start, 452 Py_ssize_t length, 453 Py_UCS4 fill_char 454 ); 455 456 /* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters. 457 Scan the string to find the maximum character. */ 458 PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( 459 int kind, 460 const void *buffer, 461 Py_ssize_t size); 462 463 464 /* --- Public PyUnicodeWriter API ----------------------------------------- */ 465 466 typedef struct PyUnicodeWriter PyUnicodeWriter; 467 468 PyAPI_FUNC(PyUnicodeWriter*) PyUnicodeWriter_Create(Py_ssize_t length); 469 PyAPI_FUNC(void) PyUnicodeWriter_Discard(PyUnicodeWriter *writer); 470 PyAPI_FUNC(PyObject*) PyUnicodeWriter_Finish(PyUnicodeWriter *writer); 471 472 PyAPI_FUNC(int) PyUnicodeWriter_WriteChar( 473 PyUnicodeWriter *writer, 474 Py_UCS4 ch); 475 PyAPI_FUNC(int) PyUnicodeWriter_WriteUTF8( 476 PyUnicodeWriter *writer, 477 const char *str, 478 Py_ssize_t size); 479 PyAPI_FUNC(int) PyUnicodeWriter_WriteASCII( 480 PyUnicodeWriter *writer, 481 const char *str, 482 Py_ssize_t size); 483 PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( 484 PyUnicodeWriter *writer, 485 const wchar_t *str, 486 Py_ssize_t size); 487 PyAPI_FUNC(int) PyUnicodeWriter_WriteUCS4( 488 PyUnicodeWriter *writer, 489 Py_UCS4 *str, 490 Py_ssize_t size); 491 492 PyAPI_FUNC(int) PyUnicodeWriter_WriteStr( 493 PyUnicodeWriter *writer, 494 PyObject *obj); 495 PyAPI_FUNC(int) PyUnicodeWriter_WriteRepr( 496 PyUnicodeWriter *writer, 497 PyObject *obj); 498 PyAPI_FUNC(int) PyUnicodeWriter_WriteSubstring( 499 PyUnicodeWriter *writer, 500 PyObject *str, 501 Py_ssize_t start, 502 Py_ssize_t end); 503 PyAPI_FUNC(int) PyUnicodeWriter_Format( 504 PyUnicodeWriter *writer, 505 const char *format, 506 ...); 507 PyAPI_FUNC(int) PyUnicodeWriter_DecodeUTF8Stateful( 508 PyUnicodeWriter *writer, 509 const char *string, /* UTF-8 encoded string */ 510 Py_ssize_t length, /* size of string */ 511 const char *errors, /* error handling */ 512 Py_ssize_t *consumed); /* bytes consumed */ 513 514 515 /* --- Private _PyUnicodeWriter API --------------------------------------- */ 516 517 typedef struct { 518 PyObject *buffer; 519 void *data; 520 int kind; 521 Py_UCS4 maxchar; 522 Py_ssize_t size; 523 Py_ssize_t pos; 524 525 /* minimum number of allocated characters (default: 0) */ 526 Py_ssize_t min_length; 527 528 /* minimum character (default: 127, ASCII) */ 529 Py_UCS4 min_char; 530 531 /* If non-zero, overallocate the buffer (default: 0). */ 532 unsigned char overallocate; 533 534 /* If readonly is 1, buffer is a shared string (cannot be modified) 535 and size is set to 0. */ 536 unsigned char readonly; 537 } _PyUnicodeWriter; 538 539 // Initialize a Unicode writer. 540 // 541 // By default, the minimum buffer size is 0 character and overallocation is 542 // disabled. Set min_length, min_char and overallocate attributes to control 543 // the allocation of the buffer. 544 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(void) _PyUnicodeWriter_Init( 545 _PyUnicodeWriter *writer); 546 547 /* Prepare the buffer to write 'length' characters 548 with the specified maximum character. 549 550 Return 0 on success, raise an exception and return -1 on error. */ 551 #define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ 552 (((MAXCHAR) <= (WRITER)->maxchar \ 553 && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ 554 ? 0 \ 555 : (((LENGTH) == 0) \ 556 ? 0 \ 557 : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) 558 559 /* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro 560 instead. */ 561 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_PrepareInternal( 562 _PyUnicodeWriter *writer, 563 Py_ssize_t length, 564 Py_UCS4 maxchar); 565 566 /* Prepare the buffer to have at least the kind KIND. 567 For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will 568 support characters in range U+000-U+FFFF. 569 570 Return 0 on success, raise an exception and return -1 on error. */ 571 #define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ 572 ((KIND) <= (WRITER)->kind \ 573 ? 0 \ 574 : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) 575 576 /* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() 577 macro instead. */ 578 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_PrepareKindInternal( 579 _PyUnicodeWriter *writer, 580 int kind); 581 582 /* Append a Unicode character. 583 Return 0 on success, raise an exception and return -1 on error. */ 584 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_WriteChar( 585 _PyUnicodeWriter *writer, 586 Py_UCS4 ch); 587 588 /* Append a Unicode string. 589 Return 0 on success, raise an exception and return -1 on error. */ 590 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_WriteStr( 591 _PyUnicodeWriter *writer, 592 PyObject *str); /* Unicode string */ 593 594 /* Append a substring of a Unicode string. 595 Return 0 on success, raise an exception and return -1 on error. */ 596 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_WriteSubstring( 597 _PyUnicodeWriter *writer, 598 PyObject *str, /* Unicode string */ 599 Py_ssize_t start, 600 Py_ssize_t end); 601 602 /* Append an ASCII-encoded byte string. 603 Return 0 on success, raise an exception and return -1 on error. */ 604 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_WriteASCIIString( 605 _PyUnicodeWriter *writer, 606 const char *str, /* ASCII-encoded byte string */ 607 Py_ssize_t len); /* number of bytes, or -1 if unknown */ 608 609 /* Append a latin1-encoded byte string. 610 Return 0 on success, raise an exception and return -1 on error. */ 611 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(int) _PyUnicodeWriter_WriteLatin1String( 612 _PyUnicodeWriter *writer, 613 const char *str, /* latin1-encoded byte string */ 614 Py_ssize_t len); /* length in bytes */ 615 616 /* Get the value of the writer as a Unicode string. Clear the 617 buffer of the writer. Raise an exception and return NULL 618 on error. */ 619 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(PyObject *) _PyUnicodeWriter_Finish( 620 _PyUnicodeWriter *writer); 621 622 /* Deallocate memory of a writer (clear its internal buffer). */ 623 _Py_DEPRECATED_EXTERNALLY(3.14) PyAPI_FUNC(void) _PyUnicodeWriter_Dealloc( 624 _PyUnicodeWriter *writer); 625 626 627 /* --- Manage the default encoding ---------------------------------------- */ 628 629 /* Returns a pointer to the default encoding (UTF-8) of the 630 Unicode object unicode. 631 632 Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation 633 in the unicodeobject. 634 635 _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to 636 support the previous internal function with the same behaviour. 637 638 Use of this API is DEPRECATED since no size information can be 639 extracted from the returned data. 640 */ 641 642 PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); 643 644 // Deprecated alias kept for backward compatibility 645 Py_DEPRECATED(3.14) static inline const char* 646 _PyUnicode_AsString(PyObject *unicode) 647 { 648 return PyUnicode_AsUTF8(unicode); 649 } 650 651 652 /* === Characters Type APIs =============================================== */ 653 654 /* These should not be used directly. Use the Py_UNICODE_IS* and 655 Py_UNICODE_TO* macros instead. 656 657 These APIs are implemented in Objects/unicodectype.c. 658 659 */ 660 661 PyAPI_FUNC(int) _PyUnicode_IsLowercase( 662 Py_UCS4 ch /* Unicode character */ 663 ); 664 665 PyAPI_FUNC(int) _PyUnicode_IsUppercase( 666 Py_UCS4 ch /* Unicode character */ 667 ); 668 669 PyAPI_FUNC(int) _PyUnicode_IsTitlecase( 670 Py_UCS4 ch /* Unicode character */ 671 ); 672 673 PyAPI_FUNC(int) _PyUnicode_IsWhitespace( 674 const Py_UCS4 ch /* Unicode character */ 675 ); 676 677 PyAPI_FUNC(int) _PyUnicode_IsLinebreak( 678 const Py_UCS4 ch /* Unicode character */ 679 ); 680 681 PyAPI_FUNC(Py_UCS4) _PyUnicode_ToLowercase( 682 Py_UCS4 ch /* Unicode character */ 683 ); 684 685 PyAPI_FUNC(Py_UCS4) _PyUnicode_ToUppercase( 686 Py_UCS4 ch /* Unicode character */ 687 ); 688 689 PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase( 690 Py_UCS4 ch /* Unicode character */ 691 ); 692 693 PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit( 694 Py_UCS4 ch /* Unicode character */ 695 ); 696 697 PyAPI_FUNC(int) _PyUnicode_ToDigit( 698 Py_UCS4 ch /* Unicode character */ 699 ); 700 701 PyAPI_FUNC(double) _PyUnicode_ToNumeric( 702 Py_UCS4 ch /* Unicode character */ 703 ); 704 705 PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit( 706 Py_UCS4 ch /* Unicode character */ 707 ); 708 709 PyAPI_FUNC(int) _PyUnicode_IsDigit( 710 Py_UCS4 ch /* Unicode character */ 711 ); 712 713 PyAPI_FUNC(int) _PyUnicode_IsNumeric( 714 Py_UCS4 ch /* Unicode character */ 715 ); 716 717 PyAPI_FUNC(int) _PyUnicode_IsPrintable( 718 Py_UCS4 ch /* Unicode character */ 719 ); 720 721 PyAPI_FUNC(int) _PyUnicode_IsAlpha( 722 Py_UCS4 ch /* Unicode character */ 723 ); 724 725 // Helper array used by Py_UNICODE_ISSPACE(). 726 PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[]; 727 728 // Since splitting on whitespace is an important use case, and 729 // whitespace in most situations is solely ASCII whitespace, we 730 // optimize for the common case by using a quick look-up table 731 // _Py_ascii_whitespace (see below) with an inlined check. 732 static inline int Py_UNICODE_ISSPACE(Py_UCS4 ch) { 733 if (ch < 128) { 734 return _Py_ascii_whitespace[ch]; 735 } 736 return _PyUnicode_IsWhitespace(ch); 737 } 738 739 #define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch) 740 #define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch) 741 #define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch) 742 #define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch) 743 744 #define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch) 745 #define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch) 746 #define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch) 747 748 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch) 749 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch) 750 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch) 751 #define Py_UNICODE_ISPRINTABLE(ch) _PyUnicode_IsPrintable(ch) 752 753 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch) 754 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch) 755 #define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch) 756 757 #define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch) 758 759 static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) { 760 return (Py_UNICODE_ISALPHA(ch) 761 || Py_UNICODE_ISDECIMAL(ch) 762 || Py_UNICODE_ISDIGIT(ch) 763 || Py_UNICODE_ISNUMERIC(ch)); 764 } 765 766 767 /* === Misc functions ===================================================== */ 768 769 // Return an interned Unicode object for an Identifier; may fail if there is no 770 // memory. 771 PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*);
