Igor Sarkisov | 64ff12e | 2020-10-06 05:32:41 -0700 | [diff] [blame] | 1 | /* |
| 2 | ******************************************************************************* |
| 3 | * |
| 4 | * Copyright (C) 1998-2010, International Business Machines |
| 5 | * Corporation and others. All Rights Reserved. |
| 6 | * |
| 7 | ******************************************************************************* |
| 8 | * |
| 9 | * Private implementation header for C collation |
| 10 | * file name: ucol_imp.h |
| 11 | * encoding: US-ASCII |
| 12 | * tab size: 8 (not used) |
| 13 | * indentation:4 |
| 14 | * |
| 15 | * created on: 2000dec11 |
| 16 | * created by: Vladimir Weinstein |
| 17 | * |
| 18 | * Modification history |
| 19 | * Date Name Comments |
| 20 | * 02/16/2001 synwee Added UCOL_GETPREVCE for the use in ucoleitr |
| 21 | * 02/27/2001 synwee Added getMaxExpansion data structure in UCollator |
| 22 | * 03/02/2001 synwee Added UCOL_IMPLICIT_CE |
| 23 | * 03/12/2001 synwee Added pointer start to collIterate. |
| 24 | */ |
| 25 | |
| 26 | #ifndef UCOL_IMP_H |
| 27 | #define UCOL_IMP_H |
| 28 | |
| 29 | #include "unicode/utypes.h" |
| 30 | |
| 31 | #define UCA_DATA_TYPE "icu" |
| 32 | #define UCA_DATA_NAME "ucadata" |
| 33 | #define INVC_DATA_TYPE "icu" |
| 34 | #define INVC_DATA_NAME "invuca" |
| 35 | |
| 36 | /** |
| 37 | * Convenience string denoting the Collation data tree |
| 38 | * @internal ICU 3.0 |
| 39 | */ |
| 40 | #define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll" |
| 41 | |
| 42 | #if !UCONFIG_NO_COLLATION |
| 43 | |
| 44 | #ifdef XP_CPLUSPLUS |
| 45 | #include "unicode/normalizer2.h" |
| 46 | #include "unicode/unistr.h" |
| 47 | #endif |
| 48 | #include "unicode/ucol.h" |
| 49 | #include "utrie.h" |
| 50 | #include "cmemory.h" |
| 51 | |
| 52 | /* This is the internal header file which contains important declarations for |
| 53 | * the collation framework. |
| 54 | * Ready to use collators are stored as binary images. Both UCA and tailorings |
| 55 | * share the same binary format. Individual files (currently only UCA) have a |
| 56 | * udata header in front of the image and should be opened using udata_open. |
| 57 | * Tailoring images are currently stored inside resource bundles and are intialized |
| 58 | * through ucol_open API. |
| 59 | * |
| 60 | * The following describes the formats for collation binaries |
| 61 | * (UCA & tailorings) and for the inverse UCA table. |
| 62 | * Substructures are described in the collation design document at |
| 63 | * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm |
| 64 | * |
| 65 | * ------------------------------------------------------------- |
| 66 | * |
| 67 | * Here is the format of binary collation image. |
| 68 | * |
| 69 | * Physical order of structures: |
| 70 | * - header (UCATableHeader) |
| 71 | * - options (UColOptionSet) |
| 72 | * - expansions (CE[]) |
| 73 | * - contractions (UChar[contractionSize] + CE[contractionSize]) |
| 74 | * - serialized UTrie with mappings of code points to CEs |
| 75 | * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount]) |
| 76 | * - two bit sets for backward processing in strcoll (identical prefixes) |
| 77 | * and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE]) |
| 78 | * - UCA constants (UCAConstants) |
| 79 | * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth]) |
| 80 | * |
| 81 | * UCATableHeader fields: |
| 82 | * |
| 83 | * int32_t size; - image size in bytes |
| 84 | * |
| 85 | * Offsets to interesting data. All offsets are in bytes. |
| 86 | * to get the address add to the header address and cast properly. |
| 87 | * Some offsets are zero if the corresponding structures are empty. |
| 88 | * |
| 89 | * Tailoring binaries that only set options and contain no mappings etc. |
| 90 | * will have all offsets 0 except for the options and expansion offsets, |
| 91 | * which give the position and length of the options array. |
| 92 | * |
| 93 | * uint32_t options; - offset to default collator options (UColOptionSet *), |
| 94 | * a set of 32-bit values. See declaration of UColOptionSet for more details |
| 95 | * |
| 96 | * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges |
| 97 | * See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store |
| 98 | * important constant values that are defined in the UCA and used for building and runtime. |
| 99 | * |
| 100 | * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth], |
| 101 | * containing contractions from the UCA. These are needed in the build process to copy UCA contractions |
| 102 | * in case the base contraction symbol is tailored. |
| 103 | * |
| 104 | * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3) |
| 105 | * |
| 106 | * uint32_t mappingPosition; - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such. |
| 107 | * Used as a primary lookup table for collation elements. |
| 108 | * |
| 109 | * uint32_t expansion; - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0. |
| 110 | * |
| 111 | * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents |
| 112 | * are aligned with the contents of contractionCEs table. 0 if no contractions. |
| 113 | * |
| 114 | * uint32_t contractionCEs; - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the |
| 115 | * in the contractionIndex table, the resulting index is used to look up corresponding CE in this table. |
| 116 | * 0 if no contractions. |
| 117 | * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs). |
| 118 | * |
| 119 | * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion |
| 120 | * and last CEs in expansions. |
| 121 | * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *). |
| 122 | * Never 0. |
| 123 | * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *) |
| 124 | * int32_t endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION |
| 125 | * for the usage model |
| 126 | * |
| 127 | * These two offsets point to byte tables that are used in the backup heuristics. |
| 128 | * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function. |
| 129 | * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP. |
| 130 | * |
| 131 | * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos |
| 132 | * (formatVersion 2.3) |
| 133 | * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition. |
| 134 | * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3) |
| 135 | * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3) |
| 136 | * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3) |
| 137 | * |
| 138 | * Various version fields |
| 139 | * UVersionInfo version; - version 4 uint8_t |
| 140 | * UVersionInfo UCAVersion; - version 4 uint8_t |
| 141 | * UVersionInfo UCDVersion; - version 4 uint8_t |
| 142 | * UVersionInfo formatVersion; - version of the format of the collation binary |
| 143 | * same formatVersion as in ucadata.icu's UDataInfo header |
| 144 | * (formatVersion 2.3) |
| 145 | * |
| 146 | * uint8_t reserved[84]; - currently unused |
| 147 | * |
| 148 | * ------------------------------------------------------------- |
| 149 | * |
| 150 | * Inverse UCA is used for constructing collators from rules. It is always an individual file |
| 151 | * and always has a UDataInfo header. |
| 152 | * here is the structure: |
| 153 | * |
| 154 | * uint32_t byteSize; - size of inverse UCA image in bytes |
| 155 | * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows) |
| 156 | * uint32_t contsSize; - size of continuation table (number of UChars in table) |
| 157 | * |
| 158 | * uint32_t table; - offset to inverse table (uint32_t *) |
| 159 | * Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation |
| 160 | * the third value is either a code unit (if there is only one code unit for element) or an index to continuation |
| 161 | * (number of code units combined with an index). |
| 162 | * table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final |
| 163 | * code unit sequence for a CE is terminated by FFFE. |
| 164 | * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE. |
| 165 | * |
| 166 | * UVersionInfo UCAVersion; - version of the UCA, read from file 4 uint8_t |
| 167 | * uint8_t padding[8]; - padding 8 uint8_t |
| 168 | * Header is followed by the table and continuation table. |
| 169 | */ |
| 170 | |
| 171 | /* let us know whether reserved fields are reset to zero or junked */ |
| 172 | #define UCOL_HEADER_MAGIC 0x20030618 |
| 173 | |
| 174 | /* UDataInfo for UCA mapping table */ |
| 175 | /* dataFormat="UCol" */ |
| 176 | #define UCA_DATA_FORMAT_0 ((uint8_t)0x55) |
| 177 | #define UCA_DATA_FORMAT_1 ((uint8_t)0x43) |
| 178 | #define UCA_DATA_FORMAT_2 ((uint8_t)0x6f) |
| 179 | #define UCA_DATA_FORMAT_3 ((uint8_t)0x6c) |
| 180 | |
| 181 | #define UCA_FORMAT_VERSION_0 ((uint8_t)2) |
| 182 | #define UCA_FORMAT_VERSION_1 ((uint8_t)3) |
| 183 | #define UCA_FORMAT_VERSION_2 ((uint8_t)0) |
| 184 | #define UCA_FORMAT_VERSION_3 ((uint8_t)0) |
| 185 | |
| 186 | /* UDataInfo for inverse UCA table */ |
| 187 | /* dataFormat="InvC" */ |
| 188 | #define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49) |
| 189 | #define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E) |
| 190 | #define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76) |
| 191 | #define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43) |
| 192 | |
| 193 | #define INVUCA_FORMAT_VERSION_0 ((uint8_t)2) |
| 194 | #define INVUCA_FORMAT_VERSION_1 ((uint8_t)1) |
| 195 | #define INVUCA_FORMAT_VERSION_2 ((uint8_t)0) |
| 196 | #define INVUCA_FORMAT_VERSION_3 ((uint8_t)0) |
| 197 | |
| 198 | /* This is the size of the stack allocated buffer for sortkey generation and similar operations */ |
| 199 | /* if it is too small, heap allocation will occur.*/ |
| 200 | /* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */ |
| 201 | #define UCOL_MAX_BUFFER 128 |
| 202 | #define UCOL_PRIMARY_MAX_BUFFER 8*UCOL_MAX_BUFFER |
| 203 | #define UCOL_SECONDARY_MAX_BUFFER UCOL_MAX_BUFFER |
| 204 | #define UCOL_TERTIARY_MAX_BUFFER UCOL_MAX_BUFFER |
| 205 | #define UCOL_CASE_MAX_BUFFER UCOL_MAX_BUFFER/4 |
| 206 | #define UCOL_QUAD_MAX_BUFFER 2*UCOL_MAX_BUFFER |
| 207 | |
| 208 | #define UCOL_NORMALIZATION_GROWTH 2 |
| 209 | #define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH |
| 210 | |
| 211 | /* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */ |
| 212 | /* Sometimes we already have a writable buffer (like in case of normalized strings). */ |
| 213 | /* |
| 214 | you can change this value to any value >= 4 if you need memory - |
| 215 | it will affect the performance, though, since we're going to malloc. |
| 216 | Note 3 is the minimum value for Thai collation and 4 is the |
| 217 | minimum number for special Jamo |
| 218 | */ |
| 219 | #define UCOL_WRITABLE_BUFFER_SIZE 256 |
| 220 | |
| 221 | /* This is the size of the buffer for expansion CE's */ |
| 222 | /* In reality we should not have to deal with expm sequences longer then 16 */ |
| 223 | /* you can change this value if you need memory */ |
| 224 | /* WARNING THIS BUFFER DOES HAVE MALLOC FALLBACK. If you make it too small, you'll get into performance trouble */ |
| 225 | /* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */ |
| 226 | /* This is the longest expansion sequence we can handle without bombing out */ |
| 227 | #define UCOL_EXPAND_CE_BUFFER_SIZE 64 |
| 228 | |
| 229 | /* This is the size to increase the buffer for expansion CE's */ |
| 230 | #define UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE 64 |
| 231 | |
| 232 | |
| 233 | /* Unsafe UChar hash table table size. */ |
| 234 | /* size is 32 bytes for 1 bit for each latin 1 char + some power of two for */ |
| 235 | /* hashing the rest of the chars. Size in bytes */ |
| 236 | #define UCOL_UNSAFECP_TABLE_SIZE 1056 |
| 237 | /* mask value down to "some power of two"-1 */ |
| 238 | /* number of bits, not num of bytes. */ |
| 239 | #define UCOL_UNSAFECP_TABLE_MASK 0x1fff |
| 240 | |
| 241 | |
| 242 | /* flags bits for collIterate.flags */ |
| 243 | /* */ |
| 244 | /* NORM - set for incremental normalize of source string */ |
| 245 | #define UCOL_ITER_NORM 1 |
| 246 | |
| 247 | #define UCOL_ITER_HASLEN 2 |
| 248 | |
| 249 | /* UCOL_ITER_INNORMBUF - set if the "pos" is in */ |
| 250 | /* the writable side buffer, handling */ |
| 251 | /* incrementally normalized characters. */ |
| 252 | #define UCOL_ITER_INNORMBUF 4 |
| 253 | |
| 254 | /* UCOL_ITER_ALLOCATED - set if this iterator has */ |
| 255 | /* malloced storage to expand a buffer. */ |
| 256 | #define UCOL_ITER_ALLOCATED 8 |
| 257 | /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana */ |
| 258 | #define UCOL_HIRAGANA_Q 16 |
| 259 | /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */ |
| 260 | /* otherwise set to false */ |
| 261 | #define UCOL_WAS_HIRAGANA 32 |
| 262 | /* UCOL_USE_ITERATOR - set this if collIterate uses a */ |
| 263 | /* character iterator instead of simply accessing string */ |
| 264 | /* by index */ |
| 265 | #define UCOL_USE_ITERATOR 64 |
| 266 | |
| 267 | #define UCOL_FORCE_HAN_IMPLICIT 128 |
| 268 | |
| 269 | #define NFC_ZERO_CC_BLOCK_LIMIT_ 0x300 |
| 270 | |
| 271 | #ifdef XP_CPLUSPLUS |
| 272 | |
| 273 | typedef struct collIterate : public U_NAMESPACE_QUALIFIER UMemory { |
| 274 | const UChar *string; /* Original string */ |
| 275 | /* UChar *start; Pointer to the start of the source string. Either points to string |
| 276 | or to writableBuffer */ |
| 277 | const UChar *endp; /* string end ptr. Is undefined for null terminated strings */ |
| 278 | const UChar *pos; /* This is position in the string. Can be to original or writable buf */ |
| 279 | |
| 280 | uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */ |
| 281 | uint32_t *CEpos; /* This is the position to which we have stored processed CEs */ |
| 282 | |
| 283 | int32_t *offsetReturn; /* This is the offset to return, if non-NULL */ |
| 284 | int32_t *offsetStore; /* This is the pointer for storing offsets */ |
| 285 | int32_t offsetRepeatCount; /* Repeat stored offset if non-zero */ |
| 286 | int32_t offsetRepeatValue; /* offset value to repeat */ |
| 287 | |
| 288 | U_NAMESPACE_QUALIFIER UnicodeString writableBuffer; |
| 289 | const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */ |
| 290 | const UCollator *coll; |
| 291 | const U_NAMESPACE_QUALIFIER Normalizer2 *nfd; |
| 292 | uint8_t flags; |
| 293 | uint8_t origFlags; |
| 294 | uint32_t *extendCEs; /* This is use if CEs is not big enough */ |
| 295 | int32_t extendCEsSize; /* Holds the size of the dynamic CEs buffer */ |
| 296 | uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */ |
| 297 | |
| 298 | int32_t *offsetBuffer; /* A dynamic buffer to hold offsets */ |
| 299 | int32_t offsetBufferSize; /* The size of the offset buffer */ |
| 300 | |
| 301 | UCharIterator *iterator; |
| 302 | /*int32_t iteratorIndex;*/ |
| 303 | } collIterate; |
| 304 | |
| 305 | #else |
| 306 | |
| 307 | typedef struct collIterate collIterate; |
| 308 | |
| 309 | #endif |
| 310 | |
| 311 | #define paddedsize(something) ((something)+((((something)%4)!=0)?(4-(something)%4):0)) |
| 312 | #define headersize (paddedsize(sizeof(UCATableHeader))+paddedsize(sizeof(UColOptionSet))) |
| 313 | |
| 314 | /* |
| 315 | struct used internally in getSpecial*CE. |
| 316 | data similar to collIterate. |
| 317 | */ |
| 318 | struct collIterateState { |
| 319 | const UChar *pos; /* This is position in the string. Can be to original or writable buf */ |
| 320 | const UChar *returnPos; |
| 321 | const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */ |
| 322 | const UChar *bufferaddress; /* address of the normalization buffer */ |
| 323 | int32_t buffersize; |
| 324 | uint8_t flags; |
| 325 | uint8_t origFlags; |
| 326 | uint32_t iteratorIndex; |
| 327 | int32_t iteratorMove; |
| 328 | }; |
| 329 | |
| 330 | U_CAPI void U_EXPORT2 |
| 331 | uprv_init_collIterate(const UCollator *collator, |
| 332 | const UChar *sourceString, int32_t sourceLen, |
| 333 | collIterate *s, UErrorCode *status); |
| 334 | |
| 335 | /* Internal functions for C test code. */ |
| 336 | U_CAPI collIterate * U_EXPORT2 |
| 337 | uprv_new_collIterate(UErrorCode *status); |
| 338 | |
| 339 | U_CAPI void U_EXPORT2 |
| 340 | uprv_delete_collIterate(collIterate *s); |
| 341 | |
| 342 | /* @return s->pos == s->endp */ |
| 343 | U_CAPI UBool U_EXPORT2 |
| 344 | uprv_collIterateAtEnd(collIterate *s); |
| 345 | |
| 346 | #ifdef XP_CPLUSPLUS |
| 347 | |
| 348 | U_NAMESPACE_BEGIN |
| 349 | |
| 350 | struct UCollationPCE; |
| 351 | typedef struct UCollationPCE UCollationPCE; |
| 352 | |
| 353 | U_NAMESPACE_END |
| 354 | |
| 355 | struct UCollationElements : public U_NAMESPACE_QUALIFIER UMemory |
| 356 | { |
| 357 | /** |
| 358 | * Struct wrapper for source data |
| 359 | */ |
| 360 | collIterate iteratordata_; |
| 361 | /** |
| 362 | * Indicates if this data has been reset. |
| 363 | */ |
| 364 | UBool reset_; |
| 365 | /** |
| 366 | * Indicates if the data should be deleted. |
| 367 | */ |
| 368 | UBool isWritable; |
| 369 | |
| 370 | /** |
| 371 | * Data for getNextProcessed, getPreviousProcessed. |
| 372 | */ |
| 373 | U_NAMESPACE_QUALIFIER UCollationPCE *pce; |
| 374 | }; |
| 375 | |
| 376 | #endif |
| 377 | |
| 378 | U_CAPI void U_EXPORT2 |
| 379 | uprv_init_pce(const struct UCollationElements *elems); |
| 380 | |
| 381 | #define UCOL_LEVELTERMINATOR 1 |
| 382 | |
| 383 | /* mask off anything but primary order */ |
| 384 | #define UCOL_PRIMARYORDERMASK 0xffff0000 |
| 385 | /* mask off anything but secondary order */ |
| 386 | #define UCOL_SECONDARYORDERMASK 0x0000ff00 |
| 387 | /* mask off anything but tertiary order */ |
| 388 | #define UCOL_TERTIARYORDERMASK 0x000000ff |
| 389 | /* primary order shift */ |
| 390 | #define UCOL_PRIMARYORDERSHIFT 16 |
| 391 | /* secondary order shift */ |
| 392 | #define UCOL_SECONDARYORDERSHIFT 8 |
| 393 | |
| 394 | #define UCOL_BYTE_SIZE_MASK 0xFF |
| 395 | |
| 396 | #define UCOL_CASE_BYTE_START 0x80 |
| 397 | #define UCOL_CASE_SHIFT_START 7 |
| 398 | |
| 399 | #define UCOL_IGNORABLE 0 |
| 400 | |
| 401 | /* get weights from a CE */ |
| 402 | #define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT) |
| 403 | #define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT) |
| 404 | #define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK) |
| 405 | |
| 406 | /** |
| 407 | * Determine if a character is a Thai vowel (which sorts after |
| 408 | * its base consonant). |
| 409 | */ |
| 410 | #define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \ |
| 411 | (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0))) |
| 412 | |
| 413 | /** |
| 414 | * Determine if a character is a Thai base consonant |
| 415 | */ |
| 416 | #define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01) |
| 417 | |
| 418 | #define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \ |
| 419 | (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \ |
| 420 | (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8))) |
| 421 | |
| 422 | /* Han character ranges */ |
| 423 | #define UCOL_FIRST_HAN 0x4E00 |
| 424 | #define UCOL_LAST_HAN 0x9FFF |
| 425 | #define UCOL_FIRST_HAN_A 0x3400 |
| 426 | #define UCOL_LAST_HAN_A 0x4DBF |
| 427 | #define UCOL_FIRST_HAN_COMPAT 0xFAE0 |
| 428 | #define UCOL_LAST_HAN_COMPAT 0xFA2F |
| 429 | |
| 430 | /* Han extension B is in plane 2 */ |
| 431 | #define UCOL_FIRST_HAN_B 0x20000 |
| 432 | #define UCOL_LAST_HAN_B 0x2A6DF |
| 433 | |
| 434 | /* Hangul range */ |
| 435 | #define UCOL_FIRST_HANGUL 0xAC00 |
| 436 | #define UCOL_LAST_HANGUL 0xD7AF |
| 437 | |
| 438 | /* Jamo ranges */ |
| 439 | #define UCOL_FIRST_L_JAMO 0x1100 |
| 440 | #define UCOL_FIRST_V_JAMO 0x1161 |
| 441 | #define UCOL_FIRST_T_JAMO 0x11A8 |
| 442 | #define UCOL_LAST_T_JAMO 0x11F9 |
| 443 | |
| 444 | |
| 445 | #if 0 |
| 446 | /* initializes collIterate structure */ |
| 447 | /* made as macro to speed up things */ |
| 448 | #define init_collIterate(collator, sourceString, sourceLen, s) { \ |
| 449 | (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \ |
| 450 | (s)->endp = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \ |
| 451 | (s)->CEpos = (s)->toReturn = (s)->CEs; \ |
| 452 | (s)->isThai = TRUE; \ |
| 453 | (s)->writableBuffer = (s)->stackWritableBuffer; \ |
| 454 | (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \ |
| 455 | (s)->coll = (collator); \ |
| 456 | (s)->fcdPosition = 0; \ |
| 457 | (s)->flags = 0; \ |
| 458 | if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \ |
| 459 | } |
| 460 | #endif |
| 461 | |
| 462 | |
| 463 | |
| 464 | /* |
| 465 | * Macro to get the maximum size of an expansion ending with the argument ce. |
| 466 | * Used in the Boyer Moore algorithm. |
| 467 | * Note for tailoring, the UCA maxexpansion table has been merged. |
| 468 | * Hence we only have to search the tailored collator only. |
| 469 | * @param coll const UCollator pointer |
| 470 | * @param order last collation element of the expansion sequence |
| 471 | * @param result size of the longest expansion with argument collation element |
| 472 | * as the last element |
| 473 | */ |
| 474 | #define UCOL_GETMAXEXPANSION(coll, order, result) { \ |
| 475 | const uint32_t *start; \ |
| 476 | const uint32_t *limit; \ |
| 477 | const uint32_t *mid; \ |
| 478 | start = (coll)->endExpansionCE; \ |
| 479 | limit = (coll)->lastEndExpansionCE; \ |
| 480 | while (start < limit - 1) { \ |
| 481 | mid = start + ((limit - start) >> 1); \ |
| 482 | if ((order) <= *mid) { \ |
| 483 | limit = mid; \ |
| 484 | } \ |
| 485 | else { \ |
| 486 | start = mid; \ |
| 487 | } \ |
| 488 | } \ |
| 489 | if (*start == order) { \ |
| 490 | result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE)); \ |
| 491 | } \ |
| 492 | else if (*limit == order) { \ |
| 493 | result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \ |
| 494 | } \ |
| 495 | else if ((order & 0xFFFF) == 0x00C0) { \ |
| 496 | result = 2; \ |
| 497 | } \ |
| 498 | else { \ |
| 499 | result = 1; \ |
| 500 | } \ |
| 501 | } |
| 502 | |
| 503 | U_CFUNC |
| 504 | uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE, collIterate *source, UErrorCode *status); |
| 505 | |
| 506 | U_CFUNC |
| 507 | uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE, |
| 508 | collIterate *source, UErrorCode *status); |
| 509 | U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll, collIterate *collationSource, UErrorCode *status); |
| 510 | U_CFUNC uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll, |
| 511 | collIterate *collationSource, |
| 512 | UErrorCode *status); |
| 513 | /* function used by C++ getCollationKey to prevent restarting the calculation */ |
| 514 | U_CFUNC int32_t |
| 515 | ucol_getSortKeyWithAllocation(const UCollator *coll, |
| 516 | const UChar *source, int32_t sourceLength, |
| 517 | uint8_t **pResult, |
| 518 | UErrorCode *pErrorCode); |
| 519 | |
| 520 | /* get some memory */ |
| 521 | void *ucol_getABuffer(const UCollator *coll, uint32_t size); |
| 522 | |
| 523 | /* worker function for generating sortkeys */ |
| 524 | U_CFUNC |
| 525 | int32_t U_CALLCONV |
| 526 | ucol_calcSortKey(const UCollator *coll, |
| 527 | const UChar *source, |
| 528 | int32_t sourceLength, |
| 529 | uint8_t **result, |
| 530 | uint32_t resultLength, |
| 531 | UBool allocatePrimary, |
| 532 | UErrorCode *status); |
| 533 | |
| 534 | U_CFUNC |
| 535 | int32_t U_CALLCONV |
| 536 | ucol_calcSortKeySimpleTertiary(const UCollator *coll, |
| 537 | const UChar *source, |
| 538 | int32_t sourceLength, |
| 539 | uint8_t **result, |
| 540 | uint32_t resultLength, |
| 541 | UBool allocatePrimary, |
| 542 | UErrorCode *status); |
| 543 | |
| 544 | U_CFUNC |
| 545 | int32_t |
| 546 | ucol_getSortKeySize(const UCollator *coll, collIterate *s, |
| 547 | int32_t currentSize, UColAttributeValue strength, |
| 548 | int32_t len); |
| 549 | /** |
| 550 | * Makes a copy of the Collator's rule data. The format is |
| 551 | * that of .col files. |
| 552 | * |
| 553 | * @param length returns the length of the data, in bytes. |
| 554 | * @param status the error status |
| 555 | * @return memory, owned by the caller, of size 'length' bytes. |
| 556 | * @internal INTERNAL USE ONLY |
| 557 | */ |
| 558 | U_CFUNC uint8_t* U_EXPORT2 |
| 559 | ucol_cloneRuleData(const UCollator *coll, int32_t *length, UErrorCode *status); |
| 560 | |
| 561 | /** |
| 562 | * Used to set requested and valid locales on a collator returned by the collator |
| 563 | * service. |
| 564 | */ |
| 565 | U_CFUNC void U_EXPORT2 |
| 566 | ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt, char *actualLocaleToAdopt); |
| 567 | |
| 568 | #define UCOL_SPECIAL_FLAG 0xF0000000 |
| 569 | #define UCOL_TAG_SHIFT 24 |
| 570 | #define UCOL_TAG_MASK 0x0F000000 |
| 571 | #define INIT_EXP_TABLE_SIZE 1024 |
| 572 | #define UCOL_NOT_FOUND 0xF0000000 |
| 573 | #define UCOL_EXPANSION 0xF1000000 |
| 574 | #define UCOL_CONTRACTION 0xF2000000 |
| 575 | #define UCOL_THAI 0xF3000000 |
| 576 | #define UCOL_UNMARKED 0x03 |
| 577 | #define UCOL_NEW_TERTIARYORDERMASK 0x0000003f |
| 578 | |
| 579 | /* Bit mask for primary collation strength. */ |
| 580 | #define UCOL_PRIMARYMASK 0xFFFF0000 |
| 581 | |
| 582 | /* Bit mask for secondary collation strength. */ |
| 583 | #define UCOL_SECONDARYMASK 0x0000FF00 |
| 584 | |
| 585 | /* Bit mask for tertiary collation strength. */ |
| 586 | #define UCOL_TERTIARYMASK 0x000000FF |
| 587 | |
| 588 | /** |
| 589 | * Internal. |
| 590 | * This indicates the last element in a UCollationElements has been consumed. |
| 591 | * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs. |
| 592 | */ |
| 593 | #define UCOL_NO_MORE_CES 0x00010101 |
| 594 | #define UCOL_NO_MORE_CES_PRIMARY 0x00010000 |
| 595 | #define UCOL_NO_MORE_CES_SECONDARY 0x00000100 |
| 596 | #define UCOL_NO_MORE_CES_TERTIARY 0x00000001 |
| 597 | |
| 598 | #define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF) |
| 599 | |
| 600 | #define UCOL_UPPER_CASE 0x80 |
| 601 | #define UCOL_MIXED_CASE 0x40 |
| 602 | #define UCOL_LOWER_CASE 0x00 |
| 603 | |
| 604 | #define UCOL_CONTINUATION_MARKER 0xC0 |
| 605 | #define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F |
| 606 | |
| 607 | #define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER) |
| 608 | #define isFlagged(CE) (((CE) & 0x80) == 0x80) |
| 609 | #define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0) |
| 610 | |
| 611 | #define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT) |
| 612 | #define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG)) |
| 613 | #define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG)) |
| 614 | #define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF) |
| 615 | #define constructSpecProcCE(CE) (UCOL_SPECIAL_FLAG | (SPEC_PROC_TAG<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF) |
| 616 | #define getContractOffset(CE) ((CE)&0xFFFFFF) |
| 617 | #define getExpansionOffset(CE) (((CE)&0x00FFFFF0)>>4) |
| 618 | #define getExpansionCount(CE) ((CE)&0xF) |
| 619 | #define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0) |
| 620 | |
| 621 | /* StringSearch internal use */ |
| 622 | #define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF) |
| 623 | #define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL) |
| 624 | #define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs) |
| 625 | #define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs) |
| 626 | #define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset) |
| 627 | #define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn) |
| 628 | #define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces) |
| 629 | |
| 630 | /* This is an enum that lists magic special byte values from the fractional UCA */ |
| 631 | /* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */ |
| 632 | |
| 633 | enum { |
| 634 | UCOL_BYTE_ZERO = 0x00, |
| 635 | UCOL_BYTE_LEVEL_SEPARATOR = 0x01, |
| 636 | UCOL_BYTE_SORTKEY_GLUE = 0x02, |
| 637 | UCOL_BYTE_SHIFT_PREFIX = 0x03, |
| 638 | UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX, |
| 639 | UCOL_BYTE_FIRST_TAILORED = 0x04, |
| 640 | UCOL_BYTE_COMMON = 0x05, |
| 641 | UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON, |
| 642 | UCOL_CODAN_PLACEHOLDER = 0x27, |
| 643 | UCOL_BYTE_LAST_LATIN_PRIMARY = 0x4C, |
| 644 | UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x4D, |
| 645 | UCOL_BYTE_UNSHIFTED_MAX = 0xFF |
| 646 | }; |
| 647 | |
| 648 | #if 0 |
| 649 | #define UCOL_RESET_TOP_VALUE 0x9F000303 |
| 650 | #define UCOL_FIRST_PRIMARY_IGNORABLE 0x00008705 |
| 651 | #define UCOL_LAST_PRIMARY_IGNORABLE 0x0000DD05 |
| 652 | #define UCOL_LAST_PRIMARY_IGNORABLE_CONT 0x000051C0 |
| 653 | #define UCOL_FIRST_SECONDARY_IGNORABLE 0x00000000 |
| 654 | #define UCOL_LAST_SECONDARY_IGNORABLE 0x00000500 |
| 655 | #define UCOL_FIRST_TERTIARY_IGNORABLE 0x00000000 |
| 656 | #define UCOL_LAST_TERTIARY_IGNORABLE 0x00000000 |
| 657 | #define UCOL_FIRST_VARIABLE 0x05070505 |
| 658 | #define UCOL_LAST_VARIABLE 0x179B0505 |
| 659 | #define UCOL_FIRST_NON_VARIABLE 0x1A200505 |
| 660 | #define UCOL_LAST_NON_VARIABLE 0x7B41058F |
| 661 | |
| 662 | #define UCOL_NEXT_TOP_VALUE 0xE8960303 |
| 663 | #define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE 0x00008905 |
| 664 | #define UCOL_NEXT_LAST_PRIMARY_IGNORABLE 0x03000303 |
| 665 | #define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705 |
| 666 | #define UCOL_NEXT_LAST_SECONDARY_IGNORABLE 0x00000500 |
| 667 | #define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE 0x00000000 |
| 668 | #define UCOL_NEXT_LAST_TERTIARY_IGNORABLE 0x00000000 |
| 669 | #define UCOL_NEXT_FIRST_VARIABLE 0x05090505 |
| 670 | #define UCOL_NEXT_LAST_VARIABLE 0x1A200505 |
| 671 | |
| 672 | #define PRIMARY_IMPLICIT_MIN 0xE8000000 |
| 673 | #define PRIMARY_IMPLICIT_MAX 0xF0000000 |
| 674 | #endif |
| 675 | |
| 676 | /* These constants can be changed - sortkey size is affected by them */ |
| 677 | #define UCOL_PROPORTION2 0.5 |
| 678 | #define UCOL_PROPORTION3 0.667 |
| 679 | |
| 680 | /* These values come from the UCA */ |
| 681 | #define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON |
| 682 | #define UCOL_COMMON_TOP2 0x86u |
| 683 | #define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1) |
| 684 | |
| 685 | #define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80 |
| 686 | #define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40 |
| 687 | #define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85 |
| 688 | #define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45 |
| 689 | #define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5 |
| 690 | |
| 691 | /* These values come from the UCA */ |
| 692 | #define UCOL_COMMON_BOT3 0x05 |
| 693 | |
| 694 | #define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86; |
| 695 | #define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3; |
| 696 | |
| 697 | #define UCOL_TOP_COUNT2 (UCOL_PROPORTION2*UCOL_TOTAL2) |
| 698 | #define UCOL_BOT_COUNT2 (UCOL_TOTAL2-UCOL_TOP_COUNT2) |
| 699 | |
| 700 | |
| 701 | #define UCOL_COMMON2 UCOL_COMMON_BOT2 |
| 702 | #define UCOL_COMMON3_UPPERFIRST 0xC5 |
| 703 | #define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3 |
| 704 | |
| 705 | #define UCOL_COMMON4 0xFF |
| 706 | |
| 707 | /* constants for case level/case first handling */ |
| 708 | /* used to instantiate UCollators fields in ucol_updateInternalState */ |
| 709 | #define UCOL_CASE_SWITCH 0xC0 |
| 710 | #define UCOL_NO_CASE_SWITCH 0x00 |
| 711 | |
| 712 | #define UCOL_REMOVE_CASE 0x3F |
| 713 | #define UCOL_KEEP_CASE 0xFF |
| 714 | |
| 715 | #define UCOL_CASE_BIT_MASK 0xC0 |
| 716 | |
| 717 | #define UCOL_TERT_CASE_MASK 0xFF |
| 718 | |
| 719 | #define UCOL_ENDOFLATINONERANGE 0xFF |
| 720 | #define UCOL_LATINONETABLELEN (UCOL_ENDOFLATINONERANGE+50) |
| 721 | #define UCOL_BAIL_OUT_CE 0xFF000000 |
| 722 | |
| 723 | |
| 724 | typedef enum { |
| 725 | NOT_FOUND_TAG = 0, |
| 726 | EXPANSION_TAG = 1, /* This code point results in an expansion */ |
| 727 | CONTRACTION_TAG = 2, /* Start of a contraction */ |
| 728 | THAI_TAG = 3, /* Thai character - do the reordering */ |
| 729 | CHARSET_TAG = 4, /* Charset processing, not yet implemented */ |
| 730 | SURROGATE_TAG = 5, /* Lead surrogate that is tailored and doesn't start a contraction */ |
| 731 | HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/ |
| 732 | LEAD_SURROGATE_TAG = 7, /* D800-DBFF*/ |
| 733 | TRAIL_SURROGATE_TAG = 8, /* DC00-DFFF*/ |
| 734 | CJK_IMPLICIT_TAG = 9, /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/ |
| 735 | IMPLICIT_TAG = 10, |
| 736 | SPEC_PROC_TAG = 11, |
| 737 | /* ICU 2.1 */ |
| 738 | LONG_PRIMARY_TAG = 12, /* This is a three byte primary with starting secondaries and tertiaries */ |
| 739 | /* It fits in a single 32 bit CE and is used instead of expansion to save */ |
| 740 | /* space without affecting the performance (hopefully) */ |
| 741 | |
| 742 | DIGIT_TAG = 13, /* COllate Digits As Numbers (CODAN) implementation */ |
| 743 | |
| 744 | CE_TAGS_COUNT |
| 745 | } UColCETags; |
| 746 | |
| 747 | /* |
| 748 | ***************************************************************************************** |
| 749 | * set to zero |
| 750 | * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF |
| 751 | ****************************************************************************************** |
| 752 | */ |
| 753 | |
| 754 | typedef struct { |
| 755 | uint32_t variableTopValue; |
| 756 | /*UColAttributeValue*/ int32_t frenchCollation; |
| 757 | /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/ |
| 758 | /*UColAttributeValue*/ int32_t caseFirst; /* who goes first, lower case or uppercase */ |
| 759 | /*UColAttributeValue*/ int32_t caseLevel; /* do we have an extra case level */ |
| 760 | /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */ |
| 761 | /*UColAttributeValue*/ int32_t strength; /* attribute for strength */ |
| 762 | /*UColAttributeValue*/ int32_t hiraganaQ; /* attribute for special Hiragana */ |
| 763 | /*UColAttributeValue*/ int32_t numericCollation; /* attribute for numeric collation */ |
| 764 | uint32_t reserved[15]; /* for future use */ |
| 765 | } UColOptionSet; |
| 766 | |
| 767 | typedef struct { |
| 768 | uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2]; /*0x00000000*/ |
| 769 | uint32_t UCA_LAST_TERTIARY_IGNORABLE[2]; /*0x00000000*/ |
| 770 | uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2]; /*0x00008705*/ |
| 771 | uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2]; /*0x00000000*/ |
| 772 | uint32_t UCA_LAST_SECONDARY_IGNORABLE[2]; /*0x00000500*/ |
| 773 | uint32_t UCA_LAST_PRIMARY_IGNORABLE[2]; /*0x0000DD05*/ |
| 774 | uint32_t UCA_FIRST_VARIABLE[2]; /*0x05070505*/ |
| 775 | uint32_t UCA_LAST_VARIABLE[2]; /*0x13CF0505*/ |
| 776 | uint32_t UCA_FIRST_NON_VARIABLE[2]; /*0x16200505*/ |
| 777 | uint32_t UCA_LAST_NON_VARIABLE[2]; /*0x767C0505*/ |
| 778 | uint32_t UCA_RESET_TOP_VALUE[2]; /*0x9F000303*/ |
| 779 | uint32_t UCA_FIRST_IMPLICIT[2]; |
| 780 | uint32_t UCA_LAST_IMPLICIT[2]; |
| 781 | uint32_t UCA_FIRST_TRAILING[2]; |
| 782 | uint32_t UCA_LAST_TRAILING[2]; |
| 783 | |
| 784 | #if 0 |
| 785 | uint32_t UCA_NEXT_TOP_VALUE[2]; /*0xE8960303*/ |
| 786 | uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE; /*0x00008905*/ |
| 787 | uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE; /*0x03000303*/ |
| 788 | uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/ |
| 789 | uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE; /*0x00000500*/ |
| 790 | uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE; /*0x00000000*/ |
| 791 | uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE; /*0x00000000*/ |
| 792 | uint32_t UCA_NEXT_FIRST_VARIABLE; /*0x05090505*/ |
| 793 | uint32_t UCA_NEXT_LAST_VARIABLE; /*0x16200505*/ |
| 794 | #endif |
| 795 | |
| 796 | uint32_t UCA_PRIMARY_TOP_MIN; |
| 797 | uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/ |
| 798 | uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/ |
| 799 | uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/ |
| 800 | uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/ |
| 801 | uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/ |
| 802 | uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/ |
| 803 | } UCAConstants; |
| 804 | |
| 805 | typedef struct { |
| 806 | int32_t size; |
| 807 | /* all the offsets are in bytes */ |
| 808 | /* to get the address add to the header address and cast properly */ |
| 809 | uint32_t options; /* these are the default options for the collator */ |
| 810 | uint32_t UCAConsts; /* structure which holds values for indirect positioning and implicit ranges */ |
| 811 | uint32_t contractionUCACombos; /* this one is needed only for UCA, to copy the appropriate contractions */ |
| 812 | uint32_t magic; /* magic number - lets us know whether reserved data is reset or junked */ |
| 813 | uint32_t mappingPosition; /* const uint8_t *mappingPosition; */ |
| 814 | uint32_t expansion; /* uint32_t *expansion; */ |
| 815 | uint32_t contractionIndex; /* UChar *contractionIndex; */ |
| 816 | uint32_t contractionCEs; /* uint32_t *contractionCEs; */ |
| 817 | uint32_t contractionSize; /* needed for various closures */ |
| 818 | /*int32_t latinOneMapping;*/ /* this is now handled in the trie itself *//* fast track to latin1 chars */ |
| 819 | |
| 820 | uint32_t endExpansionCE; /* array of last collation element in |
| 821 | expansion */ |
| 822 | uint32_t expansionCESize; /* array of maximum expansion size |
| 823 | corresponding to the expansion |
| 824 | collation elements with last element |
| 825 | in endExpansionCE*/ |
| 826 | int32_t endExpansionCECount; /* size of endExpansionCE */ |
| 827 | uint32_t unsafeCP; /* hash table of unsafe code points */ |
| 828 | uint32_t contrEndCP; /* hash table of final code points */ |
| 829 | /* in contractions. */ |
| 830 | |
| 831 | int32_t contractionUCACombosSize; /* number of UCA contraction items. */ |
| 832 | /*Length is contractionUCACombosSize*contractionUCACombosWidth*sizeof(UChar) */ |
| 833 | UBool jamoSpecial; /* is jamoSpecial */ |
| 834 | UBool isBigEndian; /* is this data big endian? from the UDataInfo header*/ |
| 835 | uint8_t charSetFamily; /* what is the charset family of this data from the UDataInfo header*/ |
| 836 | uint8_t contractionUCACombosWidth; /* width of UCA combos field */ |
| 837 | UVersionInfo version; |
| 838 | UVersionInfo UCAVersion; /* version of the UCA, read from file */ |
| 839 | UVersionInfo UCDVersion; /* UCD version, obtained by u_getUnicodeVersion */ |
| 840 | UVersionInfo formatVersion; /* format version from the UDataInfo header */ |
| 841 | uint8_t reserved[84]; /* for future use */ |
| 842 | } UCATableHeader; |
| 843 | |
| 844 | #define U_UNKNOWN_STATE 0 |
| 845 | #define U_COLLATOR_STATE 0x01 |
| 846 | #define U_STATE_LIMIT 0x02 |
| 847 | |
| 848 | /* This is the first structure in a state */ |
| 849 | /* it should be machine independent */ |
| 850 | typedef struct { |
| 851 | /* this structure is supposed to be readable on all the platforms.*/ |
| 852 | /* first 2 fields hold the size of the structure in a platform independent way */ |
| 853 | uint8_t sizeLo; |
| 854 | uint8_t sizeHi; |
| 855 | /* identifying the writing platform */ |
| 856 | uint8_t isBigEndian; |
| 857 | /* see U_CHARSET_FAMILY values in utypes.h */ |
| 858 | uint8_t charsetFamily; |
| 859 | /* version of ICU this state structure comes from */ |
| 860 | uint8_t icuVersion[4]; |
| 861 | /* What is the data following this state */ |
| 862 | uint8_t type; |
| 863 | /* more stuff to come, keep it on 16 byte boundary */ |
| 864 | uint8_t reserved[7]; |
| 865 | } UStateStruct; |
| 866 | |
| 867 | /* This structure follows UStatusStruct */ |
| 868 | /* and contains data specific for the collators */ |
| 869 | /* Endianess needs to be decided before accessing this structure */ |
| 870 | /* However, it's size IS endianess independent */ |
| 871 | typedef struct { |
| 872 | /* size of this structure */ |
| 873 | uint8_t sizeLo; |
| 874 | uint8_t sizeHi; |
| 875 | /* This state is followed by the frozen tailoring */ |
| 876 | uint8_t containsTailoring; |
| 877 | /* This state is followed by the frozen UCA */ |
| 878 | uint8_t containsUCA; |
| 879 | /* Version info - the same one */ |
| 880 | uint8_t versionInfo[4]; |
| 881 | |
| 882 | /* for charset CEs */ |
| 883 | uint8_t charsetName[32]; |
| 884 | /* this is the resolved locale name*/ |
| 885 | uint8_t locale[32]; |
| 886 | |
| 887 | /* Attributes. Open ended */ |
| 888 | /* all the following will be moved to uint32_t because of portability */ |
| 889 | /* variable top value */ |
| 890 | uint32_t variableTopValue; |
| 891 | /* attribute for handling variable elements*/ |
| 892 | uint32_t /*UColAttributeValue*/ alternateHandling; |
| 893 | /* how to handle secondary weights */ |
| 894 | uint32_t /*UColAttributeValue*/ frenchCollation; |
| 895 | /* who goes first, lower case or uppercase */ |
| 896 | uint32_t /*UColAttributeValue*/ caseFirst; |
| 897 | /* do we have an extra case level */ |
| 898 | uint32_t /*UColAttributeValue*/ caseLevel; |
| 899 | /* attribute for normalization */ |
| 900 | uint32_t /*UColAttributeValue*/ normalizationMode; |
| 901 | /* attribute for strength */ |
| 902 | uint32_t /*UColAttributeValue*/ strength; |
| 903 | /* to be immediately 16 byte aligned */ |
| 904 | uint8_t reserved[12]; |
| 905 | } UColStateStruct; |
| 906 | |
| 907 | #define UCOL_INV_SIZEMASK 0xFFF00000 |
| 908 | #define UCOL_INV_OFFSETMASK 0x000FFFFF |
| 909 | #define UCOL_INV_SHIFTVALUE 20 |
| 910 | |
| 911 | U_CDECL_BEGIN |
| 912 | |
| 913 | typedef struct { |
| 914 | uint32_t byteSize; |
| 915 | uint32_t tableSize; |
| 916 | uint32_t contsSize; |
| 917 | uint32_t table; |
| 918 | uint32_t conts; |
| 919 | UVersionInfo UCAVersion; /* version of the UCA, read from file */ |
| 920 | uint8_t padding[8]; |
| 921 | } InverseUCATableHeader; |
| 922 | |
| 923 | typedef int32_t U_CALLCONV |
| 924 | SortKeyGenerator(const UCollator *coll, |
| 925 | const UChar *source, |
| 926 | int32_t sourceLength, |
| 927 | uint8_t **result, |
| 928 | uint32_t resultLength, |
| 929 | UBool allocatePrimary, |
| 930 | UErrorCode *status); |
| 931 | |
| 932 | typedef void U_CALLCONV |
| 933 | ResourceCleaner(UCollator *coll); |
| 934 | |
| 935 | |
| 936 | struct UCollator { |
| 937 | UColOptionSet *options; |
| 938 | SortKeyGenerator *sortKeyGen; |
| 939 | uint32_t *latinOneCEs; |
| 940 | char* actualLocale; |
| 941 | char* validLocale; |
| 942 | char* requestedLocale; |
| 943 | const UChar *rules; |
| 944 | const UChar *ucaRules; |
| 945 | const UCollator *UCA; |
| 946 | const UCATableHeader *image; |
| 947 | UTrie mapping; |
| 948 | const uint32_t *latinOneMapping; |
| 949 | const uint32_t *expansion; |
| 950 | const UChar *contractionIndex; |
| 951 | const uint32_t *contractionCEs; |
| 952 | /*const uint8_t *scriptOrder;*/ |
| 953 | |
| 954 | const uint32_t *endExpansionCE; /* array of last ces in an expansion ce. |
| 955 | corresponds to expansionCESize */ |
| 956 | const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */ |
| 957 | const uint8_t *expansionCESize; /* array of the maximum size of a |
| 958 | expansion ce with the last ce |
| 959 | corresponding to endExpansionCE, |
| 960 | terminated with a null */ |
| 961 | const uint8_t *unsafeCP; /* unsafe code points hashtable */ |
| 962 | const uint8_t *contrEndCP; /* Contraction ending chars hash table */ |
| 963 | UChar minUnsafeCP; /* Smallest unsafe Code Point. */ |
| 964 | UChar minContrEndCP; /* Smallest code point at end of a contraction */ |
| 965 | |
| 966 | int32_t rulesLength; |
| 967 | int32_t latinOneTableLen; |
| 968 | |
| 969 | uint32_t variableTopValue; |
| 970 | UColAttributeValue frenchCollation; |
| 971 | UColAttributeValue alternateHandling; /* attribute for handling variable elements*/ |
| 972 | UColAttributeValue caseFirst; /* who goes first, lower case or uppercase */ |
| 973 | UColAttributeValue caseLevel; /* do we have an extra case level */ |
| 974 | UColAttributeValue normalizationMode; /* attribute for normalization */ |
| 975 | UColAttributeValue strength; /* attribute for strength */ |
| 976 | UColAttributeValue hiraganaQ; /* attribute for Hiragana */ |
| 977 | UColAttributeValue numericCollation; |
| 978 | UBool variableTopValueisDefault; |
| 979 | UBool frenchCollationisDefault; |
| 980 | UBool alternateHandlingisDefault; /* attribute for handling variable elements*/ |
| 981 | UBool caseFirstisDefault; /* who goes first, lower case or uppercase */ |
| 982 | UBool caseLevelisDefault; /* do we have an extra case level */ |
| 983 | UBool normalizationModeisDefault; /* attribute for normalization */ |
| 984 | UBool strengthisDefault; /* attribute for strength */ |
| 985 | UBool hiraganaQisDefault; /* attribute for Hiragana */ |
| 986 | UBool numericCollationisDefault; |
| 987 | UBool hasRealData; /* some collators have only options, like French, no rules */ |
| 988 | /* to speed up things, we use the UCA image, but we don't want it */ |
| 989 | /* to run around */ |
| 990 | |
| 991 | UBool freeOnClose; |
| 992 | UBool freeOptionsOnClose; |
| 993 | UBool freeRulesOnClose; |
| 994 | UBool freeImageOnClose; |
| 995 | |
| 996 | UBool latinOneUse; |
| 997 | UBool latinOneRegenTable; |
| 998 | UBool latinOneFailed; |
| 999 | |
| 1000 | int8_t tertiaryAddition; /* when switching case, we need to add or subtract different values */ |
| 1001 | uint8_t caseSwitch; |
| 1002 | uint8_t tertiaryCommon; |
| 1003 | uint8_t tertiaryMask; |
| 1004 | uint8_t tertiaryTop; /* Upper range when compressing */ |
| 1005 | uint8_t tertiaryBottom; /* Upper range when compressing */ |
| 1006 | uint8_t tertiaryTopCount; |
| 1007 | uint8_t tertiaryBottomCount; |
| 1008 | |
| 1009 | UVersionInfo dataVersion; /* Data info of UCA table */ |
| 1010 | }; |
| 1011 | |
| 1012 | U_CDECL_END |
| 1013 | |
| 1014 | /* various internal functions */ |
| 1015 | |
| 1016 | /* do not close UCA returned by ucol_initUCA! */ |
| 1017 | U_CFUNC |
| 1018 | UCollator* ucol_initUCA(UErrorCode *status); |
| 1019 | |
| 1020 | U_CFUNC |
| 1021 | UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status); |
| 1022 | |
| 1023 | U_CFUNC |
| 1024 | void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status); |
| 1025 | |
| 1026 | U_CFUNC |
| 1027 | UCollator* ucol_open_internal(const char* loc, UErrorCode* status); |
| 1028 | |
| 1029 | #if 0 |
| 1030 | U_CFUNC |
| 1031 | void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status); |
| 1032 | #endif |
| 1033 | |
| 1034 | U_CFUNC |
| 1035 | void ucol_updateInternalState(UCollator *coll, UErrorCode *status); |
| 1036 | |
| 1037 | U_CFUNC uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status); |
| 1038 | U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status); |
| 1039 | |
| 1040 | U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status); |
| 1041 | |
| 1042 | U_CAPI void U_EXPORT2 |
| 1043 | uprv_uca_initImplicitConstants(UErrorCode *status); |
| 1044 | |
| 1045 | U_CAPI uint32_t U_EXPORT2 |
| 1046 | uprv_uca_getImplicitFromRaw(UChar32 cp); |
| 1047 | |
| 1048 | /*U_CFUNC uint32_t U_EXPORT2 |
| 1049 | uprv_uca_getImplicitPrimary(UChar32 cp);*/ |
| 1050 | |
| 1051 | U_CAPI UChar32 U_EXPORT2 |
| 1052 | uprv_uca_getRawFromImplicit(uint32_t implicit); |
| 1053 | |
| 1054 | U_CAPI UChar32 U_EXPORT2 |
| 1055 | uprv_uca_getRawFromCodePoint(UChar32 i); |
| 1056 | |
| 1057 | U_CAPI UChar32 U_EXPORT2 |
| 1058 | uprv_uca_getCodePointFromRaw(UChar32 i); |
| 1059 | |
| 1060 | |
| 1061 | |
| 1062 | #ifdef XP_CPLUSPLUS |
| 1063 | /* |
| 1064 | * Test whether a character is potentially "unsafe" for use as a collation |
| 1065 | * starting point. Unsafe chars are those with combining class != 0 plus |
| 1066 | * those that are the 2nd thru nth character in a contraction sequence. |
| 1067 | * |
| 1068 | * Function is in header file because it's used in both collation and string search, |
| 1069 | * and needs to be inline for performance. |
| 1070 | */ |
| 1071 | static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) { |
| 1072 | int32_t hash; |
| 1073 | uint8_t htbyte; |
| 1074 | |
| 1075 | if (c < coll->minUnsafeCP) { |
| 1076 | return FALSE; |
| 1077 | } |
| 1078 | |
| 1079 | hash = c; |
| 1080 | if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) { |
| 1081 | if(UTF_IS_SURROGATE(c)) { |
| 1082 | /* Lead or trail surrogate */ |
| 1083 | /* These are always considered unsafe. */ |
| 1084 | return TRUE; |
| 1085 | } |
| 1086 | hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256; |
| 1087 | } |
| 1088 | htbyte = coll->unsafeCP[hash>>3]; |
| 1089 | return ((htbyte >> (hash & 7)) & 1); |
| 1090 | } |
| 1091 | #endif /* XP_CPLUSPLUS */ |
| 1092 | |
| 1093 | /* The offsetBuffer in collIterate might need to be freed to avoid memory leaks. */ |
| 1094 | void ucol_freeOffsetBuffer(collIterate *s); |
| 1095 | |
| 1096 | #endif /* #if !UCONFIG_NO_COLLATION */ |
| 1097 | |
| 1098 | #endif |