blob: 63b54aff69ae777c30130561cc473f10328b44a2 [file] [log] [blame]
Igor Sarkisov64ff12e2020-10-06 05:32:41 -07001/*
2*******************************************************************************
3*
4* Copyright (C) 1998-2010, International Business Machines
5* Corporation and others. All Rights Reserved.
6*
7*******************************************************************************
8*
9* Private implementation header for C collation
10* file name: ucol_imp.h
11* encoding: US-ASCII
12* tab size: 8 (not used)
13* indentation:4
14*
15* created on: 2000dec11
16* created by: Vladimir Weinstein
17*
18* Modification history
19* Date Name Comments
20* 02/16/2001 synwee Added UCOL_GETPREVCE for the use in ucoleitr
21* 02/27/2001 synwee Added getMaxExpansion data structure in UCollator
22* 03/02/2001 synwee Added UCOL_IMPLICIT_CE
23* 03/12/2001 synwee Added pointer start to collIterate.
24*/
25
26#ifndef UCOL_IMP_H
27#define UCOL_IMP_H
28
29#include "unicode/utypes.h"
30
31#define UCA_DATA_TYPE "icu"
32#define UCA_DATA_NAME "ucadata"
33#define INVC_DATA_TYPE "icu"
34#define INVC_DATA_NAME "invuca"
35
36/**
37 * Convenience string denoting the Collation data tree
38 * @internal ICU 3.0
39 */
40#define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll"
41
42#if !UCONFIG_NO_COLLATION
43
44#ifdef XP_CPLUSPLUS
45#include "unicode/normalizer2.h"
46#include "unicode/unistr.h"
47#endif
48#include "unicode/ucol.h"
49#include "utrie.h"
50#include "cmemory.h"
51
52/* This is the internal header file which contains important declarations for
53 * the collation framework.
54 * Ready to use collators are stored as binary images. Both UCA and tailorings
55 * share the same binary format. Individual files (currently only UCA) have a
56 * udata header in front of the image and should be opened using udata_open.
57 * Tailoring images are currently stored inside resource bundles and are intialized
58 * through ucol_open API.
59 *
60 * The following describes the formats for collation binaries
61 * (UCA & tailorings) and for the inverse UCA table.
62 * Substructures are described in the collation design document at
63 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
64 *
65 * -------------------------------------------------------------
66 *
67 * Here is the format of binary collation image.
68 *
69 * Physical order of structures:
70 * - header (UCATableHeader)
71 * - options (UColOptionSet)
72 * - expansions (CE[])
73 * - contractions (UChar[contractionSize] + CE[contractionSize])
74 * - serialized UTrie with mappings of code points to CEs
75 * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount])
76 * - two bit sets for backward processing in strcoll (identical prefixes)
77 * and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE])
78 * - UCA constants (UCAConstants)
79 * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth])
80 *
81 * UCATableHeader fields:
82 *
83 * int32_t size; - image size in bytes
84 *
85 * Offsets to interesting data. All offsets are in bytes.
86 * to get the address add to the header address and cast properly.
87 * Some offsets are zero if the corresponding structures are empty.
88 *
89 * Tailoring binaries that only set options and contain no mappings etc.
90 * will have all offsets 0 except for the options and expansion offsets,
91 * which give the position and length of the options array.
92 *
93 * uint32_t options; - offset to default collator options (UColOptionSet *),
94 * a set of 32-bit values. See declaration of UColOptionSet for more details
95 *
96 * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges
97 * See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store
98 * important constant values that are defined in the UCA and used for building and runtime.
99 *
100 * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth],
101 * containing contractions from the UCA. These are needed in the build process to copy UCA contractions
102 * in case the base contraction symbol is tailored.
103 *
104 * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3)
105 *
106 * uint32_t mappingPosition; - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such.
107 * Used as a primary lookup table for collation elements.
108 *
109 * uint32_t expansion; - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0.
110 *
111 * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents
112 * are aligned with the contents of contractionCEs table. 0 if no contractions.
113 *
114 * uint32_t contractionCEs; - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the
115 * in the contractionIndex table, the resulting index is used to look up corresponding CE in this table.
116 * 0 if no contractions.
117 * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs).
118 *
119 * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion
120 * and last CEs in expansions.
121 * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *).
122 * Never 0.
123 * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *)
124 * int32_t endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION
125 * for the usage model
126 *
127 * These two offsets point to byte tables that are used in the backup heuristics.
128 * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function.
129 * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP.
130 *
131 * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos
132 * (formatVersion 2.3)
133 * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition.
134 * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3)
135 * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3)
136 * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3)
137 *
138 * Various version fields
139 * UVersionInfo version; - version 4 uint8_t
140 * UVersionInfo UCAVersion; - version 4 uint8_t
141 * UVersionInfo UCDVersion; - version 4 uint8_t
142 * UVersionInfo formatVersion; - version of the format of the collation binary
143 * same formatVersion as in ucadata.icu's UDataInfo header
144 * (formatVersion 2.3)
145 *
146 * uint8_t reserved[84]; - currently unused
147 *
148 * -------------------------------------------------------------
149 *
150 * Inverse UCA is used for constructing collators from rules. It is always an individual file
151 * and always has a UDataInfo header.
152 * here is the structure:
153 *
154 * uint32_t byteSize; - size of inverse UCA image in bytes
155 * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows)
156 * uint32_t contsSize; - size of continuation table (number of UChars in table)
157 *
158 * uint32_t table; - offset to inverse table (uint32_t *)
159 * Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation
160 * the third value is either a code unit (if there is only one code unit for element) or an index to continuation
161 * (number of code units combined with an index).
162 * table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final
163 * code unit sequence for a CE is terminated by FFFE.
164 * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE.
165 *
166 * UVersionInfo UCAVersion; - version of the UCA, read from file 4 uint8_t
167 * uint8_t padding[8]; - padding 8 uint8_t
168 * Header is followed by the table and continuation table.
169*/
170
171/* let us know whether reserved fields are reset to zero or junked */
172#define UCOL_HEADER_MAGIC 0x20030618
173
174/* UDataInfo for UCA mapping table */
175/* dataFormat="UCol" */
176#define UCA_DATA_FORMAT_0 ((uint8_t)0x55)
177#define UCA_DATA_FORMAT_1 ((uint8_t)0x43)
178#define UCA_DATA_FORMAT_2 ((uint8_t)0x6f)
179#define UCA_DATA_FORMAT_3 ((uint8_t)0x6c)
180
181#define UCA_FORMAT_VERSION_0 ((uint8_t)2)
182#define UCA_FORMAT_VERSION_1 ((uint8_t)3)
183#define UCA_FORMAT_VERSION_2 ((uint8_t)0)
184#define UCA_FORMAT_VERSION_3 ((uint8_t)0)
185
186/* UDataInfo for inverse UCA table */
187/* dataFormat="InvC" */
188#define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49)
189#define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E)
190#define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76)
191#define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43)
192
193#define INVUCA_FORMAT_VERSION_0 ((uint8_t)2)
194#define INVUCA_FORMAT_VERSION_1 ((uint8_t)1)
195#define INVUCA_FORMAT_VERSION_2 ((uint8_t)0)
196#define INVUCA_FORMAT_VERSION_3 ((uint8_t)0)
197
198/* This is the size of the stack allocated buffer for sortkey generation and similar operations */
199/* if it is too small, heap allocation will occur.*/
200/* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */
201#define UCOL_MAX_BUFFER 128
202#define UCOL_PRIMARY_MAX_BUFFER 8*UCOL_MAX_BUFFER
203#define UCOL_SECONDARY_MAX_BUFFER UCOL_MAX_BUFFER
204#define UCOL_TERTIARY_MAX_BUFFER UCOL_MAX_BUFFER
205#define UCOL_CASE_MAX_BUFFER UCOL_MAX_BUFFER/4
206#define UCOL_QUAD_MAX_BUFFER 2*UCOL_MAX_BUFFER
207
208#define UCOL_NORMALIZATION_GROWTH 2
209#define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH
210
211/* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */
212/* Sometimes we already have a writable buffer (like in case of normalized strings). */
213/*
214you can change this value to any value >= 4 if you need memory -
215it will affect the performance, though, since we're going to malloc.
216Note 3 is the minimum value for Thai collation and 4 is the
217minimum number for special Jamo
218*/
219#define UCOL_WRITABLE_BUFFER_SIZE 256
220
221/* This is the size of the buffer for expansion CE's */
222/* In reality we should not have to deal with expm sequences longer then 16 */
223/* you can change this value if you need memory */
224/* WARNING THIS BUFFER DOES HAVE MALLOC FALLBACK. If you make it too small, you'll get into performance trouble */
225/* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */
226/* This is the longest expansion sequence we can handle without bombing out */
227#define UCOL_EXPAND_CE_BUFFER_SIZE 64
228
229/* This is the size to increase the buffer for expansion CE's */
230#define UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE 64
231
232
233/* Unsafe UChar hash table table size. */
234/* size is 32 bytes for 1 bit for each latin 1 char + some power of two for */
235/* hashing the rest of the chars. Size in bytes */
236#define UCOL_UNSAFECP_TABLE_SIZE 1056
237 /* mask value down to "some power of two"-1 */
238 /* number of bits, not num of bytes. */
239#define UCOL_UNSAFECP_TABLE_MASK 0x1fff
240
241
242/* flags bits for collIterate.flags */
243/* */
244/* NORM - set for incremental normalize of source string */
245#define UCOL_ITER_NORM 1
246
247#define UCOL_ITER_HASLEN 2
248
249 /* UCOL_ITER_INNORMBUF - set if the "pos" is in */
250 /* the writable side buffer, handling */
251 /* incrementally normalized characters. */
252#define UCOL_ITER_INNORMBUF 4
253
254 /* UCOL_ITER_ALLOCATED - set if this iterator has */
255 /* malloced storage to expand a buffer. */
256#define UCOL_ITER_ALLOCATED 8
257 /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana */
258#define UCOL_HIRAGANA_Q 16
259 /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */
260 /* otherwise set to false */
261#define UCOL_WAS_HIRAGANA 32
262 /* UCOL_USE_ITERATOR - set this if collIterate uses a */
263 /* character iterator instead of simply accessing string */
264 /* by index */
265#define UCOL_USE_ITERATOR 64
266
267#define UCOL_FORCE_HAN_IMPLICIT 128
268
269#define NFC_ZERO_CC_BLOCK_LIMIT_ 0x300
270
271#ifdef XP_CPLUSPLUS
272
273typedef struct collIterate : public U_NAMESPACE_QUALIFIER UMemory {
274 const UChar *string; /* Original string */
275 /* UChar *start; Pointer to the start of the source string. Either points to string
276 or to writableBuffer */
277 const UChar *endp; /* string end ptr. Is undefined for null terminated strings */
278 const UChar *pos; /* This is position in the string. Can be to original or writable buf */
279
280 uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */
281 uint32_t *CEpos; /* This is the position to which we have stored processed CEs */
282
283 int32_t *offsetReturn; /* This is the offset to return, if non-NULL */
284 int32_t *offsetStore; /* This is the pointer for storing offsets */
285 int32_t offsetRepeatCount; /* Repeat stored offset if non-zero */
286 int32_t offsetRepeatValue; /* offset value to repeat */
287
288 U_NAMESPACE_QUALIFIER UnicodeString writableBuffer;
289 const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
290 const UCollator *coll;
291 const U_NAMESPACE_QUALIFIER Normalizer2 *nfd;
292 uint8_t flags;
293 uint8_t origFlags;
294 uint32_t *extendCEs; /* This is use if CEs is not big enough */
295 int32_t extendCEsSize; /* Holds the size of the dynamic CEs buffer */
296 uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */
297
298 int32_t *offsetBuffer; /* A dynamic buffer to hold offsets */
299 int32_t offsetBufferSize; /* The size of the offset buffer */
300
301 UCharIterator *iterator;
302 /*int32_t iteratorIndex;*/
303} collIterate;
304
305#else
306
307typedef struct collIterate collIterate;
308
309#endif
310
311#define paddedsize(something) ((something)+((((something)%4)!=0)?(4-(something)%4):0))
312#define headersize (paddedsize(sizeof(UCATableHeader))+paddedsize(sizeof(UColOptionSet)))
313
314/*
315struct used internally in getSpecial*CE.
316data similar to collIterate.
317*/
318struct collIterateState {
319 const UChar *pos; /* This is position in the string. Can be to original or writable buf */
320 const UChar *returnPos;
321 const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
322 const UChar *bufferaddress; /* address of the normalization buffer */
323 int32_t buffersize;
324 uint8_t flags;
325 uint8_t origFlags;
326 uint32_t iteratorIndex;
327 int32_t iteratorMove;
328};
329
330U_CAPI void U_EXPORT2
331uprv_init_collIterate(const UCollator *collator,
332 const UChar *sourceString, int32_t sourceLen,
333 collIterate *s, UErrorCode *status);
334
335/* Internal functions for C test code. */
336U_CAPI collIterate * U_EXPORT2
337uprv_new_collIterate(UErrorCode *status);
338
339U_CAPI void U_EXPORT2
340uprv_delete_collIterate(collIterate *s);
341
342/* @return s->pos == s->endp */
343U_CAPI UBool U_EXPORT2
344uprv_collIterateAtEnd(collIterate *s);
345
346#ifdef XP_CPLUSPLUS
347
348U_NAMESPACE_BEGIN
349
350struct UCollationPCE;
351typedef struct UCollationPCE UCollationPCE;
352
353U_NAMESPACE_END
354
355struct UCollationElements : public U_NAMESPACE_QUALIFIER UMemory
356{
357 /**
358 * Struct wrapper for source data
359 */
360 collIterate iteratordata_;
361 /**
362 * Indicates if this data has been reset.
363 */
364 UBool reset_;
365 /**
366 * Indicates if the data should be deleted.
367 */
368 UBool isWritable;
369
370/**
371 * Data for getNextProcessed, getPreviousProcessed.
372 */
373 U_NAMESPACE_QUALIFIER UCollationPCE *pce;
374};
375
376#endif
377
378U_CAPI void U_EXPORT2
379uprv_init_pce(const struct UCollationElements *elems);
380
381#define UCOL_LEVELTERMINATOR 1
382
383/* mask off anything but primary order */
384#define UCOL_PRIMARYORDERMASK 0xffff0000
385/* mask off anything but secondary order */
386#define UCOL_SECONDARYORDERMASK 0x0000ff00
387/* mask off anything but tertiary order */
388#define UCOL_TERTIARYORDERMASK 0x000000ff
389/* primary order shift */
390#define UCOL_PRIMARYORDERSHIFT 16
391/* secondary order shift */
392#define UCOL_SECONDARYORDERSHIFT 8
393
394#define UCOL_BYTE_SIZE_MASK 0xFF
395
396#define UCOL_CASE_BYTE_START 0x80
397#define UCOL_CASE_SHIFT_START 7
398
399#define UCOL_IGNORABLE 0
400
401/* get weights from a CE */
402#define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT)
403#define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT)
404#define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK)
405
406/**
407 * Determine if a character is a Thai vowel (which sorts after
408 * its base consonant).
409 */
410#define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \
411 (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0)))
412
413/**
414 * Determine if a character is a Thai base consonant
415 */
416#define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01)
417
418#define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \
419 (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \
420 (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8)))
421
422/* Han character ranges */
423#define UCOL_FIRST_HAN 0x4E00
424#define UCOL_LAST_HAN 0x9FFF
425#define UCOL_FIRST_HAN_A 0x3400
426#define UCOL_LAST_HAN_A 0x4DBF
427#define UCOL_FIRST_HAN_COMPAT 0xFAE0
428#define UCOL_LAST_HAN_COMPAT 0xFA2F
429
430/* Han extension B is in plane 2 */
431#define UCOL_FIRST_HAN_B 0x20000
432#define UCOL_LAST_HAN_B 0x2A6DF
433
434/* Hangul range */
435#define UCOL_FIRST_HANGUL 0xAC00
436#define UCOL_LAST_HANGUL 0xD7AF
437
438/* Jamo ranges */
439#define UCOL_FIRST_L_JAMO 0x1100
440#define UCOL_FIRST_V_JAMO 0x1161
441#define UCOL_FIRST_T_JAMO 0x11A8
442#define UCOL_LAST_T_JAMO 0x11F9
443
444
445#if 0
446/* initializes collIterate structure */
447/* made as macro to speed up things */
448#define init_collIterate(collator, sourceString, sourceLen, s) { \
449 (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \
450 (s)->endp = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \
451 (s)->CEpos = (s)->toReturn = (s)->CEs; \
452 (s)->isThai = TRUE; \
453 (s)->writableBuffer = (s)->stackWritableBuffer; \
454 (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \
455 (s)->coll = (collator); \
456 (s)->fcdPosition = 0; \
457 (s)->flags = 0; \
458 if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \
459}
460#endif
461
462
463
464/*
465* Macro to get the maximum size of an expansion ending with the argument ce.
466* Used in the Boyer Moore algorithm.
467* Note for tailoring, the UCA maxexpansion table has been merged.
468* Hence we only have to search the tailored collator only.
469* @param coll const UCollator pointer
470* @param order last collation element of the expansion sequence
471* @param result size of the longest expansion with argument collation element
472* as the last element
473*/
474#define UCOL_GETMAXEXPANSION(coll, order, result) { \
475 const uint32_t *start; \
476 const uint32_t *limit; \
477 const uint32_t *mid; \
478 start = (coll)->endExpansionCE; \
479 limit = (coll)->lastEndExpansionCE; \
480 while (start < limit - 1) { \
481 mid = start + ((limit - start) >> 1); \
482 if ((order) <= *mid) { \
483 limit = mid; \
484 } \
485 else { \
486 start = mid; \
487 } \
488 } \
489 if (*start == order) { \
490 result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE)); \
491 } \
492 else if (*limit == order) { \
493 result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \
494 } \
495 else if ((order & 0xFFFF) == 0x00C0) { \
496 result = 2; \
497 } \
498 else { \
499 result = 1; \
500 } \
501}
502
503U_CFUNC
504uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE, collIterate *source, UErrorCode *status);
505
506U_CFUNC
507uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE,
508 collIterate *source, UErrorCode *status);
509U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll, collIterate *collationSource, UErrorCode *status);
510U_CFUNC uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll,
511 collIterate *collationSource,
512 UErrorCode *status);
513/* function used by C++ getCollationKey to prevent restarting the calculation */
514U_CFUNC int32_t
515ucol_getSortKeyWithAllocation(const UCollator *coll,
516 const UChar *source, int32_t sourceLength,
517 uint8_t **pResult,
518 UErrorCode *pErrorCode);
519
520/* get some memory */
521void *ucol_getABuffer(const UCollator *coll, uint32_t size);
522
523/* worker function for generating sortkeys */
524U_CFUNC
525int32_t U_CALLCONV
526ucol_calcSortKey(const UCollator *coll,
527 const UChar *source,
528 int32_t sourceLength,
529 uint8_t **result,
530 uint32_t resultLength,
531 UBool allocatePrimary,
532 UErrorCode *status);
533
534U_CFUNC
535int32_t U_CALLCONV
536ucol_calcSortKeySimpleTertiary(const UCollator *coll,
537 const UChar *source,
538 int32_t sourceLength,
539 uint8_t **result,
540 uint32_t resultLength,
541 UBool allocatePrimary,
542 UErrorCode *status);
543
544U_CFUNC
545int32_t
546ucol_getSortKeySize(const UCollator *coll, collIterate *s,
547 int32_t currentSize, UColAttributeValue strength,
548 int32_t len);
549/**
550 * Makes a copy of the Collator's rule data. The format is
551 * that of .col files.
552 *
553 * @param length returns the length of the data, in bytes.
554 * @param status the error status
555 * @return memory, owned by the caller, of size 'length' bytes.
556 * @internal INTERNAL USE ONLY
557 */
558U_CFUNC uint8_t* U_EXPORT2
559ucol_cloneRuleData(const UCollator *coll, int32_t *length, UErrorCode *status);
560
561/**
562 * Used to set requested and valid locales on a collator returned by the collator
563 * service.
564 */
565U_CFUNC void U_EXPORT2
566ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt, char *actualLocaleToAdopt);
567
568#define UCOL_SPECIAL_FLAG 0xF0000000
569#define UCOL_TAG_SHIFT 24
570#define UCOL_TAG_MASK 0x0F000000
571#define INIT_EXP_TABLE_SIZE 1024
572#define UCOL_NOT_FOUND 0xF0000000
573#define UCOL_EXPANSION 0xF1000000
574#define UCOL_CONTRACTION 0xF2000000
575#define UCOL_THAI 0xF3000000
576#define UCOL_UNMARKED 0x03
577#define UCOL_NEW_TERTIARYORDERMASK 0x0000003f
578
579/* Bit mask for primary collation strength. */
580#define UCOL_PRIMARYMASK 0xFFFF0000
581
582/* Bit mask for secondary collation strength. */
583#define UCOL_SECONDARYMASK 0x0000FF00
584
585/* Bit mask for tertiary collation strength. */
586#define UCOL_TERTIARYMASK 0x000000FF
587
588/**
589 * Internal.
590 * This indicates the last element in a UCollationElements has been consumed.
591 * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs.
592 */
593#define UCOL_NO_MORE_CES 0x00010101
594#define UCOL_NO_MORE_CES_PRIMARY 0x00010000
595#define UCOL_NO_MORE_CES_SECONDARY 0x00000100
596#define UCOL_NO_MORE_CES_TERTIARY 0x00000001
597
598#define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF)
599
600#define UCOL_UPPER_CASE 0x80
601#define UCOL_MIXED_CASE 0x40
602#define UCOL_LOWER_CASE 0x00
603
604#define UCOL_CONTINUATION_MARKER 0xC0
605#define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F
606
607#define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER)
608#define isFlagged(CE) (((CE) & 0x80) == 0x80)
609#define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0)
610
611#define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT)
612#define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG))
613#define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG))
614#define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF)
615#define constructSpecProcCE(CE) (UCOL_SPECIAL_FLAG | (SPEC_PROC_TAG<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF)
616#define getContractOffset(CE) ((CE)&0xFFFFFF)
617#define getExpansionOffset(CE) (((CE)&0x00FFFFF0)>>4)
618#define getExpansionCount(CE) ((CE)&0xF)
619#define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0)
620
621/* StringSearch internal use */
622#define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF)
623#define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL)
624#define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs)
625#define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs)
626#define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset)
627#define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn)
628#define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces)
629
630/* This is an enum that lists magic special byte values from the fractional UCA */
631/* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */
632
633enum {
634 UCOL_BYTE_ZERO = 0x00,
635 UCOL_BYTE_LEVEL_SEPARATOR = 0x01,
636 UCOL_BYTE_SORTKEY_GLUE = 0x02,
637 UCOL_BYTE_SHIFT_PREFIX = 0x03,
638 UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX,
639 UCOL_BYTE_FIRST_TAILORED = 0x04,
640 UCOL_BYTE_COMMON = 0x05,
641 UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON,
642 UCOL_CODAN_PLACEHOLDER = 0x27,
643 UCOL_BYTE_LAST_LATIN_PRIMARY = 0x4C,
644 UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x4D,
645 UCOL_BYTE_UNSHIFTED_MAX = 0xFF
646};
647
648#if 0
649#define UCOL_RESET_TOP_VALUE 0x9F000303
650#define UCOL_FIRST_PRIMARY_IGNORABLE 0x00008705
651#define UCOL_LAST_PRIMARY_IGNORABLE 0x0000DD05
652#define UCOL_LAST_PRIMARY_IGNORABLE_CONT 0x000051C0
653#define UCOL_FIRST_SECONDARY_IGNORABLE 0x00000000
654#define UCOL_LAST_SECONDARY_IGNORABLE 0x00000500
655#define UCOL_FIRST_TERTIARY_IGNORABLE 0x00000000
656#define UCOL_LAST_TERTIARY_IGNORABLE 0x00000000
657#define UCOL_FIRST_VARIABLE 0x05070505
658#define UCOL_LAST_VARIABLE 0x179B0505
659#define UCOL_FIRST_NON_VARIABLE 0x1A200505
660#define UCOL_LAST_NON_VARIABLE 0x7B41058F
661
662#define UCOL_NEXT_TOP_VALUE 0xE8960303
663#define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE 0x00008905
664#define UCOL_NEXT_LAST_PRIMARY_IGNORABLE 0x03000303
665#define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705
666#define UCOL_NEXT_LAST_SECONDARY_IGNORABLE 0x00000500
667#define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE 0x00000000
668#define UCOL_NEXT_LAST_TERTIARY_IGNORABLE 0x00000000
669#define UCOL_NEXT_FIRST_VARIABLE 0x05090505
670#define UCOL_NEXT_LAST_VARIABLE 0x1A200505
671
672#define PRIMARY_IMPLICIT_MIN 0xE8000000
673#define PRIMARY_IMPLICIT_MAX 0xF0000000
674#endif
675
676/* These constants can be changed - sortkey size is affected by them */
677#define UCOL_PROPORTION2 0.5
678#define UCOL_PROPORTION3 0.667
679
680/* These values come from the UCA */
681#define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON
682#define UCOL_COMMON_TOP2 0x86u
683#define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1)
684
685#define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80
686#define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40
687#define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85
688#define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45
689#define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5
690
691/* These values come from the UCA */
692#define UCOL_COMMON_BOT3 0x05
693
694#define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86;
695#define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3;
696
697#define UCOL_TOP_COUNT2 (UCOL_PROPORTION2*UCOL_TOTAL2)
698#define UCOL_BOT_COUNT2 (UCOL_TOTAL2-UCOL_TOP_COUNT2)
699
700
701#define UCOL_COMMON2 UCOL_COMMON_BOT2
702#define UCOL_COMMON3_UPPERFIRST 0xC5
703#define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3
704
705#define UCOL_COMMON4 0xFF
706
707/* constants for case level/case first handling */
708/* used to instantiate UCollators fields in ucol_updateInternalState */
709#define UCOL_CASE_SWITCH 0xC0
710#define UCOL_NO_CASE_SWITCH 0x00
711
712#define UCOL_REMOVE_CASE 0x3F
713#define UCOL_KEEP_CASE 0xFF
714
715#define UCOL_CASE_BIT_MASK 0xC0
716
717#define UCOL_TERT_CASE_MASK 0xFF
718
719#define UCOL_ENDOFLATINONERANGE 0xFF
720#define UCOL_LATINONETABLELEN (UCOL_ENDOFLATINONERANGE+50)
721#define UCOL_BAIL_OUT_CE 0xFF000000
722
723
724typedef enum {
725 NOT_FOUND_TAG = 0,
726 EXPANSION_TAG = 1, /* This code point results in an expansion */
727 CONTRACTION_TAG = 2, /* Start of a contraction */
728 THAI_TAG = 3, /* Thai character - do the reordering */
729 CHARSET_TAG = 4, /* Charset processing, not yet implemented */
730 SURROGATE_TAG = 5, /* Lead surrogate that is tailored and doesn't start a contraction */
731 HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/
732 LEAD_SURROGATE_TAG = 7, /* D800-DBFF*/
733 TRAIL_SURROGATE_TAG = 8, /* DC00-DFFF*/
734 CJK_IMPLICIT_TAG = 9, /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/
735 IMPLICIT_TAG = 10,
736 SPEC_PROC_TAG = 11,
737 /* ICU 2.1 */
738 LONG_PRIMARY_TAG = 12, /* This is a three byte primary with starting secondaries and tertiaries */
739 /* It fits in a single 32 bit CE and is used instead of expansion to save */
740 /* space without affecting the performance (hopefully) */
741
742 DIGIT_TAG = 13, /* COllate Digits As Numbers (CODAN) implementation */
743
744 CE_TAGS_COUNT
745} UColCETags;
746
747/*
748 *****************************************************************************************
749 * set to zero
750 * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF
751 ******************************************************************************************
752 */
753
754typedef struct {
755 uint32_t variableTopValue;
756 /*UColAttributeValue*/ int32_t frenchCollation;
757 /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/
758 /*UColAttributeValue*/ int32_t caseFirst; /* who goes first, lower case or uppercase */
759 /*UColAttributeValue*/ int32_t caseLevel; /* do we have an extra case level */
760 /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */
761 /*UColAttributeValue*/ int32_t strength; /* attribute for strength */
762 /*UColAttributeValue*/ int32_t hiraganaQ; /* attribute for special Hiragana */
763 /*UColAttributeValue*/ int32_t numericCollation; /* attribute for numeric collation */
764 uint32_t reserved[15]; /* for future use */
765} UColOptionSet;
766
767typedef struct {
768 uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2]; /*0x00000000*/
769 uint32_t UCA_LAST_TERTIARY_IGNORABLE[2]; /*0x00000000*/
770 uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2]; /*0x00008705*/
771 uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2]; /*0x00000000*/
772 uint32_t UCA_LAST_SECONDARY_IGNORABLE[2]; /*0x00000500*/
773 uint32_t UCA_LAST_PRIMARY_IGNORABLE[2]; /*0x0000DD05*/
774 uint32_t UCA_FIRST_VARIABLE[2]; /*0x05070505*/
775 uint32_t UCA_LAST_VARIABLE[2]; /*0x13CF0505*/
776 uint32_t UCA_FIRST_NON_VARIABLE[2]; /*0x16200505*/
777 uint32_t UCA_LAST_NON_VARIABLE[2]; /*0x767C0505*/
778 uint32_t UCA_RESET_TOP_VALUE[2]; /*0x9F000303*/
779 uint32_t UCA_FIRST_IMPLICIT[2];
780 uint32_t UCA_LAST_IMPLICIT[2];
781 uint32_t UCA_FIRST_TRAILING[2];
782 uint32_t UCA_LAST_TRAILING[2];
783
784#if 0
785 uint32_t UCA_NEXT_TOP_VALUE[2]; /*0xE8960303*/
786 uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE; /*0x00008905*/
787 uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE; /*0x03000303*/
788 uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/
789 uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE; /*0x00000500*/
790 uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE; /*0x00000000*/
791 uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE; /*0x00000000*/
792 uint32_t UCA_NEXT_FIRST_VARIABLE; /*0x05090505*/
793 uint32_t UCA_NEXT_LAST_VARIABLE; /*0x16200505*/
794#endif
795
796 uint32_t UCA_PRIMARY_TOP_MIN;
797 uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/
798 uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/
799 uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/
800 uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/
801 uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/
802 uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/
803} UCAConstants;
804
805typedef struct {
806 int32_t size;
807 /* all the offsets are in bytes */
808 /* to get the address add to the header address and cast properly */
809 uint32_t options; /* these are the default options for the collator */
810 uint32_t UCAConsts; /* structure which holds values for indirect positioning and implicit ranges */
811 uint32_t contractionUCACombos; /* this one is needed only for UCA, to copy the appropriate contractions */
812 uint32_t magic; /* magic number - lets us know whether reserved data is reset or junked */
813 uint32_t mappingPosition; /* const uint8_t *mappingPosition; */
814 uint32_t expansion; /* uint32_t *expansion; */
815 uint32_t contractionIndex; /* UChar *contractionIndex; */
816 uint32_t contractionCEs; /* uint32_t *contractionCEs; */
817 uint32_t contractionSize; /* needed for various closures */
818 /*int32_t latinOneMapping;*/ /* this is now handled in the trie itself *//* fast track to latin1 chars */
819
820 uint32_t endExpansionCE; /* array of last collation element in
821 expansion */
822 uint32_t expansionCESize; /* array of maximum expansion size
823 corresponding to the expansion
824 collation elements with last element
825 in endExpansionCE*/
826 int32_t endExpansionCECount; /* size of endExpansionCE */
827 uint32_t unsafeCP; /* hash table of unsafe code points */
828 uint32_t contrEndCP; /* hash table of final code points */
829 /* in contractions. */
830
831 int32_t contractionUCACombosSize; /* number of UCA contraction items. */
832 /*Length is contractionUCACombosSize*contractionUCACombosWidth*sizeof(UChar) */
833 UBool jamoSpecial; /* is jamoSpecial */
834 UBool isBigEndian; /* is this data big endian? from the UDataInfo header*/
835 uint8_t charSetFamily; /* what is the charset family of this data from the UDataInfo header*/
836 uint8_t contractionUCACombosWidth; /* width of UCA combos field */
837 UVersionInfo version;
838 UVersionInfo UCAVersion; /* version of the UCA, read from file */
839 UVersionInfo UCDVersion; /* UCD version, obtained by u_getUnicodeVersion */
840 UVersionInfo formatVersion; /* format version from the UDataInfo header */
841 uint8_t reserved[84]; /* for future use */
842} UCATableHeader;
843
844#define U_UNKNOWN_STATE 0
845#define U_COLLATOR_STATE 0x01
846#define U_STATE_LIMIT 0x02
847
848/* This is the first structure in a state */
849/* it should be machine independent */
850typedef struct {
851 /* this structure is supposed to be readable on all the platforms.*/
852 /* first 2 fields hold the size of the structure in a platform independent way */
853 uint8_t sizeLo;
854 uint8_t sizeHi;
855 /* identifying the writing platform */
856 uint8_t isBigEndian;
857 /* see U_CHARSET_FAMILY values in utypes.h */
858 uint8_t charsetFamily;
859 /* version of ICU this state structure comes from */
860 uint8_t icuVersion[4];
861 /* What is the data following this state */
862 uint8_t type;
863 /* more stuff to come, keep it on 16 byte boundary */
864 uint8_t reserved[7];
865} UStateStruct;
866
867/* This structure follows UStatusStruct */
868/* and contains data specific for the collators */
869/* Endianess needs to be decided before accessing this structure */
870/* However, it's size IS endianess independent */
871typedef struct {
872 /* size of this structure */
873 uint8_t sizeLo;
874 uint8_t sizeHi;
875 /* This state is followed by the frozen tailoring */
876 uint8_t containsTailoring;
877 /* This state is followed by the frozen UCA */
878 uint8_t containsUCA;
879 /* Version info - the same one */
880 uint8_t versionInfo[4];
881
882 /* for charset CEs */
883 uint8_t charsetName[32];
884 /* this is the resolved locale name*/
885 uint8_t locale[32];
886
887 /* Attributes. Open ended */
888 /* all the following will be moved to uint32_t because of portability */
889 /* variable top value */
890 uint32_t variableTopValue;
891 /* attribute for handling variable elements*/
892 uint32_t /*UColAttributeValue*/ alternateHandling;
893 /* how to handle secondary weights */
894 uint32_t /*UColAttributeValue*/ frenchCollation;
895 /* who goes first, lower case or uppercase */
896 uint32_t /*UColAttributeValue*/ caseFirst;
897 /* do we have an extra case level */
898 uint32_t /*UColAttributeValue*/ caseLevel;
899 /* attribute for normalization */
900 uint32_t /*UColAttributeValue*/ normalizationMode;
901 /* attribute for strength */
902 uint32_t /*UColAttributeValue*/ strength;
903 /* to be immediately 16 byte aligned */
904 uint8_t reserved[12];
905} UColStateStruct;
906
907#define UCOL_INV_SIZEMASK 0xFFF00000
908#define UCOL_INV_OFFSETMASK 0x000FFFFF
909#define UCOL_INV_SHIFTVALUE 20
910
911U_CDECL_BEGIN
912
913typedef struct {
914 uint32_t byteSize;
915 uint32_t tableSize;
916 uint32_t contsSize;
917 uint32_t table;
918 uint32_t conts;
919 UVersionInfo UCAVersion; /* version of the UCA, read from file */
920 uint8_t padding[8];
921} InverseUCATableHeader;
922
923typedef int32_t U_CALLCONV
924SortKeyGenerator(const UCollator *coll,
925 const UChar *source,
926 int32_t sourceLength,
927 uint8_t **result,
928 uint32_t resultLength,
929 UBool allocatePrimary,
930 UErrorCode *status);
931
932typedef void U_CALLCONV
933ResourceCleaner(UCollator *coll);
934
935
936struct UCollator {
937 UColOptionSet *options;
938 SortKeyGenerator *sortKeyGen;
939 uint32_t *latinOneCEs;
940 char* actualLocale;
941 char* validLocale;
942 char* requestedLocale;
943 const UChar *rules;
944 const UChar *ucaRules;
945 const UCollator *UCA;
946 const UCATableHeader *image;
947 UTrie mapping;
948 const uint32_t *latinOneMapping;
949 const uint32_t *expansion;
950 const UChar *contractionIndex;
951 const uint32_t *contractionCEs;
952 /*const uint8_t *scriptOrder;*/
953
954 const uint32_t *endExpansionCE; /* array of last ces in an expansion ce.
955 corresponds to expansionCESize */
956 const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */
957 const uint8_t *expansionCESize; /* array of the maximum size of a
958 expansion ce with the last ce
959 corresponding to endExpansionCE,
960 terminated with a null */
961 const uint8_t *unsafeCP; /* unsafe code points hashtable */
962 const uint8_t *contrEndCP; /* Contraction ending chars hash table */
963 UChar minUnsafeCP; /* Smallest unsafe Code Point. */
964 UChar minContrEndCP; /* Smallest code point at end of a contraction */
965
966 int32_t rulesLength;
967 int32_t latinOneTableLen;
968
969 uint32_t variableTopValue;
970 UColAttributeValue frenchCollation;
971 UColAttributeValue alternateHandling; /* attribute for handling variable elements*/
972 UColAttributeValue caseFirst; /* who goes first, lower case or uppercase */
973 UColAttributeValue caseLevel; /* do we have an extra case level */
974 UColAttributeValue normalizationMode; /* attribute for normalization */
975 UColAttributeValue strength; /* attribute for strength */
976 UColAttributeValue hiraganaQ; /* attribute for Hiragana */
977 UColAttributeValue numericCollation;
978 UBool variableTopValueisDefault;
979 UBool frenchCollationisDefault;
980 UBool alternateHandlingisDefault; /* attribute for handling variable elements*/
981 UBool caseFirstisDefault; /* who goes first, lower case or uppercase */
982 UBool caseLevelisDefault; /* do we have an extra case level */
983 UBool normalizationModeisDefault; /* attribute for normalization */
984 UBool strengthisDefault; /* attribute for strength */
985 UBool hiraganaQisDefault; /* attribute for Hiragana */
986 UBool numericCollationisDefault;
987 UBool hasRealData; /* some collators have only options, like French, no rules */
988 /* to speed up things, we use the UCA image, but we don't want it */
989 /* to run around */
990
991 UBool freeOnClose;
992 UBool freeOptionsOnClose;
993 UBool freeRulesOnClose;
994 UBool freeImageOnClose;
995
996 UBool latinOneUse;
997 UBool latinOneRegenTable;
998 UBool latinOneFailed;
999
1000 int8_t tertiaryAddition; /* when switching case, we need to add or subtract different values */
1001 uint8_t caseSwitch;
1002 uint8_t tertiaryCommon;
1003 uint8_t tertiaryMask;
1004 uint8_t tertiaryTop; /* Upper range when compressing */
1005 uint8_t tertiaryBottom; /* Upper range when compressing */
1006 uint8_t tertiaryTopCount;
1007 uint8_t tertiaryBottomCount;
1008
1009 UVersionInfo dataVersion; /* Data info of UCA table */
1010};
1011
1012U_CDECL_END
1013
1014/* various internal functions */
1015
1016/* do not close UCA returned by ucol_initUCA! */
1017U_CFUNC
1018UCollator* ucol_initUCA(UErrorCode *status);
1019
1020U_CFUNC
1021UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status);
1022
1023U_CFUNC
1024void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
1025
1026U_CFUNC
1027UCollator* ucol_open_internal(const char* loc, UErrorCode* status);
1028
1029#if 0
1030U_CFUNC
1031void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
1032#endif
1033
1034U_CFUNC
1035void ucol_updateInternalState(UCollator *coll, UErrorCode *status);
1036
1037U_CFUNC uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status);
1038U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status);
1039
1040U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status);
1041
1042U_CAPI void U_EXPORT2
1043uprv_uca_initImplicitConstants(UErrorCode *status);
1044
1045U_CAPI uint32_t U_EXPORT2
1046uprv_uca_getImplicitFromRaw(UChar32 cp);
1047
1048/*U_CFUNC uint32_t U_EXPORT2
1049uprv_uca_getImplicitPrimary(UChar32 cp);*/
1050
1051U_CAPI UChar32 U_EXPORT2
1052uprv_uca_getRawFromImplicit(uint32_t implicit);
1053
1054U_CAPI UChar32 U_EXPORT2
1055uprv_uca_getRawFromCodePoint(UChar32 i);
1056
1057U_CAPI UChar32 U_EXPORT2
1058uprv_uca_getCodePointFromRaw(UChar32 i);
1059
1060
1061
1062#ifdef XP_CPLUSPLUS
1063/*
1064 * Test whether a character is potentially "unsafe" for use as a collation
1065 * starting point. Unsafe chars are those with combining class != 0 plus
1066 * those that are the 2nd thru nth character in a contraction sequence.
1067 *
1068 * Function is in header file because it's used in both collation and string search,
1069 * and needs to be inline for performance.
1070 */
1071static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) {
1072 int32_t hash;
1073 uint8_t htbyte;
1074
1075 if (c < coll->minUnsafeCP) {
1076 return FALSE;
1077 }
1078
1079 hash = c;
1080 if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) {
1081 if(UTF_IS_SURROGATE(c)) {
1082 /* Lead or trail surrogate */
1083 /* These are always considered unsafe. */
1084 return TRUE;
1085 }
1086 hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256;
1087 }
1088 htbyte = coll->unsafeCP[hash>>3];
1089 return ((htbyte >> (hash & 7)) & 1);
1090}
1091#endif /* XP_CPLUSPLUS */
1092
1093/* The offsetBuffer in collIterate might need to be freed to avoid memory leaks. */
1094void ucol_freeOffsetBuffer(collIterate *s);
1095
1096#endif /* #if !UCONFIG_NO_COLLATION */
1097
1098#endif