1 /* Header for multilingual functions.
2 Copyright (C) 1992, 1995 Free Software Foundation, Inc.
3 Copyright (C) 1995 Sun Microsystems, Inc.
5 This file is part of SXEmacs
7 SXEmacs is free software: you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
12 SXEmacs is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program. If not, see <http://www.gnu.org/licenses/>. */
21 /* Synched up with: Mule 2.3. Not in FSF. */
23 /* Rewritten by Ben Wing <ben@xemacs.org>. */
25 #ifndef INCLUDED_mule_charset_h_
26 #define INCLUDED_mule_charset_h_
32 A character set (or "charset") is an ordered set of characters.
33 A particular character in a charset is indexed using one or
34 more "position codes", which are non-negative integers.
35 The number of position codes needed to identify a particular
36 character in a charset is called the "dimension" of the
37 charset. In XEmacs/Mule, all charsets have 1 or 2 dimensions,
38 and the size of all charsets (except for a few special cases)
39 is either 94, 96, 94 by 94, or 96 by 96. The range of
40 position codes used to index characters from any of these
41 types of character sets is as follows:
43 Charset type Position code 1 Position code 2
44 ------------------------------------------------------------
47 94x94 33 - 126 33 - 126
48 96x96 32 - 127 32 - 127
50 Note that in the above cases position codes do not start at
51 an expected value such as 0 or 1. The reason for this will
54 For example, Latin-1 is a 96-character charset, and JISX0208
55 (the Japanese national character set) is a 94x94-character
58 [Note that, although the ranges above define the *valid*
59 position codes for a charset, some of the slots in a particular
60 charset may in fact be empty. This is the case for JISX0208,
61 for example, where (e.g.) all the slots whose first
62 position code is in the range 118 - 127 are empty.]
64 There are three charsets that do not follow the above rules.
65 All of them have one dimension, and have ranges of position
68 Charset name Position code 1
69 ------------------------------------
72 Composite 0 - some large number
74 (The upper bound of the position code for composite characters
75 has not yet been determined, but it will probably be at
78 ASCII is the union of two subsidiary character sets:
79 Printing-ASCII (the printing ASCII character set,
80 consisting of position codes 33 - 126, like for a standard
81 94-character charset) and Control-ASCII (the non-printing
82 characters that would appear in a binary file with codes 0
85 Control-1 contains the non-printing characters that would
86 appear in a binary file with codes 128 - 159.
88 Composite contains characters that are generated by
89 overstriking one or more characters from other charsets.
91 Note that some characters in ASCII, and all characters
92 in Control-1, are "control" (non-printing) characters.
93 These have no printed representation but instead control
94 some other function of the printing (e.g. TAB or 8 moves
95 the current character position to the next tab stop).
96 All other characters in all charsets are "graphic"
97 (printing) characters.
99 When a binary file is read in, the bytes in the file are
100 assigned to character sets as follows:
102 Bytes Character set Range
103 --------------------------------------------------
104 0 - 127 ASCII 0 - 127
105 128 - 159 Control-1 0 - 31
106 160 - 255 Latin-1 32 - 127
108 This is a bit ad-hoc but gets the job done.
113 An "encoding" is a way of numerically representing
114 characters from one or more character sets. If an encoding
115 only encompasses one character set, then the position codes
116 for the characters in that character set could be used
117 directly. This is not possible, however, if more than one
118 character set is to be used in the encoding.
120 For example, the conversion detailed above between bytes in
121 a binary file and characters is effectively an encoding
122 that encompasses the three character sets ASCII, Control-1,
123 and Latin-1 in a stream of 8-bit bytes.
125 Thus, an encoding can be viewed as a way of encoding
126 characters from a specified group of character sets using a
127 stream of bytes, each of which contains a fixed number of
128 bits (but not necessarily 8, as in the common usage of
131 Here are descriptions of a couple of common
134 A. Japanese EUC (Extended Unix Code)
136 This encompasses the character sets:
138 - Katakana-JISX0201 (half-width katakana, the right half of JISX0201).
143 Note that Printing-ASCII and Katakana-JISX0201 are 94-character
144 charsets, while Japanese-JISX0208 is a 94x94-character charset.
146 The encoding is as follows:
148 Character set Representation (PC == position-code)
149 ------------- --------------
151 Japanese-JISX0208 PC1 + 0x80 | PC2 + 0x80
152 Katakana-JISX0201 0x8E | PC1 + 0x80
156 This encompasses the character sets:
158 - Latin-JISX0201 (the left half of JISX0201; this character set is
159 very similar to Printing-ASCII and is a 94-character charset)
164 Unlike Japanese EUC, this is a "modal" encoding, which
165 means that there are multiple states that the encoding can
166 be in, which affect how the bytes are to be interpreted.
167 Special sequences of bytes (called "escape sequences")
168 are used to change states.
170 The encoding is as follows:
172 Character set Representation
173 ------------- --------------
176 Katakana-JISX0201 PC1
177 Japanese-JISX0208 PC1 | PC2
179 Escape sequence ASCII equivalent Meaning
180 --------------- ---------------- -------
181 0x1B 0x28 0x42 ESC ( B invoke Printing-ASCII
182 0x1B 0x28 0x4A ESC ( J invoke Latin-JISX0201
183 0x1B 0x28 0x49 ESC ( I invoke Katakana-JISX0201
184 0x1B 0x24 0x42 ESC $ B invoke Japanese-JISX0208
186 Initially, Printing-ASCII is invoked.
188 3. Internal Mule Encodings
189 ==========================
191 In XEmacs/Mule, each character set is assigned a unique number,
192 called a "leading byte". This is used in the encodings of a
193 character. Leading bytes are in the range 0x80 - 0xFF
194 (except for ASCII, which has a leading byte of 0), although
195 some leading bytes are reserved.
197 Charsets whose leading byte is in the range 0x80 - 0x9F are
198 called "official" and are used for built-in charsets.
199 Other charsets are called "private" and have leading bytes
200 in the range 0xA0 - 0xFF; these are user-defined charsets.
204 Character set Leading byte
205 ------------- ------------
208 Dimension-1 Official 0x81 - 0x8D
211 Dimension-2 Official 0x90 - 0x99
212 (0x9A - 0x9D are free;
213 0x9E and 0x9F are reserved)
214 Dimension-1 Private 0xA0 - 0xEF
215 Dimension-2 Private 0xF0 - 0xFF
217 There are two internal encodings for characters in XEmacs/Mule.
218 One is called "string encoding" and is an 8-bit encoding that
219 is used for representing characters in a buffer or string.
220 It uses 1 to 4 bytes per character. The other is called
221 "character encoding" and is a 19-bit encoding that is used
222 for representing characters individually in a variable.
224 (In the following descriptions, we'll ignore composite
225 characters for the moment. We also give a general (structural)
226 overview first, followed later by the exact details.)
228 A. Internal String Encoding
230 ASCII characters are encoded using their position code directly.
231 Other characters are encoded using their leading byte followed
232 by their position code(s) with the high bit set. Characters
233 in private character sets have their leading byte prefixed with
234 a "leading byte prefix", which is either 0x9E or 0x9F. (No
235 character sets are ever assigned these leading bytes.) Specifically:
237 Character set Encoding (PC == position-code)
238 ------------- -------- (LB == leading-byte)
240 Control-1 LB | PC1 + 0xA0
241 Dimension-1 official LB | PC1 + 0x80
242 Dimension-1 private 0x9E | LB | PC1 + 0x80
243 Dimension-2 official LB | PC1 | PC2 + 0x80
244 Dimension-2 private 0x9F | LB | PC1 + 0x80 | PC2 + 0x80
246 The basic characteristic of this encoding is that the first byte
247 of all characters is in the range 0x00 - 0x9F, and the second and
248 following bytes of all characters is in the range 0xA0 - 0xFF.
249 This means that it is impossible to get out of sync, or more
252 1. Given any byte position, the beginning of the character it is
253 within can be determined in constant time.
254 2. Given any byte position at the beginning of a character, the
255 beginning of the next character can be determined in constant
257 3. Given any byte position at the beginning of a character, the
258 beginning of the previous character can be determined in constant
260 4. Textual searches can simply treat encoded strings as if they
261 were encoded in a one-byte-per-character fashion rather than
262 the actual multi-byte encoding.
264 None of the standard non-modal encodings meet all of these
265 conditions. For example, EUC satisfies only (2) and (3), while
266 Shift-JIS and Big5 (not yet described) satisfy only (2). (All
267 non-modal encodings must satisfy (2), in order to be unambiguous.)
269 B. Internal Character Encoding
271 One 19-bit word represents a single character. The word is
272 separated into three fields:
274 Bit number: 18 17 16 15 14 13 12 11 10 09 08 07 06 05 04 03 02 01 00
275 <------------> <------------------> <------------------>
278 Note that fields 2 and 3 hold 7 bits each, while field 1 holds 5 bits.
280 Character set Field 1 Field 2 Field 3
281 ------------- ------- ------- -------
286 Dimension-1 official 0 LB - 0x80 PC1
287 range: (01 - 0D) (20 - 7F)
288 Dimension-1 private 0 LB - 0x80 PC1
289 range: (20 - 6F) (20 - 7F)
290 Dimension-2 official LB - 0x8F PC1 PC2
291 range: (01 - 0A) (20 - 7F) (20 - 7F)
292 Dimension-2 private LB - 0xE1 PC1 PC2
293 range: (0F - 1E) (20 - 7F) (20 - 7F)
296 Note that character codes 0 - 255 are the same as the "binary encoding"
301 About Unicode support:
303 Adding Unicode support is very desirable. Unicode will likely be a
304 very common representation in the future, and thus we should
305 represent Unicode characters using three bytes instead of four.
306 This means we need to find leading bytes for Unicode. Given that
307 there are 65,536 characters in Unicode and we can attach 96x96 =
308 9,216 characters per leading byte, we need eight leading bytes for
309 Unicode. We currently have four free (0x9A - 0x9D), and with a
310 little bit of rearranging we can get five: ASCII doesn't really
311 need to take up a leading byte. (We could just as well use 0x7F,
312 with a little change to the functions that assume that 0x80 is the
313 lowest leading byte.) This means we still need to dump three
314 leading bytes and move them into private space. The CNS charsets
315 are good candidates since they are rarely used, and
316 JAPANESE_JISX0208_1978 is becoming less and less used and could
319 /************************************************************************/
320 /* Definition of leading bytes */
321 /************************************************************************/
323 #define MIN_LEADING_BYTE 0x80
324 /* These need special treatment in a string and/or character */
325 #define LEADING_BYTE_ASCII 0x8E /* Omitted in a buffer */
326 #ifdef ENABLE_COMPOSITE_CHARS
328 #define LEADING_BYTE_COMPOSITE 0x80 /* for a composite character */
329 #define LEADING_BYTE_CONTROL_1 0x8F /* represent normal 80-9F */
331 /* Note the gap in each official charset can cause core dump
332 as first and last values are used to determine whether
333 charset is defined or not in non_ascii_valid_char_p */
335 /** The following are for 1-byte characters in an official charset. **/
336 enum LEADING_BYTE_OFFICIAL_1 {
337 LEADING_BYTE_LATIN_ISO8859_1 = 0x81, /* Right half of ISO 8859-1 */
338 LEADING_BYTE_LATIN_ISO8859_2, /* 0x82 Right half of ISO 8859-2 */
339 LEADING_BYTE_LATIN_ISO8859_3, /* 0x83 Right half of ISO 8859-3 */
340 LEADING_BYTE_LATIN_ISO8859_4, /* 0x84 Right half of ISO 8859-4 */
341 LEADING_BYTE_THAI_TIS620, /* 0x85 TIS620-2533 */
342 LEADING_BYTE_GREEK_ISO8859_7, /* 0x86 Right half of ISO 8859-7 */
343 LEADING_BYTE_ARABIC_ISO8859_6, /* 0x87 Right half of ISO 8859-6 */
344 LEADING_BYTE_HEBREW_ISO8859_8, /* 0x88 Right half of ISO 8859-8 */
345 LEADING_BYTE_KATAKANA_JISX0201, /* 0x89 Right half of JIS X0201-1976 */
346 LEADING_BYTE_LATIN_JISX0201, /* 0x8A Left half of JIS X0201-1976 */
347 LEADING_BYTE_CYRILLIC_ISO8859_5, /* 0x8B Right half of ISO 8859-5 */
348 LEADING_BYTE_LATIN_ISO8859_9 /* 0x8C Right half of ISO 8859-9 */
352 #define MIN_LEADING_BYTE_OFFICIAL_1 LEADING_BYTE_LATIN_ISO8859_1
353 #define MAX_LEADING_BYTE_OFFICIAL_1 LEADING_BYTE_LATIN_ISO8859_9
355 /** The following are for 2-byte characters in an official charset. **/
356 enum LEADING_BYTE_OFFICIAL_2 {
357 LEADING_BYTE_JAPANESE_JISX0208_1978 = 0x90, /* Japanese JIS X0208-1978 */
358 LEADING_BYTE_CHINESE_GB2312, /* 0x91 Chinese Hanzi GB2312-1980 */
359 LEADING_BYTE_JAPANESE_JISX0208, /* 0x92 Japanese JIS X0208-1983 */
360 LEADING_BYTE_KOREAN_KSC5601, /* 0x93 Hangul KS C5601-1987 */
361 LEADING_BYTE_JAPANESE_JISX0212, /* 0x94 Japanese JIS X0212-1990 */
362 LEADING_BYTE_CHINESE_CNS11643_1, /* 0x95 Chinese CNS11643 Set 1 */
363 LEADING_BYTE_CHINESE_CNS11643_2, /* 0x96 Chinese CNS11643 Set 2 */
364 LEADING_BYTE_CHINESE_BIG5_1, /* 0x97 Big5 Level 1 */
365 LEADING_BYTE_CHINESE_BIG5_2 /* 0x98 Big5 Level 2 */
372 #define MIN_LEADING_BYTE_OFFICIAL_2 LEADING_BYTE_JAPANESE_JISX0208_1978
373 #define MAX_LEADING_BYTE_OFFICIAL_2 LEADING_BYTE_CHINESE_BIG5_2
375 /** The following are for 1- and 2-byte characters in a private charset. **/
377 #define PRE_LEADING_BYTE_PRIVATE_1 0x9E /* 1-byte char-set */
378 #define PRE_LEADING_BYTE_PRIVATE_2 0x9F /* 2-byte char-set */
380 #define MIN_LEADING_BYTE_PRIVATE_1 0xA0
381 #define MAX_LEADING_BYTE_PRIVATE_1 0xEF
382 #define MIN_LEADING_BYTE_PRIVATE_2 0xF0
383 #define MAX_LEADING_BYTE_PRIVATE_2 0xFF
385 #define NUM_LEADING_BYTES 128
387 /************************************************************************/
388 /* Operations on leading bytes */
389 /************************************************************************/
391 /* Is this leading byte for a private charset? */
393 #define LEADING_BYTE_PRIVATE_P(lb) ((lb) >= MIN_LEADING_BYTE_PRIVATE_1)
395 /* Is this a prefix for a private leading byte? */
397 extern_inline int LEADING_BYTE_PREFIX_P(Bufbyte lb);
398 extern_inline int LEADING_BYTE_PREFIX_P(Bufbyte lb)
400 return (lb == PRE_LEADING_BYTE_PRIVATE_1 ||
401 lb == PRE_LEADING_BYTE_PRIVATE_2);
404 /* Given a private leading byte, return the leading byte prefix stored
407 #define PRIVATE_LEADING_BYTE_PREFIX(lb) \
408 ((unsigned int) (lb) < MIN_LEADING_BYTE_PRIVATE_2 ? \
409 PRE_LEADING_BYTE_PRIVATE_1 : \
410 PRE_LEADING_BYTE_PRIVATE_2)
412 /************************************************************************/
413 /* Operations on individual bytes */
415 /************************************************************************/
417 /* These are carefully designed to work if BYTE is signed or unsigned. */
418 /* Note that SPC and DEL are considered ASCII, not control. */
420 #define BYTE_ASCII_P(byte) (((byte) & ~0x7f) == 0)
421 #define BYTE_C0_P(byte) (((byte) & ~0x1f) == 0)
422 #define BYTE_C1_P(byte) (((byte) & ~0x1f) == 0x80)
424 /************************************************************************/
425 /* Operations on individual bytes */
426 /* in a Mule-formatted string */
427 /************************************************************************/
429 /* Does BYTE represent the first byte of a character? */
431 #define BUFBYTE_FIRST_BYTE_P(byte) ((byte) < 0xA0)
433 /* Does BYTE represent the first byte of a multi-byte character? */
435 #define BUFBYTE_LEADING_BYTE_P(byte) BYTE_C1_P (byte)
437 /************************************************************************/
438 /* Information about a particular character set */
439 /************************************************************************/
441 struct Lisp_Charset {
442 struct lcrecord_header header;
446 Lisp_Object doc_string;
447 Lisp_Object registry;
448 Lisp_Object short_name;
449 Lisp_Object long_name;
451 Lisp_Object reverse_direction_charset;
453 Lisp_Object ccl_program;
455 /* Final byte of this character set in ISO2022 designating escape sequence */
458 /* Number of bytes (1 - 4) required in the internal representation
459 for characters in this character set. This is *not* the
460 same as the dimension of the character set). */
461 unsigned int rep_bytes;
463 /* Number of columns a character in this charset takes up, on TTY
464 devices. Not used for X devices. */
465 unsigned int columns;
467 /* Direction of this character set */
468 unsigned int direction;
470 /* Type of this character set (94, 96, 94x94, 96x96) */
473 /* Number of bytes used in encoding of this character set (1 or 2) */
474 unsigned int dimension;
476 /* Number of chars in each dimension (usually 94 or 96) */
479 /* Which half of font to be used to display this character set */
480 unsigned int graphic;
482 typedef struct Lisp_Charset Lisp_Charset;
484 DECLARE_LRECORD(charset, Lisp_Charset);
485 #define XCHARSET(x) XRECORD (x, charset, Lisp_Charset)
486 #define XSETCHARSET(x, p) XSETRECORD (x, p, charset)
487 #define CHARSETP(x) RECORDP (x, charset)
488 #define CHECK_CHARSET(x) CHECK_RECORD (x, charset)
489 #define CONCHECK_CHARSET(x) CONCHECK_RECORD (x, charset)
491 #define CHARSET_TYPE_94 0 /* This charset includes 94 characters. */
492 #define CHARSET_TYPE_96 1 /* This charset includes 96 characters. */
493 #define CHARSET_TYPE_94X94 2 /* This charset includes 94x94 characters. */
494 #define CHARSET_TYPE_96X96 3 /* This charset includes 96x96 characters. */
496 #define CHARSET_LEFT_TO_RIGHT 0
497 #define CHARSET_RIGHT_TO_LEFT 1
499 /* Leading byte and id have been regrouped. -- OG */
500 #define CHARSET_ID(cs) ((cs)->id)
501 #define CHARSET_LEADING_BYTE(cs) ((Bufbyte) CHARSET_ID(cs))
502 #define CHARSET_NAME(cs) ((cs)->name)
503 #define CHARSET_SHORT_NAME(cs) ((cs)->short_name)
504 #define CHARSET_LONG_NAME(cs) ((cs)->long_name)
505 #define CHARSET_REP_BYTES(cs) ((cs)->rep_bytes)
506 #define CHARSET_COLUMNS(cs) ((cs)->columns)
507 #define CHARSET_GRAPHIC(cs) ((cs)->graphic)
508 #define CHARSET_TYPE(cs) ((cs)->type)
509 #define CHARSET_DIRECTION(cs) ((cs)->direction)
510 #define CHARSET_FINAL(cs) ((cs)->final)
511 #define CHARSET_DOC_STRING(cs) ((cs)->doc_string)
512 #define CHARSET_REGISTRY(cs) ((cs)->registry)
513 #define CHARSET_CCL_PROGRAM(cs) ((cs)->ccl_program)
514 #define CHARSET_DIMENSION(cs) ((cs)->dimension)
515 #define CHARSET_CHARS(cs) ((cs)->chars)
516 #define CHARSET_REVERSE_DIRECTION_CHARSET(cs) ((cs)->reverse_direction_charset)
518 #define CHARSET_PRIVATE_P(cs) LEADING_BYTE_PRIVATE_P (CHARSET_LEADING_BYTE (cs))
520 #define XCHARSET_ID(cs) CHARSET_ID (XCHARSET (cs))
521 #define XCHARSET_NAME(cs) CHARSET_NAME (XCHARSET (cs))
522 #define XCHARSET_SHORT_NAME(cs) CHARSET_SHORT_NAME (XCHARSET (cs))
523 #define XCHARSET_LONG_NAME(cs) CHARSET_LONG_NAME (XCHARSET (cs))
524 #define XCHARSET_REP_BYTES(cs) CHARSET_REP_BYTES (XCHARSET (cs))
525 #define XCHARSET_COLUMNS(cs) CHARSET_COLUMNS (XCHARSET (cs))
526 #define XCHARSET_GRAPHIC(cs) CHARSET_GRAPHIC (XCHARSET (cs))
527 #define XCHARSET_TYPE(cs) CHARSET_TYPE (XCHARSET (cs))
528 #define XCHARSET_DIRECTION(cs) CHARSET_DIRECTION (XCHARSET (cs))
529 #define XCHARSET_FINAL(cs) CHARSET_FINAL (XCHARSET (cs))
530 #define XCHARSET_DOC_STRING(cs) CHARSET_DOC_STRING (XCHARSET (cs))
531 #define XCHARSET_REGISTRY(cs) CHARSET_REGISTRY (XCHARSET (cs))
532 #define XCHARSET_LEADING_BYTE(cs) CHARSET_LEADING_BYTE (XCHARSET (cs))
533 #define XCHARSET_CCL_PROGRAM(cs) CHARSET_CCL_PROGRAM (XCHARSET (cs))
534 #define XCHARSET_DIMENSION(cs) CHARSET_DIMENSION (XCHARSET (cs))
535 #define XCHARSET_CHARS(cs) CHARSET_CHARS (XCHARSET (cs))
536 #define XCHARSET_PRIVATE_P(cs) CHARSET_PRIVATE_P (XCHARSET (cs))
537 #define XCHARSET_REVERSE_DIRECTION_CHARSET(cs) \
538 CHARSET_REVERSE_DIRECTION_CHARSET (XCHARSET (cs))
540 struct charset_lookup {
541 /* Table of charsets indexed by leading byte. */
542 Lisp_Object charset_by_leading_byte[128];
544 /* Table of charsets indexed by type/final-byte/direction. */
545 Lisp_Object charset_by_attributes[4][128][2];
546 Bufbyte next_allocated_1_byte_leading_byte;
547 Bufbyte next_allocated_2_byte_leading_byte;
550 extern_inline Lisp_Object CHARSET_BY_LEADING_BYTE(Bufbyte lb);
551 extern_inline Lisp_Object CHARSET_BY_LEADING_BYTE(Bufbyte lb)
553 extern struct charset_lookup *chlook;
555 #ifdef ERROR_CHECK_TYPECHECK
556 /* When error-checking is on, x86 GCC 2.95.2 -O3 miscompiles the
557 following unless we introduce `tem'. */
559 type_checking_assert(tem >= 0x80 && tem <= 0xFF);
561 return chlook->charset_by_leading_byte[lb - 128];
564 extern_inline Lisp_Object
565 CHARSET_BY_ATTRIBUTES(unsigned int type, unsigned char final, int dir);
566 extern_inline Lisp_Object
567 CHARSET_BY_ATTRIBUTES(unsigned int type, unsigned char final, int dir)
569 extern struct charset_lookup *chlook;
571 type_checking_assert(type < countof(chlook->charset_by_attributes) &&
572 final < countof(chlook->charset_by_attributes[0])
574 countof(chlook->charset_by_attributes[0][0]));
575 return chlook->charset_by_attributes[type][final][dir];
578 /* Table of number of bytes in the string representation of a character
579 indexed by the first byte of that representation.
581 This value can be derived in other ways -- e.g. something like
582 XCHARSET_REP_BYTES (CHARSET_BY_LEADING_BYTE (first_byte))
583 but it's faster this way. */
584 extern const Bytecount rep_bytes_by_first_byte[0xA0];
586 /* Number of bytes in the string representation of a character. */
587 extern_inline int REP_BYTES_BY_FIRST_BYTE(Bufbyte fb);
588 extern_inline int REP_BYTES_BY_FIRST_BYTE(Bufbyte fb)
590 type_checking_assert(fb < 0xA0);
591 return rep_bytes_by_first_byte[fb];
594 /************************************************************************/
595 /* Dealing with characters */
596 /************************************************************************/
598 /* Is this character represented by more than one byte in a string? */
600 #define CHAR_MULTIBYTE_P(c) ((c) >= 0x80)
602 #define CHAR_ASCII_P(c) (!CHAR_MULTIBYTE_P (c))
604 /* The bit fields of character are divided into 3 parts:
605 FIELD1(5bits):FIELD2(7bits):FIELD3(7bits) */
607 #define CHAR_FIELD1_MASK (0x1F << 14)
608 #define CHAR_FIELD2_MASK (0x7F << 7)
609 #define CHAR_FIELD3_MASK 0x7F
611 /* Macros to access each field of a character code of C. */
613 #define CHAR_FIELD1(c) (((c) & CHAR_FIELD1_MASK) >> 14)
614 #define CHAR_FIELD2(c) (((c) & CHAR_FIELD2_MASK) >> 7)
615 #define CHAR_FIELD3(c) ((c) & CHAR_FIELD3_MASK)
617 /* Field 1, if non-zero, usually holds a leading byte for a
618 dimension-2 charset. Field 2, if non-zero, usually holds a leading
619 byte for a dimension-1 charset. */
621 /* Converting between field values and leading bytes. */
623 #define FIELD2_TO_OFFICIAL_LEADING_BYTE 0x80
624 #define FIELD2_TO_PRIVATE_LEADING_BYTE 0x80
626 #define FIELD1_TO_OFFICIAL_LEADING_BYTE 0x8F
627 #define FIELD1_TO_PRIVATE_LEADING_BYTE 0xE1
629 /* Minimum and maximum allowed values for the fields. */
631 #define MIN_CHAR_FIELD2_OFFICIAL \
632 (MIN_LEADING_BYTE_OFFICIAL_1 - FIELD2_TO_OFFICIAL_LEADING_BYTE)
633 #define MAX_CHAR_FIELD2_OFFICIAL \
634 (MAX_LEADING_BYTE_OFFICIAL_1 - FIELD2_TO_OFFICIAL_LEADING_BYTE)
636 #define MIN_CHAR_FIELD1_OFFICIAL \
637 (MIN_LEADING_BYTE_OFFICIAL_2 - FIELD1_TO_OFFICIAL_LEADING_BYTE)
638 #define MAX_CHAR_FIELD1_OFFICIAL \
639 (MAX_LEADING_BYTE_OFFICIAL_2 - FIELD1_TO_OFFICIAL_LEADING_BYTE)
641 #define MIN_CHAR_FIELD2_PRIVATE \
642 (MIN_LEADING_BYTE_PRIVATE_1 - FIELD2_TO_PRIVATE_LEADING_BYTE)
643 #define MAX_CHAR_FIELD2_PRIVATE \
644 (MAX_LEADING_BYTE_PRIVATE_1 - FIELD2_TO_PRIVATE_LEADING_BYTE)
646 #define MIN_CHAR_FIELD1_PRIVATE \
647 (MIN_LEADING_BYTE_PRIVATE_2 - FIELD1_TO_PRIVATE_LEADING_BYTE)
648 #define MAX_CHAR_FIELD1_PRIVATE \
649 (MAX_LEADING_BYTE_PRIVATE_2 - FIELD1_TO_PRIVATE_LEADING_BYTE)
651 /* Minimum character code of each <type> character. */
653 #define MIN_CHAR_OFFICIAL_TYPE9N (MIN_CHAR_FIELD2_OFFICIAL << 7)
654 #define MIN_CHAR_PRIVATE_TYPE9N (MIN_CHAR_FIELD2_PRIVATE << 7)
655 #define MIN_CHAR_OFFICIAL_TYPE9NX9N (MIN_CHAR_FIELD1_OFFICIAL << 14)
656 #define MIN_CHAR_PRIVATE_TYPE9NX9N (MIN_CHAR_FIELD1_PRIVATE << 14)
657 #define MIN_CHAR_COMPOSITION (0x1F << 14)
659 /* Leading byte of a character.
661 NOTE: This takes advantage of the fact that
662 FIELD2_TO_OFFICIAL_LEADING_BYTE and
663 FIELD2_TO_PRIVATE_LEADING_BYTE are the same.
666 extern_inline Bufbyte CHAR_LEADING_BYTE(Emchar c);
667 extern_inline Bufbyte CHAR_LEADING_BYTE(Emchar c)
670 return LEADING_BYTE_ASCII;
672 return LEADING_BYTE_CONTROL_1;
673 else if (c < MIN_CHAR_OFFICIAL_TYPE9NX9N)
674 return CHAR_FIELD2(c) + FIELD2_TO_OFFICIAL_LEADING_BYTE;
675 else if (c < MIN_CHAR_PRIVATE_TYPE9NX9N)
676 return CHAR_FIELD1(c) + FIELD1_TO_OFFICIAL_LEADING_BYTE;
677 else if (c < MIN_CHAR_COMPOSITION)
678 return CHAR_FIELD1(c) + FIELD1_TO_PRIVATE_LEADING_BYTE;
680 #ifdef ENABLE_COMPOSITE_CHARS
681 return LEADING_BYTE_COMPOSITE;
685 #endif /* ENABLE_COMPOSITE_CHARS */
689 #define CHAR_CHARSET(c) CHARSET_BY_LEADING_BYTE (CHAR_LEADING_BYTE (c))
691 /* Return a character whose charset is CHARSET and position-codes
692 are C1 and C2. TYPE9N character ignores C2.
694 NOTE: This takes advantage of the fact that
695 FIELD2_TO_OFFICIAL_LEADING_BYTE and
696 FIELD2_TO_PRIVATE_LEADING_BYTE are the same.
699 extern_inline Emchar MAKE_CHAR(Lisp_Object charset, int c1, int c2);
700 extern_inline Emchar MAKE_CHAR(Lisp_Object charset, int c1, int c2)
702 if (EQ(charset, Vcharset_ascii))
704 else if (EQ(charset, Vcharset_control_1))
706 #ifdef ENABLE_COMPOSITE_CHARS
707 else if (EQ(charset, Vcharset_composite))
708 return (0x1F << 14) | ((c1) << 7) | (c2);
710 else if (XCHARSET_DIMENSION(charset) == 1)
711 return ((XCHARSET_LEADING_BYTE(charset) -
712 FIELD2_TO_OFFICIAL_LEADING_BYTE) << 7) | (c1);
713 else if (!XCHARSET_PRIVATE_P(charset))
714 return ((XCHARSET_LEADING_BYTE(charset) -
715 FIELD1_TO_OFFICIAL_LEADING_BYTE) << 14) | ((c1) << 7) |
718 return ((XCHARSET_LEADING_BYTE(charset) -
719 FIELD1_TO_PRIVATE_LEADING_BYTE) << 14) | ((c1) << 7) |
723 /* The charset of character C is set to CHARSET, and the
724 position-codes of C are set to C1 and C2. C2 of TYPE9N character
727 /* BREAKUP_CHAR_1_UNSAFE assumes that the charset has already been
728 calculated, and just computes c1 and c2.
730 BREAKUP_CHAR also computes and stores the charset. */
732 #define BREAKUP_CHAR_1_UNSAFE(c, charset, c1, c2) \
733 XCHARSET_DIMENSION (charset) == 1 \
734 ? ((c1) = CHAR_FIELD3 (c), (c2) = 0) \
735 : ((c1) = CHAR_FIELD2 (c), \
736 (c2) = CHAR_FIELD3 (c))
738 extern_inline void breakup_char_1(Emchar c, Lisp_Object * charset, int *c1,
740 extern_inline void breakup_char_1(Emchar c, Lisp_Object * charset, int *c1,
743 *charset = CHAR_CHARSET(c);
744 BREAKUP_CHAR_1_UNSAFE(c, *charset, *c1, *c2);
747 #define BREAKUP_CHAR(c, charset, c1, c2) \
748 breakup_char_1 (c, &(charset), &(c1), &(c2))
750 #ifdef ENABLE_COMPOSITE_CHARS
751 /************************************************************************/
752 /* Composite characters */
753 /************************************************************************/
755 Emchar lookup_composite_char(Bufbyte * str, int len);
756 Lisp_Object composite_char_string(Emchar ch);
757 #endif /* ENABLE_COMPOSITE_CHARS */
759 /************************************************************************/
760 /* Exported functions */
761 /************************************************************************/
763 EXFUN(Ffind_charset, 1);
764 EXFUN(Fget_charset, 1);
766 extern Lisp_Object Vcharset_chinese_big5_1;
767 extern Lisp_Object Vcharset_chinese_big5_2;
768 extern Lisp_Object Vcharset_japanese_jisx0208;
770 Emchar Lstream_get_emchar_1(Lstream * stream, int first_char);
771 int Lstream_fput_emchar(Lstream * stream, Emchar ch);
772 void Lstream_funget_emchar(Lstream * stream, Emchar ch);
774 int copy_internal_to_external(const Bufbyte * internal, Bytecount len,
775 unsigned char *external);
776 Bytecount copy_external_to_internal(const unsigned char *external,
777 int len, Bufbyte * internal);
779 #endif /* INCLUDED_mule_charset_h_ */