2 * iso2022.c - support for ISO/IEC 2022 (alias ECMA-35).
4 * This isn't a complete implementation of ISO/IEC 2022, but it's
5 * close. It only handles decoding, because a fully general encoder
6 * isn't really useful. It can decode 8-bit and 7-bit versions, with
7 * support for single-byte and multi-byte character sets, all four
8 * containers (G0, G1, G2, and G3), using both single-shift and
9 * locking-shift sequences.
11 * The general principle is that any valid ISO/IEC 2022 sequence
12 * should either be correctly decoded or should emit an ERROR. The
13 * only exception to this is that the C0 and C1 sets are fixed as
14 * those of ISO/IEC 6429. Escape sequences for designating control
15 * sets are passed through, so a post-processor could fix them up if
18 * DOCS is not currently supported. It will be one day.
35 enum {S4
, S6
, M4
, M6
};
37 static long int null_dbcs_to_unicode(int, int);
39 const struct iso2022_subcharset
{
42 const sbcs_data
*sbcs_base
;
43 long int (*dbcs_fn
)(int, int);
44 } iso2022_subcharsets
[] = {
45 { S4
, 0, 'B', 0x00, &sbcsdata_CS_ASCII
},
47 { S4
, 0, '<', 0x80, &sbcsdata_CS_DEC_MCS
},
48 { S4
, 0, 'I', 0x80, &sbcsdata_CS_JISX0201
},
49 { S4
, 0, 'J', 0x00, &sbcsdata_CS_JISX0201
},
51 { S6
, 0, 'A', 0x80, &sbcsdata_CS_ISO8859_1
},
52 { S6
, 0, 'B', 0x80, &sbcsdata_CS_ISO8859_2
},
53 { S6
, 0, 'C', 0x80, &sbcsdata_CS_ISO8859_3
},
54 { S6
, 0, 'D', 0x80, &sbcsdata_CS_ISO8859_4
},
55 { S6
, 0, 'F', 0x80, &sbcsdata_CS_ISO8859_7
},
56 { S6
, 0, 'G', 0x80, &sbcsdata_CS_ISO8859_6
},
57 { S6
, 0, 'H', 0x80, &sbcsdata_CS_ISO8859_8
},
58 { S6
, 0, 'L', 0x80, &sbcsdata_CS_ISO8859_5
},
59 { S6
, 0, 'M', 0x80, &sbcsdata_CS_ISO8859_9
},
60 { S6
, 0, 'T', 0x80, &sbcsdata_CS_ISO8859_11
},
61 { S6
, 0, 'V', 0x80, &sbcsdata_CS_ISO8859_10
},
62 { S6
, 0, 'Y', 0x80, &sbcsdata_CS_ISO8859_13
},
63 { S6
, 0, '_', 0x80, &sbcsdata_CS_ISO8859_14
},
64 { S6
, 0, 'b', 0x80, &sbcsdata_CS_ISO8859_15
},
65 { S6
, 0, 'f', 0x80, &sbcsdata_CS_ISO8859_16
},
66 { S6
, 0, '~' }, /* empty 96-set */
68 { M4
, 0, '@' }, /* JIS C 6226-1978 */
70 { M4
, 0, 'A', -0x21, 0, &gb2312_to_unicode
},
71 { M4
, 0, 'B', -0x21, 0, &jisx0208_to_unicode
},
72 { M4
, 0, 'C', -0x21, 0, &ksx1001_to_unicode
},
73 { M4
, 0, 'D', -0x21, 0, &jisx0212_to_unicode
},
74 { M4
, 0, '~', 0, 0, &null_dbcs_to_unicode
}, /* empty 94^n-set */
75 { M6
, 0, '~', 0, 0, &null_dbcs_to_unicode
}, /* empty 96^n-set */
78 static long int null_dbcs_to_unicode(int r
, int c
)
83 /* States, or "what we're currently accumulating". */
85 IDLE
, /* None of the below */
86 SS2CHAR
, /* Accumulating a character after SS2 */
87 SS3CHAR
, /* Accumulating a character after SS3 */
88 ESCSEQ
, /* Accumulating an escape sequence */
89 ESCDROP
, /* Discarding an escape sequence */
90 ESCPASS
/* Passing through an escape sequence */
95 static void dump_state(charset_state
*s
)
97 unsigned s0
= s
->s0
, s1
= s
->s1
;
98 char const * const modes
[] = { "IDLE", "SS2CHAR", "SS3CHAR",
99 "ESCSEQ", "ESCDROP", "ESCPASS" };
101 fprintf(stderr
, "s0: %s", modes
[s0
>> 29]);
102 fprintf(stderr
, " %02x %02x %02x ", (s0
>> 16) & 0xff, (s0
>> 8) & 0xff,
104 fprintf(stderr
, "s1: LS%d LS%dR", (s1
>> 30) & 3, (s1
>> 28) & 3);
105 fprintf(stderr
, " %d %d %d %d\n", s1
& 0x7f, (s1
>> 7) & 0x7f,
106 (s1
>> 14) & 0x7f, (s1
>> 21) & 0x7f);
110 static void designate(charset_state
*state
, int container
,
111 int type
, int ibyte
, int fbyte
)
115 assert(container
>= 0 && container
<= 3);
116 assert(type
== S4
|| type
== S6
|| type
== M4
|| type
== M6
);
118 for (i
= 0; i
<= lenof(iso2022_subcharsets
); i
++) {
119 if (iso2022_subcharsets
[i
].type
== type
&&
120 iso2022_subcharsets
[i
].i
== ibyte
&&
121 iso2022_subcharsets
[i
].f
== fbyte
) {
122 state
->s1
&= ~(0x7fL
<< (container
* 7));
123 state
->s1
|= (i
<< (container
* 7));
128 * If we don't find the charset, invoke the empty one, so we
129 * output ERROR rather than garbage.
131 designate(state
, container
, type
, 0, '~');
134 static void read_iso2022(charset_spec
const *charset
, long int input_chr
,
135 charset_state
*state
,
136 void (*emit
)(void *ctx
, long int output
),
140 /* dump_state(state); */
142 * We've got 64 bits of state to play with.
144 * Locking-shift state: 2 bits each GL/GR
145 * Single-shift state: 2 bits
146 * Charset designation state: n bits each G0/G1/G2/G3
147 * MBCS/esc seq accumulation: 14 bits (assume max 4-byte sets)
148 * MBCS state: 2 bits (off, ESC, GL, GR)
149 * For no good reason, put long-term state in s1, short term in s0.
151 * s0[31:29] = state enum
152 * s0[24:0] = accumulated bytes
153 * s1[31:30] = GL locking-shift state
154 * s1[29:28] = GR locking-shift state
155 * s1[27:21] = G3 charset
156 * s1[20:14] = G2 charset
157 * s1[13:7] = G1 charset
158 * s1[6:0] = G0 charset
163 #define LOCKING_SHIFT(n,side) \
164 (state->s1 = (state->s1 & ~(3L<<(side))) | ((n ## L)<<(side)))
165 #define MODE ((state->s0 & 0xe0000000L) >> 29)
166 #define ENTER_MODE(m) (state->s0 = (state->s0 & ~0xe0000000L) | ((m)<<29))
167 #define SINGLE_SHIFT(n) ENTER_MODE(SS2CHAR - 2 + (n))
168 #define ASSERT_IDLE do { \
169 if (state->s0 != 0) emit(emitctx, ERROR); \
173 if (state
->s1
== 0) {
175 * Since there's no LS0R, this means we must just have started.
176 * Set up a sane initial state (LS0, LS1R, ASCII in G0/G1/G2/G3).
178 LOCKING_SHIFT(0, LEFT
);
179 LOCKING_SHIFT(1, RIGHT
);
180 designate(state
, 0, S4
, 0, 'B');
181 designate(state
, 1, S4
, 0, 'B');
182 designate(state
, 2, S4
, 0, 'B');
183 designate(state
, 3, S4
, 0, 'B');
186 if ((input_chr
& 0x60) == 0x00) {
187 /* C0 or C1 control */
194 LOCKING_SHIFT(0, LEFT
);
197 LOCKING_SHIFT(1, LEFT
);
206 emit(emitctx
, input_chr
);
209 } else if ((input_chr
& 0x80) || MODE
< ESCSEQ
) {
211 struct iso2022_subcharset
const *subcs
;
216 * Force idle state if we're in mid escape sequence, or in a
217 * multi-byte character with a different top bit.
219 if (MODE
>= ESCSEQ
||
220 ((state
->s0
& 0x00ff0000L
) != 0 &&
221 (((state
->s0
>> 16) ^ input_chr
) & 0x80)))
223 if (MODE
== SS2CHAR
|| MODE
== SS3CHAR
) /* Single-shift */
224 container
= MODE
- SS2CHAR
+ 2;
225 else if (input_chr
>= 0x80) /* GR */
226 container
= (state
->s1
>> 28) & 3;
228 container
= state
->s1
>> 30;
231 input_7bit
= input_chr
& ~0x80;
232 subcs
= &iso2022_subcharsets
[(state
->s1
>> (container
* 7)) & 0x7f];
233 if ((subcs
->type
== S4
|| subcs
->type
== M4
) &&
234 (input_7bit
== 0x20 || input_7bit
== 0x7f)) {
235 /* characters not in 94-char set */
236 if (is_gl
) emit(emitctx
, input_7bit
);
237 else emit(emitctx
, ERROR
);
238 } else if (subcs
->type
== M4
|| subcs
->type
== M6
) {
239 if ((state
->s0
& 0x00ff0000L
) == 0) {
240 state
->s0
|= input_chr
<< 16;
244 subcs
->dbcs_fn(((state
->s0
>> 16) & 0x7f) + subcs
->offset
,
245 input_7bit
+ subcs
->offset
));
248 if ((state
->s0
& 0x00ff0000L
) != 0)
249 emit(emitctx
, ERROR
);
250 emit(emitctx
, subcs
->sbcs_base ?
251 sbcs_to_unicode(subcs
->sbcs_base
, input_7bit
+ subcs
->offset
):
257 if (MODE
== ESCPASS
) {
258 emit(emitctx
, input_chr
);
259 if ((input_chr
& 0xf0) != 0x20)
265 * Intermediate bytes shall be any of the 16 positions of
266 * column 02 of the code table; they are denoted by the symbol
269 if ((input_chr
& 0xf0) == 0x20) {
270 if (((state
->s0
>> 16) & 0xff) == 0)
271 state
->s0
|= input_chr
<< 16;
272 else if (((state
->s0
>> 8) & 0xff) == 0)
273 state
->s0
|= input_chr
<< 8;
275 /* Long escape sequence. Switch to ESCPASS or ESCDROP. */
276 i1
= (state
->s0
>> 16) & 0xff;
277 i2
= (state
->s0
>> 8) & 0xff;
279 case '(': case ')': case '*': case '+':
280 case '-': case '.': case '/':
288 emit(emitctx
, input_chr
);
298 * Final bytes shall be any of the 79 positions of columns 03
299 * to 07 of the code table excluding position 07/15; they are
300 * denoted by the symbol F.
302 i1
= (state
->s0
>> 16) & 0xff;
303 i2
= (state
->s0
>> 8) & 0xff;
305 input_chr
= 0; /* Make sure it won't match. */
308 case 0: /* No intermediate bytes */
317 LOCKING_SHIFT(2, LEFT
);
320 LOCKING_SHIFT(3, LEFT
);
323 LOCKING_SHIFT(3, RIGHT
);
326 LOCKING_SHIFT(2, RIGHT
);
329 LOCKING_SHIFT(1, RIGHT
);
332 /* Unsupported escape sequence. Spit it back out. */
334 emit(emitctx
, input_chr
);
339 * Various coding structure facilities specify that designating
340 * a code element also invokes it. As far as I can see, invoking
341 * it now will have the same practical effect, since those
342 * facilities also ban the use of locking shifts.
345 case 'A': /* G0 element used and invoked into GL */
346 LOCKING_SHIFT(0, LEFT
);
348 case 'C': /* G0 in GL, G1 in GR */
349 case 'D': /* Ditto, at least for 8-bit codes */
350 case 'L': /* ISO 4873 (ECMA-43) level 1 */
351 case 'M': /* ISO 4873 (ECMA-43) level 2 */
352 LOCKING_SHIFT(0, LEFT
);
353 LOCKING_SHIFT(1, RIGHT
);
359 * IRR (Identify Revised Registration) is ignored here,
360 * since any revised registration must be
361 * upward-compatible with the old one, so either we'll
362 * support the new one or we'll emit ERROR when we run
363 * into a new character. In either case, there's nothing
367 case '(': /* GZD4 */ case ')': /* G1D4 */
368 case '*': /* G2D4 */ case '+': /* G3D4 */
369 designate(state
, i1
- '(', S4
, i2
, input_chr
);
371 case '-': /* G1D6 */ case '.': /* G2D6 */ case '/': /* G3D6 */
372 designate(state
, i1
- ',', S6
, i2
, input_chr
);
374 case '$': /* G?DM? */
376 case 0: /* Obsolete version of GZDM4 */
378 case '(': /* GZDM4 */ case ')': /* G1DM4 */
379 case '*': /* G2DM4 */ case '+': /* G3DM4 */
380 designate(state
, i2
- '(', M4
, 0, input_chr
);
382 case '-': /* G1DM6 */
383 case '.': /* G2DM6 */ case '/': /* G3DM6 */
384 designate(state
, i2
- ',', M6
, 0, input_chr
);
387 emit(emitctx
, ERROR
);
394 /* Unsupported nF escape sequence. Re-emit it. */
397 if (i2
) emit(emitctx
, i2
);
398 emit(emitctx
, input_chr
);
404 const charset_spec charset_CS_ISO2022
= {
405 CS_ISO2022
, read_iso2022
, NULL
, NULL
416 void iso2022_emit(void *ctx
, long output
)
418 wchar_t **p
= (wchar_t **)ctx
;
422 void iso2022_read_test(int line
, char *input
, int inlen
, ...)
425 wchar_t *p
, str
[512];
430 state
.s0
= state
.s1
= 0;
433 for (i
= 0; i
< inlen
; i
++)
434 read_iso2022(NULL
, input
[i
] & 0xFF, &state
, iso2022_emit
, &p
);
438 for (i
= 0; i
< p
- str
; i
++) {
439 l
= va_arg(ap
, long int);
441 printf("%d: correct string shorter than output\n", line
);
446 printf("%d: char %d came out as %08x, should be %08lx\n",
452 l
= va_arg(ap
, long int);
454 printf("%d: correct string longer than output\n", line
);
461 /* Macro to concoct the first three parameters of iso2022_read_test. */
462 #define TESTSTR(x) __LINE__, x, lenof(x)
466 printf("read tests beginning\n");
467 /* Simple test (Emacs sample text for Japanese, in ISO-2022-JP) */
468 iso2022_read_test(TESTSTR("Japanese (\x1b$BF|K\\8l\x1b(B)\t"
469 "\x1b$B$3$s$K$A$O\x1b(B, "
470 "\x1b$B%3%s%K%A%O\x1b(B\n"),
471 'J','a','p','a','n','e','s','e',' ','(',
472 0x65E5, 0x672C, 0x8A9E, ')', '\t',
473 0x3053, 0x3093, 0x306b, 0x3061, 0x306f, ',', ' ',
474 0x30b3, 0x30f3, 0x30cb, 0x30c1, 0x30cf, '\n', 0, -1);
475 /* Same thing in EUC-JP (with designations, and half-width katakana) */
476 iso2022_read_test(TESTSTR("\x1b$)B\x1b*I\x1b$+D"
477 "Japanese (\xc6\xfc\xcb\xdc\xb8\xec)\t"
478 "\xa4\xb3\xa4\xf3\xa4\xcb\xa4\xc1\xa4\xcf, "
479 "\x8e\xba\x8e\xdd\x8e\xc6\x8e\xc1\x8e\xca\n"),
480 'J','a','p','a','n','e','s','e',' ','(',
481 0x65E5, 0x672C, 0x8A9E, ')', '\t',
482 0x3053, 0x3093, 0x306b, 0x3061, 0x306f, ',', ' ',
483 0xff7a, 0xff9d, 0xff86, 0xff81, 0xff8a, '\n', 0, -1);
484 /* Multibyte single-shift */
485 iso2022_read_test(TESTSTR("\x1b$)B\x1b*I\x1b$+D\x8f\"/!"),
487 /* Non-existent SBCS */
488 iso2022_read_test(TESTSTR("\x1b(!Zfnord\n"),
489 ERROR
, ERROR
, ERROR
, ERROR
, ERROR
, '\n', 0, -1);
490 /* Pass-through of ordinary escape sequences, including a long one */
491 iso2022_read_test(TESTSTR("\x1b""b\x1b#5\x1b#!!!5"),
492 0x1B, 'b', 0x1B, '#', '5',
493 0x1B, '#', '!', '!', '!', '5', 0, -1);
494 /* Non-existent DBCS (also 5-byte escape sequence) */
495 iso2022_read_test(TESTSTR("\x1b$(!Bfnord!"),
496 ERROR
, ERROR
, ERROR
, 0, -1);
497 /* Incomplete DB characters */
498 iso2022_read_test(TESTSTR("\x1b$B(,(\x1b(BHi\x1b$B(,(\n"),
499 0x2501, ERROR
, 'H', 'i', 0x2501, ERROR
, '\n', 0, -1);
500 iso2022_read_test(TESTSTR("\x1b$)B\x1b*I\x1b$+D\xa4""B"),
502 iso2022_read_test(TESTSTR("\x1b$)B\x1b*I\x1b$+D\x0e\x1b|$\xa2\xaf"),
503 ERROR
, 0x02D8, 0, -1);
504 /* Incomplete escape sequence */
505 iso2022_read_test(TESTSTR("\x1b\n"), ERROR
, '\n', 0, -1);
506 iso2022_read_test(TESTSTR("\x1b-A\x1b~\x1b\xa1"), ERROR
, 0xa1, 0, -1);
507 /* Incomplete single-shift */
508 iso2022_read_test(TESTSTR("\x8e\n"), ERROR
, '\n', 0, -1);
509 iso2022_read_test(TESTSTR("\x1b$*B\x8e(\n"), ERROR
, '\n', 0, -1);
510 /* Corner cases (02/00 and 07/15) */
511 iso2022_read_test(TESTSTR("\x1b(B\x20\x7f"), 0x20, 0x7f, 0, -1);
512 iso2022_read_test(TESTSTR("\x1b(I\x20\x7f"), 0x20, 0x7f, 0, -1);
513 iso2022_read_test(TESTSTR("\x1b$B\x20\x7f"), 0x20, 0x7f, 0, -1);
514 iso2022_read_test(TESTSTR("\x1b-A\x0e\x20\x7f"), 0xa0, 0xff, 0, -1);
515 iso2022_read_test(TESTSTR("\x1b$-~\x0e\x20\x7f"), ERROR
, 0, -1);
516 iso2022_read_test(TESTSTR("\x1b)B\xa0\xff"), ERROR
, ERROR
, 0, -1);
517 iso2022_read_test(TESTSTR("\x1b)I\xa0\xff"), ERROR
, ERROR
, 0, -1);
518 iso2022_read_test(TESTSTR("\x1b$)B\xa0\xff"), ERROR
, ERROR
, 0, -1);
519 iso2022_read_test(TESTSTR("\x1b-A\x1b~\xa0\xff"), 0xa0, 0xff, 0, -1);
520 iso2022_read_test(TESTSTR("\x1b$-~\x1b~\xa0\xff"), ERROR
, 0, -1);
521 /* Designate control sets */
522 iso2022_read_test(TESTSTR("\x1b!@"), 0x1b, '!', '@', 0, -1);
523 printf("read tests completed\n");
524 printf("total: %d errors\n", total_errs
);
525 return (total_errs
!= 0);
528 #endif /* TESTMODE */
530 #else /* ENUM_CHARSETS */
532 ENUM_CHARSET(CS_ISO2022
)