2 * iso2022s.c - support for ISO-2022 subset encodings.
19 /* Functional description of a single ISO 2022 escape sequence. */
20 struct iso2022_escape
{
22 unsigned long andbits
, xorbits
;
24 * For output, these variables help us figure out which escape
25 * sequences we need to get where we want to be.
27 * `container' should be in the range 0-3, but can also be ORed
28 * with the bit flag RO to indicate that this is not a
29 * preferred container to use for this charset during output.
31 int container
, subcharset
;
37 * List of escape sequences supported in this subset. Must be
38 * in ASCII order, so that we can narrow down the list as
41 const struct iso2022_escape
*escapes
;/* must be sorted in ASCII order! */
45 * We assign indices from 0 upwards to the sub-charsets of a
46 * given ISO 2022 subset. nbytes[i] tells us how many bytes per
47 * character are required by sub-charset i. (It's a string
48 * mainly because that makes it easier to declare in C syntax
54 * The characters in this string are indices-plus-one (so that
55 * NUL can still terminate) of escape sequences in `escapes'.
56 * These escapes are output in the given sequence to reset the
57 * encoding state, unless it turns out that a given escape
58 * would not change the state at all.
63 * Initial value of s1, in case the default container contents
64 * needs to be something other than charset 0 in all cases.
65 * (Note that this must have the top bit set!)
70 * For output, some ISO 2022 subsets _mandate_ an initial shift
71 * sequence. If so, here it is so we can output it. (For the
72 * sake of basic sanity we won't bother to _require_ it on
73 * input, although it should of course be listed under
74 * `escapes' above so that we ignore it when present.)
76 char const *initial_sequence
;
79 * Is this an 8-bit ISO 2022 subset?
84 * Function calls to do the actual translation.
86 long int (*to_ucs
)(int subcharset
, unsigned long bytes
);
87 int (*from_ucs
)(long int ucs
, int *subcharset
, unsigned long *bytes
);
90 static void read_iso2022s(charset_spec
const *charset
, long int input_chr
,
92 void (*emit
)(void *ctx
, long int output
),
95 struct iso2022
const *iso
= (struct iso2022
*)charset
->data
;
98 * For reading ISO-2022 subsets, we divide up our state
99 * variables as follows:
101 * - The top byte of s0 (bits 31:24) indicates, if nonzero,
102 * that we are part-way through a recognised ISO-2022 escape
103 * sequence. Five of those bits (31:27) give the index of
104 * the first member of the escapes list matching what we
105 * have so far; the remaining three (26:24) give the number
106 * of characters we have seen so far.
108 * - The top bit of s1 (bit 31) is non-zero at all times, to
109 * indicate that we have performed any necessary
110 * initialisation. When we start, we detect a zero s1 and
111 * respond to it by initialising the default container
114 * - The next three bits of s1 (bits 30:28) indicate which
115 * _container_ is currently selected. This isn't quite as
116 * simple as it sounds, since we have to preserve memory of
117 * which of the SI/SO containers we came from when we're
118 * temporarily in SS2/SS3. Hence, what happens is:
119 * + bit 28 indicates SI/SO.
120 * + if we're in an SS2/SS3 container, that's indicated by
121 * the two bits above that being nonzero and holding
123 * + Hence: 0 is SI, 1 is SO, 4 is SS2-from-SI, 5 is
124 * SS2-from-SO, 6 is SS3-from-SI, 7 is SS3-from-SO.
125 * + For added fun: in an _8-bit_ ISO 2022 subset, we have
126 * the further special value 2, which means that we're
127 * theoretically in SI but the current character being
128 * accumulated is composed of 8-bit characters and will
129 * therefore be interpreted as if in SO.
131 * - The next nibble of s1 (27:24) indicates how many bytes
132 * have been accumulated in the current character.
134 * - The remaining three bytes of s1 are divided into four
135 * six-bit sections, and each section gives the current
136 * sub-charset selected in one of the possible containers.
137 * (Those containers are SI, SO, SS2 and SS3, respectively
138 * and in order from the bottom of s0 to the top.)
140 * - The bottom 24 bits of s0 give the accumulated character
143 * (Note that this means s1 contains all the parts of the state
144 * which might need to be operated on by escape sequences.
148 if (!(state
->s1
& 0x80000000)) {
153 * So. Firstly, we process escape sequences, if we're in the
154 * middle of one or if we see a possible introducer (SI, SO,
157 if ((state
->s0
>> 24) ||
158 (input_chr
== SO
|| input_chr
== SI
|| input_chr
== ESC
)) {
159 int n
= (state
->s0
>> 24) & 7, i
= (state
->s0
>> 27), oi
= i
, j
;
162 * If this is the start of an escape sequence, we might be
163 * in mid-character. If so, clear the character state and
164 * emit an error token for the incomplete character.
166 if (state
->s1
& 0x0F000000) {
167 state
->s1
&= ~0x0F000000;
168 state
->s0
&= 0xFF000000;
170 * If we were in the SS2 or SS3 container, we
171 * automatically exit it.
173 if (state
->s1
& 0x60000000)
174 state
->s1
&= 0x9FFFFFFF;
175 emit(emitctx
, ERROR
);
179 while (j
< iso
->nescapes
&&
180 !memcmp(iso
->escapes
[j
].sequence
,
181 iso
->escapes
[oi
].sequence
, n
)) {
182 if (iso
->escapes
[j
].sequence
[n
] < input_chr
)
187 if (i
>= iso
->nescapes
||
188 memcmp(iso
->escapes
[i
].sequence
,
189 iso
->escapes
[oi
].sequence
, n
) ||
190 iso
->escapes
[i
].sequence
[n
] != input_chr
) {
192 * This character does not appear in any valid escape
193 * sequence. Therefore, we must emit all the characters
194 * we had previously swallowed, plus this one, and
195 * return to non-escape-sequence state.
197 for (j
= 0; j
< n
; j
++)
198 emit(emitctx
, iso
->escapes
[oi
].sequence
[j
]);
199 emit(emitctx
, input_chr
);
205 * Otherwise, we have found an additional character in our
206 * escape sequence. See if we have reached the _end_ of our
207 * sequence (and therefore must process the sequence).
210 if (!iso
->escapes
[i
].sequence
[n
]) {
212 state
->s1
&= iso
->escapes
[i
].andbits
;
213 state
->s1
^= iso
->escapes
[i
].xorbits
;
218 * Failing _that_, we simply update our escape-sequence-
221 assert(i
< 32 && n
< 8);
222 state
->s0
= (i
<< 27) | (n
<< 24);
227 * If this isn't an escape sequence, it must be part of a
228 * character. One possibility is that it's a control character
229 * (00-20 or 7F-9F; also in non-8-bit ISO 2022 subsets I'm
230 * going to treat all top-half characters as controls), in
231 * which case we output it verbatim.
233 if (input_chr
< 0x21 ||
234 (input_chr
> 0x7E && (!iso
->eightbit
|| input_chr
< 0xA0))) {
236 * We might be in mid-multibyte-character. If so, clear the
237 * character state and emit an error token for the
238 * incomplete character.
240 if (state
->s1
& 0x0F000000) {
241 state
->s1
&= ~0x0F000000;
242 state
->s0
&= 0xFF000000;
243 emit(emitctx
, ERROR
);
245 * If we were in the SS2 or SS3 container, we
246 * automatically exit it.
248 if (state
->s1
& 0x60000000)
249 state
->s1
&= 0x9FFFFFFF;
252 emit(emitctx
, input_chr
);
257 * Otherwise, accumulate character data.
261 int chrlen
, cont
, subcharset
, bytes
;
264 * Verify that we've seen the right kind of character for
265 * what we're currently doing. This only matters in 8-bit
269 cont
= (state
->s1
>> 28) & 7;
271 * If cont==0, we're entitled to see either GL or GR
272 * characters. If cont==2, we expect only GR; otherwise
275 * If we see a GR character while cont==0, we set
276 * cont=2 immediately.
278 if ((cont
== 2 && !(input_chr
& 0x80)) ||
279 (cont
!= 0 && cont
!= 2 && (input_chr
& 0x80))) {
281 * Clear the previous character; it was prematurely
282 * terminated by this error.
284 state
->s1
&= ~0x0F000000;
285 state
->s0
&= 0xFF000000;
286 emit(emitctx
, ERROR
);
288 * If we were in the SS2 or SS3 container, we
289 * automatically exit it.
291 if (state
->s1
& 0x60000000)
292 state
->s1
&= 0x9FFFFFFF;
295 if (cont
== 0 && (input_chr
& 0x80)) {
296 state
->s1
|= 0x20000000;
300 /* The current character and its length. */
301 chr
= ((state
->s0
& 0x00FFFFFF) << 8) | (input_chr
& 0x7F);
302 chrlen
= ((state
->s1
>> 24) & 0xF) + 1;
303 /* The current sub-charset. */
304 cont
= (state
->s1
>> 28) & 7;
305 if (cont
> 1) cont
>>= 1;
306 subcharset
= (state
->s1
>> (6*cont
)) & 0x3F;
307 /* The number of bytes-per-character in that sub-charset. */
308 bytes
= iso
->nbytes
[subcharset
];
311 * If this character is now complete, we convert and emit
312 * it. Otherwise, we simply update the state and return.
314 if (chrlen
>= bytes
) {
315 emit(emitctx
, iso
->to_ucs(subcharset
, chr
));
318 * If we were in the SS2 or SS3 container, we
319 * automatically exit it.
321 if (state
->s1
& 0x60000000)
322 state
->s1
&= 0x9FFFFFFF;
324 state
->s0
= (state
->s0
& 0xFF000000) | chr
;
325 state
->s1
= (state
->s1
& 0xF0FFFFFF) | (chrlen
<< 24);
329 static int write_iso2022s(charset_spec
const *charset
, long int input_chr
,
330 charset_state
*state
,
331 void (*emit
)(void *ctx
, long int output
),
334 struct iso2022
const *iso
= (struct iso2022
*)charset
->data
;
335 int subcharset
, len
, i
, j
, cont
, topbit
= 0;
339 * For output, our s1 state variable contains most of the same
340 * stuff as it did for input - initial-state indicator bit,
341 * current container, and current subcharset selected in each
346 * Analyse the character and find out what subcharset it needs
349 if (input_chr
>= 0 && !iso
->from_ucs(input_chr
, &subcharset
, &bytes
))
352 if (!(state
->s1
& 0x80000000)) {
354 if (iso
->initial_sequence
)
355 for (i
= 0; iso
->initial_sequence
[i
]; i
++)
356 emit(emitctx
, iso
->initial_sequence
[i
]);
359 if (input_chr
== -1) {
360 unsigned long oldstate
;
364 * Special case: reset encoding state.
366 for (i
= 0; iso
->reset
[i
]; i
++) {
367 j
= iso
->reset
[i
] - 1;
368 oldstate
= state
->s1
;
369 state
->s1
&= iso
->escapes
[j
].andbits
;
370 state
->s1
^= iso
->escapes
[j
].xorbits
;
371 if (state
->s1
!= oldstate
) {
372 /* We must actually emit this sequence. */
373 for (k
= 0; iso
->escapes
[j
].sequence
[k
]; k
++)
374 emit(emitctx
, iso
->escapes
[j
].sequence
[k
]);
382 * Now begins the fun. We now know what subcharset we want. So
383 * we must find out which container we should select it into,
384 * select it into it if necessary, select that _container_ if
385 * necessary, and then output the given bytes.
387 for (i
= 0; i
< iso
->nescapes
; i
++)
388 if (iso
->escapes
[i
].subcharset
== subcharset
&&
389 !(iso
->escapes
[i
].container
& RO
))
391 assert(i
< iso
->nescapes
);
394 * We've found the escape sequence which would select this
395 * subcharset into a container. However, that subcharset might
396 * already _be_ selected in that container! Check before we go
397 * to the effort of emitting the sequence.
399 cont
= iso
->escapes
[i
].container
&~ RO
;
400 if (((state
->s1
>> (6*cont
)) & 0x3F) != (unsigned)subcharset
) {
401 for (j
= 0; iso
->escapes
[i
].sequence
[j
]; j
++)
402 emit(emitctx
, iso
->escapes
[i
].sequence
[j
]);
403 state
->s1
&= iso
->escapes
[i
].andbits
;
404 state
->s1
^= iso
->escapes
[i
].xorbits
;
408 * Now we know what container our subcharset is in, so we want
409 * to select that container.
412 /* SS2 or SS3; just output the sequence and be done. */
414 emit(emitctx
, 'L' + cont
); /* comes out to 'N' or 'O' */
417 * Emit SI or SO, but only if the current container isn't already
420 * Also, in an 8-bit subset, we need not do this; we'll
421 * just use 8-bit characters to output SO-container
424 if (iso
->eightbit
&& cont
== 1 && ((state
->s1
>> 28) & 7) == 0) {
426 } else if (((state
->s1
>> 28) & 7) != (unsigned)cont
) {
427 emit(emitctx
, cont ? SO
: SI
);
428 state
->s1
= (state
->s1
& 0x8FFFFFFF) | (cont
<< 28);
433 * We're done. Subcharset is selected in container, container
434 * is selected. All we need now is to write out the bytes.
436 len
= iso
->nbytes
[subcharset
];
438 emit(emitctx
, ((bytes
>> (8*len
)) & 0xFF) | topbit
);
444 * ISO-2022-JP, defined in RFC 1468.
446 static long int iso2022jp_to_ucs(int subcharset
, unsigned long bytes
)
448 switch (subcharset
) {
449 case 1: /* JIS X 0201 bottom half */
452 else if (bytes
== 0x7E)
454 /* else fall through to ASCII */
455 case 0: return bytes
; /* one-byte ASCII */
456 /* (no break needed since all control paths have returned) */
457 case 2: return jisx0208_to_unicode(((bytes
>> 8) & 0xFF) - 0x21,
458 ((bytes
) & 0xFF) - 0x21);
459 default: return ERROR
;
462 static int iso2022jp_from_ucs(long int ucs
, int *subcharset
,
463 unsigned long *bytes
)
470 } else if (ucs
== 0xA5 || ucs
== 0x203E) {
472 *bytes
= (ucs
== 0xA5 ?
0x5C : 0x7E);
474 } else if (unicode_to_jisx0208(ucs
, &r
, &c
)) {
476 *bytes
= ((r
+0x21) << 8) | (c
+0x21);
482 static const struct iso2022_escape iso2022jp_escapes
[] = {
483 {"\033$@", 0xFFFFFFC0, 0x00000002, -1, -1}, /* we ignore this one */
484 {"\033$B", 0xFFFFFFC0, 0x00000002, 0, 2},
485 {"\033(B", 0xFFFFFFC0, 0x00000000, 0, 0},
486 {"\033(J", 0xFFFFFFC0, 0x00000001, 0, 1},
488 static const struct iso2022 iso2022jp
= {
489 iso2022jp_escapes
, lenof(iso2022jp_escapes
),
490 "\1\1\2", "\3", 0x80000000, NULL
, FALSE
,
491 iso2022jp_to_ucs
, iso2022jp_from_ucs
493 const charset_spec charset_CS_ISO2022_JP
= {
494 CS_ISO2022_JP
, read_iso2022s
, write_iso2022s
, &iso2022jp
498 * ISO-2022-KR, defined in RFC 1557.
500 static long int iso2022kr_to_ucs(int subcharset
, unsigned long bytes
)
502 switch (subcharset
) {
503 case 0: return bytes
; /* one-byte ASCII */
504 case 1: return ksx1001_to_unicode(((bytes
>> 8) & 0xFF) - 0x21,
505 ((bytes
) & 0xFF) - 0x21);
506 default: return ERROR
;
509 static int iso2022kr_from_ucs(long int ucs
, int *subcharset
,
510 unsigned long *bytes
)
517 } else if (unicode_to_ksx1001(ucs
, &r
, &c
)) {
519 *bytes
= ((r
+0x21) << 8) | (c
+0x21);
525 static const struct iso2022_escape iso2022kr_escapes
[] = {
526 {"\016", 0x8FFFFFFF, 0x10000000, -1, -1},
527 {"\017", 0x8FFFFFFF, 0x00000000, 0, 0},
528 {"\033$)C", 0xFFFFF03F, 0x00000040, 1, 1}, /* bits[11:6] <- 1 */
530 static const struct iso2022 iso2022kr
= {
531 iso2022kr_escapes
, lenof(iso2022kr_escapes
),
532 "\1\2", "\2", 0x80000040, "\033$)C", FALSE
,
533 iso2022kr_to_ucs
, iso2022kr_from_ucs
535 const charset_spec charset_CS_ISO2022_KR
= {
536 CS_ISO2022_KR
, read_iso2022s
, write_iso2022s
, &iso2022kr
539 #else /* ENUM_CHARSETS */
541 ENUM_CHARSET(CS_ISO2022_JP
)
542 ENUM_CHARSET(CS_ISO2022_KR
)
544 #endif /* ENUM_CHARSETS */