diff options
| author | Ulrich Drepper <drepper@redhat.com> | 2001-05-22 22:30:18 +0000 |
|---|---|---|
| committer | Ulrich Drepper <drepper@redhat.com> | 2001-05-22 22:30:18 +0000 |
| commit | 2373b30ea829ad5dd9599b29902c80101deefc78 (patch) | |
| tree | 79874a0fea3457093e8f2834a2bf7bad41a1863a | |
| parent | 25e57b10f3943cefa888ba7f90e57bcda18ed7d8 (diff) | |
| download | glibc-2373b30ea829ad5dd9599b29902c80101deefc78.tar.gz glibc-2373b30ea829ad5dd9599b29902c80101deefc78.tar.xz glibc-2373b30ea829ad5dd9599b29902c80101deefc78.zip | |
Update.
2001-05-20 Bruno Haible <haible@clisp.cons.org> * iconvdata/cp1255.c: Completely rewritten. * iconvdata/Makefile (gen-8bit-gap-modules): Remove cp1255. * iconvdata/testdata/WINDOWS-1255: New file. * iconvdata/testdata/WINDOWS-1255..UTF8: New file. * iconvdata/TESTS: Add WINDOWS-1255 test. * iconvdata/CP1255.irreversible: New file. 2001-05-20 Bruno Haible <haible@clisp.cons.org> * iconvdata/cp1258.c: Completely rewritten. * iconvdata/Makefile (gen-8bit-gap-modules): Remove cp1258. * iconvdata/testdata/WINDOWS-1258: New file. * iconvdata/testdata/WINDOWS-1258..UTF8: New file. * iconvdata/TESTS: Add WINDOWS-1258 test. * iconvdata/tst-table-from.c (try): Reset the iconv descriptor before the main call, and flush it afterwards. (utf8_decode): Return a string, possibly containing several Unicode characters. (main): Update all utf8_decode calls. * iconvdata/CP1258.irreversible: New file. 2001-05-20 Bruno Haible <haible@clisp.cons.org> * iconv/gconv.c (__gconv): For flush without output, pass do_flush = 2. * iconv/skeleton.c: Distinguish do_flush = 1 and do_flush = 2. In the first case, set outbuf, outstart, outend, and call PREPARE_LOOP before EMIT_SHIFT_TO_INIT; then pass the output produced by this step down to the next step. In the second case, clear the state without calling EMIT_SHIFT_TO_INIT. * iconvdata/ibm930.c (EMIT_SHIFT_TO_INIT): Use outbuf instead of data->__outbuf, and outend instead of data->__outbufend. * iconvdata/ibm933.c (EMIT_SHIFT_TO_INIT): Likewise. * iconvdata/ibm935.c (EMIT_SHIFT_TO_INIT): Likewise. * iconvdata/ibm937.c (EMIT_SHIFT_TO_INIT): Likewise. * iconvdata/ibm939.c (EMIT_SHIFT_TO_INIT): Likewise. * iconvdata/iso-2022-cn.c (EMIT_SHIFT_TO_INIT): Likewise. * iconvdata/iso-2022-cn-ext.c (EMIT_SHIFT_TO_INIT): Likewise. * iconvdata/iso-2022-jp.c (EMIT_SHIFT_TO_INIT): Likewise. * iconvdata/iso-2022-kr.c (EMIT_SHIFT_TO_INIT): Likewise. * iconvdata/utf-7.c (EMIT_SHIFT_TO_INIT): Likewise. 2001-05-21 Jakub Jelinek <jakub@redhat.com> * elf/rtld.c (dl_main): Compute l_map_end for the main program. * elf/dl-sym.c (_dl_sym): Don't check for l_addr == 0. If match == _dl_loaded, caller can still come from the main program. (_dl_vsym): Likewise. * elf/dl-open.c (dl_open_worker): Don't check for l_addr == 0. * elf/dl-error.c (_dl_signal_error): Change NULL objname into "". * elf/restest2.c: New test. * elf/Makefile (tests): Add restest2. (restest2, LDFLAGS-restest2): Add rules.
| -rw-r--r-- | ChangeLog | 55 | ||||
| -rw-r--r-- | elf/Makefile | 6 | ||||
| -rw-r--r-- | elf/dl-error.c | 4 | ||||
| -rw-r--r-- | elf/dl-open.c | 7 | ||||
| -rw-r--r-- | elf/dl-sym.c | 20 | ||||
| -rw-r--r-- | elf/restest2.c | 33 | ||||
| -rw-r--r-- | elf/rtld.c | 14 | ||||
| -rw-r--r-- | iconv/gconv.c | 3 | ||||
| -rw-r--r-- | iconv/skeleton.c | 101 | ||||
| -rw-r--r-- | iconvdata/CP1255.irreversible | 34 | ||||
| -rw-r--r-- | iconvdata/CP1258.irreversible | 183 | ||||
| -rw-r--r-- | iconvdata/Makefile | 4 | ||||
| -rw-r--r-- | iconvdata/TESTS | 6 | ||||
| -rw-r--r-- | iconvdata/cp1255.c | 532 | ||||
| -rw-r--r-- | iconvdata/cp1258.c | 850 | ||||
| -rw-r--r-- | iconvdata/ibm930.c | 5 | ||||
| -rw-r--r-- | iconvdata/ibm933.c | 5 | ||||
| -rw-r--r-- | iconvdata/ibm935.c | 5 | ||||
| -rw-r--r-- | iconvdata/ibm937.c | 5 | ||||
| -rw-r--r-- | iconvdata/ibm939.c | 5 | ||||
| -rw-r--r-- | iconvdata/iso-2022-cn-ext.c | 5 | ||||
| -rw-r--r-- | iconvdata/iso-2022-cn.c | 5 | ||||
| -rw-r--r-- | iconvdata/iso-2022-jp.c | 10 | ||||
| -rw-r--r-- | iconvdata/iso-2022-kr.c | 5 | ||||
| -rw-r--r-- | iconvdata/testdata/WINDOWS-1255 | 10 | ||||
| -rw-r--r-- | iconvdata/testdata/WINDOWS-1255..UTF-8 | 10 | ||||
| -rw-r--r-- | iconvdata/testdata/WINDOWS-1258 | 4 | ||||
| -rw-r--r-- | iconvdata/testdata/WINDOWS-1258..UTF8 | 4 | ||||
| -rw-r--r-- | iconvdata/tst-table-from.c | 89 | ||||
| -rw-r--r-- | iconvdata/utf-7.c | 4 |
30 files changed, 1915 insertions, 108 deletions
diff --git a/ChangeLog b/ChangeLog index 9b1e3eb113..54e8328d31 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,58 @@ +2001-05-20 Bruno Haible <haible@clisp.cons.org> + + * iconvdata/cp1255.c: Completely rewritten. + * iconvdata/Makefile (gen-8bit-gap-modules): Remove cp1255. + * iconvdata/testdata/WINDOWS-1255: New file. + * iconvdata/testdata/WINDOWS-1255..UTF8: New file. + * iconvdata/TESTS: Add WINDOWS-1255 test. + * iconvdata/CP1255.irreversible: New file. + +2001-05-20 Bruno Haible <haible@clisp.cons.org> + + * iconvdata/cp1258.c: Completely rewritten. + * iconvdata/Makefile (gen-8bit-gap-modules): Remove cp1258. + * iconvdata/testdata/WINDOWS-1258: New file. + * iconvdata/testdata/WINDOWS-1258..UTF8: New file. + * iconvdata/TESTS: Add WINDOWS-1258 test. + * iconvdata/tst-table-from.c (try): Reset the iconv descriptor before + the main call, and flush it afterwards. + (utf8_decode): Return a string, possibly containing several Unicode + characters. + (main): Update all utf8_decode calls. + * iconvdata/CP1258.irreversible: New file. + +2001-05-20 Bruno Haible <haible@clisp.cons.org> + + * iconv/gconv.c (__gconv): For flush without output, pass do_flush = 2. + * iconv/skeleton.c: Distinguish do_flush = 1 and do_flush = 2. In the + first case, set outbuf, outstart, outend, and call PREPARE_LOOP before + EMIT_SHIFT_TO_INIT; then pass the output produced by this step down to + the next step. In the second case, clear the state without calling + EMIT_SHIFT_TO_INIT. + * iconvdata/ibm930.c (EMIT_SHIFT_TO_INIT): Use outbuf instead of + data->__outbuf, and outend instead of data->__outbufend. + * iconvdata/ibm933.c (EMIT_SHIFT_TO_INIT): Likewise. + * iconvdata/ibm935.c (EMIT_SHIFT_TO_INIT): Likewise. + * iconvdata/ibm937.c (EMIT_SHIFT_TO_INIT): Likewise. + * iconvdata/ibm939.c (EMIT_SHIFT_TO_INIT): Likewise. + * iconvdata/iso-2022-cn.c (EMIT_SHIFT_TO_INIT): Likewise. + * iconvdata/iso-2022-cn-ext.c (EMIT_SHIFT_TO_INIT): Likewise. + * iconvdata/iso-2022-jp.c (EMIT_SHIFT_TO_INIT): Likewise. + * iconvdata/iso-2022-kr.c (EMIT_SHIFT_TO_INIT): Likewise. + * iconvdata/utf-7.c (EMIT_SHIFT_TO_INIT): Likewise. + +2001-05-21 Jakub Jelinek <jakub@redhat.com> + + * elf/rtld.c (dl_main): Compute l_map_end for the main program. + * elf/dl-sym.c (_dl_sym): Don't check for l_addr == 0. + If match == _dl_loaded, caller can still come from the main program. + (_dl_vsym): Likewise. + * elf/dl-open.c (dl_open_worker): Don't check for l_addr == 0. + * elf/dl-error.c (_dl_signal_error): Change NULL objname into "". + * elf/restest2.c: New test. + * elf/Makefile (tests): Add restest2. + (restest2, LDFLAGS-restest2): Add rules. + 2001-05-22 Ulrich Drepper <drepper@redhat.com> * intl/Makefile: Add a few more dependencies to avoid races in the diff --git a/elf/Makefile b/elf/Makefile index 0668bb3251..8301683b80 100644 --- a/elf/Makefile +++ b/elf/Makefile @@ -101,7 +101,8 @@ tests = loadtest restest1 preloadtest loadfail multiload origtest resolvfail \ constload1 order $(tests-vis-$(have-protected)) noload filter unload \ reldep reldep2 reldep3 next $(tests-nodelete-$(have-z-nodelete)) \ $(tests-nodlopen-$(have-z-nodlopen)) neededtest neededtest2 \ - neededtest3 neededtest4 unload2 lateglobal initfirst global + neededtest3 neededtest4 unload2 lateglobal initfirst global \ + restest2 test-srcs = tst-pathopt tests-vis-yes = vismain tests-nodelete-yes = nodelete @@ -303,6 +304,9 @@ $(objpfx)neededtest4.out: $(objpfx)neededobj5.so $(objpfx)neededobj6.so $(objpfx)restest1: $(objpfx)testobj1.so $(objpfx)testobj1_1.so $(libdl) LDFLAGS-restest1 = -rdynamic +$(objpfx)restest2: $(libdl) +LDFLAGS-restest2 = -rdynamic + $(objpfx)restest1.out: $(test-modules) preloadtest-preloads = testobj1 testobj2 testobj3 testobj4 testobj5 diff --git a/elf/dl-error.c b/elf/dl-error.c index 1cd4cc3852..440fea35d7 100644 --- a/elf/dl-error.c +++ b/elf/dl-error.c @@ -71,6 +71,8 @@ _dl_signal_error (int errcode, const char *objname, const char *errstring) errstring = N_("DYNAMIC LINKER BUG!!!"); lcatch = tsd_getspecific (); + if (objname == NULL) + objname = ""; if (lcatch != NULL) { /* We are inside _dl_catch_error. Return to it. We have to @@ -100,7 +102,7 @@ _dl_signal_error (int errcode, const char *objname, const char *errstring) _dl_fatal_printf ("\ %s: error while loading shared libraries: %s%s%s%s%s\n", _dl_argv[0] ?: "<program name unknown>", - objname ?: "", objname && *objname ? ": " : "", + objname, *objname ? ": " : "", errstring, errcode ? ": " : "", (errcode ? __strerror_r (errcode, buffer, sizeof buffer) diff --git a/elf/dl-open.c b/elf/dl-open.c index 440da67d0d..43daf60d58 100644 --- a/elf/dl-open.c +++ b/elf/dl-open.c @@ -188,13 +188,10 @@ dl_open_worker (void *a) _dl_signal_error (0, "dlopen", N_("DST not allowed in SUID/SGID programs")); - /* We have to find out from which object the caller is calling. - Find the highest-addressed object that ADDRESS is not below. */ + /* We have to find out from which object the caller is calling. */ call_map = NULL; for (l = _dl_loaded; l; l = l->l_next) - if (l->l_addr != 0 /* Make sure we do not currently set this map up - in this moment. */ - && caller >= (const void *) l->l_map_start + if (caller >= (const void *) l->l_map_start && caller < (const void *) l->l_map_end) { /* There must be exactly one DSO for the range of the virtual diff --git a/elf/dl-sym.c b/elf/dl-sym.c index ffc36bc8f9..9ca2af81e4 100644 --- a/elf/dl-sym.c +++ b/elf/dl-sym.c @@ -41,7 +41,7 @@ _dl_sym (void *handle, const char *name, void *who) /* Find the highest-addressed object that CALLER is not below. */ for (l = _dl_loaded; l != NULL; l = l->l_next) - if (l->l_addr != 0 && caller >= l->l_map_start && caller < l->l_map_end) + if (caller >= l->l_map_start && caller < l->l_map_end) { /* There must be exactly one DSO for the range of the virtual memory. Otherwise something is really broken. */ @@ -65,8 +65,13 @@ _dl_sym (void *handle, const char *name, void *who) else { if (__builtin_expect (match == _dl_loaded, 0)) - _dl_signal_error (0, NULL, N_("\ + { + if (! _dl_loaded + || caller < _dl_loaded->l_map_start + || caller >= _dl_loaded->l_map_end) + _dl_signal_error (0, NULL, N_("\ RTLD_NEXT used in code not dynamically loaded")); + } l = match; while (l->l_loader != NULL) @@ -107,7 +112,7 @@ _dl_vsym (void *handle, const char *name, const char *version, void *who) /* Find the highest-addressed object that CALLER is not below. */ for (l = _dl_loaded; l != NULL; l = l->l_next) - if (l->l_addr != 0 && caller >= l->l_map_start && caller < l->l_map_end) + if (caller >= l->l_map_start && caller < l->l_map_end) { /* There must be exactly one DSO for the range of the virtual memory. Otherwise something is really broken. */ @@ -121,9 +126,14 @@ _dl_vsym (void *handle, const char *name, const char *version, void *who) &vers, 0, 0); else if (handle == RTLD_NEXT) { - if (match == _dl_loaded) - _dl_signal_error (0, NULL, N_("\ + if (__builtin_expect (match == _dl_loaded, 0)) + { + if (! _dl_loaded + || caller < _dl_loaded->l_map_start + || caller >= _dl_loaded->l_map_end) + _dl_signal_error (0, NULL, N_("\ RTLD_NEXT used in code not dynamically loaded")); + } l = match; while (l->l_loader != NULL) diff --git a/elf/restest2.c b/elf/restest2.c new file mode 100644 index 0000000000..f959f030a0 --- /dev/null +++ b/elf/restest2.c @@ -0,0 +1,33 @@ +#include <sys/types.h> +#include <dlfcn.h> +#include <error.h> +#include <mcheck.h> +#include <stdlib.h> +#include <unistd.h> + +pid_t pid, pid2; + +pid_t getpid(void) +{ + pid_t (*f)(void); + f = (pid_t (*)(void)) dlsym (RTLD_NEXT, "getpid"); + if (f == NULL) + error (EXIT_FAILURE, 0, "dlsym (RTLD_NEXT, \"getpid\"): %s", dlerror ()); + return (pid2 = f()) + 26; +} + +int +main (void) +{ + pid_t (*f)(void); + + mtrace (); + + f = (pid_t (*)(void)) dlsym (RTLD_DEFAULT, "getpid"); + if (f == NULL) + error (EXIT_FAILURE, 0, "dlsym (RTLD_DEFAULT, \"getpid\"): %s", dlerror ()); + pid = f(); + if (pid != pid2 + 26) + error (EXIT_FAILURE, 0, "main getpid() not called"); + return 0; +} diff --git a/elf/rtld.c b/elf/rtld.c index 8b2ca2984e..ee48af3b3b 100644 --- a/elf/rtld.c +++ b/elf/rtld.c @@ -542,8 +542,7 @@ of this helper program; chances are you did not intend to run this program.\n\ information for the program. */ } - /* It is not safe to load stuff after the main program. */ - _dl_loaded->l_map_end = ~0; + _dl_loaded->l_map_end = 0; /* Perhaps the executable has no PT_LOAD header entries at all. */ _dl_loaded->l_map_start = ~0; @@ -591,15 +590,24 @@ of this helper program; chances are you did not intend to run this program.\n\ has_interp = 1; break; case PT_LOAD: - /* Remember where the main program starts in memory. */ { ElfW(Addr) mapstart; + ElfW(Addr) allocend; + + /* Remember where the main program starts in memory. */ mapstart = _dl_loaded->l_addr + (ph->p_vaddr & ~(ph->p_align - 1)); if (_dl_loaded->l_map_start > mapstart) _dl_loaded->l_map_start = mapstart; + + /* Also where it ends. */ + allocend = _dl_loaded->l_addr + ph->p_vaddr + ph->p_memsz; + if (_dl_loaded->l_map_end < allocend) + _dl_loaded->l_map_end = allocend; } break; } + if (! _dl_loaded->l_map_end) + _dl_loaded->l_map_end = ~0; if (! _dl_rtld_map.l_libname && _dl_rtld_map.l_name) { /* We were invoked directly, so the program might not have a diff --git a/iconv/gconv.c b/iconv/gconv.c index 7cb78694e0..6a7424bcbf 100644 --- a/iconv/gconv.c +++ b/iconv/gconv.c @@ -49,7 +49,8 @@ __gconv (__gconv_t cd, const unsigned char **inbuf, /* We just flush. */ result = DL_CALL_FCT (cd->__steps->__fct, (cd->__steps, cd->__data, NULL, NULL, NULL, - irreversible, 1, 0)); + irreversible, + cd->__data[last_step].__outbuf == NULL ? 2 : 1, 0)); else { const unsigned char *last_start; diff --git a/iconv/skeleton.c b/iconv/skeleton.c index 98abc33f6d..bd8b4fe4a2 100644 --- a/iconv/skeleton.c +++ b/iconv/skeleton.c @@ -298,28 +298,93 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, dropped. */ if (__builtin_expect (do_flush, 0)) { - status = __GCONV_OK; - /* This should never happen during error handling. */ assert (outbufstart == NULL); + status = __GCONV_OK; + #ifdef EMIT_SHIFT_TO_INIT - /* Emit the escape sequence to reset the state. */ - EMIT_SHIFT_TO_INIT; -#else - /* Clear the state object. There might be bytes in there from - previous calls with CONSUME_INCOMPLETE == 1. */ - memset (data->__statep, '\0', sizeof (*data->__statep)); + if (do_flush == 1) + { + /* We preserve the initial values of the pointer variables. */ + unsigned char *outbuf = data->__outbuf; + unsigned char *outstart = outbuf; + unsigned char *outend = data->__outbufend; + +# ifdef PREPARE_LOOP + PREPARE_LOOP +# endif + +# ifdef SAVE_RESET_STATE + SAVE_RESET_STATE (1); +# endif + + /* Emit the escape sequence to reset the state. */ + EMIT_SHIFT_TO_INIT; + + /* Call the steps down the chain if there are any but only if we + successfully emitted the escape sequence. This should only + fail if the output buffer is full. If the input is invalid + it should be discarded since the user wants to start from a + clean state. */ + if (status == __GCONV_OK) + { + if (data->__flags & __GCONV_IS_LAST) + /* Store information about how many bytes are available. */ + data->__outbuf = outbuf; + else + { + /* Write out all output which was produced. */ + if (outbuf > outstart) + { + const unsigned char *outerr = outstart; + int result; + + result = DL_CALL_FCT (fct, (next_step, next_data, + &outerr, outbuf, NULL, + irreversible, 0, + consume_incomplete)); + + if (result != __GCONV_EMPTY_INPUT) + { + if (__builtin_expect (outerr != outbuf, 0)) + { + /* We have a problem. Undo the conversion. */ + outbuf = outstart; + + /* Restore the state. */ +# ifdef SAVE_RESET_STATE + SAVE_RESET_STATE (0); +# endif + } + + /* Change the status. */ + status = result; + } + } + + if (status == __GCONV_OK) + /* Now flush the remaining steps. */ + status = DL_CALL_FCT (fct, (next_step, next_data, NULL, + NULL, NULL, irreversible, 1, + consume_incomplete)); + } + } + } + else #endif - /* Call the steps down the chain if there are any but only if we - successfully emitted the escape sequence. This should only - fail if the output buffer is full. If the input is invalid - it should be discarded since the user wants to start from a - clean slate. */ - if (status == __GCONV_OK && ! (data->__flags & __GCONV_IS_LAST)) - status = DL_CALL_FCT (fct, (next_step, next_data, NULL, NULL, - NULL, irreversible, 1, - consume_incomplete)); + { + /* Clear the state object. There might be bytes in there from + previous calls with CONSUME_INCOMPLETE == 1. But don't emit + escape sequences. */ + memset (data->__statep, '\0', sizeof (*data->__statep)); + + if (! (data->__flags & __GCONV_IS_LAST)) + /* Now flush the remaining steps. */ + status = DL_CALL_FCT (fct, (next_step, next_data, NULL, NULL, + NULL, irreversible, do_flush, + consume_incomplete)); + } } else { @@ -499,7 +564,7 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, *inptrp = inptr; outbuf = outstart; - /* Reset the state. */ + /* Restore the state. */ # ifdef SAVE_RESET_STATE SAVE_RESET_STATE (0); # endif diff --git a/iconvdata/CP1255.irreversible b/iconvdata/CP1255.irreversible new file mode 100644 index 0000000000..a67efffb2c --- /dev/null +++ b/iconvdata/CP1255.irreversible @@ -0,0 +1,34 @@ +0xD6C7 0xFB1F +0xE0C7 0xFB2E +0xE0C8 0xFB2F +0xE0CC 0xFB30 +0xE1CC 0xFB31 +0xE1CF 0xFB4C +0xE2CC 0xFB32 +0xE3CC 0xFB33 +0xE4CC 0xFB34 +0xE5C9 0xFB4B +0xE5CC 0xFB35 +0xE6CC 0xFB36 +0xE8CC 0xFB38 +0xE9C4 0xFB1D +0xE9CC 0xFB39 +0xEACC 0xFB3A +0xEBCC 0xFB3B +0xEBCF 0xFB4D +0xECCC 0xFB3C +0xEECC 0xFB3E +0xF0CC 0xFB40 +0xF1CC 0xFB41 +0xF3CC 0xFB43 +0xF4CC 0xFB44 +0xF4CF 0xFB4E +0xF6CC 0xFB46 +0xF7CC 0xFB47 +0xF8CC 0xFB48 +0xF9CC 0xFB49 +0xF9CCD1 0xFB2C +0xF9CCD2 0xFB2D +0xF9D1 0xFB2A +0xF9D2 0xFB2B +0xFACC 0xFB4A diff --git a/iconvdata/CP1258.irreversible b/iconvdata/CP1258.irreversible new file mode 100644 index 0000000000..68a54812d6 --- /dev/null +++ b/iconvdata/CP1258.irreversible @@ -0,0 +1,183 @@ +0x41D2 0x1EA2 +0x41DE 0x00C3 +0x41F2 0x1EA0 +0x42F2 0x1E04 +0x43EC 0x0106 +0x44F2 0x1E0C +0x45D2 0x1EBA +0x45DE 0x1EBC +0x45F2 0x1EB8 +0x47EC 0x01F4 +0x48F2 0x1E24 +0x49CC 0x00CC +0x49D2 0x1EC8 +0x49DE 0x0128 +0x49F2 0x1ECA +0x4BEC 0x1E30 +0x4BF2 0x1E32 +0x4CEC 0x0139 +0x4CF2 0x1E36 +0x4DEC 0x1E3E +0x4DF2 0x1E42 +0x4ECC 0x01F8 +0x4EEC 0x0143 +0x4EF2 0x1E46 +0x4FCC 0x00D2 +0x4FD2 0x1ECE +0x4FDE 0x00D5 +0x4FF2 0x1ECC +0x50EC 0x1E54 +0x52EC 0x0154 +0x52F2 0x1E5A +0x53EC 0x015A +0x53F2 0x1E62 +0x54F2 0x1E6C +0x55D2 0x1EE6 +0x55DE 0x0168 +0x55F2 0x1EE4 +0x56DE 0x1E7C +0x56F2 0x1E7E +0x57CC 0x1E80 +0x57EC 0x1E82 +0x57F2 0x1E88 +0x59CC 0x1EF2 +0x59D2 0x1EF6 +0x59DE 0x1EF8 +0x59EC 0x00DD +0x59F2 0x1EF4 +0x5AEC 0x0179 +0x5AF2 0x1E92 +0x61D2 0x1EA3 +0x61DE 0x00E3 +0x61F2 0x1EA1 +0x62F2 0x1E05 +0x63EC 0x0107 +0x64F2 0x1E0D +0x65D2 0x1EBB +0x65DE 0x1EBD +0x65F2 0x1EB9 +0x67EC 0x01F5 +0x68F2 0x1E25 +0x69CC 0x00EC +0x69D2 0x1EC9 +0x69DE 0x0129 +0x69F2 0x1ECB +0x6BEC 0x1E31 +0x6BF2 0x1E33 +0x6CEC 0x013A +0x6CF2 0x1E37 +0x6DEC 0x1E3F +0x6DF2 0x1E43 +0x6ECC 0x01F9 +0x6EEC 0x0144 +0x6EF2 0x1E47 +0x6FCC 0x00F2 +0x6FD2 0x1ECF +0x6FDE 0x00F5 +0x6FF2 0x1ECD +0x70EC 0x1E55 +0x72EC 0x0155 +0x72F2 0x1E5B +0x73EC 0x015B +0x73F2 0x1E63 +0x74F2 0x1E6D +0x75D2 0x1EE7 +0x75DE 0x0169 +0x75F2 0x1EE5 +0x76DE 0x1E7D +0x76F2 0x1E7F +0x77CC 0x1E81 +0x77EC 0x1E83 +0x77F2 0x1E89 +0x79CC 0x1EF3 +0x79D2 0x1EF7 +0x79DE 0x1EF9 +0x79EC 0x00FD +0x79F2 0x1EF5 +0x7AEC 0x017A +0x7AF2 0x1E93 +0xA5EC 0x0385 +0xA8CC 0x1FED +0xA8EC 0x1FEE +0xC2CC 0x1EA6 +0xC2D2 0x1EA8 +0xC2DE 0x1EAA +0xC2EC 0x1EA4 +0xC2F2 0x1EAC +0xC3CC 0x1EB0 +0xC3D2 0x1EB2 +0xC3DE 0x1EB4 +0xC3EC 0x1EAE +0xC3F2 0x1EB6 +0xC5EC 0x01FA +0xC6EC 0x01FC +0xC7EC 0x1E08 +0xCACC 0x1EC0 +0xCAD2 0x1EC2 +0xCADE 0x1EC4 +0xCAEC 0x1EBE +0xCAF2 0x1EC6 +0xCC 0x0340 +0xCFEC 0x1E2E +0xD3DE 0x1E4C +0xD4CC 0x1ED2 +0xD4D2 0x1ED4 +0xD4DE 0x1ED6 +0xD4EC 0x1ED0 +0xD4F2 0x1ED8 +0xD5CC 0x1EDC +0xD5D2 0x1EDE +0xD5DE 0x1EE0 +0xD5EC 0x1EDA +0xD5F2 0x1EE2 +0xD6DE 0x1E4E +0xD8EC 0x01FE +0xDADE 0x1E78 +0xDCCC 0x01DB +0xDCEC 0x01D7 +0xDDCC 0x1EEA +0xDDD2 0x1EEC +0xDDDE 0x1EEE +0xDDEC 0x1EE8 +0xDDF2 0x1EF0 +0xE2CC 0x1EA7 +0xE2D2 0x1EA9 +0xE2DE 0x1EAB +0xE2EC 0x1EA5 +0xE2F2 0x1EAD +0xE3CC 0x1EB1 +0xE3D2 0x1EB3 +0xE3DE 0x1EB5 +0xE3EC 0x1EAF +0xE3F2 0x1EB7 +0xE5EC 0x01FB +0xE6EC 0x01FD +0xE7EC 0x1E09 +0xEACC 0x1EC1 +0xEAD2 0x1EC3 +0xEADE 0x1EC5 +0xEAEC 0x1EBF +0xEAF2 0x1EC7 +0xEC 0x0341 +0xEFEC 0x1E2F +0xF3DE 0x1E4D +0xF4CC 0x1ED3 +0xF4D2 0x1ED5 +0xF4DE 0x1ED7 +0xF4EC 0x1ED1 +0xF4F2 0x1ED9 +0xF5CC 0x1EDD +0xF5D2 0x1EDF +0xF5DE 0x1EE1 +0xF5EC 0x1EDB +0xF5F2 0x1EE3 +0xF6DE 0x1E4F +0xF8EC 0x01FF +0xFADE 0x1E79 +0xFCCC 0x01DC +0xFCEC 0x01D8 +0xFDCC 0x1EEB +0xFDD2 0x1EED +0xFDDE 0x1EEF +0xFDEC 0x1EE9 +0xFDF2 0x1EF1 diff --git a/iconvdata/Makefile b/iconvdata/Makefile index e9ed5ed577..0f2e37534c 100644 --- a/iconvdata/Makefile +++ b/iconvdata/Makefile @@ -167,7 +167,7 @@ gen-8bit-gap-modules := koi8-r latin-greek latin-greek-1 ibm256 ibm273 \ ibm297 ibm420 ibm424 ibm437 ibm850 ibm851 ibm852 \ ibm855 ibm857 ibm860 ibm861 ibm862 ibm863 ibm864 \ ibm865 ibm868 ibm869 ibm875 ibm880 ibm918 ibm1004 \ - ibm1026 cp1250 cp1251 cp1252 cp1253 cp1254 cp1255 \ + ibm1026 cp1250 cp1251 cp1252 cp1253 cp1254 \ cp1256 cp1257 ibm866 iso8859-5 iso8859-7 iso8859-8 \ iso8859-10 macintosh iec_p27-1 asmo_449 \ csn_369103 cwi dec-mcs ecma-cyrillic gost_19768-74 \ @@ -175,7 +175,7 @@ gen-8bit-gap-modules := koi8-r latin-greek latin-greek-1 ibm256 ibm273 \ inis-cyrillic iso_2033 iso_5427 iso_5427-ext \ iso_5428 iso_10367-box mac-is nats-dano nats-sefi \ iso8859-13 iso8859-14 iso8859-15 mac-uk sami-ws2 \ - iso-ir-197 cp1258 tis-620 koi8-u ibm874 + iso-ir-197 tis-620 koi8-u ibm874 gen-special-modules := iso8859-7jp diff --git a/iconvdata/TESTS b/iconvdata/TESTS index 5a4a844cc5..e36a63f24a 100644 --- a/iconvdata/TESTS +++ b/iconvdata/TESTS @@ -72,10 +72,16 @@ CP1251 CP1251 Y UTF8 CP1252 CP1252 Y UTF8 CP1253 CP1253 Y UTF8 CP1254 CP1254 Y UTF8 +# CP1255 and WINDOWS-1255 are the same encodings. The first test works with the +# charmap as well. The second one tests character composition/decomposition. CP1255 CP1255 Y UTF8 +WINDOWS-1255 WINDOWS-1255 N UTF8 CP1256 CP1256 Y UTF8 CP1257 CP1257 Y UTF8 +# CP1258 and WINDOWS-1258 are the same encodings. The first test works with the +# charmap as well. The second one tests character composition/decomposition. CP1258 CP1258 Y UTF8 +WINDOWS-1258 WINDOWS-1258 N UTF8 ISO-2022-JP ISO-2022-JP N UTF8 ISO-2022-JP-2 ISO-2022-JP-2 N UTF8 ISO-2022-KR ISO-2022-KR N UTF8 diff --git a/iconvdata/cp1255.c b/iconvdata/cp1255.c index b9e6ac9bc9..127f8be406 100644 --- a/iconvdata/cp1255.c +++ b/iconvdata/cp1255.c @@ -1,7 +1,8 @@ /* Conversion from and to CP1255. - Copyright (C) 1998 Free Software Foundation, Inc. + Copyright (C) 1998, 2001 Free Software Foundation, Inc. This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998. + Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998, + and Bruno Haible <haible@clisp.cons.org>, 2001. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as @@ -18,12 +19,529 @@ write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ +#include <dlfcn.h> #include <stdint.h> +#include <assert.h> -/* Get the conversion table. */ -#define TABLES <cp1255.h> +#define NELEMS(arr) (sizeof (arr) / sizeof (arr[0])) -#define CHARSET_NAME "CP1255//" -#define HAS_HOLES 1 /* Not all 256 character are defined. */ +/* Definitions used in the body of the `gconv' function. */ +#define CHARSET_NAME "CP1255//" +#define FROM_LOOP from_cp1255 +#define TO_LOOP to_cp1255 +#define DEFINE_INIT 1 +#define DEFINE_FINI 1 +#define MIN_NEEDED_FROM 1 +#define MAX_NEEDED_FROM 1 +#define MIN_NEEDED_TO 4 +#define MAX_NEEDED_TO 4 +#define PREPARE_LOOP \ + int saved_state; \ + int *statep = &data->__statep->__count; +#define EXTRA_LOOP_ARGS , statep -#include <8bit-gap.c> + +/* Since we might have to reset input pointer we must be able to save + and restore the state. */ +#define SAVE_RESET_STATE(Save) \ + if (Save) \ + saved_state = *statep; \ + else \ + *statep = saved_state + + +/* During CP1255 to UCS4 conversion, the COUNT element of the state + contains the last UCS4 character, shifted by 3 bits. */ + + +/* Since this is a stateful encoding we have to provide code which resets + the output state to the initial state. This has to be done during the + flushing. */ +#define EMIT_SHIFT_TO_INIT \ + if (data->__statep->__count != 0) \ + { \ + if (FROM_DIRECTION) \ + { \ + if (__builtin_expect (outbuf + 4 <= outend, 1)) \ + { \ + /* Write out the last character. */ \ + *((uint32_t *) outbuf)++ = data->__statep->__count >> 3; \ + data->__statep->__count = 0; \ + } \ + else \ + /* We don't have enough room in the output buffer. */ \ + status = __GCONV_FULL_OUTPUT; \ + } \ + else \ + /* We don't use shift states in the TO_DIRECTION. */ \ + data->__statep->__count = 0; \ + } + + +/* First define the conversion function from CP1255 to UCS4. */ + +static const uint16_t to_ucs4[128] = { + /* 0x80 */ + 0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, + 0x02C6, 0x2030, 0, 0x2039, 0, 0, 0, 0, + /* 0x90 */ + 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0x02DC, 0x2122, 0, 0x203A, 0, 0, 0, 0, + /* 0xA0 */ + 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x20AA, 0x00A5, 0x00A6, 0x00A7, + 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + /* 0xB0 */ + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, + 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, + /* 0xC0 */ + 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, + 0x05B8, 0x05B9, 0, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, + /* 0xD0 */ + 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3, + 0x05F4, 0, 0, 0, 0, 0, 0, 0, + /* 0xE0 */ + 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, + 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF, + /* 0xF0 */ + 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, + 0x05E8, 0x05E9, 0x05EA, 0, 0, 0x200E, 0x200F, 0, +}; + +/* CP1255 contains eight combining characters: + 0x05b4, 0x05b7, 0x05b8, 0x05b9, 0x05bc, 0x05bf, 0x05c1, 0x05c2. */ + +/* Composition tables for each of the relevant combining characters. */ +static const struct { + uint16_t base; + uint16_t composed; +} comp_table_data[] = { +#define COMP_TABLE_IDX_05B4 0 +#define COMP_TABLE_LEN_05B4 1 + { 0x05D9, 0xFB1D }, +#define COMP_TABLE_IDX_05B7 (COMP_TABLE_IDX_05B4 + COMP_TABLE_LEN_05B4) +#define COMP_TABLE_LEN_05B7 2 + { 0x05D0, 0xFB2E }, + { 0x05F2, 0xFB1F }, +#define COMP_TABLE_IDX_05B8 (COMP_TABLE_IDX_05B7 + COMP_TABLE_LEN_05B7) +#define COMP_TABLE_LEN_05B8 1 + { 0x05D0, 0xFB2F }, +#define COMP_TABLE_IDX_05B9 (COMP_TABLE_IDX_05B8 + COMP_TABLE_LEN_05B8) +#define COMP_TABLE_LEN_05B9 1 + { 0x05D5, 0xFB4B }, +#define COMP_TABLE_IDX_05BC (COMP_TABLE_IDX_05B9 + COMP_TABLE_LEN_05B9) +#define COMP_TABLE_LEN_05BC 24 + { 0x05D0, 0xFB30 }, + { 0x05D1, 0xFB31 }, + { 0x05D2, 0xFB32 }, + { 0x05D3, 0xFB33 }, + { 0x05D4, 0xFB34 }, + { 0x05D5, 0xFB35 }, + { 0x05D6, 0xFB36 }, + { 0x05D8, 0xFB38 }, + { 0x05D9, 0xFB39 }, + { 0x05DA, 0xFB3A }, + { 0x05DB, 0xFB3B }, + { 0x05DC, 0xFB3C }, + { 0x05DE, 0xFB3E }, + { 0x05E0, 0xFB40 }, + { 0x05E1, 0xFB41 }, + { 0x05E3, 0xFB43 }, + { 0x05E4, 0xFB44 }, + { 0x05E6, 0xFB46 }, + { 0x05E7, 0xFB47 }, + { 0x05E8, 0xFB48 }, + { 0x05E9, 0xFB49 }, + { 0x05EA, 0xFB4A }, + { 0xFB2A, 0xFB2C }, + { 0xFB2B, 0xFB2D }, +#define COMP_TABLE_IDX_05BF (COMP_TABLE_IDX_05BC + COMP_TABLE_LEN_05BC) +#define COMP_TABLE_LEN_05BF 3 + { 0x05D1, 0xFB4C }, + { 0x05DB, 0xFB4D }, + { 0x05E4, 0xFB4E }, +#define COMP_TABLE_IDX_05C1 (COMP_TABLE_IDX_05BF + COMP_TABLE_LEN_05BF) +#define COMP_TABLE_LEN_05C1 2 + { 0x05E9, 0xFB2A }, + { 0xFB49, 0xFB2C }, +#define COMP_TABLE_IDX_05C2 (COMP_TABLE_IDX_05C1 + COMP_TABLE_LEN_05C1) +#define COMP_TABLE_LEN_05C2 2 + { 0x05E9, 0xFB2B }, + { 0xFB49, 0xFB2D }, +#define COMP_TABLE_IDX_END (COMP_TABLE_IDX_05C2 + COMP_TABLE_LEN_05C2) +}; +/* Compile-time verification of table size. */ +typedef int verify1[(NELEMS (comp_table_data) == COMP_TABLE_IDX_END) - 1]; + +static const struct { unsigned int idx; unsigned int len; } comp_table[8] = { + { COMP_TABLE_IDX_05B4, COMP_TABLE_LEN_05B4 }, + { COMP_TABLE_IDX_05B7, COMP_TABLE_LEN_05B7 }, + { COMP_TABLE_IDX_05B8, COMP_TABLE_LEN_05B8 }, + { COMP_TABLE_IDX_05B9, COMP_TABLE_LEN_05B9 }, + { COMP_TABLE_IDX_05BC, COMP_TABLE_LEN_05BC }, + { COMP_TABLE_IDX_05BF, COMP_TABLE_LEN_05BF }, + { COMP_TABLE_IDX_05C1, COMP_TABLE_LEN_05C1 }, + { COMP_TABLE_IDX_05C2, COMP_TABLE_LEN_05C2 }, +}; + +#define MIN_NEEDED_INPUT MIN_NEEDED_FROM +#define MIN_NEEDED_OUTPUT MIN_NEEDED_TO +#define MAX_NEEDED_OUTPUT 8 +#define LOOPFCT FROM_LOOP +#define BODY \ + { \ + uint32_t ch = *inptr; \ + uint32_t last_ch; \ + int must_buffer_ch; \ + \ + if (ch >= 0x80) \ + { \ + ch = to_ucs4[ch - 0x80]; \ + if (__builtin_expect (ch == L'\0', 0)) \ + { \ + /* This is an illegal character. */ \ + if (! ignore_errors_p ()) \ + { \ + result = __GCONV_ILLEGAL_INPUT; \ + break; \ + } \ + \ + ++inptr; \ + ++*irreversible; \ + continue; \ + } \ + } \ + \ + /* Determine whether there is a buffered character pending. */ \ + last_ch = *statep >> 3; \ + \ + /* We have to buffer ch if it is a possible match in comp_table_data. */ \ + must_buffer_ch = (ch >= 0x05d0 && ch <= 0x05f2); \ + \ + if (last_ch) \ + { \ + if (ch >= 0x05b0 && ch < 0x05c5) \ + { \ + /* See whether last_ch and ch can be combined. */ \ + unsigned int i, i1, i2; \ + \ + switch (ch) \ + { \ + case 0x05b4: \ + i = 0; \ + break; \ + case 0x05b7: \ + i = 1; \ + break; \ + case 0x05b8: \ + i = 2; \ + break; \ + case 0x05b9: \ + i = 3; \ + break; \ + case 0x05bc: \ + i = 4; \ + break; \ + case 0x05bf: \ + i = 5; \ + break; \ + case 0x05c1: \ + i = 6; \ + break; \ + case 0x05c2: \ + i = 7; \ + break; \ + default: \ + goto not_combining; \ + } \ + \ + i1 = comp_table[i].idx; \ + i2 = i1 + comp_table[i].len - 1; \ + \ + if (last_ch >= comp_table_data[i1].base \ + && last_ch <= comp_table_data[i2].base) \ + { \ + for (;;) \ + { \ + i = (i1 + i2) >> 1; \ + if (last_ch == comp_table_data[i].base) \ + break; \ + if (last_ch < comp_table_data[i].base) \ + { \ + if (i1 == i) \ + goto not_combining; \ + i2 = i; \ + } \ + else \ + { \ + if (i1 != i) \ + i1 = i; \ + else \ + { \ + i = i2; \ + if (last_ch == comp_table_data[i].base) \ + break; \ + goto not_combining; \ + } \ + } \ + } \ + last_ch = comp_table_data[i].composed; \ + if (last_ch == 0xfb2a || last_ch == 0xfb2b \ + || last_ch == 0xfb49) \ + /* Buffer the combined character. */ \ + *statep = last_ch << 3; \ + else \ + { \ + /* Output the combined character. */ \ + put32 (outptr, last_ch); \ + outptr += 4; \ + *statep = 0; \ + } \ + ++inptr; \ + continue; \ + } \ + } \ + \ + not_combining: \ + /* Output the buffered character. */ \ + put32 (outptr, last_ch); \ + outptr += 4; \ + *statep = 0; \ + \ + /* If we don't have enough room to output ch as well, then deal \ + with it in another round. */ \ + if (!must_buffer_ch && __builtin_expect (outptr + 4 > outend, 0)) \ + continue; \ + } \ + \ + if (must_buffer_ch) \ + *statep = ch << 3; \ + else \ + { \ + put32 (outptr, ch); \ + outptr += 4; \ + } \ + ++inptr; \ + } +#define LOOP_NEED_FLAGS +#define EXTRA_LOOP_DECLS , int *statep +#include <iconv/loop.c> + + +/* Next, define the conversion function from UCS4 to CP1255. */ + +static const unsigned char from_ucs4[] = { +#define FROM_IDX_00 0 + 0xa0, 0xa1, 0xa2, 0xa3, 0x00, 0xa5, 0xa6, 0xa7, /* 0x00a0-0x00a7 */ + 0xa8, 0xa9, 0x00, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0x00a8-0x00af */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0x00b0-0x00b7 */ + 0xb8, 0xb9, 0x00, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0x00b8-0x00bf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00c0-0x00c7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00c8-0x00cf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, /* 0x00d0-0x00d7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00d8-0x00df */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00e0-0x00e7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00e8-0x00ef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xba, /* 0x00f0-0x00f7 */ +#define FROM_IDX_02 (FROM_IDX_00 + 88) + 0x88, 0x00, /* 0x02c6-0x02c7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x02c8-0x02cf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x02d0-0x02d7 */ + 0x00, 0x00, 0x00, 0x00, 0x98, /* 0x02d8-0x02dc */ +#define FROM_IDX_05 (FROM_IDX_02 + 23) + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0x05b0-0x05b7 */ + 0xc8, 0xc9, 0x00, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0x05b8-0x05bf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0x00, 0x00, 0x00, 0x00, /* 0x05c0-0x05c7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x05c8-0x05cf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x05d0-0x05d7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x05d8-0x05df */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0x05e0-0x05e7 */ + 0xf8, 0xf9, 0xfa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x05e8-0x05ef */ + 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, /* 0x05f0-0x05f4 */ +#define FROM_IDX_20 (FROM_IDX_05 + 69) + 0xfd, 0xfe, /* 0x200e-0x200f */ + 0x00, 0x00, 0x00, 0x96, 0x97, 0x00, 0x00, 0x00, /* 0x2010-0x2017 */ + 0x91, 0x92, 0x82, 0x00, 0x93, 0x94, 0x84, 0x00, /* 0x2018-0x201f */ + 0x86, 0x87, 0x95, 0x00, 0x00, 0x00, 0x85, 0x00, /* 0x2020-0x2027 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2028-0x202f */ + 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2030-0x2037 */ + 0x00, 0x8b, 0x9b, /* 0x2038-0x203a */ +#define FROM_IDX_FF (FROM_IDX_20 + 45) +}; +/* Compile-time verification of table size. */ +typedef int verify2[(NELEMS (from_ucs4) == FROM_IDX_FF) - 1]; + +static const unsigned char comb_table[8] = { + 0xc4, 0xc7, 0xc8, 0xc9, 0xcc, 0xcf, 0xd1, 0xd2, +}; + +/* Decomposition table for the relevant Unicode characters. */ +static const struct { + uint16_t composed; + uint16_t base; + uint32_t comb1 : 8; + int32_t comb2 : 8; +} decomp_table[] = { + { 0xFB1D, 0x05D9, 0, -1 }, + { 0xFB1F, 0x05F2, 1, -1 }, + { 0xFB2A, 0x05E9, 6, -1 }, + { 0xFB2B, 0x05E9, 7, -1 }, + { 0xFB2C, 0x05E9, 4, 6 }, + { 0xFB2D, 0x05E9, 4, 7 }, + { 0xFB2E, 0x05D0, 1, -1 }, + { 0xFB2F, 0x05D0, 2, -1 }, + { 0xFB30, 0x05D0, 4, -1 }, + { 0xFB31, 0x05D1, 4, -1 }, + { 0xFB32, 0x05D2, 4, -1 }, + { 0xFB33, 0x05D3, 4, -1 }, + { 0xFB34, 0x05D4, 4, -1 }, + { 0xFB35, 0x05D5, 4, -1 }, + { 0xFB36, 0x05D6, 4, -1 }, + { 0xFB38, 0x05D8, 4, -1 }, + { 0xFB39, 0x05D9, 4, -1 }, + { 0xFB3A, 0x05DA, 4, -1 }, + { 0xFB3B, 0x05DB, 4, -1 }, + { 0xFB3C, 0x05DC, 4, -1 }, + { 0xFB3E, 0x05DE, 4, -1 }, + { 0xFB40, 0x05E0, 4, -1 }, + { 0xFB41, 0x05E1, 4, -1 }, + { 0xFB43, 0x05E3, 4, -1 }, + { 0xFB44, 0x05E4, 4, -1 }, + { 0xFB46, 0x05E6, 4, -1 }, + { 0xFB47, 0x05E7, 4, -1 }, + { 0xFB48, 0x05E8, 4, -1 }, + { 0xFB49, 0x05E9, 4, -1 }, + { 0xFB4A, 0x05EA, 4, -1 }, + { 0xFB4B, 0x05D5, 3, -1 }, + { 0xFB4C, 0x05D1, 5, -1 }, + { 0xFB4D, 0x05DB, 5, -1 }, + { 0xFB4E, 0x05E4, 5, -1 }, +}; + +#define MIN_NEEDED_INPUT MIN_NEEDED_TO +#define MIN_NEEDED_OUTPUT MIN_NEEDED_FROM +#define MAX_NEEDED_OUTPUT 3 +#define LOOPFCT TO_LOOP +#define BODY \ + { \ + uint32_t ch = get32 (inptr); \ + \ + if (ch < 0x0080) \ + { \ + *outptr++ = ch; \ + inptr += 4; \ + } \ + else \ + { \ + unsigned char res; \ + \ + if (ch >= 0x00a0 && ch < 0x00f8) \ + res = from_ucs4[ch - 0x00a0 + FROM_IDX_00]; \ + else if (ch == 0x0192) \ + res = 0x83; \ + else if (ch >= 0x02c6 && ch < 0x02dd) \ + res = from_ucs4[ch - 0x02c6 + FROM_IDX_02]; \ + else if (ch >= 0x05b0 && ch < 0x05f5) \ + res = from_ucs4[ch - 0x05b0 + FROM_IDX_05]; \ + else if (ch >= 0x200e && ch < 0x203b) \ + res = from_ucs4[ch - 0x200e + FROM_IDX_20]; \ + else if (ch == 0x20aa) \ + res = 0xa4; \ + else if (ch == 0x20ac) \ + res = 0x80; \ + else if (ch == 0x2122) \ + res = 0x99; \ + else \ + res = 0; \ + \ + if (__builtin_expect (res != 0, 1)) \ + { \ + *outptr++ = res; \ + inptr += 4; \ + } \ + else \ + { \ + /* Try canonical decomposition. */ \ + unsigned int i1, i2; \ + \ + i1 = 0; \ + i2 = sizeof (decomp_table) / sizeof (decomp_table[0]) - 1; \ + if (ch >= decomp_table[i1].composed \ + && ch <= decomp_table[i2].composed) \ + { \ + unsigned int i; \ + \ + for (;;) \ + { \ + i = (i1 + i2) >> 1; \ + if (ch == decomp_table[i].composed) \ + break; \ + if (ch < decomp_table[i].composed) \ + { \ + if (i1 == i) \ + goto failed; \ + i2 = i; \ + } \ + else \ + { \ + if (i1 != i) \ + i1 = i; \ + else \ + { \ + i = i2; \ + if (ch == decomp_table[i].composed) \ + break; \ + goto failed; \ + } \ + } \ + } \ + \ + /* Found a canonical decomposition. */ \ + ch = decomp_table[i].base; \ + /* ch is one of 0x05d0..0x05d6, 0x05d8..0x05dc, 0x05de, \ + 0x05e0..0x05e1, 0x05e3..0x05e4, 0x05e6..0x05ea, 0x05f2. */ \ + ch = from_ucs4[ch - 0x05b0 + FROM_IDX_05]; \ + assert (ch != 0); \ + \ + if (decomp_table[i].comb2 < 0) \ + { \ + /* See whether we have room for two bytes. */ \ + if (__builtin_expect (outptr + 1 >= outend, 0)) \ + { \ + result = __GCONV_FULL_OUTPUT; \ + break; \ + } \ + \ + *outptr++ = (unsigned char) ch; \ + *outptr++ = comb_table[decomp_table[i].comb1]; \ + } \ + else \ + { \ + /* See whether we have room for three bytes. */ \ + if (__builtin_expect (outptr + 2 >= outend, 0)) \ + { \ + result = __GCONV_FULL_OUTPUT; \ + break; \ + } \ + \ + *outptr++ = (unsigned char) ch; \ + *outptr++ = comb_table[decomp_table[i].comb1]; \ + *outptr++ = comb_table[decomp_table[i].comb2]; \ + } \ + \ + inptr += 4; \ + continue; \ + } \ + \ + failed: \ + /* This is an illegal character. */ \ + STANDARD_ERR_HANDLER (4); \ + } \ + } \ + } +#define LOOP_NEED_FLAGS +#define EXTRA_LOOP_DECLS , int *statep +#include <iconv/loop.c> + + +/* Now define the toplevel functions. */ +#include <iconv/skeleton.c> diff --git a/iconvdata/cp1258.c b/iconvdata/cp1258.c index 31fe4d32ef..3636e319cb 100644 --- a/iconvdata/cp1258.c +++ b/iconvdata/cp1258.c @@ -1,7 +1,8 @@ /* Conversion from and to CP1258. - Copyright (C) 1998 Free Software Foundation, Inc. + Copyright (C) 1998, 2001 Free Software Foundation, Inc. This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998. + Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998, + and Bruno Haible <haible@clisp.cons.org>, 2001. The GNU C Library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as @@ -18,12 +19,847 @@ write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ +#include <dlfcn.h> #include <stdint.h> +#include <assert.h> -/* Get the conversion table. */ -#define TABLES <cp1258.h> +#define NELEMS(arr) (sizeof (arr) / sizeof (arr[0])) -#define CHARSET_NAME "CP1258//" -#define HAS_HOLES 1 /* Not all 256 character are defined. */ +/* Definitions used in the body of the `gconv' function. */ +#define CHARSET_NAME "CP1258//" +#define FROM_LOOP from_cp1258 +#define TO_LOOP to_cp1258 +#define DEFINE_INIT 1 +#define DEFINE_FINI 1 +#define MIN_NEEDED_FROM 1 +#define MAX_NEEDED_FROM 1 +#define MIN_NEEDED_TO 4 +#define MAX_NEEDED_TO 4 +#define PREPARE_LOOP \ + int saved_state; \ + int *statep = &data->__statep->__count; +#define EXTRA_LOOP_ARGS , statep -#include <8bit-gap.c> + +/* Since we might have to reset input pointer we must be able to save + and restore the state. */ +#define SAVE_RESET_STATE(Save) \ + if (Save) \ + saved_state = *statep; \ + else \ + *statep = saved_state + + +/* During CP1258 to UCS4 conversion, the COUNT element of the state + contains the last UCS4 character, shifted by 3 bits. */ + + +/* Since this is a stateful encoding we have to provide code which resets + the output state to the initial state. This has to be done during the + flushing. */ +#define EMIT_SHIFT_TO_INIT \ + if (data->__statep->__count != 0) \ + { \ + if (FROM_DIRECTION) \ + { \ + if (__builtin_expect (outbuf + 4 <= outend, 1)) \ + { \ + /* Write out the last character. */ \ + *((uint32_t *) outbuf)++ = data->__statep->__count >> 3; \ + data->__statep->__count = 0; \ + } \ + else \ + /* We don't have enough room in the output buffer. */ \ + status = __GCONV_FULL_OUTPUT; \ + } \ + else \ + /* We don't use shift states in the TO_DIRECTION. */ \ + data->__statep->__count = 0; \ + } + + +/* First define the conversion function from CP1258 to UCS4. */ + +static const uint16_t to_ucs4[128] = { + /* 0x80 */ + 0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, + 0x02C6, 0x2030, 0, 0x2039, 0x0152, 0, 0, 0, + /* 0x90 */ + 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0x02DC, 0x2122, 0, 0x203A, 0x0153, 0, 0, 0x0178, + /* 0xA0 */ + 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, + 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + /* 0xB0 */ + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, + 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, + /* 0xC0 */ + 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF, + /* 0xD0 */ + 0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7, + 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF, + /* 0xE0 */ + 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7, + 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0301, 0x00ED, 0x00EE, 0x00EF, + /* 0xF0 */ + 0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7, + 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0, 0x20AB, 0x00FF, +}; + +/* CP1258 contains five combining characters: + 0x0300, 0x0301, 0x0303, 0x0309, 0x0323. */ + +/* Composition tables for each of the relevant combining characters. */ +static const struct { + uint16_t base; + uint16_t composed; +} comp_table_data[] = { +#define COMP_TABLE_IDX_0300 0 +#define COMP_TABLE_LEN_0300 31 + { 0x0041, 0x00C0 }, + { 0x0045, 0x00C8 }, + { 0x0049, 0x00CC }, + { 0x004E, 0x01F8 }, + { 0x004F, 0x00D2 }, + { 0x0055, 0x00D9 }, + { 0x0057, 0x1E80 }, + { 0x0059, 0x1EF2 }, + { 0x0061, 0x00E0 }, + { 0x0065, 0x00E8 }, + { 0x0069, 0x00EC }, + { 0x006E, 0x01F9 }, + { 0x006F, 0x00F2 }, + { 0x0075, 0x00F9 }, + { 0x0077, 0x1E81 }, + { 0x0079, 0x1EF3 }, + { 0x00A8, 0x1FED }, + { 0x00C2, 0x1EA6 }, + { 0x00CA, 0x1EC0 }, + { 0x00D4, 0x1ED2 }, + { 0x00DC, 0x01DB }, + { 0x00E2, 0x1EA7 }, + { 0x00EA, 0x1EC1 }, + { 0x00F4, 0x1ED3 }, + { 0x00FC, 0x01DC }, + { 0x0102, 0x1EB0 }, + { 0x0103, 0x1EB1 }, +/*{ 0x0112, 0x1E14 },*/ +/*{ 0x0113, 0x1E15 },*/ +/*{ 0x014C, 0x1E50 },*/ +/*{ 0x014D, 0x1E51 },*/ + { 0x01A0, 0x1EDC }, + { 0x01A1, 0x1EDD }, + { 0x01AF, 0x1EEA }, + { 0x01B0, 0x1EEB }, +#define COMP_TABLE_IDX_0301 (COMP_TABLE_IDX_0300 + COMP_TABLE_LEN_0300) +#define COMP_TABLE_LEN_0301 60 + { 0x0041, 0x00C1 }, + { 0x0043, 0x0106 }, + { 0x0045, 0x00C9 }, + { 0x0047, 0x01F4 }, + { 0x0049, 0x00CD }, + { 0x004B, 0x1E30 }, + { 0x004C, 0x0139 }, + { 0x004D, 0x1E3E }, + { 0x004E, 0x0143 }, + { 0x004F, 0x00D3 }, + { 0x0050, 0x1E54 }, + { 0x0052, 0x0154 }, + { 0x0053, 0x015A }, + { 0x0055, 0x00DA }, + { 0x0057, 0x1E82 }, + { 0x0059, 0x00DD }, + { 0x005A, 0x0179 }, + { 0x0061, 0x00E1 }, + { 0x0063, 0x0107 }, + { 0x0065, 0x00E9 }, + { 0x0067, 0x01F5 }, + { 0x0069, 0x00ED }, + { 0x006B, 0x1E31 }, + { 0x006C, 0x013A }, + { 0x006D, 0x1E3F }, + { 0x006E, 0x0144 }, + { 0x006F, 0x00F3 }, + { 0x0070, 0x1E55 }, + { 0x0072, 0x0155 }, + { 0x0073, 0x015B }, + { 0x0075, 0x00FA }, + { 0x0077, 0x1E83 }, + { 0x0079, 0x00FD }, + { 0x007A, 0x017A }, + { 0x00A5, 0x0385 }, + { 0x00A8, 0x1FEE }, + { 0x00C2, 0x1EA4 }, + { 0x00C5, 0x01FA }, + { 0x00C6, 0x01FC }, + { 0x00C7, 0x1E08 }, + { 0x00CA, 0x1EBE }, + { 0x00CF, 0x1E2E }, + { 0x00D4, 0x1ED0 }, +/*{ 0x00D5, 0x1E4C },*/ + { 0x00D8, 0x01FE }, + { 0x00DC, 0x01D7 }, + { 0x00E2, 0x1EA5 }, + { 0x00E5, 0x01FB }, + { 0x00E6, 0x01FD }, + { 0x00E7, 0x1E09 }, + { 0x00EA, 0x1EBF }, + { 0x00EF, 0x1E2F }, + { 0x00F4, 0x1ED1 }, +/*{ 0x00F5, 0x1E4D },*/ + { 0x00F8, 0x01FF }, + { 0x00FC, 0x01D8 }, + { 0x0102, 0x1EAE }, + { 0x0103, 0x1EAF }, +/*{ 0x0112, 0x1E16 },*/ +/*{ 0x0113, 0x1E17 },*/ +/*{ 0x014C, 0x1E52 },*/ +/*{ 0x014D, 0x1E53 },*/ +/*{ 0x0168, 0x1E78 },*/ +/*{ 0x0169, 0x1E79 },*/ + { 0x01A0, 0x1EDA }, + { 0x01A1, 0x1EDB }, + { 0x01AF, 0x1EE8 }, + { 0x01B0, 0x1EE9 }, +#define COMP_TABLE_IDX_0303 (COMP_TABLE_IDX_0301 + COMP_TABLE_LEN_0301) +#define COMP_TABLE_LEN_0303 34 + { 0x0041, 0x00C3 }, + { 0x0045, 0x1EBC }, + { 0x0049, 0x0128 }, + { 0x004E, 0x00D1 }, + { 0x004F, 0x00D5 }, + { 0x0055, 0x0168 }, + { 0x0056, 0x1E7C }, + { 0x0059, 0x1EF8 }, + { 0x0061, 0x00E3 }, + { 0x0065, 0x1EBD }, + { 0x0069, 0x0129 }, + { 0x006E, 0x00F1 }, + { 0x006F, 0x00F5 }, + { 0x0075, 0x0169 }, + { 0x0076, 0x1E7D }, + { 0x0079, 0x1EF9 }, + { 0x00C2, 0x1EAA }, + { 0x00CA, 0x1EC4 }, + { 0x00D3, 0x1E4C }, + { 0x00D4, 0x1ED6 }, + { 0x00D6, 0x1E4E }, + { 0x00DA, 0x1E78 }, + { 0x00E2, 0x1EAB }, + { 0x00EA, 0x1EC5 }, + { 0x00F3, 0x1E4D }, + { 0x00F4, 0x1ED7 }, + { 0x00F6, 0x1E4F }, + { 0x00FA, 0x1E79 }, + { 0x0102, 0x1EB4 }, + { 0x0103, 0x1EB5 }, + { 0x01A0, 0x1EE0 }, + { 0x01A1, 0x1EE1 }, + { 0x01AF, 0x1EEE }, + { 0x01B0, 0x1EEF }, +#define COMP_TABLE_IDX_0309 (COMP_TABLE_IDX_0303 + COMP_TABLE_LEN_0303) +#define COMP_TABLE_LEN_0309 24 + { 0x0041, 0x1EA2 }, + { 0x0045, 0x1EBA }, + { 0x0049, 0x1EC8 }, + { 0x004F, 0x1ECE }, + { 0x0055, 0x1EE6 }, + { 0x0059, 0x1EF6 }, + { 0x0061, 0x1EA3 }, + { 0x0065, 0x1EBB }, + { 0x0069, 0x1EC9 }, + { 0x006F, 0x1ECF }, + { 0x0075, 0x1EE7 }, + { 0x0079, 0x1EF7 }, + { 0x00C2, 0x1EA8 }, + { 0x00CA, 0x1EC2 }, + { 0x00D4, 0x1ED4 }, + { 0x00E2, 0x1EA9 }, + { 0x00EA, 0x1EC3 }, + { 0x00F4, 0x1ED5 }, + { 0x0102, 0x1EB2 }, + { 0x0103, 0x1EB3 }, + { 0x01A0, 0x1EDE }, + { 0x01A1, 0x1EDF }, + { 0x01AF, 0x1EEC }, + { 0x01B0, 0x1EED }, +#define COMP_TABLE_IDX_0323 (COMP_TABLE_IDX_0309 + COMP_TABLE_LEN_0309) +#define COMP_TABLE_LEN_0323 50 + { 0x0041, 0x1EA0 }, + { 0x0042, 0x1E04 }, + { 0x0044, 0x1E0C }, + { 0x0045, 0x1EB8 }, + { 0x0048, 0x1E24 }, + { 0x0049, 0x1ECA }, + { 0x004B, 0x1E32 }, + { 0x004C, 0x1E36 }, + { 0x004D, 0x1E42 }, + { 0x004E, 0x1E46 }, + { 0x004F, 0x1ECC }, + { 0x0052, 0x1E5A }, + { 0x0053, 0x1E62 }, + { 0x0054, 0x1E6C }, + { 0x0055, 0x1EE4 }, + { 0x0056, 0x1E7E }, + { 0x0057, 0x1E88 }, + { 0x0059, 0x1EF4 }, + { 0x005A, 0x1E92 }, + { 0x0061, 0x1EA1 }, + { 0x0062, 0x1E05 }, + { 0x0064, 0x1E0D }, + { 0x0065, 0x1EB9 }, + { 0x0068, 0x1E25 }, + { 0x0069, 0x1ECB }, + { 0x006B, 0x1E33 }, + { 0x006C, 0x1E37 }, + { 0x006D, 0x1E43 }, + { 0x006E, 0x1E47 }, + { 0x006F, 0x1ECD }, + { 0x0072, 0x1E5B }, + { 0x0073, 0x1E63 }, + { 0x0074, 0x1E6D }, + { 0x0075, 0x1EE5 }, + { 0x0076, 0x1E7F }, + { 0x0077, 0x1E89 }, + { 0x0079, 0x1EF5 }, + { 0x007A, 0x1E93 }, + { 0x00C2, 0x1EAC }, + { 0x00CA, 0x1EC6 }, + { 0x00D4, 0x1ED8 }, + { 0x00E2, 0x1EAD }, + { 0x00EA, 0x1EC7 }, + { 0x00F4, 0x1ED9 }, + { 0x0102, 0x1EB6 }, + { 0x0103, 0x1EB7 }, + { 0x01A0, 0x1EE2 }, + { 0x01A1, 0x1EE3 }, + { 0x01AF, 0x1EF0 }, + { 0x01B0, 0x1EF1 }, +#define COMP_TABLE_IDX_END (COMP_TABLE_IDX_0323 + COMP_TABLE_LEN_0323) +}; +/* Compile-time verification of table size. */ +typedef int verify1[(NELEMS (comp_table_data) == COMP_TABLE_IDX_END) - 1]; + +static const struct { unsigned int idx; unsigned int len; } comp_table[5] = { + { COMP_TABLE_IDX_0300, COMP_TABLE_LEN_0300 }, + { COMP_TABLE_IDX_0301, COMP_TABLE_LEN_0301 }, + { COMP_TABLE_IDX_0303, COMP_TABLE_LEN_0303 }, + { COMP_TABLE_IDX_0309, COMP_TABLE_LEN_0309 }, + { COMP_TABLE_IDX_0323, COMP_TABLE_LEN_0323 }, +}; + +#define MIN_NEEDED_INPUT MIN_NEEDED_FROM +#define MIN_NEEDED_OUTPUT MIN_NEEDED_TO +#define MAX_NEEDED_OUTPUT 8 +#define LOOPFCT FROM_LOOP +#define BODY \ + { \ + uint32_t ch = *inptr; \ + uint32_t last_ch; \ + int must_buffer_ch; \ + \ + if (ch >= 0x80) \ + { \ + ch = to_ucs4[ch - 0x80]; \ + if (__builtin_expect (ch == L'\0', 0)) \ + { \ + /* This is an illegal character. */ \ + if (! ignore_errors_p ()) \ + { \ + result = __GCONV_ILLEGAL_INPUT; \ + break; \ + } \ + \ + ++inptr; \ + ++*irreversible; \ + continue; \ + } \ + } \ + \ + /* Determine whether there is a buffered character pending. */ \ + last_ch = *statep >> 3; \ + \ + /* We have to buffer ch if it is a possible match in comp_table_data. */ \ + must_buffer_ch = (ch >= 0x0041 && ch <= 0x01b0); \ + \ + if (last_ch) \ + { \ + if (ch >= 0x0300 && ch < 0x0340) \ + { \ + /* See whether last_ch and ch can be combined. */ \ + unsigned int i, i1, i2; \ + \ + switch (ch) \ + { \ + case 0x0300: \ + i = 0; \ + break; \ + case 0x0301: \ + i = 1; \ + break; \ + case 0x0303: \ + i = 2; \ + break; \ + case 0x0309: \ + i = 3; \ + break; \ + case 0x0323: \ + i = 4; \ + break; \ + default: \ + abort (); \ + } \ + \ + i1 = comp_table[i].idx; \ + i2 = i1 + comp_table[i].len - 1; \ + \ + if (last_ch >= comp_table_data[i1].base \ + && last_ch <= comp_table_data[i2].base) \ + { \ + for (;;) \ + { \ + i = (i1 + i2) >> 1; \ + if (last_ch == comp_table_data[i].base) \ + break; \ + if (last_ch < comp_table_data[i].base) \ + { \ + if (i1 == i) \ + goto not_combining; \ + i2 = i; \ + } \ + else \ + { \ + if (i1 != i) \ + i1 = i; \ + else \ + { \ + i = i2; \ + if (last_ch == comp_table_data[i].base) \ + break; \ + goto not_combining; \ + } \ + } \ + } \ + last_ch = comp_table_data[i].composed; \ + /* Output the combined character. */ \ + put32 (outptr, last_ch); \ + outptr += 4; \ + *statep = 0; \ + ++inptr; \ + continue; \ + } \ + } \ + \ + not_combining: \ + /* Output the buffered character. */ \ + put32 (outptr, last_ch); \ + outptr += 4; \ + *statep = 0; \ + \ + /* If we don't have enough room to output ch as well, then deal \ + with it in another round. */ \ + if (!must_buffer_ch && __builtin_expect (outptr + 4 > outend, 0)) \ + continue; \ + } \ + \ + if (must_buffer_ch) \ + *statep = ch << 3; \ + else \ + { \ + put32 (outptr, ch); \ + outptr += 4; \ + } \ + ++inptr; \ + } +#define LOOP_NEED_FLAGS +#define EXTRA_LOOP_DECLS , int *statep +#include <iconv/loop.c> + + +/* Next, define the conversion function from UCS4 to CP1258. */ + +static const unsigned char from_ucs4[] = { +#define FROM_IDX_00 0 + 0xc4, 0xc5, 0xc6, 0xc7, /* 0x00c4-0x00c7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0x00, 0xcd, 0xce, 0xcf, /* 0x00c8-0x00cf */ + 0x00, 0xd1, 0x00, 0xd3, 0xd4, 0x00, 0xd6, 0xd7, /* 0x00d0-0x00d7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0x00, 0x00, 0xdf, /* 0x00d8-0x00df */ + 0xe0, 0xe1, 0xe2, 0x00, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x00e0-0x00e7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0x00, 0xed, 0xee, 0xef, /* 0x00e8-0x00ef */ + 0x00, 0xf1, 0x00, 0xf3, 0xf4, 0x00, 0xf6, 0xf7, /* 0x00f0-0x00f7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x00, 0x00, 0xff, /* 0x00f8-0x00ff */ + 0x00, 0x00, 0xc3, 0xe3, 0x00, 0x00, 0x00, 0x00, /* 0x0100-0x0107 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0108-0x010f */ + 0xd0, 0xf0, /* 0x0110-0x0111 */ +#define FROM_IDX_01 (FROM_IDX_00 + 78) + 0x8c, 0x9c, 0x00, 0x00, 0x00, 0x00, /* 0x0152-0x0157 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0158-0x015f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0160-0x0167 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0168-0x016f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0170-0x0177 */ + 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0178-0x017f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0180-0x0187 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0188-0x018f */ + 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0190-0x0197 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0198-0x019f */ + 0xd5, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x01a0-0x01a7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdd, /* 0x01a8-0x01af */ + 0xfd, /* 0x01b0-0x01b0 */ +#define FROM_IDX_02 (FROM_IDX_01 + 95) + 0x88, 0x00, /* 0x02c6-0x02c7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x02c8-0x02cf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x02d0-0x02d7 */ + 0x00, 0x00, 0x00, 0x00, 0x98, /* 0x02d8-0x02dc */ +#define FROM_IDX_03 (FROM_IDX_02 + 23) + 0xcc, 0xec, 0x00, 0xde, 0x00, 0x00, 0x00, 0x00, /* 0x0300-0x0307 */ + 0x00, 0xd2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0308-0x030f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0310-0x0317 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0318-0x031f */ + 0x00, 0x00, 0x00, 0xf2, /* 0x0320-0x0323 */ +#define FROM_IDX_20 (FROM_IDX_03 + 36) + 0x96, 0x97, 0x00, 0x00, 0x00, /* 0x2013-0x2017 */ + 0x91, 0x92, 0x82, 0x00, 0x93, 0x94, 0x84, 0x00, /* 0x2018-0x201f */ + 0x86, 0x87, 0x95, 0x00, 0x00, 0x00, 0x85, 0x00, /* 0x2020-0x2027 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2028-0x202f */ + 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2030-0x2037 */ + 0x00, 0x8b, 0x9b, /* 0x2038-0x203a */ +#define FROM_IDX_FF (FROM_IDX_20 + 40) +}; +/* Compile-time verification of table size. */ +typedef int verify2[(NELEMS (from_ucs4) == FROM_IDX_FF) - 1]; + +static const unsigned char comb_table[5] = { + 0xcc, 0xec, 0xde, 0xd2, 0xf2, +}; + +/* Decomposition table for the relevant Unicode characters. */ +static const struct { + uint16_t composed; + uint32_t base : 12; + uint32_t comb1 : 4; +} decomp_table[] = { + { 0x00c0, 0x0041, 0 }, + { 0x00c1, 0x0041, 1 }, + { 0x00c3, 0x0041, 2 }, + { 0x00c8, 0x0045, 0 }, + { 0x00c9, 0x0045, 1 }, + { 0x00cc, 0x0049, 0 }, + { 0x00cd, 0x0049, 1 }, + { 0x00d1, 0x004e, 2 }, + { 0x00d2, 0x004f, 0 }, + { 0x00d3, 0x004f, 1 }, + { 0x00d5, 0x004f, 2 }, + { 0x00d9, 0x0055, 0 }, + { 0x00da, 0x0055, 1 }, + { 0x00dd, 0x0059, 1 }, + { 0x00e0, 0x0061, 0 }, + { 0x00e1, 0x0061, 1 }, + { 0x00e3, 0x0061, 2 }, + { 0x00e8, 0x0065, 0 }, + { 0x00e9, 0x0065, 1 }, + { 0x00ec, 0x0069, 0 }, + { 0x00ed, 0x0069, 1 }, + { 0x00f1, 0x006e, 2 }, + { 0x00f2, 0x006f, 0 }, + { 0x00f3, 0x006f, 1 }, + { 0x00f5, 0x006f, 2 }, + { 0x00f9, 0x0075, 0 }, + { 0x00fa, 0x0075, 1 }, + { 0x00fd, 0x0079, 1 }, + { 0x0106, 0x0043, 1 }, + { 0x0107, 0x0063, 1 }, + { 0x0128, 0x0049, 2 }, + { 0x0129, 0x0069, 2 }, + { 0x0139, 0x004c, 1 }, + { 0x013a, 0x006c, 1 }, + { 0x0143, 0x004e, 1 }, + { 0x0144, 0x006e, 1 }, + { 0x0154, 0x0052, 1 }, + { 0x0155, 0x0072, 1 }, + { 0x015a, 0x0053, 1 }, + { 0x015b, 0x0073, 1 }, + { 0x0168, 0x0055, 2 }, + { 0x0169, 0x0075, 2 }, + { 0x0179, 0x005a, 1 }, + { 0x017a, 0x007a, 1 }, + { 0x01d7, 0x00dc, 1 }, + { 0x01d8, 0x00fc, 1 }, + { 0x01db, 0x00dc, 0 }, + { 0x01dc, 0x00fc, 0 }, + { 0x01f4, 0x0047, 1 }, + { 0x01f5, 0x0067, 1 }, + { 0x01f8, 0x004e, 0 }, + { 0x01f9, 0x006e, 0 }, + { 0x01fa, 0x00c5, 1 }, + { 0x01fb, 0x00e5, 1 }, + { 0x01fc, 0x00c6, 1 }, + { 0x01fd, 0x00e6, 1 }, + { 0x01fe, 0x00d8, 1 }, + { 0x01ff, 0x00f8, 1 }, + { 0x0385, 0x00a5, 1 }, + { 0x1e04, 0x0042, 4 }, + { 0x1e05, 0x0062, 4 }, + { 0x1e08, 0x00c7, 1 }, + { 0x1e09, 0x00e7, 1 }, + { 0x1e0c, 0x0044, 4 }, + { 0x1e0d, 0x0064, 4 }, + { 0x1e24, 0x0048, 4 }, + { 0x1e25, 0x0068, 4 }, + { 0x1e2e, 0x00cf, 1 }, + { 0x1e2f, 0x00ef, 1 }, + { 0x1e30, 0x004b, 1 }, + { 0x1e31, 0x006b, 1 }, + { 0x1e32, 0x004b, 4 }, + { 0x1e33, 0x006b, 4 }, + { 0x1e36, 0x004c, 4 }, + { 0x1e37, 0x006c, 4 }, + { 0x1e3e, 0x004d, 1 }, + { 0x1e3f, 0x006d, 1 }, + { 0x1e42, 0x004d, 4 }, + { 0x1e43, 0x006d, 4 }, + { 0x1e46, 0x004e, 4 }, + { 0x1e47, 0x006e, 4 }, + { 0x1e4c, 0x00d3, 2 }, /*{ 0x1e4c, 0x00d5, 1 }, { 0x1e4c, 0x004f, 1, 2 },*/ + { 0x1e4d, 0x00f3, 2 }, /*{ 0x1e4d, 0x00f5, 1 }, { 0x1e4d, 0x006f, 1, 2 },*/ + { 0x1e4e, 0x00d6, 2 }, + { 0x1e4f, 0x00f6, 2 }, + { 0x1e54, 0x0050, 1 }, + { 0x1e55, 0x0070, 1 }, + { 0x1e5a, 0x0052, 4 }, + { 0x1e5b, 0x0072, 4 }, + { 0x1e62, 0x0053, 4 }, + { 0x1e63, 0x0073, 4 }, + { 0x1e6c, 0x0054, 4 }, + { 0x1e6d, 0x0074, 4 }, + { 0x1e78, 0x00da, 2 }, /*{ 0x1e78, 0x0168, 1 }, { 0x1e78, 0x0055, 1, 2 },*/ + { 0x1e79, 0x00fa, 2 }, /*{ 0x1e79, 0x0169, 1 }, { 0x1e79, 0x0075, 1, 2 },*/ + { 0x1e7c, 0x0056, 2 }, + { 0x1e7d, 0x0076, 2 }, + { 0x1e7e, 0x0056, 4 }, + { 0x1e7f, 0x0076, 4 }, + { 0x1e80, 0x0057, 0 }, + { 0x1e81, 0x0077, 0 }, + { 0x1e82, 0x0057, 1 }, + { 0x1e83, 0x0077, 1 }, + { 0x1e88, 0x0057, 4 }, + { 0x1e89, 0x0077, 4 }, + { 0x1e92, 0x005a, 4 }, + { 0x1e93, 0x007a, 4 }, + { 0x1ea0, 0x0041, 4 }, + { 0x1ea1, 0x0061, 4 }, + { 0x1ea2, 0x0041, 3 }, + { 0x1ea3, 0x0061, 3 }, + { 0x1ea4, 0x00c2, 1 }, + { 0x1ea5, 0x00e2, 1 }, + { 0x1ea6, 0x00c2, 0 }, + { 0x1ea7, 0x00e2, 0 }, + { 0x1ea8, 0x00c2, 3 }, + { 0x1ea9, 0x00e2, 3 }, + { 0x1eaa, 0x00c2, 2 }, + { 0x1eab, 0x00e2, 2 }, + { 0x1eac, 0x00c2, 4 }, + { 0x1ead, 0x00e2, 4 }, + { 0x1eae, 0x0102, 1 }, + { 0x1eaf, 0x0103, 1 }, + { 0x1eb0, 0x0102, 0 }, + { 0x1eb1, 0x0103, 0 }, + { 0x1eb2, 0x0102, 3 }, + { 0x1eb3, 0x0103, 3 }, + { 0x1eb4, 0x0102, 2 }, + { 0x1eb5, 0x0103, 2 }, + { 0x1eb6, 0x0102, 4 }, + { 0x1eb7, 0x0103, 4 }, + { 0x1eb8, 0x0045, 4 }, + { 0x1eb9, 0x0065, 4 }, + { 0x1eba, 0x0045, 3 }, + { 0x1ebb, 0x0065, 3 }, + { 0x1ebc, 0x0045, 2 }, + { 0x1ebd, 0x0065, 2 }, + { 0x1ebe, 0x00ca, 1 }, + { 0x1ebf, 0x00ea, 1 }, + { 0x1ec0, 0x00ca, 0 }, + { 0x1ec1, 0x00ea, 0 }, + { 0x1ec2, 0x00ca, 3 }, + { 0x1ec3, 0x00ea, 3 }, + { 0x1ec4, 0x00ca, 2 }, + { 0x1ec5, 0x00ea, 2 }, + { 0x1ec6, 0x00ca, 4 }, + { 0x1ec7, 0x00ea, 4 }, + { 0x1ec8, 0x0049, 3 }, + { 0x1ec9, 0x0069, 3 }, + { 0x1eca, 0x0049, 4 }, + { 0x1ecb, 0x0069, 4 }, + { 0x1ecc, 0x004f, 4 }, + { 0x1ecd, 0x006f, 4 }, + { 0x1ece, 0x004f, 3 }, + { 0x1ecf, 0x006f, 3 }, + { 0x1ed0, 0x00d4, 1 }, + { 0x1ed1, 0x00f4, 1 }, + { 0x1ed2, 0x00d4, 0 }, + { 0x1ed3, 0x00f4, 0 }, + { 0x1ed4, 0x00d4, 3 }, + { 0x1ed5, 0x00f4, 3 }, + { 0x1ed6, 0x00d4, 2 }, + { 0x1ed7, 0x00f4, 2 }, + { 0x1ed8, 0x00d4, 4 }, + { 0x1ed9, 0x00f4, 4 }, + { 0x1eda, 0x01a0, 1 }, + { 0x1edb, 0x01a1, 1 }, + { 0x1edc, 0x01a0, 0 }, + { 0x1edd, 0x01a1, 0 }, + { 0x1ede, 0x01a0, 3 }, + { 0x1edf, 0x01a1, 3 }, + { 0x1ee0, 0x01a0, 2 }, + { 0x1ee1, 0x01a1, 2 }, + { 0x1ee2, 0x01a0, 4 }, + { 0x1ee3, 0x01a1, 4 }, + { 0x1ee4, 0x0055, 4 }, + { 0x1ee5, 0x0075, 4 }, + { 0x1ee6, 0x0055, 3 }, + { 0x1ee7, 0x0075, 3 }, + { 0x1ee8, 0x01af, 1 }, + { 0x1ee9, 0x01b0, 1 }, + { 0x1eea, 0x01af, 0 }, + { 0x1eeb, 0x01b0, 0 }, + { 0x1eec, 0x01af, 3 }, + { 0x1eed, 0x01b0, 3 }, + { 0x1eee, 0x01af, 2 }, + { 0x1eef, 0x01b0, 2 }, + { 0x1ef0, 0x01af, 4 }, + { 0x1ef1, 0x01b0, 4 }, + { 0x1ef2, 0x0059, 0 }, + { 0x1ef3, 0x0079, 0 }, + { 0x1ef4, 0x0059, 4 }, + { 0x1ef5, 0x0079, 4 }, + { 0x1ef6, 0x0059, 3 }, + { 0x1ef7, 0x0079, 3 }, + { 0x1ef8, 0x0059, 2 }, + { 0x1ef9, 0x0079, 2 }, + { 0x1fed, 0x00a8, 0 }, + { 0x1fee, 0x00a8, 1 }, +}; + +#define MIN_NEEDED_INPUT MIN_NEEDED_TO +#define MIN_NEEDED_OUTPUT MIN_NEEDED_FROM +#define MAX_NEEDED_OUTPUT 2 +#define LOOPFCT TO_LOOP +#define BODY \ + { \ + uint32_t ch = get32 (inptr); \ + \ + if (ch < 0x0080 || (ch >= 0x00a0 && ch < 0x00c3)) \ + { \ + *outptr++ = ch; \ + inptr += 4; \ + } \ + else \ + { \ + unsigned char res; \ + \ + if (ch >= 0x00c4 && ch < 0x0112) \ + res = from_ucs4[ch - 0x00c4 + FROM_IDX_00]; \ + else if (ch >= 0x0152 && ch < 0x01b1) \ + res = from_ucs4[ch - 0x0152 + FROM_IDX_01]; \ + else if (ch >= 0x02c6 && ch < 0x02dd) \ + res = from_ucs4[ch - 0x02c6 + FROM_IDX_02]; \ + else if (ch >= 0x0300 && ch < 0x0324) \ + res = from_ucs4[ch - 0x0300 + FROM_IDX_03]; \ + else if (ch >= 0x0340 && ch < 0x0342) /* Vietnamese tone marks */ \ + res = from_ucs4[ch - 0x0340 + FROM_IDX_03]; \ + else if (ch >= 0x2013 && ch < 0x203b) \ + res = from_ucs4[ch - 0x2013 + FROM_IDX_20]; \ + else if (ch == 0x20ab) \ + res = 0xfe; \ + else if (ch == 0x20ac) \ + res = 0x80; \ + else if (ch == 0x2122) \ + res = 0x99; \ + else \ + res = 0; \ + \ + if (__builtin_expect (res != 0, 1)) \ + { \ + *outptr++ = res; \ + inptr += 4; \ + } \ + else \ + { \ + /* Try canonical decomposition. */ \ + unsigned int i1, i2; \ + \ + i1 = 0; \ + i2 = sizeof (decomp_table) / sizeof (decomp_table[0]) - 1; \ + if (ch >= decomp_table[i1].composed \ + && ch <= decomp_table[i2].composed) \ + { \ + unsigned int i; \ + \ + for (;;) \ + { \ + i = (i1 + i2) >> 1; \ + if (ch == decomp_table[i].composed) \ + break; \ + if (ch < decomp_table[i].composed) \ + { \ + if (i1 == i) \ + goto failed; \ + i2 = i; \ + } \ + else \ + { \ + if (i1 != i) \ + i1 = i; \ + else \ + { \ + i = i2; \ + if (ch == decomp_table[i].composed) \ + break; \ + goto failed; \ + } \ + } \ + } \ + \ + /* Found a canonical decomposition. */ \ + ch = decomp_table[i].base; \ + /* ch is one of 0x0020, 0x0041..0x005a, 0x0061..0x007a, \ + 0x00a5, 0x00a8, 0x00c2, 0x00c5..0x00c7, 0x00ca, 0x00cf, \ + 0x00d3, 0x00d4, 0x00d6, 0x00d8, 0x00da, 0x00dc, 0x00e2, \ + 0x00e5..0x00e7, 0x00ea, 0x00ef, 0x00f3, 0x00f4, 0x00f6, \ + 0x00f8, 0x00fc, 0x0102, 0x0103, 0x01a0, 0x01a1, 0x01af, \ + 0x01b0. */ \ + if (ch >= 0x0100) \ + { \ + if (ch < 0x0112) \ + ch = from_ucs4[ch - 0x00c4 + FROM_IDX_00]; \ + else \ + ch = from_ucs4[ch - 0x0152 + FROM_IDX_01]; \ + } \ + assert (ch != 0); \ + \ + /* See whether we have room for two bytes. */ \ + if (__builtin_expect (outptr + 1 >= outend, 0)) \ + { \ + result = __GCONV_FULL_OUTPUT; \ + break; \ + } \ + \ + *outptr++ = (unsigned char) ch; \ + *outptr++ = comb_table[decomp_table[i].comb1]; \ + inptr += 4; \ + continue; \ + } \ + \ + failed: \ + /* This is an illegal character. */ \ + STANDARD_ERR_HANDLER (4); \ + } \ + } \ + } +#define LOOP_NEED_FLAGS +#define EXTRA_LOOP_DECLS , int *statep +#include <iconv/loop.c> + + +/* Now define the toplevel functions. */ +#include <iconv/skeleton.c> diff --git a/iconvdata/ibm930.c b/iconvdata/ibm930.c index 1a78b90e01..0d09b7bec2 100644 --- a/iconvdata/ibm930.c +++ b/iconvdata/ibm930.c @@ -56,18 +56,15 @@ data->__statep->__count &= 7; \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit `SI'. */ \ - if (__builtin_expect (outbuf >= data->__outbufend, 0)) \ + if (__builtin_expect (outbuf >= outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ { \ /* Write out the shift sequence. */ \ *outbuf++ = SI; \ - data->__outbuf = outbuf; \ data->__statep->__count &= 7; \ } \ } \ diff --git a/iconvdata/ibm933.c b/iconvdata/ibm933.c index 9a9b053627..92a9d5b52a 100644 --- a/iconvdata/ibm933.c +++ b/iconvdata/ibm933.c @@ -56,18 +56,15 @@ data->__statep->__count &= 7; \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit `SI'. */ \ - if (__builtin_expect (outbuf >= data->__outbufend, 0)) \ + if (__builtin_expect (outbuf >= outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ { \ /* Write out the shift sequence. */ \ *outbuf++ = SI; \ - data->__outbuf = outbuf; \ data->__statep->__count &= 7; \ } \ } \ diff --git a/iconvdata/ibm935.c b/iconvdata/ibm935.c index 04b8e9dd38..9728894b0b 100644 --- a/iconvdata/ibm935.c +++ b/iconvdata/ibm935.c @@ -56,18 +56,15 @@ data->__statep->__count &= 7; \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit `SI'. */ \ - if (__builtin_expect (outbuf >= data->__outbufend, 0)) \ + if (__builtin_expect (outbuf >= outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ { \ /* Write out the shift sequence. */ \ *outbuf++ = SI; \ - data->__outbuf = outbuf; \ data->__statep->__count &= 7; \ } \ } \ diff --git a/iconvdata/ibm937.c b/iconvdata/ibm937.c index 4fee56a506..3dca817914 100644 --- a/iconvdata/ibm937.c +++ b/iconvdata/ibm937.c @@ -56,18 +56,15 @@ data->__statep->__count &= 7; \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit `SI'. */ \ - if (__builtin_expect (outbuf >= data->__outbufend, 0)) \ + if (__builtin_expect (outbuf >= outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ { \ /* Write out the shift sequence. */ \ *outbuf++ = SI; \ - data->__outbuf = outbuf; \ data->__statep->__count &= 7; \ } \ } \ diff --git a/iconvdata/ibm939.c b/iconvdata/ibm939.c index d50d880ab0..02307a0084 100644 --- a/iconvdata/ibm939.c +++ b/iconvdata/ibm939.c @@ -56,18 +56,15 @@ data->__statep->__count &= 7; \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit `SI'. */ \ - if (__builtin_expect (outbuf >= data->__outbufend, 0)) \ + if (__builtin_expect (outbuf >= outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ { \ /* Write out the shift sequence. */ \ *outbuf++ = SI; \ - data->__outbuf = outbuf; \ data->__statep->__count &= 7; \ } \ } \ diff --git a/iconvdata/iso-2022-cn-ext.c b/iconvdata/iso-2022-cn-ext.c index ab8a988550..7c59e48eaa 100644 --- a/iconvdata/iso-2022-cn-ext.c +++ b/iconvdata/iso-2022-cn-ext.c @@ -131,11 +131,9 @@ enum data->__statep->__count = ASCII_set << 3; \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit `SI'. */ \ - if (outbuf == data->__outbufend) \ + if (__builtin_expect (outbuf == outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ @@ -144,7 +142,6 @@ enum *outbuf++ = SI; \ if (data->__flags & __GCONV_IS_LAST) \ *irreversible += 1; \ - data->__outbuf = outbuf; \ data->__statep->__count = ASCII_set << 3; \ } \ } \ diff --git a/iconvdata/iso-2022-cn.c b/iconvdata/iso-2022-cn.c index 6040e1294f..6eeeea3606 100644 --- a/iconvdata/iso-2022-cn.c +++ b/iconvdata/iso-2022-cn.c @@ -83,18 +83,15 @@ enum data->__statep->__count = ASCII_set; \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit `SI'. */ \ - if (__builtin_expect (outbuf == data->__outbufend, 0)) \ + if (__builtin_expect (outbuf == outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ { \ /* Write out the shift sequence. */ \ *outbuf++ = SI; \ - data->__outbuf = outbuf; \ data->__statep->__count = ASCII_set; \ } \ } \ diff --git a/iconvdata/iso-2022-jp.c b/iconvdata/iso-2022-jp.c index 1d9164d818..8825dcc0fd 100644 --- a/iconvdata/iso-2022-jp.c +++ b/iconvdata/iso-2022-jp.c @@ -193,10 +193,11 @@ gconv_end (struct __gconv_step *data) the output state to the initial state. This has to be done during the flushing. */ #define EMIT_SHIFT_TO_INIT \ + /* Avoid warning about unused variable 'var'. */ \ + (void) var; \ + \ if ((data->__statep->__count & ~7) != ASCII_set) \ { \ - enum direction dir = ((struct iso2022jp_data *) step->__data)->dir; \ - \ if (dir == from_iso2022jp) \ { \ /* It's easy, we don't have to emit anything, we just reset the \ @@ -207,11 +208,9 @@ gconv_end (struct __gconv_step *data) } \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit the sequence `Esc ( B'. */ \ - if (__builtin_expect (outbuf + 3 > data->__outbufend, 0)) \ + if (__builtin_expect (outbuf + 3 > outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ @@ -220,7 +219,6 @@ gconv_end (struct __gconv_step *data) *outbuf++ = ESC; \ *outbuf++ = '('; \ *outbuf++ = 'B'; \ - data->__outbuf = outbuf; \ /* Note that this also clears the G2 designation. */ \ data->__statep->__count &= ~7; \ data->__statep->__count |= ASCII_set; \ diff --git a/iconvdata/iso-2022-kr.c b/iconvdata/iso-2022-kr.c index c0280fccea..6547768ab2 100644 --- a/iconvdata/iso-2022-kr.c +++ b/iconvdata/iso-2022-kr.c @@ -85,18 +85,15 @@ enum } \ else \ { \ - unsigned char *outbuf = data->__outbuf; \ - \ /* We are not in the initial state. To switch back we have \ to emit `SI'. */ \ - if (__builtin_expect (outbuf == data->__outbufend, 0)) \ + if (__builtin_expect (outbuf == outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ { \ /* Write out the shift sequence. */ \ *outbuf++ = SI; \ - data->__outbuf = outbuf; \ data->__statep->__count = ASCII_set; \ } \ } \ diff --git a/iconvdata/testdata/WINDOWS-1255 b/iconvdata/testdata/WINDOWS-1255 new file mode 100644 index 0000000000..6b0726e0b3 --- /dev/null +++ b/iconvdata/testdata/WINDOWS-1255 @@ -0,0 +1,10 @@ +ãÌÈí +öÄôÇøÀãÌÅòÇ +ëÌÄðÌÄéí +òÈøåÉá +ãÌÆáÆø +ùÑÀçÄéï +áÌÈøÈã +àÇøÀáÌÆä +çÉùÑÆêÀ +îÇëÌÇú áÌÀëåÉøåÉú diff --git a/iconvdata/testdata/WINDOWS-1255..UTF-8 b/iconvdata/testdata/WINDOWS-1255..UTF-8 new file mode 100644 index 0000000000..04806ba9dc --- /dev/null +++ b/iconvdata/testdata/WINDOWS-1255..UTF-8 @@ -0,0 +1,10 @@ +ï¬³Ö¸× +צִפַרְדֵּעַ +כִּï€Ö´×™× +עָרï‹×‘ +דֶּבֶר +שְׁחִין +בָּרָד +אַרְבֶּה +חֹשֶׁךְ +מַכַּת בְּכï‹×¨ï‹×ª diff --git a/iconvdata/testdata/WINDOWS-1258 b/iconvdata/testdata/WINDOWS-1258 new file mode 100644 index 0000000000..13baac25b4 --- /dev/null +++ b/iconvdata/testdata/WINDOWS-1258 @@ -0,0 +1,4 @@ +// The last line of this file is missing the end-of-line terminator +// on purpose, in order to test that the conversion empties the bit buffer +// and shifts back to the initial state at the end of the conversion. +Vietnamese (Tiêìng Viêòt) Chào baòn \ No newline at end of file diff --git a/iconvdata/testdata/WINDOWS-1258..UTF8 b/iconvdata/testdata/WINDOWS-1258..UTF8 new file mode 100644 index 0000000000..8016dbd83e --- /dev/null +++ b/iconvdata/testdata/WINDOWS-1258..UTF8 @@ -0,0 +1,4 @@ +// The last line of this file is missing the end-of-line terminator +// on purpose, in order to test that the conversion empties the bit buffer +// and shifts back to the initial state at the end of the conversion. +Vietnamese (Tiếng Việt) Chà o bạn \ No newline at end of file diff --git a/iconvdata/tst-table-from.c b/iconvdata/tst-table-from.c index 5a2fa9d47a..b24632e1b9 100644 --- a/iconvdata/tst-table-from.c +++ b/iconvdata/tst-table-from.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2000 Free Software Foundation, Inc. +/* Copyright (C) 2000-2001 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Bruno Haible <haible@clisp.cons.org>, 2000. @@ -64,9 +64,13 @@ try (iconv_t cd, unsigned char buf[], unsigned int buflen, unsigned char *out) size_t inbytesleft = buflen; char *outbuf = (char *) out; size_t outbytesleft = 6; - size_t result = iconv (cd, - (char **) &inbuf, &inbytesleft, - &outbuf, &outbytesleft); + size_t result; + + iconv (cd, NULL, NULL, NULL, NULL); + result = iconv (cd, (char **) &inbuf, &inbytesleft, &outbuf, &outbytesleft); + if (result != (size_t)(-1)) + result = iconv (cd, NULL, NULL, &outbuf, &outbytesleft); + if (result == (size_t)(-1)) { if (errno == EILSEQ) @@ -100,17 +104,68 @@ try (iconv_t cd, unsigned char buf[], unsigned int buflen, unsigned char *out) } } -/* Returns the out[] buffer as a Unicode value. */ -static unsigned int +/* Returns the out[] buffer as a Unicode value, formatted as 0x%04X. */ +static const char * utf8_decode (const unsigned char *out, unsigned int outlen) { - return (outlen==1 ? out[0] : - outlen==2 ? ((out[0] & 0x1f) << 6) + (out[1] & 0x3f) : - outlen==3 ? ((out[0] & 0x0f) << 12) + ((out[1] & 0x3f) << 6) + (out[2] & 0x3f) : - outlen==4 ? ((out[0] & 0x07) << 18) + ((out[1] & 0x3f) << 12) + ((out[2] & 0x3f) << 6) + (out[3] & 0x3f) : - outlen==5 ? ((out[0] & 0x03) << 24) + ((out[1] & 0x3f) << 18) + ((out[2] & 0x3f) << 12) + ((out[3] & 0x3f) << 6) + (out[4] & 0x3f) : - outlen==6 ? ((out[0] & 0x01) << 30) + ((out[1] & 0x3f) << 24) + ((out[2] & 0x3f) << 18) + ((out[3] & 0x3f) << 12) + ((out[4] & 0x3f) << 6) + (out[5] & 0x3f) : - 0xfffd); + static char hexbuf[42]; + char *p = hexbuf; + + while (outlen > 0) + { + if (p > hexbuf) + *p++ = ' '; + + if (out[0] < 0x80) + { + sprintf (p, "0x%04X", out[0]); + out += 1; outlen -= 1; + } + else if (out[0] >= 0xc0 && out[0] < 0xe0 && outlen >= 2) + { + sprintf (p, "0x%04X", ((out[0] & 0x1f) << 6) + (out[1] & 0x3f)); + out += 2; outlen -= 2; + } + else if (out[0] >= 0xe0 && out[0] < 0xf0 && outlen >= 3) + { + sprintf (p, "0x%04X", ((out[0] & 0x0f) << 12) + + ((out[1] & 0x3f) << 6) + (out[2] & 0x3f)); + out += 3; outlen -= 3; + } + else if (out[0] >= 0xf0 && out[0] < 0xf8 && outlen >= 4) + { + sprintf (p, "0x%04X", ((out[0] & 0x07) << 18) + + ((out[1] & 0x3f) << 12) + + ((out[2] & 0x3f) << 6) + (out[3] & 0x3f)); + out += 4; outlen -= 4; + } + else if (out[0] >= 0xf8 && out[0] < 0xfc && outlen >= 5) + { + sprintf (p, "0x%04X", ((out[0] & 0x03) << 24) + + ((out[1] & 0x3f) << 18) + + ((out[2] & 0x3f) << 12) + + ((out[3] & 0x3f) << 6) + (out[4] & 0x3f)); + out += 5; outlen -= 5; + } + else if (out[0] >= 0xfc && out[0] < 0xfe && outlen >= 6) + { + sprintf (p, "0x%04X", ((out[0] & 0x01) << 30) + + ((out[1] & 0x3f) << 24) + + ((out[2] & 0x3f) << 18) + + ((out[3] & 0x3f) << 12) + + ((out[4] & 0x3f) << 6) + (out[5] & 0x3f)); + out += 6; outlen -= 6; + } + else + { + sprintf (p, "0x????"); + out += 1; outlen -= 1; + } + + p += strlen (p); + } + + return hexbuf; } int @@ -148,7 +203,7 @@ main (int argc, char *argv[]) } else if (result > 0) { - printf ("0x%02X\t0x%04X\n", + printf ("0x%02X\t%s\n", i0, utf8_decode (out, result)); } else @@ -162,7 +217,7 @@ main (int argc, char *argv[]) } else if (result > 0) { - printf ("0x%02X%02X\t0x%04X\n", + printf ("0x%02X%02X\t%s\n", i0, i1, utf8_decode (out, result)); } else @@ -176,7 +231,7 @@ main (int argc, char *argv[]) } else if (result > 0) { - printf ("0x%02X%02X%02X\t0x%04X\n", + printf ("0x%02X%02X%02X\t%s\n", i0, i1, i2, utf8_decode (out, result)); } else if (strcmp (charset, "UTF-8")) @@ -190,7 +245,7 @@ main (int argc, char *argv[]) } else if (result > 0) { - printf ("0x%02X%02X%02X%02X\t0x%04X\n", + printf ("0x%02X%02X%02X%02X\t%s\n", i0, i1, i2, i3, utf8_decode (out, result)); } diff --git a/iconvdata/utf-7.c b/iconvdata/utf-7.c index f02063d334..98b66751e7 100644 --- a/iconvdata/utf-7.c +++ b/iconvdata/utf-7.c @@ -533,10 +533,9 @@ base64 (unsigned int i) if (state & 0x18) \ { \ /* Deactivate base64 encoding. */ \ - unsigned char *outbuf = data->__outbuf; \ size_t count = ((state & 0x18) >= 0x10) + 1; \ \ - if (__builtin_expect (outbuf + count > data->__outbufend, 0)) \ + if (__builtin_expect (outbuf + count > outend, 0)) \ /* We don't have enough room in the output buffer. */ \ status = __GCONV_FULL_OUTPUT; \ else \ @@ -546,7 +545,6 @@ base64 (unsigned int i) *outbuf++ = base64 ((state >> 3) & ~3); \ *outbuf++ = '-'; \ \ - data->__outbuf = outbuf; \ data->__statep->__count = 0; \ } \ } \ |