diff options
Diffstat (limited to 'iconv')
-rw-r--r-- | iconv/gconv.h | 2 | ||||
-rw-r--r-- | iconv/gconv_int.h | 2 | ||||
-rw-r--r-- | iconv/gconv_simple.c | 177 | ||||
-rw-r--r-- | iconv/loop.c | 105 | ||||
-rw-r--r-- | iconv/skeleton.c | 100 |
5 files changed, 358 insertions, 28 deletions
diff --git a/iconv/gconv.h b/iconv/gconv.h index 85b9a3a..6d79b07 100644 --- a/iconv/gconv.h +++ b/iconv/gconv.h @@ -59,7 +59,7 @@ struct __gconv_loaded_object; /* Type of a conversion function. */ typedef int (*__gconv_fct) (struct __gconv_step *, struct __gconv_step_data *, __const unsigned char **, __const unsigned char *, - size_t *, int); + size_t *, int, int); /* Constructor and destructor for local data for conversion step. */ typedef int (*__gconv_init_fct) (struct __gconv_step *); diff --git a/iconv/gconv_int.h b/iconv/gconv_int.h index 626e45e..bc5d003 100644 --- a/iconv/gconv_int.h +++ b/iconv/gconv_int.h @@ -161,7 +161,7 @@ extern void __gconv_get_builtin_trans (const char *__name, struct __gconv_step_data *__data, \ const unsigned char **__inbuf, \ const unsigned char *__inbufend, size_t *__written, \ - int __do_flush) + int __do_flush, int __consume_incomplete) __BUILTIN_TRANS (__gconv_transform_ascii_internal); __BUILTIN_TRANS (__gconv_transform_internal_ascii); diff --git a/iconv/gconv_simple.c b/iconv/gconv_simple.c index 9710eb1..d06db5a 100644 --- a/iconv/gconv_simple.c +++ b/iconv/gconv_simple.c @@ -145,6 +145,45 @@ internal_ucs4_loop_unaligned (const unsigned char **inptrp, } #endif + +static inline int +internal_ucs4_loop_single (const unsigned char **inptrp, + const unsigned char *inend, + unsigned char **outptrp, unsigned char *outend, + mbstate_t *state, void *data, size_t *converted) +{ + size_t cnt = state->__count & 7; + + while (*inptrp < inend && cnt < 4) + state->__value.__wchb[cnt++] = *(*inptrp)++; + + if (cnt < 4) + { + /* Still not enough bytes. Store the ones in the input buffer. */ + state->__count &= ~7; + state->__count |= cnt; + + return __GCONV_INCOMPLETE_INPUT; + } + +#if __BYTE_ORDER == __LITTLE_ENDIAN + (*outptrp)[0] = state->__value.__wchb[3]; + (*outptrp)[1] = state->__value.__wchb[2]; + (*outptrp)[2] = state->__value.__wchb[1]; + (*outptrp)[3] = state->__value.__wchb[0]; +#elif __BYTE_ORDER == __BIG_ENDIAN + /* XXX unaligned */ + *(*((uint32_t **) outptrp)++) = state->__value.__wch; +#else +# error "This endianess is not supported." +#endif + + /* Clear the state buffer. */ + state->__count &= ~7; + + return __GCONV_OK; +} + #include <iconv/skeleton.c> @@ -244,6 +283,43 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, } #endif + +static inline int +internal_ucs4le_loop_single (const unsigned char **inptrp, + const unsigned char *inend, + unsigned char **outptrp, unsigned char *outend, + mbstate_t *state, void *data, size_t *converted) +{ + size_t cnt = state->__count & 7; + + while (*inptrp < inend && cnt < 4) + state->__value.__wchb[cnt++] = *(*inptrp)++; + + if (cnt < 4) + { + /* Still not enough bytes. Store the ones in the input buffer. */ + state->__count &= ~7; + state->__count |= cnt; + + return __GCONV_INCOMPLETE_INPUT; + } + +#if __BYTE_ORDER == __BIG_ENDIAN + (*outptrp)[0] = state->__value.__wchb[3]; + (*outptrp)[1] = state->__value.__wchb[2]; + (*outptrp)[2] = state->__value.__wchb[1]; + (*outptrp)[3] = state->__value.__wchb[0]; +#else + /* XXX unaligned */ + *(*((uint32_t **) outptrp)++) = state->__value.__wch; +#endif + + /* Clear the state buffer. */ + state->__count &= ~7; + + return __GCONV_OK; +} + #include <iconv/skeleton.c> @@ -256,6 +332,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, #define FROM_LOOP ascii_internal_loop #define TO_LOOP ascii_internal_loop /* This is not used. */ #define FUNCTION_NAME __gconv_transform_ascii_internal +#define ONE_DIRECTION 1 #define MIN_NEEDED_INPUT MIN_NEEDED_FROM #define MIN_NEEDED_OUTPUT MIN_NEEDED_TO @@ -270,6 +347,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, } \ \ /* It's an one byte sequence. */ \ + /* XXX unaligned. */ \ *((uint32_t *) outptr)++ = *inptr++; \ } #include <iconv/loop.c> @@ -285,6 +363,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, #define FROM_LOOP internal_ascii_loop #define TO_LOOP internal_ascii_loop /* This is not used. */ #define FUNCTION_NAME __gconv_transform_internal_ascii +#define ONE_DIRECTION 1 #define MIN_NEEDED_INPUT MIN_NEEDED_FROM #define MIN_NEEDED_OUTPUT MIN_NEEDED_TO @@ -315,6 +394,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, #define FROM_LOOP internal_utf8_loop #define TO_LOOP internal_utf8_loop /* This is not used. */ #define FUNCTION_NAME __gconv_transform_internal_utf8 +#define ONE_DIRECTION 1 #define MIN_NEEDED_INPUT MIN_NEEDED_FROM #define MIN_NEEDED_OUTPUT MIN_NEEDED_TO @@ -375,6 +455,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, #define FROM_LOOP utf8_internal_loop #define TO_LOOP utf8_internal_loop /* This is not used. */ #define FUNCTION_NAME __gconv_transform_utf8_internal +#define ONE_DIRECTION 1 #define MIN_NEEDED_INPUT MIN_NEEDED_FROM #define MAX_NEEDED_INPUT MAX_NEEDED_FROM @@ -438,8 +519,13 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, \ if (NEED_LENGTH_TEST && inptr + cnt > inend) \ { \ - /* We don't have enough input. */ \ - result = __GCONV_INCOMPLETE_INPUT; \ + /* We don't have enough input. But before we report that check \ + that all the bytes are correct. */ \ + for (i = 1; inptr + i < inend; ++i) \ + if ((inptr[i] & 0xc0) != 0x80) \ + break; \ + result = (inptr + i == inend \ + ? __GCONV_INCOMPLETE_INPUT : __GCONV_ILLEGAL_INPUT); \ break; \ } \ \ @@ -472,6 +558,89 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, /* Now adjust the pointers and store the result. */ \ *((uint32_t *) outptr)++ = ch; \ } + +#define STORE_REST \ + { \ + /* We store the remaining bytes while converting them into the UCS4 \ + format. We can assume that the first byte in the buffer is \ + correct and that it requires a larger number of bytes than there \ + are in the input buffer. */ \ + wint_t ch = **inptrp; \ + size_t cnt; \ + \ + state->__count = inend - *inptrp; \ + \ + if (ch >= 0xc2 && ch < 0xe0) \ + { \ + /* We expect two bytes. The first byte cannot be 0xc0 or \ + 0xc1, otherwise the wide character could have been \ + represented using a single byte. */ \ + cnt = 2; \ + ch &= 0x1f; \ + } \ + else if ((ch & 0xf0) == 0xe0) \ + { \ + /* We expect three bytes. */ \ + cnt = 3; \ + ch &= 0x0f; \ + } \ + else if ((ch & 0xf8) == 0xf0) \ + { \ + /* We expect four bytes. */ \ + cnt = 4; \ + ch &= 0x07; \ + } \ + else if ((ch & 0xfc) == 0xf8) \ + { \ + /* We expect five bytes. */ \ + cnt = 5; \ + ch &= 0x03; \ + } \ + else \ + { \ + /* We expect six bytes. */ \ + cnt = 6; \ + ch &= 0x01; \ + } \ + \ + /* The first byte is already consumed. */ \ + --cnt; \ + while (++(*inptrp) < inend) \ + { \ + ch <<= 6; \ + ch |= **inptrp & 0x3f; \ + --cnt; \ + } \ + \ + /* Shift for the so far missing bytes. */ \ + ch <<= cnt * 6; \ + \ + /* Store the value. */ \ + state->__value.__wch = ch; \ + } + +#define UNPACK_BYTES \ + { \ + wint_t wch = state->__value.__wch; \ + inlen = state->__count; \ + \ + if (state->__value.__wch <= 0x7ff) \ + bytebuf[0] = 0xc0; \ + else if (state->__value.__wch <= 0xffff) \ + bytebuf[0] = 0xe0; \ + else if (state->__value.__wch <= 0x1fffff) \ + bytebuf[0] = 0xf0; \ + else if (state->__value.__wch <= 0x3ffffff) \ + bytebuf[0] = 0xf8; \ + else \ + bytebuf[0] = 0xfc; \ + \ + while (inlen-- > 1) \ + bytebuf[inlen] = 0x80 | (wch & 0x3f); \ + \ + bytebuf[0] |= wch; \ + } + #include <iconv/loop.c> #include <iconv/skeleton.c> @@ -485,6 +654,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, #define FROM_LOOP ucs2_internal_loop #define TO_LOOP ucs2_internal_loop /* This is not used. */ #define FUNCTION_NAME __gconv_transform_ucs2_internal +#define ONE_DIRECTION 1 #define MIN_NEEDED_INPUT MIN_NEEDED_FROM #define MIN_NEEDED_OUTPUT MIN_NEEDED_TO @@ -504,6 +674,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, #define FROM_LOOP internal_ucs2_loop #define TO_LOOP internal_ucs2_loop /* This is not used. */ #define FUNCTION_NAME __gconv_transform_internal_ucs2 +#define ONE_DIRECTION 1 #define MIN_NEEDED_INPUT MIN_NEEDED_FROM #define MIN_NEEDED_OUTPUT MIN_NEEDED_TO @@ -530,6 +701,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, #define FROM_LOOP ucs2reverse_internal_loop #define TO_LOOP ucs2reverse_internal_loop/* This is not used.*/ #define FUNCTION_NAME __gconv_transform_ucs2reverse_internal +#define ONE_DIRECTION 1 #define MIN_NEEDED_INPUT MIN_NEEDED_FROM #define MIN_NEEDED_OUTPUT MIN_NEEDED_TO @@ -550,6 +722,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, #define FROM_LOOP internal_ucs2reverse_loop #define TO_LOOP internal_ucs2reverse_loop/* This is not used.*/ #define FUNCTION_NAME __gconv_transform_internal_ucs2reverse +#define ONE_DIRECTION 1 #define MIN_NEEDED_INPUT MIN_NEEDED_FROM #define MIN_NEEDED_OUTPUT MIN_NEEDED_TO diff --git a/iconv/loop.c b/iconv/loop.c index 1e54033..ffdd24d 100644 --- a/iconv/loop.c +++ b/iconv/loop.c @@ -45,6 +45,7 @@ UPDATE_PARAMS code to store result in params. */ +#include <assert.h> #include <endian.h> #include <gconv.h> #include <stdint.h> @@ -261,23 +262,109 @@ FCTNAME (LOOPFCT) (const unsigned char **inptrp, const unsigned char *inend, } -#undef get16 -#undef get32 -#undef put16 -#undef put32 -#undef unaligned - /* Include the file a second time to define the function to define the function to handle unaligned access. */ #if !defined DEFINE_UNALIGNED && !defined _STRING_ARCH_unaligned \ && MIN_NEEDED_FROM != 1 && MAX_NEEDED_FROM % MIN_NEEDED_FROM == 0 \ && MIN_NEEDED_TO != 1 && MAX_NEEDED_TO % MIN_NEEDED_TO == 0 +# undef get16 +# undef get32 +# undef put16 +# undef put32 +# undef unaligned + # define DEFINE_UNALIGNED # include "loop.c" # undef DEFINE_UNALIGNED #endif +#if MAX_NEEDED_INPUT > 1 +# define SINGLE(fct) SINGLE2 (fct) +# define SINGLE2(fct) fct##_single +static inline int +SINGLE(LOOPFCT) (const unsigned char **inptrp, const unsigned char *inend, + unsigned char **outptrp, unsigned char *outend, + mbstate_t *state, void *data, size_t *converted + EXTRA_LOOP_DECLS) +{ + int result = __GCONV_OK; + unsigned char bytebuf[MAX_NEEDED_INPUT]; + const unsigned char *inptr = *inptrp; + unsigned char *outptr = *outptrp; + size_t inlen; + +#ifdef INIT_PARAMS + INIT_PARAMS; +#endif + +#ifdef UNPACK_BYTES + UNPACK_BYTES +#else + /* Add the bytes from the state to the input buffer. */ + for (inlen = 0; inlen < (state->__count & 7); ++ inlen) + bytebuf[inlen] = state->__value.__wchb[inlen]; +#endif + + /* Are there enough bytes in the input buffer? */ + if (inptr + (MAX_NEEDED_INPUT - inlen) > inend) + { +#ifdef STORE_REST + *inptrp = inend; + inptr = bytebuf; + inptrp = &inptr; + inend = &bytebuf[inlen]; + + STORE_REST +#else + /* We don't have enough input for another complete input + character. */ + while (inptr < inend) + state->__value.__wchb[inlen++] = *inptr++; +#endif + + return __GCONV_INCOMPLETE_INPUT; + } + + /* Enough space in output buffer. */ + if ((MIN_NEEDED_OUTPUT != 1 && outptr + MIN_NEEDED_OUTPUT > outend) + || (MIN_NEEDED_OUTPUT == 1 && outptr >= outend)) + /* Overflow in the output buffer. */ + return __GCONV_FULL_OUTPUT; + + /* Now add characters from the normal input buffer. */ + do + bytebuf[inlen++] = *inptr++; + while (inlen < MAX_NEEDED_INPUT); + + inptr = bytebuf; + inend = &inptr[MAX_NEEDED_INPUT]; + do + { + BODY + } + while (0); + + if (result == __GCONV_OK) + { + /* We successfully converted the character (maybe even more). + Update the pointers passed in. */ + assert (inptr - bytebuf > (state->__count & 7)); + + *inptrp += inptr - bytebuf - (state->__count & 7); + *outptrp = outptr; + + /* Clear the state buffer. */ + state->__count &= ~7; + } + + return result; +} +# undef SINGLE +# undef SINGLE2 +#endif + + /* We remove the macro definitions so that we can include this file again for the definition of another function. */ #undef MIN_NEEDED_INPUT @@ -290,3 +377,9 @@ FCTNAME (LOOPFCT) (const unsigned char **inptrp, const unsigned char *inend, #undef EXTRA_LOOP_DECLS #undef INIT_PARAMS #undef UPDATE_PARAMS +#undef get16 +#undef get32 +#undef put16 +#undef put32 +#undef unaligned +#undef UNPACK_BYTES diff --git a/iconv/skeleton.c b/iconv/skeleton.c index 27b1cab..50ee45d 100644 --- a/iconv/skeleton.c +++ b/iconv/skeleton.c @@ -192,11 +192,11 @@ static int to_object; (outbuf - outerr) is always divisible by MIN_NEEDED_TO. */ # define RESET_INPUT_BUFFER \ if (MIN_NEEDED_FROM % MIN_NEEDED_TO == 0) \ - *inbuf -= (outbuf - outerr) * (MIN_NEEDED_FROM / MIN_NEEDED_TO); \ + *inptrp -= (outbuf - outerr) * (MIN_NEEDED_FROM / MIN_NEEDED_TO); \ else if (MIN_NEEDED_TO % MIN_NEEDED_FROM == 0) \ - *inbuf -= (outbuf - outerr) / (MIN_NEEDED_TO / MIN_NEEDED_FROM); \ + *inptrp -= (outbuf - outerr) / (MIN_NEEDED_TO / MIN_NEEDED_FROM); \ else \ - *inbuf -= ((outbuf - outerr) / MIN_NEEDED_TO) * MIN_NEEDED_FROM + *inptrp -= ((outbuf - outerr) / MIN_NEEDED_TO) * MIN_NEEDED_FROM # endif #endif @@ -263,10 +263,15 @@ gconv_init (struct __gconv_step *step) # define FUNCTION_NAME gconv #endif +/* The macros are used to access the function to convert single characters. */ +#define SINGLE(fct) SINGLE2 (fct) +#define SINGLE2(fct) fct##_single + + int FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, - const unsigned char **inbuf, const unsigned char *inbufend, - size_t *written, int do_flush) + const unsigned char **inptrp, const unsigned char *inend, + size_t *written, int do_flush, int consume_incomplete) { struct __gconv_step *next_step = step + 1; struct __gconv_step_data *next_data = data + 1; @@ -288,12 +293,12 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, successfully emitted the escape sequence. */ if (status == __GCONV_OK && ! data->__is_last) status = DL_CALL_FCT (fct, (next_step, next_data, NULL, NULL, - written, 1)); + written, 1, consume_incomplete)); } else { /* We preserve the initial values of the pointer variables. */ - const unsigned char *inptr = *inbuf; + const unsigned char *inptr = *inptrp; unsigned char *outbuf = data->__outbuf; unsigned char *outend = data->__outbufend; unsigned char *outstart; @@ -314,6 +319,36 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, PREPARE_LOOP #endif +#if MAX_NEEDED_FROM > 1 || MAX_NEEDED_TO > 1 + /* If the function is used to implement the mb*towc*() or wc*tomb*() + functions we must test whether any bytes from the last call are + stored in the `state' object. */ + if (((MAX_NEEDED_FROM > 1 && FROM_DIRECTION) + || (MAX_NEEDED_TO > 1 && !FROM_DIRECTION)) + && consume_incomplete && (data->__statep->__count & 7) != 0) + { + /* Yep, we have some bytes left over. Process them now. */ + +# if MAX_NEEDED_FROM > 1 + if (MAX_NEEDED_TO == 1 || FROM_DIRECTION) + status = SINGLE(FROM_LOOP) (inptrp, inend, &outbuf, outend, + data->__statep, step->__data, + &converted EXTRA_LOOP_ARGS); +# endif +# if MAX_NEEDED_FROM > 1 && MAX_NEEDED_TO > 1 && !ONE_DIRECTION + else +# endif +# if MAX_NEEDED_TO > 1 && !ONE_DIRECTION + status = SINGLE(TO_LOOP) (inptrp, inend, &outbuf, outend, + data->__statep, step->__data, + &converted EXTRA_LOOP_ARGS); +# endif + + if (status != __GCONV_OK) + return status; + } +#endif + #if !defined _STRING_ARCH_unaligned \ && MIN_NEEDED_FROM != 1 && MAX_NEEDED_FROM % MIN_NEEDED_FROM == 0 \ && MIN_NEEDED_TO != 1 && MAX_NEEDED_TO % MIN_NEEDED_TO == 0 @@ -335,7 +370,7 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, do { /* Remember the start value for this round. */ - inptr = *inbuf; + inptr = *inptrp; /* The outbuf buffer is empty. */ outstart = outbuf; @@ -347,12 +382,12 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, { if (FROM_DIRECTION) /* Run the conversion loop. */ - status = FROM_LOOP (inbuf, inbufend, &outbuf, outend, + status = FROM_LOOP (inptrp, inend, &outbuf, outend, data->__statep, step->__data, &converted EXTRA_LOOP_ARGS); else /* Run the conversion loop. */ - status = TO_LOOP (inbuf, inbufend, &outbuf, outend, + status = TO_LOOP (inptrp, inend, &outbuf, outend, data->__statep, step->__data, &converted EXTRA_LOOP_ARGS); } @@ -363,13 +398,13 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, { if (FROM_DIRECTION) /* Run the conversion loop. */ - status = GEN_unaligned (FROM_LOOP) (inbuf, inbufend, &outbuf, + status = GEN_unaligned (FROM_LOOP) (inptrp, inend, &outbuf, outend, data->__statep, step->__data, &converted EXTRA_LOOP_ARGS); else /* Run the conversion loop. */ - status = GEN_unaligned (TO_LOOP) (inbuf, inbufend, &outbuf, + status = GEN_unaligned (TO_LOOP) (inptrp, inend, &outbuf, outend, data->__statep, step->__data, &converted EXTRA_LOOP_ARGS); @@ -399,7 +434,8 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, int result; result = DL_CALL_FCT (fct, (next_step, next_data, &outerr, - outbuf, written, 0)); + outbuf, written, 0, + consume_incomplete)); if (result != __GCONV_EMPTY_INPUT) { @@ -413,7 +449,7 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, size_t nstatus; /* Reload the pointers. */ - *inbuf = inptr; + *inptrp = inptr; outbuf = outstart; /* Reset the state. */ @@ -423,16 +459,16 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, if (FROM_DIRECTION) /* Run the conversion loop. */ - nstatus = FROM_LOOP ((const unsigned char **) inbuf, - (const unsigned char *) inbufend, + nstatus = FROM_LOOP ((const unsigned char **) inptrp, + (const unsigned char *) inend, (unsigned char **) &outbuf, (unsigned char *) outerr, data->__statep, step->__data, &converted EXTRA_LOOP_ARGS); else /* Run the conversion loop. */ - nstatus = TO_LOOP ((const unsigned char **) inbuf, - (const unsigned char *) inbufend, + nstatus = TO_LOOP ((const unsigned char **) inptrp, + (const unsigned char *) inend, (unsigned char **) &outbuf, (unsigned char *) outerr, data->__statep, step->__data, @@ -465,6 +501,32 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, #ifdef END_LOOP END_LOOP #endif + + /* If we are supposed to consume all character store now all of the + remaining characters in the `state' object. */ +#if MAX_NEEDED_FROM > 1 || MAX_NEEDED_TO > 1 + if (((MAX_NEEDED_FROM > 1 && FROM_DIRECTION) + || (MAX_NEEDED_TO > 1 && !FROM_DIRECTION)) + && consume_incomplete && status == __GCONV_INCOMPLETE_INPUT) + { +# ifdef STORE_REST + mbstate_t *state = data->__statep; + + STORE_REST +# else + size_t cnt; + + /* Make sure the remaining bytes fit into the state objects + buffer. */ + assert (inend - *inptrp < 4); + + for (cnt = 0; *inptrp < inend; ++cnt) + data->__statep->__value.__wchb[cnt] = *(*inptrp)++; + data->__statep->__count &= ~7; + data->__statep->__count |= cnt; +# endif + } +#endif } return status; @@ -487,3 +549,5 @@ FUNCTION_NAME (struct __gconv_step *step, struct __gconv_step_data *data, #undef FUNCTION_NAME #undef PREPARE_LOOP #undef END_LOOP +#undef ONE_DIRECTION +#undef STORE_REST |