X-Git-Url: https://perl5.git.perl.org/perl5.git/blobdiff_plain/8f8d40ab34e002001c7e6f500a987fa1dfb3bd78..64844641e1be28fdf8b7bba9436537339624f40b:/pp_pack.c diff --git a/pp_pack.c b/pp_pack.c index 834e723..3fa0e74af 100644 --- a/pp_pack.c +++ b/pp_pack.c @@ -1,7 +1,7 @@ /* pp_pack.c * * Copyright (C) 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, - * 2000, 2001, 2002, 2003, by Larry Wall and others + * 2000, 2001, 2002, 2003, 2004, 2005, 2006, by Larry Wall and others * * You may distribute under the terms of either the GNU General Public * License or the Artistic License, as specified in the README file. @@ -16,20 +16,69 @@ * some salt. */ +/* This file contains pp ("push/pop") functions that + * execute the opcodes that make up a perl program. A typical pp function + * expects to find its arguments on the stack, and usually pushes its + * results onto the stack, hence the 'pp' terminology. Each OP structure + * contains a pointer to the relevant pp_foo() function. + * + * This particular file just contains pp_pack() and pp_unpack(). See the + * other pp*.c files for the rest of the pp_ functions. + */ + + #include "EXTERN.h" #define PERL_IN_PP_PACK_C #include "perl.h" -/* - * The compiler on Concurrent CX/UX systems has a subtle bug which only - * seems to show up when compiling pp.c - it generates the wrong double - * precision constant value for (double)UV_MAX when used inline in the body - * of the code below, so this makes a static variable up front (which the - * compiler seems to get correct) and uses it in place of UV_MAX below. - */ -#ifdef CXUX_BROKEN_CONSTANT_CONVERT -static double UV_MAX_cxux = ((double)UV_MAX); +/* Types used by pack/unpack */ +typedef enum { + e_no_len, /* no length */ + e_number, /* number, [] */ + e_star /* asterisk */ +} howlen_t; + +typedef struct tempsym { + const char* patptr; /* current template char */ + const char* patend; /* one after last char */ + const char* grpbeg; /* 1st char of ()-group */ + const char* grpend; /* end of ()-group */ + I32 code; /* template code (!<>) */ + I32 length; /* length/repeat count */ + howlen_t howlen; /* how length is given */ + int level; /* () nesting level */ + U32 flags; /* /=4, comma=2, pack=1 */ + /* and group modifiers */ + STRLEN strbeg; /* offset of group start */ + struct tempsym *previous; /* previous group */ +} tempsym_t; + +#define TEMPSYM_INIT(symptr, p, e, f) \ + STMT_START { \ + (symptr)->patptr = (p); \ + (symptr)->patend = (e); \ + (symptr)->grpbeg = NULL; \ + (symptr)->grpend = NULL; \ + (symptr)->grpend = NULL; \ + (symptr)->code = 0; \ + (symptr)->length = 0; \ + (symptr)->howlen = 0; \ + (symptr)->level = 0; \ + (symptr)->flags = (f); \ + (symptr)->strbeg = 0; \ + (symptr)->previous = NULL; \ + } STMT_END + +#if PERL_VERSION >= 9 +# define PERL_PACK_CAN_BYTEORDER +# define PERL_PACK_CAN_SHRIEKSIGN +#endif + +#ifndef CHAR_BIT +# define CHAR_BIT 8 #endif +/* Maximum number of bytes to which a byte can grow due to upgrade */ +#define UTF8_EXPAND 2 /* * Offset for integer pack/unpack. @@ -55,41 +104,69 @@ static double UV_MAX_cxux = ((double)UV_MAX); /* CROSSCOMPILE and MULTIARCH are going to affect pp_pack() and pp_unpack(). --jhi Feb 1999 */ -#if SHORTSIZE != SIZE16 || LONGSIZE != SIZE32 -# define PERL_NATINT_PACK -#endif - -#if LONGSIZE > 4 && defined(_CRAY) -# if BYTEORDER == 0x12345678 -# define OFF16(p) (char*)(p) -# define OFF32(p) (char*)(p) +#if U16SIZE > SIZE16 || U32SIZE > SIZE32 +# if BYTEORDER == 0x1234 || BYTEORDER == 0x12345678 /* little-endian */ +# define OFF16(p) ((char*)(p)) +# define OFF32(p) ((char*)(p)) # else -# if BYTEORDER == 0x87654321 +# if BYTEORDER == 0x4321 || BYTEORDER == 0x87654321 /* big-endian */ # define OFF16(p) ((char*)(p) + (sizeof(U16) - SIZE16)) # define OFF32(p) ((char*)(p) + (sizeof(U32) - SIZE32)) # else - }}}} bad cray byte order + ++++ bad cray byte order # endif # endif -# define COPY16(s,p) (*(p) = 0, Copy(s, OFF16(p), SIZE16, char)) -# define COPY32(s,p) (*(p) = 0, Copy(s, OFF32(p), SIZE32, char)) -# define COPYNN(s,p,n) (*(p) = 0, Copy(s, (char *)(p), n, char)) -# define CAT16(sv,p) sv_catpvn(sv, OFF16(p), SIZE16) -# define CAT32(sv,p) sv_catpvn(sv, OFF32(p), SIZE32) #else -# define COPY16(s,p) Copy(s, p, SIZE16, char) -# define COPY32(s,p) Copy(s, p, SIZE32, char) -# define COPYNN(s,p,n) Copy(s, (char *)(p), n, char) -# define CAT16(sv,p) sv_catpvn(sv, (char*)(p), SIZE16) -# define CAT32(sv,p) sv_catpvn(sv, (char*)(p), SIZE32) +# define OFF16(p) ((char *) (p)) +# define OFF32(p) ((char *) (p)) #endif +/* Only to be used inside a loop (see the break) */ +#define SHIFT16(utf8, s, strend, p, datumtype) STMT_START { \ + if (utf8) { \ + if (!uni_to_bytes(aTHX_ &(s), strend, OFF16(p), SIZE16, datumtype)) break; \ + } else { \ + Copy(s, OFF16(p), SIZE16, char); \ + (s) += SIZE16; \ + } \ +} STMT_END + +/* Only to be used inside a loop (see the break) */ +#define SHIFT32(utf8, s, strend, p, datumtype) STMT_START { \ + if (utf8) { \ + if (!uni_to_bytes(aTHX_ &(s), strend, OFF32(p), SIZE32, datumtype)) break; \ + } else { \ + Copy(s, OFF32(p), SIZE32, char); \ + (s) += SIZE32; \ + } \ +} STMT_END + +#define PUSH16(utf8, cur, p) PUSH_BYTES(utf8, cur, OFF16(p), SIZE16) +#define PUSH32(utf8, cur, p) PUSH_BYTES(utf8, cur, OFF32(p), SIZE32) + +/* Only to be used inside a loop (see the break) */ +#define SHIFT_VAR(utf8, s, strend, var, datumtype) \ +STMT_START { \ + if (utf8) { \ + if (!uni_to_bytes(aTHX_ &s, strend, \ + (char *) &var, sizeof(var), datumtype)) break;\ + } else { \ + Copy(s, (char *) &var, sizeof(var), char); \ + s += sizeof(var); \ + } \ +} STMT_END + +#define PUSH_VAR(utf8, aptr, var) \ + PUSH_BYTES(utf8, aptr, &(var), sizeof(var)) + /* Avoid stack overflow due to pathological templates. 100 should be plenty. */ #define MAX_SUB_TEMPLATE_LEVEL 100 -/* flags */ +/* flags (note that type modifiers can also be used as flags!) */ +#define FLAG_WAS_UTF8 0x40 +#define FLAG_PARSE_UTF8 0x20 /* Parse as utf8 */ #define FLAG_UNPACK_ONLY_ONE 0x10 -#define FLAG_UNPACK_DO_UTF8 0x08 +#define FLAG_DO_UTF8 0x08 /* The underlying string is utf8 */ #define FLAG_SLASH 0x04 #define FLAG_COMMA 0x02 #define FLAG_PACK 0x01 @@ -100,10 +177,9 @@ S_mul128(pTHX_ SV *sv, U8 m) STRLEN len; char *s = SvPV(sv, len); char *t; - U32 i = 0; if (!strnEQ(s, "0000", 4)) { /* need to grow sv */ - SV *tmpNew = newSVpvn("0000000000", 10); + SV * const tmpNew = newSVpvs("0000000000"); sv_catsv(tmpNew, sv); SvREFCNT_dec(sv); /* free old sv */ @@ -114,7 +190,7 @@ S_mul128(pTHX_ SV *sv, U8 m) while (!*t) /* trailing '\0'? */ t--; while (t > s) { - i = ((*t - '0') << 7) + m; + const U32 i = ((*t - '0') << 7) + m; *(t--) = '0' + (char)(i % 10); m = (char)(i / 10); } @@ -134,185 +210,685 @@ S_mul128(pTHX_ SV *sv, U8 m) #define ISUUCHAR(ch) (memchr(PL_uuemap, (ch), sizeof(PL_uuemap)-1) || (ch) == ' ') #endif +/* type modifiers */ #define TYPE_IS_SHRIEKING 0x100 +#define TYPE_IS_BIG_ENDIAN 0x200 +#define TYPE_IS_LITTLE_ENDIAN 0x400 +#define TYPE_IS_PACK 0x800 +#define TYPE_ENDIANNESS_MASK (TYPE_IS_BIG_ENDIAN|TYPE_IS_LITTLE_ENDIAN) +#define TYPE_MODIFIERS(t) ((t) & ~0xFF) +#define TYPE_NO_MODIFIERS(t) ((t) & 0xFF) + +#ifdef PERL_PACK_CAN_SHRIEKSIGN +# define SHRIEKING_ALLOWED_TYPES "sSiIlLxXnNvV@." +#else +# define SHRIEKING_ALLOWED_TYPES "sSiIlLxX" +#endif -/* Returns the sizeof() struct described by pat */ -STATIC I32 -S_measure_struct(pTHX_ register tempsym_t* symptr) -{ - register I32 len = 0; - register I32 total = 0; - int star; +#ifndef PERL_PACK_CAN_BYTEORDER +/* Put "can't" first because it is shorter */ +# define TYPE_ENDIANNESS(t) 0 +# define TYPE_NO_ENDIANNESS(t) (t) + +# define ENDIANNESS_ALLOWED_TYPES "" + +# define DO_BO_UNPACK(var, type) +# define DO_BO_PACK(var, type) +# define DO_BO_UNPACK_PTR(var, type, pre_cast, post_cast) +# define DO_BO_PACK_PTR(var, type, pre_cast, post_cast) +# define DO_BO_UNPACK_N(var, type) +# define DO_BO_PACK_N(var, type) +# define DO_BO_UNPACK_P(var) +# define DO_BO_PACK_P(var) +# define DO_BO_UNPACK_PC(var) +# define DO_BO_PACK_PC(var) + +#else /* PERL_PACK_CAN_BYTEORDER */ + +# define TYPE_ENDIANNESS(t) ((t) & TYPE_ENDIANNESS_MASK) +# define TYPE_NO_ENDIANNESS(t) ((t) & ~TYPE_ENDIANNESS_MASK) + +# define ENDIANNESS_ALLOWED_TYPES "sSiIlLqQjJfFdDpP(" + +# define DO_BO_UNPACK(var, type) \ + STMT_START { \ + switch (TYPE_ENDIANNESS(datumtype)) { \ + case TYPE_IS_BIG_ENDIAN: var = my_betoh ## type (var); break; \ + case TYPE_IS_LITTLE_ENDIAN: var = my_letoh ## type (var); break; \ + default: break; \ + } \ + } STMT_END + +# define DO_BO_PACK(var, type) \ + STMT_START { \ + switch (TYPE_ENDIANNESS(datumtype)) { \ + case TYPE_IS_BIG_ENDIAN: var = my_htobe ## type (var); break; \ + case TYPE_IS_LITTLE_ENDIAN: var = my_htole ## type (var); break; \ + default: break; \ + } \ + } STMT_END + +# define DO_BO_UNPACK_PTR(var, type, pre_cast, post_cast) \ + STMT_START { \ + switch (TYPE_ENDIANNESS(datumtype)) { \ + case TYPE_IS_BIG_ENDIAN: \ + var = (post_cast*) my_betoh ## type ((pre_cast) var); \ + break; \ + case TYPE_IS_LITTLE_ENDIAN: \ + var = (post_cast *) my_letoh ## type ((pre_cast) var); \ + break; \ + default: \ + break; \ + } \ + } STMT_END + +# define DO_BO_PACK_PTR(var, type, pre_cast, post_cast) \ + STMT_START { \ + switch (TYPE_ENDIANNESS(datumtype)) { \ + case TYPE_IS_BIG_ENDIAN: \ + var = (post_cast *) my_htobe ## type ((pre_cast) var); \ + break; \ + case TYPE_IS_LITTLE_ENDIAN: \ + var = (post_cast *) my_htole ## type ((pre_cast) var); \ + break; \ + default: \ + break; \ + } \ + } STMT_END + +# define BO_CANT_DOIT(action, type) \ + STMT_START { \ + switch (TYPE_ENDIANNESS(datumtype)) { \ + case TYPE_IS_BIG_ENDIAN: \ + Perl_croak(aTHX_ "Can't %s big-endian %ss on this " \ + "platform", #action, #type); \ + break; \ + case TYPE_IS_LITTLE_ENDIAN: \ + Perl_croak(aTHX_ "Can't %s little-endian %ss on this " \ + "platform", #action, #type); \ + break; \ + default: \ + break; \ + } \ + } STMT_END + +# if PTRSIZE == INTSIZE +# define DO_BO_UNPACK_P(var) DO_BO_UNPACK_PTR(var, i, int, void) +# define DO_BO_PACK_P(var) DO_BO_PACK_PTR(var, i, int, void) +# define DO_BO_UNPACK_PC(var) DO_BO_UNPACK_PTR(var, i, int, char) +# define DO_BO_PACK_PC(var) DO_BO_PACK_PTR(var, i, int, char) +# elif PTRSIZE == LONGSIZE +# define DO_BO_UNPACK_P(var) DO_BO_UNPACK_PTR(var, l, long, void) +# define DO_BO_PACK_P(var) DO_BO_PACK_PTR(var, l, long, void) +# define DO_BO_UNPACK_PC(var) DO_BO_UNPACK_PTR(var, l, long, char) +# define DO_BO_PACK_PC(var) DO_BO_PACK_PTR(var, l, long, char) +# elif PTRSIZE == IVSIZE +# define DO_BO_UNPACK_P(var) DO_BO_UNPACK_PTR(var, l, IV, void) +# define DO_BO_PACK_P(var) DO_BO_PACK_PTR(var, l, IV, void) +# define DO_BO_UNPACK_PC(var) DO_BO_UNPACK_PTR(var, l, IV, char) +# define DO_BO_PACK_PC(var) DO_BO_PACK_PTR(var, l, IV, char) +# else +# define DO_BO_UNPACK_P(var) BO_CANT_DOIT(unpack, pointer) +# define DO_BO_PACK_P(var) BO_CANT_DOIT(pack, pointer) +# define DO_BO_UNPACK_PC(var) BO_CANT_DOIT(unpack, pointer) +# define DO_BO_PACK_PC(var) BO_CANT_DOIT(pack, pointer) +# endif - register int size; +# if defined(my_htolen) && defined(my_letohn) && \ + defined(my_htoben) && defined(my_betohn) +# define DO_BO_UNPACK_N(var, type) \ + STMT_START { \ + switch (TYPE_ENDIANNESS(datumtype)) { \ + case TYPE_IS_BIG_ENDIAN: my_betohn(&var, sizeof(type)); break;\ + case TYPE_IS_LITTLE_ENDIAN: my_letohn(&var, sizeof(type)); break;\ + default: break; \ + } \ + } STMT_END + +# define DO_BO_PACK_N(var, type) \ + STMT_START { \ + switch (TYPE_ENDIANNESS(datumtype)) { \ + case TYPE_IS_BIG_ENDIAN: my_htoben(&var, sizeof(type)); break;\ + case TYPE_IS_LITTLE_ENDIAN: my_htolen(&var, sizeof(type)); break;\ + default: break; \ + } \ + } STMT_END +# else +# define DO_BO_UNPACK_N(var, type) BO_CANT_DOIT(unpack, type) +# define DO_BO_PACK_N(var, type) BO_CANT_DOIT(pack, type) +# endif - while (next_symbol(symptr)) { +#endif /* PERL_PACK_CAN_BYTEORDER */ - switch( symptr->howlen ){ - case e_no_len: - case e_number: - len = symptr->length; - break; - case e_star: - Perl_croak(aTHX_ "Within []-length '*' not allowed in %s", - symptr->flags & FLAG_PACK ? "pack" : "unpack" ); - break; - } +#define PACK_SIZE_CANNOT_CSUM 0x80 +#define PACK_SIZE_UNPREDICTABLE 0x40 /* Not a fixed size element */ +#define PACK_SIZE_MASK 0x3F - switch(symptr->code) { - default: - Perl_croak(aTHX_ "Invalid type '%c' in %s", - (int)symptr->code, - symptr->flags & FLAG_PACK ? "pack" : "unpack" ); - case '@': - case '/': - case 'U': /* XXXX Is it correct? */ - case 'w': - case 'u': - Perl_croak(aTHX_ "Within []-length '%c' not allowed in %s", - (int)symptr->code, - symptr->flags & FLAG_PACK ? "pack" : "unpack" ); - case '%': - size = 0; - break; - case '(': - { - tempsym_t savsym = *symptr; - symptr->patptr = savsym.grpbeg; - symptr->patend = savsym.grpend; - /* XXXX Theoretically, we need to measure many times at different - positions, since the subexpression may contain - alignment commands, but be not of aligned length. - Need to detect this and croak(). */ - size = measure_struct(symptr); - *symptr = savsym; - break; - } - case 'X' | TYPE_IS_SHRIEKING: - /* XXXX Is this useful? Then need to treat MEASURE_BACKWARDS. */ - if (!len) /* Avoid division by 0 */ - len = 1; - len = total % len; /* Assumed: the start is aligned. */ - /* FALL THROUGH */ - case 'X': - size = -1; - if (total < len) - Perl_croak(aTHX_ "'X' outside of string in %s", - symptr->flags & FLAG_PACK ? "pack" : "unpack" ); - break; - case 'x' | TYPE_IS_SHRIEKING: - if (!len) /* Avoid division by 0 */ - len = 1; - star = total % len; /* Assumed: the start is aligned. */ - if (star) /* Other portable ways? */ - len = len - star; - else - len = 0; - /* FALL THROUGH */ - case 'x': - case 'A': - case 'Z': - case 'a': - case 'c': - case 'C': - size = 1; - break; - case 'B': - case 'b': - len = (len + 7)/8; - size = 1; - break; - case 'H': - case 'h': - len = (len + 1)/2; - size = 1; - break; - case 's' | TYPE_IS_SHRIEKING: -#if SHORTSIZE != SIZE16 - size = sizeof(short); - break; +/* These tables are regenerated by genpacksizetables.pl (and then hand pasted + in). You're unlikely ever to need to regenerate them. */ + +#if TYPE_IS_SHRIEKING != 0x100 + ++++shriek offset should be 256 +#endif + +typedef U8 packprops_t; +#if 'J'-'I' == 1 +/* ASCII */ +STATIC const packprops_t packprops[512] = { + /* normal */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, + /* C */ sizeof(unsigned char) | PACK_SIZE_UNPREDICTABLE, +#if defined(HAS_LONG_DOUBLE) && defined(USE_LONG_DOUBLE) + /* D */ LONG_DOUBLESIZE, #else - /* FALL THROUGH */ + 0, #endif - case 's': - size = SIZE16; - break; - case 'S' | TYPE_IS_SHRIEKING: -#if SHORTSIZE != SIZE16 - size = sizeof(unsigned short); - break; + 0, + /* F */ NVSIZE, + 0, 0, + /* I */ sizeof(unsigned int), + /* J */ UVSIZE, + 0, + /* L */ SIZE32, + 0, + /* N */ SIZE32, + 0, 0, +#if defined(HAS_QUAD) + /* Q */ sizeof(Uquad_t), #else - /* FALL THROUGH */ + 0, #endif - case 'v': - case 'n': - case 'S': - size = SIZE16; - break; - case 'i' | TYPE_IS_SHRIEKING: - case 'i': - size = sizeof(int); - break; - case 'I' | TYPE_IS_SHRIEKING: - case 'I': - size = sizeof(unsigned int); - break; - case 'j': - size = IVSIZE; - break; - case 'J': - size = UVSIZE; - break; - case 'l' | TYPE_IS_SHRIEKING: -#if LONGSIZE != SIZE32 - size = sizeof(long); - break; + 0, + /* S */ SIZE16, + 0, + /* U */ sizeof(char) | PACK_SIZE_UNPREDICTABLE, + /* V */ SIZE32, + /* W */ sizeof(unsigned char) | PACK_SIZE_UNPREDICTABLE, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* c */ sizeof(char), + /* d */ sizeof(double), + 0, + /* f */ sizeof(float), + 0, 0, + /* i */ sizeof(int), + /* j */ IVSIZE, + 0, + /* l */ SIZE32, + 0, + /* n */ SIZE16, + 0, + /* p */ sizeof(char *) | PACK_SIZE_CANNOT_CSUM, +#if defined(HAS_QUAD) + /* q */ sizeof(Quad_t), #else - /* FALL THROUGH */ + 0, #endif - case 'l': - size = SIZE32; - break; - case 'L' | TYPE_IS_SHRIEKING: -#if LONGSIZE != SIZE32 - size = sizeof(unsigned long); - break; + 0, + /* s */ SIZE16, + 0, 0, + /* v */ SIZE16, + /* w */ sizeof(char) | PACK_SIZE_UNPREDICTABLE | PACK_SIZE_CANNOT_CSUM, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + /* shrieking */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* I */ sizeof(unsigned int), + 0, 0, + /* L */ sizeof(unsigned long), + 0, +#if defined(PERL_PACK_CAN_SHRIEKSIGN) + /* N */ SIZE32, #else - /* FALL THROUGH */ + 0, #endif - case 'V': - case 'N': - case 'L': - size = SIZE32; - break; - case 'P': - len = 1; - /* FALL THROUGH */ - case 'p': - size = sizeof(char*); - break; -#ifdef HAS_QUAD - case 'q': - size = sizeof(Quad_t); - break; - case 'Q': - size = sizeof(Uquad_t); - break; + 0, 0, 0, 0, + /* S */ sizeof(unsigned short), + 0, 0, +#if defined(PERL_PACK_CAN_SHRIEKSIGN) + /* V */ SIZE32, +#else + 0, #endif - case 'f': - size = sizeof(float); - break; - case 'd': - size = sizeof(double); - break; - case 'F': - size = NVSIZE; - break; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, + /* i */ sizeof(int), + 0, 0, + /* l */ sizeof(long), + 0, +#if defined(PERL_PACK_CAN_SHRIEKSIGN) + /* n */ SIZE16, +#else + 0, +#endif + 0, 0, 0, 0, + /* s */ sizeof(short), + 0, 0, +#if defined(PERL_PACK_CAN_SHRIEKSIGN) + /* v */ SIZE16, +#else + 0, +#endif + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; +#else +/* EBCDIC (or bust) */ +STATIC const packprops_t packprops[512] = { + /* normal */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, + /* c */ sizeof(char), + /* d */ sizeof(double), + 0, + /* f */ sizeof(float), + 0, 0, + /* i */ sizeof(int), + 0, 0, 0, 0, 0, 0, 0, + /* j */ IVSIZE, + 0, + /* l */ SIZE32, + 0, + /* n */ SIZE16, + 0, + /* p */ sizeof(char *) | PACK_SIZE_CANNOT_CSUM, +#if defined(HAS_QUAD) + /* q */ sizeof(Quad_t), +#else + 0, +#endif + 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* s */ SIZE16, + 0, 0, + /* v */ SIZE16, + /* w */ sizeof(char) | PACK_SIZE_UNPREDICTABLE | PACK_SIZE_CANNOT_CSUM, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* C */ sizeof(unsigned char) | PACK_SIZE_UNPREDICTABLE, #if defined(HAS_LONG_DOUBLE) && defined(USE_LONG_DOUBLE) - case 'D': - size = LONG_DOUBLESIZE; + /* D */ LONG_DOUBLESIZE, +#else + 0, +#endif + 0, + /* F */ NVSIZE, + 0, 0, + /* I */ sizeof(unsigned int), + 0, 0, 0, 0, 0, 0, 0, + /* J */ UVSIZE, + 0, + /* L */ SIZE32, + 0, + /* N */ SIZE32, + 0, 0, +#if defined(HAS_QUAD) + /* Q */ sizeof(Uquad_t), +#else + 0, +#endif + 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* S */ SIZE16, + 0, + /* U */ sizeof(char) | PACK_SIZE_UNPREDICTABLE, + /* V */ SIZE32, + /* W */ sizeof(unsigned char) | PACK_SIZE_UNPREDICTABLE, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* shrieking */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* i */ sizeof(int), + 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* l */ sizeof(long), + 0, +#if defined(PERL_PACK_CAN_SHRIEKSIGN) + /* n */ SIZE16, +#else + 0, +#endif + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* s */ sizeof(short), + 0, 0, +#if defined(PERL_PACK_CAN_SHRIEKSIGN) + /* v */ SIZE16, +#else + 0, +#endif + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, + /* I */ sizeof(unsigned int), + 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* L */ sizeof(unsigned long), + 0, +#if defined(PERL_PACK_CAN_SHRIEKSIGN) + /* N */ SIZE32, +#else + 0, +#endif + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* S */ sizeof(unsigned short), + 0, 0, +#if defined(PERL_PACK_CAN_SHRIEKSIGN) + /* V */ SIZE32, +#else + 0, +#endif + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; +#endif + +STATIC U8 +uni_to_byte(pTHX_ const char **s, const char *end, I32 datumtype) +{ + STRLEN retlen; + UV val = utf8n_to_uvchr((U8 *) *s, end-*s, &retlen, + ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); + /* We try to process malformed UTF-8 as much as possible (preferrably with + warnings), but these two mean we make no progress in the string and + might enter an infinite loop */ + if (retlen == (STRLEN) -1 || retlen == 0) + Perl_croak(aTHX_ "Malformed UTF-8 string in '%c' format in unpack", + (int) TYPE_NO_MODIFIERS(datumtype)); + if (val >= 0x100) { + if (ckWARN(WARN_UNPACK)) + Perl_warner(aTHX_ packWARN(WARN_UNPACK), + "Character in '%c' format wrapped in unpack", + (int) TYPE_NO_MODIFIERS(datumtype)); + val &= 0xff; + } + *s += retlen; + return (U8)val; +} + +#define SHIFT_BYTE(utf8, s, strend, datumtype) ((utf8) ? \ + uni_to_byte(aTHX_ &(s), (strend), (datumtype)) : \ + *(U8 *)(s)++) + +STATIC bool +uni_to_bytes(pTHX_ const char **s, const char *end, const char *buf, int buf_len, I32 datumtype) +{ + UV val; + STRLEN retlen; + const char *from = *s; + int bad = 0; + const U32 flags = ckWARN(WARN_UTF8) ? + UTF8_CHECK_ONLY : (UTF8_CHECK_ONLY | UTF8_ALLOW_ANY); + for (;buf_len > 0; buf_len--) { + if (from >= end) return FALSE; + val = utf8n_to_uvchr((U8 *) from, end-from, &retlen, flags); + if (retlen == (STRLEN) -1 || retlen == 0) { + from += UTF8SKIP(from); + bad |= 1; + } else from += retlen; + if (val >= 0x100) { + bad |= 2; + val &= 0xff; + } + *(U8 *)buf++ = (U8)val; + } + /* We have enough characters for the buffer. Did we have problems ? */ + if (bad) { + if (bad & 1) { + /* Rewalk the string fragment while warning */ + const char *ptr; + const int flags = ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY; + for (ptr = *s; ptr < from; ptr += UTF8SKIP(ptr)) { + if (ptr >= end) break; + utf8n_to_uvuni((U8 *) ptr, end-ptr, &retlen, flags); + } + if (from > end) from = end; + } + if ((bad & 2) && ckWARN(WARN_UNPACK)) + Perl_warner(aTHX_ packWARN(datumtype & TYPE_IS_PACK ? + WARN_PACK : WARN_UNPACK), + "Character(s) in '%c' format wrapped in %s", + (int) TYPE_NO_MODIFIERS(datumtype), + datumtype & TYPE_IS_PACK ? "pack" : "unpack"); + } + *s = from; + return TRUE; +} + +STATIC bool +next_uni_uu(pTHX_ const char **s, const char *end, I32 *out) +{ + dVAR; + STRLEN retlen; + const UV val = utf8n_to_uvchr((U8 *) *s, end-*s, &retlen, UTF8_CHECK_ONLY); + if (val >= 0x100 || !ISUUCHAR(val) || + retlen == (STRLEN) -1 || retlen == 0) { + *out = 0; + return FALSE; + } + *out = PL_uudmap[val] & 077; + *s += retlen; + return TRUE; +} + +STATIC char * +S_bytes_to_uni(pTHX_ const U8 *start, STRLEN len, char *dest) { + const U8 * const end = start + len; + + while (start < end) { + U8 buffer[UTF8_MAXLEN]; + const int length = + uvuni_to_utf8_flags(buffer, NATIVE_TO_UNI(*start), 0) - buffer; + switch(length) { + case 1: + *dest++ = buffer[0]; + break; + case 2: + *dest++ = buffer[0]; + *dest++ = buffer[1]; + break; + default: + Perl_croak(aTHX_ "Perl bug: value %d UTF-8 expands to %d bytes", + *start, length); + } + start++; + } + return dest; +} + +#define PUSH_BYTES(utf8, cur, buf, len) \ +STMT_START { \ + if (utf8) \ + (cur) = bytes_to_uni((U8 *) buf, len, (cur)); \ + else { \ + Copy(buf, cur, len, char); \ + (cur) += (len); \ + } \ +} STMT_END + +#define GROWING(utf8, cat, start, cur, in_len) \ +STMT_START { \ + STRLEN glen = (in_len); \ + if (utf8) glen *= UTF8_EXPAND; \ + if ((cur) + glen >= (start) + SvLEN(cat)) { \ + (start) = sv_exp_grow(cat, glen); \ + (cur) = (start) + SvCUR(cat); \ + } \ +} STMT_END + +#define PUSH_GROWING_BYTES(utf8, cat, start, cur, buf, in_len) \ +STMT_START { \ + const STRLEN glen = (in_len); \ + STRLEN gl = glen; \ + if (utf8) gl *= UTF8_EXPAND; \ + if ((cur) + gl >= (start) + SvLEN(cat)) { \ + *cur = '\0'; \ + SvCUR_set((cat), (cur) - (start)); \ + (start) = sv_exp_grow(cat, gl); \ + (cur) = (start) + SvCUR(cat); \ + } \ + PUSH_BYTES(utf8, cur, buf, glen); \ +} STMT_END + +#define PUSH_BYTE(utf8, s, byte) \ +STMT_START { \ + if (utf8) { \ + const U8 au8 = (byte); \ + (s) = bytes_to_uni(&au8, 1, (s)); \ + } else *(U8 *)(s)++ = (byte); \ +} STMT_END + +/* Only to be used inside a loop (see the break) */ +#define NEXT_UNI_VAL(val, cur, str, end, utf8_flags) \ +STMT_START { \ + STRLEN retlen; \ + if (str >= end) break; \ + val = utf8n_to_uvchr((U8 *) str, end-str, &retlen, utf8_flags); \ + if (retlen == (STRLEN) -1 || retlen == 0) { \ + *cur = '\0'; \ + Perl_croak(aTHX_ "Malformed UTF-8 string in pack"); \ + } \ + str += retlen; \ +} STMT_END + +static const char *_action( const tempsym_t* symptr ) +{ + return ( symptr->flags & FLAG_PACK ) ? "pack" : "unpack"; +} + +/* Returns the sizeof() struct described by pat */ +STATIC I32 +S_measure_struct(pTHX_ tempsym_t* symptr) +{ + I32 total = 0; + + while (next_symbol(symptr)) { + I32 len; + int size; + + switch (symptr->howlen) { + case e_star: + Perl_croak(aTHX_ "Within []-length '*' not allowed in %s", + _action( symptr ) ); + break; + default: + /* e_no_len and e_number */ + len = symptr->length; break; + } + + size = packprops[TYPE_NO_ENDIANNESS(symptr->code)] & PACK_SIZE_MASK; + if (!size) { + int star; + /* endianness doesn't influence the size of a type */ + switch(TYPE_NO_ENDIANNESS(symptr->code)) { + default: + Perl_croak(aTHX_ "Invalid type '%c' in %s", + (int)TYPE_NO_MODIFIERS(symptr->code), + _action( symptr ) ); +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case '.' | TYPE_IS_SHRIEKING: + case '@' | TYPE_IS_SHRIEKING: #endif + case '@': + case '.': + case '/': + case 'U': /* XXXX Is it correct? */ + case 'w': + case 'u': + Perl_croak(aTHX_ "Within []-length '%c' not allowed in %s", + (int) TYPE_NO_MODIFIERS(symptr->code), + _action( symptr ) ); + case '%': + size = 0; + break; + case '(': + { + tempsym_t savsym = *symptr; + symptr->patptr = savsym.grpbeg; + symptr->patend = savsym.grpend; + /* XXXX Theoretically, we need to measure many times at + different positions, since the subexpression may contain + alignment commands, but be not of aligned length. + Need to detect this and croak(). */ + size = measure_struct(symptr); + *symptr = savsym; + break; + } + case 'X' | TYPE_IS_SHRIEKING: + /* XXXX Is this useful? Then need to treat MEASURE_BACKWARDS. + */ + if (!len) /* Avoid division by 0 */ + len = 1; + len = total % len; /* Assumed: the start is aligned. */ + /* FALL THROUGH */ + case 'X': + size = -1; + if (total < len) + Perl_croak(aTHX_ "'X' outside of string in %s", _action( symptr ) ); + break; + case 'x' | TYPE_IS_SHRIEKING: + if (!len) /* Avoid division by 0 */ + len = 1; + star = total % len; /* Assumed: the start is aligned. */ + if (star) /* Other portable ways? */ + len = len - star; + else + len = 0; + /* FALL THROUGH */ + case 'x': + case 'A': + case 'Z': + case 'a': + size = 1; + break; + case 'B': + case 'b': + len = (len + 7)/8; + size = 1; + break; + case 'H': + case 'h': + len = (len + 1)/2; + size = 1; + break; + + case 'P': + len = 1; + size = sizeof(char*); + break; + } } total += len * size; } @@ -323,11 +899,11 @@ S_measure_struct(pTHX_ register tempsym_t* symptr) /* locate matching closing parenthesis or bracket * returns char pointer to char after match, or NULL */ -STATIC char * -S_group_end(pTHX_ register char *patptr, register char *patend, char ender) +STATIC const char * +S_group_end(pTHX_ register const char *patptr, register const char *patend, char ender) { while (patptr < patend) { - char c = *patptr++; + const char c = *patptr++; if (isSPACE(c)) continue; @@ -351,9 +927,9 @@ S_group_end(pTHX_ register char *patptr, register char *patend, char ender) /* Convert unsigned decimal number to binary. * Expects a pointer to the first digit and address of length variable * Advances char pointer to 1st non-digit char and returns number - */ -STATIC char * -S_get_num(pTHX_ register char *patptr, I32 *lenptr ) + */ +STATIC const char * +S_get_num(pTHX_ register const char *patptr, I32 *lenptr ) { I32 len = *patptr++ - '0'; while (isDIGIT(*patptr)) { @@ -369,10 +945,10 @@ S_get_num(pTHX_ register char *patptr, I32 *lenptr ) * locates next template code and count */ STATIC bool -S_next_symbol(pTHX_ register tempsym_t* symptr ) +S_next_symbol(pTHX_ tempsym_t* symptr ) { - register char* patptr = symptr->patptr; - register char* patend = symptr->patend; + const char* patptr = symptr->patptr; + const char* const patend = symptr->patend; symptr->flags &= ~FLAG_SLASH; @@ -386,43 +962,92 @@ S_next_symbol(pTHX_ register tempsym_t* symptr ) if (patptr < patend) patptr++; } else { - /* We should have found a template code */ + /* We should have found a template code */ I32 code = *patptr++ & 0xFF; + U32 inherited_modifiers = 0; if (code == ','){ /* grandfather in commas but with a warning */ if (((symptr->flags & FLAG_COMMA) == 0) && ckWARN(WARN_UNPACK)){ symptr->flags |= FLAG_COMMA; Perl_warner(aTHX_ packWARN(WARN_UNPACK), - "Invalid type ',' in %s", - symptr->flags & FLAG_PACK ? "pack" : "unpack" ); + "Invalid type ',' in %s", _action( symptr ) ); } continue; } - + /* for '(', skip to ')' */ - if (code == '(') { + if (code == '(') { if( isDIGIT(*patptr) || *patptr == '*' || *patptr == '[' ) Perl_croak(aTHX_ "()-group starts with a count in %s", - symptr->flags & FLAG_PACK ? "pack" : "unpack" ); + _action( symptr ) ); symptr->grpbeg = patptr; patptr = 1 + ( symptr->grpend = group_end(patptr, patend, ')') ); if( symptr->level >= MAX_SUB_TEMPLATE_LEVEL ) Perl_croak(aTHX_ "Too deeply nested ()-groups in %s", - symptr->flags & FLAG_PACK ? "pack" : "unpack" ); + _action( symptr ) ); + } + + /* look for group modifiers to inherit */ + if (TYPE_ENDIANNESS(symptr->flags)) { + if (strchr(ENDIANNESS_ALLOWED_TYPES, TYPE_NO_MODIFIERS(code))) + inherited_modifiers |= TYPE_ENDIANNESS(symptr->flags); } - /* test for '!' modifier */ - if (patptr < patend && *patptr == '!') { - static const char natstr[] = "sSiIlLxX"; - patptr++; - if (strchr(natstr, code)) - code |= TYPE_IS_SHRIEKING; - else - Perl_croak(aTHX_ "'!' allowed only after types %s in %s", - natstr, symptr->flags & FLAG_PACK ? "pack" : "unpack" ); + /* look for modifiers */ + while (patptr < patend) { + const char *allowed; + I32 modifier; + switch (*patptr) { + case '!': + modifier = TYPE_IS_SHRIEKING; + allowed = SHRIEKING_ALLOWED_TYPES; + break; +#ifdef PERL_PACK_CAN_BYTEORDER + case '>': + modifier = TYPE_IS_BIG_ENDIAN; + allowed = ENDIANNESS_ALLOWED_TYPES; + break; + case '<': + modifier = TYPE_IS_LITTLE_ENDIAN; + allowed = ENDIANNESS_ALLOWED_TYPES; + break; +#endif /* PERL_PACK_CAN_BYTEORDER */ + default: + allowed = ""; + modifier = 0; + break; + } + + if (modifier == 0) + break; + + if (!strchr(allowed, TYPE_NO_MODIFIERS(code))) + Perl_croak(aTHX_ "'%c' allowed only after types %s in %s", *patptr, + allowed, _action( symptr ) ); + + if (TYPE_ENDIANNESS(code | modifier) == TYPE_ENDIANNESS_MASK) + Perl_croak(aTHX_ "Can't use both '<' and '>' after type '%c' in %s", + (int) TYPE_NO_MODIFIERS(code), _action( symptr ) ); + else if (TYPE_ENDIANNESS(code | modifier | inherited_modifiers) == + TYPE_ENDIANNESS_MASK) + Perl_croak(aTHX_ "Can't use '%c' in a group with different byte-order in %s", + *patptr, _action( symptr ) ); + + if ((code & modifier) && ckWARN(WARN_UNPACK)) { + Perl_warner(aTHX_ packWARN(WARN_UNPACK), + "Duplicate modifier '%c' after '%c' in %s", + *patptr, (int) TYPE_NO_MODIFIERS(code), + _action( symptr ) ); + } + + code |= modifier; + patptr++; } - /* look for count and/or / */ + /* inherit modifiers */ + code |= inherited_modifiers; + + /* look for count and/or / */ if (patptr < patend) { if (isDIGIT(*patptr)) { patptr = get_num( patptr, &symptr->length ); @@ -433,7 +1058,7 @@ S_next_symbol(pTHX_ register tempsym_t* symptr ) symptr->howlen = e_star; } else if (*patptr == '[') { - char* lenptr = ++patptr; + const char* lenptr = ++patptr; symptr->howlen = e_number; patptr = group_end( patptr, patend, ']' ) + 1; /* what kind of [] is it? */ @@ -441,7 +1066,7 @@ S_next_symbol(pTHX_ register tempsym_t* symptr ) lenptr = get_num( lenptr, &symptr->length ); if( *lenptr != ']' ) Perl_croak(aTHX_ "Malformed integer in [] in %s", - symptr->flags & FLAG_PACK ? "pack" : "unpack"); + _action( symptr ) ); } else { tempsym_t savsym = *symptr; symptr->patend = patptr-1; @@ -465,13 +1090,13 @@ S_next_symbol(pTHX_ register tempsym_t* symptr ) if (patptr < patend) patptr++; } else { - if( *patptr == '/' ){ + if (*patptr == '/') { symptr->flags |= FLAG_SLASH; patptr++; - if( patptr < patend && - (isDIGIT(*patptr) || *patptr == '*' || *patptr == '[') ) + if (patptr < patend && + (isDIGIT(*patptr) || *patptr == '*' || *patptr == '[')) Perl_croak(aTHX_ "'/' does not take a repeat count in %s", - symptr->flags & FLAG_PACK ? "pack" : "unpack" ); + _action( symptr ) ); } break; } @@ -483,31 +1108,50 @@ S_next_symbol(pTHX_ register tempsym_t* symptr ) } symptr->code = code; - symptr->patptr = patptr; + symptr->patptr = patptr; return TRUE; } } - symptr->patptr = patptr; + symptr->patptr = patptr; return FALSE; } /* -=for apidoc unpack_str - -The engine implementing unpack() Perl function. Note: parameters strbeg, new_s -and ocnt are not used. This call should not be used, use unpackstring instead. - -=cut */ - -I32 -Perl_unpack_str(pTHX_ char *pat, register char *patend, register char *s, char *strbeg, char *strend, char **new_s, I32 ocnt, U32 flags) + There is no way to cleanly handle the case where we should process the + string per byte in its upgraded form while it's really in downgraded form + (e.g. estimates like strend-s as an upper bound for the number of + characters left wouldn't work). So if we foresee the need of this + (pattern starts with U or contains U0), we want to work on the encoded + version of the string. Users are advised to upgrade their pack string + themselves if they need to do a lot of unpacks like this on it +*/ +STATIC bool +need_utf8(const char *pat, const char *patend) { - tempsym_t sym = { 0 }; - sym.patptr = pat; - sym.patend = patend; - sym.flags = flags; + bool first = TRUE; + while (pat < patend) { + if (pat[0] == '#') { + pat++; + pat = (const char *) memchr(pat, '\n', patend-pat); + if (!pat) return FALSE; + } else if (pat[0] == 'U') { + if (first || pat[1] == '0') return TRUE; + } else first = FALSE; + pat++; + } + return FALSE; +} - return unpack_rec(&sym, s, s, strend, NULL ); +STATIC char +first_symbol(const char *pat, const char *patend) { + while (pat < patend) { + if (pat[0] != '#') return pat[0]; + pat++; + pat = (const char *) memchr(pat, '\n', patend-pat); + if (!pat) return 0; + pat++; + } + return 0; } /* @@ -520,85 +1164,92 @@ Issue C before and C after the call to this function. =cut */ I32 -Perl_unpackstring(pTHX_ char *pat, register char *patend, register char *s, char *strend, U32 flags) +Perl_unpackstring(pTHX_ const char *pat, const char *patend, const char *s, const char *strend, U32 flags) { - tempsym_t sym = { 0 }; - sym.patptr = pat; - sym.patend = patend; - sym.flags = flags; + tempsym_t sym; + + if (flags & FLAG_DO_UTF8) flags |= FLAG_WAS_UTF8; + else if (need_utf8(pat, patend)) { + /* We probably should try to avoid this in case a scalar context call + wouldn't get to the "U0" */ + STRLEN len = strend - s; + s = (char *) bytes_to_utf8((U8 *) s, &len); + SAVEFREEPV(s); + strend = s + len; + flags |= FLAG_DO_UTF8; + } + + if (first_symbol(pat, patend) != 'U' && (flags & FLAG_DO_UTF8)) + flags |= FLAG_PARSE_UTF8; + + TEMPSYM_INIT(&sym, pat, patend, flags); return unpack_rec(&sym, s, s, strend, NULL ); } STATIC I32 -S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, char *strend, char **new_s ) +S_unpack_rec(pTHX_ tempsym_t* symptr, const char *s, const char *strbeg, const char *strend, const char **new_s ) { - dSP; - I32 datumtype; - register I32 len = 0; - register I32 bits = 0; - register char *str; + dVAR; dSP; SV *sv; - I32 start_sp_offset = SP - PL_stack_base; + const I32 start_sp_offset = SP - PL_stack_base; howlen_t howlen; - /* These must not be in registers: */ - short ashort; - int aint; - long along; -#ifdef HAS_QUAD - Quad_t aquad; -#endif - U16 aushort; - unsigned int auint; - U32 aulong; -#ifdef HAS_QUAD - Uquad_t auquad; -#endif - char *aptr; - float afloat; - double adouble; I32 checksum = 0; UV cuv = 0; NV cdouble = 0.0; - const int bits_in_uv = 8 * sizeof(cuv); - char* strrelbeg = s; + const int bits_in_uv = CHAR_BIT * sizeof(cuv); bool beyond = FALSE; - bool unpack_only_one = (symptr->flags & FLAG_UNPACK_ONLY_ONE) != 0; - - IV aiv; - UV auv; - NV anv; -#if defined(HAS_LONG_DOUBLE) && defined(USE_LONG_DOUBLE) - long double aldouble; -#endif + bool explicit_length; + const bool unpack_only_one = (symptr->flags & FLAG_UNPACK_ONLY_ONE) != 0; + bool utf8 = (symptr->flags & FLAG_PARSE_UTF8) ? 1 : 0; + symptr->strbeg = s - strbeg; while (next_symbol(symptr)) { - datumtype = symptr->code; + packprops_t props; + I32 len; + I32 datumtype = symptr->code; /* do first one only unless in list context - / is implemented by unpacking the count, then poping it from the + / is implemented by unpacking the count, then popping it from the stack, so must check that we're not in the middle of a / */ if ( unpack_only_one && (SP - PL_stack_base == start_sp_offset + 1) && (datumtype != '/') ) /* XXX can this be omitted */ break; - switch( howlen = symptr->howlen ){ - case e_no_len: - case e_number: - len = symptr->length; + switch (howlen = symptr->howlen) { + case e_star: + len = strend - strbeg; /* long enough */ break; - case e_star: - len = strend - strbeg; /* long enough */ + default: + /* e_no_len and e_number */ + len = symptr->length; break; } + explicit_length = TRUE; redo_switch: beyond = s >= strend; - switch(datumtype) { + + props = packprops[TYPE_NO_ENDIANNESS(datumtype)]; + if (props) { + /* props nonzero means we can process this letter. */ + const long size = props & PACK_SIZE_MASK; + const long howmany = (strend - s) / size; + if (len > howmany) + len = howmany; + + if (!checksum || (props & PACK_SIZE_CANNOT_CSUM)) { + if (len && unpack_only_one) len = 1; + EXTEND(SP, len); + EXTEND_MORTAL(len); + } + } + + switch(TYPE_NO_ENDIANNESS(datumtype)) { default: - Perl_croak(aTHX_ "Invalid type '%c' in unpack", (int)datumtype ); + Perl_croak(aTHX_ "Invalid type '%c' in unpack", (int)TYPE_NO_MODIFIERS(datumtype) ); case '%': if (howlen == e_no_len) @@ -610,51 +1261,137 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c break; case '(': { - char *ss = s; /* Move from register */ tempsym_t savsym = *symptr; + const U32 group_modifiers = TYPE_MODIFIERS(datumtype & ~symptr->flags); + symptr->flags |= group_modifiers; symptr->patend = savsym.grpend; + symptr->previous = &savsym; symptr->level++; PUTBACK; while (len--) { symptr->patptr = savsym.grpbeg; - unpack_rec(symptr, ss, strbeg, strend, &ss ); - if (ss == strend && savsym.howlen == e_star) + if (utf8) symptr->flags |= FLAG_PARSE_UTF8; + else symptr->flags &= ~FLAG_PARSE_UTF8; + unpack_rec(symptr, s, strbeg, strend, &s); + if (s == strend && savsym.howlen == e_star) break; /* No way to continue */ } SPAGAIN; - s = ss; - savsym.flags = symptr->flags; + savsym.flags = symptr->flags & ~group_modifiers; *symptr = savsym; break; } +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case '.' | TYPE_IS_SHRIEKING: +#endif + case '.': { + const char *from; + SV *sv; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + const bool u8 = utf8 && !(datumtype & TYPE_IS_SHRIEKING); +#else /* PERL_PACK_CAN_SHRIEKSIGN */ + const bool u8 = utf8; +#endif + if (howlen == e_star) from = strbeg; + else if (len <= 0) from = s; + else { + tempsym_t *group = symptr; + + while (--len && group) group = group->previous; + from = group ? strbeg + group->strbeg : strbeg; + } + sv = from <= s ? + newSVuv( u8 ? (UV) utf8_length((const U8*)from, (const U8*)s) : (UV) (s-from)) : + newSViv(-(u8 ? (IV) utf8_length((const U8*)s, (const U8*)from) : (IV) (from-s))); + XPUSHs(sv_2mortal(sv)); + break; + } +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case '@' | TYPE_IS_SHRIEKING: +#endif case '@': - if (len > strend - strrelbeg) - Perl_croak(aTHX_ "'@' outside of string in unpack"); - s = strrelbeg + len; + s = strbeg + symptr->strbeg; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + if (utf8 && !(datumtype & TYPE_IS_SHRIEKING)) +#else /* PERL_PACK_CAN_SHRIEKSIGN */ + if (utf8) +#endif + { + while (len > 0) { + if (s >= strend) + Perl_croak(aTHX_ "'@' outside of string in unpack"); + s += UTF8SKIP(s); + len--; + } + if (s > strend) + Perl_croak(aTHX_ "'@' outside of string with malformed UTF-8 in unpack"); + } else { + if (strend-s < len) + Perl_croak(aTHX_ "'@' outside of string in unpack"); + s += len; + } break; case 'X' | TYPE_IS_SHRIEKING: if (!len) /* Avoid division by 0 */ len = 1; - len = (s - strbeg) % len; + if (utf8) { + const char *hop, *last; + I32 l = len; + hop = last = strbeg; + while (hop < s) { + hop += UTF8SKIP(hop); + if (--l == 0) { + last = hop; + l = len; + } + } + if (last > s) + Perl_croak(aTHX_ "Malformed UTF-8 string in unpack"); + s = last; + break; + } + len = (s - strbeg) % len; /* FALL THROUGH */ case 'X': - if (len > s - strbeg) - Perl_croak(aTHX_ "'X' outside of string in unpack" ); - s -= len; - break; - case 'x' | TYPE_IS_SHRIEKING: - if (!len) /* Avoid division by 0 */ - len = 1; - aint = (s - strbeg) % len; - if (aint) /* Other portable ways? */ - len = len - aint; - else - len = 0; - /* FALL THROUGH */ + if (utf8) { + while (len > 0) { + if (s <= strbeg) + Perl_croak(aTHX_ "'X' outside of string in unpack"); + while (--s, UTF8_IS_CONTINUATION(*s)) { + if (s <= strbeg) + Perl_croak(aTHX_ "'X' outside of string in unpack"); + } + len--; + } + } else { + if (len > s - strbeg) + Perl_croak(aTHX_ "'X' outside of string in unpack" ); + s -= len; + } + break; + case 'x' | TYPE_IS_SHRIEKING: { + I32 ai32; + if (!len) /* Avoid division by 0 */ + len = 1; + if (utf8) ai32 = utf8_length((U8 *) strbeg, (U8 *) s) % len; + else ai32 = (s - strbeg) % len; + if (ai32 == 0) break; + len -= ai32; + } + /* FALL THROUGH */ case 'x': - if (len > strend - s) - Perl_croak(aTHX_ "'x' outside of string in unpack"); - s += len; + if (utf8) { + while (len>0) { + if (s >= strend) + Perl_croak(aTHX_ "'x' outside of string in unpack"); + s += UTF8SKIP(s); + len--; + } + } else { + if (len > strend - s) + Perl_croak(aTHX_ "'x' outside of string in unpack"); + s += len; + } break; case '/': Perl_croak(aTHX_ "'/' must follow a numeric type in unpack"); @@ -662,40 +1399,72 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c case 'A': case 'Z': case 'a': - if (len > strend - s) - len = strend - s; - if (checksum) - goto uchar_checksum; - sv = NEWSV(35, len); - sv_setpvn(sv, s, len); - if (len > 0 && (datumtype == 'A' || datumtype == 'Z')) { - aptr = s; /* borrow register */ - if (datumtype == 'Z') { /* 'Z' strips stuff after first null */ - s = SvPVX(sv); - while (*s) - s++; - if (howlen == e_star) /* exact for 'Z*' */ - len = s - SvPVX(sv) + 1; - } - else { /* 'A' strips both nulls and spaces */ - s = SvPVX(sv) + len - 1; - while (s >= SvPVX(sv) && (!*s || isSPACE(*s))) - s--; - *++s = '\0'; + if (checksum) { + /* Preliminary length estimate is assumed done in 'W' */ + if (len > strend - s) len = strend - s; + goto W_checksum; + } + if (utf8) { + I32 l; + const char *hop; + for (l=len, hop=s; l>0; l--, hop += UTF8SKIP(hop)) { + if (hop >= strend) { + if (hop > strend) + Perl_croak(aTHX_ "Malformed UTF-8 string in unpack"); + break; + } } - SvCUR_set(sv, s - SvPVX(sv)); - s = aptr; /* unborrow register */ + if (hop > strend) + Perl_croak(aTHX_ "Malformed UTF-8 string in unpack"); + len = hop - s; + } else if (len > strend - s) + len = strend - s; + + if (datumtype == 'Z') { + /* 'Z' strips stuff after first null */ + const char *ptr, *end; + end = s + len; + for (ptr = s; ptr < end; ptr++) if (*ptr == 0) break; + sv = newSVpvn(s, ptr-s); + if (howlen == e_star) /* exact for 'Z*' */ + len = ptr-s + (ptr != strend ? 1 : 0); + } else if (datumtype == 'A') { + /* 'A' strips both nulls and spaces */ + const char *ptr; + if (utf8 && (symptr->flags & FLAG_WAS_UTF8)) { + for (ptr = s+len-1; ptr >= s; ptr--) + if (*ptr != 0 && !UTF8_IS_CONTINUATION(*ptr) && + !is_utf8_space((U8 *) ptr)) break; + if (ptr >= s) ptr += UTF8SKIP(ptr); + else ptr++; + if (ptr > s+len) + Perl_croak(aTHX_ "Malformed UTF-8 string in unpack"); + } else { + for (ptr = s+len-1; ptr >= s; ptr--) + if (*ptr != 0 && !isSPACE(*ptr)) break; + ptr++; + } + sv = newSVpvn(s, ptr-s); + } else sv = newSVpvn(s, len); + + if (utf8) { + SvUTF8_on(sv); + /* Undo any upgrade done due to need_utf8() */ + if (!(symptr->flags & FLAG_WAS_UTF8)) + sv_utf8_downgrade(sv, 0); } - s += len; XPUSHs(sv_2mortal(sv)); + s += len; break; case 'B': - case 'b': + case 'b': { + char *str; if (howlen == e_star || len > (strend - s) * 8) len = (strend - s) * 8; if (checksum) { if (!PL_bitcount) { - Newz(601, PL_bitcount, 256, char); + int bits; + Newxz(PL_bitcount, 256, char); for (bits = 1; bits < 256; bits++) { if (bits & 1) PL_bitcount[bits]++; if (bits & 2) PL_bitcount[bits]++; @@ -707,280 +1476,248 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c if (bits & 128) PL_bitcount[bits]++; } } - while (len >= 8) { - cuv += PL_bitcount[*(unsigned char*)s++]; - len -= 8; - } - if (len) { - bits = *s; - if (datumtype == 'b') { + if (utf8) + while (len >= 8 && s < strend) { + cuv += PL_bitcount[uni_to_byte(aTHX_ &s, strend, datumtype)]; + len -= 8; + } + else + while (len >= 8) { + cuv += PL_bitcount[*(U8 *)s++]; + len -= 8; + } + if (len && s < strend) { + U8 bits; + bits = SHIFT_BYTE(utf8, s, strend, datumtype); + if (datumtype == 'b') while (len-- > 0) { if (bits & 1) cuv++; bits >>= 1; } - } - else { + else while (len-- > 0) { - if (bits & 128) cuv++; + if (bits & 0x80) cuv++; bits <<= 1; } - } } break; } - sv = NEWSV(35, len + 1); - SvCUR_set(sv, len); + + sv = sv_2mortal(newSV(len ? len : 1)); SvPOK_on(sv); str = SvPVX(sv); if (datumtype == 'b') { - aint = len; - for (len = 0; len < aint; len++) { - if (len & 7) /*SUPPRESS 595*/ - bits >>= 1; - else - bits = *s++; - *str++ = '0' + (bits & 1); + U8 bits = 0; + const I32 ai32 = len; + for (len = 0; len < ai32; len++) { + if (len & 7) bits >>= 1; + else if (utf8) { + if (s >= strend) break; + bits = uni_to_byte(aTHX_ &s, strend, datumtype); + } else bits = *(U8 *) s++; + *str++ = bits & 1 ? '1' : '0'; } - } - else { - aint = len; - for (len = 0; len < aint; len++) { - if (len & 7) - bits <<= 1; - else - bits = *s++; - *str++ = '0' + ((bits & 128) != 0); + } else { + U8 bits = 0; + const I32 ai32 = len; + for (len = 0; len < ai32; len++) { + if (len & 7) bits <<= 1; + else if (utf8) { + if (s >= strend) break; + bits = uni_to_byte(aTHX_ &s, strend, datumtype); + } else bits = *(U8 *) s++; + *str++ = bits & 0x80 ? '1' : '0'; } } *str = '\0'; - XPUSHs(sv_2mortal(sv)); + SvCUR_set(sv, str - SvPVX_const(sv)); + XPUSHs(sv); break; + } case 'H': - case 'h': + case 'h': { + char *str; + /* Preliminary length estimate, acceptable for utf8 too */ if (howlen == e_star || len > (strend - s) * 2) len = (strend - s) * 2; - sv = NEWSV(35, len + 1); - SvCUR_set(sv, len); + sv = sv_2mortal(newSV(len ? len : 1)); SvPOK_on(sv); str = SvPVX(sv); if (datumtype == 'h') { - aint = len; - for (len = 0; len < aint; len++) { - if (len & 1) - bits >>= 4; - else - bits = *s++; + U8 bits = 0; + I32 ai32 = len; + for (len = 0; len < ai32; len++) { + if (len & 1) bits >>= 4; + else if (utf8) { + if (s >= strend) break; + bits = uni_to_byte(aTHX_ &s, strend, datumtype); + } else bits = * (U8 *) s++; *str++ = PL_hexdigit[bits & 15]; } - } - else { - aint = len; - for (len = 0; len < aint; len++) { - if (len & 1) - bits <<= 4; - else - bits = *s++; + } else { + U8 bits = 0; + const I32 ai32 = len; + for (len = 0; len < ai32; len++) { + if (len & 1) bits <<= 4; + else if (utf8) { + if (s >= strend) break; + bits = uni_to_byte(aTHX_ &s, strend, datumtype); + } else bits = *(U8 *) s++; *str++ = PL_hexdigit[(bits >> 4) & 15]; } } *str = '\0'; - XPUSHs(sv_2mortal(sv)); + SvCUR_set(sv, str - SvPVX_const(sv)); + XPUSHs(sv); break; + } case 'c': - if (len > strend - s) - len = strend - s; - if (checksum) { - while (len-- > 0) { - aint = *s++; - if (aint >= 128) /* fake up signed chars */ - aint -= 256; - if (checksum > bits_in_uv) - cdouble += (NV)aint; - else - cuv += aint; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - aint = *s++; - if (aint >= 128) /* fake up signed chars */ - aint -= 256; - sv = NEWSV(36, 0); - sv_setiv(sv, (IV)aint); - PUSHs(sv_2mortal(sv)); - } + while (len-- > 0) { + int aint = SHIFT_BYTE(utf8, s, strend, datumtype); + if (aint >= 128) /* fake up signed chars */ + aint -= 256; + if (!checksum) + PUSHs(sv_2mortal(newSViv((IV)aint))); + else if (checksum > bits_in_uv) + cdouble += (NV)aint; + else + cuv += aint; } break; case 'C': - unpack_C: /* unpack U will jump here if not UTF-8 */ + case 'W': + W_checksum: if (len == 0) { - symptr->flags &= ~FLAG_UNPACK_DO_UTF8; + if (explicit_length && datumtype == 'C') + /* Switch to "character" mode */ + utf8 = (symptr->flags & FLAG_DO_UTF8) ? 1 : 0; break; } - if (len > strend - s) - len = strend - s; - if (checksum) { - uchar_checksum: - while (len-- > 0) { - auint = *s++ & 255; - cuv += auint; + if (datumtype == 'C' ? + (symptr->flags & FLAG_DO_UTF8) && + !(symptr->flags & FLAG_WAS_UTF8) : utf8) { + while (len-- > 0 && s < strend) { + STRLEN retlen; + const UV val = utf8n_to_uvchr((U8 *) s, strend-s, &retlen, + ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); + if (retlen == (STRLEN) -1 || retlen == 0) + Perl_croak(aTHX_ "Malformed UTF-8 string in unpack"); + s += retlen; + if (!checksum) + PUSHs(sv_2mortal(newSVuv((UV) val))); + else if (checksum > bits_in_uv) + cdouble += (NV) val; + else + cuv += val; } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); + } else if (!checksum) while (len-- > 0) { - auint = *s++ & 255; - sv = NEWSV(37, 0); - sv_setiv(sv, (IV)auint); - PUSHs(sv_2mortal(sv)); - } + const U8 ch = *(U8 *) s++; + PUSHs(sv_2mortal(newSVuv((UV) ch))); } + else if (checksum > bits_in_uv) + while (len-- > 0) cdouble += (NV) *(U8 *) s++; + else + while (len-- > 0) cuv += *(U8 *) s++; break; case 'U': if (len == 0) { - symptr->flags |= FLAG_UNPACK_DO_UTF8; - break; - } - if ((symptr->flags & FLAG_UNPACK_DO_UTF8) == 0) - goto unpack_C; - if (len > strend - s) - len = strend - s; - if (checksum) { - while (len-- > 0 && s < strend) { - STRLEN alen; - auint = NATIVE_TO_UNI(utf8n_to_uvchr((U8*)s, strend - s, &alen, ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANYUV)); - along = alen; - s += along; - if (checksum > bits_in_uv) - cdouble += (NV)auint; + if (explicit_length) { + /* Switch to "bytes in UTF-8" mode */ + if (symptr->flags & FLAG_DO_UTF8) utf8 = 0; else - cuv += auint; + /* Should be impossible due to the need_utf8() test */ + Perl_croak(aTHX_ "U0 mode on a byte string"); } + break; } - else { - if (len && unpack_only_one) - len = 1; + if (len > strend - s) len = strend - s; + if (!checksum) { + if (len && unpack_only_one) len = 1; EXTEND(SP, len); EXTEND_MORTAL(len); - while (len-- > 0 && s < strend) { - STRLEN alen; - auint = NATIVE_TO_UNI(utf8n_to_uvchr((U8*)s, strend - s, &alen, ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANYUV)); - along = alen; - s += along; - sv = NEWSV(37, 0); - sv_setuv(sv, (UV)auint); - PUSHs(sv_2mortal(sv)); - } + } + while (len-- > 0 && s < strend) { + STRLEN retlen; + UV auv; + if (utf8) { + U8 result[UTF8_MAXLEN]; + const char *ptr = s; + STRLEN len; + /* Bug: warns about bad utf8 even if we are short on bytes + and will break out of the loop */ + if (!uni_to_bytes(aTHX_ &ptr, strend, (char *) result, 1, + 'U')) + break; + len = UTF8SKIP(result); + if (!uni_to_bytes(aTHX_ &ptr, strend, + (char *) &result[1], len-1, 'U')) break; + auv = utf8n_to_uvuni(result, len, &retlen, ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANYUV); + s = ptr; + } else { + auv = utf8n_to_uvuni((U8*)s, strend - s, &retlen, ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANYUV); + if (retlen == (STRLEN) -1 || retlen == 0) + Perl_croak(aTHX_ "Malformed UTF-8 string in unpack"); + s += retlen; + } + if (!checksum) + PUSHs(sv_2mortal(newSVuv((UV) auv))); + else if (checksum > bits_in_uv) + cdouble += (NV) auv; + else + cuv += auv; } break; case 's' | TYPE_IS_SHRIEKING: #if SHORTSIZE != SIZE16 - along = (strend - s) / sizeof(short); - if (len > along) - len = along; - if (checksum) { - short ashort; - while (len-- > 0) { - COPYNN(s, &ashort, sizeof(short)); - s += sizeof(short); - if (checksum > bits_in_uv) - cdouble += (NV)ashort; - else - cuv += ashort; - - } - } - else { + while (len-- > 0) { short ashort; - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - COPYNN(s, &ashort, sizeof(short)); - s += sizeof(short); - sv = NEWSV(38, 0); - sv_setiv(sv, (IV)ashort); - PUSHs(sv_2mortal(sv)); - } + SHIFT_VAR(utf8, s, strend, ashort, datumtype); + DO_BO_UNPACK(ashort, s); + if (!checksum) + PUSHs(sv_2mortal(newSViv((IV)ashort))); + else if (checksum > bits_in_uv) + cdouble += (NV)ashort; + else + cuv += ashort; } break; #else /* Fallthrough! */ #endif case 's': - along = (strend - s) / SIZE16; - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - COPY16(s, &ashort); -#if SHORTSIZE > SIZE16 - if (ashort > 32767) - ashort -= 65536; -#endif - s += SIZE16; - if (checksum > bits_in_uv) - cdouble += (NV)ashort; - else - cuv += ashort; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); + while (len-- > 0) { + I16 ai16; - while (len-- > 0) { - COPY16(s, &ashort); -#if SHORTSIZE > SIZE16 - if (ashort > 32767) - ashort -= 65536; +#if U16SIZE > SIZE16 + ai16 = 0; #endif - s += SIZE16; - sv = NEWSV(38, 0); - sv_setiv(sv, (IV)ashort); - PUSHs(sv_2mortal(sv)); - } + SHIFT16(utf8, s, strend, &ai16, datumtype); + DO_BO_UNPACK(ai16, 16); +#if U16SIZE > SIZE16 + if (ai16 > 32767) + ai16 -= 65536; +#endif + if (!checksum) + PUSHs(sv_2mortal(newSViv((IV)ai16))); + else if (checksum > bits_in_uv) + cdouble += (NV)ai16; + else + cuv += ai16; } break; case 'S' | TYPE_IS_SHRIEKING: #if SHORTSIZE != SIZE16 - along = (strend - s) / sizeof(unsigned short); - if (len > along) - len = along; - if (checksum) { + while (len-- > 0) { unsigned short aushort; - while (len-- > 0) { - COPYNN(s, &aushort, sizeof(unsigned short)); - s += sizeof(unsigned short); - if (checksum > bits_in_uv) - cdouble += (NV)aushort; - else - cuv += aushort; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - unsigned short aushort; - COPYNN(s, &aushort, sizeof(unsigned short)); - s += sizeof(unsigned short); - sv = NEWSV(39, 0); - sv_setiv(sv, (UV)aushort); - PUSHs(sv_2mortal(sv)); - } + SHIFT_VAR(utf8, s, strend, aushort, datumtype); + DO_BO_UNPACK(aushort, s); + if (!checksum) + PUSHs(sv_2mortal(newSVuv((UV) aushort))); + else if (checksum > bits_in_uv) + cdouble += (NV)aushort; + else + cuv += aushort; } break; #else @@ -989,298 +1726,173 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c case 'v': case 'n': case 'S': - along = (strend - s) / SIZE16; - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - COPY16(s, &aushort); - s += SIZE16; -#ifdef HAS_NTOHS - if (datumtype == 'n') - aushort = PerlSock_ntohs(aushort); -#endif -#ifdef HAS_VTOHS - if (datumtype == 'v') - aushort = vtohs(aushort); + while (len-- > 0) { + U16 au16; +#if U16SIZE > SIZE16 + au16 = 0; #endif - if (checksum > bits_in_uv) - cdouble += (NV)aushort; - else - cuv += aushort; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - COPY16(s, &aushort); - s += SIZE16; - sv = NEWSV(39, 0); + SHIFT16(utf8, s, strend, &au16, datumtype); + DO_BO_UNPACK(au16, 16); #ifdef HAS_NTOHS - if (datumtype == 'n') - aushort = PerlSock_ntohs(aushort); + if (datumtype == 'n') + au16 = PerlSock_ntohs(au16); #endif #ifdef HAS_VTOHS - if (datumtype == 'v') - aushort = vtohs(aushort); + if (datumtype == 'v') + au16 = vtohs(au16); #endif - sv_setiv(sv, (UV)aushort); - PUSHs(sv_2mortal(sv)); - } + if (!checksum) + PUSHs(sv_2mortal(newSVuv((UV)au16))); + else if (checksum > bits_in_uv) + cdouble += (NV) au16; + else + cuv += au16; } break; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case 'v' | TYPE_IS_SHRIEKING: + case 'n' | TYPE_IS_SHRIEKING: + while (len-- > 0) { + I16 ai16; +# if U16SIZE > SIZE16 + ai16 = 0; +# endif + SHIFT16(utf8, s, strend, &ai16, datumtype); +# ifdef HAS_NTOHS + if (datumtype == ('n' | TYPE_IS_SHRIEKING)) + ai16 = (I16) PerlSock_ntohs((U16) ai16); +# endif /* HAS_NTOHS */ +# ifdef HAS_VTOHS + if (datumtype == ('v' | TYPE_IS_SHRIEKING)) + ai16 = (I16) vtohs((U16) ai16); +# endif /* HAS_VTOHS */ + if (!checksum) + PUSHs(sv_2mortal(newSViv((IV)ai16))); + else if (checksum > bits_in_uv) + cdouble += (NV) ai16; + else + cuv += ai16; + } + break; +#endif /* PERL_PACK_CAN_SHRIEKSIGN */ case 'i': case 'i' | TYPE_IS_SHRIEKING: - along = (strend - s) / sizeof(int); - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &aint, 1, int); - s += sizeof(int); - if (checksum > bits_in_uv) - cdouble += (NV)aint; - else - cuv += aint; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - Copy(s, &aint, 1, int); - s += sizeof(int); - sv = NEWSV(40, 0); -#ifdef __osf__ - /* Without the dummy below unpack("i", pack("i",-1)) - * return 0xFFffFFff instead of -1 for Digital Unix V4.0 - * cc with optimization turned on. - * - * The bug was detected in - * DEC C V5.8-009 on Digital UNIX V4.0 (Rev. 1091) (V4.0E) - * with optimization (-O4) turned on. - * DEC C V5.2-040 on Digital UNIX V4.0 (Rev. 564) (V4.0B) - * does not have this problem even with -O4. - * - * This bug was reported as DECC_BUGS 1431 - * and tracked internally as GEM_BUGS 7775. - * - * The bug is fixed in - * Tru64 UNIX V5.0: Compaq C V6.1-006 or later - * UNIX V4.0F support: DEC C V5.9-006 or later - * UNIX V4.0E support: DEC C V5.8-011 or later - * and also in DTK. - * - * See also few lines later for the same bug. - */ - (aint) ? - sv_setiv(sv, (IV)aint) : -#endif - sv_setiv(sv, (IV)aint); - PUSHs(sv_2mortal(sv)); - } + while (len-- > 0) { + int aint; + SHIFT_VAR(utf8, s, strend, aint, datumtype); + DO_BO_UNPACK(aint, i); + if (!checksum) + PUSHs(sv_2mortal(newSViv((IV)aint))); + else if (checksum > bits_in_uv) + cdouble += (NV)aint; + else + cuv += aint; } break; case 'I': case 'I' | TYPE_IS_SHRIEKING: - along = (strend - s) / sizeof(unsigned int); - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &auint, 1, unsigned int); - s += sizeof(unsigned int); - if (checksum > bits_in_uv) - cdouble += (NV)auint; - else - cuv += auint; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - Copy(s, &auint, 1, unsigned int); - s += sizeof(unsigned int); - sv = NEWSV(41, 0); -#ifdef __osf__ - /* Without the dummy below unpack("I", pack("I",0xFFFFFFFF)) - * returns 1.84467440737096e+19 instead of 0xFFFFFFFF. - * See details few lines earlier. */ - (auint) ? - sv_setuv(sv, (UV)auint) : -#endif - sv_setuv(sv, (UV)auint); - PUSHs(sv_2mortal(sv)); - } + while (len-- > 0) { + unsigned int auint; + SHIFT_VAR(utf8, s, strend, auint, datumtype); + DO_BO_UNPACK(auint, i); + if (!checksum) + PUSHs(sv_2mortal(newSVuv((UV)auint))); + else if (checksum > bits_in_uv) + cdouble += (NV)auint; + else + cuv += auint; } break; case 'j': - along = (strend - s) / IVSIZE; - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &aiv, 1, IV); - s += IVSIZE; - if (checksum > bits_in_uv) - cdouble += (NV)aiv; - else - cuv += aiv; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - Copy(s, &aiv, 1, IV); - s += IVSIZE; - sv = NEWSV(40, 0); - sv_setiv(sv, aiv); - PUSHs(sv_2mortal(sv)); - } + while (len-- > 0) { + IV aiv; + SHIFT_VAR(utf8, s, strend, aiv, datumtype); +#if IVSIZE == INTSIZE + DO_BO_UNPACK(aiv, i); +#elif IVSIZE == LONGSIZE + DO_BO_UNPACK(aiv, l); +#elif defined(HAS_QUAD) && IVSIZE == U64SIZE + DO_BO_UNPACK(aiv, 64); +#else + Perl_croak(aTHX_ "'j' not supported on this platform"); +#endif + if (!checksum) + PUSHs(sv_2mortal(newSViv(aiv))); + else if (checksum > bits_in_uv) + cdouble += (NV)aiv; + else + cuv += aiv; } break; case 'J': - along = (strend - s) / UVSIZE; - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &auv, 1, UV); - s += UVSIZE; - if (checksum > bits_in_uv) - cdouble += (NV)auv; - else - cuv += auv; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - Copy(s, &auv, 1, UV); - s += UVSIZE; - sv = NEWSV(41, 0); - sv_setuv(sv, auv); - PUSHs(sv_2mortal(sv)); - } + while (len-- > 0) { + UV auv; + SHIFT_VAR(utf8, s, strend, auv, datumtype); +#if IVSIZE == INTSIZE + DO_BO_UNPACK(auv, i); +#elif IVSIZE == LONGSIZE + DO_BO_UNPACK(auv, l); +#elif defined(HAS_QUAD) && IVSIZE == U64SIZE + DO_BO_UNPACK(auv, 64); +#else + Perl_croak(aTHX_ "'J' not supported on this platform"); +#endif + if (!checksum) + PUSHs(sv_2mortal(newSVuv(auv))); + else if (checksum > bits_in_uv) + cdouble += (NV)auv; + else + cuv += auv; } break; case 'l' | TYPE_IS_SHRIEKING: #if LONGSIZE != SIZE32 - along = (strend - s) / sizeof(long); - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - COPYNN(s, &along, sizeof(long)); - s += sizeof(long); - if (checksum > bits_in_uv) - cdouble += (NV)along; - else - cuv += along; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - COPYNN(s, &along, sizeof(long)); - s += sizeof(long); - sv = NEWSV(42, 0); - sv_setiv(sv, (IV)along); - PUSHs(sv_2mortal(sv)); - } + while (len-- > 0) { + long along; + SHIFT_VAR(utf8, s, strend, along, datumtype); + DO_BO_UNPACK(along, l); + if (!checksum) + PUSHs(sv_2mortal(newSViv((IV)along))); + else if (checksum > bits_in_uv) + cdouble += (NV)along; + else + cuv += along; } break; #else /* Fallthrough! */ #endif case 'l': - along = (strend - s) / SIZE32; - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { -#if LONGSIZE > SIZE32 && INTSIZE == SIZE32 - I32 along; -#endif - COPY32(s, &along); -#if LONGSIZE > SIZE32 - if (along > 2147483647) - along -= 4294967296; -#endif - s += SIZE32; - if (checksum > bits_in_uv) - cdouble += (NV)along; - else - cuv += along; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { -#if LONGSIZE > SIZE32 && INTSIZE == SIZE32 - I32 along; + while (len-- > 0) { + I32 ai32; +#if U32SIZE > SIZE32 + ai32 = 0; #endif - COPY32(s, &along); -#if LONGSIZE > SIZE32 - if (along > 2147483647) - along -= 4294967296; + SHIFT32(utf8, s, strend, &ai32, datumtype); + DO_BO_UNPACK(ai32, 32); +#if U32SIZE > SIZE32 + if (ai32 > 2147483647) ai32 -= 4294967296; #endif - s += SIZE32; - sv = NEWSV(42, 0); - sv_setiv(sv, (IV)along); - PUSHs(sv_2mortal(sv)); - } + if (!checksum) + PUSHs(sv_2mortal(newSViv((IV)ai32))); + else if (checksum > bits_in_uv) + cdouble += (NV)ai32; + else + cuv += ai32; } break; case 'L' | TYPE_IS_SHRIEKING: #if LONGSIZE != SIZE32 - along = (strend - s) / sizeof(unsigned long); - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - unsigned long aulong; - COPYNN(s, &aulong, sizeof(unsigned long)); - s += sizeof(unsigned long); - if (checksum > bits_in_uv) - cdouble += (NV)aulong; - else - cuv += aulong; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - unsigned long aulong; - COPYNN(s, &aulong, sizeof(unsigned long)); - s += sizeof(unsigned long); - sv = NEWSV(43, 0); - sv_setuv(sv, (UV)aulong); - PUSHs(sv_2mortal(sv)); - } + while (len-- > 0) { + unsigned long aulong; + SHIFT_VAR(utf8, s, strend, aulong, datumtype); + DO_BO_UNPACK(aulong, l); + if (!checksum) + PUSHs(sv_2mortal(newSVuv((UV)aulong))); + else if (checksum > bits_in_uv) + cdouble += (NV)aulong; + else + cuv += aulong; } break; #else @@ -1289,101 +1901,94 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c case 'V': case 'N': case 'L': - along = (strend - s) / SIZE32; - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - COPY32(s, &aulong); - s += SIZE32; -#ifdef HAS_NTOHL - if (datumtype == 'N') - aulong = PerlSock_ntohl(aulong); -#endif -#ifdef HAS_VTOHL - if (datumtype == 'V') - aulong = vtohl(aulong); + while (len-- > 0) { + U32 au32; +#if U32SIZE > SIZE32 + au32 = 0; #endif - if (checksum > bits_in_uv) - cdouble += (NV)aulong; - else - cuv += aulong; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - COPY32(s, &aulong); - s += SIZE32; + SHIFT32(utf8, s, strend, &au32, datumtype); + DO_BO_UNPACK(au32, 32); #ifdef HAS_NTOHL - if (datumtype == 'N') - aulong = PerlSock_ntohl(aulong); + if (datumtype == 'N') + au32 = PerlSock_ntohl(au32); #endif #ifdef HAS_VTOHL - if (datumtype == 'V') - aulong = vtohl(aulong); + if (datumtype == 'V') + au32 = vtohl(au32); #endif - sv = NEWSV(43, 0); - sv_setuv(sv, (UV)aulong); - PUSHs(sv_2mortal(sv)); - } + if (!checksum) + PUSHs(sv_2mortal(newSVuv((UV)au32))); + else if (checksum > bits_in_uv) + cdouble += (NV)au32; + else + cuv += au32; + } + break; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case 'V' | TYPE_IS_SHRIEKING: + case 'N' | TYPE_IS_SHRIEKING: + while (len-- > 0) { + I32 ai32; +# if U32SIZE > SIZE32 + ai32 = 0; +# endif + SHIFT32(utf8, s, strend, &ai32, datumtype); +# ifdef HAS_NTOHL + if (datumtype == ('N' | TYPE_IS_SHRIEKING)) + ai32 = (I32)PerlSock_ntohl((U32)ai32); +# endif +# ifdef HAS_VTOHL + if (datumtype == ('V' | TYPE_IS_SHRIEKING)) + ai32 = (I32)vtohl((U32)ai32); +# endif + if (!checksum) + PUSHs(sv_2mortal(newSViv((IV)ai32))); + else if (checksum > bits_in_uv) + cdouble += (NV)ai32; + else + cuv += ai32; } break; +#endif /* PERL_PACK_CAN_SHRIEKSIGN */ case 'p': - along = (strend - s) / sizeof(char*); - if (len > along) - len = along; - EXTEND(SP, len); - EXTEND_MORTAL(len); while (len-- > 0) { - if (sizeof(char*) > strend - s) - break; - else { - Copy(s, &aptr, 1, char*); - s += sizeof(char*); - } - sv = NEWSV(44, 0); - if (aptr) - sv_setpv(sv, aptr); - PUSHs(sv_2mortal(sv)); + const char *aptr; + SHIFT_VAR(utf8, s, strend, aptr, datumtype); + DO_BO_UNPACK_PC(aptr); + /* newSVpv generates undef if aptr is NULL */ + PUSHs(sv_2mortal(newSVpv(aptr, 0))); } break; case 'w': - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); { UV auv = 0; U32 bytes = 0; - - while ((len > 0) && (s < strend)) { - auv = (auv << 7) | (*s & 0x7f); + + while (len > 0 && s < strend) { + U8 ch; + ch = SHIFT_BYTE(utf8, s, strend, datumtype); + auv = (auv << 7) | (ch & 0x7f); /* UTF8_IS_XXXXX not right here - using constant 0x80 */ - if ((U8)(*s++) < 0x80) { + if (ch < 0x80) { bytes = 0; - sv = NEWSV(40, 0); - sv_setuv(sv, auv); - PUSHs(sv_2mortal(sv)); + PUSHs(sv_2mortal(newSVuv(auv))); len--; auv = 0; + continue; } - else if (++bytes >= sizeof(UV)) { /* promote to string */ - char *t; - STRLEN n_a; + if (++bytes >= sizeof(UV)) { /* promote to string */ + const char *t; sv = Perl_newSVpvf(aTHX_ "%.*"UVf, (int)TYPE_DIGITS(UV), auv); while (s < strend) { - sv = mul128(sv, (U8)(*s & 0x7f)); - if (!(*s++ & 0x80)) { + ch = SHIFT_BYTE(utf8, s, strend, datumtype); + sv = mul128(sv, (U8)(ch & 0x7f)); + if (!(ch & 0x80)) { bytes = 0; break; } } - t = SvPV(sv, n_a); + t = SvPV_nolen_const(sv); while (*t == '0') t++; sv_chop(sv, t); @@ -1400,189 +2005,88 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c if (symptr->howlen == e_star) Perl_croak(aTHX_ "'P' must have an explicit size in unpack"); EXTEND(SP, 1); - if (sizeof(char*) > strend - s) - break; - else { - Copy(s, &aptr, 1, char*); - s += sizeof(char*); + if (s + sizeof(char*) <= strend) { + char *aptr; + SHIFT_VAR(utf8, s, strend, aptr, datumtype); + DO_BO_UNPACK_PC(aptr); + /* newSVpvn generates undef if aptr is NULL */ + PUSHs(sv_2mortal(newSVpvn(aptr, len))); } - sv = NEWSV(44, 0); - if (aptr) - sv_setpvn(sv, aptr, len); - PUSHs(sv_2mortal(sv)); break; #ifdef HAS_QUAD case 'q': - along = (strend - s) / sizeof(Quad_t); - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &aquad, 1, Quad_t); - s += sizeof(Quad_t); - if (checksum > bits_in_uv) - cdouble += (NV)aquad; - else - cuv += aquad; - } + while (len-- > 0) { + Quad_t aquad; + SHIFT_VAR(utf8, s, strend, aquad, datumtype); + DO_BO_UNPACK(aquad, 64); + if (!checksum) + PUSHs(sv_2mortal(aquad >= IV_MIN && aquad <= IV_MAX ? + newSViv((IV)aquad) : newSVnv((NV)aquad))); + else if (checksum > bits_in_uv) + cdouble += (NV)aquad; + else + cuv += aquad; } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - if (s + sizeof(Quad_t) > strend) - aquad = 0; - else { - Copy(s, &aquad, 1, Quad_t); - s += sizeof(Quad_t); - } - sv = NEWSV(42, 0); - if (aquad >= IV_MIN && aquad <= IV_MAX) - sv_setiv(sv, (IV)aquad); - else - sv_setnv(sv, (NV)aquad); - PUSHs(sv_2mortal(sv)); - } - } break; case 'Q': - along = (strend - s) / sizeof(Uquad_t); - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &auquad, 1, Uquad_t); - s += sizeof(Uquad_t); - if (checksum > bits_in_uv) - cdouble += (NV)auquad; - else - cuv += auquad; - } + while (len-- > 0) { + Uquad_t auquad; + SHIFT_VAR(utf8, s, strend, auquad, datumtype); + DO_BO_UNPACK(auquad, 64); + if (!checksum) + PUSHs(sv_2mortal(auquad <= UV_MAX ? + newSVuv((UV)auquad):newSVnv((NV)auquad))); + else if (checksum > bits_in_uv) + cdouble += (NV)auquad; + else + cuv += auquad; } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - if (s + sizeof(Uquad_t) > strend) - auquad = 0; - else { - Copy(s, &auquad, 1, Uquad_t); - s += sizeof(Uquad_t); - } - sv = NEWSV(43, 0); - if (auquad <= UV_MAX) - sv_setuv(sv, (UV)auquad); - else - sv_setnv(sv, (NV)auquad); - PUSHs(sv_2mortal(sv)); - } - } break; -#endif +#endif /* HAS_QUAD */ /* float and double added gnb@melba.bby.oz.au 22/11/89 */ case 'f': - along = (strend - s) / sizeof(float); - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &afloat, 1, float); - s += sizeof(float); + while (len-- > 0) { + float afloat; + SHIFT_VAR(utf8, s, strend, afloat, datumtype); + DO_BO_UNPACK_N(afloat, float); + if (!checksum) + PUSHs(sv_2mortal(newSVnv((NV)afloat))); + else cdouble += afloat; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - Copy(s, &afloat, 1, float); - s += sizeof(float); - sv = NEWSV(47, 0); - sv_setnv(sv, (NV)afloat); - PUSHs(sv_2mortal(sv)); - } } break; case 'd': - along = (strend - s) / sizeof(double); - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &adouble, 1, double); - s += sizeof(double); + while (len-- > 0) { + double adouble; + SHIFT_VAR(utf8, s, strend, adouble, datumtype); + DO_BO_UNPACK_N(adouble, double); + if (!checksum) + PUSHs(sv_2mortal(newSVnv((NV)adouble))); + else cdouble += adouble; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - Copy(s, &adouble, 1, double); - s += sizeof(double); - sv = NEWSV(48, 0); - sv_setnv(sv, (NV)adouble); - PUSHs(sv_2mortal(sv)); - } } break; case 'F': - along = (strend - s) / NVSIZE; - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &anv, 1, NV); - s += NVSIZE; + while (len-- > 0) { + NV anv; + SHIFT_VAR(utf8, s, strend, anv, datumtype); + DO_BO_UNPACK_N(anv, NV); + if (!checksum) + PUSHs(sv_2mortal(newSVnv(anv))); + else cdouble += anv; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - Copy(s, &anv, 1, NV); - s += NVSIZE; - sv = NEWSV(48, 0); - sv_setnv(sv, anv); - PUSHs(sv_2mortal(sv)); - } } break; #if defined(HAS_LONG_DOUBLE) && defined(USE_LONG_DOUBLE) case 'D': - along = (strend - s) / LONG_DOUBLESIZE; - if (len > along) - len = along; - if (checksum) { - while (len-- > 0) { - Copy(s, &aldouble, 1, long double); - s += LONG_DOUBLESIZE; + while (len-- > 0) { + long double aldouble; + SHIFT_VAR(utf8, s, strend, aldouble, datumtype); + DO_BO_UNPACK_N(aldouble, long double); + if (!checksum) + PUSHs(sv_2mortal(newSVnv((NV)aldouble))); + else cdouble += aldouble; - } - } - else { - if (len && unpack_only_one) - len = 1; - EXTEND(SP, len); - EXTEND_MORTAL(len); - while (len-- > 0) { - Copy(s, &aldouble, 1, long double); - s += LONG_DOUBLESIZE; - sv = NEWSV(48, 0); - sv_setnv(sv, (NV)aldouble); - PUSHs(sv_2mortal(sv)); - } } break; #endif @@ -1593,9 +2097,9 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c * (and just as fast as doing character arithmetic) */ if (PL_uudmap['M'] == 0) { - int i; + size_t i; - for (i = 0; i < sizeof(PL_uuemap); i += 1) + for (i = 0; i < sizeof(PL_uuemap); ++i) PL_uudmap[(U8)PL_uuemap[i]] = i; /* * Because ' ' and '`' map to the same value, @@ -1603,78 +2107,108 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c */ PL_uudmap[' '] = 0; } - - along = (strend - s) * 3 / 4; - sv = NEWSV(42, along); - if (along) - SvPOK_on(sv); - while (s < strend && *s > ' ' && ISUUCHAR(*s)) { - I32 a, b, c, d; - char hunk[4]; - - hunk[3] = '\0'; - len = PL_uudmap[*(U8*)s++] & 077; - while (len > 0) { - if (s < strend && ISUUCHAR(*s)) - a = PL_uudmap[*(U8*)s++] & 077; - else - a = 0; - if (s < strend && ISUUCHAR(*s)) - b = PL_uudmap[*(U8*)s++] & 077; - else - b = 0; - if (s < strend && ISUUCHAR(*s)) - c = PL_uudmap[*(U8*)s++] & 077; - else - c = 0; - if (s < strend && ISUUCHAR(*s)) - d = PL_uudmap[*(U8*)s++] & 077; - else - d = 0; - hunk[0] = (char)((a << 2) | (b >> 4)); - hunk[1] = (char)((b << 4) | (c >> 2)); - hunk[2] = (char)((c << 6) | d); - sv_catpvn(sv, hunk, (len > 3) ? 3 : len); - len -= 3; + { + const STRLEN l = (STRLEN) (strend - s) * 3 / 4; + sv = sv_2mortal(newSV(l)); + if (l) SvPOK_on(sv); + } + if (utf8) { + while (next_uni_uu(aTHX_ &s, strend, &len)) { + I32 a, b, c, d; + char hunk[4]; + + hunk[3] = '\0'; + while (len > 0) { + next_uni_uu(aTHX_ &s, strend, &a); + next_uni_uu(aTHX_ &s, strend, &b); + next_uni_uu(aTHX_ &s, strend, &c); + next_uni_uu(aTHX_ &s, strend, &d); + hunk[0] = (char)((a << 2) | (b >> 4)); + hunk[1] = (char)((b << 4) | (c >> 2)); + hunk[2] = (char)((c << 6) | d); + sv_catpvn(sv, hunk, (len > 3) ? 3 : len); + len -= 3; + } + if (s < strend) { + if (*s == '\n') { + s++; + } + else { + /* possible checksum byte */ + const char *skip = s+UTF8SKIP(s); + if (skip < strend && *skip == '\n') + s = skip+1; + } + } + } + } else { + while (s < strend && *s > ' ' && ISUUCHAR(*s)) { + I32 a, b, c, d; + char hunk[4]; + + hunk[3] = '\0'; + len = PL_uudmap[*(U8*)s++] & 077; + while (len > 0) { + if (s < strend && ISUUCHAR(*s)) + a = PL_uudmap[*(U8*)s++] & 077; + else + a = 0; + if (s < strend && ISUUCHAR(*s)) + b = PL_uudmap[*(U8*)s++] & 077; + else + b = 0; + if (s < strend && ISUUCHAR(*s)) + c = PL_uudmap[*(U8*)s++] & 077; + else + c = 0; + if (s < strend && ISUUCHAR(*s)) + d = PL_uudmap[*(U8*)s++] & 077; + else + d = 0; + hunk[0] = (char)((a << 2) | (b >> 4)); + hunk[1] = (char)((b << 4) | (c >> 2)); + hunk[2] = (char)((c << 6) | d); + sv_catpvn(sv, hunk, (len > 3) ? 3 : len); + len -= 3; + } + if (*s == '\n') + s++; + else /* possible checksum byte */ + if (s + 1 < strend && s[1] == '\n') + s += 2; } - if (*s == '\n') - s++; - else /* possible checksum byte */ - if (s + 1 < strend && s[1] == '\n') - s += 2; } - XPUSHs(sv_2mortal(sv)); + XPUSHs(sv); break; } if (checksum) { - sv = NEWSV(42, 0); - if (strchr("fFdD", datumtype) || + if (strchr("fFdD", TYPE_NO_MODIFIERS(datumtype)) || (checksum > bits_in_uv && - strchr("csSiIlLnNUvVqQjJ", datumtype&0xFF)) ) { - NV trouble; + strchr("cCsSiIlLnNUWvVqQjJ", TYPE_NO_MODIFIERS(datumtype))) ) { + NV trouble, anv; - adouble = (NV) (1 << (checksum & 15)); + anv = (NV) (1 << (checksum & 15)); while (checksum >= 16) { checksum -= 16; - adouble *= 65536.0; + anv *= 65536.0; } while (cdouble < 0.0) - cdouble += adouble; - cdouble = Perl_modf(cdouble / adouble, &trouble) * adouble; - sv_setnv(sv, cdouble); + cdouble += anv; + cdouble = Perl_modf(cdouble / anv, &trouble) * anv; + sv = newSVnv(cdouble); } else { if (checksum < bits_in_uv) { UV mask = ((UV)1 << checksum) - 1; cuv &= mask; } - sv_setuv(sv, cuv); + sv = newSVuv(cuv); } XPUSHs(sv_2mortal(sv)); checksum = 0; } - + if (symptr->flags & FLAG_SLASH){ if (SP - PL_stack_base - start_sp_offset <= 0) Perl_croak(aTHX_ "'/' must follow a numeric type in unpack"); @@ -1694,6 +2228,7 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c Perl_croak(aTHX_ "Code missing after '/' in unpack" ); } datumtype = symptr->code; + explicit_length = FALSE; goto redo_switch; } } @@ -1706,28 +2241,22 @@ S_unpack_rec(pTHX_ register tempsym_t* symptr, register char *s, char *strbeg, c PP(pp_unpack) { + dVAR; dSP; dPOPPOPssrl; I32 gimme = GIMME_V; STRLEN llen; STRLEN rlen; - register char *pat = SvPV(left, llen); -#ifdef PACKED_IS_OCTETS - /* Packed side is assumed to be octets - so force downgrade if it - has been UTF-8 encoded by accident - */ - register char *s = SvPVbyte(right, rlen); -#else - register char *s = SvPV(right, rlen); -#endif - char *strend = s + rlen; - register char *patend = pat + llen; - register I32 cnt; + const char *pat = SvPV_const(left, llen); + const char *s = SvPV_const(right, rlen); + const char *strend = s + rlen; + const char *patend = pat + llen; + I32 cnt; PUTBACK; cnt = unpackstring(pat, patend, s, strend, ((gimme == G_SCALAR) ? FLAG_UNPACK_ONLY_ONE : 0) - | (DO_UTF8(right) ? FLAG_UNPACK_DO_UTF8 : 0)); + | (DO_UTF8(right) ? FLAG_DO_UTF8 : 0)); SPAGAIN; if ( !cnt && gimme == G_SCALAR ) @@ -1735,43 +2264,37 @@ PP(pp_unpack) RETURN; } -STATIC void -S_doencodes(pTHX_ register SV *sv, register char *s, register I32 len) +STATIC U8 * +doencodes(U8 *h, const char *s, I32 len) { - char hunk[5]; - - *hunk = PL_uuemap[len]; - sv_catpvn(sv, hunk, 1); - hunk[4] = '\0'; + *h++ = PL_uuemap[len]; while (len > 2) { - hunk[0] = PL_uuemap[(077 & (*s >> 2))]; - hunk[1] = PL_uuemap[(077 & (((*s << 4) & 060) | ((s[1] >> 4) & 017)))]; - hunk[2] = PL_uuemap[(077 & (((s[1] << 2) & 074) | ((s[2] >> 6) & 03)))]; - hunk[3] = PL_uuemap[(077 & (s[2] & 077))]; - sv_catpvn(sv, hunk, 4); + *h++ = PL_uuemap[(077 & (s[0] >> 2))]; + *h++ = PL_uuemap[(077 & (((s[0] << 4) & 060) | ((s[1] >> 4) & 017)))]; + *h++ = PL_uuemap[(077 & (((s[1] << 2) & 074) | ((s[2] >> 6) & 03)))]; + *h++ = PL_uuemap[(077 & (s[2] & 077))]; s += 3; len -= 3; } if (len > 0) { - char r = (len > 1 ? s[1] : '\0'); - hunk[0] = PL_uuemap[(077 & (*s >> 2))]; - hunk[1] = PL_uuemap[(077 & (((*s << 4) & 060) | ((r >> 4) & 017)))]; - hunk[2] = PL_uuemap[(077 & ((r << 2) & 074))]; - hunk[3] = PL_uuemap[0]; - sv_catpvn(sv, hunk, 4); + const char r = (len > 1 ? s[1] : '\0'); + *h++ = PL_uuemap[(077 & (s[0] >> 2))]; + *h++ = PL_uuemap[(077 & (((s[0] << 4) & 060) | ((r >> 4) & 017)))]; + *h++ = PL_uuemap[(077 & ((r << 2) & 074))]; + *h++ = PL_uuemap[0]; } - sv_catpvn(sv, "\n", 1); + *h++ = '\n'; + return h; } STATIC SV * -S_is_an_int(pTHX_ char *s, STRLEN l) +S_is_an_int(pTHX_ const char *s, STRLEN l) { - STRLEN n_a; - SV *result = newSVpvn(s, l); - char *result_c = SvPV(result, n_a); /* convenience */ - char *out = result_c; - bool skip = 1; - bool ignore = 0; + SV *result = newSVpvn(s, l); + char *const result_c = SvPV_nolen(result); /* convenience */ + char *out = result_c; + bool skip = 1; + bool ignore = 0; while (*s) { switch (*s) { @@ -1816,532 +2339,891 @@ S_is_an_int(pTHX_ char *s, STRLEN l) STATIC int S_div128(pTHX_ SV *pnum, bool *done) { - STRLEN len; - char *s = SvPV(pnum, len); - int m = 0; - int r = 0; - char *t = s; - - *done = 1; - while (*t) { - int i; - - i = m * 10 + (*t - '0'); - m = i & 0x7F; - r = (i >> 7); /* r < 10 */ - if (r) { - *done = 0; - } - *(t++) = '0' + r; - } - *(t++) = '\0'; - SvCUR_set(pnum, (STRLEN) (t - s)); - return (m); -} - - - -/* -=for apidoc pack_cat - -The engine implementing pack() Perl function. Note: parameters next_in_list and -flags are not used. This call should not be used; use packlist instead. - -=cut */ - - -void -Perl_pack_cat(pTHX_ SV *cat, char *pat, register char *patend, register SV **beglist, SV **endlist, SV ***next_in_list, U32 flags) -{ - tempsym_t sym = { 0 }; - sym.patptr = pat; - sym.patend = patend; - sym.flags = FLAG_PACK; - - (void)pack_rec( cat, &sym, beglist, endlist ); + STRLEN len; + char * const s = SvPV(pnum, len); + char *t = s; + int m = 0; + + *done = 1; + while (*t) { + const int i = m * 10 + (*t - '0'); + const int r = (i >> 7); /* r < 10 */ + m = i & 0x7F; + if (r) { + *done = 0; + } + *(t++) = '0' + r; + } + *(t++) = '\0'; + SvCUR_set(pnum, (STRLEN) (t - s)); + return (m); } - /* =for apidoc packlist The engine implementing pack() Perl function. -=cut */ - +=cut +*/ void -Perl_packlist(pTHX_ SV *cat, char *pat, register char *patend, register SV **beglist, SV **endlist ) +Perl_packlist(pTHX_ SV *cat, const char *pat, const char *patend, register SV **beglist, SV **endlist ) { - tempsym_t sym = { 0 }; - sym.patptr = pat; - sym.patend = patend; - sym.flags = FLAG_PACK; + dVAR; + tempsym_t sym; + + TEMPSYM_INIT(&sym, pat, patend, FLAG_PACK); + + /* We're going to do changes through SvPVX(cat). Make sure it's valid. + Also make sure any UTF8 flag is loaded */ + SvPV_force_nolen(cat); + if (DO_UTF8(cat)) + sym.flags |= FLAG_PARSE_UTF8 | FLAG_DO_UTF8; (void)pack_rec( cat, &sym, beglist, endlist ); } +/* like sv_utf8_upgrade, but also repoint the group start markers */ +STATIC void +marked_upgrade(pTHX_ SV *sv, tempsym_t *sym_ptr) { + STRLEN len; + tempsym_t *group; + const char *from_ptr, *from_start, *from_end, **marks, **m; + char *to_start, *to_ptr; + + if (SvUTF8(sv)) return; + + from_start = SvPVX_const(sv); + from_end = from_start + SvCUR(sv); + for (from_ptr = from_start; from_ptr < from_end; from_ptr++) + if (!NATIVE_IS_INVARIANT(*from_ptr)) break; + if (from_ptr == from_end) { + /* Simple case: no character needs to be changed */ + SvUTF8_on(sv); + return; + } + + len = (from_end-from_ptr)*UTF8_EXPAND+(from_ptr-from_start)+1; + Newx(to_start, len, char); + Copy(from_start, to_start, from_ptr-from_start, char); + to_ptr = to_start + (from_ptr-from_start); + + Newx(marks, sym_ptr->level+2, const char *); + for (group=sym_ptr; group; group = group->previous) + marks[group->level] = from_start + group->strbeg; + marks[sym_ptr->level+1] = from_end+1; + for (m = marks; *m < from_ptr; m++) + *m = to_start + (*m-from_start); + + for (;from_ptr < from_end; from_ptr++) { + while (*m == from_ptr) *m++ = to_ptr; + to_ptr = (char *) uvchr_to_utf8((U8 *) to_ptr, *(U8 *) from_ptr); + } + *to_ptr = 0; + + while (*m == from_ptr) *m++ = to_ptr; + if (m != marks + sym_ptr->level+1) { + Safefree(marks); + Safefree(to_start); + Perl_croak(aTHX_ "Assertion: marks beyond string end"); + } + for (group=sym_ptr; group; group = group->previous) + group->strbeg = marks[group->level] - to_start; + Safefree(marks); + + if (SvOOK(sv)) { + if (SvIVX(sv)) { + SvLEN_set(sv, SvLEN(sv) + SvIVX(sv)); + from_start -= SvIVX(sv); + SvIV_set(sv, 0); + } + SvFLAGS(sv) &= ~SVf_OOK; + } + if (SvLEN(sv) != 0) + Safefree(from_start); + SvPV_set(sv, to_start); + SvCUR_set(sv, to_ptr - to_start); + SvLEN_set(sv, len); + SvUTF8_on(sv); +} + +/* Exponential string grower. Makes string extension effectively O(n) + needed says how many extra bytes we need (not counting the final '\0') + Only grows the string if there is an actual lack of space +*/ +STATIC char * +S_sv_exp_grow(pTHX_ SV *sv, STRLEN needed) { + const STRLEN cur = SvCUR(sv); + const STRLEN len = SvLEN(sv); + STRLEN extend; + if (len - cur > needed) return SvPVX(sv); + extend = needed > len ? needed : len; + return SvGROW(sv, len+extend+1); +} STATIC SV ** -S_pack_rec(pTHX_ SV *cat, register tempsym_t* symptr, register SV **beglist, SV **endlist ) +S_pack_rec(pTHX_ SV *cat, tempsym_t* symptr, SV **beglist, SV **endlist ) { - register I32 items; - STRLEN fromlen; - register I32 len = 0; - SV *fromstr; - /*SUPPRESS 442*/ - static char null10[] = {0,0,0,0,0,0,0,0,0,0}; - static char *space10 = " "; - bool found; - - /* These must not be in registers: */ - char achar; - I16 ashort; - int aint; - unsigned int auint; - I32 along; - U32 aulong; - IV aiv; - UV auv; - NV anv; -#if defined(HAS_LONG_DOUBLE) && defined(USE_LONG_DOUBLE) - long double aldouble; -#endif -#ifdef HAS_QUAD - Quad_t aquad; - Uquad_t auquad; -#endif - char *aptr; - float afloat; - double adouble; - int strrelbeg = SvCUR(cat); + dVAR; tempsym_t lookahead; - - items = endlist - beglist; - found = next_symbol( symptr ); - -#ifndef PACKED_IS_OCTETS - if (symptr->level == 0 && found && symptr->code == 'U' ){ - SvUTF8_on(cat); + I32 items = endlist - beglist; + bool found = next_symbol(symptr); + bool utf8 = (symptr->flags & FLAG_PARSE_UTF8) ? 1 : 0; + bool warn_utf8 = ckWARN(WARN_UTF8); + + if (symptr->level == 0 && found && symptr->code == 'U') { + marked_upgrade(aTHX_ cat, symptr); + symptr->flags |= FLAG_DO_UTF8; + utf8 = 0; } -#endif + symptr->strbeg = SvCUR(cat); while (found) { - SV *lengthcode = Nullsv; -#define NEXTFROM ( lengthcode ? lengthcode : items-- > 0 ? *beglist++ : &PL_sv_no) - + SV *fromstr; + STRLEN fromlen; + I32 len; + SV *lengthcode = NULL; I32 datumtype = symptr->code; - howlen_t howlen; + howlen_t howlen = symptr->howlen; + char *start = SvPVX(cat); + char *cur = start + SvCUR(cat); - switch( howlen = symptr->howlen ){ - case e_no_len: - case e_number: - len = symptr->length; +#define NEXTFROM (lengthcode ? lengthcode : items-- > 0 ? *beglist++ : &PL_sv_no) + + switch (howlen) { + case e_star: + len = strchr("@Xxu", TYPE_NO_MODIFIERS(datumtype)) ? + 0 : items; break; - case e_star: - len = strchr("@Xxu", datumtype) ? 0 : items; + default: + /* e_no_len and e_number */ + len = symptr->length; break; } + if (len) { + packprops_t props = packprops[TYPE_NO_ENDIANNESS(datumtype)]; + + if (props && !(props & PACK_SIZE_UNPREDICTABLE)) { + /* We can process this letter. */ + STRLEN size = props & PACK_SIZE_MASK; + GROWING(utf8, cat, start, cur, (STRLEN) len * size); + } + } + /* Look ahead for next symbol. Do we have code/code? */ lookahead = *symptr; found = next_symbol(&lookahead); - if ( symptr->flags & FLAG_SLASH ) { - if (found){ - if ( 0 == strchr( "aAZ", lookahead.code ) || - e_star != lookahead.howlen ) - Perl_croak(aTHX_ "'/' must be followed by 'a*', 'A*' or 'Z*' in pack"); - lengthcode = sv_2mortal(newSViv(sv_len(items > 0 - ? *beglist : &PL_sv_no) - + (lookahead.code == 'Z' ? 1 : 0))); + if (symptr->flags & FLAG_SLASH) { + IV count; + if (!found) Perl_croak(aTHX_ "Code missing after '/' in pack"); + if (strchr("aAZ", lookahead.code)) { + if (lookahead.howlen == e_number) count = lookahead.length; + else { + if (items > 0) { + if (SvGAMAGIC(*beglist)) { + /* Avoid reading the active data more than once + by copying it to a temporary. */ + STRLEN len; + const char *const pv = SvPV_const(*beglist, len); + SV *const temp = sv_2mortal(newSVpvn(pv, len)); + if (SvUTF8(*beglist)) + SvUTF8_on(temp); + *beglist = temp; + } + count = DO_UTF8(*beglist) ? + sv_len_utf8(*beglist) : sv_len(*beglist); + } + else count = 0; + if (lookahead.code == 'Z') count++; + } } else { - Perl_croak(aTHX_ "Code missing after '/' in pack"); - } + if (lookahead.howlen == e_number && lookahead.length < items) + count = lookahead.length; + else count = items; + } + lookahead.howlen = e_number; + lookahead.length = count; + lengthcode = sv_2mortal(newSViv(count)); } - switch(datumtype) { + /* Code inside the switch must take care to properly update + cat (CUR length and '\0' termination) if it updated *cur and + doesn't simply leave using break */ + switch(TYPE_NO_ENDIANNESS(datumtype)) { default: - Perl_croak(aTHX_ "Invalid type '%c' in pack", (int)datumtype); + Perl_croak(aTHX_ "Invalid type '%c' in pack", + (int) TYPE_NO_MODIFIERS(datumtype)); case '%': Perl_croak(aTHX_ "'%%' may not be used in pack"); + { + char *from; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case '.' | TYPE_IS_SHRIEKING: +#endif + case '.': + if (howlen == e_star) from = start; + else if (len == 0) from = cur; + else { + tempsym_t *group = symptr; + + while (--len && group) group = group->previous; + from = group ? start + group->strbeg : start; + } + fromstr = NEXTFROM; + len = SvIV(fromstr); + goto resize; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case '@' | TYPE_IS_SHRIEKING: +#endif case '@': - len += strrelbeg - SvCUR(cat); - if (len > 0) - goto grow; - len = -len; - if (len > 0) + from = start + symptr->strbeg; + resize: +#ifdef PERL_PACK_CAN_SHRIEKSIGN + if (utf8 && !(datumtype & TYPE_IS_SHRIEKING)) +#else /* PERL_PACK_CAN_SHRIEKSIGN */ + if (utf8) +#endif + if (len >= 0) { + while (len && from < cur) { + from += UTF8SKIP(from); + len--; + } + if (from > cur) + Perl_croak(aTHX_ "Malformed UTF-8 string in pack"); + if (len) { + /* Here we know from == cur */ + grow: + GROWING(0, cat, start, cur, len); + Zero(cur, len, char); + cur += len; + } else if (from < cur) { + len = cur - from; + goto shrink; + } else goto no_change; + } else { + cur = from; + len = -len; + goto utf8_shrink; + } + else { + len -= cur - from; + if (len > 0) goto grow; + if (len == 0) goto no_change; + len = -len; goto shrink; + } break; - case '(': - { + } + case '(': { tempsym_t savsym = *symptr; + U32 group_modifiers = TYPE_MODIFIERS(datumtype & ~symptr->flags); + symptr->flags |= group_modifiers; symptr->patend = savsym.grpend; symptr->level++; + symptr->previous = &lookahead; while (len--) { + U32 was_utf8; + if (utf8) symptr->flags |= FLAG_PARSE_UTF8; + else symptr->flags &= ~FLAG_PARSE_UTF8; + was_utf8 = SvUTF8(cat); symptr->patptr = savsym.grpbeg; - beglist = pack_rec(cat, symptr, beglist, endlist ); + beglist = pack_rec(cat, symptr, beglist, endlist); + if (SvUTF8(cat) != was_utf8) + /* This had better be an upgrade while in utf8==0 mode */ + utf8 = 1; + if (savsym.howlen == e_star && beglist == endlist) break; /* No way to continue */ } - lookahead.flags = symptr->flags; - *symptr = savsym; - break; + lookahead.flags = symptr->flags & ~group_modifiers; + goto no_change; } case 'X' | TYPE_IS_SHRIEKING: if (!len) /* Avoid division by 0 */ len = 1; - len = (SvCUR(cat)) % len; + if (utf8) { + char *hop, *last; + I32 l = len; + hop = last = start; + while (hop < cur) { + hop += UTF8SKIP(hop); + if (--l == 0) { + last = hop; + l = len; + } + } + if (last > cur) + Perl_croak(aTHX_ "Malformed UTF-8 string in pack"); + cur = last; + break; + } + len = (cur-start) % len; /* FALL THROUGH */ case 'X': - shrink: - if ((I32)SvCUR(cat) < len) - Perl_croak(aTHX_ "'X' outside of string in pack"); - SvCUR(cat) -= len; - *SvEND(cat) = '\0'; - break; - case 'x' | TYPE_IS_SHRIEKING: + if (utf8) { + if (len < 1) goto no_change; + utf8_shrink: + while (len > 0) { + if (cur <= start) + Perl_croak(aTHX_ "'%c' outside of string in pack", + (int) TYPE_NO_MODIFIERS(datumtype)); + while (--cur, UTF8_IS_CONTINUATION(*cur)) { + if (cur <= start) + Perl_croak(aTHX_ "'%c' outside of string in pack", + (int) TYPE_NO_MODIFIERS(datumtype)); + } + len--; + } + } else { + shrink: + if (cur - start < len) + Perl_croak(aTHX_ "'%c' outside of string in pack", + (int) TYPE_NO_MODIFIERS(datumtype)); + cur -= len; + } + if (cur < start+symptr->strbeg) { + /* Make sure group starts don't point into the void */ + tempsym_t *group; + const STRLEN length = cur-start; + for (group = symptr; + group && length < group->strbeg; + group = group->previous) group->strbeg = length; + lookahead.strbeg = length; + } + break; + case 'x' | TYPE_IS_SHRIEKING: { + I32 ai32; if (!len) /* Avoid division by 0 */ len = 1; - aint = (SvCUR(cat)) % len; - if (aint) /* Other portable ways? */ - len = len - aint; - else - len = 0; - /* FALL THROUGH */ - + if (utf8) ai32 = utf8_length((U8 *) start, (U8 *) cur) % len; + else ai32 = (cur - start) % len; + if (ai32 == 0) goto no_change; + len -= ai32; + } + /* FALL THROUGH */ case 'x': - grow: - while (len >= 10) { - sv_catpvn(cat, null10, 10); - len -= 10; - } - sv_catpvn(cat, null10, len); - break; + goto grow; case 'A': case 'Z': - case 'a': + case 'a': { + const char *aptr; + fromstr = NEXTFROM; - aptr = SvPV(fromstr, fromlen); - if (howlen == e_star) { - len = fromlen; - if (datumtype == 'Z') - ++len; - } - if ((I32)fromlen >= len) { - sv_catpvn(cat, aptr, len); - if (datumtype == 'Z') - *(SvEND(cat)-1) = '\0'; - } - else { - sv_catpvn(cat, aptr, fromlen); + aptr = SvPV_const(fromstr, fromlen); + if (DO_UTF8(fromstr)) { + const char *end, *s; + + if (!utf8 && !SvUTF8(cat)) { + marked_upgrade(aTHX_ cat, symptr); + lookahead.flags |= FLAG_DO_UTF8; + lookahead.strbeg = symptr->strbeg; + utf8 = 1; + start = SvPVX(cat); + cur = start + SvCUR(cat); + } + if (howlen == e_star) { + if (utf8) goto string_copy; + len = fromlen+1; + } + s = aptr; + end = aptr + fromlen; + fromlen = datumtype == 'Z' ? len-1 : len; + while ((I32) fromlen > 0 && s < end) { + s += UTF8SKIP(s); + fromlen--; + } + if (s > end) + Perl_croak(aTHX_ "Malformed UTF-8 string in pack"); + if (utf8) { + len = fromlen; + if (datumtype == 'Z') len++; + fromlen = s-aptr; + len += fromlen; + + goto string_copy; + } + fromlen = len - fromlen; + if (datumtype == 'Z') fromlen--; + if (howlen == e_star) { + len = fromlen; + if (datumtype == 'Z') len++; + } + GROWING(0, cat, start, cur, len); + if (!uni_to_bytes(aTHX_ &aptr, end, cur, fromlen, + datumtype | TYPE_IS_PACK)) + Perl_croak(aTHX_ "Perl bug: predicted utf8 length not available"); + cur += fromlen; len -= fromlen; - if (datumtype == 'A') { - while (len >= 10) { - sv_catpvn(cat, space10, 10); - len -= 10; - } - sv_catpvn(cat, space10, len); + } else if (utf8) { + if (howlen == e_star) { + len = fromlen; + if (datumtype == 'Z') len++; } - else { - while (len >= 10) { - sv_catpvn(cat, null10, 10); - len -= 10; - } - sv_catpvn(cat, null10, len); + if (len <= (I32) fromlen) { + fromlen = len; + if (datumtype == 'Z' && fromlen > 0) fromlen--; } + /* assumes a byte expands to at most UTF8_EXPAND bytes on + upgrade, so: + expected_length <= from_len*UTF8_EXPAND + (len-from_len) */ + GROWING(0, cat, start, cur, fromlen*(UTF8_EXPAND-1)+len); + len -= fromlen; + while (fromlen > 0) { + cur = (char *) uvchr_to_utf8((U8 *) cur, * (U8 *) aptr); + aptr++; + fromlen--; + } + } else { + string_copy: + if (howlen == e_star) { + len = fromlen; + if (datumtype == 'Z') len++; + } + if (len <= (I32) fromlen) { + fromlen = len; + if (datumtype == 'Z' && fromlen > 0) fromlen--; + } + GROWING(0, cat, start, cur, len); + Copy(aptr, cur, fromlen, char); + cur += fromlen; + len -= fromlen; } + memset(cur, datumtype == 'A' ? ' ' : '\0', len); + cur += len; break; + } case 'B': - case 'b': - { - register char *str; - I32 saveitems; + case 'b': { + const char *str, *end; + I32 l, field_len; + U8 bits; + bool utf8_source; + U32 utf8_flags; - fromstr = NEXTFROM; - saveitems = items; - str = SvPV(fromstr, fromlen); - if (howlen == e_star) - len = fromlen; - aint = SvCUR(cat); - SvCUR(cat) += (len+7)/8; - SvGROW(cat, SvCUR(cat) + 1); - aptr = SvPVX(cat) + aint; - if (len > (I32)fromlen) - len = fromlen; - aint = len; - items = 0; - if (datumtype == 'B') { - for (len = 0; len++ < aint;) { - items |= *str++ & 1; - if (len & 7) - items <<= 1; - else { - *aptr++ = items & 0xff; - items = 0; - } + fromstr = NEXTFROM; + str = SvPV_const(fromstr, fromlen); + end = str + fromlen; + if (DO_UTF8(fromstr)) { + utf8_source = TRUE; + utf8_flags = warn_utf8 ? 0 : UTF8_ALLOW_ANY; + } else { + utf8_source = FALSE; + utf8_flags = 0; /* Unused, but keep compilers happy */ + } + if (howlen == e_star) len = fromlen; + field_len = (len+7)/8; + GROWING(utf8, cat, start, cur, field_len); + if (len > (I32)fromlen) len = fromlen; + bits = 0; + l = 0; + if (datumtype == 'B') + while (l++ < len) { + if (utf8_source) { + UV val = 0; + NEXT_UNI_VAL(val, cur, str, end, utf8_flags); + bits |= val & 1; + } else bits |= *str++ & 1; + if (l & 7) bits <<= 1; + else { + PUSH_BYTE(utf8, cur, bits); + bits = 0; } } - else { - for (len = 0; len++ < aint;) { - if (*str++ & 1) - items |= 128; - if (len & 7) - items >>= 1; - else { - *aptr++ = items & 0xff; - items = 0; - } + else + /* datumtype == 'b' */ + while (l++ < len) { + if (utf8_source) { + UV val = 0; + NEXT_UNI_VAL(val, cur, str, end, utf8_flags); + if (val & 1) bits |= 0x80; + } else if (*str++ & 1) + bits |= 0x80; + if (l & 7) bits >>= 1; + else { + PUSH_BYTE(utf8, cur, bits); + bits = 0; } } - if (aint & 7) { - if (datumtype == 'B') - items <<= 7 - (aint & 7); - else - items >>= 7 - (aint & 7); - *aptr++ = items & 0xff; - } - str = SvPVX(cat) + SvCUR(cat); - while (aptr <= str) - *aptr++ = '\0'; - - items = saveitems; + l--; + if (l & 7) { + if (datumtype == 'B') + bits <<= 7 - (l & 7); + else + bits >>= 7 - (l & 7); + PUSH_BYTE(utf8, cur, bits); + l += 7; } + /* Determine how many chars are left in the requested field */ + l /= 8; + if (howlen == e_star) field_len = 0; + else field_len -= l; + Zero(cur, field_len, char); + cur += field_len; break; + } case 'H': - case 'h': - { - register char *str; - I32 saveitems; + case 'h': { + const char *str, *end; + I32 l, field_len; + U8 bits; + bool utf8_source; + U32 utf8_flags; - fromstr = NEXTFROM; - saveitems = items; - str = SvPV(fromstr, fromlen); - if (howlen == e_star) - len = fromlen; - aint = SvCUR(cat); - SvCUR(cat) += (len+1)/2; - SvGROW(cat, SvCUR(cat) + 1); - aptr = SvPVX(cat) + aint; - if (len > (I32)fromlen) - len = fromlen; - aint = len; - items = 0; - if (datumtype == 'H') { - for (len = 0; len++ < aint;) { - if (isALPHA(*str)) - items |= ((*str++ & 15) + 9) & 15; + fromstr = NEXTFROM; + str = SvPV_const(fromstr, fromlen); + end = str + fromlen; + if (DO_UTF8(fromstr)) { + utf8_source = TRUE; + utf8_flags = warn_utf8 ? 0 : UTF8_ALLOW_ANY; + } else { + utf8_source = FALSE; + utf8_flags = 0; /* Unused, but keep compilers happy */ + } + if (howlen == e_star) len = fromlen; + field_len = (len+1)/2; + GROWING(utf8, cat, start, cur, field_len); + if (!utf8 && len > (I32)fromlen) len = fromlen; + bits = 0; + l = 0; + if (datumtype == 'H') + while (l++ < len) { + if (utf8_source) { + UV val = 0; + NEXT_UNI_VAL(val, cur, str, end, utf8_flags); + if (val < 256 && isALPHA(val)) + bits |= (val + 9) & 0xf; else - items |= *str++ & 15; - if (len & 1) - items <<= 4; - else { - *aptr++ = items & 0xff; - items = 0; - } + bits |= val & 0xf; + } else if (isALPHA(*str)) + bits |= (*str++ + 9) & 0xf; + else + bits |= *str++ & 0xf; + if (l & 1) bits <<= 4; + else { + PUSH_BYTE(utf8, cur, bits); + bits = 0; } } - else { - for (len = 0; len++ < aint;) { - if (isALPHA(*str)) - items |= (((*str++ & 15) + 9) & 15) << 4; + else + while (l++ < len) { + if (utf8_source) { + UV val = 0; + NEXT_UNI_VAL(val, cur, str, end, utf8_flags); + if (val < 256 && isALPHA(val)) + bits |= ((val + 9) & 0xf) << 4; else - items |= (*str++ & 15) << 4; - if (len & 1) - items >>= 4; - else { - *aptr++ = items & 0xff; - items = 0; - } + bits |= (val & 0xf) << 4; + } else if (isALPHA(*str)) + bits |= ((*str++ + 9) & 0xf) << 4; + else + bits |= (*str++ & 0xf) << 4; + if (l & 1) bits >>= 4; + else { + PUSH_BYTE(utf8, cur, bits); + bits = 0; } } - if (aint & 1) - *aptr++ = items & 0xff; - str = SvPVX(cat) + SvCUR(cat); - while (aptr <= str) - *aptr++ = '\0'; - - items = saveitems; + l--; + if (l & 1) { + PUSH_BYTE(utf8, cur, bits); + l++; } + /* Determine how many chars are left in the requested field */ + l /= 2; + if (howlen == e_star) field_len = 0; + else field_len -= l; + Zero(cur, field_len, char); + cur += field_len; break; - case 'C': + } case 'c': while (len-- > 0) { + IV aiv; + fromstr = NEXTFROM; + aiv = SvIV(fromstr); + if ((-128 > aiv || aiv > 127) && + ckWARN(WARN_PACK)) + Perl_warner(aTHX_ packWARN(WARN_PACK), + "Character in 'c' format wrapped in pack"); + PUSH_BYTE(utf8, cur, (U8)(aiv & 0xff)); + } + break; + case 'C': + if (len == 0) { + utf8 = (symptr->flags & FLAG_DO_UTF8) ? 1 : 0; + break; + } + GROWING(0, cat, start, cur, len); + while (len-- > 0) { + IV aiv; + fromstr = NEXTFROM; + aiv = SvIV(fromstr); + if ((0 > aiv || aiv > 0xff) && + ckWARN(WARN_PACK)) + Perl_warner(aTHX_ packWARN(WARN_PACK), + "Character in 'C' format wrapped in pack"); + *cur++ = (char)(aiv & 0xff); + } + break; + case 'W': { + char *end; + U8 in_bytes = IN_BYTES; + + end = start+SvLEN(cat)-1; + if (utf8) end -= UTF8_MAXLEN-1; + while (len-- > 0) { + UV auv; fromstr = NEXTFROM; - switch (datumtype) { - case 'C': - aint = SvIV(fromstr); - if ((aint < 0 || aint > 255) && - ckWARN(WARN_PACK)) - Perl_warner(aTHX_ packWARN(WARN_PACK), - "Character in 'C' format wrapped in pack"); - achar = aint & 255; - sv_catpvn(cat, &achar, sizeof(char)); - break; - case 'c': - aint = SvIV(fromstr); - if ((aint < -128 || aint > 127) && - ckWARN(WARN_PACK)) - Perl_warner(aTHX_ packWARN(WARN_PACK), - "Character in 'c' format wrapped in pack" ); - achar = aint & 255; - sv_catpvn(cat, &achar, sizeof(char)); - break; + auv = SvUV(fromstr); + if (in_bytes) auv = auv % 0x100; + if (utf8) { + W_utf8: + if (cur > end) { + *cur = '\0'; + SvCUR_set(cat, cur - start); + + GROWING(0, cat, start, cur, len+UTF8_MAXLEN); + end = start+SvLEN(cat)-UTF8_MAXLEN; + } + cur = (char *) uvuni_to_utf8_flags((U8 *) cur, + NATIVE_TO_UNI(auv), + warn_utf8 ? + 0 : UNICODE_ALLOW_ANY); + } else { + if (auv >= 0x100) { + if (!SvUTF8(cat)) { + *cur = '\0'; + SvCUR_set(cat, cur - start); + marked_upgrade(aTHX_ cat, symptr); + lookahead.flags |= FLAG_DO_UTF8; + lookahead.strbeg = symptr->strbeg; + utf8 = 1; + start = SvPVX(cat); + cur = start + SvCUR(cat); + end = start+SvLEN(cat)-UTF8_MAXLEN; + goto W_utf8; + } + if (ckWARN(WARN_PACK)) + Perl_warner(aTHX_ packWARN(WARN_PACK), + "Character in 'W' format wrapped in pack"); + auv &= 0xff; + } + if (cur >= end) { + *cur = '\0'; + SvCUR_set(cat, cur - start); + GROWING(0, cat, start, cur, len+1); + end = start+SvLEN(cat)-1; + } + *(U8 *) cur++ = (U8)auv; } } break; - case 'U': + } + case 'U': { + char *end; + + if (len == 0) { + if (!(symptr->flags & FLAG_DO_UTF8)) { + marked_upgrade(aTHX_ cat, symptr); + lookahead.flags |= FLAG_DO_UTF8; + lookahead.strbeg = symptr->strbeg; + } + utf8 = 0; + goto no_change; + } + + end = start+SvLEN(cat); + if (!utf8) end -= UTF8_MAXLEN; while (len-- > 0) { + UV auv; fromstr = NEXTFROM; - auint = UNI_TO_NATIVE(SvUV(fromstr)); - SvGROW(cat, SvCUR(cat) + UTF8_MAXLEN + 1); - SvCUR_set(cat, - (char*)uvchr_to_utf8_flags((U8*)SvEND(cat), - auint, - ckWARN(WARN_UTF8) ? - 0 : UNICODE_ALLOW_ANY) - - SvPVX(cat)); + auv = SvUV(fromstr); + if (utf8) { + U8 buffer[UTF8_MAXLEN], *endb; + endb = uvuni_to_utf8_flags(buffer, auv, + warn_utf8 ? + 0 : UNICODE_ALLOW_ANY); + if (cur+(endb-buffer)*UTF8_EXPAND >= end) { + *cur = '\0'; + SvCUR_set(cat, cur - start); + GROWING(0, cat, start, cur, + len+(endb-buffer)*UTF8_EXPAND); + end = start+SvLEN(cat); + } + cur = bytes_to_uni(buffer, endb-buffer, cur); + } else { + if (cur >= end) { + *cur = '\0'; + SvCUR_set(cat, cur - start); + GROWING(0, cat, start, cur, len+UTF8_MAXLEN); + end = start+SvLEN(cat)-UTF8_MAXLEN; + } + cur = (char *) uvuni_to_utf8_flags((U8 *) cur, auv, + warn_utf8 ? + 0 : UNICODE_ALLOW_ANY); + } } - *SvEND(cat) = '\0'; break; + } /* Float and double added by gnb@melba.bby.oz.au 22/11/89 */ case 'f': while (len-- > 0) { + float afloat; + NV anv; fromstr = NEXTFROM; + anv = SvNV(fromstr); #ifdef __VOS__ -/* VOS does not automatically map a floating-point overflow - during conversion from double to float into infinity, so we - do it by hand. This code should either be generalized for - any OS that needs it, or removed if and when VOS implements - posix-976 (suggestion to support mapping to infinity). - Paul.Green@stratus.com 02-04-02. */ - if (SvNV(fromstr) > FLT_MAX) - afloat = _float_constants[0]; /* single prec. inf. */ - else if (SvNV(fromstr) < -FLT_MAX) - afloat = _float_constants[0]; /* single prec. inf. */ - else afloat = (float)SvNV(fromstr); -#else + /* VOS does not automatically map a floating-point overflow + during conversion from double to float into infinity, so we + do it by hand. This code should either be generalized for + any OS that needs it, or removed if and when VOS implements + posix-976 (suggestion to support mapping to infinity). + Paul.Green@stratus.com 02-04-02. */ + if (anv > FLT_MAX) + afloat = _float_constants[0]; /* single prec. inf. */ + else if (anv < -FLT_MAX) + afloat = _float_constants[0]; /* single prec. inf. */ + else afloat = (float) anv; +#else /* __VOS__ */ # if defined(VMS) && !defined(__IEEE_FP) -/* IEEE fp overflow shenanigans are unavailable on VAX and optional - * on Alpha; fake it if we don't have them. - */ - if (SvNV(fromstr) > FLT_MAX) - afloat = FLT_MAX; - else if (SvNV(fromstr) < -FLT_MAX) - afloat = -FLT_MAX; - else afloat = (float)SvNV(fromstr); + /* IEEE fp overflow shenanigans are unavailable on VAX and optional + * on Alpha; fake it if we don't have them. + */ + if (anv > FLT_MAX) + afloat = FLT_MAX; + else if (anv < -FLT_MAX) + afloat = -FLT_MAX; + else afloat = (float)anv; # else - afloat = (float)SvNV(fromstr); + afloat = (float)anv; # endif -#endif - sv_catpvn(cat, (char *)&afloat, sizeof (float)); +#endif /* __VOS__ */ + DO_BO_PACK_N(afloat, float); + PUSH_VAR(utf8, cur, afloat); } break; case 'd': while (len-- > 0) { + double adouble; + NV anv; fromstr = NEXTFROM; + anv = SvNV(fromstr); #ifdef __VOS__ -/* VOS does not automatically map a floating-point overflow - during conversion from long double to double into infinity, - so we do it by hand. This code should either be generalized - for any OS that needs it, or removed if and when VOS - implements posix-976 (suggestion to support mapping to - infinity). Paul.Green@stratus.com 02-04-02. */ - if (SvNV(fromstr) > DBL_MAX) - adouble = _double_constants[0]; /* double prec. inf. */ - else if (SvNV(fromstr) < -DBL_MAX) - adouble = _double_constants[0]; /* double prec. inf. */ - else adouble = (double)SvNV(fromstr); -#else + /* VOS does not automatically map a floating-point overflow + during conversion from long double to double into infinity, + so we do it by hand. This code should either be generalized + for any OS that needs it, or removed if and when VOS + implements posix-976 (suggestion to support mapping to + infinity). Paul.Green@stratus.com 02-04-02. */ + if (anv > DBL_MAX) + adouble = _double_constants[0]; /* double prec. inf. */ + else if (anv < -DBL_MAX) + adouble = _double_constants[0]; /* double prec. inf. */ + else adouble = (double) anv; +#else /* __VOS__ */ # if defined(VMS) && !defined(__IEEE_FP) -/* IEEE fp overflow shenanigans are unavailable on VAX and optional - * on Alpha; fake it if we don't have them. - */ - if (SvNV(fromstr) > DBL_MAX) - adouble = DBL_MAX; - else if (SvNV(fromstr) < -DBL_MAX) - adouble = -DBL_MAX; - else adouble = (double)SvNV(fromstr); + /* IEEE fp overflow shenanigans are unavailable on VAX and optional + * on Alpha; fake it if we don't have them. + */ + if (anv > DBL_MAX) + adouble = DBL_MAX; + else if (anv < -DBL_MAX) + adouble = -DBL_MAX; + else adouble = (double)anv; # else - adouble = (double)SvNV(fromstr); + adouble = (double)anv; # endif -#endif - sv_catpvn(cat, (char *)&adouble, sizeof (double)); +#endif /* __VOS__ */ + DO_BO_PACK_N(adouble, double); + PUSH_VAR(utf8, cur, adouble); } break; - case 'F': + case 'F': { + NV anv; + Zero(&anv, 1, NV); /* can be long double with unused bits */ while (len-- > 0) { fromstr = NEXTFROM; anv = SvNV(fromstr); - sv_catpvn(cat, (char *)&anv, NVSIZE); + DO_BO_PACK_N(anv, NV); + PUSH_VAR(utf8, cur, anv); } break; + } #if defined(HAS_LONG_DOUBLE) && defined(USE_LONG_DOUBLE) - case 'D': + case 'D': { + long double aldouble; + /* long doubles can have unused bits, which may be nonzero */ + Zero(&aldouble, 1, long double); while (len-- > 0) { fromstr = NEXTFROM; aldouble = (long double)SvNV(fromstr); - sv_catpvn(cat, (char *)&aldouble, LONG_DOUBLESIZE); + DO_BO_PACK_N(aldouble, long double); + PUSH_VAR(utf8, cur, aldouble); } break; + } +#endif +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case 'n' | TYPE_IS_SHRIEKING: #endif case 'n': while (len-- > 0) { + I16 ai16; fromstr = NEXTFROM; - ashort = (I16)SvIV(fromstr); + ai16 = (I16)SvIV(fromstr); #ifdef HAS_HTONS - ashort = PerlSock_htons(ashort); + ai16 = PerlSock_htons(ai16); #endif - CAT16(cat, &ashort); + PUSH16(utf8, cur, &ai16); } break; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case 'v' | TYPE_IS_SHRIEKING: +#endif case 'v': while (len-- > 0) { + I16 ai16; fromstr = NEXTFROM; - ashort = (I16)SvIV(fromstr); + ai16 = (I16)SvIV(fromstr); #ifdef HAS_HTOVS - ashort = htovs(ashort); + ai16 = htovs(ai16); #endif - CAT16(cat, &ashort); + PUSH16(utf8, cur, &ai16); } break; case 'S' | TYPE_IS_SHRIEKING: #if SHORTSIZE != SIZE16 - { + while (len-- > 0) { unsigned short aushort; - - while (len-- > 0) { - fromstr = NEXTFROM; - aushort = SvUV(fromstr); - sv_catpvn(cat, (char *)&aushort, sizeof(unsigned short)); - } - } + fromstr = NEXTFROM; + aushort = SvUV(fromstr); + DO_BO_PACK(aushort, s); + PUSH_VAR(utf8, cur, aushort); + } break; #else /* Fall through! */ #endif case 'S': - { - U16 aushort; - - while (len-- > 0) { - fromstr = NEXTFROM; - aushort = (U16)SvUV(fromstr); - CAT16(cat, &aushort); - } - + while (len-- > 0) { + U16 au16; + fromstr = NEXTFROM; + au16 = (U16)SvUV(fromstr); + DO_BO_PACK(au16, 16); + PUSH16(utf8, cur, &au16); } break; case 's' | TYPE_IS_SHRIEKING: #if SHORTSIZE != SIZE16 - { + while (len-- > 0) { short ashort; - - while (len-- > 0) { - fromstr = NEXTFROM; - ashort = SvIV(fromstr); - sv_catpvn(cat, (char *)&ashort, sizeof(short)); - } + fromstr = NEXTFROM; + ashort = SvIV(fromstr); + DO_BO_PACK(ashort, s); + PUSH_VAR(utf8, cur, ashort); } break; #else @@ -2349,49 +3231,76 @@ S_pack_rec(pTHX_ SV *cat, register tempsym_t* symptr, register SV **beglist, SV #endif case 's': while (len-- > 0) { + I16 ai16; fromstr = NEXTFROM; - ashort = (I16)SvIV(fromstr); - CAT16(cat, &ashort); + ai16 = (I16)SvIV(fromstr); + DO_BO_PACK(ai16, 16); + PUSH16(utf8, cur, &ai16); } break; case 'I': case 'I' | TYPE_IS_SHRIEKING: while (len-- > 0) { + unsigned int auint; fromstr = NEXTFROM; auint = SvUV(fromstr); - sv_catpvn(cat, (char*)&auint, sizeof(unsigned int)); + DO_BO_PACK(auint, i); + PUSH_VAR(utf8, cur, auint); } break; case 'j': while (len-- > 0) { + IV aiv; fromstr = NEXTFROM; aiv = SvIV(fromstr); - sv_catpvn(cat, (char*)&aiv, IVSIZE); +#if IVSIZE == INTSIZE + DO_BO_PACK(aiv, i); +#elif IVSIZE == LONGSIZE + DO_BO_PACK(aiv, l); +#elif defined(HAS_QUAD) && IVSIZE == U64SIZE + DO_BO_PACK(aiv, 64); +#else + Perl_croak(aTHX_ "'j' not supported on this platform"); +#endif + PUSH_VAR(utf8, cur, aiv); } break; case 'J': while (len-- > 0) { + UV auv; fromstr = NEXTFROM; auv = SvUV(fromstr); - sv_catpvn(cat, (char*)&auv, UVSIZE); +#if UVSIZE == INTSIZE + DO_BO_PACK(auv, i); +#elif UVSIZE == LONGSIZE + DO_BO_PACK(auv, l); +#elif defined(HAS_QUAD) && UVSIZE == U64SIZE + DO_BO_PACK(auv, 64); +#else + Perl_croak(aTHX_ "'J' not supported on this platform"); +#endif + PUSH_VAR(utf8, cur, auv); } break; case 'w': while (len-- > 0) { + NV anv; fromstr = NEXTFROM; anv = SvNV(fromstr); - if (anv < 0) + if (anv < 0) { + *cur = '\0'; + SvCUR_set(cat, cur - start); Perl_croak(aTHX_ "Cannot compress negative numbers in pack"); + } /* 0xFFFFFFFFFFFFFFFF may cast to 18446744073709551616.0, which is == UV_MAX_P1. IOK is fine (instead of UV_only), as any negative IVs will have already been got by the croak() above. IOK is untrue for fractions, so we test them against UV_MAX_P1. */ - if (SvIOK(fromstr) || anv < UV_MAX_P1) - { - char buf[(sizeof(UV)*8)/7+1]; + if (SvIOK(fromstr) || anv < UV_MAX_P1) { + char buf[(sizeof(UV)*CHAR_BIT)/7+1]; char *in = buf + sizeof(buf); UV auv = SvUV(fromstr); @@ -2400,29 +3309,10 @@ S_pack_rec(pTHX_ SV *cat, register tempsym_t* symptr, register SV **beglist, SV auv >>= 7; } while (auv); buf[sizeof(buf) - 1] &= 0x7f; /* clear continue bit */ - sv_catpvn(cat, in, (buf + sizeof(buf)) - in); - } - else if (SvPOKp(fromstr)) { /* decimal string arithmetics */ - char *from, *result, *in; - SV *norm; - STRLEN len; - bool done; - - /* Copy string and check for compliance */ - from = SvPV(fromstr, len); - if ((norm = is_an_int(from, len)) == NULL) - Perl_croak(aTHX_ "Can only compress unsigned integers in pack"); - - New('w', result, len, char); - in = result + len; - done = FALSE; - while (!done) - *--in = div128(norm, &done) | 0x80; - result[len - 1] &= 0x7F; /* clear continue bit */ - sv_catpvn(cat, in, (result + len) - in); - Safefree(result); - SvREFCNT_dec(norm); /* free norm */ - } + PUSH_GROWING_BYTES(utf8, cat, start, cur, + in, (buf + sizeof(buf)) - in); + } else if (SvPOKp(fromstr)) + goto w_string; else if (SvNOKp(fromstr)) { /* 10**NV_MAX_10_EXP is the largest power of 10 so 10**(NV_MAX_10_EXP+1) is definately unrepresentable @@ -2435,110 +3325,118 @@ S_pack_rec(pTHX_ SV *cat, register tempsym_t* symptr, register SV **beglist, SV floating-point value. */ #ifdef NV_MAX_10_EXP -/* char buf[1 + (int)((NV_MAX_10_EXP + 1) * 0.47456)]; -- invalid C */ + /* char buf[1 + (int)((NV_MAX_10_EXP + 1) * 0.47456)]; -- invalid C */ char buf[1 + (int)((NV_MAX_10_EXP + 1) / 2)]; /* valid C */ #else -/* char buf[1 + (int)((308 + 1) * 0.47456)]; -- invalid C */ + /* char buf[1 + (int)((308 + 1) * 0.47456)]; -- invalid C */ char buf[1 + (int)((308 + 1) / 2)]; /* valid C */ #endif char *in = buf + sizeof(buf); - anv = Perl_floor(anv); + anv = Perl_floor(anv); do { - NV next = Perl_floor(anv / 128); + const NV next = Perl_floor(anv / 128); if (in <= buf) /* this cannot happen ;-) */ Perl_croak(aTHX_ "Cannot compress integer in pack"); *--in = (unsigned char)(anv - (next * 128)) | 0x80; anv = next; } while (anv > 0); buf[sizeof(buf) - 1] &= 0x7f; /* clear continue bit */ - sv_catpvn(cat, in, (buf + sizeof(buf)) - in); - } - else { - char *from, *result, *in; + PUSH_GROWING_BYTES(utf8, cat, start, cur, + in, (buf + sizeof(buf)) - in); + } else { + const char *from; + char *result, *in; SV *norm; STRLEN len; bool done; + w_string: /* Copy string and check for compliance */ - from = SvPV(fromstr, len); + from = SvPV_const(fromstr, len); if ((norm = is_an_int(from, len)) == NULL) Perl_croak(aTHX_ "Can only compress unsigned integers in pack"); - New('w', result, len, char); + Newx(result, len, char); in = result + len; done = FALSE; - while (!done) - *--in = div128(norm, &done) | 0x80; + while (!done) *--in = div128(norm, &done) | 0x80; result[len - 1] &= 0x7F; /* clear continue bit */ - sv_catpvn(cat, in, (result + len) - in); + PUSH_GROWING_BYTES(utf8, cat, start, cur, + in, (result + len) - in); Safefree(result); SvREFCNT_dec(norm); /* free norm */ - } + } } break; case 'i': case 'i' | TYPE_IS_SHRIEKING: while (len-- > 0) { + int aint; fromstr = NEXTFROM; aint = SvIV(fromstr); - sv_catpvn(cat, (char*)&aint, sizeof(int)); + DO_BO_PACK(aint, i); + PUSH_VAR(utf8, cur, aint); } break; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case 'N' | TYPE_IS_SHRIEKING: +#endif case 'N': while (len-- > 0) { + U32 au32; fromstr = NEXTFROM; - aulong = SvUV(fromstr); + au32 = SvUV(fromstr); #ifdef HAS_HTONL - aulong = PerlSock_htonl(aulong); + au32 = PerlSock_htonl(au32); #endif - CAT32(cat, &aulong); + PUSH32(utf8, cur, &au32); } break; +#ifdef PERL_PACK_CAN_SHRIEKSIGN + case 'V' | TYPE_IS_SHRIEKING: +#endif case 'V': while (len-- > 0) { + U32 au32; fromstr = NEXTFROM; - aulong = SvUV(fromstr); + au32 = SvUV(fromstr); #ifdef HAS_HTOVL - aulong = htovl(aulong); + au32 = htovl(au32); #endif - CAT32(cat, &aulong); + PUSH32(utf8, cur, &au32); } break; case 'L' | TYPE_IS_SHRIEKING: #if LONGSIZE != SIZE32 - { + while (len-- > 0) { unsigned long aulong; - - while (len-- > 0) { - fromstr = NEXTFROM; - aulong = SvUV(fromstr); - sv_catpvn(cat, (char *)&aulong, sizeof(unsigned long)); - } + fromstr = NEXTFROM; + aulong = SvUV(fromstr); + DO_BO_PACK(aulong, l); + PUSH_VAR(utf8, cur, aulong); } break; #else /* Fall though! */ #endif case 'L': - { - while (len-- > 0) { - fromstr = NEXTFROM; - aulong = SvUV(fromstr); - CAT32(cat, &aulong); - } + while (len-- > 0) { + U32 au32; + fromstr = NEXTFROM; + au32 = SvUV(fromstr); + DO_BO_PACK(au32, 32); + PUSH32(utf8, cur, &au32); } break; case 'l' | TYPE_IS_SHRIEKING: #if LONGSIZE != SIZE32 - { + while (len-- > 0) { long along; - - while (len-- > 0) { - fromstr = NEXTFROM; - along = SvIV(fromstr); - sv_catpvn(cat, (char *)&along, sizeof(long)); - } + fromstr = NEXTFROM; + along = SvIV(fromstr); + DO_BO_PACK(along, l); + PUSH_VAR(utf8, cur, along); } break; #else @@ -2546,78 +3444,115 @@ S_pack_rec(pTHX_ SV *cat, register tempsym_t* symptr, register SV **beglist, SV #endif case 'l': while (len-- > 0) { + I32 ai32; fromstr = NEXTFROM; - along = SvIV(fromstr); - CAT32(cat, &along); + ai32 = SvIV(fromstr); + DO_BO_PACK(ai32, 32); + PUSH32(utf8, cur, &ai32); } break; #ifdef HAS_QUAD case 'Q': while (len-- > 0) { + Uquad_t auquad; fromstr = NEXTFROM; - auquad = (Uquad_t)SvUV(fromstr); - sv_catpvn(cat, (char*)&auquad, sizeof(Uquad_t)); + auquad = (Uquad_t) SvUV(fromstr); + DO_BO_PACK(auquad, 64); + PUSH_VAR(utf8, cur, auquad); } break; case 'q': while (len-- > 0) { + Quad_t aquad; fromstr = NEXTFROM; aquad = (Quad_t)SvIV(fromstr); - sv_catpvn(cat, (char*)&aquad, sizeof(Quad_t)); + DO_BO_PACK(aquad, 64); + PUSH_VAR(utf8, cur, aquad); } break; -#endif +#endif /* HAS_QUAD */ case 'P': len = 1; /* assume SV is correct length */ + GROWING(utf8, cat, start, cur, sizeof(char *)); /* Fall through! */ case 'p': while (len-- > 0) { + const char *aptr; + fromstr = NEXTFROM; - if (fromstr == &PL_sv_undef) - aptr = NULL; + SvGETMAGIC(fromstr); + if (!SvOK(fromstr)) aptr = NULL; else { - STRLEN n_a; /* XXX better yet, could spirit away the string to * a safe spot and hang on to it until the result * of pack() (and all copies of the result) are * gone. */ - if (ckWARN(WARN_PACK) && (SvTEMP(fromstr) - || (SvPADTMP(fromstr) - && !SvREADONLY(fromstr)))) - { + if ((SvTEMP(fromstr) || (SvPADTMP(fromstr) && + !SvREADONLY(fromstr))) && ckWARN(WARN_PACK)) { Perl_warner(aTHX_ packWARN(WARN_PACK), - "Attempt to pack pointer to temporary value"); + "Attempt to pack pointer to temporary value"); } if (SvPOK(fromstr) || SvNIOK(fromstr)) - aptr = SvPV(fromstr,n_a); + aptr = SvPV_nomg_const_nolen(fromstr); else - aptr = SvPV_force(fromstr,n_a); + aptr = SvPV_force_flags_nolen(fromstr, 0); } - sv_catpvn(cat, (char*)&aptr, sizeof(char*)); + DO_BO_PACK_PC(aptr); + PUSH_VAR(utf8, cur, aptr); } break; - case 'u': + case 'u': { + const char *aptr, *aend; + bool from_utf8; + fromstr = NEXTFROM; - aptr = SvPV(fromstr, fromlen); - SvGROW(cat, fromlen * 4 / 3); - if (len <= 2) - len = 45; - else - len = len / 3 * 3; + if (len <= 2) len = 45; + else len = len / 3 * 3; + if (len >= 64) { + if (ckWARN(WARN_PACK)) + Perl_warner(aTHX_ packWARN(WARN_PACK), + "Field too wide in 'u' format in pack"); + len = 63; + } + aptr = SvPV_const(fromstr, fromlen); + from_utf8 = DO_UTF8(fromstr); + if (from_utf8) { + aend = aptr + fromlen; + fromlen = sv_len_utf8(fromstr); + } else aend = NULL; /* Unused, but keep compilers happy */ + GROWING(utf8, cat, start, cur, (fromlen+2) / 3 * 4 + (fromlen+len-1)/len * 2); while (fromlen > 0) { + U8 *end; I32 todo; + U8 hunk[1+63/3*4+1]; if ((I32)fromlen > len) todo = len; else todo = fromlen; - doencodes(cat, aptr, todo); + if (from_utf8) { + char buffer[64]; + if (!uni_to_bytes(aTHX_ &aptr, aend, buffer, todo, + 'u' | TYPE_IS_PACK)) { + *cur = '\0'; + SvCUR_set(cat, cur - start); + Perl_croak(aTHX_ "Assertion: string is shorter than advertised"); + } + end = doencodes(hunk, buffer, todo); + } else { + end = doencodes(hunk, aptr, todo); + aptr += todo; + } + PUSH_BYTES(utf8, cur, hunk, end-hunk); fromlen -= todo; - aptr += todo; } break; } + } + *cur = '\0'; + SvCUR_set(cat, cur - start); + no_change: *symptr = lookahead; } return beglist; @@ -2627,14 +3562,16 @@ S_pack_rec(pTHX_ SV *cat, register tempsym_t* symptr, register SV **beglist, SV PP(pp_pack) { - dSP; dMARK; dORIGMARK; dTARGET; + dVAR; dSP; dMARK; dORIGMARK; dTARGET; register SV *cat = TARG; STRLEN fromlen; - register char *pat = SvPVx(*++MARK, fromlen); - register char *patend = pat + fromlen; + SV *pat_sv = *++MARK; + register const char *pat = SvPV_const(pat_sv, fromlen); + register const char *patend = pat + fromlen; MARK++; sv_setpvn(cat, "", 0); + SvUTF8_off(cat); packlist(cat, pat, patend, MARK, SP + 1); @@ -2644,3 +3581,12 @@ PP(pp_pack) RETURN; } +/* + * Local variables: + * c-indentation-style: bsd + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + * + * ex: set ts=8 sts=4 sw=4 noet: + */