utf8.c: Generalize static fcn return for indeterminate result

Prior to this commit, isFF_OVERLONG() returned a boolean, with 0 also
indicating that there wasn't enough information to make a determination.
I realized that I was forgetting that 0 wasn't necessarily definitive
while coding.  By changing the API to return 3 values, forgetting that
won't likely happen.

This and the next several commits change several other functions that
have the same predicament.

diff --git a/embed.fnc b/embed.fnc
index 20f2987..6799895 100644 (file)
--- a/embed.fnc
+++ b/embed.fnc
@@ -1724,7 +1724,7 @@ EpM       |char * |_byte_dump_string                                      \
 #if defined(PERL_IN_UTF8_C)
 inR    |bool   |does_utf8_overflow|NN const U8 * const s|NN const U8 * e
 inR    |bool   |is_utf8_overlong_given_start_byte_ok|NN const U8 * const s|const STRLEN len
-inR    |bool   |isFF_OVERLONG  |NN const U8 * const s|const STRLEN len
+inR    |int    |isFF_OVERLONG  |NN const U8 * const s|const STRLEN len
 sMR    |char * |unexpected_non_continuation_text                       \
                |NN const U8 * const s                                  \
                |STRLEN print_len                                       \

diff --git a/proto.h b/proto.h
index a8f6de8..dd87483 100644 (file)
--- a/proto.h
+++ b/proto.h
@@ -5825,7 +5825,7 @@ PERL_STATIC_INLINE bool   S_does_utf8_overflow(const U8 * const s, const U8 * e)
 #endif
 
 #ifndef PERL_NO_INLINE_FUNCTIONS
-PERL_STATIC_INLINE bool        S_isFF_OVERLONG(const U8 * const s, const STRLEN len)
+PERL_STATIC_INLINE int S_isFF_OVERLONG(const U8 * const s, const STRLEN len)
                        __attribute__warn_unused_result__;
 #define PERL_ARGS_ASSERT_ISFF_OVERLONG \
        assert(s)

diff --git a/utf8.c b/utf8.c
index 1735b81..c05866a 100644 (file)
--- a/utf8.c
+++ b/utf8.c
@@ -531,7 +531,7 @@ S_does_utf8_overflow(const U8 * const s, const U8 * e)
     /* On 32 bit ASCII machines, many overlongs that start with FF don't
      * overflow */
 
-    if (isFF_OVERLONG(s, len)) {
+    if (isFF_OVERLONG(s, len) > 0) {
         const U8 max_32_bit_overlong[] = "\xFF\x80\x80\x80\x80\x80\x80\x84";
         return memGE(s, max_32_bit_overlong,
                                 MIN(len, sizeof(max_32_bit_overlong) - 1));
@@ -616,21 +616,38 @@ S_is_utf8_overlong_given_start_byte_ok(const U8 * const s, const STRLEN len)
     }
 
     /* Check for the FF overlong */
-    return isFF_OVERLONG(s, len);
+    return isFF_OVERLONG(s, len) > 0;
 }
 
-PERL_STATIC_INLINE bool
+PERL_STATIC_INLINE int
 S_isFF_OVERLONG(const U8 * const s, const STRLEN len)
 {
+    /* Returns an int indicating whether or not the UTF-8 sequence from 's' to
+     * 'e' - 1 is an overlong beginning with \xFF.  It returns 1 if it is; 0 if
+     * it isn't, and -1 if there isn't enough information to tell.  This last
+     * return value can happen if the sequence is incomplete, missing some
+     * trailing bytes that would form a complete character.  If there are
+     * enough bytes to make a definitive decision, this function does so. */
+
     PERL_ARGS_ASSERT_ISFF_OVERLONG;
 
-    /* Check for the FF overlong.  This happens only if all these bytes match;
-     * what comes after them doesn't matter.  See tables in utf8.h,
+    /* To be an FF overlong, all the available bytes must match */
+    if (LIKELY(memNE(s, FF_OVERLONG_PREFIX,
+                     MIN(len, sizeof(FF_OVERLONG_PREFIX) - 1))))
+    {
+        return 0;
+    }
+
+    /* To be an FF overlong sequence, all the bytes in FF_OVERLONG_PREFIX must
+     * be there; what comes after them doesn't matter.  See tables in utf8.h,
      * utfebcdic.h. */
+    if (len >= sizeof(FF_OVERLONG_PREFIX) - 1) {
+        return 1;
+    }
 
-    return    len >= sizeof(FF_OVERLONG_PREFIX) - 1
-           && UNLIKELY(memEQ(s, FF_OVERLONG_PREFIX,
-                                            sizeof(FF_OVERLONG_PREFIX) - 1));
+    /* The missing bytes could cause the result to go one way or the other, so
+     * the result is indeterminate */
+    return -1;
 }
 
 #undef F0_ABOVE_OVERLONG