Prepare for Unicode 11.0

Unicode 11 has some new data files needed for it, and some changes in
the boundary rules that need to be accounted for.  This does all that
can be done without causing tests to fail.  The LB algorithm has
changed, and tests would fail if we included the code changes needed for
that change in this commit.  Instead those few lines will come as part
of the Unicode 11.0 commit.

diff --git a/charclass_invlists.h b/charclass_invlists.h
index 2580074..c0f3ea3 100644 (file)
--- a/charclass_invlists.h
+++ b/charclass_invlists.h
@@ -19757,7 +19757,8 @@ typedef enum {
        GCB_T = 15,
        GCB_V = 16,
        GCB_ZWJ = 17,
-       GCB_EDGE = 18
+       GCB_XPG_XX = 18,
+       GCB_EDGE = 19
 } GCB_enum;
 
 static const GCB_enum _Perl_GCB_invmap[] = { /* for ASCII/Latin1 */
@@ -23191,7 +23192,8 @@ typedef enum {
        GCB_T = 15,
        GCB_V = 16,
        GCB_ZWJ = 17,
-       GCB_EDGE = 18
+       GCB_XPG_XX = 18,
+       GCB_EDGE = 19
 } GCB_enum;
 
 static const GCB_enum _Perl_GCB_invmap[] = { /* for EBCDIC 1047 */
@@ -26625,7 +26627,8 @@ typedef enum {
        GCB_T = 15,
        GCB_V = 16,
        GCB_ZWJ = 17,
-       GCB_EDGE = 18
+       GCB_XPG_XX = 18,
+       GCB_EDGE = 19
 } GCB_enum;
 
 static const GCB_enum _Perl_GCB_invmap[] = { /* for EBCDIC 037 */
@@ -83045,8 +83048,10 @@ typedef enum {
        WB_Regional_Indicator = 20,
        WB_Single_Quote = 21,
        WB_ZWJ = 22,
-       WB_EDGE = 23,
-       WB_UNKNOWN = 24
+       WB_XPG_XX = 23,
+       WB_XPG_LE = 24,
+       WB_EDGE = 25,
+       WB_UNKNOWN = 26
 } WB_enum;
 
 static const WB_enum _Perl_WB_invmap[] = { /* for ASCII/Latin1 */
@@ -86570,8 +86575,10 @@ typedef enum {
        WB_Regional_Indicator = 20,
        WB_Single_Quote = 21,
        WB_ZWJ = 22,
-       WB_EDGE = 23,
-       WB_UNKNOWN = 24
+       WB_XPG_XX = 23,
+       WB_XPG_LE = 24,
+       WB_EDGE = 25,
+       WB_UNKNOWN = 26
 } WB_enum;
 
 static const WB_enum _Perl_WB_invmap[] = { /* for EBCDIC 1047 */
@@ -90114,8 +90121,10 @@ typedef enum {
        WB_Regional_Indicator = 20,
        WB_Single_Quote = 21,
        WB_ZWJ = 22,
-       WB_EDGE = 23,
-       WB_UNKNOWN = 24
+       WB_XPG_XX = 23,
+       WB_XPG_LE = 24,
+       WB_EDGE = 25,
+       WB_UNKNOWN = 26
 } WB_enum;
 
 static const WB_enum _Perl_WB_invmap[] = { /* for EBCDIC 037 */
@@ -374150,33 +374159,37 @@ static const UV * const PL_uni_prop_ptrs[] = {
 
 #if defined(PERL_IN_REGEXEC_C)
 
-#define GCB_NOBREAK      0
-#define GCB_BREAKABLE    1
-#define GCB_RI_then_RI   2
-#define GCB_EX_then_EM   3
-
-static const U8 GCB_table[19][19] = {
-   /* 'edg' stands for 'EDGE' */
-/*        XX CR CN EB EBG EM EX GAZ  L LF LV LVT PP RI SM  T  V ZWJ edg */
-/* XX */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0,  1 },
-/* CR */ { 1, 1, 1, 1,  1, 1, 1,  1, 1, 0, 1,  1, 1, 1, 1, 1, 1,  1,  1 },
-/* CN */ { 1, 1, 1, 1,  1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1, 1, 1,  1,  1 },
-/* EB */ { 1, 1, 1, 1,  1, 0, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0,  1 },
-/* EBG*/ { 1, 1, 1, 1,  1, 0, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0,  1 },
-/* EM */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0,  1 },
-/* EX */ { 1, 1, 1, 1,  1, 3, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0,  1 },
-/* GAZ*/ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0,  1 },
-/* L  */ { 1, 1, 1, 1,  1, 1, 0,  1, 0, 1, 0,  0, 1, 1, 0, 1, 0,  0,  1 },
-/* LF */ { 1, 1, 1, 1,  1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1, 1, 1,  1,  1 },
-/* LV */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 0, 0,  0,  1 },
-/* LVT*/ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 0, 1,  0,  1 },
-/* PP */ { 0, 1, 1, 0,  0, 0, 0,  0, 0, 1, 0,  0, 0, 0, 0, 0, 0,  0,  1 },
-/* RI */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 2, 0, 1, 1,  0,  1 },
-/* SM */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0,  1 },
-/* T  */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 0, 1,  0,  1 },
-/* V  */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 0, 0,  0,  1 },
-/* ZWJ*/ { 1, 1, 1, 1,  0, 1, 0,  0, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0,  1 },
-/* edg*/ { 1, 1, 1, 1,  1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1, 1, 1,  1,  0 }
+#define GCB_NOBREAK                0
+#define GCB_BREAKABLE              1
+#define GCB_RI_then_RI             2
+#define GCB_EX_then_EM             3
+#define GCB_Maybe_Emoji_NonBreak   4
+
+static const U8 GCB_table[20][20] = {
+   /* 'edg' stands for 'EDGE'; other lowercase names are placeholders for
+    * property values not defined until a later Unicode release, so are
+    * irrelevant in this one, as they are not assigned to any code points */
+/*        XX CR CN EB EBG EM EX GAZ  L LF LV LVT PP RI SM  T  V ZWJ  a edg */
+/* XX */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 1,  1 },
+/* CR */ { 1, 1, 1, 1,  1, 1, 1,  1, 1, 0, 1,  1, 1, 1, 1, 1, 1,  1, 1,  1 },
+/* CN */ { 1, 1, 1, 1,  1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1, 1, 1,  1, 1,  1 },
+/* EB */ { 1, 1, 1, 1,  1, 0, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 1,  1 },
+/* EBG*/ { 1, 1, 1, 1,  1, 0, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 1,  1 },
+/* EM */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 1,  1 },
+/* EX */ { 1, 1, 1, 1,  1, 3, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 1,  1 },
+/* GAZ*/ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 1,  1 },
+/* L  */ { 1, 1, 1, 1,  1, 1, 0,  1, 0, 1, 0,  0, 1, 1, 0, 1, 0,  0, 1,  1 },
+/* LF */ { 1, 1, 1, 1,  1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1, 1, 1,  1, 1,  1 },
+/* LV */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 0, 0,  0, 1,  1 },
+/* LVT*/ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 0, 1,  0, 1,  1 },
+/* PP */ { 0, 1, 1, 0,  0, 0, 0,  0, 0, 1, 0,  0, 0, 0, 0, 0, 0,  0, 0,  1 },
+/* RI */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 2, 0, 1, 1,  0, 1,  1 },
+/* SM */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 1,  1 },
+/* T  */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 0, 1,  0, 1,  1 },
+/* V  */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 0, 0,  0, 1,  1 },
+/* ZWJ*/ { 1, 1, 1, 1,  0, 1, 0,  0, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 4,  1 },
+/* a  */ { 1, 1, 1, 1,  1, 1, 0,  1, 1, 1, 1,  1, 1, 1, 0, 1, 1,  0, 1,  1 },
+/* edg*/ { 1, 1, 1, 1,  1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1, 1, 1,  1, 1,  0 }
 };
 
 #define LB_NOBREAK                        0
@@ -374246,34 +374259,38 @@ static const U8 LB_table[39][39] = {
 #define WB_NU_then_MB_or_MN_or_SQ        14
 #define WB_RI_then_RI                    16
 
-static const U8 WB_table[24][24] = {
+static const U8 WB_table[26][26] = {
    /* 'Ext' stands for 'Extend'; 'edg' stands for 'EDGE'; 'hs' stands for
-    * 'Perl_Tailored_HSpace'; 'unk' stands for 'UNKNOWN' */
-/*        XX LE CR DQ EB EBG EM Ext EX FO GAZ HL KA LF ML MN MB NL NU hs RI SQ ZWJ edg */
-/* XX */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0,  1 },
-/* LE */ { 1, 0, 1, 1, 1,  1, 1,  0, 0, 0,  1, 0, 1, 1, 9, 1, 9, 1, 0, 1, 1, 9,  0,  1 },
-/* CR */ { 1, 1, 0, 1, 1,  1, 1,  1, 1, 1,  1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1,  1,  1 },
-/* DQ */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0,  1 },
-/* EB */ { 1, 1, 1, 1, 1,  1, 0,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0,  1 },
-/* EBG*/ { 1, 1, 1, 1, 1,  1, 0,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0,  1 },
-/* EM */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0,  1 },
-/* Ext*/ { 3, 3, 1, 3, 3,  3, 3,  0, 3, 0,  3, 3, 3, 1, 3, 3, 3, 1, 3, 1, 3, 3,  0,  1 },
-/* EX */ { 1, 0, 1, 1, 1,  1, 1,  0, 0, 0,  1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1,  0,  1 },
-/* FO */ { 3, 3, 1, 3, 3,  3, 3,  0, 3, 0,  3, 3, 3, 1, 3, 3, 3, 1, 3, 1, 3, 3,  0,  1 },
-/* GAZ*/ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0,  1 },
-/* HL */ { 1, 0, 1, 7, 1,  1, 1,  0, 0, 0,  1, 0, 1, 1, 9, 1, 9, 1, 0, 1, 1, 8,  0,  1 },
-/* KA */ { 1, 1, 1, 1, 1,  1, 1,  0, 0, 0,  1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0,  1 },
-/* LF */ { 1, 1, 0, 1, 1,  1, 1,  1, 1, 1,  1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1,  1,  1 },
-/* ML */ { 1,11, 1, 1, 1,  1, 1,  0, 1, 0,  1,11, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0,  1 },
-/* MN */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1,13, 1, 1, 1,  0,  1 },
-/* MB */ { 1,11, 1, 1, 1,  1, 1,  0, 1, 0,  1,11, 1, 1, 1, 1, 1, 1,13, 1, 1, 1,  0,  1 },
-/* NL */ { 1, 1, 0, 1, 1,  1, 1,  1, 1, 1,  1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1,  1,  1 },
-/* NU */ { 1, 0, 1, 1, 1,  1, 1,  0, 0, 0,  1, 0, 1, 1, 1,15,15, 1, 0, 1, 1,15,  0,  1 },
-/* hs */ { 1, 1, 0, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 0, 1, 1, 1, 0, 1, 2, 1, 1,  0,  1 },
-/* RI */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1,16, 1,  0,  1 },
-/* SQ */ { 1,11, 1, 1, 1,  1, 1,  0, 1, 0,  1,11, 1, 1, 1, 1, 1, 1,13, 1, 1, 1,  0,  1 },
-/* ZWJ*/ { 3, 3, 1, 3, 3,  0, 3,  0, 3, 0,  0, 3, 3, 1, 3, 3, 3, 1, 3, 1, 3, 3,  0,  1 },
-/* edg*/ { 1, 1, 1, 1, 1,  1, 1,  1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  1,  0 }
+    * 'Perl_Tailored_HSpace'; 'unk' stands for 'UNKNOWN'; other lowercase names are
+    * placeholders for property values not defined until a later Unicode release, so are
+    * irrelevant in this one, as they are not assigned to any code points */
+/*        XX LE CR DQ EB EBG EM Ext EX FO GAZ HL KA LF ML MN MB NL NU hs RI SQ ZWJ  b  a edg */
+/* XX */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1, 1,  1 },
+/* LE */ { 1, 0, 1, 1, 1,  1, 1,  0, 0, 0,  1, 0, 1, 1, 9, 1, 9, 1, 0, 1, 1, 9,  0, 1, 1,  1 },
+/* CR */ { 1, 1, 0, 1, 1,  1, 1,  1, 1, 1,  1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1,  1, 1, 1,  1 },
+/* DQ */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1, 1,  1 },
+/* EB */ { 1, 1, 1, 1, 1,  1, 0,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1, 1,  1 },
+/* EBG*/ { 1, 1, 1, 1, 1,  1, 0,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1, 1,  1 },
+/* EM */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1, 1,  1 },
+/* Ext*/ { 3, 3, 1, 3, 3,  3, 3,  0, 3, 0,  3, 3, 3, 1, 3, 3, 3, 1, 3, 1, 3, 3,  0, 3, 3,  1 },
+/* EX */ { 1, 0, 1, 1, 1,  1, 1,  0, 0, 0,  1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1,  0, 1, 0,  1 },
+/* FO */ { 3, 3, 1, 3, 3,  3, 3,  0, 3, 0,  3, 3, 3, 1, 3, 3, 3, 1, 3, 1, 3, 3,  0, 3, 3,  1 },
+/* GAZ*/ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1, 1,  1 },
+/* HL */ { 1, 0, 1, 7, 1,  1, 1,  0, 0, 0,  1, 0, 1, 1, 9, 1, 9, 1, 0, 1, 1, 8,  0, 1, 0,  1 },
+/* KA */ { 1, 1, 1, 1, 1,  1, 1,  0, 0, 0,  1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1, 1,  1 },
+/* LF */ { 1, 1, 0, 1, 1,  1, 1,  1, 1, 1,  1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1,  1, 1, 1,  1 },
+/* ML */ { 1,11, 1, 1, 1,  1, 1,  0, 1, 0,  1,11, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1,11,  1 },
+/* MN */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1,13, 1, 1, 1,  0, 1, 1,  1 },
+/* MB */ { 1,11, 1, 1, 1,  1, 1,  0, 1, 0,  1,11, 1, 1, 1, 1, 1, 1,13, 1, 1, 1,  0, 1,11,  1 },
+/* NL */ { 1, 1, 0, 1, 1,  1, 1,  1, 1, 1,  1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1,  1, 1, 1,  1 },
+/* NU */ { 1, 0, 1, 1, 1,  1, 1,  0, 0, 0,  1, 0, 1, 1, 1,15,15, 1, 0, 1, 1,15,  0, 1, 0,  1 },
+/* hs */ { 1, 1, 0, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 0, 1, 1, 1, 0, 1, 2, 1, 1,  0, 1, 1,  1 },
+/* RI */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1,16, 1,  0, 1, 1,  1 },
+/* SQ */ { 1,11, 1, 1, 1,  1, 1,  0, 1, 0,  1,11, 1, 1, 1, 1, 1, 1,13, 1, 1, 1,  0, 1,11,  1 },
+/* ZWJ*/ { 3, 3, 1, 3, 3,  0, 3,  0, 3, 0,  0, 3, 3, 1, 3, 3, 3, 1, 3, 1, 3, 3,  0, 0, 0,  1 },
+/* b  */ { 1, 1, 1, 1, 1,  1, 1,  0, 1, 0,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  0, 1, 1,  1 },
+/* a  */ { 1, 0, 1, 1, 1,  1, 1,  0, 0, 0,  1, 0, 1, 1, 9, 1, 9, 1, 0, 1, 1, 9,  0, 1, 0,  1 },
+/* edg*/ { 1, 1, 1, 1, 1,  1, 1,  1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1,  0 }
 };
 
 #endif /* defined(PERL_IN_REGEXEC_C) */
@@ -374323,9 +374340,9 @@ static const U8 WB_table[24][24] = {
  * be0f129691d479aa38646e4ca0ec1ee576ae7f75b0300a5624a7fa862fa8abba lib/unicore/extracted/DLineBreak.txt
  * 92449d354d9f6b6f2f97a292ebb59f6344ffdeb83d120d7d23e569c43ba67cd5 lib/unicore/extracted/DNumType.txt
  * e3a319527153b0c6c0c549b40fc6f3a01a7a0dcd6620784391db25901df3b154 lib/unicore/extracted/DNumValues.txt
- * 5c4387f0eb0a4d3ec12ce24309495ba2fe2ff8d6c20e11eb19258ee347a83e5d lib/unicore/mktables
+ * 22c0974193811f3f52d01b33c394c22b87541385fb68d5a3502749ec6bf077aa lib/unicore/mktables
  * 21653d2744fdd071f9ef138c805393901bb9547cf3e777ebf50215a191f986ea lib/unicore/version
  * 4bb677187a1a64e39d48f2e341b5ecb6c99857e49d7a79cf503bd8a3c709999b regen/charset_translations.pl
  * 03e51b0f07beebd5da62ab943899aa4934eee1f792fa27c1fb638c33bf4ac6ea regen/mk_PL_charclass.pl
- * b015b9e6db7b31e919c494042c9c4b81f2b77352ff2a03a0b123f09a5818f8dd regen/mk_invlists.pl
+ * 507fb07e9b2257fe365525cc485c02ca756f13ca2186412c8792f1d03496313f regen/mk_invlists.pl
  * ex: set ro: */

diff --git a/lib/unicore/mktables b/lib/unicore/mktables
index b1cdaeb..f45946e 100644 (file)
--- a/lib/unicore/mktables
+++ b/lib/unicore/mktables
@@ -895,6 +895,19 @@ if ($v_version gt v3.2.0) {
                                 'Canonical_Combining_Class=Attached_Below_Left'
 }
 
+# Obsoleted
+if ($v_version ge v11.0.0) {
+    push @tables_that_may_be_empty, qw(
+                                       Grapheme_Cluster_Break=E_Base
+                                       Grapheme_Cluster_Break=E_Base_GAZ
+                                       Grapheme_Cluster_Break=E_Modifier
+                                       Grapheme_Cluster_Break=Glue_After_Zwj
+                                       Word_Break=E_Base
+                                       Word_Break=E_Base_GAZ
+                                       Word_Break=E_Modifier
+                                       Word_Break=Glue_After_Zwj);
+}
+
 # Enum values for to_output_map() method in the Map_Table package. (0 is don't
 # output)
 my $EXTERNAL_MAP = 1;
@@ -13474,6 +13487,24 @@ sub  filter_script_extensions_line {
     return;
 }
 
+sub setup_emojidata {
+    my $prop_ref = Property->new('XPG',
+                                 Full_Name => 'Extended_Pictographic',
+    );
+    $prop_ref->set_fate($PLACEHOLDER,
+                        "Not part of the Unicode Character Database");
+}
+
+sub filter_emojidata_line {
+    # We only are interested in this single property from this non-UCD data
+    # file, and we turn it into a Perl property, so that it isn't accessible
+    # to the users
+
+    $_ = "" unless /\bExtended_Pictographic\b/;
+
+    return;
+}
+
 sub generate_hst {
 
     # Populates the Hangul Syllable Type property from first principles
@@ -15446,22 +15477,52 @@ END
     }
 
     # Perl tailors the WordBreak property so that \b{wb} doesn't split
-    # adjacent spaces into separate words.  First create a copy of the regular
-    # WB property as '_Perl_WB'.  (On Unicode releases earlier than when WB
-    # was defined for, this will already have been done by the substitute file
-    # portion for 'Input_file' code for WB.)
+    # adjacent spaces into separate words.  Unicode 11.0 moved in that
+    # direction, but left TAB,  FIGURE SPACE (U+2007), and (ironically) NO
+    # BREAK SPACE as breaking, so we retained the original Perl customization.
+    # To do this, in the Perl copy of WB, simply replace the mappings of
+    # horizontal space characters that otherwise would map to the default or
+    # the 11.0 'WSegSpace' to instead map to our tailoring.
     my $perl_wb = property_ref('_Perl_WB');
-    # And simply replace the mappings of horizontal space characters that
-    # otherwise would map to the default to instead map to our tailoring.
     my $default = $perl_wb->default_map;
     for my $range ($Blank->ranges) {
         for my $i ($range->start .. $range->end) {
-            next unless $perl_wb->value_of($i) eq $default;
+            my $value = $perl_wb->value_of($i);
+
+            next unless $value eq $default || $value eq 'WSegSpace';
             $perl_wb->add_map($i, $i, 'Perl_Tailored_HSpace',
                               Replace => $UNCONDITIONALLY);
         }
     }
 
+    # Also starting in Unicode 11.0, rules for some of the boundary types are
+    # based on a non-UCD property (which we have read in if it exists).
+    # Recall that these boundary properties partition the code points into
+    # equivalence classes (represented as enums).
+    #
+    # The loop below goes through each code point that matches the non-UCD
+    # property, and for each current equivalence class containing such a code
+    # point, splits it so that those that are in both are now in a newly
+    # created equivalence class whose name is a combination of the property
+    # and the old class name, leaving unchanged everything that doesn't match
+    # the non-UCD property.
+    my $pictographic_emoji = property_ref('XPG');
+    if (defined $pictographic_emoji) {
+        foreach my $base_property (property_ref('GCB'),
+                                   property_ref('WB'))
+        {
+            my $property = property_ref('_Perl_' . $base_property->name);
+            foreach my $range ($pictographic_emoji->table('Y')->ranges) {
+                foreach my $i ($range->start .. $range->end) {
+                    my $current = $property->value_of($i);
+                    $current = $property->table($current)->short_name;
+                    $property->add_map($i, $i, 'XPG_' . $current,
+                                       Replace => $UNCONDITIONALLY);
+                }
+            }
+        }
+    }
+
     # Create a version of the LineBreak property with the mappings that are
     # omitted in the default algorithm remapped to what
     # http://www.unicode.org/reports/tr14 says they should be.
@@ -19917,6 +19978,19 @@ my @input_file_objects = (
     Input_file->new('NushuSources.txt', v10.0.0,
                     Skip => 'Specifies source material for Nushu characters',
                    ),
+    Input_file->new('EquivalentUnifiedIdeograph.txt', v11.0.0,
+                    Property => 'Equivalent_Unified_Ideograph',
+                    Has_Missings_Defaults => $NOT_IGNORED,
+                   ),
+    Input_file->new('EmojiData.txt', v11.0.0,
+                    # Is in UAX #51 and not the UCD, so must be updated
+                    # separately, and the first line edited to indicate the
+                    # UCD release we're pretending it to be in.  The UTC says
+                    # this is a transitional state.
+                    Pre_Handler => \&setup_emojidata,
+                    Has_Missings_Defaults => $NOT_IGNORED,
+                    Each_Line_Handler => \&filter_emojidata_line,
+                   ),
 );
 
 # End of all the preliminaries.

diff --git a/regcharclass.h b/regcharclass.h
index 5596ee7..7e9909c 100644 (file)
--- a/regcharclass.h
+++ b/regcharclass.h
@@ -1907,7 +1907,7 @@
  * be0f129691d479aa38646e4ca0ec1ee576ae7f75b0300a5624a7fa862fa8abba lib/unicore/extracted/DLineBreak.txt
  * 92449d354d9f6b6f2f97a292ebb59f6344ffdeb83d120d7d23e569c43ba67cd5 lib/unicore/extracted/DNumType.txt
  * e3a319527153b0c6c0c549b40fc6f3a01a7a0dcd6620784391db25901df3b154 lib/unicore/extracted/DNumValues.txt
- * 5c4387f0eb0a4d3ec12ce24309495ba2fe2ff8d6c20e11eb19258ee347a83e5d lib/unicore/mktables
+ * 22c0974193811f3f52d01b33c394c22b87541385fb68d5a3502749ec6bf077aa lib/unicore/mktables
  * 21653d2744fdd071f9ef138c805393901bb9547cf3e777ebf50215a191f986ea lib/unicore/version
  * 4bb677187a1a64e39d48f2e341b5ecb6c99857e49d7a79cf503bd8a3c709999b regen/charset_translations.pl
  * f4fe24c2a005b636f6e7418df060c6c9a57ddcae4219b54d0cb18baf312dbc3a regen/regcharclass.pl

diff --git a/regen/mk_invlists.pl b/regen/mk_invlists.pl
index 4e92351..464801c 100644 (file)
--- a/regen/mk_invlists.pl
+++ b/regen/mk_invlists.pl
@@ -1135,6 +1135,7 @@ sub output_GCB_table() {
         GCB_BREAKABLE                    => 1,
         GCB_RI_then_RI                   => 2,   # Rules 12 and 13
         GCB_EX_then_EM                   => 3,   # Rule 10
+        GCB_Maybe_Emoji_NonBreak         => 4,
     );
 
     # The table is constructed in reverse order of the rules, to make the
@@ -1161,8 +1162,16 @@ sub output_GCB_table() {
     $gcb_table[$gcb_enums{'Regional_Indicator'}]
               [$gcb_enums{'Regional_Indicator'}] = $gcb_actions{GCB_RI_then_RI};
 
+    # Post 11.0: GB11  \p{Extended_Pictographic} Extend* ZWJ
+    #                                               × \p{Extended_Pictographic}
+    $gcb_table[$gcb_enums{'ZWJ'}][$gcb_enums{'XPG_XX'}] =
+                                         $gcb_actions{GCB_Maybe_Emoji_NonBreak};
+
+    # This and the rule GB10 obsolete starting with Unicode 11.0, can be left
+    # in as there are no code points that match, so the code won't ever get
+    # executed.
     # Do not break within emoji modifier sequences or emoji zwj sequences.
-    # GB11  ZWJ  × ( Glue_After_Zwj | E_Base_GAZ )
+    # Pre 11.0: GB11  ZWJ  × ( Glue_After_Zwj | E_Base_GAZ )
     $gcb_table[$gcb_enums{'ZWJ'}][$gcb_enums{'Glue_After_Zwj'}] = 0;
     $gcb_table[$gcb_enums{'ZWJ'}][$gcb_enums{'E_Base_GAZ'}] = 0;
 
@@ -1922,6 +1931,8 @@ sub output_WB_table() {
     # WB13b  ExtendNumLet  ×  (ALetter | Hebrew_Letter | Numeric | Katakana)
     $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'ALetter'}]
                                                 = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'XPG_LE'}]
+                                                = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'Hebrew_Letter'}]
                                                 = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'ExtendNumLet'}][$wb_enums{'Numeric'}]
@@ -1933,6 +1944,8 @@ sub output_WB_table() {
     #        × ExtendNumLet
     $wb_table[$wb_enums{'ALetter'}][$wb_enums{'ExtendNumLet'}]
                                                 = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'XPG_LE'}][$wb_enums{'ExtendNumLet'}]
+                                                = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'ExtendNumLet'}]
                                                 = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'Numeric'}][$wb_enums{'ExtendNumLet'}]
@@ -1969,12 +1982,16 @@ sub output_WB_table() {
     # WB10  Numeric  ×  (ALetter | Hebrew_Letter)
     $wb_table[$wb_enums{'Numeric'}][$wb_enums{'ALetter'}]
                                                 = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'Numeric'}][$wb_enums{'XPG_LE'}]
+                                                = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'Numeric'}][$wb_enums{'Hebrew_Letter'}]
                                                 = $wb_actions{'WB_NOBREAK'};
 
     # WB9  (ALetter | Hebrew_Letter)  ×  Numeric
     $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Numeric'}]
                                                 = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'XPG_LE'}][$wb_enums{'Numeric'}]
+                                                = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Numeric'}]
                                                 = $wb_actions{'WB_NOBREAK'};
 
@@ -1999,14 +2016,20 @@ sub output_WB_table() {
     #       × (ALetter | Hebrew_Letter)
     $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'ALetter'}]
                             += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+    $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'XPG_LE'}]
+                            += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
     $wb_table[$wb_enums{'MidNumLet'}][$wb_enums{'Hebrew_Letter'}]
                             += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
     $wb_table[$wb_enums{'MidLetter'}][$wb_enums{'ALetter'}]
                             += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+    $wb_table[$wb_enums{'MidLetter'}][$wb_enums{'XPG_LE'}]
+                            += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
     $wb_table[$wb_enums{'MidLetter'}][$wb_enums{'Hebrew_Letter'}]
                             += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
     $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'ALetter'}]
                             += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
+    $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'XPG_LE'}]
+                            += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
     $wb_table[$wb_enums{'Single_Quote'}][$wb_enums{'Hebrew_Letter'}]
                             += $wb_actions{'WB_MB_or_ML_or_SQ_then_LE_or_HL'};
 
@@ -2014,14 +2037,20 @@ sub output_WB_table() {
     #       | Single_Quote) (ALetter | Hebrew_Letter)
     $wb_table[$wb_enums{'ALetter'}][$wb_enums{'MidNumLet'}]
                             += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+    $wb_table[$wb_enums{'XPG_LE'}][$wb_enums{'MidNumLet'}]
+                            += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
     $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'MidNumLet'}]
                             += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
     $wb_table[$wb_enums{'ALetter'}][$wb_enums{'MidLetter'}]
                             += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+    $wb_table[$wb_enums{'XPG_LE'}][$wb_enums{'MidLetter'}]
+                            += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
     $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'MidLetter'}]
                             += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
     $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Single_Quote'}]
                             += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
+    $wb_table[$wb_enums{'XPG_LE'}][$wb_enums{'Single_Quote'}]
+                            += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
     $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Single_Quote'}]
                             += $wb_actions{'WB_LE_or_HL_then_MB_or_ML_or_SQ'};
 
@@ -2029,12 +2058,20 @@ sub output_WB_table() {
     # WB5  (ALetter | Hebrew_Letter)  ×  (ALetter | Hebrew_Letter)
     $wb_table[$wb_enums{'ALetter'}][$wb_enums{'ALetter'}]
                                                     = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'XPG_LE'}][$wb_enums{'ALetter'}]
+                                                    = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'ALetter'}][$wb_enums{'Hebrew_Letter'}]
                                                     = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'XPG_LE'}][$wb_enums{'Hebrew_Letter'}]
+                                                    = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'ALetter'}]
                                                     = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'XPG_LE'}]
+                                                    = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'Hebrew_Letter'}][$wb_enums{'Hebrew_Letter'}]
                                                     = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'XPG_LE'}][$wb_enums{'XPG_LE'}]
+                                                    = $wb_actions{'WB_NOBREAK'};
 
     # Ignore Format and Extend characters, except after sot, CR, LF, and
     # Newline.  This also has the effect of: Any × (Format | Extend | ZWJ)
@@ -2062,12 +2099,22 @@ sub output_WB_table() {
         $wb_table[$i][$wb_enums{'Format'}] = $wb_actions{'WB_NOBREAK'};
     }
 
+    # Keep horizontal whitespace together
+    # Use perl's tailoring instead
+    # WB3d WSegSpace × WSegSpace
+    #$wb_table[$wb_enums{'WSegSpace'}][$wb_enums{'WSegSpace'}]
+    #                                               = $wb_actions{'WB_NOBREAK'};
+
     # Do not break within emoji zwj sequences.
     # WB3c ZWJ × ( Glue_After_Zwj | EBG )
     $wb_table[$wb_enums{'ZWJ'}][$wb_enums{'Glue_After_Zwj'}]
                                                 = $wb_actions{'WB_NOBREAK'};
     $wb_table[$wb_enums{'ZWJ'}][$wb_enums{'E_Base_GAZ'}]
                                                 = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'ZWJ'}][$wb_enums{'XPG_XX'}]
+                                                = $wb_actions{'WB_NOBREAK'};
+    $wb_table[$wb_enums{'ZWJ'}][$wb_enums{'XPG_LE'}]
+                                                = $wb_actions{'WB_NOBREAK'};
 
     # Break before and after newlines
     # WB3b     ÷  (Newline | CR | LF)
@@ -2167,10 +2214,10 @@ my @props;
 push @props, sort { prop_name_for_cmp($a) cmp prop_name_for_cmp($b) } qw(
                     &NonL1_Perl_Non_Final_Folds
                     &UpperLatin1
-                    _Perl_GCB,E_Base,E_Base_GAZ,E_Modifier,Glue_After_Zwj,LV,Prepend,Regional_Indicator,SpacingMark,ZWJ,EDGE
+                    _Perl_GCB,E_Base,E_Base_GAZ,E_Modifier,Glue_After_Zwj,LV,Prepend,Regional_Indicator,SpacingMark,ZWJ,XPG_XX,EDGE
                     _Perl_LB,Close_Parenthesis,Hebrew_Letter,Next_Line,Regional_Indicator,ZWJ,Contingent_Break,E_Base,E_Modifier,H2,H3,JL,JT,JV,Word_Joiner,EDGE
                     _Perl_SB,SContinue,CR,Extend,LF,EDGE
-                    _Perl_WB,CR,Double_Quote,E_Base,E_Base_GAZ,E_Modifier,Extend,Glue_After_Zwj,Hebrew_Letter,LF,MidNumLet,Newline,Regional_Indicator,Single_Quote,ZWJ,EDGE,UNKNOWN
+                    _Perl_WB,CR,Double_Quote,E_Base,E_Base_GAZ,E_Modifier,Extend,Glue_After_Zwj,Hebrew_Letter,LF,MidNumLet,Newline,Regional_Indicator,Single_Quote,ZWJ,XPG_XX,XPG_LE,Perl_Tailored_HSpace,EDGE,UNKNOWN
                     _Perl_SCX,Latin,Inherited,Unknown,Kore,Jpan,Hanb,INVALID
                     Lowercase_Mapping
                     Titlecase_Mapping

diff --git a/regexec.c b/regexec.c
index 45084be..d0c4e6b 100644 (file)
--- a/regexec.c
+++ b/regexec.c
@@ -4699,6 +4699,24 @@ S_isGCB(pTHX_ const GCB_enum before, const GCB_enum after, const U8 * const strb
                 return prev != GCB_E_Base && prev != GCB_E_Base_GAZ;
             }
 
+        case GCB_Maybe_Emoji_NonBreak:
+
+            {
+
+            /* Do not break within emoji modifier sequences or emoji zwj sequences.
+              GB11 \p{Extended_Pictographic} Extend* ZWJ × \p{Extended_Pictographic}
+              */
+                U8 * temp_pos = (U8 *) curpos;
+                GCB_enum prev;
+
+                do {
+                    prev = backup_one_GCB(strbeg, &temp_pos, utf8_target);
+                }
+                while (prev == GCB_Extend);
+
+                return prev != GCB_XPG_XX;
+            }
+
         default:
             break;
     }

diff --git a/uni_keywords.h b/uni_keywords.h
index 2b22ac3..1c2e2dc 100644 (file)
--- a/uni_keywords.h
+++ b/uni_keywords.h
@@ -6834,10 +6834,10 @@ MPH_VALt match_uniprop( const unsigned char * const key, const U16 key_len ) {
  * be0f129691d479aa38646e4ca0ec1ee576ae7f75b0300a5624a7fa862fa8abba lib/unicore/extracted/DLineBreak.txt
  * 92449d354d9f6b6f2f97a292ebb59f6344ffdeb83d120d7d23e569c43ba67cd5 lib/unicore/extracted/DNumType.txt
  * e3a319527153b0c6c0c549b40fc6f3a01a7a0dcd6620784391db25901df3b154 lib/unicore/extracted/DNumValues.txt
- * 5c4387f0eb0a4d3ec12ce24309495ba2fe2ff8d6c20e11eb19258ee347a83e5d lib/unicore/mktables
+ * 22c0974193811f3f52d01b33c394c22b87541385fb68d5a3502749ec6bf077aa lib/unicore/mktables
  * 21653d2744fdd071f9ef138c805393901bb9547cf3e777ebf50215a191f986ea lib/unicore/version
  * 4bb677187a1a64e39d48f2e341b5ecb6c99857e49d7a79cf503bd8a3c709999b regen/charset_translations.pl
  * 03e51b0f07beebd5da62ab943899aa4934eee1f792fa27c1fb638c33bf4ac6ea regen/mk_PL_charclass.pl
- * b015b9e6db7b31e919c494042c9c4b81f2b77352ff2a03a0b123f09a5818f8dd regen/mk_invlists.pl
+ * 507fb07e9b2257fe365525cc485c02ca756f13ca2186412c8792f1d03496313f regen/mk_invlists.pl
  * 1adf61210fe457d9fe2e953f9646cac257b1c00226a84435d9c2642051b81024 regen/mph.pl
  * ex: set ro: */