This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
APItest/t/utf8_warn_base.pl: Display mnemonics on error
[perl5.git] / ext / XS-APItest / t / utf8_warn_base.pl
CommitLineData
6aa905cf
KW
1#!perl -w
2
3# This is a base file to be used by various .t's in its directory
4
5use strict;
6use Test::More;
7
8BEGIN {
9 use_ok('XS::APItest');
10 require 'charset_tools.pl';
11 require './t/utf8_setup.pl';
12};
13
14$|=1;
15
16no warnings 'deprecated'; # Some of the below are above IV_MAX on 32 bit
17 # machines, and that is tested elsewhere
18
19use XS::APItest;
20use Data::Dumper;
21
9cdc3054 22my @warnings_gotten;
6aa905cf
KW
23
24use warnings 'utf8';
9cdc3054 25local $SIG{__WARN__} = sub { push @warnings_gotten, @_ };
6aa905cf
KW
26
27sub nonportable_regex ($) {
28
29 # Returns a pattern that matches the non-portable message raised either
30 # for the specific input code point, or the one generated when there
31 # is some malformation that precludes the message containing the specific
32 # code point
33
34 my $code_point = shift;
35
36 my $string = sprintf '(Code point 0x%X is not Unicode, and'
37 . '|Any UTF-8 sequence that starts with'
38 . ' "(\\\x[[:xdigit:]]{2})+" is for a'
39 . ' non-Unicode code point, and is) not portable',
40 $code_point;
41 return qr/$string/;
42}
43
44# Now test the cases where a legal code point is generated, but may or may not
45# be allowed/warned on.
46my @tests = (
9cdc3054 47 # ($testname, $bytes, $disallow_flags, $controlling_warning_category,
b7e1f4b2 48 # $allowed_uv, $needed_to_discern_len )
6aa905cf
KW
49 [ "lowest surrogate",
50 (isASCII) ? "\xed\xa0\x80" : I8_to_native("\xf1\xb6\xa0\xa0"),
b7e1f4b2 51 $::UTF8_DISALLOW_SURROGATE,
6aa905cf 52 'surrogate', 0xD800,
6aa905cf
KW
53 ],
54 [ "a middle surrogate",
55 (isASCII) ? "\xed\xa4\x8d" : I8_to_native("\xf1\xb6\xa8\xad"),
b7e1f4b2 56 $::UTF8_DISALLOW_SURROGATE,
6aa905cf 57 'surrogate', 0xD90D,
6aa905cf
KW
58 ],
59 [ "highest surrogate",
60 (isASCII) ? "\xed\xbf\xbf" : I8_to_native("\xf1\xb7\xbf\xbf"),
b7e1f4b2 61 $::UTF8_DISALLOW_SURROGATE,
6aa905cf 62 'surrogate', 0xDFFF,
6aa905cf
KW
63 ],
64 [ "first non_unicode",
65 (isASCII) ? "\xf4\x90\x80\x80" : I8_to_native("\xf9\xa2\xa0\xa0\xa0"),
b7e1f4b2 66 $::UTF8_DISALLOW_SUPER,
6aa905cf 67 'non_unicode', 0x110000,
6aa905cf 68 2,
6aa905cf
KW
69 ],
70 [ "non_unicode whose first byte tells that",
71 (isASCII) ? "\xf5\x80\x80\x80" : I8_to_native("\xfa\xa0\xa0\xa0\xa0"),
b7e1f4b2 72 $::UTF8_DISALLOW_SUPER,
6aa905cf
KW
73 'non_unicode',
74 (isASCII) ? 0x140000 : 0x200000,
6aa905cf 75 1,
6aa905cf
KW
76 ],
77 [ "first of 32 consecutive non-character code points",
78 (isASCII) ? "\xef\xb7\x90" : I8_to_native("\xf1\xbf\xae\xb0"),
b7e1f4b2 79 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 80 'nonchar', 0xFDD0,
6aa905cf
KW
81 ],
82 [ "a mid non-character code point of the 32 consecutive ones",
83 (isASCII) ? "\xef\xb7\xa0" : I8_to_native("\xf1\xbf\xaf\xa0"),
b7e1f4b2 84 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 85 'nonchar', 0xFDE0,
6aa905cf
KW
86 ],
87 [ "final of 32 consecutive non-character code points",
88 (isASCII) ? "\xef\xb7\xaf" : I8_to_native("\xf1\xbf\xaf\xaf"),
b7e1f4b2 89 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 90 'nonchar', 0xFDEF,
6aa905cf
KW
91 ],
92 [ "non-character code point U+FFFE",
93 (isASCII) ? "\xef\xbf\xbe" : I8_to_native("\xf1\xbf\xbf\xbe"),
b7e1f4b2 94 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 95 'nonchar', 0xFFFE,
6aa905cf
KW
96 ],
97 [ "non-character code point U+FFFF",
98 (isASCII) ? "\xef\xbf\xbf" : I8_to_native("\xf1\xbf\xbf\xbf"),
b7e1f4b2 99 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 100 'nonchar', 0xFFFF,
6aa905cf
KW
101 ],
102 [ "non-character code point U+1FFFE",
103 (isASCII) ? "\xf0\x9f\xbf\xbe" : I8_to_native("\xf3\xbf\xbf\xbe"),
b7e1f4b2 104 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 105 'nonchar', 0x1FFFE,
6aa905cf
KW
106 ],
107 [ "non-character code point U+1FFFF",
108 (isASCII) ? "\xf0\x9f\xbf\xbf" : I8_to_native("\xf3\xbf\xbf\xbf"),
b7e1f4b2 109 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 110 'nonchar', 0x1FFFF,
6aa905cf
KW
111 ],
112 [ "non-character code point U+2FFFE",
113 (isASCII) ? "\xf0\xaf\xbf\xbe" : I8_to_native("\xf5\xbf\xbf\xbe"),
b7e1f4b2 114 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 115 'nonchar', 0x2FFFE,
6aa905cf
KW
116 ],
117 [ "non-character code point U+2FFFF",
118 (isASCII) ? "\xf0\xaf\xbf\xbf" : I8_to_native("\xf5\xbf\xbf\xbf"),
b7e1f4b2 119 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 120 'nonchar', 0x2FFFF,
6aa905cf
KW
121 ],
122 [ "non-character code point U+3FFFE",
123 (isASCII) ? "\xf0\xbf\xbf\xbe" : I8_to_native("\xf7\xbf\xbf\xbe"),
b7e1f4b2 124 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 125 'nonchar', 0x3FFFE,
6aa905cf
KW
126 ],
127 [ "non-character code point U+3FFFF",
128 (isASCII) ? "\xf0\xbf\xbf\xbf" : I8_to_native("\xf7\xbf\xbf\xbf"),
b7e1f4b2 129 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 130 'nonchar', 0x3FFFF,
6aa905cf
KW
131 ],
132 [ "non-character code point U+4FFFE",
133 (isASCII) ? "\xf1\x8f\xbf\xbe" : I8_to_native("\xf8\xa9\xbf\xbf\xbe"),
b7e1f4b2 134 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 135 'nonchar', 0x4FFFE,
6aa905cf
KW
136 ],
137 [ "non-character code point U+4FFFF",
138 (isASCII) ? "\xf1\x8f\xbf\xbf" : I8_to_native("\xf8\xa9\xbf\xbf\xbf"),
b7e1f4b2 139 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 140 'nonchar', 0x4FFFF,
6aa905cf
KW
141 ],
142 [ "non-character code point U+5FFFE",
143 (isASCII) ? "\xf1\x9f\xbf\xbe" : I8_to_native("\xf8\xab\xbf\xbf\xbe"),
b7e1f4b2 144 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 145 'nonchar', 0x5FFFE,
6aa905cf
KW
146 ],
147 [ "non-character code point U+5FFFF",
148 (isASCII) ? "\xf1\x9f\xbf\xbf" : I8_to_native("\xf8\xab\xbf\xbf\xbf"),
b7e1f4b2 149 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 150 'nonchar', 0x5FFFF,
6aa905cf
KW
151 ],
152 [ "non-character code point U+6FFFE",
153 (isASCII) ? "\xf1\xaf\xbf\xbe" : I8_to_native("\xf8\xad\xbf\xbf\xbe"),
b7e1f4b2 154 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 155 'nonchar', 0x6FFFE,
6aa905cf
KW
156 ],
157 [ "non-character code point U+6FFFF",
158 (isASCII) ? "\xf1\xaf\xbf\xbf" : I8_to_native("\xf8\xad\xbf\xbf\xbf"),
b7e1f4b2 159 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 160 'nonchar', 0x6FFFF,
6aa905cf
KW
161 ],
162 [ "non-character code point U+7FFFE",
163 (isASCII) ? "\xf1\xbf\xbf\xbe" : I8_to_native("\xf8\xaf\xbf\xbf\xbe"),
b7e1f4b2 164 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 165 'nonchar', 0x7FFFE,
6aa905cf
KW
166 ],
167 [ "non-character code point U+7FFFF",
168 (isASCII) ? "\xf1\xbf\xbf\xbf" : I8_to_native("\xf8\xaf\xbf\xbf\xbf"),
b7e1f4b2 169 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 170 'nonchar', 0x7FFFF,
6aa905cf
KW
171 ],
172 [ "non-character code point U+8FFFE",
173 (isASCII) ? "\xf2\x8f\xbf\xbe" : I8_to_native("\xf8\xb1\xbf\xbf\xbe"),
b7e1f4b2 174 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 175 'nonchar', 0x8FFFE,
6aa905cf
KW
176 ],
177 [ "non-character code point U+8FFFF",
178 (isASCII) ? "\xf2\x8f\xbf\xbf" : I8_to_native("\xf8\xb1\xbf\xbf\xbf"),
b7e1f4b2 179 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 180 'nonchar', 0x8FFFF,
6aa905cf
KW
181 ],
182 [ "non-character code point U+9FFFE",
183 (isASCII) ? "\xf2\x9f\xbf\xbe" : I8_to_native("\xf8\xb3\xbf\xbf\xbe"),
b7e1f4b2 184 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 185 'nonchar', 0x9FFFE,
6aa905cf
KW
186 ],
187 [ "non-character code point U+9FFFF",
188 (isASCII) ? "\xf2\x9f\xbf\xbf" : I8_to_native("\xf8\xb3\xbf\xbf\xbf"),
b7e1f4b2 189 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 190 'nonchar', 0x9FFFF,
6aa905cf
KW
191 ],
192 [ "non-character code point U+AFFFE",
193 (isASCII) ? "\xf2\xaf\xbf\xbe" : I8_to_native("\xf8\xb5\xbf\xbf\xbe"),
b7e1f4b2 194 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 195 'nonchar', 0xAFFFE,
6aa905cf
KW
196 ],
197 [ "non-character code point U+AFFFF",
198 (isASCII) ? "\xf2\xaf\xbf\xbf" : I8_to_native("\xf8\xb5\xbf\xbf\xbf"),
b7e1f4b2 199 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 200 'nonchar', 0xAFFFF,
6aa905cf
KW
201 ],
202 [ "non-character code point U+BFFFE",
203 (isASCII) ? "\xf2\xbf\xbf\xbe" : I8_to_native("\xf8\xb7\xbf\xbf\xbe"),
b7e1f4b2 204 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 205 'nonchar', 0xBFFFE,
6aa905cf
KW
206 ],
207 [ "non-character code point U+BFFFF",
208 (isASCII) ? "\xf2\xbf\xbf\xbf" : I8_to_native("\xf8\xb7\xbf\xbf\xbf"),
b7e1f4b2 209 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 210 'nonchar', 0xBFFFF,
6aa905cf
KW
211 ],
212 [ "non-character code point U+CFFFE",
213 (isASCII) ? "\xf3\x8f\xbf\xbe" : I8_to_native("\xf8\xb9\xbf\xbf\xbe"),
b7e1f4b2 214 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 215 'nonchar', 0xCFFFE,
6aa905cf
KW
216 ],
217 [ "non-character code point U+CFFFF",
218 (isASCII) ? "\xf3\x8f\xbf\xbf" : I8_to_native("\xf8\xb9\xbf\xbf\xbf"),
b7e1f4b2 219 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 220 'nonchar', 0xCFFFF,
6aa905cf
KW
221 ],
222 [ "non-character code point U+DFFFE",
223 (isASCII) ? "\xf3\x9f\xbf\xbe" : I8_to_native("\xf8\xbb\xbf\xbf\xbe"),
b7e1f4b2 224 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 225 'nonchar', 0xDFFFE,
6aa905cf
KW
226 ],
227 [ "non-character code point U+DFFFF",
228 (isASCII) ? "\xf3\x9f\xbf\xbf" : I8_to_native("\xf8\xbb\xbf\xbf\xbf"),
b7e1f4b2 229 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 230 'nonchar', 0xDFFFF,
6aa905cf
KW
231 ],
232 [ "non-character code point U+EFFFE",
233 (isASCII) ? "\xf3\xaf\xbf\xbe" : I8_to_native("\xf8\xbd\xbf\xbf\xbe"),
b7e1f4b2 234 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 235 'nonchar', 0xEFFFE,
6aa905cf
KW
236 ],
237 [ "non-character code point U+EFFFF",
238 (isASCII) ? "\xf3\xaf\xbf\xbf" : I8_to_native("\xf8\xbd\xbf\xbf\xbf"),
b7e1f4b2 239 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 240 'nonchar', 0xEFFFF,
6aa905cf
KW
241 ],
242 [ "non-character code point U+FFFFE",
243 (isASCII) ? "\xf3\xbf\xbf\xbe" : I8_to_native("\xf8\xbf\xbf\xbf\xbe"),
b7e1f4b2 244 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 245 'nonchar', 0xFFFFE,
6aa905cf
KW
246 ],
247 [ "non-character code point U+FFFFF",
248 (isASCII) ? "\xf3\xbf\xbf\xbf" : I8_to_native("\xf8\xbf\xbf\xbf\xbf"),
b7e1f4b2 249 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 250 'nonchar', 0xFFFFF,
6aa905cf
KW
251 ],
252 [ "non-character code point U+10FFFE",
253 (isASCII) ? "\xf4\x8f\xbf\xbe" : I8_to_native("\xf9\xa1\xbf\xbf\xbe"),
b7e1f4b2 254 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 255 'nonchar', 0x10FFFE,
6aa905cf
KW
256 ],
257 [ "non-character code point U+10FFFF",
258 (isASCII) ? "\xf4\x8f\xbf\xbf" : I8_to_native("\xf9\xa1\xbf\xbf\xbf"),
b7e1f4b2 259 $::UTF8_DISALLOW_NONCHAR,
6aa905cf 260 'nonchar', 0x10FFFF,
6aa905cf 261 ],
76513bdc
KW
262 [ "requires at least 32 bits",
263 (isASCII)
264 ? "\xfe\x82\x80\x80\x80\x80\x80"
265 : I8_to_native(
266 "\xff\xa0\xa0\xa0\xa0\xa0\xa0\xa2\xa0\xa0\xa0\xa0\xa0\xa0"),
267 # This code point is chosen so that it is representable in a UV on
268 # 32-bit machines
b7e1f4b2 269 $::UTF8_DISALLOW_ABOVE_31_BIT,
76513bdc 270 'utf8', 0x80000000,
76513bdc 271 (isASCII) ? 1 : 8,
76513bdc
KW
272 ],
273 [ "highest 32 bit code point",
274 (isASCII)
275 ? "\xfe\x83\xbf\xbf\xbf\xbf\xbf"
276 : I8_to_native(
277 "\xff\xa0\xa0\xa0\xa0\xa0\xa0\xa3\xbf\xbf\xbf\xbf\xbf\xbf"),
b7e1f4b2 278 $::UTF8_DISALLOW_ABOVE_31_BIT,
76513bdc 279 'utf8', 0xFFFFFFFF,
76513bdc 280 (isASCII) ? 1 : 8,
76513bdc
KW
281 ],
282 [ "requires at least 32 bits, and use SUPER-type flags, instead of"
283 . " ABOVE_31_BIT",
284 (isASCII)
285 ? "\xfe\x82\x80\x80\x80\x80\x80"
286 : I8_to_native(
287 "\xff\xa0\xa0\xa0\xa0\xa0\xa0\xa2\xa0\xa0\xa0\xa0\xa0\xa0"),
b7e1f4b2 288 $::UTF8_DISALLOW_SUPER,
76513bdc 289 'utf8', 0x80000000,
76513bdc 290 1,
76513bdc
KW
291 ],
292 [ "overflow with warnings/disallow for more than 31 bits",
293 # This tests the interaction of WARN_ABOVE_31_BIT/DISALLOW_ABOVE_31_BIT
294 # with overflow. The overflow malformation is never allowed, so
295 # preventing it takes precedence if the ABOVE_31_BIT options would
296 # otherwise allow in an overflowing value. The ASCII code points (1
297 # for 32-bits; 1 for 64) were chosen because the old overflow
298 # detection algorithm did not catch them; this means this test also
299 # checks for that fix. The EBCDIC are arbitrary overflowing ones
300 # since we have no reports of failures with it.
301 (($::is64bit)
302 ? ((isASCII)
303 ? "\xff\x80\x90\x90\x90\xbf\xbf\xbf\xbf\xbf\xbf\xbf\xbf"
304 : I8_to_native(
305 "\xff\xB0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0"))
306 : ((isASCII)
307 ? "\xfe\x86\x80\x80\x80\x80\x80"
308 : I8_to_native(
309 "\xff\xa0\xa0\xa0\xa0\xa0\xa0\xa4\xa0\xa0\xa0\xa0\xa0\xa0"))),
76513bdc 310 $::UTF8_DISALLOW_ABOVE_31_BIT,
04f42bf6 311 'utf8', -1,
76513bdc 312 (isASCII || $::is64bit) ? 2 : 8,
76513bdc 313 ],
6aa905cf
KW
314);
315
316if (! $::is64bit) {
317 if (isASCII) {
318 no warnings qw{portable overflow};
319 push @tests,
320 [ "Lowest 33 bit code point: overflow",
321 "\xFE\x84\x80\x80\x80\x80\x80",
b7e1f4b2 322 $::UTF8_DISALLOW_ABOVE_31_BIT,
04f42bf6 323 'utf8', -1,
b3169593 324 1,
6aa905cf
KW
325 ];
326 }
327}
328else {
329 no warnings qw{portable overflow};
330 push @tests,
331 [ "More than 32 bits",
332 (isASCII)
333 ? "\xff\x80\x80\x80\x80\x80\x81\x80\x80\x80\x80\x80\x80"
334 : I8_to_native(
335 "\xff\xa0\xa0\xa0\xa0\xa0\xa2\xa0\xa0\xa0\xa0\xa0\xa0\xa0"),
b7e1f4b2 336 $::UTF8_DISALLOW_ABOVE_31_BIT,
6aa905cf 337 'utf8', 0x1000000000,
b3169593 338 (isASCII) ? 1 : 7,
6aa905cf
KW
339 ];
340 if (! isASCII) {
341 push @tests, # These could falsely show wrongly in a naive
342 # implementation
343 [ "requires at least 32 bits",
344 I8_to_native(
345 "\xff\xa0\xa0\xa0\xa0\xa0\xa1\xa0\xa0\xa0\xa0\xa0\xa0\xa0"),
b7e1f4b2 346 $::UTF8_DISALLOW_ABOVE_31_BIT,
6aa905cf 347 'utf8', 0x800000000,
b3169593 348 7,
6aa905cf
KW
349 ],
350 [ "requires at least 32 bits",
351 I8_to_native(
352 "\xff\xa0\xa0\xa0\xa0\xa1\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0"),
b7e1f4b2 353 $::UTF8_DISALLOW_ABOVE_31_BIT,
6aa905cf 354 'utf8', 0x10000000000,
b3169593 355 6,
6aa905cf
KW
356 ],
357 [ "requires at least 32 bits",
358 I8_to_native(
359 "\xff\xa0\xa0\xa0\xa1\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0"),
b7e1f4b2 360 $::UTF8_DISALLOW_ABOVE_31_BIT,
6aa905cf 361 'utf8', 0x200000000000,
b3169593 362 5,
6aa905cf
KW
363 ],
364 [ "requires at least 32 bits",
365 I8_to_native(
366 "\xff\xa0\xa0\xa1\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0"),
b7e1f4b2 367 $::UTF8_DISALLOW_ABOVE_31_BIT,
6aa905cf 368 'utf8', 0x4000000000000,
b3169593 369 4,
6aa905cf
KW
370 ],
371 [ "requires at least 32 bits",
372 I8_to_native(
373 "\xff\xa0\xa1\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0"),
b7e1f4b2 374 $::UTF8_DISALLOW_ABOVE_31_BIT,
6aa905cf 375 'utf8', 0x80000000000000,
b3169593 376 3,
6aa905cf
KW
377 ],
378 [ "requires at least 32 bits",
379 I8_to_native(
380 "\xff\xa1\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0\xa0"),
b7e1f4b2 381 $::UTF8_DISALLOW_ABOVE_31_BIT,
6aa905cf 382 'utf8', 0x1000000000000000,
b3169593 383 2,
6aa905cf
KW
384 ];
385 }
386}
387
5722c46d
KW
388sub flags_to_text($$)
389{
390 my ($flags, $flags_to_text_ref) = @_;
391
392 # Returns a string containing a mnemonic representation of the bits that
393 # are set in the $flags. These are assumed to be flag bits. The return
394 # looks like "FOO|BAR|BAZ". The second parameter is a reference to an
395 # array that gives the textual representation of all the possible flags.
396 # Element 0 is the text for the bit 0 flag; element 1 for bit 1; .... If
397 # no bits at all are set the string "0" is returned;
398
399 my @flag_text;
400 my $shift = 0;
401
402 return "0" if $flags == 0;
403
404 while ($flags) {
405 #diag sprintf "%x", $flags;
406 if ($flags & 1) {
407 push @flag_text, $flags_to_text_ref->[$shift];
408 }
409 $shift++;
410 $flags >>= 1;
411 }
412
413 return join "|", @flag_text;
414}
415
416# Possible flag returns from utf8n_to_uvchr_error(). These should have G_,
417# instead of A_, D_, but the prefixes will be used in a a later commit, so
418# minimize churn by having them here.
419my @utf8n_flags_to_text = ( qw(
420 A_EMPTY
421 A_CONTINUATION
422 A_NON_CONTINUATION
423 A_SHORT
424 A_LONG
425 A_LONG_AND_ITS_VALUE
426 PLACEHOLDER
427 A_OVERFLOW
428 D_SURROGATE
429 W_SURROGATE
430 D_NONCHAR
431 W_NONCHAR
432 D_SUPER
433 W_SUPER
434 D_ABOVE_31_BIT
435 W_ABOVE_31_BIT
436 CHECK_ONLY
437 NO_CONFIDENCE_IN_CURLEN_
438 ) );
439
6aa905cf
KW
440# This test is split into this number of files.
441my $num_test_files = $ENV{TEST_JOBS} || 1;
442$num_test_files = 10 if $num_test_files > 10;
443
444my $test_count = -1;
445foreach my $test (@tests) {
446 $test_count++;
447 next if $test_count % $num_test_files != $::TEST_CHUNK;
448
b7e1f4b2 449 my ($testname, $bytes, $disallow_flags,
9cdc3054 450 $controlling_warning_category, $allowed_uv, $needed_to_discern_len
6aa905cf
KW
451 ) = @$test;
452
453 my $length = length $bytes;
04f42bf6 454 my $will_overflow = $allowed_uv < 0;
6aa905cf 455
b7e1f4b2
KW
456 # The convention is that the got flag is the same value as the disallow
457 # one, and the warn flag is the next bit over. If this were violated, the
458 # tests here should start failing. We could do an eval under no strict to
459 # be sure.
460 my $expected_error_flags = $disallow_flags;
461 my $warn_flags = $disallow_flags << 1;
462
3022ad00
KW
463 my $message;
464 if ($allowed_uv > 0x7FFFFFFF) {
465 $message = nonportable_regex($allowed_uv);
466 }
467 elsif ($allowed_uv > 0x10FFFF) {
468 $message = qr/(not Unicode|for a non-Unicode code point).* may not be portable/;
469 }
470 elsif ($allowed_uv >= 0xD800 && $allowed_uv <= 0xDFFF) {
471 $message = qr/surrogate/;
67e45424 472 $needed_to_discern_len = 2 unless defined $needed_to_discern_len;
3022ad00
KW
473 }
474 elsif ( ($allowed_uv >= 0xFDD0 && $allowed_uv <= 0xFDEF)
475 || ($allowed_uv & 0xFFFE) == 0xFFFE)
476 {
477 $message = qr/Unicode non-character.*is not recommended for open interchange/;
67e45424 478 $needed_to_discern_len = $length unless defined $needed_to_discern_len;
3022ad00
KW
479 }
480 elsif ($will_overflow) {
481 $message = qr/overflows/;
482 }
483 else {
484 die "Can't figure out what type of warning to test for $testname"
485 }
486
67e45424
KW
487 die 'Didn\'t set $needed_to_discern_len for ' . $testname
488 unless defined $needed_to_discern_len;
489
6aa905cf
KW
490 {
491 use warnings;
9cdc3054 492 undef @warnings_gotten;
6aa905cf
KW
493 my $ret = test_isUTF8_CHAR($bytes, $length);
494 my $ret_flags = test_isUTF8_CHAR_flags($bytes, $length, 0);
495 if ($will_overflow) {
496 is($ret, 0, "isUTF8_CHAR() $testname: returns 0");
497 is($ret_flags, 0, "isUTF8_CHAR_flags() $testname: returns 0");
498 }
499 else {
500 is($ret, $length,
501 "isUTF8_CHAR() $testname: returns expected length: $length");
502 is($ret_flags, $length, "isUTF8_CHAR_flags(...,0) $testname:"
503 . " returns expected length: $length");
504 }
9cdc3054 505 is(scalar @warnings_gotten, 0,
6aa905cf
KW
506 "isUTF8_CHAR() and isUTF8_CHAR()_flags $testname: generated"
507 . " no warnings")
9cdc3054 508 or output_warnings(@warnings_gotten);
6aa905cf 509
9cdc3054 510 undef @warnings_gotten;
6aa905cf
KW
511 $ret = test_isSTRICT_UTF8_CHAR($bytes, $length);
512 if ($will_overflow) {
513 is($ret, 0, "isSTRICT_UTF8_CHAR() $testname: returns 0");
514 }
515 else {
516 my $expected_ret = ( $testname =~ /surrogate|non-character/
517 || $allowed_uv > 0x10FFFF)
518 ? 0
519 : $length;
520 is($ret, $expected_ret, "isSTRICT_UTF8_CHAR() $testname: returns"
521 . " expected length: $expected_ret");
522 $ret = test_isUTF8_CHAR_flags($bytes, $length,
523 $::UTF8_DISALLOW_ILLEGAL_INTERCHANGE);
524 is($ret, $expected_ret,
525 "isUTF8_CHAR_flags('DISALLOW_ILLEGAL_INTERCHANGE')"
526 . " acts like isSTRICT_UTF8_CHAR");
527 }
9cdc3054 528 is(scalar @warnings_gotten, 0,
6aa905cf
KW
529 "isSTRICT_UTF8_CHAR() and isUTF8_CHAR_flags $testname:"
530 . " generated no warnings")
9cdc3054 531 or output_warnings(@warnings_gotten);
6aa905cf 532
9cdc3054 533 undef @warnings_gotten;
6aa905cf
KW
534 $ret = test_isC9_STRICT_UTF8_CHAR($bytes, $length);
535 if ($will_overflow) {
536 is($ret, 0, "isC9_STRICT_UTF8_CHAR() $testname: returns 0");
537 }
538 else {
539 my $expected_ret = ( $testname =~ /surrogate/
540 || $allowed_uv > 0x10FFFF)
541 ? 0
542 : $length;
543 is($ret, $expected_ret, "isC9_STRICT_UTF8_CHAR() $testname:"
544 ." returns expected length: $expected_ret");
545 $ret = test_isUTF8_CHAR_flags($bytes, $length,
546 $::UTF8_DISALLOW_ILLEGAL_C9_INTERCHANGE);
547 is($ret, $expected_ret,
548 "isUTF8_CHAR_flags('DISALLOW_ILLEGAL_C9_INTERCHANGE')"
549 . " acts like isC9_STRICT_UTF8_CHAR");
550 }
9cdc3054 551 is(scalar @warnings_gotten, 0,
6aa905cf
KW
552 "isC9_STRICT_UTF8_CHAR() and isUTF8_CHAR_flags $testname:"
553 . " generated no warnings")
9cdc3054 554 or output_warnings(@warnings_gotten);
6aa905cf
KW
555
556 # Test partial character handling, for each byte not a full character
557 for my $j (1.. $length - 1) {
558
559 # Skip the test for the interaction between overflow and above-31
560 # bit. It is really testing other things than the partial
561 # character tests, for which other tests in this file are
562 # sufficient
04f42bf6 563 last if $will_overflow;
6aa905cf
KW
564
565 foreach my $disallow_flag (0, $disallow_flags) {
566 my $partial = substr($bytes, 0, $j);
567 my $ret_should_be;
568 my $comment;
569 if ($disallow_flag) {
570 $ret_should_be = 0;
571 $comment = "disallowed";
572 if ($j < $needed_to_discern_len) {
573 $ret_should_be = 1;
574 $comment .= ", but need $needed_to_discern_len bytes"
575 . " to discern:";
576 }
577 }
578 else {
579 $ret_should_be = 1;
580 $comment = "allowed";
581 }
582
9cdc3054 583 undef @warnings_gotten;
6aa905cf
KW
584
585 $ret = test_is_utf8_valid_partial_char_flags($partial, $j,
586 $disallow_flag);
587 is($ret, $ret_should_be,
588 "$testname: is_utf8_valid_partial_char_flags("
589 . display_bytes($partial)
590 . "), $comment: returns $ret_should_be");
9cdc3054 591 is(scalar @warnings_gotten, 0,
6aa905cf
KW
592 "$testname: is_utf8_valid_partial_char_flags()"
593 . " generated no warnings")
9cdc3054 594 or output_warnings(@warnings_gotten);
6aa905cf
KW
595 }
596 }
597 }
598
599 # This is more complicated than the malformations tested earlier, as there
600 # are several orthogonal variables involved. We test all the subclasses
601 # of utf8 warnings to verify they work with and without the utf8 class,
602 # and don't have effects on other sublass warnings
9cdc3054 603 foreach my $trial_warning_category ('utf8', 'surrogate', 'nonchar', 'non_unicode') {
6aa905cf
KW
604 foreach my $warn_flag (0, $warn_flags) {
605 foreach my $disallow_flag (0, $disallow_flags) {
606 foreach my $do_warning (0, 1) {
607
608 # We try each of the above with various combinations of
609 # malformations that can occur on the same input sequence.
610 foreach my $short ("", "short") {
611 foreach my $unexpected_noncont ("",
612 "unexpected non-continuation")
613 {
614 foreach my $overlong ("", "overlong") {
615
b3169593
KW
616 # If we're creating an overlong, it can't be longer than
617 # the maximum length, so skip if we're already at that
618 # length.
619 next if $overlong && $length >= $::max_bytes;
6aa905cf
KW
620
621 my @malformations;
9cdc3054 622 my @expected_return_flags;
6aa905cf
KW
623 push @malformations, $short if $short;
624 push @malformations, $unexpected_noncont
625 if $unexpected_noncont;
626 push @malformations, $overlong if $overlong;
627
628 # The overflow malformation test in the input
629 # array is coerced into being treated like one of
630 # the others.
631 if ($will_overflow) {
632 push @malformations, 'overflow';
9cdc3054 633 push @expected_return_flags, $::UTF8_GOT_OVERFLOW;
6aa905cf
KW
634 }
635
636 my $malformations_name = join "/", @malformations;
637 $malformations_name .= " malformation"
638 if $malformations_name;
639 $malformations_name .= "s" if @malformations > 1;
640 my $this_bytes = $bytes;
641 my $this_length = $length;
642 my $expected_uv = $allowed_uv;
b3169593 643 my $this_expected_len = $length;
6aa905cf
KW
644 my $this_needed_to_discern_len = $needed_to_discern_len;
645 if ($malformations_name) {
646 $expected_uv = 0;
647
648 # Coerce the input into the desired
649 # malformation
650 if ($malformations_name =~ /overlong/) {
651
652 # For an overlong, we convert the original
653 # start byte into a continuation byte with
654 # the same data bits as originally. ...
655 substr($this_bytes, 0, 1)
656 = start_byte_to_cont(substr($this_bytes,
657 0, 1));
658
659 # ... Then we prepend it with a known
660 # overlong sequence. This should evaluate
661 # to the exact same code point as the
662 # original.
663 $this_bytes
664 = I8_to_native("\xff")
dbb8d798 665 . (I8_to_native(chr $::lowest_continuation)
6aa905cf
KW
666 x ( $::max_bytes - 1 - length($this_bytes)))
667 . $this_bytes;
668 $this_length = length($this_bytes);
669 $this_needed_to_discern_len
670 = $::max_bytes - ($this_expected_len
671 - $this_needed_to_discern_len);
672 $this_expected_len = $::max_bytes;
9cdc3054 673 push @expected_return_flags, $::UTF8_GOT_LONG;
6aa905cf
KW
674 }
675 if ($malformations_name =~ /short/) {
676
677 # Just tell the test to not look far
678 # enough into the input.
679 $this_length--;
680 $this_expected_len--;
9cdc3054 681 push @expected_return_flags, $::UTF8_GOT_SHORT;
6aa905cf
KW
682 }
683 if ($malformations_name
684 =~ /non-continuation/)
685 {
686 # Change the final continuation byte into
687 # a non one.
688 my $pos = ($short) ? -2 : -1;
689 substr($this_bytes, $pos, 1) = '?';
690 $this_expected_len--;
9cdc3054 691 push @expected_return_flags,
6aa905cf
KW
692 $::UTF8_GOT_NON_CONTINUATION;
693 }
694 }
695
696 my $eval_warn = $do_warning
9cdc3054
KW
697 ? "use warnings '$trial_warning_category'"
698 : $trial_warning_category eq "utf8"
6aa905cf
KW
699 ? "no warnings 'utf8'"
700 : ( "use warnings 'utf8';"
9cdc3054 701 . " no warnings '$trial_warning_category'");
6aa905cf
KW
702
703 # Is effectively disallowed if we've set up a
704 # malformation, even if the flag indicates it is
705 # allowed. Fix up test name to indicate this as
706 # well
707 my $disallowed = $disallow_flag
708 || $malformations_name;
709 my $this_name = "utf8n_to_uvchr_error() $testname: "
710 . (($disallow_flag)
711 ? 'disallowed'
712 : $disallowed
713 ? $disallowed
714 : 'allowed');
715 $this_name .= ", $eval_warn";
716 $this_name .= ", " . (($warn_flag)
717 ? 'with warning flag'
718 : 'no warning flag');
719
9cdc3054 720 undef @warnings_gotten;
6aa905cf
KW
721 my $ret_ref;
722 my $display_bytes = display_bytes($this_bytes);
723 my $call = " Call was: $eval_warn; \$ret_ref"
724 . " = test_utf8n_to_uvchr_error("
725 . "'$display_bytes', $this_length,"
726 . "$warn_flag"
727 . "|$disallow_flag)";
728 my $eval_text = "$eval_warn; \$ret_ref"
729 . " = test_utf8n_to_uvchr_error("
730 . "'$this_bytes',"
731 . " $this_length, $warn_flag"
732 . "|$disallow_flag)";
733 eval "$eval_text";
734 if (! ok ("$@ eq ''",
735 "$this_name: eval succeeded"))
736 {
737 diag "\$!='$!'; eval'd=\"$call\"";
738 next;
739 }
740 if ($disallowed) {
741 is($ret_ref->[0], 0, "$this_name: Returns 0")
742 or diag $call;
743 }
744 else {
745 is($ret_ref->[0], $expected_uv,
746 "$this_name: Returns expected uv: "
747 . sprintf("0x%04X", $expected_uv))
748 or diag $call;
749 }
750 is($ret_ref->[1], $this_expected_len,
751 "$this_name: Returns expected length:"
752 . " $this_expected_len")
753 or diag $call;
754
9cdc3054 755 my $returned_flags = $ret_ref->[2];
6aa905cf 756
9cdc3054
KW
757 for (my $i = @expected_return_flags - 1; $i >= 0; $i--) {
758 if (ok($expected_return_flags[$i] & $returned_flags,
6aa905cf
KW
759 "Expected and got error bit return"
760 . " for $malformations[$i] malformation"))
761 {
9cdc3054 762 $returned_flags &= ~$expected_return_flags[$i];
6aa905cf 763 }
9cdc3054 764 splice @expected_return_flags, $i, 1;
6aa905cf 765 }
9cdc3054 766 is(scalar @expected_return_flags, 0,
6aa905cf 767 "Got all the expected malformation errors")
9cdc3054 768 or diag Dumper \@expected_return_flags;
6aa905cf
KW
769
770 if ( $this_expected_len >= $this_needed_to_discern_len
771 && ($warn_flag || $disallow_flag))
772 {
9cdc3054 773 is($returned_flags, $expected_error_flags,
6aa905cf
KW
774 "Got the correct error flag")
775 or diag $call;
776 }
777 else {
5722c46d
KW
778 is($returned_flags, 0, "Got no other error flag")
779 or
780
781 # We strip off any prefixes from the flag names
782 diag "The unexpected flags were: "
783 . (flags_to_text($returned_flags,
784 \@utf8n_flags_to_text)
785 =~ s/ \b [A-Z] _ //xgr);
6aa905cf
KW
786 }
787
788 if (@malformations) {
9cdc3054 789 if (! $do_warning && $trial_warning_category eq 'utf8') {
6aa905cf
KW
790 goto no_warnings_expected;
791 }
792
793 # Check that each malformation generates a
794 # warning, removing that warning if found
795 MALFORMATION:
796 foreach my $malformation (@malformations) {
9cdc3054
KW
797 foreach (my $i = 0; $i < @warnings_gotten; $i++) {
798 if ($warnings_gotten[$i] =~ /$malformation/) {
6aa905cf
KW
799 pass("Expected and got"
800 . "'$malformation' warning");
9cdc3054 801 splice @warnings_gotten, $i, 1;
6aa905cf
KW
802 next MALFORMATION;
803 }
804 }
805 fail("Expected '$malformation' warning"
806 . " but didn't get it");
807
808 }
809 }
810
811 # Any overflow will override any super or above-31
812 # warnings.
813 goto no_warnings_expected
814 if $will_overflow || $this_expected_len
815 < $this_needed_to_discern_len;
816
817 if ( ! $do_warning
9cdc3054
KW
818 && ( $trial_warning_category eq 'utf8'
819 || $trial_warning_category eq $controlling_warning_category))
6aa905cf
KW
820 {
821 goto no_warnings_expected;
822 }
823 elsif ($warn_flag) {
9cdc3054 824 if (is(scalar @warnings_gotten, 1,
6aa905cf
KW
825 "$this_name: Got a single warning "))
826 {
9cdc3054 827 like($warnings_gotten[0], $message,
6aa905cf
KW
828 "$this_name: Got expected warning")
829 or diag $call;
830 }
831 else {
832 diag $call;
9cdc3054
KW
833 if (scalar @warnings_gotten) {
834 output_warnings(@warnings_gotten);
6aa905cf
KW
835 }
836 }
837 }
838 else {
9cdc3054 839
6aa905cf 840 no_warnings_expected:
9cdc3054 841 unless (is(scalar @warnings_gotten, 0,
6aa905cf
KW
842 "$this_name: Got no warnings"))
843 {
844 diag $call;
9cdc3054 845 output_warnings(@warnings_gotten);
6aa905cf
KW
846 }
847 }
848
849 # Check CHECK_ONLY results when the input is
850 # disallowed. Do this when actually disallowed,
851 # not just when the $disallow_flag is set
852 if ($disallowed) {
9cdc3054 853 undef @warnings_gotten;
6aa905cf
KW
854 $ret_ref = test_utf8n_to_uvchr_error(
855 $this_bytes, $this_length,
856 $disallow_flag|$::UTF8_CHECK_ONLY);
857 is($ret_ref->[0], 0,
858 "$this_name, CHECK_ONLY: Returns 0")
859 or diag $call;
860 is($ret_ref->[1], -1,
861 "$this_name: CHECK_ONLY: returns -1 for length")
862 or diag $call;
9cdc3054 863 if (! is(scalar @warnings_gotten, 0,
6aa905cf
KW
864 "$this_name, CHECK_ONLY: no warnings"
865 . " generated"))
866 {
867 diag $call;
9cdc3054 868 output_warnings(@warnings_gotten);
6aa905cf
KW
869 }
870 }
871
872 # Now repeat some of the above, but for
873 # uvchr_to_utf8_flags(). Since this comes from an
874 # existing code point, it hasn't overflowed, and
875 # isn't malformed.
876 next if @malformations;
877
878 # The warning and disallow flags passed in are for
879 # utf8n_to_uvchr_error(). Convert them for
880 # uvchr_to_utf8_flags().
881 my $uvchr_warn_flag = 0;
882 my $uvchr_disallow_flag = 0;
883 if ($warn_flag) {
884 if ($warn_flag == $::UTF8_WARN_SURROGATE) {
885 $uvchr_warn_flag = $::UNICODE_WARN_SURROGATE
886 }
887 elsif ($warn_flag == $::UTF8_WARN_NONCHAR) {
888 $uvchr_warn_flag = $::UNICODE_WARN_NONCHAR
889 }
890 elsif ($warn_flag == $::UTF8_WARN_SUPER) {
891 $uvchr_warn_flag = $::UNICODE_WARN_SUPER
892 }
893 elsif ($warn_flag == $::UTF8_WARN_ABOVE_31_BIT) {
894 $uvchr_warn_flag
895 = $::UNICODE_WARN_ABOVE_31_BIT;
896 }
897 else {
898 fail(sprintf "Unexpected warn flag: %x",
899 $warn_flag);
900 next;
901 }
902 }
903 if ($disallow_flag) {
904 if ($disallow_flag == $::UTF8_DISALLOW_SURROGATE)
905 {
906 $uvchr_disallow_flag
907 = $::UNICODE_DISALLOW_SURROGATE;
908 }
909 elsif ($disallow_flag == $::UTF8_DISALLOW_NONCHAR)
910 {
911 $uvchr_disallow_flag
912 = $::UNICODE_DISALLOW_NONCHAR;
913 }
914 elsif ($disallow_flag == $::UTF8_DISALLOW_SUPER) {
915 $uvchr_disallow_flag
916 = $::UNICODE_DISALLOW_SUPER;
917 }
918 elsif ($disallow_flag
919 == $::UTF8_DISALLOW_ABOVE_31_BIT)
920 {
921 $uvchr_disallow_flag =
922 $::UNICODE_DISALLOW_ABOVE_31_BIT;
923 }
924 else {
925 fail(sprintf "Unexpected disallow flag: %x",
926 $disallow_flag);
927 next;
928 }
929 }
930
931 $disallowed = $uvchr_disallow_flag;
932
933 $this_name = "uvchr_to_utf8_flags() $testname: "
934 . (($uvchr_disallow_flag)
935 ? 'disallowed'
936 : ($disallowed)
937 ? 'ABOVE_31_BIT allowed'
938 : 'allowed');
939 $this_name .= ", $eval_warn";
940 $this_name .= ", " . (($uvchr_warn_flag)
941 ? 'with warning flag'
942 : 'no warning flag');
943
9cdc3054 944 undef @warnings_gotten;
6aa905cf
KW
945 my $ret;
946 my $warn_flag = sprintf "0x%x", $uvchr_warn_flag;
947 my $disallow_flag = sprintf "0x%x",
948 $uvchr_disallow_flag;
949 $call = sprintf(" Call was: $eval_warn; \$ret"
950 . " = test_uvchr_to_utf8_flags("
951 . " 0x%x, $warn_flag|$disallow_flag)",
952 $allowed_uv);
953 $eval_text = "$eval_warn; \$ret ="
954 . " test_uvchr_to_utf8_flags("
955 . "$allowed_uv, $warn_flag|"
956 . "$disallow_flag)";
957 eval "$eval_text";
958 if (! ok ("$@ eq ''", "$this_name: eval succeeded"))
959 {
960 diag "\$!='$!'; eval'd=\"$eval_text\"";
961 next;
962 }
963 if ($disallowed) {
964 is($ret, undef, "$this_name: Returns undef")
965 or diag $call;
966 }
967 else {
968 is($ret, $bytes, "$this_name: Returns expected string")
969 or diag $call;
970 }
971 if (! $do_warning
9cdc3054 972 && ($trial_warning_category eq 'utf8' || $trial_warning_category eq $controlling_warning_category))
6aa905cf 973 {
9cdc3054 974 if (!is(scalar @warnings_gotten, 0,
6aa905cf
KW
975 "$this_name: No warnings generated"))
976 {
977 diag $call;
9cdc3054 978 output_warnings(@warnings_gotten);
6aa905cf
KW
979 }
980 }
981 elsif ( $uvchr_warn_flag
9cdc3054
KW
982 && ( $trial_warning_category eq 'utf8'
983 || $trial_warning_category eq $controlling_warning_category))
6aa905cf 984 {
9cdc3054 985 if (is(scalar @warnings_gotten, 1,
6aa905cf
KW
986 "$this_name: Got a single warning "))
987 {
9cdc3054 988 like($warnings_gotten[0], $message,
6aa905cf
KW
989 "$this_name: Got expected warning")
990 or diag $call;
991 }
992 else {
993 diag $call;
9cdc3054
KW
994 output_warnings(@warnings_gotten)
995 if scalar @warnings_gotten;
6aa905cf
KW
996 }
997 }
998 }
999 }
1000 }
1001 }
1002 }
1003 }
1004 }
1005}
1006
1007done_testing;