This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
t/re/regexp_unicode_prop.t: Extract code into a function
[perl5.git] / t / re / regexp_unicode_prop.t
CommitLineData
84281c31
A
1#!./perl
2#
3# Tests that have to do with checking whether characters have (or not have)
4# certain Unicode properties; belong (or not belong) to blocks, scripts, etc.
5#
6
7use strict;
8use warnings;
9use 5.010;
10
f3b02925 11BEGIN {
b5efbd1f 12 chdir 't' if -d 't';
f3b02925
NC
13 require './test.pl';
14 skip_all_if_miniperl("no dynamic loading on miniperl, no File::Spec (used by charnames)");
15}
84281c31
A
16
17sub run_tests;
18
3a0825d5
KW
19sub get_str_name($) {
20 my $char = shift;
21
22 my ($str, $name);
23
24 if ($char =~ /^\\/) {
25 $str = eval qq ["$char"];
26 $name = qq ["$char"];
27 }
28 elsif ($char =~ /^0x([0-9A-Fa-f]+)$/) {
29 $str = chr hex $1;
30 $name = "chr ($char)";
31 }
32 else {
33 $str = $char;
34 $name = qq ["$char"];
35 }
36
37 return ($str, $name);
38}
39
84281c31
A
40#
41# This is the data to test.
42#
43# This is a hash; keys are the property to test.
44# Values are arrays containing characters to test. The characters can
45# have the following formats:
46# '\N{CHARACTER NAME}' - Use character with that name
47# '\x{1234}' - Use character with that hex escape
48# '0x1234' - Use chr() to get that character
49# "a" - Character to use
50#
51# If a character entry starts with ! the character does not belong to the class
52#
53# If the class is just single letter, we use both \pL and \p{L}
54#
55
56use charnames ':full';
57
58my @CLASSES = (
59 L => ["a", "A"],
60 Ll => ["b", "!B"],
61 Lu => ["!c", "C"],
62 IsLl => ["d", "!D"],
63 IsLu => ["!e", "E"],
64 LC => ["f", "!1"],
65 'L&' => ["g", "!2"],
66 'Lowercase Letter' => ["h", "!H"],
67
68 Common => ["!i", "3"],
69 Inherited => ["!j", '\x{300}'],
70
71 InBasicLatin => ['\N{LATIN CAPITAL LETTER A}'],
72 InLatin1Supplement => ['\N{LATIN CAPITAL LETTER A WITH GRAVE}'],
73 InLatinExtendedA => ['\N{LATIN CAPITAL LETTER A WITH MACRON}'],
74 InLatinExtendedB => ['\N{LATIN SMALL LETTER B WITH STROKE}'],
75 InKatakana => ['\N{KATAKANA LETTER SMALL A}'],
76 IsLatin => ["0x100", "0x212b"],
77 IsHebrew => ["0x5d0", "0xfb4f"],
78 IsGreek => ["0x37a", "0x386", "!0x387", "0x388",
79 "0x38a", "!0x38b", "0x38c"],
80 HangulSyllables => ['\x{AC00}'],
81 'Script=Latin' => ['\x{0100}'],
82 'Block=LatinExtendedA' => ['\x{0100}'],
83 'Category=UppercaseLetter' => ['\x{0100}'],
84
85 #
86 # It's ok to repeat class names.
87 #
88 InLatin1Supplement =>
425224e1 89 ['!\N{U+7f}', '\N{U+80}', '\N{U+ff}', '!\x{100}'],
84281c31 90 InLatinExtendedA =>
425224e1 91 ['!\N{U+7f}', '!\N{U+80}', '!\N{U+ff}', '\x{100}'],
84281c31
A
92
93 #
94 # Properties are case-insensitive, and may have whitespace,
95 # dashes and underscores.
96 #
425224e1 97 'in-latin1_SUPPLEMENT' => ['\N{U+80}',
84281c31
A
98 '\N{LATIN SMALL LETTER Y WITH DIAERESIS}'],
99 ' ^ In Latin 1 Supplement '
425224e1
KW
100 => ['!\N{U+80}', '\N{COFFIN}'],
101 'latin-1 supplement' => ['\N{U+80}', "0xDF"],
84281c31
A
102
103);
104
3b9fb374
KW
105 my @USER_DEFINED_PROPERTIES = (
106 #
107 # User defined properties
108 #
109 InKana1 => ['\x{3040}', '!\x{303F}'],
110 InKana2 => ['\x{3040}', '!\x{303F}'],
111 InKana3 => ['\x{3041}', '!\x{3040}'],
112 InNotKana => ['\x{3040}', '!\x{3041}'],
113 InConsonant => ['d', '!e'],
114 IsSyriac1 => ['\x{0712}', '!\x{072F}'],
115 IsSyriac1KanaMark => ['\x{309A}', '!\x{3090}'],
116 IsSyriac1KanaMark => ['\x{0730}', '!\x{0712}'],
117 '# User-defined character properties may lack \n at the end',
118 InGreekSmall => ['\N{GREEK SMALL LETTER PI}',
119 '\N{GREEK SMALL LETTER FINAL SIGMA}'],
120 InGreekCapital => ['\N{GREEK CAPITAL LETTER PI}', '!\x{03A2}'],
121 Dash => ['-'],
122 ASCII_Hex_Digit => ['!-', 'A'],
123 IsAsciiHexAndDash => ['-', 'A'],
124
125 # This overrides the official one
126 InLatin1 => ['\x{0100}', '!\x{00FF}'],
127 );
128
129 my @USER_CASELESS_PROPERTIES = (
130 #
131 # User defined properties which differ depending on /i. Second entry
132 # is false normally, true under /i
133 #
134 'IsMyUpper' => ["M", "!m" ],
135 );
19c4061a 136
84281c31
A
137
138#
139# From the short properties we populate POSIX-like classes.
140#
141my %SHORT_PROPERTIES = (
142 'Ll' => ['m', '\N{CYRILLIC SMALL LETTER A}'],
143 'Lu' => ['M', '\N{GREEK CAPITAL LETTER ALPHA}'],
144 'Lo' => ['\N{HIRAGANA LETTER SMALL A}'],
99870f4d
KW
145 # is also in other alphabetic
146 'Mn' => ['\N{HEBREW POINT RAFE}'],
84281c31
A
147 'Nd' => ["0", '\N{ARABIC-INDIC DIGIT ZERO}'],
148 'Pc' => ["_"],
149 'Po' => ["!"],
150 'Zs' => [" "],
151 'Cc' => ['\x{00}'],
152);
153
154#
155# Illegal properties
156#
d658a8a8
DM
157my @ILLEGAL_PROPERTIES =
158 qw[q qrst f foo isfoo infoo ISfoo INfoo Is::foo In::foo];
84281c31
A
159
160my %d;
161
162while (my ($class, $chars) = each %SHORT_PROPERTIES) {
163 push @{$d {IsAlpha}} => map {$class =~ /^[LM]/ ? $_ : "!$_"} @$chars;
164 push @{$d {IsAlnum}} => map {$class =~ /^[LMN]./ ? $_ : "!$_"} @$chars;
165 push @{$d {IsASCII}} => map {length ($_) == 1 || $_ eq '\x{00}'
166 ? $_ : "!$_"} @$chars;
167 push @{$d {IsCntrl}} => map {$class =~ /^C/ ? $_ : "!$_"} @$chars;
168 push @{$d {IsBlank}} => map {$class =~ /^Z[lps]/ ? $_ : "!$_"} @$chars;
169 push @{$d {IsDigit}} => map {$class =~ /^Nd$/ ? $_ : "!$_"} @$chars;
170 push @{$d {IsGraph}} => map {$class =~ /^([LMNPS]|Co)/
171 ? $_ : "!$_"} @$chars;
172 push @{$d {IsPrint}} => map {$class =~ /^([LMNPS]|Co|Zs)/
173 ? $_ : "!$_"} @$chars;
174 push @{$d {IsLower}} => map {$class =~ /^Ll$/ ? $_ : "!$_"} @$chars;
175 push @{$d {IsUpper}} => map {$class =~ /^L[ut]/ ? $_ : "!$_"} @$chars;
176 push @{$d {IsPunct}} => map {$class =~ /^P/ ? $_ : "!$_"} @$chars;
177 push @{$d {IsWord}} => map {$class =~ /^[LMN]/ || $_ eq "_"
178 ? $_ : "!$_"} @$chars;
179 push @{$d {IsSpace}} => map {$class =~ /^Z/ ||
425224e1
KW
180 length ($_) == 1 && utf8::native_to_unicode(ord ($_)) >= 0x09
181 && utf8::native_to_unicode(ord ($_)) <= 0x0D
84281c31
A
182 ? $_ : "!$_"} @$chars;
183}
184
84281c31
A
185push @CLASSES => "# Short properties" => %SHORT_PROPERTIES,
186 "# POSIX like properties" => %d,
187 "# User defined properties" => @USER_DEFINED_PROPERTIES;
188
189
190#
191# Calculate the number of tests.
192#
193my $count = 0;
194for (my $i = 0; $i < @CLASSES; $i += 2) {
195 $i ++, redo if $CLASSES [$i] =~ /^\h*#\h*(.*)/;
72d70e56 196 $count += 2 * (length $CLASSES [$i] == 1 ? 4 : 2) * @{$CLASSES [$i + 1]};
84281c31 197}
72d70e56
NC
198$count += 4 * @ILLEGAL_PROPERTIES;
199$count += 4 * grep {length $_ == 1} @ILLEGAL_PROPERTIES;
200$count += 8 * @USER_CASELESS_PROPERTIES;
710d3eb3 201$count += 1; # Test for pkg:IsMyLower
84281c31 202
72d70e56 203plan(tests => $count);
84281c31
A
204
205run_tests unless caller ();
206
207sub match {
19c4061a
KW
208 my ($char, $match, $nomatch, $caseless) = @_;
209 $caseless = "" unless defined $caseless;
210 $caseless = 'i' if $caseless;
84281c31 211
3a0825d5 212 my ($str, $name) = get_str_name($char);
84281c31 213
19c4061a 214 undef $@;
707250fe
KW
215 my $pat = "qr/$match/$caseless";
216 my $match_pat = eval $pat;
217 is($@, '', "$pat compiled correctly to a regexp: $@");
72d70e56 218 like($str, $match_pat, "$name correctly matched");
19c4061a
KW
219
220 undef $@;
707250fe
KW
221 $pat = "qr/$nomatch/$caseless";
222 my $nomatch_pat = eval $pat;
223 is($@, '', "$pat compiled correctly to a regexp: $@");
72d70e56 224 unlike($str, $nomatch_pat, "$name correctly did not match");
84281c31
A
225}
226
227sub run_tests {
228
229 while (@CLASSES) {
230 my $class = shift @CLASSES;
231 if ($class =~ /^\h*#\h*(.*)/) {
232 print "# $1\n";
233 next;
234 }
235 last unless @CLASSES;
236 my $chars = shift @CLASSES;
237 my @in = grep {!/^!./} @$chars;
238 my @out = map {s/^!(?=.)//; $_} grep { /^!./} @$chars;
239 my $in_pat = eval qq ['\\p{$class}'];
240 my $out_pat = eval qq ['\\P{$class}'];
241
242 match $_, $in_pat, $out_pat for @in;
243 match $_, $out_pat, $in_pat for @out;
244
9c8ea558 245 if (1 == length $class) { # Repeat without braces if name length 1
84281c31
A
246 my $in_pat = eval qq ['\\p$class'];
247 my $out_pat = eval qq ['\\P$class'];
248
249 match $_, $in_pat, $out_pat for @in;
250 match $_, $out_pat, $in_pat for @out;
251 }
252 }
253
254
84281c31
A
255 print "# Illegal properties\n";
256 foreach my $p (@ILLEGAL_PROPERTIES) {
4003ea29
KW
257 my $pat;
258 if ($p =~ /::/) {
259 $pat = qr /^Illegal user-defined property name/;
260 }
261 else {
262 $pat = qr /^Can't find Unicode property definition/;
263 }
264
84281c31
A
265 undef $@;
266 my $r = eval "'a' =~ /\\p{$p}/; 1";
72d70e56
NC
267 is($r, undef, "Unknown Unicode property \\p{$p}");
268 like($@, $pat, "Unknown Unicode property \\p{$p}");
84281c31
A
269 undef $@;
270 my $s = eval "'a' =~ /\\P{$p}/; 1";
72d70e56
NC
271 is($s, undef, "Unknown Unicode property \\p{$p}");
272 like($@, $pat, "Unknown Unicode property \\p{$p}");
84281c31
A
273 if (length $p == 1) {
274 undef $@;
275 my $r = eval "'a' =~ /\\p$p/; 1";
72d70e56
NC
276 is($r, undef, "Unknown Unicode property \\p$p");
277 like($@, $pat, "Unknown Unicode property \\p$p");
84281c31
A
278 undef $@;
279 my $s = eval "'a' =~ /\\P$p/; 1";
72d70e56
NC
280 is($r, undef, "Unknown Unicode property \\P$p");
281 like($@, $pat, "Unknown Unicode property \\P$p");
84281c31
A
282 }
283 }
19c4061a
KW
284
285 print "# User-defined properties with /i differences\n";
49fdc6e8 286 while (my $class = shift @USER_CASELESS_PROPERTIES) {
19c4061a
KW
287 my $chars_ref = shift @USER_CASELESS_PROPERTIES;
288 my @in = grep {!/^!./} @$chars_ref;
289 my @out = map {s/^!(?=.)//; $_} grep { /^!./} @$chars_ref;
290 my $in_pat = eval qq ['\\p{$class}'];
291 my $out_pat = eval qq ['\\P{$class}'];
292
293 # Verify works as regularly for not /i
294 match $_, $in_pat, $out_pat for @in;
295 match $_, $out_pat, $in_pat for @out;
296
297 # Verify that adding /i doesn't change the in set.
298 match $_, $in_pat, $out_pat, 'i' for @in;
299
300 # Verify that adding /i does change the out set to match.
301 match $_, $in_pat, $out_pat, 'i' for @out;
302 }
84281c31
A
303}
304
305
306#
307# User defined properties
308#
309
310sub InKana1 {<<'--'}
3113040 309F
31230A0 30FF
313--
314
315sub InKana2 {<<'--'}
316+utf8::InHiragana
317+utf8::InKatakana
318--
319
320sub InKana3 {<<'--'}
321+utf8::InHiragana
322+utf8::InKatakana
323-utf8::IsCn
324--
325
326sub InNotKana {<<'--'}
327!utf8::InHiragana
328-utf8::InKatakana
329+utf8::IsCn
330--
331
425224e1
KW
332sub InConsonant {
333
334 my $return = "+utf8::Lowercase\n&utf8::ASCII\n";
335 $return .= sprintf("-%X\n", ord "a");
336 $return .= sprintf("-%X\n", ord "e");
337 $return .= sprintf("-%X\n", ord "i");
338 $return .= sprintf("-%X\n", ord "o");
339 $return .= sprintf("-%X\n", ord "u");
340 return $return;
341}
84281c31
A
342
343sub IsSyriac1 {<<'--'}
3440712 072C
3450730 074A
346--
347
84281c31
A
348sub InGreekSmall {return "03B1\t03C9"}
349sub InGreekCapital {return "0391\t03A9\n-03A2"}
350
d658a8a8 351sub IsAsciiHexAndDash {<<'--'}
84281c31
A
352+utf8::ASCII_Hex_Digit
353+utf8::Dash
354--
355
eb765580
KW
356sub InLatin1 {
357 return "0100\t10FFFF";
358}
359
19c4061a
KW
360sub IsMyUpper {
361 my $caseless = shift;
425224e1
KW
362 return "+utf8::"
363 . (($caseless)
364 ? 'Alphabetic'
365 : 'Uppercase')
366 . "\n&utf8::ASCII";
19c4061a
KW
367}
368
710d3eb3
KW
369{ # This has to be done here and not like the others, because we have to
370 # make sure that the property is not known until after the regex is
371 # compiled. It was previously getting confused about the pkg and /i
372 # combination
373
374 my $mylower = qr/\p{pkg::IsMyLower}/i;
375
3b9fb374
KW
376sub pkg::IsMyLower {
377 my $caseless = shift;
378 return "+utf8::"
379 . (($caseless)
380 ? 'Alphabetic'
381 : 'Lowercase')
382 . "\n&utf8::ASCII";
383}
710d3eb3
KW
384
385 like("A", $mylower, "Not available until runtime user-defined property with pkg:: and /i works");
386
387}
388
8fd2c59a
KW
389# Verify that can use user-defined properties inside another one
390sub IsSyriac1KanaMark {<<'--'}
391+main::IsSyriac1
392+main::InKana3
393&utf8::IsMark
394--
395
d658a8a8
DM
396# fake user-defined properties; these subs shouldn't be called, because
397# their names don't start with In or Is
398
399sub f { die }
400sub foo { die }
401sub isfoo { die }
402sub infoo { die }
403sub ISfoo { die }
404sub INfoo { die }
405sub Is::foo { die }
406sub In::foo { die }
0a441b11
KW
407
4081;
84281c31 409__END__