This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
t/re/regexp_unicode_prop.t: White-space only
[perl5.git] / t / re / regexp_unicode_prop.t
CommitLineData
84281c31
A
1#!./perl
2#
3# Tests that have to do with checking whether characters have (or not have)
4# certain Unicode properties; belong (or not belong) to blocks, scripts, etc.
5#
6
7use strict;
8use warnings;
9use 5.010;
10
f3b02925 11BEGIN {
b5efbd1f 12 chdir 't' if -d 't';
f3b02925
NC
13 require './test.pl';
14 skip_all_if_miniperl("no dynamic loading on miniperl, no File::Spec (used by charnames)");
15}
84281c31
A
16
17sub run_tests;
18
19#
20# This is the data to test.
21#
22# This is a hash; keys are the property to test.
23# Values are arrays containing characters to test. The characters can
24# have the following formats:
25# '\N{CHARACTER NAME}' - Use character with that name
26# '\x{1234}' - Use character with that hex escape
27# '0x1234' - Use chr() to get that character
28# "a" - Character to use
29#
30# If a character entry starts with ! the character does not belong to the class
31#
32# If the class is just single letter, we use both \pL and \p{L}
33#
34
35use charnames ':full';
36
37my @CLASSES = (
38 L => ["a", "A"],
39 Ll => ["b", "!B"],
40 Lu => ["!c", "C"],
41 IsLl => ["d", "!D"],
42 IsLu => ["!e", "E"],
43 LC => ["f", "!1"],
44 'L&' => ["g", "!2"],
45 'Lowercase Letter' => ["h", "!H"],
46
47 Common => ["!i", "3"],
48 Inherited => ["!j", '\x{300}'],
49
50 InBasicLatin => ['\N{LATIN CAPITAL LETTER A}'],
51 InLatin1Supplement => ['\N{LATIN CAPITAL LETTER A WITH GRAVE}'],
52 InLatinExtendedA => ['\N{LATIN CAPITAL LETTER A WITH MACRON}'],
53 InLatinExtendedB => ['\N{LATIN SMALL LETTER B WITH STROKE}'],
54 InKatakana => ['\N{KATAKANA LETTER SMALL A}'],
55 IsLatin => ["0x100", "0x212b"],
56 IsHebrew => ["0x5d0", "0xfb4f"],
57 IsGreek => ["0x37a", "0x386", "!0x387", "0x388",
58 "0x38a", "!0x38b", "0x38c"],
59 HangulSyllables => ['\x{AC00}'],
60 'Script=Latin' => ['\x{0100}'],
61 'Block=LatinExtendedA' => ['\x{0100}'],
62 'Category=UppercaseLetter' => ['\x{0100}'],
63
64 #
65 # It's ok to repeat class names.
66 #
67 InLatin1Supplement =>
425224e1 68 ['!\N{U+7f}', '\N{U+80}', '\N{U+ff}', '!\x{100}'],
84281c31 69 InLatinExtendedA =>
425224e1 70 ['!\N{U+7f}', '!\N{U+80}', '!\N{U+ff}', '\x{100}'],
84281c31
A
71
72 #
73 # Properties are case-insensitive, and may have whitespace,
74 # dashes and underscores.
75 #
425224e1 76 'in-latin1_SUPPLEMENT' => ['\N{U+80}',
84281c31
A
77 '\N{LATIN SMALL LETTER Y WITH DIAERESIS}'],
78 ' ^ In Latin 1 Supplement '
425224e1
KW
79 => ['!\N{U+80}', '\N{COFFIN}'],
80 'latin-1 supplement' => ['\N{U+80}', "0xDF"],
84281c31
A
81
82);
83
3b9fb374
KW
84 my @USER_DEFINED_PROPERTIES = (
85 #
86 # User defined properties
87 #
88 InKana1 => ['\x{3040}', '!\x{303F}'],
89 InKana2 => ['\x{3040}', '!\x{303F}'],
90 InKana3 => ['\x{3041}', '!\x{3040}'],
91 InNotKana => ['\x{3040}', '!\x{3041}'],
92 InConsonant => ['d', '!e'],
93 IsSyriac1 => ['\x{0712}', '!\x{072F}'],
94 IsSyriac1KanaMark => ['\x{309A}', '!\x{3090}'],
95 IsSyriac1KanaMark => ['\x{0730}', '!\x{0712}'],
96 '# User-defined character properties may lack \n at the end',
97 InGreekSmall => ['\N{GREEK SMALL LETTER PI}',
98 '\N{GREEK SMALL LETTER FINAL SIGMA}'],
99 InGreekCapital => ['\N{GREEK CAPITAL LETTER PI}', '!\x{03A2}'],
100 Dash => ['-'],
101 ASCII_Hex_Digit => ['!-', 'A'],
102 IsAsciiHexAndDash => ['-', 'A'],
103
104 # This overrides the official one
105 InLatin1 => ['\x{0100}', '!\x{00FF}'],
106 );
107
108 my @USER_CASELESS_PROPERTIES = (
109 #
110 # User defined properties which differ depending on /i. Second entry
111 # is false normally, true under /i
112 #
113 'IsMyUpper' => ["M", "!m" ],
114 );
19c4061a 115
84281c31
A
116
117#
118# From the short properties we populate POSIX-like classes.
119#
120my %SHORT_PROPERTIES = (
121 'Ll' => ['m', '\N{CYRILLIC SMALL LETTER A}'],
122 'Lu' => ['M', '\N{GREEK CAPITAL LETTER ALPHA}'],
123 'Lo' => ['\N{HIRAGANA LETTER SMALL A}'],
99870f4d
KW
124 # is also in other alphabetic
125 'Mn' => ['\N{HEBREW POINT RAFE}'],
84281c31
A
126 'Nd' => ["0", '\N{ARABIC-INDIC DIGIT ZERO}'],
127 'Pc' => ["_"],
128 'Po' => ["!"],
129 'Zs' => [" "],
130 'Cc' => ['\x{00}'],
131);
132
133#
134# Illegal properties
135#
d658a8a8
DM
136my @ILLEGAL_PROPERTIES =
137 qw[q qrst f foo isfoo infoo ISfoo INfoo Is::foo In::foo];
84281c31
A
138
139my %d;
140
141while (my ($class, $chars) = each %SHORT_PROPERTIES) {
142 push @{$d {IsAlpha}} => map {$class =~ /^[LM]/ ? $_ : "!$_"} @$chars;
143 push @{$d {IsAlnum}} => map {$class =~ /^[LMN]./ ? $_ : "!$_"} @$chars;
144 push @{$d {IsASCII}} => map {length ($_) == 1 || $_ eq '\x{00}'
145 ? $_ : "!$_"} @$chars;
146 push @{$d {IsCntrl}} => map {$class =~ /^C/ ? $_ : "!$_"} @$chars;
147 push @{$d {IsBlank}} => map {$class =~ /^Z[lps]/ ? $_ : "!$_"} @$chars;
148 push @{$d {IsDigit}} => map {$class =~ /^Nd$/ ? $_ : "!$_"} @$chars;
149 push @{$d {IsGraph}} => map {$class =~ /^([LMNPS]|Co)/
150 ? $_ : "!$_"} @$chars;
151 push @{$d {IsPrint}} => map {$class =~ /^([LMNPS]|Co|Zs)/
152 ? $_ : "!$_"} @$chars;
153 push @{$d {IsLower}} => map {$class =~ /^Ll$/ ? $_ : "!$_"} @$chars;
154 push @{$d {IsUpper}} => map {$class =~ /^L[ut]/ ? $_ : "!$_"} @$chars;
155 push @{$d {IsPunct}} => map {$class =~ /^P/ ? $_ : "!$_"} @$chars;
156 push @{$d {IsWord}} => map {$class =~ /^[LMN]/ || $_ eq "_"
157 ? $_ : "!$_"} @$chars;
158 push @{$d {IsSpace}} => map {$class =~ /^Z/ ||
425224e1
KW
159 length ($_) == 1 && utf8::native_to_unicode(ord ($_)) >= 0x09
160 && utf8::native_to_unicode(ord ($_)) <= 0x0D
84281c31
A
161 ? $_ : "!$_"} @$chars;
162}
163
84281c31
A
164push @CLASSES => "# Short properties" => %SHORT_PROPERTIES,
165 "# POSIX like properties" => %d,
166 "# User defined properties" => @USER_DEFINED_PROPERTIES;
167
168
169#
170# Calculate the number of tests.
171#
172my $count = 0;
173for (my $i = 0; $i < @CLASSES; $i += 2) {
174 $i ++, redo if $CLASSES [$i] =~ /^\h*#\h*(.*)/;
72d70e56 175 $count += 2 * (length $CLASSES [$i] == 1 ? 4 : 2) * @{$CLASSES [$i + 1]};
84281c31 176}
72d70e56
NC
177$count += 4 * @ILLEGAL_PROPERTIES;
178$count += 4 * grep {length $_ == 1} @ILLEGAL_PROPERTIES;
179$count += 8 * @USER_CASELESS_PROPERTIES;
710d3eb3 180$count += 1; # Test for pkg:IsMyLower
84281c31 181
72d70e56 182plan(tests => $count);
84281c31
A
183
184run_tests unless caller ();
185
186sub match {
19c4061a
KW
187 my ($char, $match, $nomatch, $caseless) = @_;
188 $caseless = "" unless defined $caseless;
189 $caseless = 'i' if $caseless;
84281c31
A
190
191 my ($str, $name);
192
8d4c8383
BF
193 if ($char =~ /^\\/) {
194 $str = eval qq ["$char"];
195 $name = qq ["$char"];
196 }
197 elsif ($char =~ /^0x([0-9A-Fa-f]+)$/) {
198 $str = chr hex $1;
199 $name = "chr ($char)";
200 }
201 else {
202 $str = $char;
203 $name = qq ["$char"];
84281c31
A
204 }
205
19c4061a 206 undef $@;
707250fe
KW
207 my $pat = "qr/$match/$caseless";
208 my $match_pat = eval $pat;
209 is($@, '', "$pat compiled correctly to a regexp: $@");
72d70e56 210 like($str, $match_pat, "$name correctly matched");
19c4061a
KW
211
212 undef $@;
707250fe
KW
213 $pat = "qr/$nomatch/$caseless";
214 my $nomatch_pat = eval $pat;
215 is($@, '', "$pat compiled correctly to a regexp: $@");
72d70e56 216 unlike($str, $nomatch_pat, "$name correctly did not match");
84281c31
A
217}
218
219sub run_tests {
220
221 while (@CLASSES) {
222 my $class = shift @CLASSES;
223 if ($class =~ /^\h*#\h*(.*)/) {
224 print "# $1\n";
225 next;
226 }
227 last unless @CLASSES;
228 my $chars = shift @CLASSES;
229 my @in = grep {!/^!./} @$chars;
230 my @out = map {s/^!(?=.)//; $_} grep { /^!./} @$chars;
231 my $in_pat = eval qq ['\\p{$class}'];
232 my $out_pat = eval qq ['\\P{$class}'];
233
234 match $_, $in_pat, $out_pat for @in;
235 match $_, $out_pat, $in_pat for @out;
236
9c8ea558 237 if (1 == length $class) { # Repeat without braces if name length 1
84281c31
A
238 my $in_pat = eval qq ['\\p$class'];
239 my $out_pat = eval qq ['\\P$class'];
240
241 match $_, $in_pat, $out_pat for @in;
242 match $_, $out_pat, $in_pat for @out;
243 }
244 }
245
246
84281c31
A
247 print "# Illegal properties\n";
248 foreach my $p (@ILLEGAL_PROPERTIES) {
4003ea29
KW
249 my $pat;
250 if ($p =~ /::/) {
251 $pat = qr /^Illegal user-defined property name/;
252 }
253 else {
254 $pat = qr /^Can't find Unicode property definition/;
255 }
256
84281c31
A
257 undef $@;
258 my $r = eval "'a' =~ /\\p{$p}/; 1";
72d70e56
NC
259 is($r, undef, "Unknown Unicode property \\p{$p}");
260 like($@, $pat, "Unknown Unicode property \\p{$p}");
84281c31
A
261 undef $@;
262 my $s = eval "'a' =~ /\\P{$p}/; 1";
72d70e56
NC
263 is($s, undef, "Unknown Unicode property \\p{$p}");
264 like($@, $pat, "Unknown Unicode property \\p{$p}");
84281c31
A
265 if (length $p == 1) {
266 undef $@;
267 my $r = eval "'a' =~ /\\p$p/; 1";
72d70e56
NC
268 is($r, undef, "Unknown Unicode property \\p$p");
269 like($@, $pat, "Unknown Unicode property \\p$p");
84281c31
A
270 undef $@;
271 my $s = eval "'a' =~ /\\P$p/; 1";
72d70e56
NC
272 is($r, undef, "Unknown Unicode property \\P$p");
273 like($@, $pat, "Unknown Unicode property \\P$p");
84281c31
A
274 }
275 }
19c4061a
KW
276
277 print "# User-defined properties with /i differences\n";
49fdc6e8 278 while (my $class = shift @USER_CASELESS_PROPERTIES) {
19c4061a
KW
279 my $chars_ref = shift @USER_CASELESS_PROPERTIES;
280 my @in = grep {!/^!./} @$chars_ref;
281 my @out = map {s/^!(?=.)//; $_} grep { /^!./} @$chars_ref;
282 my $in_pat = eval qq ['\\p{$class}'];
283 my $out_pat = eval qq ['\\P{$class}'];
284
285 # Verify works as regularly for not /i
286 match $_, $in_pat, $out_pat for @in;
287 match $_, $out_pat, $in_pat for @out;
288
289 # Verify that adding /i doesn't change the in set.
290 match $_, $in_pat, $out_pat, 'i' for @in;
291
292 # Verify that adding /i does change the out set to match.
293 match $_, $in_pat, $out_pat, 'i' for @out;
294 }
84281c31
A
295}
296
297
298#
299# User defined properties
300#
301
302sub InKana1 {<<'--'}
3033040 309F
30430A0 30FF
305--
306
307sub InKana2 {<<'--'}
308+utf8::InHiragana
309+utf8::InKatakana
310--
311
312sub InKana3 {<<'--'}
313+utf8::InHiragana
314+utf8::InKatakana
315-utf8::IsCn
316--
317
318sub InNotKana {<<'--'}
319!utf8::InHiragana
320-utf8::InKatakana
321+utf8::IsCn
322--
323
425224e1
KW
324sub InConsonant {
325
326 my $return = "+utf8::Lowercase\n&utf8::ASCII\n";
327 $return .= sprintf("-%X\n", ord "a");
328 $return .= sprintf("-%X\n", ord "e");
329 $return .= sprintf("-%X\n", ord "i");
330 $return .= sprintf("-%X\n", ord "o");
331 $return .= sprintf("-%X\n", ord "u");
332 return $return;
333}
84281c31
A
334
335sub IsSyriac1 {<<'--'}
3360712 072C
3370730 074A
338--
339
84281c31
A
340sub InGreekSmall {return "03B1\t03C9"}
341sub InGreekCapital {return "0391\t03A9\n-03A2"}
342
d658a8a8 343sub IsAsciiHexAndDash {<<'--'}
84281c31
A
344+utf8::ASCII_Hex_Digit
345+utf8::Dash
346--
347
eb765580
KW
348sub InLatin1 {
349 return "0100\t10FFFF";
350}
351
19c4061a
KW
352sub IsMyUpper {
353 my $caseless = shift;
425224e1
KW
354 return "+utf8::"
355 . (($caseless)
356 ? 'Alphabetic'
357 : 'Uppercase')
358 . "\n&utf8::ASCII";
19c4061a
KW
359}
360
710d3eb3
KW
361{ # This has to be done here and not like the others, because we have to
362 # make sure that the property is not known until after the regex is
363 # compiled. It was previously getting confused about the pkg and /i
364 # combination
365
366 my $mylower = qr/\p{pkg::IsMyLower}/i;
367
3b9fb374
KW
368sub pkg::IsMyLower {
369 my $caseless = shift;
370 return "+utf8::"
371 . (($caseless)
372 ? 'Alphabetic'
373 : 'Lowercase')
374 . "\n&utf8::ASCII";
375}
710d3eb3
KW
376
377 like("A", $mylower, "Not available until runtime user-defined property with pkg:: and /i works");
378
379}
380
8fd2c59a
KW
381# Verify that can use user-defined properties inside another one
382sub IsSyriac1KanaMark {<<'--'}
383+main::IsSyriac1
384+main::InKana3
385&utf8::IsMark
386--
387
d658a8a8
DM
388# fake user-defined properties; these subs shouldn't be called, because
389# their names don't start with In or Is
390
391sub f { die }
392sub foo { die }
393sub isfoo { die }
394sub infoo { die }
395sub ISfoo { die }
396sub INfoo { die }
397sub Is::foo { die }
398sub In::foo { die }
0a441b11
KW
399
4001;
84281c31 401__END__