This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Upgrade to Digest 1.11 with a minor Pod fix. Also my first commit :)
[perl5.git] / ext / Encode / Encode.pm
CommitLineData
10c5ecbb 1#
56ff7374 2# $Id: Encode.pm,v 2.11 2005/08/05 10:58:25 dankogai Exp dankogai $
10c5ecbb 3#
2c674647 4package Encode;
51ef4e11 5use strict;
56ff7374 6our $VERSION = sprintf "%d.%02d", q$Revision: 2.11 $ =~ /(\d+)/g;
8f139f4c 7sub DEBUG () { 0 }
6d1c0808 8use XSLoader ();
10c5ecbb 9XSLoader::load(__PACKAGE__, $VERSION);
2c674647 10
2c674647 11require Exporter;
7e19fb92 12use base qw/Exporter/;
2c674647 13
4411f3b6 14# Public, encouraged API is exported by default
85982a32
JH
15
16our @EXPORT = qw(
17 decode decode_utf8 encode encode_utf8
a0d8a30e 18 encodings find_encoding clone_encoding
4411f3b6
NIS
19);
20
b7a5c9de 21our @FB_FLAGS = qw(DIE_ON_ERR WARN_ON_ERR RETURN_ON_ERR LEAVE_SRC
56ff7374 22 PERLQQ HTMLCREF XMLCREF STOP_AT_PARTIAL);
b7a5c9de 23our @FB_CONSTS = qw(FB_DEFAULT FB_CROAK FB_QUIET FB_WARN
af1f55d9 24 FB_PERLQQ FB_HTMLCREF FB_XMLCREF);
85982a32 25
51ef4e11 26our @EXPORT_OK =
6d1c0808 27 (
85982a32
JH
28 qw(
29 _utf8_off _utf8_on define_encoding from_to is_16bit is_8bit
30 is_utf8 perlio_ok resolve_alias utf8_downgrade utf8_upgrade
31 ),
32 @FB_FLAGS, @FB_CONSTS,
33 );
34
6d1c0808 35our %EXPORT_TAGS =
85982a32
JH
36 (
37 all => [ @EXPORT, @EXPORT_OK ],
38 fallbacks => [ @FB_CONSTS ],
39 fallback_all => [ @FB_CONSTS, @FB_FLAGS ],
40 );
41
4411f3b6 42# Documentation moved after __END__ for speed - NI-S
2c674647 43
a63c962f 44our $ON_EBCDIC = (ord("A") == 193);
f2a2953c 45
5d030b67
JH
46use Encode::Alias;
47
5129552c
JH
48# Make a %Encoding package variable to allow a certain amount of cheating
49our %Encoding;
aae85ceb
DK
50our %ExtModule;
51require Encode::Config;
52eval { require Encode::ConfigLocal };
5129552c 53
656753f8
NIS
54sub encodings
55{
5129552c 56 my $class = shift;
fc17bd48
JH
57 my %enc;
58 if (@_ and $_[0] eq ":all"){
59 %enc = ( %Encoding, %ExtModule );
60 }else{
61 %enc = %Encoding;
62 for my $mod (map {m/::/o ? $_ : "Encode::$_" } @_){
8f139f4c 63 DEBUG and warn $mod;
fc17bd48
JH
64 for my $enc (keys %ExtModule){
65 $ExtModule{$enc} eq $mod and $enc{$enc} = $mod;
66 }
67 }
5129552c
JH
68 }
69 return
ce912cd4 70 sort { lc $a cmp lc $b }
fc17bd48 71 grep {!/^(?:Internal|Unicode|Guess)$/o} keys %enc;
51ef4e11
NIS
72}
73
85982a32 74sub perlio_ok{
0ab8f81e 75 my $obj = ref($_[0]) ? $_[0] : find_encoding($_[0]);
011b2d2f 76 $obj->can("perlio_ok") and return $obj->perlio_ok();
0ab8f81e 77 return 0; # safety net
85982a32
JH
78}
79
51ef4e11
NIS
80sub define_encoding
81{
18586f54
NIS
82 my $obj = shift;
83 my $name = shift;
5129552c 84 $Encoding{$name} = $obj;
18586f54
NIS
85 my $lc = lc($name);
86 define_alias($lc => $obj) unless $lc eq $name;
10c5ecbb 87 while (@_){
18586f54 88 my $alias = shift;
10c5ecbb 89 define_alias($alias, $obj);
18586f54
NIS
90 }
91 return $obj;
656753f8
NIS
92}
93
656753f8
NIS
94sub getEncoding
95{
10c5ecbb
JH
96 my ($class, $name, $skip_external) = @_;
97
a0d8a30e 98 ref($name) && $name->can('renew') and return $name;
10c5ecbb 99 exists $Encoding{$name} and return $Encoding{$name};
18586f54 100 my $lc = lc $name;
10c5ecbb 101 exists $Encoding{$lc} and return $Encoding{$lc};
c50d192e 102
5129552c 103 my $oc = $class->find_alias($name);
10c5ecbb
JH
104 defined($oc) and return $oc;
105 $lc ne $name and $oc = $class->find_alias($lc);
106 defined($oc) and return $oc;
c50d192e 107
c731e18e 108 unless ($skip_external)
d1ed7747 109 {
c731e18e
JH
110 if (my $mod = $ExtModule{$name} || $ExtModule{$lc}){
111 $mod =~ s,::,/,g ; $mod .= '.pm';
112 eval{ require $mod; };
10c5ecbb 113 exists $Encoding{$name} and return $Encoding{$name};
c731e18e 114 }
d1ed7747 115 }
18586f54 116 return;
656753f8
NIS
117}
118
a0d8a30e 119sub find_encoding($;$)
4411f3b6 120{
10c5ecbb 121 my ($name, $skip_external) = @_;
dd9703c9 122 return __PACKAGE__->getEncoding($name,$skip_external);
4411f3b6
NIS
123}
124
a0d8a30e 125sub resolve_alias($){
fcb875d4
JH
126 my $obj = find_encoding(shift);
127 defined $obj and return $obj->name;
128 return;
129}
130
a0d8a30e
DK
131sub clone_encoding($){
132 my $obj = find_encoding(shift);
133 ref $obj or return;
134 eval { require Storable };
135 $@ and return;
136 return Storable::dclone($obj);
137}
138
b2704119 139sub encode($$;$)
4411f3b6 140{
e8c86ba6 141 my ($name, $string, $check) = @_;
0f7c507f 142 return undef unless defined $string;
78589665 143 $string .= '' if ref $string; # stringify;
b2704119 144 $check ||=0;
18586f54 145 my $enc = find_encoding($name);
10c5ecbb
JH
146 unless(defined $enc){
147 require Carp;
148 Carp::croak("Unknown encoding '$name'");
149 }
18586f54 150 my $octets = $enc->encode($string,$check);
7f0d54d7 151 $_[1] = $string if $check and !($check & LEAVE_SRC());
18586f54 152 return $octets;
4411f3b6
NIS
153}
154
b2704119 155sub decode($$;$)
4411f3b6 156{
18586f54 157 my ($name,$octets,$check) = @_;
0f7c507f 158 return undef unless defined $octets;
78589665 159 $octets .= '' if ref $octets;
b2704119 160 $check ||=0;
18586f54 161 my $enc = find_encoding($name);
10c5ecbb
JH
162 unless(defined $enc){
163 require Carp;
164 Carp::croak("Unknown encoding '$name'");
165 }
18586f54 166 my $string = $enc->decode($octets,$check);
7f0d54d7 167 $_[1] = $octets if $check and !($check & LEAVE_SRC());
18586f54 168 return $string;
4411f3b6
NIS
169}
170
b2704119 171sub from_to($$$;$)
4411f3b6 172{
18586f54 173 my ($string,$from,$to,$check) = @_;
0f7c507f 174 return undef unless defined $string;
b2704119 175 $check ||=0;
18586f54 176 my $f = find_encoding($from);
10c5ecbb
JH
177 unless (defined $f){
178 require Carp;
179 Carp::croak("Unknown encoding '$from'");
180 }
18586f54 181 my $t = find_encoding($to);
10c5ecbb
JH
182 unless (defined $t){
183 require Carp;
184 Carp::croak("Unknown encoding '$to'");
185 }
18586f54
NIS
186 my $uni = $f->decode($string,$check);
187 return undef if ($check && length($string));
a999c27c 188 $string = $t->encode($uni,$check);
18586f54 189 return undef if ($check && length($uni));
3ef515df 190 return defined($_[0] = $string) ? length($string) : undef ;
4411f3b6
NIS
191}
192
b2704119 193sub encode_utf8($)
4411f3b6 194{
18586f54 195 my ($str) = @_;
c731e18e 196 utf8::encode($str);
18586f54 197 return $str;
4411f3b6
NIS
198}
199
c2cbba7d 200sub decode_utf8($;$)
4411f3b6 201{
c2cbba7d
RGS
202 my ($str, $check) = @_;
203 if ($check){
204 return decode("utf8", $str, $check);
205 }else{
dc4a2e29 206 return decode("utf8", $str);
c2cbba7d
RGS
207 return $str;
208 }
5ad8ef52
NIS
209}
210
b536bf57 211predefine_encodings(1);
f2a2953c
JH
212
213#
214# This is to restore %Encoding if really needed;
215#
10c5ecbb 216
f2a2953c 217sub predefine_encodings{
10c5ecbb 218 use Encode::Encoding;
b536bf57
DK
219 no warnings 'redefine';
220 my $use_xs = shift;
6d1c0808 221 if ($ON_EBCDIC) {
f2a2953c
JH
222 # was in Encode::UTF_EBCDIC
223 package Encode::UTF_EBCDIC;
10c5ecbb 224 push @Encode::UTF_EBCDIC::ISA, 'Encode::Encoding';
f2a2953c
JH
225 *decode = sub{
226 my ($obj,$str,$chk) = @_;
227 my $res = '';
228 for (my $i = 0; $i < length($str); $i++) {
6d1c0808 229 $res .=
f2a2953c
JH
230 chr(utf8::unicode_to_native(ord(substr($str,$i,1))));
231 }
232 $_[1] = '' if $chk;
233 return $res;
234 };
235 *encode = sub{
236 my ($obj,$str,$chk) = @_;
237 my $res = '';
238 for (my $i = 0; $i < length($str); $i++) {
6d1c0808 239 $res .=
f2a2953c
JH
240 chr(utf8::native_to_unicode(ord(substr($str,$i,1))));
241 }
242 $_[1] = '' if $chk;
243 return $res;
244 };
6d1c0808 245 $Encode::Encoding{Unicode} =
c731e18e 246 bless {Name => "UTF_EBCDIC"} => "Encode::UTF_EBCDIC";
6d1c0808 247 } else {
f2a2953c 248 package Encode::Internal;
10c5ecbb 249 push @Encode::Internal::ISA, 'Encode::Encoding';
f2a2953c
JH
250 *decode = sub{
251 my ($obj,$str,$chk) = @_;
252 utf8::upgrade($str);
253 $_[1] = '' if $chk;
254 return $str;
255 };
256 *encode = \&decode;
6d1c0808 257 $Encode::Encoding{Unicode} =
c731e18e 258 bless {Name => "Internal"} => "Encode::Internal";
f2a2953c
JH
259 }
260
261 {
262 # was in Encode::utf8
263 package Encode::utf8;
10c5ecbb 264 push @Encode::utf8::ISA, 'Encode::Encoding';
b536bf57
DK
265 #
266 if ($use_xs){
8f139f4c 267 Encode::DEBUG and warn __PACKAGE__, " XS on";
b536bf57
DK
268 *decode = \&decode_xs;
269 *encode = \&encode_xs;
270 }else{
8f139f4c 271 Encode::DEBUG and warn __PACKAGE__, " XS off";
b536bf57
DK
272 *decode = sub{
273 my ($obj,$octets,$chk) = @_;
274 my $str = Encode::decode_utf8($octets);
275 if (defined $str) {
276 $_[1] = '' if $chk;
277 return $str;
278 }
279 return undef;
280 };
281 *encode = sub {
282 my ($obj,$string,$chk) = @_;
283 my $octets = Encode::encode_utf8($string);
284 $_[1] = '' if $chk;
285 return $octets;
286 };
287 }
220e2d4e
IH
288 *cat_decode = sub{ # ($obj, $dst, $src, $pos, $trm, $chk)
289 my ($obj, undef, undef, $pos, $trm) = @_; # currently ignores $chk
290 my ($rdst, $rsrc, $rpos) = \@_[1,2,3];
291 use bytes;
292 if ((my $npos = index($$rsrc, $trm, $pos)) >= 0) {
293 $$rdst .= substr($$rsrc, $pos, $npos - $pos + length($trm));
294 $$rpos = $npos + length($trm);
295 return 1;
296 }
297 $$rdst .= substr($$rsrc, $pos);
298 $$rpos = length($$rsrc);
299 return '';
300 };
b7a5c9de 301 $Encode::Encoding{utf8} =
c731e18e 302 bless {Name => "utf8"} => "Encode::utf8";
7f0d54d7
RGS
303 $Encode::Encoding{"utf-8-strict"} =
304 bless {Name => "utf-8-strict", strict_utf8 => 1 } => "Encode::utf8";
f2a2953c 305 }
f2a2953c
JH
306}
307
656753f8
NIS
3081;
309
2a936312
NIS
310__END__
311
4411f3b6
NIS
312=head1 NAME
313
314Encode - character encodings
315
316=head1 SYNOPSIS
317
318 use Encode;
319
67d7b5ef
JH
320=head2 Table of Contents
321
0ab8f81e 322Encode consists of a collection of modules whose details are too big
67d7b5ef 323to fit in one document. This POD itself explains the top-level APIs
6d1c0808 324and general topics at a glance. For other topics and more details,
0ab8f81e 325see the PODs below:
67d7b5ef
JH
326
327 Name Description
328 --------------------------------------------------------
6d1c0808 329 Encode::Alias Alias definitions to encodings
67d7b5ef
JH
330 Encode::Encoding Encode Implementation Base Class
331 Encode::Supported List of Supported Encodings
332 Encode::CN Simplified Chinese Encodings
333 Encode::JP Japanese Encodings
334 Encode::KR Korean Encodings
335 Encode::TW Traditional Chinese Encodings
336 --------------------------------------------------------
337
4411f3b6
NIS
338=head1 DESCRIPTION
339
47bfe92f 340The C<Encode> module provides the interfaces between Perl's strings
67d7b5ef
JH
341and the rest of the system. Perl strings are sequences of
342B<characters>.
343
344The repertoire of characters that Perl can represent is at least that
345defined by the Unicode Consortium. On most platforms the ordinal
346values of the characters (as returned by C<ord(ch)>) is the "Unicode
347codepoint" for the character (the exceptions are those platforms where
348the legacy encoding is some variant of EBCDIC rather than a super-set
349of ASCII - see L<perlebcdic>).
350
0ab8f81e 351Traditionally, computer data has been moved around in 8-bit chunks
67d7b5ef
JH
352often called "bytes". These chunks are also known as "octets" in
353networking standards. Perl is widely used to manipulate data of many
354types - not only strings of characters representing human or computer
0ab8f81e 355languages but also "binary" data being the machine's representation of
67d7b5ef
JH
356numbers, pixels in an image - or just about anything.
357
0ab8f81e 358When Perl is processing "binary data", the programmer wants Perl to
67d7b5ef 359process "sequences of bytes". This is not a problem for Perl - as a
0ab8f81e 360byte has 256 possible values, it easily fits in Perl's much larger
67d7b5ef
JH
361"logical character".
362
363=head2 TERMINOLOGY
4411f3b6 364
7e19fb92 365=over 2
21938dfa 366
67d7b5ef
JH
367=item *
368
369I<character>: a character in the range 0..(2**32-1) (or more).
370(What Perl's strings are made of.)
371
372=item *
373
374I<byte>: a character in the range 0..255
375(A special case of a Perl character.)
376
377=item *
378
379I<octet>: 8 bits of data, with ordinal values 0..255
0ab8f81e 380(Term for bytes passed to or from a non-Perl context, e.g. a disk file.)
67d7b5ef
JH
381
382=back
4411f3b6 383
67d7b5ef 384=head1 PERL ENCODING API
4411f3b6 385
7e19fb92 386=over 2
4411f3b6 387
b7a5c9de 388=item $octets = encode(ENCODING, $string [, CHECK])
4411f3b6 389
0ab8f81e 390Encodes a string from Perl's internal form into I<ENCODING> and returns
67d7b5ef 391a sequence of octets. ENCODING can be either a canonical name or
0ab8f81e
JH
392an alias. For encoding names and aliases, see L</"Defining Aliases">.
393For CHECK, see L</"Handling Malformed Data">.
4411f3b6 394
b7a5c9de 395For example, to convert a string from Perl's internal format to
6d1c0808 396iso-8859-1 (also known as Latin1),
681a7c68 397
b7a5c9de 398 $octets = encode("iso-8859-1", $string);
7e19fb92 399
b7a5c9de
JH
400B<CAVEAT>: When you run C<$octets = encode("utf8", $string)>, then $octets
401B<may not be equal to> $string. Though they both contain the same data, the utf8 flag
7e19fb92
JH
402for $octets is B<always> off. When you encode anything, utf8 flag of
403the result is always off, even when it contains completely valid utf8
404string. See L</"The UTF-8 flag"> below.
681a7c68 405
7f0d54d7 406If the $string is C<undef> then C<undef> is returned.
4089adc4 407
b7a5c9de 408=item $string = decode(ENCODING, $octets [, CHECK])
4411f3b6 409
0ab8f81e
JH
410Decodes a sequence of octets assumed to be in I<ENCODING> into Perl's
411internal form and returns the resulting string. As in encode(),
412ENCODING can be either a canonical name or an alias. For encoding names
413and aliases, see L</"Defining Aliases">. For CHECK, see
47bfe92f
JH
414L</"Handling Malformed Data">.
415
b7a5c9de 416For example, to convert ISO-8859-1 data to a string in Perl's internal format:
681a7c68 417
b7a5c9de 418 $string = decode("iso-8859-1", $octets);
681a7c68 419
b7a5c9de
JH
420B<CAVEAT>: When you run C<$string = decode("utf8", $octets)>, then $string
421B<may not be equal to> $octets. Though they both contain the same data,
422the utf8 flag for $string is on unless $octets entirely consists of
7e19fb92
JH
423ASCII data (or EBCDIC on EBCDIC machines). See L</"The UTF-8 flag">
424below.
47bfe92f 425
7f0d54d7 426If the $string is C<undef> then C<undef> is returned.
4089adc4 427
b7a5c9de 428=item [$length =] from_to($octets, FROM_ENC, TO_ENC [, CHECK])
7e19fb92 429
b7a5c9de
JH
430Converts B<in-place> data between two encodings. The data in $octets
431must be encoded as octets and not as characters in Perl's internal
f9d05ba3
RGS
432format. For example, to convert ISO-8859-1 data to Microsoft's CP1250
433encoding:
2b106fbe 434
b7a5c9de 435 from_to($octets, "iso-8859-1", "cp1250");
2b106fbe
JH
436
437and to convert it back:
438
b7a5c9de 439 from_to($octets, "cp1250", "iso-8859-1");
4411f3b6 440
ab97ca19 441Note that because the conversion happens in place, the data to be
0ab8f81e 442converted cannot be a string constant; it must be a scalar variable.
ab97ca19 443
f9d05ba3
RGS
444from_to() returns the length of the converted string in octets on
445success, I<undef> on error.
3ef515df 446
b7a5c9de 447B<CAVEAT>: The following operations look the same but are not quite so;
7e19fb92 448
b7a5c9de 449 from_to($data, "iso-8859-1", "utf8"); #1
7e19fb92 450 $data = decode("iso-8859-1", $data); #2
4411f3b6 451
b7a5c9de 452Both #1 and #2 make $data consist of a completely valid UTF-8 string
7e19fb92 453but only #2 turns utf8 flag on. #1 is equivalent to
f2a2953c 454
7e19fb92 455 $data = encode("utf8", decode("iso-8859-1", $data));
f2a2953c 456
7e19fb92 457See L</"The UTF-8 flag"> below.
f2a2953c
JH
458
459=item $octets = encode_utf8($string);
460
7e19fb92 461Equivalent to C<$octets = encode("utf8", $string);> The characters
b7a5c9de
JH
462that comprise $string are encoded in Perl's internal format and the
463result is returned as a sequence of octets. All possible
7e19fb92
JH
464characters have a UTF-8 representation so this function cannot fail.
465
f2a2953c
JH
466
467=item $string = decode_utf8($octets [, CHECK]);
468
7e19fb92 469equivalent to C<$string = decode("utf8", $octets [, CHECK])>.
b7a5c9de 470The sequence of octets represented by
7e19fb92
JH
471$octets is decoded from UTF-8 into a sequence of logical
472characters. Not all sequences of octets form valid UTF-8 encodings, so
473it is possible for this call to fail. For CHECK, see
474L</"Handling Malformed Data">.
f2a2953c
JH
475
476=back
477
51ef4e11
NIS
478=head2 Listing available encodings
479
5129552c
JH
480 use Encode;
481 @list = Encode->encodings();
482
483Returns a list of the canonical names of the available encodings that
484are loaded. To get a list of all available encodings including the
485ones that are not loaded yet, say
486
487 @all_encodings = Encode->encodings(":all");
488
0ab8f81e 489Or you can give the name of a specific module.
5129552c 490
c731e18e
JH
491 @with_jp = Encode->encodings("Encode::JP");
492
493When "::" is not in the name, "Encode::" is assumed.
51ef4e11 494
c731e18e 495 @ebcdic = Encode->encodings("EBCDIC");
5d030b67 496
0ab8f81e 497To find out in detail which encodings are supported by this package,
5d030b67 498see L<Encode::Supported>.
51ef4e11
NIS
499
500=head2 Defining Aliases
501
0ab8f81e 502To add a new alias to a given encoding, use:
67d7b5ef 503
5129552c
JH
504 use Encode;
505 use Encode::Alias;
a63c962f 506 define_alias(newName => ENCODING);
51ef4e11 507
3ef515df 508After that, newName can be used as an alias for ENCODING.
f2a2953c
JH
509ENCODING may be either the name of an encoding or an
510I<encoding object>
51ef4e11 511
fcb875d4
JH
512But before you do so, make sure the alias is nonexistent with
513C<resolve_alias()>, which returns the canonical name thereof.
514i.e.
515
516 Encode::resolve_alias("latin1") eq "iso-8859-1" # true
517 Encode::resolve_alias("iso-8859-12") # false; nonexistent
518 Encode::resolve_alias($name) eq $name # true if $name is canonical
519
0ab8f81e
JH
520resolve_alias() does not need C<use Encode::Alias>; it can be
521exported via C<use Encode qw(resolve_alias)>.
fcb875d4 522
0ab8f81e 523See L<Encode::Alias> for details.
51ef4e11 524
85982a32 525=head1 Encoding via PerlIO
4411f3b6 526
b7a5c9de 527If your perl supports I<PerlIO> (which is the default), you can use a PerlIO layer to decode
0ab8f81e
JH
528and encode directly via a filehandle. The following two examples
529are totally identical in their functionality.
4411f3b6 530
85982a32
JH
531 # via PerlIO
532 open my $in, "<:encoding(shiftjis)", $infile or die;
533 open my $out, ">:encoding(euc-jp)", $outfile or die;
b7a5c9de 534 while(<$in>){ print $out $_; }
8e86646e 535
85982a32 536 # via from_to
0ab8f81e
JH
537 open my $in, "<", $infile or die;
538 open my $out, ">", $outfile or die;
b7a5c9de 539 while(<$in>){
0ab8f81e 540 from_to($_, "shiftjis", "euc-jp", 1);
b7a5c9de 541 print $out $_;
85982a32 542 }
4411f3b6 543
b7a5c9de 544Unfortunately, it may be that encodings are PerlIO-savvy. You can check
0ab8f81e
JH
545if your encoding is supported by PerlIO by calling the C<perlio_ok>
546method.
547
548 Encode::perlio_ok("hz"); # False
549 find_encoding("euc-cn")->perlio_ok; # True where PerlIO is available
550
551 use Encode qw(perlio_ok); # exported upon request
552 perlio_ok("euc-jp")
4411f3b6 553
0ab8f81e 554Fortunately, all encodings that come with Encode core are PerlIO-savvy
f9d05ba3
RGS
555except for hz and ISO-2022-kr. For gory details, see
556L<Encode::Encoding> and L<Encode::PerlIO>.
4411f3b6 557
85982a32 558=head1 Handling Malformed Data
4411f3b6 559
f9d05ba3
RGS
560The optional I<CHECK> argument is used as follows. When you omit it,
561Encode::FB_DEFAULT ( == 0 ) is assumed.
562
563=over 2
564
3c4b39be 565=item B<NOTE:> Not all encoding support this feature
f9d05ba3
RGS
566
567Some encodings ignore I<CHECK> argument. For example,
568L<Encode::Unicode> ignores I<CHECK> and it always croaks on error.
569
570=back
571
572Now here is the list of I<CHECK> values available
47bfe92f 573
151b5d36
JH
574=over 2
575
85982a32 576=item I<CHECK> = Encode::FB_DEFAULT ( == 0)
47bfe92f 577
f9d05ba3 578If I<CHECK> is 0, (en|de)code will put a I<substitution character> in
78589665
RGS
579place of a malformed character. When you encode, E<lt>subcharE<gt>
580will be used. When you decode the code point C<0xFFFD> is used. If
581the data is supposed to be UTF-8, an optional lexical warning
582(category utf8) is given.
e9692b5b 583
7e19fb92 584=item I<CHECK> = Encode::FB_CROAK ( == 1)
e9692b5b 585
b7a5c9de 586If I<CHECK> is 1, methods will die on error immediately with an error
0ab8f81e 587message. Therefore, when I<CHECK> is set to 1, you should trap the
f9d05ba3 588error with eval{} unless you really want to let it die.
47bfe92f 589
85982a32 590=item I<CHECK> = Encode::FB_QUIET
47bfe92f 591
85982a32 592If I<CHECK> is set to Encode::FB_QUIET, (en|de)code will immediately
f9d05ba3
RGS
593return the portion of the data that has been processed so far when an
594error occurs. The data argument will be overwritten with everything
595after that point (that is, the unprocessed part of data). This is
596handy when you have to call decode repeatedly in the case where your
597source data may contain partial multi-byte character sequences,
598(i.e. you are reading with a fixed-width buffer). Here is a sample
599code that does exactly this:
4411f3b6 600
78589665
RGS
601 my $buffer = ''; my $string = '';
602 while(read $fh, $buffer, 256, length($buffer)){
603 $string .= decode($encoding, $buffer, Encode::FB_QUIET);
604 # $buffer now contains the unprocessed partial character
85982a32 605 }
1768d7eb 606
85982a32 607=item I<CHECK> = Encode::FB_WARN
67d7b5ef 608
0ab8f81e
JH
609This is the same as above, except that it warns on error. Handy when
610you are debugging the mode above.
85982a32
JH
611
612=item perlqq mode (I<CHECK> = Encode::FB_PERLQQ)
613
af1f55d9
JH
614=item HTML charref mode (I<CHECK> = Encode::FB_HTMLCREF)
615
616=item XML charref mode (I<CHECK> = Encode::FB_XMLCREF)
617
85982a32
JH
618For encodings that are implemented by Encode::XS, CHECK ==
619Encode::FB_PERLQQ turns (en|de)code into C<perlqq> fallback mode.
620
b7a5c9de
JH
621When you decode, C<\xI<HH>> will be inserted for a malformed character,
622where I<HH> is the hex representation of the octet that could not be
623decoded to utf8. And when you encode, C<\x{I<HHHH>}> will be inserted,
624where I<HHHH> is the Unicode ID of the character that cannot be found
0ab8f81e 625in the character repertoire of the encoding.
85982a32 626
af1f55d9 627HTML/XML character reference modes are about the same, in place of
78589665
RGS
628C<\x{I<HHHH>}>, HTML uses C<&#I<NNN>;> where I<NNN> is a decimal number and
629XML uses C<&#xI<HHHH>;> where I<HHHH> is the hexadecimal number.
af1f55d9 630
7f0d54d7
RGS
631In Encode 2.10 or later, C<LEAVE_SRC> is also implied.
632
85982a32
JH
633=item The bitmask
634
0ab8f81e
JH
635These modes are actually set via a bitmask. Here is how the FB_XX
636constants are laid out. You can import the FB_XX constants via
637C<use Encode qw(:fallbacks)>; you can import the generic bitmask
638constants via C<use Encode qw(:fallback_all)>.
85982a32 639
b0b300a3
JH
640 FB_DEFAULT FB_CROAK FB_QUIET FB_WARN FB_PERLQQ
641 DIE_ON_ERR 0x0001 X
4089adc4 642 WARN_ON_ERR 0x0002 X
b0b300a3 643 RETURN_ON_ERR 0x0004 X X
7f0d54d7 644 LEAVE_SRC 0x0008 X
b0b300a3 645 PERLQQ 0x0100 X
b7a5c9de
JH
646 HTMLCREF 0x0200
647 XMLCREF 0x0400
67d7b5ef 648
151b5d36
JH
649=back
650
0ab8f81e 651=head2 Unimplemented fallback schemes
67d7b5ef 652
0ab8f81e 653In the future, you will be able to use a code reference to a callback
f2a2953c 654function for the value of I<CHECK> but its API is still undecided.
67d7b5ef 655
982a4085
JH
656The fallback scheme does not work on EBCDIC platforms.
657
67d7b5ef
JH
658=head1 Defining Encodings
659
660To define a new encoding, use:
661
b7a5c9de 662 use Encode qw(define_encoding);
67d7b5ef
JH
663 define_encoding($object, 'canonicalName' [, alias...]);
664
665I<canonicalName> will be associated with I<$object>. The object
0ab8f81e 666should provide the interface described in L<Encode::Encoding>.
67d7b5ef 667If more than two arguments are provided then additional
b7a5c9de 668arguments are taken as aliases for I<$object>.
67d7b5ef 669
f2a2953c
JH
670See L<Encode::Encoding> for more details.
671
7e19fb92
JH
672=head1 The UTF-8 flag
673
674Before the introduction of utf8 support in perl, The C<eq> operator
b7a5c9de
JH
675just compared the strings represented by two scalars. Beginning with
676perl 5.8, C<eq> compares two strings with simultaneous consideration
677of I<the utf8 flag>. To explain why we made it so, I will quote page
678402 of C<Programming Perl, 3rd ed.>
7e19fb92
JH
679
680=over 2
681
682=item Goal #1:
683
684Old byte-oriented programs should not spontaneously break on the old
685byte-oriented data they used to work on.
686
687=item Goal #2:
688
689Old byte-oriented programs should magically start working on the new
690character-oriented data when appropriate.
691
692=item Goal #3:
693
694Programs should run just as fast in the new character-oriented mode
695as in the old byte-oriented mode.
696
697=item Goal #4:
698
699Perl should remain one language, rather than forking into a
700byte-oriented Perl and a character-oriented Perl.
701
702=back
703
704Back when C<Programming Perl, 3rd ed.> was written, not even Perl 5.6.0
705was born and many features documented in the book remained
b7a5c9de
JH
706unimplemented for a long time. Perl 5.8 corrected this and the introduction
707of the UTF-8 flag is one of them. You can think of this perl notion as of a
708byte-oriented mode (utf8 flag off) and a character-oriented mode (utf8
7e19fb92
JH
709flag on).
710
711Here is how Encode takes care of the utf8 flag.
712
4bdf5738 713=over 2
7e19fb92
JH
714
715=item *
716
717When you encode, the resulting utf8 flag is always off.
718
151b5d36 719=item *
7e19fb92 720
b7a5c9de 721When you decode, the resulting utf8 flag is on unless you can
7e19fb92
JH
722unambiguously represent data. Here is the definition of
723dis-ambiguity.
724
b7a5c9de 725After C<$utf8 = decode('foo', $octet);>,
7e19fb92
JH
726
727 When $octet is... The utf8 flag in $utf8 is
728 ---------------------------------------------
729 In ASCII only (or EBCDIC only) OFF
730 In ISO-8859-1 ON
731 In any other Encoding ON
732 ---------------------------------------------
733
3c4b39be 734As you see, there is one exception, In ASCII. That way you can assume
7e19fb92
JH
735Goal #1. And with Encode Goal #2 is assumed but you still have to be
736careful in such cases mentioned in B<CAVEAT> paragraphs.
737
738This utf8 flag is not visible in perl scripts, exactly for the same
739reason you cannot (or you I<don't have to>) see if a scalar contains a
740string, integer, or floating point number. But you can still peek
741and poke these if you will. See the section below.
742
743=back
744
745=head2 Messing with Perl's Internals
4411f3b6 746
47bfe92f 747The following API uses parts of Perl's internals in the current
0ab8f81e 748implementation. As such, they are efficient but may change.
4411f3b6 749
7e19fb92 750=over 2
4411f3b6 751
a63c962f 752=item is_utf8(STRING [, CHECK])
4411f3b6 753
0ab8f81e 754[INTERNAL] Tests whether the UTF-8 flag is turned on in the STRING.
47bfe92f
JH
755If CHECK is true, also checks the data in STRING for being well-formed
756UTF-8. Returns true if successful, false otherwise.
4411f3b6 757
2c246b25 758As of perl 5.8.1, L<utf8> also has utf8::is_utf8().
b5ab1f6f 759
a63c962f 760=item _utf8_on(STRING)
4411f3b6 761
0ab8f81e 762[INTERNAL] Turns on the UTF-8 flag in STRING. The data in STRING is
4411f3b6
NIS
763B<not> checked for being well-formed UTF-8. Do not use unless you
764B<know> that the STRING is well-formed UTF-8. Returns the previous
0ab8f81e
JH
765state of the UTF-8 flag (so please don't treat the return value as
766indicating success or failure), or C<undef> if STRING is not a string.
4411f3b6 767
a63c962f 768=item _utf8_off(STRING)
4411f3b6 769
0ab8f81e
JH
770[INTERNAL] Turns off the UTF-8 flag in STRING. Do not use frivolously.
771Returns the previous state of the UTF-8 flag (so please don't treat the
772return value as indicating success or failure), or C<undef> if STRING is
4411f3b6
NIS
773not a string.
774
775=back
776
7f0d54d7
RGS
777=head1 UTF-8 vs. utf8
778
779 ....We now view strings not as sequences of bytes, but as sequences
780 of numbers in the range 0 .. 2**32-1 (or in the case of 64-bit
781 computers, 0 .. 2**64-1) -- Programming Perl, 3rd ed.
782
783That has been the perl's notion of UTF-8 but official UTF-8 is more
784strict; Its ranges is much narrower (0 .. 10FFFF), some sequences are
785not allowed (i.e. Those used in the surrogate pair, 0xFFFE, et al).
786
787Now that is overruled by Larry Wall himself.
788
789 From: Larry Wall <larry@wall.org>
790 Date: December 04, 2004 11:51:58 JST
791 To: perl-unicode@perl.org
792 Subject: Re: Make Encode.pm support the real UTF-8
793 Message-Id: <20041204025158.GA28754@wall.org>
794
795 On Fri, Dec 03, 2004 at 10:12:12PM +0000, Tim Bunce wrote:
796 : I've no problem with 'utf8' being perl's unrestricted uft8 encoding,
797 : but "UTF-8" is the name of the standard and should give the
798 : corresponding behaviour.
799
800 For what it's worth, that's how I've always kept them straight in my
801 head.
802
803 Also for what it's worth, Perl 6 will mostly default to strict but
804 make it easy to switch back to lax.
805
806 Larry
807
808Do you copy? As of Perl 5.8.7, B<UTF-8> means strict, official UTF-8
809while B<utf8> means liberal, lax, version thereof. And Encode version
8102.10 or later thus groks the difference between C<UTF-8> and C"utf8".
811
812 encode("utf8", "\x{FFFF_FFFF}", 1); # okay
813 encode("UTF-8", "\x{FFFF_FFFF}", 1); # croaks
814
815C<UTF-8> in Encode is actually a canonical name for C<utf-8-strict>.
816Yes, the hyphen between "UTF" and "8" is important. Without it Encode
817goes "liberal"
818
819 find_encoding("UTF-8")->name # is 'utf-8-strict'
820 find_encoding("utf-8")->name # ditto. names are case insensitive
821 find_encoding("utf8")->name # ditto. "_" are treated as "-"
822 find_encoding("UTF8")->name # is 'utf8'.
823
824
4411f3b6
NIS
825=head1 SEE ALSO
826
5d030b67
JH
827L<Encode::Encoding>,
828L<Encode::Supported>,
6d1c0808 829L<Encode::PerlIO>,
5d030b67 830L<encoding>,
6d1c0808
JH
831L<perlebcdic>,
832L<perlfunc/open>,
833L<perlunicode>,
834L<utf8>,
5d030b67 835the Perl Unicode Mailing List E<lt>perl-unicode@perl.orgE<gt>
4411f3b6 836
85982a32 837=head1 MAINTAINER
aae85ceb
DK
838
839This project was originated by Nick Ing-Simmons and later maintained
7e19fb92
JH
840by Dan Kogai E<lt>dankogai@dan.co.jpE<gt>. See AUTHORS for a full
841list of people involved. For any questions, use
b7a5c9de 842E<lt>perl-unicode@perl.orgE<gt> so we can all share.
aae85ceb 843
4411f3b6 844=cut