This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Encode: synch with CPAN version 3.01
[perl5.git] / cpan / Encode / Encode.pm
CommitLineData
10c5ecbb 1#
e06f52f0 2# $Id: Encode.pm,v 3.01 2019/03/13 00:25:25 dankogai Exp $
10c5ecbb 3#
2c674647 4package Encode;
51ef4e11 5use strict;
656ebd29 6use warnings;
e46d9735 7use constant DEBUG => !!$ENV{PERL_ENCODE_DEBUG};
3f60a930
AC
8our $VERSION;
9BEGIN {
e06f52f0 10 $VERSION = sprintf "%d.%02d", q$Revision: 3.01 $ =~ /(\d+)/g;
3f60a930
AC
11 require XSLoader;
12 XSLoader::load( __PACKAGE__, $VERSION );
13}
2c674647 14
369b9ffe 15use Exporter 5.57 'import';
2c674647 16
20797ee1 17use Carp ();
3f60a930
AC
18our @CARP_NOT = qw(Encode::Encoder);
19
4411f3b6 20# Public, encouraged API is exported by default
85982a32
JH
21
22our @EXPORT = qw(
0a8c69ed 23 decode decode_utf8 encode encode_utf8 str2bytes bytes2str
15f5e486 24 encodings find_encoding find_mime_encoding clone_encoding
4411f3b6 25);
d1256cb1
RGS
26our @FB_FLAGS = qw(
27 DIE_ON_ERR WARN_ON_ERR RETURN_ON_ERR LEAVE_SRC
28 PERLQQ HTMLCREF XMLCREF STOP_AT_PARTIAL
29);
30our @FB_CONSTS = qw(
31 FB_DEFAULT FB_CROAK FB_QUIET FB_WARN
32 FB_PERLQQ FB_HTMLCREF FB_XMLCREF
33);
34our @EXPORT_OK = (
35 qw(
36 _utf8_off _utf8_on define_encoding from_to is_16bit is_8bit
37 is_utf8 perlio_ok resolve_alias utf8_downgrade utf8_upgrade
85982a32 38 ),
d1256cb1
RGS
39 @FB_FLAGS, @FB_CONSTS,
40);
85982a32 41
d1256cb1
RGS
42our %EXPORT_TAGS = (
43 all => [ @EXPORT, @EXPORT_OK ],
0263186c
NC
44 default => [ @EXPORT ],
45 fallbacks => [ @FB_CONSTS ],
d1256cb1
RGS
46 fallback_all => [ @FB_CONSTS, @FB_FLAGS ],
47);
85982a32 48
4411f3b6 49# Documentation moved after __END__ for speed - NI-S
2c674647 50
d1256cb1 51our $ON_EBCDIC = ( ord("A") == 193 );
f2a2953c 52
3f60a930
AC
53use Encode::Alias ();
54use Encode::MIME::Name;
55
56use Storable;
5d030b67 57
5129552c
JH
58# Make a %Encoding package variable to allow a certain amount of cheating
59our %Encoding;
aae85ceb
DK
60our %ExtModule;
61require Encode::Config;
2fd0906e
SH
62# See
63# https://bugzilla.redhat.com/show_bug.cgi?id=435505#c2
0a225b3c 64# to find why sig handlers inside eval{} are disabled.
2fd0906e
SH
65eval {
66 local $SIG{__DIE__};
67 local $SIG{__WARN__};
b8097e94
TC
68 local @INC = @INC;
69 pop @INC if $INC[-1] eq '.';
2fd0906e
SH
70 require Encode::ConfigLocal;
71};
5129552c 72
d1256cb1 73sub encodings {
fc17bd48 74 my %enc;
64a9a3c0
CBW
75 my $arg = $_[1] || '';
76 if ( $arg eq ":all" ) {
d1256cb1 77 %enc = ( %Encoding, %ExtModule );
5129552c 78 }
d1256cb1
RGS
79 else {
80 %enc = %Encoding;
b9370cdb 81 for my $mod ( map { m/::/ ? $_ : "Encode::$_" } @_ ) {
d1256cb1
RGS
82 DEBUG and warn $mod;
83 for my $enc ( keys %ExtModule ) {
84 $ExtModule{$enc} eq $mod and $enc{$enc} = $mod;
85 }
86 }
87 }
88 return sort { lc $a cmp lc $b }
89 grep { !/^(?:Internal|Unicode|Guess)$/o } keys %enc;
51ef4e11
NIS
90}
91
d1256cb1
RGS
92sub perlio_ok {
93 my $obj = ref( $_[0] ) ? $_[0] : find_encoding( $_[0] );
011b2d2f 94 $obj->can("perlio_ok") and return $obj->perlio_ok();
d1256cb1 95 return 0; # safety net
85982a32
JH
96}
97
d1256cb1 98sub define_encoding {
18586f54
NIS
99 my $obj = shift;
100 my $name = shift;
5129552c 101 $Encoding{$name} = $obj;
18586f54 102 my $lc = lc($name);
d1256cb1
RGS
103 define_alias( $lc => $obj ) unless $lc eq $name;
104 while (@_) {
105 my $alias = shift;
106 define_alias( $alias, $obj );
18586f54 107 }
3f60a930
AC
108 my $class = ref($obj);
109 push @Encode::CARP_NOT, $class unless grep { $_ eq $class } @Encode::CARP_NOT;
110 push @Encode::Encoding::CARP_NOT, $class unless grep { $_ eq $class } @Encode::Encoding::CARP_NOT;
18586f54 111 return $obj;
656753f8
NIS
112}
113
d1256cb1
RGS
114sub getEncoding {
115 my ( $class, $name, $skip_external ) = @_;
10c5ecbb 116
15f5e486
SH
117 defined($name) or return;
118
b9370cdb
CBW
119 $name =~ s/\s+//g; # https://rt.cpan.org/Ticket/Display.html?id=65796
120
a0d8a30e 121 ref($name) && $name->can('renew') and return $name;
10c5ecbb 122 exists $Encoding{$name} and return $Encoding{$name};
18586f54 123 my $lc = lc $name;
10c5ecbb 124 exists $Encoding{$lc} and return $Encoding{$lc};
c50d192e 125
5129552c 126 my $oc = $class->find_alias($name);
10c5ecbb
JH
127 defined($oc) and return $oc;
128 $lc ne $name and $oc = $class->find_alias($lc);
129 defined($oc) and return $oc;
c50d192e 130
d1256cb1
RGS
131 unless ($skip_external) {
132 if ( my $mod = $ExtModule{$name} || $ExtModule{$lc} ) {
133 $mod =~ s,::,/,g;
134 $mod .= '.pm';
135 eval { require $mod; };
136 exists $Encoding{$name} and return $Encoding{$name};
137 }
d1ed7747 138 }
18586f54 139 return;
656753f8
NIS
140}
141
3f60a930
AC
142# HACK: These two functions must be defined in Encode and because of
143# cyclic dependency between Encode and Encode::Alias, Exporter does not work
144sub find_alias {
145 goto &Encode::Alias::find_alias;
146}
147sub define_alias {
148 goto &Encode::Alias::define_alias;
149}
150
d1256cb1
RGS
151sub find_encoding($;$) {
152 my ( $name, $skip_external ) = @_;
153 return __PACKAGE__->getEncoding( $name, $skip_external );
4411f3b6
NIS
154}
155
15f5e486
SH
156sub find_mime_encoding($;$) {
157 my ( $mime_name, $skip_external ) = @_;
15f5e486
SH
158 my $name = Encode::MIME::Name::get_encode_name( $mime_name );
159 return find_encoding( $name, $skip_external );
160}
161
d1256cb1 162sub resolve_alias($) {
fcb875d4
JH
163 my $obj = find_encoding(shift);
164 defined $obj and return $obj->name;
165 return;
166}
167
d1256cb1 168sub clone_encoding($) {
a0d8a30e
DK
169 my $obj = find_encoding(shift);
170 ref $obj or return;
a0d8a30e
DK
171 return Storable::dclone($obj);
172}
173
3f60a930
AC
174onBOOT;
175
176if ($ON_EBCDIC) {
177 package Encode::UTF_EBCDIC;
178 use parent 'Encode::Encoding';
179 my $obj = bless { Name => "UTF_EBCDIC" } => "Encode::UTF_EBCDIC";
180 Encode::define_encoding($obj, 'Unicode');
181 sub decode {
182 my ( undef, $str, $chk ) = @_;
183 my $res = '';
184 for ( my $i = 0 ; $i < length($str) ; $i++ ) {
185 $res .=
186 chr(
187 utf8::unicode_to_native( ord( substr( $str, $i, 1 ) ) )
188 );
189 }
190 $_[1] = '' if $chk;
191 return $res;
d1256cb1 192 }
3f60a930
AC
193 sub encode {
194 my ( undef, $str, $chk ) = @_;
195 my $res = '';
196 for ( my $i = 0 ; $i < length($str) ; $i++ ) {
197 $res .=
198 chr(
199 utf8::native_to_unicode( ord( substr( $str, $i, 1 ) ) )
200 );
201 }
202 $_[1] = '' if $chk;
203 return $res;
f2a2953c 204 }
3f60a930
AC
205} else {
206 package Encode::Internal;
207 use parent 'Encode::Encoding';
208 my $obj = bless { Name => "Internal" } => "Encode::Internal";
209 Encode::define_encoding($obj, 'Unicode');
210 sub decode {
211 my ( undef, $str, $chk ) = @_;
212 utf8::upgrade($str);
213 $_[1] = '' if $chk;
214 return $str;
33bbbd9c 215 }
3f60a930
AC
216 *encode = \&decode;
217}
d1256cb1 218
3f60a930
AC
219{
220 # https://rt.cpan.org/Public/Bug/Display.html?id=103253
221 package Encode::XS;
222 use parent 'Encode::Encoding';
223}
d1256cb1 224
3f60a930
AC
225{
226 package Encode::utf8;
227 use parent 'Encode::Encoding';
228 my %obj = (
229 'utf8' => { Name => 'utf8' },
230 'utf-8-strict' => { Name => 'utf-8-strict', strict_utf8 => 1 }
231 );
232 for ( keys %obj ) {
233 bless $obj{$_} => __PACKAGE__;
234 Encode::define_encoding( $obj{$_} => $_ );
235 }
236 sub cat_decode {
237 # ($obj, $dst, $src, $pos, $trm, $chk)
238 # currently ignores $chk
239 my ( undef, undef, undef, $pos, $trm ) = @_;
240 my ( $rdst, $rsrc, $rpos ) = \@_[ 1, 2, 3 ];
241 use bytes;
242 if ( ( my $npos = index( $$rsrc, $trm, $pos ) ) >= 0 ) {
243 $$rdst .=
244 substr( $$rsrc, $pos, $npos - $pos + length($trm) );
245 $$rpos = $npos + length($trm);
246 return 1;
d1256cb1 247 }
3f60a930
AC
248 $$rdst .= substr( $$rsrc, $pos );
249 $$rpos = length($$rsrc);
250 return '';
f2a2953c 251 }
f2a2953c
JH
252}
253
656753f8
NIS
2541;
255
2a936312
NIS
256__END__
257
4411f3b6
NIS
258=head1 NAME
259
b9370cdb 260Encode - character encodings in Perl
4411f3b6
NIS
261
262=head1 SYNOPSIS
263
84678a67
SH
264 use Encode qw(decode encode);
265 $characters = decode('UTF-8', $octets, Encode::FB_CROAK);
266 $octets = encode('UTF-8', $characters, Encode::FB_CROAK);
4411f3b6 267
67d7b5ef
JH
268=head2 Table of Contents
269
b9370cdb
CBW
270Encode consists of a collection of modules whose details are too extensive
271to fit in one document. This one itself explains the top-level APIs
6d1c0808 272and general topics at a glance. For other topics and more details,
b9370cdb 273see the documentation for these modules:
67d7b5ef 274
84678a67
SH
275=over 2
276
277=item L<Encode::Alias> - Alias definitions to encodings
278
279=item L<Encode::Encoding> - Encode Implementation Base Class
280
281=item L<Encode::Supported> - List of Supported Encodings
282
283=item L<Encode::CN> - Simplified Chinese Encodings
284
285=item L<Encode::JP> - Japanese Encodings
286
287=item L<Encode::KR> - Korean Encodings
288
289=item L<Encode::TW> - Traditional Chinese Encodings
290
291=back
67d7b5ef 292
4411f3b6
NIS
293=head1 DESCRIPTION
294
b9370cdb 295The C<Encode> module provides the interface between Perl strings
67d7b5ef 296and the rest of the system. Perl strings are sequences of
b9370cdb 297I<characters>.
67d7b5ef 298
b9370cdb 299The repertoire of characters that Perl can represent is a superset of those
67d7b5ef 300defined by the Unicode Consortium. On most platforms the ordinal
b9370cdb
CBW
301values of a character as returned by C<ord(I<S>)> is the I<Unicode
302codepoint> for that character. The exceptions are platforms where
303the legacy encoding is some variant of EBCDIC rather than a superset
304of ASCII; see L<perlebcdic>.
305
306During recent history, data is moved around a computer in 8-bit chunks,
307often called "bytes" but also known as "octets" in standards documents.
308Perl is widely used to manipulate data of many types: not only strings of
309characters representing human or computer languages, but also "binary"
310data, being the machine's representation of numbers, pixels in an image, or
311just about anything.
67d7b5ef 312
0ab8f81e 313When Perl is processing "binary data", the programmer wants Perl to
b9370cdb 314process "sequences of bytes". This is not a problem for Perl: because a
0ab8f81e 315byte has 256 possible values, it easily fits in Perl's much larger
67d7b5ef
JH
316"logical character".
317
84678a67
SH
318This document mostly explains the I<how>. L<perlunitut> and L<perlunifaq>
319explain the I<why>.
4411f3b6 320
84678a67 321=head2 TERMINOLOGY
21938dfa 322
84678a67 323=head3 character
67d7b5ef 324
84678a67 325A character in the range 0 .. 2**32-1 (or more);
b9370cdb 326what Perl's strings are made of.
67d7b5ef 327
84678a67 328=head3 byte
67d7b5ef 329
84678a67
SH
330A character in the range 0..255;
331a special case of a Perl character.
67d7b5ef 332
84678a67 333=head3 octet
67d7b5ef 334
84678a67
SH
3358 bits of data, with ordinal values 0..255;
336term for bytes passed to or from a non-Perl context, such as a disk file,
337standard I/O stream, database, command-line argument, environment variable,
338socket etc.
4411f3b6 339
b9370cdb 340=head1 THE PERL ENCODING API
4411f3b6 341
84678a67
SH
342=head2 Basic methods
343
344=head3 encode
4411f3b6 345
84678a67 346 $octets = encode(ENCODING, STRING[, CHECK])
4411f3b6 347
b9370cdb
CBW
348Encodes the scalar value I<STRING> from Perl's internal form into
349I<ENCODING> and returns a sequence of octets. I<ENCODING> can be either a
350canonical name or an alias. For encoding names and aliases, see
351L</"Defining Aliases">. For CHECK, see L</"Handling Malformed Data">.
4411f3b6 352
313d8687
SH
353B<CAVEAT>: the input scalar I<STRING> might be modified in-place depending
354on what is set in CHECK. See L</LEAVE_SRC> if you want your inputs to be
355left unchanged.
356
b9370cdb
CBW
357For example, to convert a string from Perl's internal format into
358ISO-8859-1, also known as Latin1:
681a7c68 359
b7a5c9de 360 $octets = encode("iso-8859-1", $string);
7e19fb92 361
3f60a930 362B<CAVEAT>: When you run C<$octets = encode("UTF-8", $string)>, then
b9370cdb
CBW
363$octets I<might not be equal to> $string. Though both contain the
364same data, the UTF8 flag for $octets is I<always> off. When you
365encode anything, the UTF8 flag on the result is always off, even when it
3f60a930 366contains a completely valid UTF-8 string. See L</"The UTF8 flag"> below.
681a7c68 367
b9370cdb 368If the $string is C<undef>, then C<undef> is returned.
4089adc4 369
3f60a930
AC
370C<str2bytes> may be used as an alias for C<encode>.
371
84678a67
SH
372=head3 decode
373
374 $string = decode(ENCODING, OCTETS[, CHECK])
4411f3b6 375
b9370cdb
CBW
376This function returns the string that results from decoding the scalar
377value I<OCTETS>, assumed to be a sequence of octets in I<ENCODING>, into
127a7155 378Perl's internal form. As with encode(),
b9370cdb
CBW
379I<ENCODING> can be either a canonical name or an alias. For encoding names
380and aliases, see L</"Defining Aliases">; for I<CHECK>, see L</"Handling
381Malformed Data">.
47bfe92f 382
313d8687
SH
383B<CAVEAT>: the input scalar I<OCTETS> might be modified in-place depending
384on what is set in CHECK. See L</LEAVE_SRC> if you want your inputs to be
385left unchanged.
386
b9370cdb
CBW
387For example, to convert ISO-8859-1 data into a string in Perl's
388internal format:
681a7c68 389
b7a5c9de 390 $string = decode("iso-8859-1", $octets);
681a7c68 391
3f60a930 392B<CAVEAT>: When you run C<$string = decode("UTF-8", $octets)>, then $string
b9370cdb 393I<might not be equal to> $octets. Though both contain the same data, the
0a225b3c 394UTF8 flag for $string is on. See L</"The UTF8 flag">
7e19fb92 395below.
47bfe92f 396
b9370cdb 397If the $string is C<undef>, then C<undef> is returned.
4089adc4 398
3f60a930
AC
399C<bytes2str> may be used as an alias for C<decode>.
400
84678a67
SH
401=head3 find_encoding
402
403 [$obj =] find_encoding(ENCODING)
44b3b9c7 404
b9370cdb
CBW
405Returns the I<encoding object> corresponding to I<ENCODING>. Returns
406C<undef> if no matching I<ENCODING> is find. The returned object is
407what does the actual encoding or decoding.
44b3b9c7 408
3f60a930 409 $string = decode($name, $bytes);
44b3b9c7
SP
410
411is in fact
412
3f60a930 413 $string = do {
b9370cdb
CBW
414 $obj = find_encoding($name);
415 croak qq(encoding "$name" not found) unless ref $obj;
416 $obj->decode($bytes);
417 };
44b3b9c7
SP
418
419with more error checking.
420
b9370cdb 421You can therefore save time by reusing this object as follows;
44b3b9c7 422
b9370cdb
CBW
423 my $enc = find_encoding("iso-8859-1");
424 while(<>) {
3f60a930
AC
425 my $string = $enc->decode($_);
426 ... # now do something with $string;
b9370cdb 427 }
44b3b9c7 428
84678a67
SH
429Besides L</decode> and L</encode>, other methods are
430available as well. For instance, C<name()> returns the canonical
44b3b9c7
SP
431name of the encoding object.
432
433 find_encoding("latin1")->name; # iso-8859-1
434
435See L<Encode::Encoding> for details.
436
15f5e486
SH
437=head3 find_mime_encoding
438
439 [$obj =] find_mime_encoding(MIME_ENCODING)
440
441Returns the I<encoding object> corresponding to I<MIME_ENCODING>. Acts
442same as C<find_encoding()> but C<mime_name()> of returned object must
443match to I<MIME_ENCODING>. So as opposite of C<find_encoding()>
444canonical names and aliases are not used when searching for object.
445
446 find_mime_encoding("utf8"); # returns undef because "utf8" is not valid I<MIME_ENCODING>
447 find_mime_encoding("utf-8"); # returns encode object "utf-8-strict"
448 find_mime_encoding("UTF-8"); # same as "utf-8" because I<MIME_ENCODING> is case insensitive
449 find_mime_encoding("utf-8-strict"); returns undef because "utf-8-strict" is not valid I<MIME_ENCODING>
450
84678a67
SH
451=head3 from_to
452
453 [$length =] from_to($octets, FROM_ENC, TO_ENC [, CHECK])
7e19fb92 454
b9370cdb
CBW
455Converts I<in-place> data between two encodings. The data in $octets
456must be encoded as octets and I<not> as characters in Perl's internal
457format. For example, to convert ISO-8859-1 data into Microsoft's CP1250
f9d05ba3 458encoding:
2b106fbe 459
b7a5c9de 460 from_to($octets, "iso-8859-1", "cp1250");
2b106fbe
JH
461
462and to convert it back:
463
b7a5c9de 464 from_to($octets, "cp1250", "iso-8859-1");
4411f3b6 465
b9370cdb
CBW
466Because the conversion happens in place, the data to be
467converted cannot be a string constant: it must be a scalar variable.
ab97ca19 468
84678a67 469C<from_to()> returns the length of the converted string in octets on success,
b9370cdb 470and C<undef> on error.
3ef515df 471
b9370cdb 472B<CAVEAT>: The following operations may look the same, but are not:
7e19fb92 473
3f60a930 474 from_to($data, "iso-8859-1", "UTF-8"); #1
7e19fb92 475 $data = decode("iso-8859-1", $data); #2
4411f3b6 476
b9370cdb
CBW
477Both #1 and #2 make $data consist of a completely valid UTF-8 string,
478but only #2 turns the UTF8 flag on. #1 is equivalent to:
f2a2953c 479
3f60a930 480 $data = encode("UTF-8", decode("iso-8859-1", $data));
f2a2953c 481
2575c402 482See L</"The UTF8 flag"> below.
f2a2953c 483
b9370cdb 484Also note that:
7828f908
RGS
485
486 from_to($octets, $from, $to, $check);
487
127a7155 488is equivalent to:
7828f908
RGS
489
490 $octets = encode($to, decode($from, $octets), $check);
491
b9370cdb
CBW
492Yes, it does I<not> respect the $check during decoding. It is
493deliberately done that way. If you need minute control, use C<decode>
494followed by C<encode> as follows:
7828f908
RGS
495
496 $octets = encode($to, decode($from, $octets, $check_from), $check_to);
497
84678a67
SH
498=head3 encode_utf8
499
500 $octets = encode_utf8($string);
f2a2953c 501
b9370cdb
CBW
502Equivalent to C<$octets = encode("utf8", $string)>. The characters in
503$string are encoded in Perl's internal format, and the result is returned
504as a sequence of octets. Because all possible characters in Perl have a
3f60a930
AC
505(loose, not strict) utf8 representation, this function cannot fail.
506
507B<WARNING>: do not use this function for data exchange as it can produce
508not strict utf8 $octets! For strictly valid UTF-8 output use
509C<$octets = encode("UTF-8", $string)>.
f2a2953c 510
84678a67
SH
511=head3 decode_utf8
512
513 $string = decode_utf8($octets [, CHECK]);
f2a2953c 514
b9370cdb
CBW
515Equivalent to C<$string = decode("utf8", $octets [, CHECK])>.
516The sequence of octets represented by $octets is decoded
3f60a930
AC
517from (loose, not strict) utf8 into a sequence of logical characters.
518Because not all sequences of octets are valid not strict utf8,
b9370cdb
CBW
519it is quite possible for this function to fail.
520For CHECK, see L</"Handling Malformed Data">.
f2a2953c 521
3f60a930
AC
522B<WARNING>: do not use this function for data exchange as it can produce
523$string with not strict utf8 representation! For strictly valid UTF-8
524$string representation use C<$string = decode("UTF-8", $octets [, CHECK])>.
525
313d8687
SH
526B<CAVEAT>: the input I<$octets> might be modified in-place depending on
527what is set in CHECK. See L</LEAVE_SRC> if you want your inputs to be
528left unchanged.
529
51ef4e11
NIS
530=head2 Listing available encodings
531
5129552c
JH
532 use Encode;
533 @list = Encode->encodings();
534
b9370cdb
CBW
535Returns a list of canonical names of available encodings that have already
536been loaded. To get a list of all available encodings including those that
537have not yet been loaded, say:
5129552c
JH
538
539 @all_encodings = Encode->encodings(":all");
540
b9370cdb 541Or you can give the name of a specific module:
5129552c 542
c731e18e
JH
543 @with_jp = Encode->encodings("Encode::JP");
544
b9370cdb 545When "C<::>" is not in the name, "C<Encode::>" is assumed.
51ef4e11 546
c731e18e 547 @ebcdic = Encode->encodings("EBCDIC");
5d030b67 548
0ab8f81e 549To find out in detail which encodings are supported by this package,
5d030b67 550see L<Encode::Supported>.
51ef4e11
NIS
551
552=head2 Defining Aliases
553
0ab8f81e 554To add a new alias to a given encoding, use:
67d7b5ef 555
5129552c
JH
556 use Encode;
557 use Encode::Alias;
b9370cdb 558 define_alias(NEWNAME => ENCODING);
51ef4e11 559
b9370cdb 560After that, I<NEWNAME> can be used as an alias for I<ENCODING>.
84678a67 561I<ENCODING> may be either the name of an encoding or an
b9370cdb 562I<encoding object>.
51ef4e11 563
b9370cdb 564Before you do that, first make sure the alias is nonexistent using
fcb875d4 565C<resolve_alias()>, which returns the canonical name thereof.
b9370cdb 566For example:
fcb875d4
JH
567
568 Encode::resolve_alias("latin1") eq "iso-8859-1" # true
569 Encode::resolve_alias("iso-8859-12") # false; nonexistent
570 Encode::resolve_alias($name) eq $name # true if $name is canonical
571
84678a67 572C<resolve_alias()> does not need C<use Encode::Alias>; it can be
b9370cdb 573imported via C<use Encode qw(resolve_alias)>.
fcb875d4 574
0ab8f81e 575See L<Encode::Alias> for details.
51ef4e11 576
742555bd
SP
577=head2 Finding IANA Character Set Registry names
578
579The canonical name of a given encoding does not necessarily agree with
b9370cdb
CBW
580IANA Character Set Registry, commonly seen as C<< Content-Type:
581text/plain; charset=I<WHATEVER> >>. For most cases, the canonical name
582works, but sometimes it does not, most notably with "utf-8-strict".
742555bd 583
84678a67 584As of C<Encode> version 2.21, a new method C<mime_name()> is therefore added.
742555bd
SP
585
586 use Encode;
b9370cdb 587 my $enc = find_encoding("UTF-8");
742555bd
SP
588 warn $enc->name; # utf-8-strict
589 warn $enc->mime_name; # UTF-8
590
591See also: L<Encode::Encoding>
592
85982a32 593=head1 Encoding via PerlIO
4411f3b6 594
b9370cdb
CBW
595If your perl supports C<PerlIO> (which is the default), you can use a
596C<PerlIO> layer to decode and encode directly via a filehandle. The
597following two examples are fully identical in functionality:
598
599 ### Version 1 via PerlIO
600 open(INPUT, "< :encoding(shiftjis)", $infile)
601 || die "Can't open < $infile for reading: $!";
602 open(OUTPUT, "> :encoding(euc-jp)", $outfile)
603 || die "Can't open > $output for writing: $!";
604 while (<INPUT>) { # auto decodes $_
605 print OUTPUT; # auto encodes $_
606 }
607 close(INPUT) || die "can't close $infile: $!";
608 close(OUTPUT) || die "can't close $outfile: $!";
609
610 ### Version 2 via from_to()
611 open(INPUT, "< :raw", $infile)
612 || die "Can't open < $infile for reading: $!";
613 open(OUTPUT, "> :raw", $outfile)
614 || die "Can't open > $output for writing: $!";
615
616 while (<INPUT>) {
617 from_to($_, "shiftjis", "euc-jp", 1); # switch encoding
618 print OUTPUT; # emit raw (but properly encoded) data
619 }
620 close(INPUT) || die "can't close $infile: $!";
621 close(OUTPUT) || die "can't close $outfile: $!";
8e86646e 622
b9370cdb
CBW
623In the first version above, you let the appropriate encoding layer
624handle the conversion. In the second, you explicitly translate
625from one encoding to the other.
4411f3b6 626
127a7155 627Unfortunately, it may be that encodings are not C<PerlIO>-savvy. You can check
b9370cdb
CBW
628to see whether your encoding is supported by C<PerlIO> by invoking the
629C<perlio_ok> method on it:
0ab8f81e 630
b9370cdb
CBW
631 Encode::perlio_ok("hz"); # false
632 find_encoding("euc-cn")->perlio_ok; # true wherever PerlIO is available
0ab8f81e 633
b9370cdb 634 use Encode qw(perlio_ok); # imported upon request
0ab8f81e 635 perlio_ok("euc-jp")
4411f3b6 636
b9370cdb 637Fortunately, all encodings that come with C<Encode> core are C<PerlIO>-savvy
84678a67 638except for C<hz> and C<ISO-2022-kr>. For the gory details, see
f9d05ba3 639L<Encode::Encoding> and L<Encode::PerlIO>.
4411f3b6 640
85982a32 641=head1 Handling Malformed Data
4411f3b6 642
b9370cdb
CBW
643The optional I<CHECK> argument tells C<Encode> what to do when
644encountering malformed data. Without I<CHECK>, C<Encode::FB_DEFAULT>
645(== 0) is assumed.
8e180e82 646
b9370cdb
CBW
647As of version 2.12, C<Encode> supports coderef values for C<CHECK>;
648see below.
f9d05ba3 649
84678a67
SH
650B<NOTE:> Not all encodings support this feature.
651Some encodings ignore the I<CHECK> argument. For example,
f9d05ba3
RGS
652L<Encode::Unicode> ignores I<CHECK> and it always croaks on error.
653
84678a67 654=head2 List of I<CHECK> values
47bfe92f 655
84678a67 656=head3 FB_DEFAULT
151b5d36 657
84678a67 658 I<CHECK> = Encode::FB_DEFAULT ( == 0)
47bfe92f 659
b9370cdb
CBW
660If I<CHECK> is 0, encoding and decoding replace any malformed character
661with a I<substitution character>. When you encode, I<SUBCHAR> is used.
662When you decode, the Unicode REPLACEMENT CHARACTER, code point U+FFFD, is
663used. If the data is supposed to be UTF-8, an optional lexical warning of
664warning category C<"utf8"> is given.
e9692b5b 665
84678a67
SH
666=head3 FB_CROAK
667
668 I<CHECK> = Encode::FB_CROAK ( == 1)
e9692b5b 669
b9370cdb
CBW
670If I<CHECK> is 1, methods immediately die with an error
671message. Therefore, when I<CHECK> is 1, you should trap
672exceptions with C<eval{}>, unless you really want to let it C<die>.
47bfe92f 673
84678a67
SH
674=head3 FB_QUIET
675
676 I<CHECK> = Encode::FB_QUIET
47bfe92f 677
b9370cdb 678If I<CHECK> is set to C<Encode::FB_QUIET>, encoding and decoding immediately
f9d05ba3 679return the portion of the data that has been processed so far when an
b9370cdb
CBW
680error occurs. The data argument is overwritten with everything
681after that point; that is, the unprocessed portion of the data. This is
682handy when you have to call C<decode> repeatedly in the case where your
f9d05ba3 683source data may contain partial multi-byte character sequences,
b9370cdb
CBW
684(that is, you are reading with a fixed-width buffer). Here's some sample
685code to do exactly that:
4411f3b6 686
b9370cdb
CBW
687 my($buffer, $string) = ("", "");
688 while (read($fh, $buffer, 256, length($buffer))) {
689 $string .= decode($encoding, $buffer, Encode::FB_QUIET);
690 # $buffer now contains the unprocessed partial character
691 }
1768d7eb 692
84678a67
SH
693=head3 FB_WARN
694
695 I<CHECK> = Encode::FB_WARN
67d7b5ef 696
b9370cdb
CBW
697This is the same as C<FB_QUIET> above, except that instead of being silent
698on errors, it issues a warning. This is handy for when you are debugging.
85982a32 699
20797ee1
DK
700B<CAVEAT>: All warnings from Encode module are reported, independently of
701L<pragma warnings|warnings> settings. If you want to follow settings of
702lexical warnings configured by L<pragma warnings|warnings> then append
703also check value C<ENCODE::ONLY_PRAGMA_WARNINGS>. This value is available
704since Encode version 2.99.
705
84678a67
SH
706=head3 FB_PERLQQ FB_HTMLCREF FB_XMLCREF
707
708=over 2
709
85982a32
JH
710=item perlqq mode (I<CHECK> = Encode::FB_PERLQQ)
711
af1f55d9
JH
712=item HTML charref mode (I<CHECK> = Encode::FB_HTMLCREF)
713
714=item XML charref mode (I<CHECK> = Encode::FB_XMLCREF)
715
84678a67
SH
716=back
717
b9370cdb
CBW
718For encodings that are implemented by the C<Encode::XS> module, C<CHECK> C<==>
719C<Encode::FB_PERLQQ> puts C<encode> and C<decode> into C<perlqq> fallback mode.
85982a32 720
b9370cdb
CBW
721When you decode, C<\xI<HH>> is inserted for a malformed character, where
722I<HH> is the hex representation of the octet that could not be decoded to
723utf8. When you encode, C<\x{I<HHHH>}> will be inserted, where I<HHHH> is
724the Unicode code point (in any number of hex digits) of the character that
725cannot be found in the character repertoire of the encoding.
85982a32 726
b9370cdb
CBW
727The HTML/XML character reference modes are about the same. In place of
728C<\x{I<HHHH>}>, HTML uses C<&#I<NNN>;> where I<NNN> is a decimal number, and
78589665 729XML uses C<&#xI<HHHH>;> where I<HHHH> is the hexadecimal number.
af1f55d9 730
b9370cdb 731In C<Encode> 2.10 or later, C<LEAVE_SRC> is also implied.
7f0d54d7 732
84678a67 733=head3 The bitmask
85982a32 734
b9370cdb
CBW
735These modes are all actually set via a bitmask. Here is how the C<FB_I<XXX>>
736constants are laid out. You can import the C<FB_I<XXX>> constants via
737C<use Encode qw(:fallbacks)>, and you can import the generic bitmask
0ab8f81e 738constants via C<use Encode qw(:fallback_all)>.
85982a32 739
b0b300a3
JH
740 FB_DEFAULT FB_CROAK FB_QUIET FB_WARN FB_PERLQQ
741 DIE_ON_ERR 0x0001 X
4089adc4 742 WARN_ON_ERR 0x0002 X
b0b300a3 743 RETURN_ON_ERR 0x0004 X X
7f0d54d7 744 LEAVE_SRC 0x0008 X
b0b300a3 745 PERLQQ 0x0100 X
b7a5c9de
JH
746 HTMLCREF 0x0200
747 XMLCREF 0x0400
67d7b5ef 748
84678a67 749=head3 LEAVE_SRC
44b3b9c7 750
84678a67 751 Encode::LEAVE_SRC
51e4e64d 752
b9370cdb 753If the C<Encode::LEAVE_SRC> bit is I<not> set but I<CHECK> is set, then the
64a9a3c0 754source string to encode() or decode() will be overwritten in place.
b9370cdb 755If you're not interested in this, then bitwise-OR it with the bitmask.
51e4e64d 756
0dbed2e5 757=head2 coderef for CHECK
8e180e82 758
b9370cdb 759As of C<Encode> 2.12, C<CHECK> can also be a code reference which takes the
c7981a06
SH
760ordinal value of the unmapped character as an argument and returns
761octets that represent the fallback character. For instance:
67d7b5ef 762
8e180e82 763 $ascii = encode("ascii", $utf8, sub{ sprintf "<U+%04X>", shift });
67d7b5ef 764
b9370cdb 765Acts like C<FB_PERLQQ> but U+I<XXXX> is used instead of C<\x{I<XXXX>}>.
982a4085 766
3f60a930
AC
767Fallback for C<decode> must return decoded string (sequence of characters)
768and takes a list of ordinal values as its arguments. So for
127a7155 769example if you wish to decode octets as UTF-8, and use ISO-8859-15 as
c7981a06
SH
770a fallback for bytes that are not valid UTF-8, you could write
771
772 $str = decode 'UTF-8', $octets, sub {
3f60a930
AC
773 my $tmp = join '', map chr, @_;
774 return decode 'ISO-8859-15', $tmp;
c7981a06
SH
775 };
776
67d7b5ef
JH
777=head1 Defining Encodings
778
779To define a new encoding, use:
780
b7a5c9de 781 use Encode qw(define_encoding);
b9370cdb 782 define_encoding($object, CANONICAL_NAME [, alias...]);
67d7b5ef 783
b9370cdb 784I<CANONICAL_NAME> will be associated with I<$object>. The object
0ab8f81e 785should provide the interface described in L<Encode::Encoding>.
b9370cdb
CBW
786If more than two arguments are provided, additional
787arguments are considered aliases for I<$object>.
67d7b5ef 788
b9370cdb 789See L<Encode::Encoding> for details.
f2a2953c 790
2575c402 791=head1 The UTF8 flag
7e19fb92 792
b9370cdb 793Before the introduction of Unicode support in Perl, The C<eq> operator
b7a5c9de 794just compared the strings represented by two scalars. Beginning with
b9370cdb
CBW
795Perl 5.8, C<eq> compares two strings with simultaneous consideration of
796I<the UTF8 flag>. To explain why we made it so, I quote from page 402 of
797I<Programming Perl, 3rd ed.>
7e19fb92
JH
798
799=over 2
800
801=item Goal #1:
802
803Old byte-oriented programs should not spontaneously break on the old
804byte-oriented data they used to work on.
805
806=item Goal #2:
807
808Old byte-oriented programs should magically start working on the new
809character-oriented data when appropriate.
810
811=item Goal #3:
812
813Programs should run just as fast in the new character-oriented mode
814as in the old byte-oriented mode.
815
816=item Goal #4:
817
818Perl should remain one language, rather than forking into a
819byte-oriented Perl and a character-oriented Perl.
820
821=back
822
b9370cdb
CBW
823When I<Programming Perl, 3rd ed.> was written, not even Perl 5.6.0 had been
824born yet, many features documented in the book remained unimplemented for a
825long time. Perl 5.8 corrected much of this, and the introduction of the
826UTF8 flag is one of them. You can think of there being two fundamentally
827different kinds of strings and string-operations in Perl: one a
828byte-oriented mode for when the internal UTF8 flag is off, and the other a
829character-oriented mode for when the internal UTF8 flag is on.
7e19fb92 830
b9370cdb
CBW
831This UTF8 flag is not visible in Perl scripts, exactly for the same reason
832you cannot (or rather, you I<don't have to>) see whether a scalar contains
833a string, an integer, or a floating-point number. But you can still peek
834and poke these if you will. See the next section.
7e19fb92 835
7e19fb92 836=head2 Messing with Perl's Internals
4411f3b6 837
47bfe92f 838The following API uses parts of Perl's internals in the current
b9370cdb
CBW
839implementation. As such, they are efficient but may change in a future
840release.
4411f3b6 841
84678a67 842=head3 is_utf8
4411f3b6 843
84678a67 844 is_utf8(STRING [, CHECK])
4411f3b6 845
b9370cdb
CBW
846[INTERNAL] Tests whether the UTF8 flag is turned on in the I<STRING>.
847If I<CHECK> is true, also checks whether I<STRING> contains well-formed
47bfe92f 848UTF-8. Returns true if successful, false otherwise.
4411f3b6 849
13a15ab6
CBW
850Typically only necessary for debugging and testing. Don't use this flag as
851a marker to distinguish character and binary data, that should be decided
852for each variable when you write your code.
853
854B<CAVEAT>: If I<STRING> has UTF8 flag set, it does B<NOT> mean that
855I<STRING> is UTF-8 encoded and vice-versa.
856
b9370cdb 857As of Perl 5.8.1, L<utf8> also has the C<utf8::is_utf8> function.
b5ab1f6f 858
84678a67
SH
859=head3 _utf8_on
860
861 _utf8_on(STRING)
4411f3b6 862
b9370cdb
CBW
863[INTERNAL] Turns the I<STRING>'s internal UTF8 flag B<on>. The I<STRING>
864is I<not> checked for containing only well-formed UTF-8. Do not use this
865unless you I<know with absolute certainty> that the STRING holds only
866well-formed UTF-8. Returns the previous state of the UTF8 flag (so please
867don't treat the return value as indicating success or failure), or C<undef>
868if I<STRING> is not a string.
4411f3b6 869
b9370cdb 870B<NOTE>: For security reasons, this function does not work on tainted values.
64bc6d54 871
84678a67
SH
872=head3 _utf8_off
873
874 _utf8_off(STRING)
4411f3b6 875
b9370cdb
CBW
876[INTERNAL] Turns the I<STRING>'s internal UTF8 flag B<off>. Do not use
877frivolously. Returns the previous state of the UTF8 flag, or C<undef> if
878I<STRING> is not a string. Do not treat the return value as indicative of
879success or failure, because that isn't what it means: it is only the
880previous setting.
4411f3b6 881
b9370cdb 882B<NOTE>: For security reasons, this function does not work on tainted values.
64bc6d54 883
2575c402 884=head1 UTF-8 vs. utf8 vs. UTF8
7f0d54d7
RGS
885
886 ....We now view strings not as sequences of bytes, but as sequences
887 of numbers in the range 0 .. 2**32-1 (or in the case of 64-bit
888 computers, 0 .. 2**64-1) -- Programming Perl, 3rd ed.
889
b9370cdb
CBW
890That has historically been Perl's notion of UTF-8, as that is how UTF-8 was
891first conceived by Ken Thompson when he invented it. However, thanks to
892later revisions to the applicable standards, official UTF-8 is now rather
893stricter than that. For example, its range is much narrower (0 .. 0x10_FFFF
894to cover only 21 bits instead of 32 or 64 bits) and some sequences
895are not allowed, like those used in surrogate pairs, the 31 non-character
896code points 0xFDD0 .. 0xFDEF, the last two code points in I<any> plane
897(0xI<XX>_FFFE and 0xI<XX>_FFFF), all non-shortest encodings, etc.
7f0d54d7 898
b9370cdb
CBW
899The former default in which Perl would always use a loose interpretation of
900UTF-8 has now been overruled:
7f0d54d7
RGS
901
902 From: Larry Wall <larry@wall.org>
903 Date: December 04, 2004 11:51:58 JST
904 To: perl-unicode@perl.org
905 Subject: Re: Make Encode.pm support the real UTF-8
906 Message-Id: <20041204025158.GA28754@wall.org>
b9370cdb 907
7f0d54d7
RGS
908 On Fri, Dec 03, 2004 at 10:12:12PM +0000, Tim Bunce wrote:
909 : I've no problem with 'utf8' being perl's unrestricted uft8 encoding,
910 : but "UTF-8" is the name of the standard and should give the
911 : corresponding behaviour.
b9370cdb 912
7f0d54d7
RGS
913 For what it's worth, that's how I've always kept them straight in my
914 head.
b9370cdb 915
7f0d54d7
RGS
916 Also for what it's worth, Perl 6 will mostly default to strict but
917 make it easy to switch back to lax.
b9370cdb 918
7f0d54d7
RGS
919 Larry
920
b9370cdb
CBW
921Got that? As of Perl 5.8.7, B<"UTF-8"> means UTF-8 in its current
922sense, which is conservative and strict and security-conscious, whereas
923B<"utf8"> means UTF-8 in its former sense, which was liberal and loose and
924lax. C<Encode> version 2.10 or later thus groks this subtle but critically
925important distinction between C<"UTF-8"> and C<"utf8">.
7f0d54d7
RGS
926
927 encode("utf8", "\x{FFFF_FFFF}", 1); # okay
928 encode("UTF-8", "\x{FFFF_FFFF}", 1); # croaks
929
b9370cdb
CBW
930In the C<Encode> module, C<"UTF-8"> is actually a canonical name for
931C<"utf-8-strict">. That hyphen between the C<"UTF"> and the C<"8"> is
932critical; without it, C<Encode> goes "liberal" and (perhaps overly-)permissive:
7f0d54d7
RGS
933
934 find_encoding("UTF-8")->name # is 'utf-8-strict'
935 find_encoding("utf-8")->name # ditto. names are case insensitive
b9370cdb 936 find_encoding("utf_8")->name # ditto. "_" are treated as "-"
7f0d54d7
RGS
937 find_encoding("UTF8")->name # is 'utf8'.
938
b9370cdb
CBW
939Perl's internal UTF8 flag is called "UTF8", without a hyphen. It indicates
940whether a string is internally encoded as "utf8", also without a hyphen.
7f0d54d7 941
4411f3b6
NIS
942=head1 SEE ALSO
943
5d030b67
JH
944L<Encode::Encoding>,
945L<Encode::Supported>,
6d1c0808 946L<Encode::PerlIO>,
5d030b67 947L<encoding>,
6d1c0808
JH
948L<perlebcdic>,
949L<perlfunc/open>,
370462a2 950L<perlunicode>, L<perluniintro>, L<perlunifaq>, L<perlunitut>
6d1c0808 951L<utf8>,
84678a67 952the Perl Unicode Mailing List L<http://lists.perl.org/list/perl-unicode.html>
4411f3b6 953
85982a32 954=head1 MAINTAINER
aae85ceb 955
b9370cdb 956This project was originated by the late Nick Ing-Simmons and later
fd172611 957maintained by Dan Kogai I<< <dankogai@cpan.org> >>. See AUTHORS
b9370cdb
CBW
958for a full list of people involved. For any questions, send mail to
959I<< <perl-unicode@perl.org> >> so that we can all share.
aae85ceb 960
b9370cdb
CBW
961While Dan Kogai retains the copyright as a maintainer, credit
962should go to all those involved. See AUTHORS for a list of those
963who submitted code to the project.
d1256cb1
RGS
964
965=head1 COPYRIGHT
966
28e02325 967Copyright 2002-2014 Dan Kogai I<< <dankogai@cpan.org> >>.
d1256cb1
RGS
968
969This library is free software; you can redistribute it and/or modify
970it under the same terms as Perl itself.
971
4411f3b6 972=cut