This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
[Unicode::Normalize] mkheader tweak
[perl5.git] / ext / Encode / Encode.pm
CommitLineData
10c5ecbb
JH
1#
2# $Id: Encode.pm,v 1.63 2002/04/27 18:59:50 dankogai Exp $
3#
2c674647 4package Encode;
51ef4e11 5use strict;
10c5ecbb 6our $VERSION = do { my @r = (q$Revision: 1.63 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r };
5129552c 7our $DEBUG = 0;
6d1c0808 8use XSLoader ();
10c5ecbb 9XSLoader::load(__PACKAGE__, $VERSION);
2c674647 10
2c674647 11require Exporter;
7e19fb92 12use base qw/Exporter/;
2c674647 13
4411f3b6 14# Public, encouraged API is exported by default
85982a32
JH
15
16our @EXPORT = qw(
17 decode decode_utf8 encode encode_utf8
18 encodings find_encoding
4411f3b6
NIS
19);
20
af1f55d9
JH
21our @FB_FLAGS = qw(DIE_ON_ERR WARN_ON_ERR RETURN_ON_ERR LEAVE_SRC
22 PERLQQ HTMLCREF XMLCREF);
23our @FB_CONSTS = qw(FB_DEFAULT FB_CROAK FB_QUIET FB_WARN
24 FB_PERLQQ FB_HTMLCREF FB_XMLCREF);
85982a32 25
51ef4e11 26our @EXPORT_OK =
6d1c0808 27 (
85982a32
JH
28 qw(
29 _utf8_off _utf8_on define_encoding from_to is_16bit is_8bit
30 is_utf8 perlio_ok resolve_alias utf8_downgrade utf8_upgrade
31 ),
32 @FB_FLAGS, @FB_CONSTS,
33 );
34
6d1c0808 35our %EXPORT_TAGS =
85982a32
JH
36 (
37 all => [ @EXPORT, @EXPORT_OK ],
38 fallbacks => [ @FB_CONSTS ],
39 fallback_all => [ @FB_CONSTS, @FB_FLAGS ],
40 );
41
4411f3b6 42# Documentation moved after __END__ for speed - NI-S
2c674647 43
a63c962f 44our $ON_EBCDIC = (ord("A") == 193);
f2a2953c 45
5d030b67
JH
46use Encode::Alias;
47
5129552c
JH
48# Make a %Encoding package variable to allow a certain amount of cheating
49our %Encoding;
aae85ceb
DK
50our %ExtModule;
51require Encode::Config;
52eval { require Encode::ConfigLocal };
5129552c 53
656753f8
NIS
54sub encodings
55{
5129552c 56 my $class = shift;
fc17bd48
JH
57 my %enc;
58 if (@_ and $_[0] eq ":all"){
59 %enc = ( %Encoding, %ExtModule );
60 }else{
61 %enc = %Encoding;
62 for my $mod (map {m/::/o ? $_ : "Encode::$_" } @_){
63 $DEBUG and warn $mod;
64 for my $enc (keys %ExtModule){
65 $ExtModule{$enc} eq $mod and $enc{$enc} = $mod;
66 }
67 }
5129552c
JH
68 }
69 return
ce912cd4 70 sort { lc $a cmp lc $b }
fc17bd48 71 grep {!/^(?:Internal|Unicode|Guess)$/o} keys %enc;
51ef4e11
NIS
72}
73
85982a32 74sub perlio_ok{
0ab8f81e 75 my $obj = ref($_[0]) ? $_[0] : find_encoding($_[0]);
011b2d2f 76 $obj->can("perlio_ok") and return $obj->perlio_ok();
0ab8f81e 77 return 0; # safety net
85982a32
JH
78}
79
51ef4e11
NIS
80sub define_encoding
81{
18586f54
NIS
82 my $obj = shift;
83 my $name = shift;
5129552c 84 $Encoding{$name} = $obj;
18586f54
NIS
85 my $lc = lc($name);
86 define_alias($lc => $obj) unless $lc eq $name;
10c5ecbb 87 while (@_){
18586f54 88 my $alias = shift;
10c5ecbb 89 define_alias($alias, $obj);
18586f54
NIS
90 }
91 return $obj;
656753f8
NIS
92}
93
656753f8
NIS
94sub getEncoding
95{
10c5ecbb
JH
96 my ($class, $name, $skip_external) = @_;
97
98 ref($name) && $name->can('new_sequence') and return $name;
99 exists $Encoding{$name} and return $Encoding{$name};
18586f54 100 my $lc = lc $name;
10c5ecbb 101 exists $Encoding{$lc} and return $Encoding{$lc};
c50d192e 102
5129552c 103 my $oc = $class->find_alias($name);
10c5ecbb
JH
104 defined($oc) and return $oc;
105 $lc ne $name and $oc = $class->find_alias($lc);
106 defined($oc) and return $oc;
c50d192e 107
c731e18e 108 unless ($skip_external)
d1ed7747 109 {
c731e18e
JH
110 if (my $mod = $ExtModule{$name} || $ExtModule{$lc}){
111 $mod =~ s,::,/,g ; $mod .= '.pm';
112 eval{ require $mod; };
10c5ecbb 113 exists $Encoding{$name} and return $Encoding{$name};
c731e18e 114 }
d1ed7747 115 }
18586f54 116 return;
656753f8
NIS
117}
118
4411f3b6
NIS
119sub find_encoding
120{
10c5ecbb 121 my ($name, $skip_external) = @_;
dd9703c9 122 return __PACKAGE__->getEncoding($name,$skip_external);
4411f3b6
NIS
123}
124
fcb875d4
JH
125sub resolve_alias {
126 my $obj = find_encoding(shift);
127 defined $obj and return $obj->name;
128 return;
129}
130
b2704119 131sub encode($$;$)
4411f3b6 132{
18586f54 133 my ($name,$string,$check) = @_;
b2704119 134 $check ||=0;
18586f54 135 my $enc = find_encoding($name);
10c5ecbb
JH
136 unless(defined $enc){
137 require Carp;
138 Carp::croak("Unknown encoding '$name'");
139 }
18586f54
NIS
140 my $octets = $enc->encode($string,$check);
141 return undef if ($check && length($string));
142 return $octets;
4411f3b6
NIS
143}
144
b2704119 145sub decode($$;$)
4411f3b6 146{
18586f54 147 my ($name,$octets,$check) = @_;
b2704119 148 $check ||=0;
18586f54 149 my $enc = find_encoding($name);
10c5ecbb
JH
150 unless(defined $enc){
151 require Carp;
152 Carp::croak("Unknown encoding '$name'");
153 }
18586f54
NIS
154 my $string = $enc->decode($octets,$check);
155 $_[1] = $octets if $check;
156 return $string;
4411f3b6
NIS
157}
158
b2704119 159sub from_to($$$;$)
4411f3b6 160{
18586f54 161 my ($string,$from,$to,$check) = @_;
b2704119 162 $check ||=0;
18586f54 163 my $f = find_encoding($from);
10c5ecbb
JH
164 unless (defined $f){
165 require Carp;
166 Carp::croak("Unknown encoding '$from'");
167 }
18586f54 168 my $t = find_encoding($to);
10c5ecbb
JH
169 unless (defined $t){
170 require Carp;
171 Carp::croak("Unknown encoding '$to'");
172 }
18586f54
NIS
173 my $uni = $f->decode($string,$check);
174 return undef if ($check && length($string));
a999c27c 175 $string = $t->encode($uni,$check);
18586f54 176 return undef if ($check && length($uni));
3ef515df 177 return defined($_[0] = $string) ? length($string) : undef ;
4411f3b6
NIS
178}
179
b2704119 180sub encode_utf8($)
4411f3b6 181{
18586f54 182 my ($str) = @_;
c731e18e 183 utf8::encode($str);
18586f54 184 return $str;
4411f3b6
NIS
185}
186
b2704119 187sub decode_utf8($)
4411f3b6 188{
18586f54
NIS
189 my ($str) = @_;
190 return undef unless utf8::decode($str);
191 return $str;
5ad8ef52
NIS
192}
193
f2a2953c
JH
194predefine_encodings();
195
196#
197# This is to restore %Encoding if really needed;
198#
10c5ecbb 199
f2a2953c 200sub predefine_encodings{
10c5ecbb 201 use Encode::Encoding;
6d1c0808 202 if ($ON_EBCDIC) {
f2a2953c
JH
203 # was in Encode::UTF_EBCDIC
204 package Encode::UTF_EBCDIC;
10c5ecbb 205 push @Encode::UTF_EBCDIC::ISA, 'Encode::Encoding';
f2a2953c
JH
206 *decode = sub{
207 my ($obj,$str,$chk) = @_;
208 my $res = '';
209 for (my $i = 0; $i < length($str); $i++) {
6d1c0808 210 $res .=
f2a2953c
JH
211 chr(utf8::unicode_to_native(ord(substr($str,$i,1))));
212 }
213 $_[1] = '' if $chk;
214 return $res;
215 };
216 *encode = sub{
217 my ($obj,$str,$chk) = @_;
218 my $res = '';
219 for (my $i = 0; $i < length($str); $i++) {
6d1c0808 220 $res .=
f2a2953c
JH
221 chr(utf8::native_to_unicode(ord(substr($str,$i,1))));
222 }
223 $_[1] = '' if $chk;
224 return $res;
225 };
6d1c0808 226 $Encode::Encoding{Unicode} =
c731e18e 227 bless {Name => "UTF_EBCDIC"} => "Encode::UTF_EBCDIC";
6d1c0808 228 } else {
f2a2953c 229 package Encode::Internal;
10c5ecbb 230 push @Encode::Internal::ISA, 'Encode::Encoding';
f2a2953c
JH
231 *decode = sub{
232 my ($obj,$str,$chk) = @_;
233 utf8::upgrade($str);
234 $_[1] = '' if $chk;
235 return $str;
236 };
237 *encode = \&decode;
6d1c0808 238 $Encode::Encoding{Unicode} =
c731e18e 239 bless {Name => "Internal"} => "Encode::Internal";
f2a2953c
JH
240 }
241
242 {
243 # was in Encode::utf8
244 package Encode::utf8;
10c5ecbb 245 push @Encode::utf8::ISA, 'Encode::Encoding';
f2a2953c
JH
246 *decode = sub{
247 my ($obj,$octets,$chk) = @_;
248 my $str = Encode::decode_utf8($octets);
249 if (defined $str) {
250 $_[1] = '' if $chk;
251 return $str;
252 }
253 return undef;
254 };
255 *encode = sub {
256 my ($obj,$string,$chk) = @_;
257 my $octets = Encode::encode_utf8($string);
258 $_[1] = '' if $chk;
259 return $octets;
260 };
0ab8f81e 261 $Encode::Encoding{utf8} =
c731e18e 262 bless {Name => "utf8"} => "Encode::utf8";
f2a2953c 263 }
f2a2953c
JH
264}
265
656753f8
NIS
2661;
267
2a936312
NIS
268__END__
269
4411f3b6
NIS
270=head1 NAME
271
272Encode - character encodings
273
274=head1 SYNOPSIS
275
276 use Encode;
277
67d7b5ef
JH
278=head2 Table of Contents
279
0ab8f81e 280Encode consists of a collection of modules whose details are too big
67d7b5ef 281to fit in one document. This POD itself explains the top-level APIs
6d1c0808 282and general topics at a glance. For other topics and more details,
0ab8f81e 283see the PODs below:
67d7b5ef
JH
284
285 Name Description
286 --------------------------------------------------------
6d1c0808 287 Encode::Alias Alias definitions to encodings
67d7b5ef
JH
288 Encode::Encoding Encode Implementation Base Class
289 Encode::Supported List of Supported Encodings
290 Encode::CN Simplified Chinese Encodings
291 Encode::JP Japanese Encodings
292 Encode::KR Korean Encodings
293 Encode::TW Traditional Chinese Encodings
294 --------------------------------------------------------
295
4411f3b6
NIS
296=head1 DESCRIPTION
297
47bfe92f 298The C<Encode> module provides the interfaces between Perl's strings
67d7b5ef
JH
299and the rest of the system. Perl strings are sequences of
300B<characters>.
301
302The repertoire of characters that Perl can represent is at least that
303defined by the Unicode Consortium. On most platforms the ordinal
304values of the characters (as returned by C<ord(ch)>) is the "Unicode
305codepoint" for the character (the exceptions are those platforms where
306the legacy encoding is some variant of EBCDIC rather than a super-set
307of ASCII - see L<perlebcdic>).
308
0ab8f81e 309Traditionally, computer data has been moved around in 8-bit chunks
67d7b5ef
JH
310often called "bytes". These chunks are also known as "octets" in
311networking standards. Perl is widely used to manipulate data of many
312types - not only strings of characters representing human or computer
0ab8f81e 313languages but also "binary" data being the machine's representation of
67d7b5ef
JH
314numbers, pixels in an image - or just about anything.
315
0ab8f81e 316When Perl is processing "binary data", the programmer wants Perl to
67d7b5ef 317process "sequences of bytes". This is not a problem for Perl - as a
0ab8f81e 318byte has 256 possible values, it easily fits in Perl's much larger
67d7b5ef
JH
319"logical character".
320
321=head2 TERMINOLOGY
4411f3b6 322
7e19fb92 323=over 2
21938dfa 324
67d7b5ef
JH
325=item *
326
327I<character>: a character in the range 0..(2**32-1) (or more).
328(What Perl's strings are made of.)
329
330=item *
331
332I<byte>: a character in the range 0..255
333(A special case of a Perl character.)
334
335=item *
336
337I<octet>: 8 bits of data, with ordinal values 0..255
0ab8f81e 338(Term for bytes passed to or from a non-Perl context, e.g. a disk file.)
67d7b5ef
JH
339
340=back
4411f3b6 341
67d7b5ef
JH
342The marker [INTERNAL] marks Internal Implementation Details, in
343general meant only for those who think they know what they are doing,
344and such details may change in future releases.
345
346=head1 PERL ENCODING API
4411f3b6 347
7e19fb92 348=over 2
4411f3b6 349
f2a2953c 350=item $octets = encode(ENCODING, $string[, CHECK])
4411f3b6 351
0ab8f81e 352Encodes a string from Perl's internal form into I<ENCODING> and returns
67d7b5ef 353a sequence of octets. ENCODING can be either a canonical name or
0ab8f81e
JH
354an alias. For encoding names and aliases, see L</"Defining Aliases">.
355For CHECK, see L</"Handling Malformed Data">.
4411f3b6 356
0ab8f81e 357For example, to convert (internally UTF-8 encoded) Unicode string to
6d1c0808 358iso-8859-1 (also known as Latin1),
681a7c68 359
7e19fb92
JH
360 $octets = encode("iso-8859-1", $utf8);
361
362B<CAVEAT>: When you C<$octets = encode("utf8", $utf8)>, then $octets
363B<ne> $utf8. Though they both contain the same data, the utf8 flag
364for $octets is B<always> off. When you encode anything, utf8 flag of
365the result is always off, even when it contains completely valid utf8
366string. See L</"The UTF-8 flag"> below.
681a7c68 367
f2a2953c 368=item $string = decode(ENCODING, $octets[, CHECK])
4411f3b6 369
0ab8f81e
JH
370Decodes a sequence of octets assumed to be in I<ENCODING> into Perl's
371internal form and returns the resulting string. As in encode(),
372ENCODING can be either a canonical name or an alias. For encoding names
373and aliases, see L</"Defining Aliases">. For CHECK, see
47bfe92f
JH
374L</"Handling Malformed Data">.
375
0ab8f81e 376For example, to convert ISO-8859-1 data to UTF-8:
681a7c68 377
67d7b5ef 378 $utf8 = decode("iso-8859-1", $latin1);
681a7c68 379
7e19fb92
JH
380B<CAVEAT>: When you C<$utf8 = encode("utf8", $octets)>, then $utf8
381B<may not be equal to> $utf8. Though they both contain the same data,
382the utf8 flag for $utf8 is on unless $octets entirely conststs of
383ASCII data (or EBCDIC on EBCDIC machines). See L</"The UTF-8 flag">
384below.
47bfe92f 385
7e19fb92
JH
386=item [$length =] from_to($string, FROM_ENC, TO_ENC [, CHECK])
387
388Converts B<in-place> data between two encodings. For example, to
389convert ISO-8859-1 data to UTF-8:
2b106fbe 390
7e19fb92 391 from_to($data, "iso-8859-1", "utf8");
2b106fbe
JH
392
393and to convert it back:
394
7e19fb92 395 from_to($data, "utf8", "iso-8859-1");
4411f3b6 396
ab97ca19 397Note that because the conversion happens in place, the data to be
0ab8f81e 398converted cannot be a string constant; it must be a scalar variable.
ab97ca19 399
0ab8f81e 400from_to() returns the length of the converted string on success, undef
3ef515df
JH
401otherwise.
402
7e19fb92
JH
403B<CAVEAT>: The following operations look the same but not quite so;
404
405 from_to($data, "iso-8859-1", "utf8"); #1
406 $data = decode("iso-8859-1", $data); #2
4411f3b6 407
7e19fb92
JH
408Both #1 and #2 makes $data consists of completely valid UTF-8 string
409but only #2 turns utf8 flag on. #1 is equivalent to
f2a2953c 410
7e19fb92 411 $data = encode("utf8", decode("iso-8859-1", $data));
f2a2953c 412
7e19fb92 413See L</"The UTF-8 flag"> below.
f2a2953c
JH
414
415=item $octets = encode_utf8($string);
416
7e19fb92
JH
417Equivalent to C<$octets = encode("utf8", $string);> The characters
418that comprise $string are encoded in Perl's superset of UTF-8 and the
419resulting octets are returned as a sequence of bytes. All possible
420characters have a UTF-8 representation so this function cannot fail.
421
f2a2953c
JH
422
423=item $string = decode_utf8($octets [, CHECK]);
424
7e19fb92
JH
425equivalent to C<$string = decode("utf8", $octets [, CHECK])>.
426decode_utf8($octets [, CHECK]); The sequence of octets represented by
427$octets is decoded from UTF-8 into a sequence of logical
428characters. Not all sequences of octets form valid UTF-8 encodings, so
429it is possible for this call to fail. For CHECK, see
430L</"Handling Malformed Data">.
f2a2953c
JH
431
432=back
433
51ef4e11
NIS
434=head2 Listing available encodings
435
5129552c
JH
436 use Encode;
437 @list = Encode->encodings();
438
439Returns a list of the canonical names of the available encodings that
440are loaded. To get a list of all available encodings including the
441ones that are not loaded yet, say
442
443 @all_encodings = Encode->encodings(":all");
444
0ab8f81e 445Or you can give the name of a specific module.
5129552c 446
c731e18e
JH
447 @with_jp = Encode->encodings("Encode::JP");
448
449When "::" is not in the name, "Encode::" is assumed.
51ef4e11 450
c731e18e 451 @ebcdic = Encode->encodings("EBCDIC");
5d030b67 452
0ab8f81e 453To find out in detail which encodings are supported by this package,
5d030b67 454see L<Encode::Supported>.
51ef4e11
NIS
455
456=head2 Defining Aliases
457
0ab8f81e 458To add a new alias to a given encoding, use:
67d7b5ef 459
5129552c
JH
460 use Encode;
461 use Encode::Alias;
a63c962f 462 define_alias(newName => ENCODING);
51ef4e11 463
3ef515df 464After that, newName can be used as an alias for ENCODING.
f2a2953c
JH
465ENCODING may be either the name of an encoding or an
466I<encoding object>
51ef4e11 467
fcb875d4
JH
468But before you do so, make sure the alias is nonexistent with
469C<resolve_alias()>, which returns the canonical name thereof.
470i.e.
471
472 Encode::resolve_alias("latin1") eq "iso-8859-1" # true
473 Encode::resolve_alias("iso-8859-12") # false; nonexistent
474 Encode::resolve_alias($name) eq $name # true if $name is canonical
475
0ab8f81e
JH
476resolve_alias() does not need C<use Encode::Alias>; it can be
477exported via C<use Encode qw(resolve_alias)>.
fcb875d4 478
0ab8f81e 479See L<Encode::Alias> for details.
51ef4e11 480
85982a32 481=head1 Encoding via PerlIO
4411f3b6 482
0ab8f81e
JH
483If your perl supports I<PerlIO>, you can use a PerlIO layer to decode
484and encode directly via a filehandle. The following two examples
485are totally identical in their functionality.
4411f3b6 486
85982a32
JH
487 # via PerlIO
488 open my $in, "<:encoding(shiftjis)", $infile or die;
489 open my $out, ">:encoding(euc-jp)", $outfile or die;
490 while(<>){ print; }
8e86646e 491
85982a32 492 # via from_to
0ab8f81e
JH
493 open my $in, "<", $infile or die;
494 open my $out, ">", $outfile or die;
6d1c0808 495 while(<>){
0ab8f81e 496 from_to($_, "shiftjis", "euc-jp", 1);
85982a32 497 }
4411f3b6 498
0ab8f81e
JH
499Unfortunately, there may be encodings are PerlIO-savvy. You can check
500if your encoding is supported by PerlIO by calling the C<perlio_ok>
501method.
502
503 Encode::perlio_ok("hz"); # False
504 find_encoding("euc-cn")->perlio_ok; # True where PerlIO is available
505
506 use Encode qw(perlio_ok); # exported upon request
507 perlio_ok("euc-jp")
4411f3b6 508
0ab8f81e
JH
509Fortunately, all encodings that come with Encode core are PerlIO-savvy
510except for hz and ISO-2022-kr. See L<Encode::Encoding> for details.
4411f3b6 511
0ab8f81e 512For gory details, see L<Encode::PerlIO>.
4411f3b6 513
85982a32 514=head1 Handling Malformed Data
4411f3b6 515
7e19fb92 516=over 2
47bfe92f 517
0ab8f81e
JH
518The I<CHECK> argument is used as follows. When you omit it,
519the behaviour is the same as if you had passed a value of 0 for
520I<CHECK>.
47bfe92f 521
85982a32 522=item I<CHECK> = Encode::FB_DEFAULT ( == 0)
47bfe92f 523
0ab8f81e
JH
524If I<CHECK> is 0, (en|de)code will put a I<substitution character>
525in place of a malformed character. For UCM-based encodings,
526E<lt>subcharE<gt> will be used. For Unicode, "\x{FFFD}" is used.
527If the data is supposed to be UTF-8, an optional lexical warning
528(category utf8) is given.
e9692b5b 529
7e19fb92 530=item I<CHECK> = Encode::FB_CROAK ( == 1)
e9692b5b 531
0ab8f81e
JH
532If I<CHECK> is 1, methods will die immediately with an error
533message. Therefore, when I<CHECK> is set to 1, you should trap the
534fatal error with eval{} unless you really want to let it die on error.
47bfe92f 535
85982a32 536=item I<CHECK> = Encode::FB_QUIET
47bfe92f 537
85982a32 538If I<CHECK> is set to Encode::FB_QUIET, (en|de)code will immediately
0ab8f81e
JH
539return the portion of the data that has been processed so far when
540an error occurs. The data argument will be overwritten with
541everything after that point (that is, the unprocessed part of data).
542This is handy when you have to call decode repeatedly in the case
543where your source data may contain partial multi-byte character
544sequences, for example because you are reading with a fixed-width
545buffer. Here is some sample code that does exactly this:
4411f3b6 546
85982a32
JH
547 my $data = '';
548 while(defined(read $fh, $buffer, 256)){
0ab8f81e 549 # buffer may end in a partial character so we append
85982a32
JH
550 $data .= $buffer;
551 $utf8 .= decode($encoding, $data, ENCODE::FB_QUIET);
0ab8f81e 552 # $data now contains the unprocessed partial character
85982a32 553 }
1768d7eb 554
85982a32 555=item I<CHECK> = Encode::FB_WARN
67d7b5ef 556
0ab8f81e
JH
557This is the same as above, except that it warns on error. Handy when
558you are debugging the mode above.
85982a32
JH
559
560=item perlqq mode (I<CHECK> = Encode::FB_PERLQQ)
561
af1f55d9
JH
562=item HTML charref mode (I<CHECK> = Encode::FB_HTMLCREF)
563
564=item XML charref mode (I<CHECK> = Encode::FB_XMLCREF)
565
85982a32
JH
566For encodings that are implemented by Encode::XS, CHECK ==
567Encode::FB_PERLQQ turns (en|de)code into C<perlqq> fallback mode.
568
0ab8f81e
JH
569When you decode, '\xI<XX>' will be inserted for a malformed character,
570where I<XX> is the hex representation of the octet that could not be
571decoded to utf8. And when you encode, '\x{I<xxxx>}' will be inserted,
572where I<xxxx> is the Unicode ID of the character that cannot be found
573in the character repertoire of the encoding.
85982a32 574
af1f55d9
JH
575HTML/XML character reference modes are about the same, in place of
576\x{I<xxxx>}, HTML uses &#I<1234>; where I<1234> is a decimal digit and
577XML uses &#xI<abcd>; where I<abcd> is the hexadecimal digit.
578
85982a32
JH
579=item The bitmask
580
0ab8f81e
JH
581These modes are actually set via a bitmask. Here is how the FB_XX
582constants are laid out. You can import the FB_XX constants via
583C<use Encode qw(:fallbacks)>; you can import the generic bitmask
584constants via C<use Encode qw(:fallback_all)>.
85982a32 585
b0b300a3
JH
586 FB_DEFAULT FB_CROAK FB_QUIET FB_WARN FB_PERLQQ
587 DIE_ON_ERR 0x0001 X
588 WARN_ON_ER 0x0002 X
589 RETURN_ON_ERR 0x0004 X X
590 LEAVE_SRC 0x0008
591 PERLQQ 0x0100 X
af1f55d9
JH
592 HTMLCREF 0x0200
593 XMLCREF 0x0400
67d7b5ef 594
0ab8f81e 595=head2 Unimplemented fallback schemes
67d7b5ef 596
0ab8f81e 597In the future, you will be able to use a code reference to a callback
f2a2953c 598function for the value of I<CHECK> but its API is still undecided.
67d7b5ef
JH
599
600=head1 Defining Encodings
601
602To define a new encoding, use:
603
604 use Encode qw(define_alias);
605 define_encoding($object, 'canonicalName' [, alias...]);
606
607I<canonicalName> will be associated with I<$object>. The object
0ab8f81e 608should provide the interface described in L<Encode::Encoding>.
67d7b5ef 609If more than two arguments are provided then additional
0ab8f81e 610arguments are taken as aliases for I<$object>, as for C<define_alias>.
67d7b5ef 611
f2a2953c
JH
612See L<Encode::Encoding> for more details.
613
7e19fb92
JH
614=head1 The UTF-8 flag
615
616Before the introduction of utf8 support in perl, The C<eq> operator
617just compares internal data of the scalars. Now C<eq> means internal
618data equality AND I<the utf8 flag>. To explain why we made it so, I
619will quote page 402 of C<Programming Perl, 3rd ed.>
620
621=over 2
622
623=item Goal #1:
624
625Old byte-oriented programs should not spontaneously break on the old
626byte-oriented data they used to work on.
627
628=item Goal #2:
629
630Old byte-oriented programs should magically start working on the new
631character-oriented data when appropriate.
632
633=item Goal #3:
634
635Programs should run just as fast in the new character-oriented mode
636as in the old byte-oriented mode.
637
638=item Goal #4:
639
640Perl should remain one language, rather than forking into a
641byte-oriented Perl and a character-oriented Perl.
642
643=back
644
645Back when C<Programming Perl, 3rd ed.> was written, not even Perl 5.6.0
646was born and many features documented in the book remained
647unimplemented. Perl 5.8 hopefully correct this and the introduction
648of UTF-8 flag is one of them. You can think this perl notion of
649byte-oriented mode (utf8 flag off) and character-oriented mode (utf8
650flag on).
651
652Here is how Encode takes care of the utf8 flag.
653
4bdf5738 654=over 2
7e19fb92
JH
655
656=item *
657
658When you encode, the resulting utf8 flag is always off.
659
660=item
661
662When you decode, the resuting utf8 flag is on unless you can
663unambiguously represent data. Here is the definition of
664dis-ambiguity.
665
666 After C<$utf8 = decode('foo', $octet);>,
667
668 When $octet is... The utf8 flag in $utf8 is
669 ---------------------------------------------
670 In ASCII only (or EBCDIC only) OFF
671 In ISO-8859-1 ON
672 In any other Encoding ON
673 ---------------------------------------------
674
675As you see, there is one exception, In ASCII. That way you can assue
676Goal #1. And with Encode Goal #2 is assumed but you still have to be
677careful in such cases mentioned in B<CAVEAT> paragraphs.
678
679This utf8 flag is not visible in perl scripts, exactly for the same
680reason you cannot (or you I<don't have to>) see if a scalar contains a
681string, integer, or floating point number. But you can still peek
682and poke these if you will. See the section below.
683
684=back
685
686=head2 Messing with Perl's Internals
4411f3b6 687
47bfe92f 688The following API uses parts of Perl's internals in the current
0ab8f81e 689implementation. As such, they are efficient but may change.
4411f3b6 690
7e19fb92 691=over 2
4411f3b6 692
a63c962f 693=item is_utf8(STRING [, CHECK])
4411f3b6 694
0ab8f81e 695[INTERNAL] Tests whether the UTF-8 flag is turned on in the STRING.
47bfe92f
JH
696If CHECK is true, also checks the data in STRING for being well-formed
697UTF-8. Returns true if successful, false otherwise.
4411f3b6 698
a63c962f 699=item _utf8_on(STRING)
4411f3b6 700
0ab8f81e 701[INTERNAL] Turns on the UTF-8 flag in STRING. The data in STRING is
4411f3b6
NIS
702B<not> checked for being well-formed UTF-8. Do not use unless you
703B<know> that the STRING is well-formed UTF-8. Returns the previous
0ab8f81e
JH
704state of the UTF-8 flag (so please don't treat the return value as
705indicating success or failure), or C<undef> if STRING is not a string.
4411f3b6 706
a63c962f 707=item _utf8_off(STRING)
4411f3b6 708
0ab8f81e
JH
709[INTERNAL] Turns off the UTF-8 flag in STRING. Do not use frivolously.
710Returns the previous state of the UTF-8 flag (so please don't treat the
711return value as indicating success or failure), or C<undef> if STRING is
4411f3b6
NIS
712not a string.
713
714=back
715
716=head1 SEE ALSO
717
5d030b67
JH
718L<Encode::Encoding>,
719L<Encode::Supported>,
6d1c0808 720L<Encode::PerlIO>,
5d030b67 721L<encoding>,
6d1c0808
JH
722L<perlebcdic>,
723L<perlfunc/open>,
724L<perlunicode>,
725L<utf8>,
5d030b67 726the Perl Unicode Mailing List E<lt>perl-unicode@perl.orgE<gt>
4411f3b6 727
85982a32 728=head1 MAINTAINER
aae85ceb
DK
729
730This project was originated by Nick Ing-Simmons and later maintained
7e19fb92
JH
731by Dan Kogai E<lt>dankogai@dan.co.jpE<gt>. See AUTHORS for a full
732list of people involved. For any questions, use
733E<lt>perl-unicode@perl.orgE<gt> so we can all share share.
aae85ceb 734
4411f3b6 735=cut