This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Must Uppercase.
[perl5.git] / ext / Encode / Encode.pm
CommitLineData
2c674647 1package Encode;
51ef4e11 2use strict;
2c674647 3
b8a524e9 4our $VERSION = '0.02';
2c674647
JH
5
6require DynaLoader;
7require Exporter;
8
51ef4e11 9our @ISA = qw(Exporter DynaLoader);
2c674647 10
4411f3b6 11# Public, encouraged API is exported by default
51ef4e11 12our @EXPORT = qw (
4411f3b6
NIS
13 encode
14 decode
15 encode_utf8
16 decode_utf8
17 find_encoding
51ef4e11 18 encodings
4411f3b6
NIS
19);
20
51ef4e11 21our @EXPORT_OK =
2c674647 22 qw(
51ef4e11
NIS
23 define_encoding
24 define_alias
2c674647
JH
25 from_to
26 is_utf8
4411f3b6
NIS
27 is_8bit
28 is_16bit
a12c0f56
NIS
29 utf8_upgrade
30 utf8_downgrade
4411f3b6
NIS
31 _utf8_on
32 _utf8_off
2c674647
JH
33 );
34
35bootstrap Encode ();
36
4411f3b6 37# Documentation moved after __END__ for speed - NI-S
2c674647 38
bf230f3d
NIS
39use Carp;
40
51ef4e11
NIS
41# Make a %encoding package variable to allow a certain amount of cheating
42our %encoding;
43my @alias; # ordered matching list
44my %alias; # cached known aliases
f7ac3676 45
6d6a7c8d
NIS
46 # 0 1 2 3 4 5 6 7 8 9 10
47our @latin2iso_num = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 );
48
f7ac3676
JH
49our %winlatin2cp = (
50 'Latin1' => 1252,
51 'Latin2' => 1250,
52 'Cyrillic' => 1251,
53 'Baltic' => 1257,
54 'Greek' => 1253,
55 'Turkish' => 1254,
56 'Hebrew' => 1255,
57 'Arabic' => 1256,
58 'Baltic' => 1257,
59 'Vietnamese' => 1258,
60 );
5345d506 61
656753f8
NIS
62sub encodings
63{
64 my ($class) = @_;
51ef4e11
NIS
65 return keys %encoding;
66}
67
68sub findAlias
69{
70 my $class = shift;
71 local $_ = shift;
72 unless (exists $alias{$_})
656753f8 73 {
51ef4e11 74 for (my $i=0; $i < @alias; $i += 2)
656753f8 75 {
51ef4e11
NIS
76 my $alias = $alias[$i];
77 my $val = $alias[$i+1];
78 my $new;
79 if (ref($alias) eq 'Regexp' && $_ =~ $alias)
5345d506 80 {
51ef4e11
NIS
81 $new = eval $val;
82 }
83 elsif (ref($alias) eq 'CODE')
84 {
85 $new = &{$alias}($val)
86 }
5ad8ef52 87 elsif (lc($_) eq lc($alias))
51ef4e11
NIS
88 {
89 $new = $val;
90 }
91 if (defined($new))
92 {
93 next if $new eq $_; # avoid (direct) recursion on bugs
94 my $enc = (ref($new)) ? $new : find_encoding($new);
95 if ($enc)
5345d506 96 {
51ef4e11
NIS
97 $alias{$_} = $enc;
98 last;
5345d506
NIS
99 }
100 }
656753f8 101 }
5345d506 102 }
51ef4e11 103 return $alias{$_};
5345d506
NIS
104}
105
51ef4e11 106sub define_alias
5345d506 107{
51ef4e11 108 while (@_)
5345d506 109 {
51ef4e11
NIS
110 my ($alias,$name) = splice(@_,0,2);
111 push(@alias, $alias => $name);
656753f8 112 }
51ef4e11
NIS
113}
114
016cb72c 115# Allow variants of iso-8859-1 etc.
d6089a2a 116define_alias( qr/^iso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' );
016cb72c 117
7faf300d
JH
118# At least HP-UX has these.
119define_alias( qr/^iso8859(\d+)$/i => '"iso-8859-$1"' );
120
f7ac3676
JH
121# More HP stuff.
122define_alias( qr/^(?:hp-)?(arabic|greek|hebrew|kana|roman|thai|turkish)8$/i => '"${1}8"' );
123
58d53262
JH
124# This is a font issue, not an encoding issue.
125# (The currency symbol of the Latin 1 upper half
126# has been redefined as the euro symbol.)
127define_alias( qr/^(.+)\@euro$/i => '"$1"' );
128
016cb72c 129# Allow latin-1 style names as well
7faf300d 130define_alias( qr/^(?:iso[-_]?)?latin[-_]?(\d+)$/i => '"iso-8859-$latin2iso_num[$1]"' );
016cb72c 131
f7ac3676 132# Allow winlatin1 style names as well
cf91068f 133define_alias( qr/^win(latin[12]|cyrillic|baltic|greek|turkish|hebrew|arabic|baltic|vietnamese)$/i => '"cp$winlatin2cp{\u$1}"' );
f7ac3676 134
016cb72c
NIS
135# Common names for non-latin prefered MIME names
136define_alias( 'ascii' => 'US-ascii',
137 'cyrillic' => 'iso-8859-5',
138 'arabic' => 'iso-8859-6',
139 'greek' => 'iso-8859-7',
f7ac3676
JH
140 'hebrew' => 'iso-8859-8',
141 'thai' => 'iso-8859-11',
142 'tis620' => 'iso-8859-11',
143 );
016cb72c 144
7faf300d
JH
145# At least AIX has IBM-NNN (surprisingly...) instead of cpNNN.
146define_alias( qr/^ibm[-_]?(\d\d\d\d?)$/i => '"cp$1"');
147
58d53262
JH
148# Standardize on the dashed versions.
149define_alias( qr/^utf8$/i => 'utf-8' );
7faf300d 150define_alias( qr/^koi8r$/i => 'koi8-r' );
f7ac3676
JH
151define_alias( qr/^koi8u$/i => 'koi8-u' );
152
153# TODO: HP-UX '8' encodings arabic8 greek8 hebrew8 kana8 thai8 turkish8
154# TODO: HP-UX '15' encodings japanese15 korean15 roi15
155# TODO: Cyrillic encoding ISO-IR-111 (useful?)
156# TODO: Chinese encodings GB18030 GBK Big5-HSKCS EUC-TW
157# TODO: Armenian encoding ARMSCII-8
158# TODO: Hebrew encoding ISO-8859-8-1
159# TODO: Thai encoding TCVN
160# TODO: Korean encoding Johab
161# TODO: Vietnamese encodings VISCII VPS
162# TODO: Japanese encoding JIS (not the same as SJIS)
163# TODO: Mac Asian+African encodings: Arabic Armenian Bengali Burmese
164# ChineseSimp ChineseTrad Devanagari Ethiopic ExtArabic
165# Farsi Georgian Gujarati Gurmukhi Hebrew Japanese
166# Kannada Khmer Korean Laotian Malayalam Mongolian
167# Oriya Sinhalese Symbol Tamil Telugu Tibetan Vietnamese
168# TODO: what is the Japanese 'UJIS' encoding seen in some Linuxes?
58d53262 169
016cb72c
NIS
170# Map white space and _ to '-'
171define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' );
172
51ef4e11
NIS
173sub define_encoding
174{
175 my $obj = shift;
176 my $name = shift;
177 $encoding{$name} = $obj;
178 my $lc = lc($name);
179 define_alias($lc => $obj) unless $lc eq $name;
180 while (@_)
656753f8 181 {
51ef4e11
NIS
182 my $alias = shift;
183 define_alias($alias,$obj);
656753f8 184 }
51ef4e11 185 return $obj;
656753f8
NIS
186}
187
656753f8
NIS
188sub getEncoding
189{
190 my ($class,$name) = @_;
5345d506 191 my $enc;
0f43fc90
NIS
192 if (ref($name) && $name->can('new_sequence'))
193 {
194 return $name;
195 }
51ef4e11 196 if (exists $encoding{$name})
656753f8 197 {
51ef4e11
NIS
198 return $encoding{$name};
199 }
200 else
201 {
202 return $class->findAlias($name);
656753f8 203 }
656753f8
NIS
204}
205
4411f3b6
NIS
206sub find_encoding
207{
208 my ($name) = @_;
209 return __PACKAGE__->getEncoding($name);
210}
211
212sub encode
213{
214 my ($name,$string,$check) = @_;
215 my $enc = find_encoding($name);
216 croak("Unknown encoding '$name'") unless defined $enc;
50d26985 217 my $octets = $enc->encode($string,$check);
4411f3b6
NIS
218 return undef if ($check && length($string));
219 return $octets;
220}
221
222sub decode
223{
224 my ($name,$octets,$check) = @_;
225 my $enc = find_encoding($name);
226 croak("Unknown encoding '$name'") unless defined $enc;
50d26985 227 my $string = $enc->decode($octets,$check);
4411f3b6
NIS
228 return undef if ($check && length($octets));
229 return $string;
230}
231
232sub from_to
233{
234 my ($string,$from,$to,$check) = @_;
235 my $f = find_encoding($from);
236 croak("Unknown encoding '$from'") unless defined $f;
237 my $t = find_encoding($to);
238 croak("Unknown encoding '$to'") unless defined $t;
50d26985 239 my $uni = $f->decode($string,$check);
4411f3b6 240 return undef if ($check && length($string));
50d26985 241 $string = $t->encode($uni,$check);
4411f3b6
NIS
242 return undef if ($check && length($uni));
243 return length($_[0] = $string);
244}
245
246sub encode_utf8
247{
248 my ($str) = @_;
1b026014 249 utf8::encode($str);
4411f3b6
NIS
250 return $str;
251}
252
253sub decode_utf8
254{
255 my ($str) = @_;
1b026014 256 return undef unless utf8::decode($str);
4411f3b6
NIS
257 return $str;
258}
259
50d26985
NIS
260package Encode::Encoding;
261# Base class for classes which implement encodings
4edaa979 262
51ef4e11
NIS
263sub Define
264{
265 my $obj = shift;
266 my $canonical = shift;
267 $obj = bless { Name => $canonical },$obj unless ref $obj;
268 # warn "$canonical => $obj\n";
269 Encode::define_encoding($obj, $canonical, @_);
270}
271
272sub name { shift->{'Name'} }
273
50d26985 274# Temporary legacy methods
4edaa979
NIS
275sub toUnicode { shift->decode(@_) }
276sub fromUnicode { shift->encode(@_) }
277
278sub new_sequence { return $_[0] }
50d26985
NIS
279
280package Encode::XS;
281use base 'Encode::Encoding';
282
5ad8ef52 283package Encode::Internal;
50d26985 284use base 'Encode::Encoding';
656753f8 285
9b37254d 286# Dummy package that provides the encode interface but leaves data
1b026014 287# as UTF-X encoded. It is here so that from_to() works.
656753f8 288
5ad8ef52
NIS
289__PACKAGE__->Define('Internal');
290
291Encode::define_alias( 'Unicode' => 'Internal' ) if ord('A') == 65;
656753f8 292
50d26985 293sub decode
a12c0f56
NIS
294{
295 my ($obj,$str,$chk) = @_;
1b026014 296 utf8::upgrade($str);
a12c0f56
NIS
297 $_[1] = '' if $chk;
298 return $str;
299}
656753f8 300
50d26985 301*encode = \&decode;
656753f8 302
5ad8ef52
NIS
303package Encoding::Unicode;
304use base 'Encode::Encoding';
305
306__PACKAGE__->Define('Unicode') unless ord('A') == 65;
307
308sub decode
309{
310 my ($obj,$str,$chk) = @_;
311 my $res = '';
312 for (my $i = 0; $i < length($str); $i++)
313 {
314 $res .= chr(utf8::unicode_to_native(ord(substr($str,$i,1))));
315 }
316 $_[1] = '' if $chk;
317 return $res;
318}
319
320sub encode
321{
322 my ($obj,$str,$chk) = @_;
323 my $res = '';
324 for (my $i = 0; $i < length($str); $i++)
325 {
326 $res .= chr(utf8::native_to_unicode(ord(substr($str,$i,1))));
327 }
328 $_[1] = '' if $chk;
329 return $res;
330}
331
332
4411f3b6 333package Encode::utf8;
50d26985 334use base 'Encode::Encoding';
4411f3b6
NIS
335# package to allow long-hand
336# $octets = encode( utf8 => $string );
337#
338
51ef4e11 339__PACKAGE__->Define(qw(UTF-8 utf8));
4411f3b6 340
50d26985 341sub decode
4411f3b6
NIS
342{
343 my ($obj,$octets,$chk) = @_;
2a936312 344 my $str = Encode::decode_utf8($octets);
4411f3b6
NIS
345 if (defined $str)
346 {
347 $_[1] = '' if $chk;
348 return $str;
349 }
350 return undef;
351}
352
50d26985 353sub encode
4411f3b6
NIS
354{
355 my ($obj,$string,$chk) = @_;
2a936312 356 my $octets = Encode::encode_utf8($string);
4411f3b6
NIS
357 $_[1] = '' if $chk;
358 return $octets;
4411f3b6
NIS
359}
360
9b37254d 361package Encode::iso10646_1;
50d26985 362use base 'Encode::Encoding';
51ef4e11 363# Encoding is 16-bit network order Unicode (no surogates)
9b37254d 364# Used for X font encodings
87714904 365
8040349a 366__PACKAGE__->Define(qw(UCS-2 iso-10646-1));
87714904 367
50d26985 368sub decode
87714904
NIS
369{
370 my ($obj,$str,$chk) = @_;
371 my $uni = '';
372 while (length($str))
373 {
5dcbab34 374 my $code = unpack('n',substr($str,0,2,'')) & 0xffff;
87714904
NIS
375 $uni .= chr($code);
376 }
377 $_[1] = $str if $chk;
8040349a 378 utf8::upgrade($uni);
87714904
NIS
379 return $uni;
380}
381
50d26985 382sub encode
87714904
NIS
383{
384 my ($obj,$uni,$chk) = @_;
385 my $str = '';
386 while (length($uni))
387 {
388 my $ch = substr($uni,0,1,'');
389 my $x = ord($ch);
390 unless ($x < 32768)
391 {
392 last if ($chk);
393 $x = 0;
394 }
5dcbab34 395 $str .= pack('n',$x);
656753f8 396 }
bf230f3d 397 $_[1] = $uni if $chk;
656753f8
NIS
398 return $str;
399}
400
79019f4f
MS
401package Encode::ucs_2le;
402use base 'Encode::Encoding';
403
404__PACKAGE__->Define(qw(UCS-2le UCS-2LE ucs-2le));
405
406sub decode
407{
408 my ($obj,$str,$chk) = @_;
409 my $uni = '';
410 while (length($str))
411 {
412 my $code = unpack('v',substr($str,0,2,'')) & 0xffff;
413 $uni .= chr($code);
414 }
415 $_[1] = $str if $chk;
416 utf8::upgrade($uni);
417 return $uni;
418}
419
420sub encode
421{
422 my ($obj,$uni,$chk) = @_;
423 my $str = '';
424 while (length($uni))
425 {
426 my $ch = substr($uni,0,1,'');
427 my $x = ord($ch);
428 unless ($x < 32768)
429 {
430 last if ($chk);
431 $x = 0;
432 }
433 $str .= pack('v',$x);
434 }
435 $_[1] = $uni if $chk;
436 return $str;
437}
438
4411f3b6
NIS
439# switch back to Encode package in case we ever add AutoLoader
440package Encode;
441
656753f8
NIS
4421;
443
2a936312
NIS
444__END__
445
4411f3b6
NIS
446=head1 NAME
447
448Encode - character encodings
449
450=head1 SYNOPSIS
451
452 use Encode;
453
454=head1 DESCRIPTION
455
47bfe92f
JH
456The C<Encode> module provides the interfaces between Perl's strings
457and the rest of the system. Perl strings are sequences of B<characters>.
4411f3b6
NIS
458
459The repertoire of characters that Perl can represent is at least that
47bfe92f
JH
460defined by the Unicode Consortium. On most platforms the ordinal
461values of the characters (as returned by C<ord(ch)>) is the "Unicode
462codepoint" for the character (the exceptions are those platforms where
463the legacy encoding is some variant of EBCDIC rather than a super-set
464of ASCII - see L<perlebcdic>).
4411f3b6
NIS
465
466Traditionaly computer data has been moved around in 8-bit chunks
467often called "bytes". These chunks are also known as "octets" in
468networking standards. Perl is widely used to manipulate data of
469many types - not only strings of characters representing human or
470computer languages but also "binary" data being the machines representation
471of numbers, pixels in an image - or just about anything.
472
47bfe92f
JH
473When Perl is processing "binary data" the programmer wants Perl to process
474"sequences of bytes". This is not a problem for Perl - as a byte has 256
475possible values it easily fits in Perl's much larger "logical character".
4411f3b6
NIS
476
477=head2 TERMINOLOGY
478
4ac9195f 479=over 4
4411f3b6
NIS
480
481=item *
482
483I<character>: a character in the range 0..(2**32-1) (or more).
47bfe92f 484(What Perl's strings are made of.)
4411f3b6
NIS
485
486=item *
487
488I<byte>: a character in the range 0..255
47bfe92f 489(A special case of a Perl character.)
4411f3b6
NIS
490
491=item *
492
493I<octet>: 8 bits of data, with ordinal values 0..255
47bfe92f 494(Term for bytes passed to or from a non-Perl context, e.g. disk file.)
4411f3b6
NIS
495
496=back
497
498The marker [INTERNAL] marks Internal Implementation Details, in
499general meant only for those who think they know what they are doing,
500and such details may change in future releases.
501
502=head1 ENCODINGS
503
504=head2 Characteristics of an Encoding
505
506An encoding has a "repertoire" of characters that it can represent,
507and for each representable character there is at least one sequence of
508octets that represents it.
509
510=head2 Types of Encodings
511
512Encodings can be divided into the following types:
513
514=over 4
515
516=item * Fixed length 8-bit (or less) encodings.
517
518Each character is a single octet so may have a repertoire of up to
519256 characters. ASCII and iso-8859-* are typical examples.
520
521=item * Fixed length 16-bit encodings
522
523Each character is two octets so may have a repertoire of up to
47bfe92f 52465 536 characters. Unicode's UCS-2 is an example. Also used for
4411f3b6
NIS
525encodings for East Asian languages.
526
527=item * Fixed length 32-bit encodings.
528
529Not really very "encoded" encodings. The Unicode code points
530are just represented as 4-octet integers. None the less because
531different architectures use different representations of integers
532(so called "endian") there at least two disctinct encodings.
533
534=item * Multi-byte encodings
535
536The number of octets needed to represent a character varies.
537UTF-8 is a particularly complex but regular case of a multi-byte
538encoding. Several East Asian countries use a multi-byte encoding
539where 1-octet is used to cover western roman characters and Asian
540characters get 2-octets.
541(UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets
542to represent a Unicode code point.)
543
544=item * "Escape" encodings.
545
546These encodings embed "escape sequences" into the octet sequence
547which describe how the following octets are to be interpreted.
548The iso-2022-* family is typical. Following the escape sequence
549octets are encoded by an "embedded" encoding (which will be one
550of the above types) until another escape sequence switches to
551a different "embedded" encoding.
552
553These schemes are very flexible and can handle mixed languages but are
47bfe92f
JH
554very complex to process (and have state). No escape encodings are
555implemented for Perl yet.
4411f3b6
NIS
556
557=back
558
559=head2 Specifying Encodings
560
561Encodings can be specified to the API described below in two ways:
562
563=over 4
564
565=item 1. By name
566
47bfe92f
JH
567Encoding names are strings with characters taken from a restricted
568repertoire. See L</"Encoding Names">.
4411f3b6
NIS
569
570=item 2. As an object
571
572Encoding objects are returned by C<find_encoding($name)>.
573
574=back
575
576=head2 Encoding Names
577
578Encoding names are case insensitive. White space in names is ignored.
47bfe92f
JH
579In addition an encoding may have aliases. Each encoding has one
580"canonical" name. The "canonical" name is chosen from the names of
581the encoding by picking the first in the following sequence:
4411f3b6
NIS
582
583=over 4
584
585=item * The MIME name as defined in IETF RFC-XXXX.
586
587=item * The name in the IANA registry.
588
589=item * The name used by the the organization that defined it.
590
591=back
592
593Because of all the alias issues, and because in the general case
594encodings have state C<Encode> uses the encoding object internally
595once an operation is in progress.
596
4411f3b6
NIS
597=head1 PERL ENCODING API
598
599=head2 Generic Encoding Interface
600
601=over 4
602
603=item *
604
605 $bytes = encode(ENCODING, $string[, CHECK])
606
47bfe92f
JH
607Encodes string from Perl's internal form into I<ENCODING> and returns
608a sequence of octets. For CHECK see L</"Handling Malformed Data">.
4411f3b6
NIS
609
610=item *
611
612 $string = decode(ENCODING, $bytes[, CHECK])
613
47bfe92f
JH
614Decode sequence of octets assumed to be in I<ENCODING> into Perl's
615internal form and returns the resulting string. For CHECK see
616L</"Handling Malformed Data">.
617
618=item *
619
620 from_to($string, FROM_ENCODING, TO_ENCODING[, CHECK])
621
2b106fbe
JH
622Convert B<in-place> the data between two encodings. How did the data
623in $string originally get to be in FROM_ENCODING? Either using
e9692b5b 624encode() or through PerlIO: See L</"Encoding and IO">. For CHECK
2b106fbe
JH
625see L</"Handling Malformed Data">.
626
627For example to convert ISO 8859-1 data to UTF-8:
628
629 from_to($data, "iso-8859-1", "utf-8");
630
631and to convert it back:
632
633 from_to($data, "utf-8", "iso-8859-1");
4411f3b6 634
ab97ca19
JH
635Note that because the conversion happens in place, the data to be
636converted cannot be a string constant, it must be a scalar variable.
637
4411f3b6
NIS
638=back
639
640=head2 Handling Malformed Data
641
642If CHECK is not set, C<undef> is returned. If the data is supposed to
47bfe92f
JH
643be UTF-8, an optional lexical warning (category utf8) is given. If
644CHECK is true but not a code reference, dies.
4411f3b6 645
47bfe92f
JH
646It would desirable to have a way to indicate that transform should use
647the encodings "replacement character" - no such mechanism is defined yet.
4411f3b6
NIS
648
649It is also planned to allow I<CHECK> to be a code reference.
650
47bfe92f
JH
651This is not yet implemented as there are design issues with what its
652arguments should be and how it returns its results.
4411f3b6
NIS
653
654=over 4
655
656=item Scheme 1
657
658Passed remaining fragment of string being processed.
659Modifies it in place to remove bytes/characters it can understand
660and returns a string used to represent them.
661e.g.
662
663 sub fixup {
664 my $ch = substr($_[0],0,1,'');
665 return sprintf("\x{%02X}",ord($ch);
666 }
667
668This scheme is close to how underlying C code for Encode works, but gives
669the fixup routine very little context.
670
671=item Scheme 2
672
47bfe92f
JH
673Passed original string, and an index into it of the problem area, and
674output string so far. Appends what it will to output string and
675returns new index into original string. For example:
4411f3b6
NIS
676
677 sub fixup {
678 # my ($s,$i,$d) = @_;
679 my $ch = substr($_[0],$_[1],1);
680 $_[2] .= sprintf("\x{%02X}",ord($ch);
681 return $_[1]+1;
682 }
683
47bfe92f
JH
684This scheme gives maximal control to the fixup routine but is more
685complicated to code, and may need internals of Encode to be tweaked to
686keep original string intact.
4411f3b6
NIS
687
688=item Other Schemes
689
690Hybrids of above.
691
692Multiple return values rather than in-place modifications.
693
694Index into the string could be pos($str) allowing s/\G...//.
695
696=back
697
698=head2 UTF-8 / utf8
699
700The Unicode consortium defines the UTF-8 standard as a way of encoding
47bfe92f
JH
701the entire Unicode repertiore as sequences of octets. This encoding is
702expected to become very widespread. Perl can use this form internaly
703to represent strings, so conversions to and from this form are
704particularly efficient (as octets in memory do not have to change,
705just the meta-data that tells Perl how to treat them).
4411f3b6
NIS
706
707=over 4
708
709=item *
710
711 $bytes = encode_utf8($string);
712
47bfe92f 713The characters that comprise string are encoded in Perl's superset of UTF-8
4411f3b6
NIS
714and the resulting octets returned as a sequence of bytes. All possible
715characters have a UTF-8 representation so this function cannot fail.
716
717=item *
718
719 $string = decode_utf8($bytes [,CHECK]);
720
47bfe92f
JH
721The sequence of octets represented by $bytes is decoded from UTF-8
722into a sequence of logical characters. Not all sequences of octets
723form valid UTF-8 encodings, so it is possible for this call to fail.
724For CHECK see L</"Handling Malformed Data">.
4411f3b6
NIS
725
726=back
727
728=head2 Other Encodings of Unicode
729
47bfe92f
JH
730UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. UCS-2 can only
731represent 0..0xFFFF, while UTF-16 has a "surrogate pair" scheme which
732allows it to cover the whole Unicode range.
4411f3b6 733
8040349a 734Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that
47bfe92f
JH
735happens to be the name used by that representation when used with X11
736fonts.
4411f3b6
NIS
737
738UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters
739can be considered as being in this form without encoding. An encoding
47bfe92f
JH
740to transfer strings in this form (e.g. to write them to a file) would
741need to
4411f3b6 742
c079d275 743 pack('L*', unpack('U*', $string)); # native
4411f3b6 744 or
c079d275 745 pack('V*', unpack('U*', $string)); # little-endian
4411f3b6 746 or
c079d275 747 pack('N*', unpack('U*', $string)); # big-endian
4411f3b6 748
c079d275 749depending on the endianness required.
4411f3b6 750
51ef4e11 751No UTF-32 encodings are implemented yet.
4411f3b6 752
47bfe92f
JH
753Both UCS-2 and UCS-4 style encodings can have "byte order marks" by
754representing the code point 0xFFFE as the very first thing in a file.
4411f3b6 755
51ef4e11
NIS
756=head2 Listing available encodings
757
758 use Encode qw(encodings);
759 @list = encodings();
760
761Returns a list of the canonical names of the available encodings.
762
763=head2 Defining Aliases
764
765 use Encode qw(define_alias);
766 define_alias( newName => ENCODING);
767
47bfe92f
JH
768Allows newName to be used as am alias for ENCODING. ENCODING may be
769either the name of an encoding or and encoding object (as above).
51ef4e11
NIS
770
771Currently I<newName> can be specified in the following ways:
772
773=over 4
774
775=item As a simple string.
776
777=item As a qr// compiled regular expression, e.g.:
778
779 define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' );
780
47bfe92f
JH
781In this case if I<ENCODING> is not a reference it is C<eval>-ed to
782allow C<$1> etc. to be subsituted. The example is one way to names as
783used in X11 font names to alias the MIME names for the iso-8859-*
784family.
51ef4e11
NIS
785
786=item As a code reference, e.g.:
787
788 define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , '');
789
790In this case C<$_> will be set to the name that is being looked up and
47bfe92f
JH
791I<ENCODING> is passed to the sub as its first argument. The example
792is another way to names as used in X11 font names to alias the MIME
793names for the iso-8859-* family.
51ef4e11
NIS
794
795=back
796
797=head2 Defining Encodings
798
e9692b5b
JH
799 use Encode qw(define_alias);
800 define_encoding( $object, 'canonicalName' [,alias...]);
51ef4e11 801
47bfe92f
JH
802Causes I<canonicalName> to be associated with I<$object>. The object
803should provide the interface described in L</"IMPLEMENTATION CLASSES">
804below. If more than two arguments are provided then additional
805arguments are taken as aliases for I<$object> as for C<define_alias>.
51ef4e11 806
4411f3b6
NIS
807=head1 Encoding and IO
808
809It is very common to want to do encoding transformations when
810reading or writing files, network connections, pipes etc.
47bfe92f 811If Perl is configured to use the new 'perlio' IO system then
4411f3b6
NIS
812C<Encode> provides a "layer" (See L<perliol>) which can transform
813data as it is read or written.
814
8e86646e
JH
815Here is how the blind poet would modernise the encoding:
816
42234700 817 use Encode;
8e86646e
JH
818 open(my $iliad,'<:encoding(iso-8859-7)','iliad.greek');
819 open(my $utf8,'>:utf8','iliad.utf8');
820 my @epic = <$iliad>;
821 print $utf8 @epic;
822 close($utf8);
823 close($illiad);
4411f3b6
NIS
824
825In addition the new IO system can also be configured to read/write
826UTF-8 encoded characters (as noted above this is efficient):
827
e9692b5b
JH
828 open(my $fh,'>:utf8','anything');
829 print $fh "Any \x{0021} string \N{SMILEY FACE}\n";
4411f3b6
NIS
830
831Either of the above forms of "layer" specifications can be made the default
832for a lexical scope with the C<use open ...> pragma. See L<open>.
833
834Once a handle is open is layers can be altered using C<binmode>.
835
47bfe92f 836Without any such configuration, or if Perl itself is built using
4411f3b6
NIS
837system's own IO, then write operations assume that file handle accepts
838only I<bytes> and will C<die> if a character larger than 255 is
839written to the handle. When reading, each octet from the handle
840becomes a byte-in-a-character. Note that this default is the same
47bfe92f
JH
841behaviour as bytes-only languages (including Perl before v5.6) would
842have, and is sufficient to handle native 8-bit encodings
843e.g. iso-8859-1, EBCDIC etc. and any legacy mechanisms for handling
844other encodings and binary data.
845
846In other cases it is the programs responsibility to transform
847characters into bytes using the API above before doing writes, and to
848transform the bytes read from a handle into characters before doing
849"character operations" (e.g. C<lc>, C</\W+/>, ...).
850
47bfe92f
JH
851You can also use PerlIO to convert larger amounts of data you don't
852want to bring into memory. For example to convert between ISO 8859-1
853(Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines):
854
e9692b5b
JH
855 open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!;
856 open(G, ">:utf8", "data.utf") or die $!;
857 while (<F>) { print G }
858
859 # Could also do "print G <F>" but that would pull
860 # the whole file into memory just to write it out again.
861
862More examples:
47bfe92f 863
e9692b5b
JH
864 open(my $f, "<:encoding(cp1252)")
865 open(my $g, ">:encoding(iso-8859-2)")
866 open(my $h, ">:encoding(latin9)") # iso-8859-15
47bfe92f
JH
867
868See L<PerlIO> for more information.
4411f3b6
NIS
869
870=head1 Encoding How to ...
871
872To do:
873
874=over 4
875
876=item * IO with mixed content (faking iso-2020-*)
877
878=item * MIME's Content-Length:
879
880=item * UTF-8 strings in binary data.
881
47bfe92f 882=item * Perl/Encode wrappers on non-Unicode XS modules.
4411f3b6
NIS
883
884=back
885
886=head1 Messing with Perl's Internals
887
47bfe92f
JH
888The following API uses parts of Perl's internals in the current
889implementation. As such they are efficient, but may change.
4411f3b6
NIS
890
891=over 4
892
4411f3b6
NIS
893=item * is_utf8(STRING [, CHECK])
894
895[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING.
47bfe92f
JH
896If CHECK is true, also checks the data in STRING for being well-formed
897UTF-8. Returns true if successful, false otherwise.
4411f3b6
NIS
898
899=item * valid_utf8(STRING)
900
47bfe92f
JH
901[INTERNAL] Test whether STRING is in a consistent state. Will return
902true if string is held as bytes, or is well-formed UTF-8 and has the
903UTF-8 flag on. Main reason for this routine is to allow Perl's
904testsuite to check that operations have left strings in a consistent
905state.
4411f3b6
NIS
906
907=item *
908
909 _utf8_on(STRING)
910
911[INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is
912B<not> checked for being well-formed UTF-8. Do not use unless you
913B<know> that the STRING is well-formed UTF-8. Returns the previous
914state of the UTF-8 flag (so please don't test the return value as
915I<not> success or failure), or C<undef> if STRING is not a string.
916
917=item *
918
919 _utf8_off(STRING)
920
921[INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously.
922Returns the previous state of the UTF-8 flag (so please don't test the
923return value as I<not> success or failure), or C<undef> if STRING is
924not a string.
925
926=back
927
4edaa979
NIS
928=head1 IMPLEMENTATION CLASSES
929
930As mentioned above encodings are (in the current implementation at least)
931defined by objects. The mapping of encoding name to object is via the
51ef4e11 932C<%encodings> hash.
4edaa979
NIS
933
934The values of the hash can currently be either strings or objects.
935The string form may go away in the future. The string form occurs
936when C<encodings()> has scanned C<@INC> for loadable encodings but has
937not actually loaded the encoding in question. This is because the
47bfe92f 938current "loading" process is all Perl and a bit slow.
4edaa979 939
47bfe92f
JH
940Once an encoding is loaded then value of the hash is object which
941implements the encoding. The object should provide the following
942interface:
4edaa979
NIS
943
944=over 4
945
946=item -E<gt>name
947
948Should return the string representing the canonical name of the encoding.
949
950=item -E<gt>new_sequence
951
47bfe92f
JH
952This is a placeholder for encodings with state. It should return an
953object which implements this interface, all current implementations
954return the original object.
4edaa979
NIS
955
956=item -E<gt>encode($string,$check)
957
47bfe92f
JH
958Should return the octet sequence representing I<$string>. If I<$check>
959is true it should modify I<$string> in place to remove the converted
960part (i.e. the whole string unless there is an error). If an error
961occurs it should return the octet sequence for the fragment of string
962that has been converted, and modify $string in-place to remove the
963converted part leaving it starting with the problem fragment.
4edaa979 964
47bfe92f
JH
965If check is is false then C<encode> should make a "best effort" to
966convert the string - for example by using a replacement character.
4edaa979
NIS
967
968=item -E<gt>decode($octets,$check)
969
47bfe92f
JH
970Should return the string that I<$octets> represents. If I<$check> is
971true it should modify I<$octets> in place to remove the converted part
972(i.e. the whole sequence unless there is an error). If an error
973occurs it should return the fragment of string that has been
974converted, and modify $octets in-place to remove the converted part
4edaa979
NIS
975leaving it starting with the problem fragment.
976
47bfe92f
JH
977If check is is false then C<decode> should make a "best effort" to
978convert the string - for example by using Unicode's "\x{FFFD}" as a
979replacement character.
4edaa979
NIS
980
981=back
982
47bfe92f
JH
983It should be noted that the check behaviour is different from the
984outer public API. The logic is that the "unchecked" case is useful
985when encoding is part of a stream which may be reporting errors
986(e.g. STDERR). In such cases it is desirable to get everything
987through somehow without causing additional errors which obscure the
988original one. Also the encoding is best placed to know what the
989correct replacement character is, so if that is the desired behaviour
990then letting low level code do it is the most efficient.
991
992In contrast if check is true, the scheme above allows the encoding to
993do as much as it can and tell layer above how much that was. What is
994lacking at present is a mechanism to report what went wrong. The most
995likely interface will be an additional method call to the object, or
996perhaps (to avoid forcing per-stream objects on otherwise stateless
997encodings) and additional parameter.
998
999It is also highly desirable that encoding classes inherit from
1000C<Encode::Encoding> as a base class. This allows that class to define
1001additional behaviour for all encoding objects. For example built in
1002Unicode, UCS-2 and UTF-8 classes use :
51ef4e11
NIS
1003
1004 package Encode::MyEncoding;
1005 use base qw(Encode::Encoding);
1006
1007 __PACKAGE__->Define(qw(myCanonical myAlias));
1008
47bfe92f
JH
1009To create an object with bless {Name => ...},$class, and call
1010define_encoding. They inherit their C<name> method from
1011C<Encode::Encoding>.
4edaa979
NIS
1012
1013=head2 Compiled Encodings
1014
47bfe92f
JH
1015F<Encode.xs> provides a class C<Encode::XS> which provides the
1016interface described above. It calls a generic octet-sequence to
1017octet-sequence "engine" that is driven by tables (defined in
1018F<encengine.c>). The same engine is used for both encode and
1019decode. C<Encode:XS>'s C<encode> forces Perl's characters to their
1020UTF-8 form and then treats them as just another multibyte
1021encoding. C<Encode:XS>'s C<decode> transforms the sequence and then
1022turns the UTF-8-ness flag as that is the form that the tables are
1023defined to produce. For details of the engine see the comments in
1024F<encengine.c>.
1025
1026The tables are produced by the Perl script F<compile> (the name needs
1027to change so we can eventually install it somewhere). F<compile> can
1028currently read two formats:
4edaa979
NIS
1029
1030=over 4
1031
1032=item *.enc
1033
47bfe92f
JH
1034This is a coined format used by Tcl. It is documented in
1035Encode/EncodeFormat.pod.
4edaa979
NIS
1036
1037=item *.ucm
1038
1039This is the semi-standard format used by IBM's ICU package.
1040
1041=back
1042
1043F<compile> can write the following forms:
1044
1045=over 4
1046
1047=item *.ucm
1048
1049See above - the F<Encode/*.ucm> files provided with the distribution have
1050been created from the original Tcl .enc files using this approach.
1051
1052=item *.c
1053
1054Produces tables as C data structures - this is used to build in encodings
1055into F<Encode.so>/F<Encode.dll>.
1056
1057=item *.xs
1058
47bfe92f
JH
1059In theory this allows encodings to be stand-alone loadable Perl
1060extensions. The process has not yet been tested. The plan is to use
1061this approach for large East Asian encodings.
4edaa979
NIS
1062
1063=back
1064
47bfe92f
JH
1065The set of encodings built-in to F<Encode.so>/F<Encode.dll> is
1066determined by F<Makefile.PL>. The current set is as follows:
4edaa979
NIS
1067
1068=over 4
1069
1070=item ascii and iso-8859-*
1071
1072That is all the common 8-bit "western" encodings.
1073
1074=item IBM-1047 and two other variants of EBCDIC.
1075
47bfe92f
JH
1076These are the same variants that are supported by EBCDIC Perl as
1077"native" encodings. They are included to prove "reversibility" of
1078some constructs in EBCDIC Perl.
4edaa979
NIS
1079
1080=item symbol and dingbats as used by Tk on X11.
1081
47bfe92f 1082(The reason Encode got started was to support Perl/Tk.)
4edaa979
NIS
1083
1084=back
1085
47bfe92f
JH
1086That set is rather ad hoc and has been driven by the needs of the
1087tests rather than the needs of typical applications. It is likely
1088to be rationalized.
4edaa979 1089
4411f3b6
NIS
1090=head1 SEE ALSO
1091
47bfe92f 1092L<perlunicode>, L<perlebcdic>, L<perlfunc/open>, L<PerlIO>
4411f3b6
NIS
1093
1094=cut
1095