This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Encode/IO doc tweaks.
[perl5.git] / ext / Encode / Encode.pm
CommitLineData
2c674647 1package Encode;
51ef4e11 2use strict;
2c674647 3
b8a524e9 4our $VERSION = '0.02';
2c674647
JH
5
6require DynaLoader;
7require Exporter;
8
51ef4e11 9our @ISA = qw(Exporter DynaLoader);
2c674647 10
4411f3b6 11# Public, encouraged API is exported by default
51ef4e11 12our @EXPORT = qw (
4411f3b6
NIS
13 encode
14 decode
15 encode_utf8
16 decode_utf8
17 find_encoding
51ef4e11 18 encodings
4411f3b6
NIS
19);
20
51ef4e11 21our @EXPORT_OK =
2c674647 22 qw(
51ef4e11
NIS
23 define_encoding
24 define_alias
2c674647
JH
25 from_to
26 is_utf8
4411f3b6
NIS
27 is_8bit
28 is_16bit
a12c0f56
NIS
29 utf8_upgrade
30 utf8_downgrade
4411f3b6
NIS
31 _utf8_on
32 _utf8_off
2c674647
JH
33 );
34
35bootstrap Encode ();
36
4411f3b6 37# Documentation moved after __END__ for speed - NI-S
2c674647 38
bf230f3d
NIS
39use Carp;
40
51ef4e11
NIS
41# Make a %encoding package variable to allow a certain amount of cheating
42our %encoding;
43my @alias; # ordered matching list
44my %alias; # cached known aliases
6d6a7c8d
NIS
45 # 0 1 2 3 4 5 6 7 8 9 10
46our @latin2iso_num = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 );
47
5345d506 48
656753f8
NIS
49sub encodings
50{
51 my ($class) = @_;
51ef4e11
NIS
52 return keys %encoding;
53}
54
55sub findAlias
56{
57 my $class = shift;
58 local $_ = shift;
59 unless (exists $alias{$_})
656753f8 60 {
51ef4e11 61 for (my $i=0; $i < @alias; $i += 2)
656753f8 62 {
51ef4e11
NIS
63 my $alias = $alias[$i];
64 my $val = $alias[$i+1];
65 my $new;
66 if (ref($alias) eq 'Regexp' && $_ =~ $alias)
5345d506 67 {
51ef4e11
NIS
68 $new = eval $val;
69 }
70 elsif (ref($alias) eq 'CODE')
71 {
72 $new = &{$alias}($val)
73 }
5ad8ef52 74 elsif (lc($_) eq lc($alias))
51ef4e11
NIS
75 {
76 $new = $val;
77 }
78 if (defined($new))
79 {
80 next if $new eq $_; # avoid (direct) recursion on bugs
81 my $enc = (ref($new)) ? $new : find_encoding($new);
82 if ($enc)
5345d506 83 {
51ef4e11
NIS
84 $alias{$_} = $enc;
85 last;
5345d506
NIS
86 }
87 }
656753f8 88 }
5345d506 89 }
51ef4e11 90 return $alias{$_};
5345d506
NIS
91}
92
51ef4e11 93sub define_alias
5345d506 94{
51ef4e11 95 while (@_)
5345d506 96 {
51ef4e11
NIS
97 my ($alias,$name) = splice(@_,0,2);
98 push(@alias, $alias => $name);
656753f8 99 }
51ef4e11
NIS
100}
101
016cb72c 102# Allow variants of iso-8859-1 etc.
d6089a2a 103define_alias( qr/^iso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' );
016cb72c
NIS
104
105# Allow latin-1 style names as well
016cb72c
NIS
106define_alias( qr/^latin[-_]?(\d+)$/i => '"iso-8859-$latin2iso_num[$1]"' );
107
108# Common names for non-latin prefered MIME names
109define_alias( 'ascii' => 'US-ascii',
110 'cyrillic' => 'iso-8859-5',
111 'arabic' => 'iso-8859-6',
112 'greek' => 'iso-8859-7',
113 'hebrew' => 'iso-8859-8');
114
51ef4e11
NIS
115define_alias( 'ibm-1047' => 'cp1047');
116
016cb72c
NIS
117# Map white space and _ to '-'
118define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' );
119
51ef4e11
NIS
120sub define_encoding
121{
122 my $obj = shift;
123 my $name = shift;
124 $encoding{$name} = $obj;
125 my $lc = lc($name);
126 define_alias($lc => $obj) unless $lc eq $name;
127 while (@_)
656753f8 128 {
51ef4e11
NIS
129 my $alias = shift;
130 define_alias($alias,$obj);
656753f8 131 }
51ef4e11 132 return $obj;
656753f8
NIS
133}
134
656753f8
NIS
135sub getEncoding
136{
137 my ($class,$name) = @_;
5345d506 138 my $enc;
0f43fc90
NIS
139 if (ref($name) && $name->can('new_sequence'))
140 {
141 return $name;
142 }
51ef4e11 143 if (exists $encoding{$name})
656753f8 144 {
51ef4e11
NIS
145 return $encoding{$name};
146 }
147 else
148 {
149 return $class->findAlias($name);
656753f8 150 }
656753f8
NIS
151}
152
4411f3b6
NIS
153sub find_encoding
154{
155 my ($name) = @_;
156 return __PACKAGE__->getEncoding($name);
157}
158
159sub encode
160{
161 my ($name,$string,$check) = @_;
162 my $enc = find_encoding($name);
163 croak("Unknown encoding '$name'") unless defined $enc;
50d26985 164 my $octets = $enc->encode($string,$check);
4411f3b6
NIS
165 return undef if ($check && length($string));
166 return $octets;
167}
168
169sub decode
170{
171 my ($name,$octets,$check) = @_;
172 my $enc = find_encoding($name);
173 croak("Unknown encoding '$name'") unless defined $enc;
50d26985 174 my $string = $enc->decode($octets,$check);
4411f3b6
NIS
175 return undef if ($check && length($octets));
176 return $string;
177}
178
179sub from_to
180{
181 my ($string,$from,$to,$check) = @_;
182 my $f = find_encoding($from);
183 croak("Unknown encoding '$from'") unless defined $f;
184 my $t = find_encoding($to);
185 croak("Unknown encoding '$to'") unless defined $t;
50d26985 186 my $uni = $f->decode($string,$check);
4411f3b6 187 return undef if ($check && length($string));
50d26985 188 $string = $t->encode($uni,$check);
4411f3b6
NIS
189 return undef if ($check && length($uni));
190 return length($_[0] = $string);
191}
192
193sub encode_utf8
194{
195 my ($str) = @_;
1b026014 196 utf8::encode($str);
4411f3b6
NIS
197 return $str;
198}
199
200sub decode_utf8
201{
202 my ($str) = @_;
1b026014 203 return undef unless utf8::decode($str);
4411f3b6
NIS
204 return $str;
205}
206
50d26985
NIS
207package Encode::Encoding;
208# Base class for classes which implement encodings
4edaa979 209
51ef4e11
NIS
210sub Define
211{
212 my $obj = shift;
213 my $canonical = shift;
214 $obj = bless { Name => $canonical },$obj unless ref $obj;
215 # warn "$canonical => $obj\n";
216 Encode::define_encoding($obj, $canonical, @_);
217}
218
219sub name { shift->{'Name'} }
220
50d26985 221# Temporary legacy methods
4edaa979
NIS
222sub toUnicode { shift->decode(@_) }
223sub fromUnicode { shift->encode(@_) }
224
225sub new_sequence { return $_[0] }
50d26985
NIS
226
227package Encode::XS;
228use base 'Encode::Encoding';
229
5ad8ef52 230package Encode::Internal;
50d26985 231use base 'Encode::Encoding';
656753f8 232
9b37254d 233# Dummy package that provides the encode interface but leaves data
1b026014 234# as UTF-X encoded. It is here so that from_to() works.
656753f8 235
5ad8ef52
NIS
236__PACKAGE__->Define('Internal');
237
238Encode::define_alias( 'Unicode' => 'Internal' ) if ord('A') == 65;
656753f8 239
50d26985 240sub decode
a12c0f56
NIS
241{
242 my ($obj,$str,$chk) = @_;
1b026014 243 utf8::upgrade($str);
a12c0f56
NIS
244 $_[1] = '' if $chk;
245 return $str;
246}
656753f8 247
50d26985 248*encode = \&decode;
656753f8 249
5ad8ef52
NIS
250package Encoding::Unicode;
251use base 'Encode::Encoding';
252
253__PACKAGE__->Define('Unicode') unless ord('A') == 65;
254
255sub decode
256{
257 my ($obj,$str,$chk) = @_;
258 my $res = '';
259 for (my $i = 0; $i < length($str); $i++)
260 {
261 $res .= chr(utf8::unicode_to_native(ord(substr($str,$i,1))));
262 }
263 $_[1] = '' if $chk;
264 return $res;
265}
266
267sub encode
268{
269 my ($obj,$str,$chk) = @_;
270 my $res = '';
271 for (my $i = 0; $i < length($str); $i++)
272 {
273 $res .= chr(utf8::native_to_unicode(ord(substr($str,$i,1))));
274 }
275 $_[1] = '' if $chk;
276 return $res;
277}
278
279
4411f3b6 280package Encode::utf8;
50d26985 281use base 'Encode::Encoding';
4411f3b6
NIS
282# package to allow long-hand
283# $octets = encode( utf8 => $string );
284#
285
51ef4e11 286__PACKAGE__->Define(qw(UTF-8 utf8));
4411f3b6 287
50d26985 288sub decode
4411f3b6
NIS
289{
290 my ($obj,$octets,$chk) = @_;
2a936312 291 my $str = Encode::decode_utf8($octets);
4411f3b6
NIS
292 if (defined $str)
293 {
294 $_[1] = '' if $chk;
295 return $str;
296 }
297 return undef;
298}
299
50d26985 300sub encode
4411f3b6
NIS
301{
302 my ($obj,$string,$chk) = @_;
2a936312 303 my $octets = Encode::encode_utf8($string);
4411f3b6
NIS
304 $_[1] = '' if $chk;
305 return $octets;
4411f3b6
NIS
306}
307
9b37254d 308package Encode::iso10646_1;
50d26985 309use base 'Encode::Encoding';
51ef4e11 310# Encoding is 16-bit network order Unicode (no surogates)
9b37254d 311# Used for X font encodings
87714904 312
8040349a 313__PACKAGE__->Define(qw(UCS-2 iso-10646-1));
87714904 314
50d26985 315sub decode
87714904
NIS
316{
317 my ($obj,$str,$chk) = @_;
318 my $uni = '';
319 while (length($str))
320 {
5dcbab34 321 my $code = unpack('n',substr($str,0,2,'')) & 0xffff;
87714904
NIS
322 $uni .= chr($code);
323 }
324 $_[1] = $str if $chk;
8040349a 325 utf8::upgrade($uni);
87714904
NIS
326 return $uni;
327}
328
50d26985 329sub encode
87714904
NIS
330{
331 my ($obj,$uni,$chk) = @_;
332 my $str = '';
333 while (length($uni))
334 {
335 my $ch = substr($uni,0,1,'');
336 my $x = ord($ch);
337 unless ($x < 32768)
338 {
339 last if ($chk);
340 $x = 0;
341 }
5dcbab34 342 $str .= pack('n',$x);
656753f8 343 }
bf230f3d 344 $_[1] = $uni if $chk;
656753f8
NIS
345 return $str;
346}
347
4411f3b6
NIS
348# switch back to Encode package in case we ever add AutoLoader
349package Encode;
350
656753f8
NIS
3511;
352
2a936312
NIS
353__END__
354
4411f3b6
NIS
355=head1 NAME
356
357Encode - character encodings
358
359=head1 SYNOPSIS
360
361 use Encode;
362
363=head1 DESCRIPTION
364
47bfe92f
JH
365The C<Encode> module provides the interfaces between Perl's strings
366and the rest of the system. Perl strings are sequences of B<characters>.
4411f3b6
NIS
367
368The repertoire of characters that Perl can represent is at least that
47bfe92f
JH
369defined by the Unicode Consortium. On most platforms the ordinal
370values of the characters (as returned by C<ord(ch)>) is the "Unicode
371codepoint" for the character (the exceptions are those platforms where
372the legacy encoding is some variant of EBCDIC rather than a super-set
373of ASCII - see L<perlebcdic>).
4411f3b6
NIS
374
375Traditionaly computer data has been moved around in 8-bit chunks
376often called "bytes". These chunks are also known as "octets" in
377networking standards. Perl is widely used to manipulate data of
378many types - not only strings of characters representing human or
379computer languages but also "binary" data being the machines representation
380of numbers, pixels in an image - or just about anything.
381
47bfe92f
JH
382When Perl is processing "binary data" the programmer wants Perl to process
383"sequences of bytes". This is not a problem for Perl - as a byte has 256
384possible values it easily fits in Perl's much larger "logical character".
4411f3b6
NIS
385
386=head2 TERMINOLOGY
387
4ac9195f 388=over 4
4411f3b6
NIS
389
390=item *
391
392I<character>: a character in the range 0..(2**32-1) (or more).
47bfe92f 393(What Perl's strings are made of.)
4411f3b6
NIS
394
395=item *
396
397I<byte>: a character in the range 0..255
47bfe92f 398(A special case of a Perl character.)
4411f3b6
NIS
399
400=item *
401
402I<octet>: 8 bits of data, with ordinal values 0..255
47bfe92f 403(Term for bytes passed to or from a non-Perl context, e.g. disk file.)
4411f3b6
NIS
404
405=back
406
407The marker [INTERNAL] marks Internal Implementation Details, in
408general meant only for those who think they know what they are doing,
409and such details may change in future releases.
410
411=head1 ENCODINGS
412
413=head2 Characteristics of an Encoding
414
415An encoding has a "repertoire" of characters that it can represent,
416and for each representable character there is at least one sequence of
417octets that represents it.
418
419=head2 Types of Encodings
420
421Encodings can be divided into the following types:
422
423=over 4
424
425=item * Fixed length 8-bit (or less) encodings.
426
427Each character is a single octet so may have a repertoire of up to
428256 characters. ASCII and iso-8859-* are typical examples.
429
430=item * Fixed length 16-bit encodings
431
432Each character is two octets so may have a repertoire of up to
47bfe92f 43365 536 characters. Unicode's UCS-2 is an example. Also used for
4411f3b6
NIS
434encodings for East Asian languages.
435
436=item * Fixed length 32-bit encodings.
437
438Not really very "encoded" encodings. The Unicode code points
439are just represented as 4-octet integers. None the less because
440different architectures use different representations of integers
441(so called "endian") there at least two disctinct encodings.
442
443=item * Multi-byte encodings
444
445The number of octets needed to represent a character varies.
446UTF-8 is a particularly complex but regular case of a multi-byte
447encoding. Several East Asian countries use a multi-byte encoding
448where 1-octet is used to cover western roman characters and Asian
449characters get 2-octets.
450(UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets
451to represent a Unicode code point.)
452
453=item * "Escape" encodings.
454
455These encodings embed "escape sequences" into the octet sequence
456which describe how the following octets are to be interpreted.
457The iso-2022-* family is typical. Following the escape sequence
458octets are encoded by an "embedded" encoding (which will be one
459of the above types) until another escape sequence switches to
460a different "embedded" encoding.
461
462These schemes are very flexible and can handle mixed languages but are
47bfe92f
JH
463very complex to process (and have state). No escape encodings are
464implemented for Perl yet.
4411f3b6
NIS
465
466=back
467
468=head2 Specifying Encodings
469
470Encodings can be specified to the API described below in two ways:
471
472=over 4
473
474=item 1. By name
475
47bfe92f
JH
476Encoding names are strings with characters taken from a restricted
477repertoire. See L</"Encoding Names">.
4411f3b6
NIS
478
479=item 2. As an object
480
481Encoding objects are returned by C<find_encoding($name)>.
482
483=back
484
485=head2 Encoding Names
486
487Encoding names are case insensitive. White space in names is ignored.
47bfe92f
JH
488In addition an encoding may have aliases. Each encoding has one
489"canonical" name. The "canonical" name is chosen from the names of
490the encoding by picking the first in the following sequence:
4411f3b6
NIS
491
492=over 4
493
494=item * The MIME name as defined in IETF RFC-XXXX.
495
496=item * The name in the IANA registry.
497
498=item * The name used by the the organization that defined it.
499
500=back
501
502Because of all the alias issues, and because in the general case
503encodings have state C<Encode> uses the encoding object internally
504once an operation is in progress.
505
4411f3b6
NIS
506=head1 PERL ENCODING API
507
508=head2 Generic Encoding Interface
509
510=over 4
511
512=item *
513
514 $bytes = encode(ENCODING, $string[, CHECK])
515
47bfe92f
JH
516Encodes string from Perl's internal form into I<ENCODING> and returns
517a sequence of octets. For CHECK see L</"Handling Malformed Data">.
4411f3b6
NIS
518
519=item *
520
521 $string = decode(ENCODING, $bytes[, CHECK])
522
47bfe92f
JH
523Decode sequence of octets assumed to be in I<ENCODING> into Perl's
524internal form and returns the resulting string. For CHECK see
525L</"Handling Malformed Data">.
526
527=item *
528
529 from_to($string, FROM_ENCODING, TO_ENCODING[, CHECK])
530
2b106fbe
JH
531Convert B<in-place> the data between two encodings. How did the data
532in $string originally get to be in FROM_ENCODING? Either using
e9692b5b 533encode() or through PerlIO: See L</"Encoding and IO">. For CHECK
2b106fbe
JH
534see L</"Handling Malformed Data">.
535
536For example to convert ISO 8859-1 data to UTF-8:
537
538 from_to($data, "iso-8859-1", "utf-8");
539
540and to convert it back:
541
542 from_to($data, "utf-8", "iso-8859-1");
4411f3b6 543
ab97ca19
JH
544Note that because the conversion happens in place, the data to be
545converted cannot be a string constant, it must be a scalar variable.
546
4411f3b6
NIS
547=back
548
549=head2 Handling Malformed Data
550
551If CHECK is not set, C<undef> is returned. If the data is supposed to
47bfe92f
JH
552be UTF-8, an optional lexical warning (category utf8) is given. If
553CHECK is true but not a code reference, dies.
4411f3b6 554
47bfe92f
JH
555It would desirable to have a way to indicate that transform should use
556the encodings "replacement character" - no such mechanism is defined yet.
4411f3b6
NIS
557
558It is also planned to allow I<CHECK> to be a code reference.
559
47bfe92f
JH
560This is not yet implemented as there are design issues with what its
561arguments should be and how it returns its results.
4411f3b6
NIS
562
563=over 4
564
565=item Scheme 1
566
567Passed remaining fragment of string being processed.
568Modifies it in place to remove bytes/characters it can understand
569and returns a string used to represent them.
570e.g.
571
572 sub fixup {
573 my $ch = substr($_[0],0,1,'');
574 return sprintf("\x{%02X}",ord($ch);
575 }
576
577This scheme is close to how underlying C code for Encode works, but gives
578the fixup routine very little context.
579
580=item Scheme 2
581
47bfe92f
JH
582Passed original string, and an index into it of the problem area, and
583output string so far. Appends what it will to output string and
584returns new index into original string. For example:
4411f3b6
NIS
585
586 sub fixup {
587 # my ($s,$i,$d) = @_;
588 my $ch = substr($_[0],$_[1],1);
589 $_[2] .= sprintf("\x{%02X}",ord($ch);
590 return $_[1]+1;
591 }
592
47bfe92f
JH
593This scheme gives maximal control to the fixup routine but is more
594complicated to code, and may need internals of Encode to be tweaked to
595keep original string intact.
4411f3b6
NIS
596
597=item Other Schemes
598
599Hybrids of above.
600
601Multiple return values rather than in-place modifications.
602
603Index into the string could be pos($str) allowing s/\G...//.
604
605=back
606
607=head2 UTF-8 / utf8
608
609The Unicode consortium defines the UTF-8 standard as a way of encoding
47bfe92f
JH
610the entire Unicode repertiore as sequences of octets. This encoding is
611expected to become very widespread. Perl can use this form internaly
612to represent strings, so conversions to and from this form are
613particularly efficient (as octets in memory do not have to change,
614just the meta-data that tells Perl how to treat them).
4411f3b6
NIS
615
616=over 4
617
618=item *
619
620 $bytes = encode_utf8($string);
621
47bfe92f 622The characters that comprise string are encoded in Perl's superset of UTF-8
4411f3b6
NIS
623and the resulting octets returned as a sequence of bytes. All possible
624characters have a UTF-8 representation so this function cannot fail.
625
626=item *
627
628 $string = decode_utf8($bytes [,CHECK]);
629
47bfe92f
JH
630The sequence of octets represented by $bytes is decoded from UTF-8
631into a sequence of logical characters. Not all sequences of octets
632form valid UTF-8 encodings, so it is possible for this call to fail.
633For CHECK see L</"Handling Malformed Data">.
4411f3b6
NIS
634
635=back
636
637=head2 Other Encodings of Unicode
638
47bfe92f
JH
639UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. UCS-2 can only
640represent 0..0xFFFF, while UTF-16 has a "surrogate pair" scheme which
641allows it to cover the whole Unicode range.
4411f3b6 642
8040349a 643Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that
47bfe92f
JH
644happens to be the name used by that representation when used with X11
645fonts.
4411f3b6
NIS
646
647UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters
648can be considered as being in this form without encoding. An encoding
47bfe92f
JH
649to transfer strings in this form (e.g. to write them to a file) would
650need to
4411f3b6
NIS
651
652 pack('L',map(chr($_),split(//,$string))); # native
653 or
654 pack('V',map(chr($_),split(//,$string))); # little-endian
655 or
656 pack('N',map(chr($_),split(//,$string))); # big-endian
657
658depending on the endian required.
659
51ef4e11 660No UTF-32 encodings are implemented yet.
4411f3b6 661
47bfe92f
JH
662Both UCS-2 and UCS-4 style encodings can have "byte order marks" by
663representing the code point 0xFFFE as the very first thing in a file.
4411f3b6 664
51ef4e11
NIS
665=head2 Listing available encodings
666
667 use Encode qw(encodings);
668 @list = encodings();
669
670Returns a list of the canonical names of the available encodings.
671
672=head2 Defining Aliases
673
674 use Encode qw(define_alias);
675 define_alias( newName => ENCODING);
676
47bfe92f
JH
677Allows newName to be used as am alias for ENCODING. ENCODING may be
678either the name of an encoding or and encoding object (as above).
51ef4e11
NIS
679
680Currently I<newName> can be specified in the following ways:
681
682=over 4
683
684=item As a simple string.
685
686=item As a qr// compiled regular expression, e.g.:
687
688 define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' );
689
47bfe92f
JH
690In this case if I<ENCODING> is not a reference it is C<eval>-ed to
691allow C<$1> etc. to be subsituted. The example is one way to names as
692used in X11 font names to alias the MIME names for the iso-8859-*
693family.
51ef4e11
NIS
694
695=item As a code reference, e.g.:
696
697 define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , '');
698
699In this case C<$_> will be set to the name that is being looked up and
47bfe92f
JH
700I<ENCODING> is passed to the sub as its first argument. The example
701is another way to names as used in X11 font names to alias the MIME
702names for the iso-8859-* family.
51ef4e11
NIS
703
704=back
705
706=head2 Defining Encodings
707
e9692b5b
JH
708 use Encode qw(define_alias);
709 define_encoding( $object, 'canonicalName' [,alias...]);
51ef4e11 710
47bfe92f
JH
711Causes I<canonicalName> to be associated with I<$object>. The object
712should provide the interface described in L</"IMPLEMENTATION CLASSES">
713below. If more than two arguments are provided then additional
714arguments are taken as aliases for I<$object> as for C<define_alias>.
51ef4e11 715
4411f3b6
NIS
716=head1 Encoding and IO
717
718It is very common to want to do encoding transformations when
719reading or writing files, network connections, pipes etc.
47bfe92f 720If Perl is configured to use the new 'perlio' IO system then
4411f3b6
NIS
721C<Encode> provides a "layer" (See L<perliol>) which can transform
722data as it is read or written.
723
e9692b5b
JH
724 open(my $ilyad,'>:encoding(iso-8859-7)','ilyad.greek');
725 print $ilyad @epic;
4411f3b6
NIS
726
727In addition the new IO system can also be configured to read/write
728UTF-8 encoded characters (as noted above this is efficient):
729
e9692b5b
JH
730 open(my $fh,'>:utf8','anything');
731 print $fh "Any \x{0021} string \N{SMILEY FACE}\n";
4411f3b6
NIS
732
733Either of the above forms of "layer" specifications can be made the default
734for a lexical scope with the C<use open ...> pragma. See L<open>.
735
736Once a handle is open is layers can be altered using C<binmode>.
737
47bfe92f 738Without any such configuration, or if Perl itself is built using
4411f3b6
NIS
739system's own IO, then write operations assume that file handle accepts
740only I<bytes> and will C<die> if a character larger than 255 is
741written to the handle. When reading, each octet from the handle
742becomes a byte-in-a-character. Note that this default is the same
47bfe92f
JH
743behaviour as bytes-only languages (including Perl before v5.6) would
744have, and is sufficient to handle native 8-bit encodings
745e.g. iso-8859-1, EBCDIC etc. and any legacy mechanisms for handling
746other encodings and binary data.
747
748In other cases it is the programs responsibility to transform
749characters into bytes using the API above before doing writes, and to
750transform the bytes read from a handle into characters before doing
751"character operations" (e.g. C<lc>, C</\W+/>, ...).
752
47bfe92f
JH
753You can also use PerlIO to convert larger amounts of data you don't
754want to bring into memory. For example to convert between ISO 8859-1
755(Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines):
756
e9692b5b
JH
757 open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!;
758 open(G, ">:utf8", "data.utf") or die $!;
759 while (<F>) { print G }
760
761 # Could also do "print G <F>" but that would pull
762 # the whole file into memory just to write it out again.
763
764More examples:
47bfe92f 765
e9692b5b
JH
766 open(my $f, "<:encoding(cp1252)")
767 open(my $g, ">:encoding(iso-8859-2)")
768 open(my $h, ">:encoding(latin9)") # iso-8859-15
47bfe92f
JH
769
770See L<PerlIO> for more information.
4411f3b6
NIS
771
772=head1 Encoding How to ...
773
774To do:
775
776=over 4
777
778=item * IO with mixed content (faking iso-2020-*)
779
780=item * MIME's Content-Length:
781
782=item * UTF-8 strings in binary data.
783
47bfe92f 784=item * Perl/Encode wrappers on non-Unicode XS modules.
4411f3b6
NIS
785
786=back
787
788=head1 Messing with Perl's Internals
789
47bfe92f
JH
790The following API uses parts of Perl's internals in the current
791implementation. As such they are efficient, but may change.
4411f3b6
NIS
792
793=over 4
794
4411f3b6
NIS
795=item * is_utf8(STRING [, CHECK])
796
797[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING.
47bfe92f
JH
798If CHECK is true, also checks the data in STRING for being well-formed
799UTF-8. Returns true if successful, false otherwise.
4411f3b6
NIS
800
801=item * valid_utf8(STRING)
802
47bfe92f
JH
803[INTERNAL] Test whether STRING is in a consistent state. Will return
804true if string is held as bytes, or is well-formed UTF-8 and has the
805UTF-8 flag on. Main reason for this routine is to allow Perl's
806testsuite to check that operations have left strings in a consistent
807state.
4411f3b6
NIS
808
809=item *
810
811 _utf8_on(STRING)
812
813[INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is
814B<not> checked for being well-formed UTF-8. Do not use unless you
815B<know> that the STRING is well-formed UTF-8. Returns the previous
816state of the UTF-8 flag (so please don't test the return value as
817I<not> success or failure), or C<undef> if STRING is not a string.
818
819=item *
820
821 _utf8_off(STRING)
822
823[INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously.
824Returns the previous state of the UTF-8 flag (so please don't test the
825return value as I<not> success or failure), or C<undef> if STRING is
826not a string.
827
828=back
829
4edaa979
NIS
830=head1 IMPLEMENTATION CLASSES
831
832As mentioned above encodings are (in the current implementation at least)
833defined by objects. The mapping of encoding name to object is via the
51ef4e11 834C<%encodings> hash.
4edaa979
NIS
835
836The values of the hash can currently be either strings or objects.
837The string form may go away in the future. The string form occurs
838when C<encodings()> has scanned C<@INC> for loadable encodings but has
839not actually loaded the encoding in question. This is because the
47bfe92f 840current "loading" process is all Perl and a bit slow.
4edaa979 841
47bfe92f
JH
842Once an encoding is loaded then value of the hash is object which
843implements the encoding. The object should provide the following
844interface:
4edaa979
NIS
845
846=over 4
847
848=item -E<gt>name
849
850Should return the string representing the canonical name of the encoding.
851
852=item -E<gt>new_sequence
853
47bfe92f
JH
854This is a placeholder for encodings with state. It should return an
855object which implements this interface, all current implementations
856return the original object.
4edaa979
NIS
857
858=item -E<gt>encode($string,$check)
859
47bfe92f
JH
860Should return the octet sequence representing I<$string>. If I<$check>
861is true it should modify I<$string> in place to remove the converted
862part (i.e. the whole string unless there is an error). If an error
863occurs it should return the octet sequence for the fragment of string
864that has been converted, and modify $string in-place to remove the
865converted part leaving it starting with the problem fragment.
4edaa979 866
47bfe92f
JH
867If check is is false then C<encode> should make a "best effort" to
868convert the string - for example by using a replacement character.
4edaa979
NIS
869
870=item -E<gt>decode($octets,$check)
871
47bfe92f
JH
872Should return the string that I<$octets> represents. If I<$check> is
873true it should modify I<$octets> in place to remove the converted part
874(i.e. the whole sequence unless there is an error). If an error
875occurs it should return the fragment of string that has been
876converted, and modify $octets in-place to remove the converted part
4edaa979
NIS
877leaving it starting with the problem fragment.
878
47bfe92f
JH
879If check is is false then C<decode> should make a "best effort" to
880convert the string - for example by using Unicode's "\x{FFFD}" as a
881replacement character.
4edaa979
NIS
882
883=back
884
47bfe92f
JH
885It should be noted that the check behaviour is different from the
886outer public API. The logic is that the "unchecked" case is useful
887when encoding is part of a stream which may be reporting errors
888(e.g. STDERR). In such cases it is desirable to get everything
889through somehow without causing additional errors which obscure the
890original one. Also the encoding is best placed to know what the
891correct replacement character is, so if that is the desired behaviour
892then letting low level code do it is the most efficient.
893
894In contrast if check is true, the scheme above allows the encoding to
895do as much as it can and tell layer above how much that was. What is
896lacking at present is a mechanism to report what went wrong. The most
897likely interface will be an additional method call to the object, or
898perhaps (to avoid forcing per-stream objects on otherwise stateless
899encodings) and additional parameter.
900
901It is also highly desirable that encoding classes inherit from
902C<Encode::Encoding> as a base class. This allows that class to define
903additional behaviour for all encoding objects. For example built in
904Unicode, UCS-2 and UTF-8 classes use :
51ef4e11
NIS
905
906 package Encode::MyEncoding;
907 use base qw(Encode::Encoding);
908
909 __PACKAGE__->Define(qw(myCanonical myAlias));
910
47bfe92f
JH
911To create an object with bless {Name => ...},$class, and call
912define_encoding. They inherit their C<name> method from
913C<Encode::Encoding>.
4edaa979
NIS
914
915=head2 Compiled Encodings
916
47bfe92f
JH
917F<Encode.xs> provides a class C<Encode::XS> which provides the
918interface described above. It calls a generic octet-sequence to
919octet-sequence "engine" that is driven by tables (defined in
920F<encengine.c>). The same engine is used for both encode and
921decode. C<Encode:XS>'s C<encode> forces Perl's characters to their
922UTF-8 form and then treats them as just another multibyte
923encoding. C<Encode:XS>'s C<decode> transforms the sequence and then
924turns the UTF-8-ness flag as that is the form that the tables are
925defined to produce. For details of the engine see the comments in
926F<encengine.c>.
927
928The tables are produced by the Perl script F<compile> (the name needs
929to change so we can eventually install it somewhere). F<compile> can
930currently read two formats:
4edaa979
NIS
931
932=over 4
933
934=item *.enc
935
47bfe92f
JH
936This is a coined format used by Tcl. It is documented in
937Encode/EncodeFormat.pod.
4edaa979
NIS
938
939=item *.ucm
940
941This is the semi-standard format used by IBM's ICU package.
942
943=back
944
945F<compile> can write the following forms:
946
947=over 4
948
949=item *.ucm
950
951See above - the F<Encode/*.ucm> files provided with the distribution have
952been created from the original Tcl .enc files using this approach.
953
954=item *.c
955
956Produces tables as C data structures - this is used to build in encodings
957into F<Encode.so>/F<Encode.dll>.
958
959=item *.xs
960
47bfe92f
JH
961In theory this allows encodings to be stand-alone loadable Perl
962extensions. The process has not yet been tested. The plan is to use
963this approach for large East Asian encodings.
4edaa979
NIS
964
965=back
966
47bfe92f
JH
967The set of encodings built-in to F<Encode.so>/F<Encode.dll> is
968determined by F<Makefile.PL>. The current set is as follows:
4edaa979
NIS
969
970=over 4
971
972=item ascii and iso-8859-*
973
974That is all the common 8-bit "western" encodings.
975
976=item IBM-1047 and two other variants of EBCDIC.
977
47bfe92f
JH
978These are the same variants that are supported by EBCDIC Perl as
979"native" encodings. They are included to prove "reversibility" of
980some constructs in EBCDIC Perl.
4edaa979
NIS
981
982=item symbol and dingbats as used by Tk on X11.
983
47bfe92f 984(The reason Encode got started was to support Perl/Tk.)
4edaa979
NIS
985
986=back
987
47bfe92f
JH
988That set is rather ad hoc and has been driven by the needs of the
989tests rather than the needs of typical applications. It is likely
990to be rationalized.
4edaa979 991
4411f3b6
NIS
992=head1 SEE ALSO
993
47bfe92f 994L<perlunicode>, L<perlebcdic>, L<perlfunc/open>, L<PerlIO>
4411f3b6
NIS
995
996=cut
997