This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Integrate mainline.
[perl5.git] / ext / Encode / Encode.pm
CommitLineData
2c674647 1package Encode;
51ef4e11 2use strict;
2c674647 3
51ef4e11 4our $VERSION = 0.02;
2c674647
JH
5
6require DynaLoader;
7require Exporter;
8
51ef4e11 9our @ISA = qw(Exporter DynaLoader);
2c674647 10
4411f3b6 11# Public, encouraged API is exported by default
51ef4e11 12our @EXPORT = qw (
4411f3b6
NIS
13 encode
14 decode
15 encode_utf8
16 decode_utf8
17 find_encoding
51ef4e11 18 encodings
4411f3b6
NIS
19);
20
51ef4e11 21our @EXPORT_OK =
2c674647 22 qw(
51ef4e11
NIS
23 define_encoding
24 define_alias
2c674647
JH
25 from_to
26 is_utf8
4411f3b6
NIS
27 is_8bit
28 is_16bit
a12c0f56
NIS
29 utf8_upgrade
30 utf8_downgrade
4411f3b6
NIS
31 _utf8_on
32 _utf8_off
2c674647
JH
33 );
34
35bootstrap Encode ();
36
4411f3b6 37# Documentation moved after __END__ for speed - NI-S
2c674647 38
bf230f3d
NIS
39use Carp;
40
51ef4e11
NIS
41# Make a %encoding package variable to allow a certain amount of cheating
42our %encoding;
43my @alias; # ordered matching list
44my %alias; # cached known aliases
5345d506 45
656753f8
NIS
46sub encodings
47{
48 my ($class) = @_;
51ef4e11
NIS
49 return keys %encoding;
50}
51
52sub findAlias
53{
54 my $class = shift;
55 local $_ = shift;
56 unless (exists $alias{$_})
656753f8 57 {
51ef4e11 58 for (my $i=0; $i < @alias; $i += 2)
656753f8 59 {
51ef4e11
NIS
60 my $alias = $alias[$i];
61 my $val = $alias[$i+1];
62 my $new;
63 if (ref($alias) eq 'Regexp' && $_ =~ $alias)
5345d506 64 {
51ef4e11
NIS
65 $new = eval $val;
66 }
67 elsif (ref($alias) eq 'CODE')
68 {
69 $new = &{$alias}($val)
70 }
71 elsif (lc($_) eq $alias)
72 {
73 $new = $val;
74 }
75 if (defined($new))
76 {
77 next if $new eq $_; # avoid (direct) recursion on bugs
78 my $enc = (ref($new)) ? $new : find_encoding($new);
79 if ($enc)
5345d506 80 {
51ef4e11
NIS
81 $alias{$_} = $enc;
82 last;
5345d506
NIS
83 }
84 }
656753f8 85 }
5345d506 86 }
51ef4e11 87 return $alias{$_};
5345d506
NIS
88}
89
51ef4e11 90sub define_alias
5345d506 91{
51ef4e11 92 while (@_)
5345d506 93 {
51ef4e11
NIS
94 my ($alias,$name) = splice(@_,0,2);
95 push(@alias, $alias => $name);
656753f8 96 }
51ef4e11
NIS
97}
98
016cb72c 99# Allow variants of iso-8859-1 etc.
d6089a2a 100define_alias( qr/^iso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' );
016cb72c
NIS
101
102# Allow latin-1 style names as well
103 # 0 1 2 3 4 5 6 7 8 9 10
104my @latin2iso_num = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 );
105define_alias( qr/^latin[-_]?(\d+)$/i => '"iso-8859-$latin2iso_num[$1]"' );
106
107# Common names for non-latin prefered MIME names
108define_alias( 'ascii' => 'US-ascii',
109 'cyrillic' => 'iso-8859-5',
110 'arabic' => 'iso-8859-6',
111 'greek' => 'iso-8859-7',
112 'hebrew' => 'iso-8859-8');
113
51ef4e11
NIS
114define_alias( 'ibm-1047' => 'cp1047');
115
016cb72c
NIS
116# Map white space and _ to '-'
117define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' );
118
51ef4e11
NIS
119sub define_encoding
120{
121 my $obj = shift;
122 my $name = shift;
123 $encoding{$name} = $obj;
124 my $lc = lc($name);
125 define_alias($lc => $obj) unless $lc eq $name;
126 while (@_)
656753f8 127 {
51ef4e11
NIS
128 my $alias = shift;
129 define_alias($alias,$obj);
656753f8 130 }
51ef4e11 131 return $obj;
656753f8
NIS
132}
133
656753f8
NIS
134sub getEncoding
135{
136 my ($class,$name) = @_;
5345d506 137 my $enc;
51ef4e11 138 if (exists $encoding{$name})
656753f8 139 {
51ef4e11
NIS
140 return $encoding{$name};
141 }
142 else
143 {
144 return $class->findAlias($name);
656753f8 145 }
656753f8
NIS
146}
147
4411f3b6
NIS
148sub find_encoding
149{
150 my ($name) = @_;
151 return __PACKAGE__->getEncoding($name);
152}
153
154sub encode
155{
156 my ($name,$string,$check) = @_;
157 my $enc = find_encoding($name);
158 croak("Unknown encoding '$name'") unless defined $enc;
50d26985 159 my $octets = $enc->encode($string,$check);
4411f3b6
NIS
160 return undef if ($check && length($string));
161 return $octets;
162}
163
164sub decode
165{
166 my ($name,$octets,$check) = @_;
167 my $enc = find_encoding($name);
168 croak("Unknown encoding '$name'") unless defined $enc;
50d26985 169 my $string = $enc->decode($octets,$check);
4411f3b6
NIS
170 return undef if ($check && length($octets));
171 return $string;
172}
173
174sub from_to
175{
176 my ($string,$from,$to,$check) = @_;
177 my $f = find_encoding($from);
178 croak("Unknown encoding '$from'") unless defined $f;
179 my $t = find_encoding($to);
180 croak("Unknown encoding '$to'") unless defined $t;
50d26985 181 my $uni = $f->decode($string,$check);
4411f3b6 182 return undef if ($check && length($string));
50d26985 183 $string = $t->encode($uni,$check);
4411f3b6
NIS
184 return undef if ($check && length($uni));
185 return length($_[0] = $string);
186}
187
188sub encode_utf8
189{
190 my ($str) = @_;
1b026014 191 utf8::encode($str);
4411f3b6
NIS
192 return $str;
193}
194
195sub decode_utf8
196{
197 my ($str) = @_;
1b026014 198 return undef unless utf8::decode($str);
4411f3b6
NIS
199 return $str;
200}
201
50d26985
NIS
202package Encode::Encoding;
203# Base class for classes which implement encodings
4edaa979 204
51ef4e11
NIS
205sub Define
206{
207 my $obj = shift;
208 my $canonical = shift;
209 $obj = bless { Name => $canonical },$obj unless ref $obj;
210 # warn "$canonical => $obj\n";
211 Encode::define_encoding($obj, $canonical, @_);
212}
213
214sub name { shift->{'Name'} }
215
50d26985 216# Temporary legacy methods
4edaa979
NIS
217sub toUnicode { shift->decode(@_) }
218sub fromUnicode { shift->encode(@_) }
219
220sub new_sequence { return $_[0] }
50d26985
NIS
221
222package Encode::XS;
223use base 'Encode::Encoding';
224
656753f8 225package Encode::Unicode;
50d26985 226use base 'Encode::Encoding';
656753f8 227
9b37254d 228# Dummy package that provides the encode interface but leaves data
1b026014 229# as UTF-X encoded. It is here so that from_to() works.
656753f8 230
51ef4e11 231__PACKAGE__->Define('Unicode');
656753f8 232
50d26985 233sub decode
a12c0f56
NIS
234{
235 my ($obj,$str,$chk) = @_;
1b026014 236 utf8::upgrade($str);
a12c0f56
NIS
237 $_[1] = '' if $chk;
238 return $str;
239}
656753f8 240
50d26985 241*encode = \&decode;
656753f8 242
4411f3b6 243package Encode::utf8;
50d26985 244use base 'Encode::Encoding';
4411f3b6
NIS
245# package to allow long-hand
246# $octets = encode( utf8 => $string );
247#
248
51ef4e11 249__PACKAGE__->Define(qw(UTF-8 utf8));
4411f3b6 250
50d26985 251sub decode
4411f3b6
NIS
252{
253 my ($obj,$octets,$chk) = @_;
2a936312 254 my $str = Encode::decode_utf8($octets);
4411f3b6
NIS
255 if (defined $str)
256 {
257 $_[1] = '' if $chk;
258 return $str;
259 }
260 return undef;
261}
262
50d26985 263sub encode
4411f3b6
NIS
264{
265 my ($obj,$string,$chk) = @_;
2a936312 266 my $octets = Encode::encode_utf8($string);
4411f3b6
NIS
267 $_[1] = '' if $chk;
268 return $octets;
4411f3b6
NIS
269}
270
9b37254d 271package Encode::iso10646_1;
50d26985 272use base 'Encode::Encoding';
51ef4e11 273# Encoding is 16-bit network order Unicode (no surogates)
9b37254d 274# Used for X font encodings
87714904 275
8040349a 276__PACKAGE__->Define(qw(UCS-2 iso-10646-1));
87714904 277
50d26985 278sub decode
87714904
NIS
279{
280 my ($obj,$str,$chk) = @_;
281 my $uni = '';
282 while (length($str))
283 {
5dcbab34 284 my $code = unpack('n',substr($str,0,2,'')) & 0xffff;
87714904
NIS
285 $uni .= chr($code);
286 }
287 $_[1] = $str if $chk;
8040349a 288 utf8::upgrade($uni);
87714904
NIS
289 return $uni;
290}
291
50d26985 292sub encode
87714904
NIS
293{
294 my ($obj,$uni,$chk) = @_;
295 my $str = '';
296 while (length($uni))
297 {
298 my $ch = substr($uni,0,1,'');
299 my $x = ord($ch);
300 unless ($x < 32768)
301 {
302 last if ($chk);
303 $x = 0;
304 }
5dcbab34 305 $str .= pack('n',$x);
656753f8 306 }
bf230f3d 307 $_[1] = $uni if $chk;
656753f8
NIS
308 return $str;
309}
310
4411f3b6
NIS
311# switch back to Encode package in case we ever add AutoLoader
312package Encode;
313
656753f8
NIS
3141;
315
2a936312
NIS
316__END__
317
4411f3b6
NIS
318=head1 NAME
319
320Encode - character encodings
321
322=head1 SYNOPSIS
323
324 use Encode;
325
326=head1 DESCRIPTION
327
328The C<Encode> module provides the interfaces between perl's strings
329and the rest of the system. Perl strings are sequences of B<characters>.
330
331The repertoire of characters that Perl can represent is at least that
332defined by the Unicode Consortium. On most platforms the ordinal values
333of the characters (as returned by C<ord(ch)>) is the "Unicode codepoint" for
334the character (the exceptions are those platforms where the legacy
335encoding is some variant of EBCDIC rather than a super-set of ASCII
336- see L<perlebcdic>).
337
338Traditionaly computer data has been moved around in 8-bit chunks
339often called "bytes". These chunks are also known as "octets" in
340networking standards. Perl is widely used to manipulate data of
341many types - not only strings of characters representing human or
342computer languages but also "binary" data being the machines representation
343of numbers, pixels in an image - or just about anything.
344
345When perl is processing "binary data" the programmer wants perl to process
346"sequences of bytes". This is not a problem for perl - as a byte has 256
347possible values it easily fits in perl's much larger "logical character".
348
349=head2 TERMINOLOGY
350
4ac9195f 351=over 4
4411f3b6
NIS
352
353=item *
354
355I<character>: a character in the range 0..(2**32-1) (or more).
356(What perl's strings are made of.)
357
358=item *
359
360I<byte>: a character in the range 0..255
361(A special case of a perl character.)
362
363=item *
364
365I<octet>: 8 bits of data, with ordinal values 0..255
366(Term for bytes passed to or from a non-perl context, e.g. disk file.)
367
368=back
369
370The marker [INTERNAL] marks Internal Implementation Details, in
371general meant only for those who think they know what they are doing,
372and such details may change in future releases.
373
374=head1 ENCODINGS
375
376=head2 Characteristics of an Encoding
377
378An encoding has a "repertoire" of characters that it can represent,
379and for each representable character there is at least one sequence of
380octets that represents it.
381
382=head2 Types of Encodings
383
384Encodings can be divided into the following types:
385
386=over 4
387
388=item * Fixed length 8-bit (or less) encodings.
389
390Each character is a single octet so may have a repertoire of up to
391256 characters. ASCII and iso-8859-* are typical examples.
392
393=item * Fixed length 16-bit encodings
394
395Each character is two octets so may have a repertoire of up to
39665,536 characters. Unicode's UCS-2 is an example. Also used for
397encodings for East Asian languages.
398
399=item * Fixed length 32-bit encodings.
400
401Not really very "encoded" encodings. The Unicode code points
402are just represented as 4-octet integers. None the less because
403different architectures use different representations of integers
404(so called "endian") there at least two disctinct encodings.
405
406=item * Multi-byte encodings
407
408The number of octets needed to represent a character varies.
409UTF-8 is a particularly complex but regular case of a multi-byte
410encoding. Several East Asian countries use a multi-byte encoding
411where 1-octet is used to cover western roman characters and Asian
412characters get 2-octets.
413(UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets
414to represent a Unicode code point.)
415
416=item * "Escape" encodings.
417
418These encodings embed "escape sequences" into the octet sequence
419which describe how the following octets are to be interpreted.
420The iso-2022-* family is typical. Following the escape sequence
421octets are encoded by an "embedded" encoding (which will be one
422of the above types) until another escape sequence switches to
423a different "embedded" encoding.
424
425These schemes are very flexible and can handle mixed languages but are
426very complex to process (and have state).
427No escape encodings are implemented for perl yet.
428
429=back
430
431=head2 Specifying Encodings
432
433Encodings can be specified to the API described below in two ways:
434
435=over 4
436
437=item 1. By name
438
439Encoding names are strings with characters taken from a restricted repertoire.
440See L</"Encoding Names">.
441
442=item 2. As an object
443
444Encoding objects are returned by C<find_encoding($name)>.
445
446=back
447
448=head2 Encoding Names
449
450Encoding names are case insensitive. White space in names is ignored.
451In addition an encoding may have aliases. Each encoding has one "canonical" name.
452The "canonical" name is chosen from the names of the encoding by picking
453the first in the following sequence:
454
455=over 4
456
457=item * The MIME name as defined in IETF RFC-XXXX.
458
459=item * The name in the IANA registry.
460
461=item * The name used by the the organization that defined it.
462
463=back
464
465Because of all the alias issues, and because in the general case
466encodings have state C<Encode> uses the encoding object internally
467once an operation is in progress.
468
4411f3b6
NIS
469=head1 PERL ENCODING API
470
471=head2 Generic Encoding Interface
472
473=over 4
474
475=item *
476
477 $bytes = encode(ENCODING, $string[, CHECK])
478
479Encodes string from perl's internal form into I<ENCODING> and returns a
480sequence of octets.
481See L</"Handling Malformed Data">.
482
483=item *
484
485 $string = decode(ENCODING, $bytes[, CHECK])
486
487Decode sequence of octets assumed to be in I<ENCODING> into perls internal
488form and returns the resuting string.
489See L</"Handling Malformed Data">.
490
491=back
492
493=head2 Handling Malformed Data
494
495If CHECK is not set, C<undef> is returned. If the data is supposed to
496be UTF-8, an optional lexical warning (category utf8) is given.
497If CHECK is true but not a code reference, dies.
498
499It would desirable to have a way to indicate that transform should use the
500encodings "replacement character" - no such mechanism is defined yet.
501
502It is also planned to allow I<CHECK> to be a code reference.
503
504This is not yet implemented as there are design issues with what its arguments
505should be and how it returns its results.
506
507=over 4
508
509=item Scheme 1
510
511Passed remaining fragment of string being processed.
512Modifies it in place to remove bytes/characters it can understand
513and returns a string used to represent them.
514e.g.
515
516 sub fixup {
517 my $ch = substr($_[0],0,1,'');
518 return sprintf("\x{%02X}",ord($ch);
519 }
520
521This scheme is close to how underlying C code for Encode works, but gives
522the fixup routine very little context.
523
524=item Scheme 2
525
526Passed original string, and an index into it of the problem area,
527and output string so far.
528Appends what it will to output string and returns new index into
529original string.
530e.g.
531
532 sub fixup {
533 # my ($s,$i,$d) = @_;
534 my $ch = substr($_[0],$_[1],1);
535 $_[2] .= sprintf("\x{%02X}",ord($ch);
536 return $_[1]+1;
537 }
538
539This scheme gives maximal control to the fixup routine but is more complicated
540to code, and may need internals of Encode to be tweaked to keep original
541string intact.
542
543=item Other Schemes
544
545Hybrids of above.
546
547Multiple return values rather than in-place modifications.
548
549Index into the string could be pos($str) allowing s/\G...//.
550
551=back
552
553=head2 UTF-8 / utf8
554
555The Unicode consortium defines the UTF-8 standard as a way of encoding
556the entire Unicode repertiore as sequences of octets. This encoding
557is expected to become very widespread. Perl can use this form internaly
558to represent strings, so conversions to and from this form are particularly
559efficient (as octets in memory do not have to change, just the meta-data
560that tells perl how to treat them).
561
562=over 4
563
564=item *
565
566 $bytes = encode_utf8($string);
567
568The characters that comprise string are encoded in perl's superset of UTF-8
569and the resulting octets returned as a sequence of bytes. All possible
570characters have a UTF-8 representation so this function cannot fail.
571
572=item *
573
574 $string = decode_utf8($bytes [,CHECK]);
575
576The sequence of octets represented by $bytes is decoded from UTF-8 into
577a sequence of logical characters. Not all sequences of octets form valid
578UTF-8 encodings, so it is possible for this call to fail.
579See L</"Handling Malformed Data">.
580
581=back
582
583=head2 Other Encodings of Unicode
584
585UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks.
586UCS-2 can only represent 0..0xFFFF, while UTF-16 has a "surogate pair"
587scheme which allows it to cover the whole Unicode range.
588
8040349a 589Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that
4411f3b6
NIS
590happens to be the name used by that representation when used with X11 fonts.
591
592UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters
593can be considered as being in this form without encoding. An encoding
594to transfer strings in this form (e.g. to write them to a file) would need to
595
596 pack('L',map(chr($_),split(//,$string))); # native
597 or
598 pack('V',map(chr($_),split(//,$string))); # little-endian
599 or
600 pack('N',map(chr($_),split(//,$string))); # big-endian
601
602depending on the endian required.
603
51ef4e11 604No UTF-32 encodings are implemented yet.
4411f3b6
NIS
605
606Both UCS-2 and UCS-4 style encodings can have "byte order marks" by representing
607the code point 0xFFFE as the very first thing in a file.
608
51ef4e11
NIS
609=head2 Listing available encodings
610
611 use Encode qw(encodings);
612 @list = encodings();
613
614Returns a list of the canonical names of the available encodings.
615
616=head2 Defining Aliases
617
618 use Encode qw(define_alias);
619 define_alias( newName => ENCODING);
620
621Allows newName to be used as am alias for ENCODING. ENCODING may be either the
622name of an encoding or and encoding object (as above).
623
624Currently I<newName> can be specified in the following ways:
625
626=over 4
627
628=item As a simple string.
629
630=item As a qr// compiled regular expression, e.g.:
631
632 define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' );
633
634In this case if I<ENCODING> is not a reference it is C<eval>-ed to allow
635C<$1> etc. to be subsituted.
636The example is one way to names as used in X11 font names to alias the MIME names for the
637iso-8859-* family.
638
639=item As a code reference, e.g.:
640
641 define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , '');
642
643In this case C<$_> will be set to the name that is being looked up and
644I<ENCODING> is passed to the sub as its first argument.
645The example is another way to names as used in X11 font names to alias the MIME names for
646the iso-8859-* family.
647
648=back
649
650=head2 Defining Encodings
651
652 use Encode qw(define_alias);
653 define_encoding( $object, 'canonicalName' [,alias...]);
654
655Causes I<canonicalName> to be associated with I<$object>.
656The object should provide the interface described in L</"IMPLEMENTATION CLASSES"> below.
657If more than two arguments are provided then additional arguments are taken
658as aliases for I<$object> as for C<define_alias>.
659
4411f3b6
NIS
660=head1 Encoding and IO
661
662It is very common to want to do encoding transformations when
663reading or writing files, network connections, pipes etc.
664If perl is configured to use the new 'perlio' IO system then
665C<Encode> provides a "layer" (See L<perliol>) which can transform
666data as it is read or written.
667
51ef4e11 668 open(my $ilyad,'>:encoding(iso-8859-7)','ilyad.greek');
4411f3b6
NIS
669 print $ilyad @epic;
670
671In addition the new IO system can also be configured to read/write
672UTF-8 encoded characters (as noted above this is efficient):
673
674 open(my $fh,'>:utf8','anything');
675 print $fh "Any \x{0021} string \N{SMILEY FACE}\n";
676
677Either of the above forms of "layer" specifications can be made the default
678for a lexical scope with the C<use open ...> pragma. See L<open>.
679
680Once a handle is open is layers can be altered using C<binmode>.
681
682Without any such configuration, or if perl itself is built using
683system's own IO, then write operations assume that file handle accepts
684only I<bytes> and will C<die> if a character larger than 255 is
685written to the handle. When reading, each octet from the handle
686becomes a byte-in-a-character. Note that this default is the same
687behaviour as bytes-only languages (including perl before v5.6) would have,
688and is sufficient to handle native 8-bit encodings e.g. iso-8859-1,
689EBCDIC etc. and any legacy mechanisms for handling other encodings
690and binary data.
691
692In other cases it is the programs responsibility
693to transform characters into bytes using the API above before
694doing writes, and to transform the bytes read from a handle into characters
695before doing "character operations" (e.g. C<lc>, C</\W+/>, ...).
696
697=head1 Encoding How to ...
698
699To do:
700
701=over 4
702
703=item * IO with mixed content (faking iso-2020-*)
704
705=item * MIME's Content-Length:
706
707=item * UTF-8 strings in binary data.
708
709=item * perl/Encode wrappers on non-Unicode XS modules.
710
711=back
712
713=head1 Messing with Perl's Internals
714
715The following API uses parts of perl's internals in the current implementation.
716As such they are efficient, but may change.
717
718=over 4
719
4411f3b6
NIS
720=item * is_utf8(STRING [, CHECK])
721
722[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING.
723If CHECK is true, also checks the data in STRING for being
724well-formed UTF-8. Returns true if successful, false otherwise.
725
726=item * valid_utf8(STRING)
727
728[INTERNAL] Test whether STRING is in a consistent state.
729Will return true if string is held as bytes, or is well-formed UTF-8
730and has the UTF-8 flag on.
731Main reason for this routine is to allow perl's testsuite to check
732that operations have left strings in a consistent state.
733
734=item *
735
736 _utf8_on(STRING)
737
738[INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is
739B<not> checked for being well-formed UTF-8. Do not use unless you
740B<know> that the STRING is well-formed UTF-8. Returns the previous
741state of the UTF-8 flag (so please don't test the return value as
742I<not> success or failure), or C<undef> if STRING is not a string.
743
744=item *
745
746 _utf8_off(STRING)
747
748[INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously.
749Returns the previous state of the UTF-8 flag (so please don't test the
750return value as I<not> success or failure), or C<undef> if STRING is
751not a string.
752
753=back
754
4edaa979
NIS
755=head1 IMPLEMENTATION CLASSES
756
757As mentioned above encodings are (in the current implementation at least)
758defined by objects. The mapping of encoding name to object is via the
51ef4e11 759C<%encodings> hash.
4edaa979
NIS
760
761The values of the hash can currently be either strings or objects.
762The string form may go away in the future. The string form occurs
763when C<encodings()> has scanned C<@INC> for loadable encodings but has
764not actually loaded the encoding in question. This is because the
765current "loading" process is all perl and a bit slow.
766
767Once an encoding is loaded then value of the hash is object which implements
768the encoding. The object should provide the following interface:
769
770=over 4
771
772=item -E<gt>name
773
774Should return the string representing the canonical name of the encoding.
775
776=item -E<gt>new_sequence
777
778This is a placeholder for encodings with state. It should return an object
779which implements this interface, all current implementations return the
780original object.
781
782=item -E<gt>encode($string,$check)
783
784Should return the octet sequence representing I<$string>. If I<$check> is true
785it should modify I<$string> in place to remove the converted part (i.e.
786the whole string unless there is an error).
787If an error occurs it should return the octet sequence for the
788fragment of string that has been converted, and modify $string in-place
789to remove the converted part leaving it starting with the problem fragment.
790
791If check is is false then C<encode> should make a "best effort" to convert
792the string - for example by using a replacement character.
793
794=item -E<gt>decode($octets,$check)
795
796Should return the string that I<$octets> represents. If I<$check> is true
797it should modify I<$octets> in place to remove the converted part (i.e.
798the whole sequence unless there is an error).
799If an error occurs it should return the fragment of string
800that has been converted, and modify $octets in-place to remove the converted part
801leaving it starting with the problem fragment.
802
803If check is is false then C<decode> should make a "best effort" to convert
804the string - for example by using Unicode's "\x{FFFD}" as a replacement character.
805
806=back
807
808It should be noted that the check behaviour is different from the outer
809public API. The logic is that the "unchecked" case is useful when
810encoding is part of a stream which may be reporting errors (e.g. STDERR).
811In such cases it is desirable to get everything through somehow without
812causing additional errors which obscure the original one. Also the encoding
813is best placed to know what the correct replacement character is, so if that
814is the desired behaviour then letting low level code do it is the most efficient.
815
816In contrast if check is true, the scheme above allows the encoding to do as
817much as it can and tell layer above how much that was. What is lacking
818at present is a mechanism to report what went wrong. The most likely interface
819will be an additional method call to the object, or perhaps
820(to avoid forcing per-stream objects on otherwise stateless encodings)
821and additional parameter.
822
823It is also highly desirable that encoding classes inherit from C<Encode::Encoding>
824as a base class. This allows that class to define additional behaviour for
51ef4e11
NIS
825all encoding objects. For example built in Unicode, UCS-2 and UTF-8 classes
826use :
827
828 package Encode::MyEncoding;
829 use base qw(Encode::Encoding);
830
831 __PACKAGE__->Define(qw(myCanonical myAlias));
832
833To create an object with bless {Name => ...},$class, and call define_encoding.
834They inherit their C<name> method from C<Encode::Encoding>.
4edaa979
NIS
835
836=head2 Compiled Encodings
837
838F<Encode.xs> provides a class C<Encode::XS> which provides the interface described
839above. It calls a generic octet-sequence to octet-sequence "engine" that is
840driven by tables (defined in F<encengine.c>). The same engine is used for both
841encode and decode. C<Encode:XS>'s C<encode> forces perl's characters to their UTF-8 form
842and then treats them as just another multibyte encoding. C<Encode:XS>'s C<decode> transforms
843the sequence and then turns the UTF-8-ness flag as that is the form that the tables
844are defined to produce. For details of the engine see the comments in F<encengine.c>.
845
846The tables are produced by the perl script F<compile> (the name needs to change so
847we can eventually install it somewhere). F<compile> can currently read two formats:
848
849=over 4
850
851=item *.enc
852
853This is a coined format used by Tcl. It is documented in Encode/EncodeFormat.pod.
854
855=item *.ucm
856
857This is the semi-standard format used by IBM's ICU package.
858
859=back
860
861F<compile> can write the following forms:
862
863=over 4
864
865=item *.ucm
866
867See above - the F<Encode/*.ucm> files provided with the distribution have
868been created from the original Tcl .enc files using this approach.
869
870=item *.c
871
872Produces tables as C data structures - this is used to build in encodings
873into F<Encode.so>/F<Encode.dll>.
874
875=item *.xs
876
877In theory this allows encodings to be stand-alone loadable perl extensions.
878The process has not yet been tested. The plan is to use this approach
879for large East Asian encodings.
880
881=back
882
883The set of encodings built-in to F<Encode.so>/F<Encode.dll> is determined by
884F<Makefile.PL>. The current set is as follows:
885
886=over 4
887
888=item ascii and iso-8859-*
889
890That is all the common 8-bit "western" encodings.
891
892=item IBM-1047 and two other variants of EBCDIC.
893
894These are the same variants that are supported by EBCDIC perl as "native" encodings.
895They are included to prove "reversibility" of some constructs in EBCDIC perl.
896
897=item symbol and dingbats as used by Tk on X11.
898
899(The reason Encode got started was to support perl/Tk.)
900
901=back
902
903That set is rather ad. hoc. and has been driven by the needs of the tests rather
904than the needs of typical applications. It is likely to be rationalized.
905
4411f3b6
NIS
906=head1 SEE ALSO
907
908L<perlunicode>, L<perlebcdic>, L<perlfunc/open>
909
910=cut
911
912
2a936312 913