From 1b2c56c862bbec4d95cac5086b866641985b8bdb Mon Sep 17 00:00:00 2001 From: Jarkko Hietaniemi Date: Wed, 20 Mar 2002 21:26:09 +0000 Subject: [PATCH] Upgrade to Encode 0.94, from Dan Kogai. p4raw-id: //depot/perl@15376 --- MANIFEST | 3 +- ext/Encode/AUTHORS | 10 +- ext/Encode/CN/CN.pm | 4 +- ext/Encode/Changes | 31 +- ext/Encode/Encode.pm | 471 +----------------- ext/Encode/JP/JP.pm | 4 +- ext/Encode/KR/KR.pm | 4 +- ext/Encode/MANIFEST | 1 + ext/Encode/TW/TW.pm | 17 +- ext/Encode/lib/Encode/Description.pod | 821 ++++++++++++++++++++++++++++++++ ext/Encode/lib/Encode/Encoding.pm | 177 ++++++- ext/Encode/lib/Encode/JP/ISO_2022_JP.pm | 4 +- ext/Encode/lib/Encode/JP/JIS.pm | 4 +- ext/Encode/lib/Encode/Tcl.pm | 4 +- ext/Encode/t/CJKalias.t | 8 +- 15 files changed, 1077 insertions(+), 486 deletions(-) create mode 100644 ext/Encode/lib/Encode/Description.pod diff --git a/MANIFEST b/MANIFEST index 8f39306..04a4628 100644 --- a/MANIFEST +++ b/MANIFEST @@ -204,7 +204,7 @@ ext/Encode/compile Encode extension ext/Encode/encengine.c Encode extension ext/Encode/encode.h Encode extension ext/Encode/Encode.pm Encode extension -ext/Encode/Encode.xs Encode extension +ext/Encode/Encode.xs Encode extension ext/Encode/Encode/11643-1.enc Encode table ext/Encode/Encode/11643-2.enc Encode table ext/Encode/Encode/2022-cn.enc Encode table @@ -340,6 +340,7 @@ ext/Encode/JP/Makefile.PL Encode extension ext/Encode/KR/KR.pm Encode extension ext/Encode/KR/Makefile.PL Encode extension ext/Encode/lib/Encode/CN/HZ.pm Encode extension +ext/Encode/lib/Encode/Description.pod Encode extension ext/Encode/lib/Encode/Encoding.pm Encode extension ext/Encode/lib/Encode/Internal.pm Encode extension ext/Encode/lib/Encode/iso10646_1.pm Encode extension diff --git a/ext/Encode/AUTHORS b/ext/Encode/AUTHORS index 5f9869e..a131c51 100644 --- a/ext/Encode/AUTHORS +++ b/ext/Encode/AUTHORS @@ -9,7 +9,9 @@ # # This list is in alphabetical order. -- -Autrijus Tang -Dan Kogai -Jarkko Hietaniemi -Nick Ing-Simmons +Anton Tagunov +Autrijus Tang +Dan Kogai +Jarkko Hietaniemi +Nick Ing-Simmons +SADAHIRO Tomoyuki diff --git a/ext/Encode/CN/CN.pm b/ext/Encode/CN/CN.pm index 198eeb5..12a9fd3 100644 --- a/ext/Encode/CN/CN.pm +++ b/ext/Encode/CN/CN.pm @@ -1,5 +1,5 @@ package Encode::CN; -our $VERSION = do { my @r = (q$Revision: 0.93 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +our $VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; use Encode; use Encode::CN::HZ; @@ -17,7 +17,7 @@ Encode::CN - China-based Chinese Encodings =head1 SYNOPSIS - use Encode 'encode'; + use Encode qw/encode decode/; $euc_cn = encode("euc-cn", $utf8); # loads Encode::CN implicitly $utf8 = decode("euc-cn", $euc_cn); # ditto diff --git a/ext/Encode/Changes b/ext/Encode/Changes index 48c6aa3..b48001e 100644 --- a/ext/Encode/Changes +++ b/ext/Encode/Changes @@ -1,8 +1,37 @@ # Revision history for Perl extension Encode. # -# $Id$ +# $Id: Changes,v 0.94 2002/03/20 19:59:38 dankogai Exp dankogai $ # +0.94 Thu Mar 21 2002 ++ lib/Encode/Description.pod +! lib/Encode/Encoding.pm + Now the pod in Encode.pm is abridged as programming references. + lib/Encode/Description.pod contains the original, detailed description + and Encode::Encoding explains how to write your own module to + add new encodings. So far, lib/Encode/Description.pod contains + the whole pod once in Encode.pm. This is intentional. +! Encode.pm + Pod revisions by Anton Tagunov + Message-Id: <517178431.20020320174824@motor.ru> +! lib/Encode/Tcl.pm + all occrance of Encode::Tcl::Extended removed including pod +! t/CJKalias.t + test now checks $encoding->name only; $encoding->{name} are + no longer check to find the canonical name. +! lib/Encode/JP/JIS.pm +! lib/Encode/JP/ISO_2022_JP.pm + ->name() added to be more compliant with API +! CN/CN.pm +! JP/JP.pm +! KR/KR.pm +! TW/TW.pm +! t/CJKalias.t + Patch by Autrijus to add aliases to TW and fixes to POD + Message-Id: <20020320090619.GA24774@not.autrijus.org> +! AUTHORS + SADAHIRO Tomoyuki added as should. My apologies. + 0.93 Wed Mar 20 2002 * First release to be uploaded to CPAN. For prehistoric changes, please see Changes file of perl distibution as well as diff --git a/ext/Encode/Encode.pm b/ext/Encode/Encode.pm index 92dd1db..8dc14c3 100644 --- a/ext/Encode/Encode.pm +++ b/ext/Encode/Encode.pm @@ -1,6 +1,6 @@ package Encode; use strict; -our $VERSION = do { my @r = (q$Revision: 0.93 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +our $VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; require DynaLoader; require Exporter; @@ -203,7 +203,6 @@ define_alias( qr/^gbk$/i => '"cp936"'); # TODO: Thai encoding TCVN # TODO: Korean encoding Johab # TODO: Vietnamese encodings VPS -# TODO: Japanese encoding JIS (not the same as SJIS) # TODO: Mac Asian+African encodings: Arabic Armenian Bengali Burmese # ChineseSimp ChineseTrad Devanagari Ethiopic ExtArabic # Farsi Georgian Gujarati Gurmukhi Hebrew Japanese @@ -340,239 +339,8 @@ Encode - character encodings The C module provides the interfaces between Perl's strings and the rest of the system. Perl strings are sequences of B. -The repertoire of characters that Perl can represent is at least that -defined by the Unicode Consortium. On most platforms the ordinal -values of the characters (as returned by C) is the "Unicode -codepoint" for the character (the exceptions are those platforms where -the legacy encoding is some variant of EBCDIC rather than a super-set -of ASCII - see L). - -Traditionaly computer data has been moved around in 8-bit chunks -often called "bytes". These chunks are also known as "octets" in -networking standards. Perl is widely used to manipulate data of -many types - not only strings of characters representing human or -computer languages but also "binary" data being the machines representation -of numbers, pixels in an image - or just about anything. - -When Perl is processing "binary data" the programmer wants Perl to process -"sequences of bytes". This is not a problem for Perl - as a byte has 256 -possible values it easily fits in Perl's much larger "logical character". - -Due to size concerns, each of B (Chinese, Japanese & Korean) modules -are not loaded in memory until the first time they're used. Although you -don't have to C the corresponding B(B|B|B|B) -modules first, be aware that those encodings will not be in C<%encodings> -until their module is loaded (either implicitly through using encodings -contained in the same module, or via an explicit C). - -=head2 TERMINOLOGY - -=over 4 - -=item * - -I: a character in the range 0..(2**32-1) (or more). -(What Perl's strings are made of.) - -=item * - -I: a character in the range 0..255 -(A special case of a Perl character.) - -=item * - -I: 8 bits of data, with ordinal values 0..255 -(Term for bytes passed to or from a non-Perl context, e.g. disk file.) - -=back - -The marker [INTERNAL] marks Internal Implementation Details, in -general meant only for those who think they know what they are doing, -and such details may change in future releases. - -=head1 ENCODINGS - -=head2 Characteristics of an Encoding - -An encoding has a "repertoire" of characters that it can represent, -and for each representable character there is at least one sequence of -octets that represents it. - -=head2 Types of Encodings - -Encodings can be divided into the following types: - -=over 4 - -=item * Fixed length 8-bit (or less) encodings. - -Each character is a single octet so may have a repertoire of up to -256 characters. ASCII and iso-8859-* are typical examples. - -=item * Fixed length 16-bit encodings - -Each character is two octets so may have a repertoire of up to -65 536 characters. Unicode's UCS-2 is an example. Also used for -encodings for East Asian languages. - -=item * Fixed length 32-bit encodings. - -Not really very "encoded" encodings. The Unicode code points -are just represented as 4-octet integers. None the less because -different architectures use different representations of integers -(so called "endian") there at least two disctinct encodings. - -=item * Multi-byte encodings - -The number of octets needed to represent a character varies. -UTF-8 is a particularly complex but regular case of a multi-byte -encoding. Several East Asian countries use a multi-byte encoding -where 1-octet is used to cover western roman characters and Asian -characters get 2-octets. -(UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets -to represent a Unicode code point.) - -=item * "Escape" encodings. - -These encodings embed "escape sequences" into the octet sequence -which describe how the following octets are to be interpreted. -The iso-2022-* family is typical. Following the escape sequence -octets are encoded by an "embedded" encoding (which will be one -of the above types) until another escape sequence switches to -a different "embedded" encoding. - -These schemes are very flexible and can handle mixed languages but are -very complex to process (and have state). No escape encodings are -implemented for Perl yet. - -=back - -=head2 Specifying Encodings - -Encodings can be specified to the API described below in two ways: - -=over 4 - -=item 1. By name - -Encoding names are strings with characters taken from a restricted -repertoire. See L. - -=item 2. As an object - -Encoding objects are returned by C. -If the second parameter is true, Encode will refrain from loading external -modules for CJK encodings. - -=back - -=head2 Encoding Names - -Encoding names are case insensitive. White space in names is ignored. -In addition an encoding may have aliases. Each encoding has one -"canonical" name. The "canonical" name is chosen from the names of -the encoding by picking the first in the following sequence: - -=over 4 - -=item * The MIME name as defined in IETF RFCs. - -=item * The name in the IANA registry. - -=item * The name used by the organization that defined it. - -=back - -Because of all the alias issues, and because in the general case -encodings have state C uses the encoding object internally -once an operation is in progress. - -As of Perl 5.8.0, at least the following encodings are recognized -(the => marks aliases): - - ASCII - - US-ASCII => ASCII - -The Unicode: - - UTF-8 - UTF-16 - UCS-2 - - ISO 10646-1 => UCS-2 - -The ISO 8859 and KOI: - - ISO 8859-1 ISO 8859-6 ISO 8859-11 KOI8-F - ISO 8859-2 ISO 8859-7 (12 doesn't exist) KOI8-R - ISO 8859-3 ISO 8859-8 ISO 8859-13 KOI8-U - ISO 8859-4 ISO 8859-9 ISO 8859-14 - ISO 8859-5 ISO 8859-10 ISO 8859-15 - ISO 8859-16 - - Latin1 => 8859-1 Latin6 => 8859-10 - Latin2 => 8859-2 Latin7 => 8859-13 - Latin3 => 8859-3 Latin8 => 8859-14 - Latin4 => 8859-4 Latin9 => 8859-15 - Latin5 => 8859-9 Latin10 => 8859-16 - - Cyrillic => 8859-5 - Arabic => 8859-6 - Greek => 8859-7 - Hebrew => 8859-8 - Thai => 8859-11 - TIS620 => 8859-11 - -The CJKV: Chinese, Japanese, Korean, Vietnamese: - - ISO 2022 ISO 2022 JP-1 JIS 0201 GB 1988 Big5 EUC-CN - ISO 2022 CN ISO 2022 JP-2 JIS 0208 GB 2312 HZ EUC-JP - ISO 2022 JP ISO 2022 KR JIS 0210 GB 12345 CNS 11643 EUC-JP-0212 - Shift-JIS GBK Big5-HKSCS EUC-KR - VISCII ISO-IR-165 - -(Due to size concerns, additional Chinese encodings including C, -C and C are distributed separately on CPAN, under the name -L.) - -The PC codepages: - - CP37 CP852 CP861 CP866 CP949 CP1251 CP1256 - CP424 CP855 CP862 CP869 CP950 CP1252 CP1257 - CP737 CP856 CP863 CP874 CP1006 CP1253 CP1258 - CP775 CP857 CP864 CP932 CP1047 CP1254 - CP850 CP860 CP865 CP936 CP1250 CP1255 - - WinLatin1 => CP1252 - WinLatin2 => CP1250 - WinCyrillic => CP1251 - WinGreek => CP1253 - WinTurkiskh => CP1254 - WinHebrew => CP1255 - WinArabic => CP1256 - WinBaltic => CP1257 - WinVietnamese => CP1258 - -(All the CPI are available also as IBMI.) - -The Mac codepages: - - MacCentralEuropean MacJapanese - MacCroatian MacRoman - MacCyrillic MacRomanian - MacDingbats MacSami - MacGreek MacThai - MacIcelandic MacTurkish - MacUkraine - -Miscellaneous: - - 7bit-greek IR-197 - 7bit-kana NeXTstep - 7bit-latin1 POSIX-BC - DingBats Roman8 - GSM 0338 Symbol +To find more about character encodings, please consult +L . This document focuses on programming references. =head1 PERL ENCODING API @@ -600,7 +368,7 @@ Decode sequence of octets assumed to be in I into Perl's internal form and returns the resulting string. For CHECK see L. -For example to convert ISO 8859-1 data to UTF-8: +For example to convert ISO-8859-1 data to UTF-8: $utf8 = decode("latin1", $latin1); @@ -613,7 +381,7 @@ in $string originally get to be in FROM_ENCODING? Either using encode() or through PerlIO: See L. For CHECK see L. -For example to convert ISO 8859-1 data to UTF-8: +For example to convert ISO-8859-1 data to UTF-8: from_to($data, "iso-8859-1", "utf-8"); @@ -714,46 +482,6 @@ For CHECK see L. =back -=head2 Other Encodings of Unicode - -UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. UCS-2 can only -represent 0..0xFFFF, while UTF-16 has a I scheme which -allows it to cover the whole Unicode range. - -Surrogates are code points set aside to encode the 0x01000..0x10FFFF -range of Unicode code points in pairs of 16-bit units. The I are the range 0xD800..0xDBFF, and the I -are the range 0xDC00..0xDFFFF. The surrogate encoding is - - $hi = ($uni - 0x10000) / 0x400 + 0xD800; - $lo = ($uni - 0x10000) % 0x400 + 0xDC00; - -and the decoding is - - $uni = 0x10000 + ($hi - 0xD8000) * 0x400 + ($lo - 0xDC00); - -Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that -happens to be the name used by that representation when used with X11 -fonts. - -UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters -can be considered as being in this form without encoding. An encoding -to transfer strings in this form (e.g. to write them to a file) would -need to - - pack('L*', unpack('U*', $string)); # native - or - pack('V*', unpack('U*', $string)); # little-endian - or - pack('N*', unpack('U*', $string)); # big-endian - -depending on the endianness required. - -No UTF-32 encodings are implemented yet. - -Both UCS-2 and UCS-4 style encodings can have "byte order marks" by -representing the code point 0xFFFE as the very first thing in a file. - =head2 Listing available encodings use Encode qw(encodings); @@ -782,7 +510,8 @@ Currently I can be specified in the following ways: In this case if I is not a reference it is C-ed to allow C<$1> etc. to be subsituted. The example is one way to names as used in X11 font names to alias the MIME names for the iso-8859-* -family. +family. Note the double quote inside the single quote. If you are +using regex here, you have to do so or it won't work in this case. =item As a code reference, e.g.: @@ -795,13 +524,13 @@ names for the iso-8859-* family. =back -=head2 Defining Encodings +=head1 Defining Encodings use Encode qw(define_alias); define_encoding( $object, 'canonicalName' [,alias...]); Causes I to be associated with I<$object>. The object -should provide the interface described in L +should provide the interface described in L below. If more than two arguments are provided then additional arguments are taken as aliases for I<$object> as for C. @@ -850,7 +579,7 @@ transform the bytes read from a handle into characters before doing "character operations" (e.g. C, C, ...). You can also use PerlIO to convert larger amounts of data you don't -want to bring into memory. For example to convert between ISO 8859-1 +want to bring into memory. For example to convert between ISO-8859-1 (Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines): open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!; @@ -871,22 +600,6 @@ See L for more information. See also L for how to change the default encoding of the data in your script. -=head1 Encoding How to ... - -To do: - -=over 4 - -=item * IO with mixed content (faking iso-2020-*) - -=item * MIME's Content-Length: - -=item * UTF-8 strings in binary data. - -=item * Perl/Encode wrappers on non-Unicode XS modules. - -=back - =head1 Messing with Perl's Internals The following API uses parts of Perl's internals in the current @@ -921,172 +634,10 @@ not a string. =back -=head1 IMPLEMENTATION CLASSES - -As mentioned above encodings are (in the current implementation at least) -defined by objects. The mapping of encoding name to object is via the -C<%encodings> hash. - -The values of the hash can currently be either strings or objects. -The string form may go away in the future. The string form occurs -when C has scanned C<@INC> for loadable encodings but has -not actually loaded the encoding in question. This is because the -current "loading" process is all Perl and a bit slow. - -Once an encoding is loaded then value of the hash is object which -implements the encoding. The object should provide the following -interface: - -=over 4 - -=item -Ename - -Should return the string representing the canonical name of the encoding. - -=item -Enew_sequence - -This is a placeholder for encodings with state. It should return an -object which implements this interface, all current implementations -return the original object. - -=item -Eencode($string,$check) - -Should return the octet sequence representing I<$string>. If I<$check> -is true it should modify I<$string> in place to remove the converted -part (i.e. the whole string unless there is an error). If an error -occurs it should return the octet sequence for the fragment of string -that has been converted, and modify $string in-place to remove the -converted part leaving it starting with the problem fragment. - -If check is is false then C should make a "best effort" to -convert the string - for example by using a replacement character. - -=item -Edecode($octets,$check) - -Should return the string that I<$octets> represents. If I<$check> is -true it should modify I<$octets> in place to remove the converted part -(i.e. the whole sequence unless there is an error). If an error -occurs it should return the fragment of string that has been -converted, and modify $octets in-place to remove the converted part -leaving it starting with the problem fragment. - -If check is is false then C should make a "best effort" to -convert the string - for example by using Unicode's "\x{FFFD}" as a -replacement character. - -=back - -It should be noted that the check behaviour is different from the -outer public API. The logic is that the "unchecked" case is useful -when encoding is part of a stream which may be reporting errors -(e.g. STDERR). In such cases it is desirable to get everything -through somehow without causing additional errors which obscure the -original one. Also the encoding is best placed to know what the -correct replacement character is, so if that is the desired behaviour -then letting low level code do it is the most efficient. - -In contrast if check is true, the scheme above allows the encoding to -do as much as it can and tell layer above how much that was. What is -lacking at present is a mechanism to report what went wrong. The most -likely interface will be an additional method call to the object, or -perhaps (to avoid forcing per-stream objects on otherwise stateless -encodings) and additional parameter. - -It is also highly desirable that encoding classes inherit from -C as a base class. This allows that class to define -additional behaviour for all encoding objects. For example built in -Unicode, UCS-2 and UTF-8 classes use : - - package Encode::MyEncoding; - use base qw(Encode::Encoding); - - __PACKAGE__->Define(qw(myCanonical myAlias)); - -To create an object with bless {Name => ...},$class, and call -define_encoding. They inherit their C method from -C. - -=head2 Compiled Encodings - -F provides a class C which provides the -interface described above. It calls a generic octet-sequence to -octet-sequence "engine" that is driven by tables (defined in -F). The same engine is used for both encode and -decode. C's C forces Perl's characters to their -UTF-8 form and then treats them as just another multibyte -encoding. C's C transforms the sequence and then -turns the UTF-8-ness flag as that is the form that the tables are -defined to produce. For details of the engine see the comments in -F. - -The tables are produced by the Perl script F (the name needs -to change so we can eventually install it somewhere). F can -currently read two formats: - -=over 4 - -=item *.enc - -This is a coined format used by Tcl. It is documented in -Encode/EncodeFormat.pod. - -=item *.ucm - -This is the semi-standard format used by IBM's ICU package. - -=back - -F can write the following forms: - -=over 4 - -=item *.ucm - -See above - the F files provided with the distribution have -been created from the original Tcl .enc files using this approach. - -=item *.c - -Produces tables as C data structures - this is used to build in encodings -into F/F. - -=item *.xs - -In theory this allows encodings to be stand-alone loadable Perl -extensions. The process has not yet been tested. The plan is to use -this approach for large East Asian encodings. - -=back - -The set of encodings built-in to F/F is -determined by F. The current set is as follows: - -=over 4 - -=item ascii and iso-8859-* - -That is all the common 8-bit "western" encodings. - -=item IBM-1047 and two other variants of EBCDIC. - -These are the same variants that are supported by EBCDIC Perl as -"native" encodings. They are included to prove "reversibility" of -some constructs in EBCDIC Perl. - -=item symbol and dingbats as used by Tk on X11. - -(The reason Encode got started was to support Perl/Tk.) - -=back - -That set is rather ad hoc and has been driven by the needs of the -tests rather than the needs of typical applications. It is likely -to be rationalized. - =head1 SEE ALSO L, L, L, L, L, -L, the Perl Unicode Mailing List Eperl-unicode@perl.orgE +L, L, L the Perl Unicode Mailing List Eperl-unicode@perl.orgE =cut diff --git a/ext/Encode/JP/JP.pm b/ext/Encode/JP/JP.pm index e60b912..3091a99 100644 --- a/ext/Encode/JP/JP.pm +++ b/ext/Encode/JP/JP.pm @@ -5,7 +5,7 @@ BEGIN { } } use Encode; -our $VERSION = do { my @r = (q$Revision: 0.93 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +our $VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; use XSLoader; XSLoader::load('Encode::JP',$VERSION); @@ -28,7 +28,7 @@ Encode::JP - Japanese Encodings =head1 SYNOPSIS - use Encode 'encode'; + use Encode qw/encode decode/; $euc_jp = encode("euc-jp", $utf8); # loads Encode::JP implicitly $utf8 = decode("euc-jp", $euc_jp); # ditto diff --git a/ext/Encode/KR/KR.pm b/ext/Encode/KR/KR.pm index 0823fa9..2a6507a 100644 --- a/ext/Encode/KR/KR.pm +++ b/ext/Encode/KR/KR.pm @@ -1,5 +1,5 @@ package Encode::KR; -our $VERSION = do { my @r = (q$Revision: 0.93 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +our $VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; use Encode; use XSLoader; @@ -16,7 +16,7 @@ Encode::KR - Korean Encodings =head1 SYNOPSIS - use Encode 'encode'; + use Encode qw/encode decode/; $euc_kr = encode("euc-kr", $utf8); # loads Encode::KR implicitly $utf8 = decode("euc-kr", $euc_kr); # ditto diff --git a/ext/Encode/MANIFEST b/ext/Encode/MANIFEST index ec9fc4d..4d5c6fa 100644 --- a/ext/Encode/MANIFEST +++ b/ext/Encode/MANIFEST @@ -147,6 +147,7 @@ compile Encode extension encengine.c Encode extension encode.h Encode extension lib/Encode/CN/HZ.pm Encode extension +lib/Encode/Description.pod General topics on character encodings lib/Encode/Encoding.pm Encode extension lib/Encode/Internal.pm Encode extension lib/Encode/JP/Constants.pm Encode extension diff --git a/ext/Encode/TW/TW.pm b/ext/Encode/TW/TW.pm index 61b8271..3daa2a1 100644 --- a/ext/Encode/TW/TW.pm +++ b/ext/Encode/TW/TW.pm @@ -1,10 +1,13 @@ package Encode::TW; -our $VERSION = do { my @r = (q$Revision: 0.92 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +our $VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; use Encode; use XSLoader; XSLoader::load('Encode::TW',$VERSION); +Encode::define_alias( qr/big-?5$/i => '"big5"' ); +Encode::define_alias( qr/big5-hk(?:scs)?/i => '"big5-hkscs"' ); + 1; __END__ =head1 NAME @@ -13,7 +16,7 @@ Encode::TW - Taiwan-based Chinese Encodings =head1 SYNOPSIS - use Encode 'encode'; + use Encode qw/encode decode/; $big5 = encode("big5", $utf8); # loads Encode::TW implicitly $utf8 = decode("big5", $big5); # ditto @@ -22,9 +25,13 @@ Encode::TW - Taiwan-based Chinese Encodings This module implements Taiwan-based Chinese charset encodings. Encodings supported are as follows. - big5 The original Big5 encoding - big5-hkscs Big5 plus Cantonese characters in Hong Kong - cp950 Code Page 950 (Big5 + Microsoft vendor mappings) + Canonical Alias Description + -------------------------------------------------------------------- + big5 /big-?5$/i The original Big5 encoding + big5-hkscs /big5-hk(scs)?$/i Big5 plus Cantonese characters in + Hong Kong + cp950 Code Page 950 + (Big5 + Microsoft vendor mappings) To find how to use this module in detail, see L. diff --git a/ext/Encode/lib/Encode/Description.pod b/ext/Encode/lib/Encode/Description.pod new file mode 100644 index 0000000..aa3a0af --- /dev/null +++ b/ext/Encode/lib/Encode/Description.pod @@ -0,0 +1,821 @@ + +=head1 NAME + +Encode - character encodings + +=head1 SYNOPSIS + + use Encode; + +=head1 DESCRIPTION + +The C module provides the interfaces between Perl's strings +and the rest of the system. Perl strings are sequences of B. + +The repertoire of characters that Perl can represent is at least that +defined by the Unicode Consortium. On most platforms the ordinal +values of the characters (as returned by C) is the "Unicode +codepoint" for the character (the exceptions are those platforms where +the legacy encoding is some variant of EBCDIC rather than a super-set +of ASCII - see L). + +Traditionaly computer data has been moved around in 8-bit chunks +often called "bytes". These chunks are also known as "octets" in +networking standards. Perl is widely used to manipulate data of +many types - not only strings of characters representing human or +computer languages but also "binary" data being the machines representation +of numbers, pixels in an image - or just about anything. + +When Perl is processing "binary data" the programmer wants Perl to process +"sequences of bytes". This is not a problem for Perl - as a byte has 256 +possible values it easily fits in Perl's much larger "logical character". + +Due to size concerns, each of B (Chinese, Japanese & Korean) modules +are not loaded in memory until the first time they're used. Although you +don't have to C the corresponding B(B|B|B|B) +modules first, be aware that those encodings will not be in C<%encodings> +until their module is loaded (either implicitly through using encodings +contained in the same module, or via an explicit C). + +=head2 TERMINOLOGY + +=over 4 + +=item * + +I: a character in the range 0..(2**32-1) (or more). +(What Perl's strings are made of.) + +=item * + +I: a character in the range 0..255 +(A special case of a Perl character.) + +=item * + +I: 8 bits of data, with ordinal values 0..255 +(Term for bytes passed to or from a non-Perl context, e.g. disk file.) + +=back + +The marker [INTERNAL] marks Internal Implementation Details, in +general meant only for those who think they know what they are doing, +and such details may change in future releases. + +=head1 ENCODINGS + +=head2 Characteristics of an Encoding + +An encoding has a "repertoire" of characters that it can represent, +and for each representable character there is at least one sequence of +octets that represents it. + +=head2 Types of Encodings + +Encodings can be divided into the following types: + +=over 4 + +=item * Fixed length 8-bit (or less) encodings. + +Each character is a single octet so may have a repertoire of up to +256 characters. ASCII and iso-8859-* are typical examples. + +=item * Fixed length 16-bit encodings + +Each character is two octets so may have a repertoire of up to +65 536 characters. Unicode's UCS-2 is an example. Also used for +encodings for East Asian languages. + +=item * Fixed length 32-bit encodings. + +Not really very "encoded" encodings. The Unicode code points +are just represented as 4-octet integers. None the less because +different architectures use different representations of integers +(so called "endian") there at least two disctinct encodings. + +=item * Multi-byte encodings + +The number of octets needed to represent a character varies. +UTF-8 is a particularly complex but regular case of a multi-byte +encoding. Several East Asian countries use a multi-byte encoding +where 1-octet is used to cover western roman characters and Asian +characters get 2-octets. +(UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets +to represent a Unicode code point.) + +=item * "Escape" encodings. + +These encodings embed "escape sequences" into the octet sequence +which describe how the following octets are to be interpreted. +The iso-2022-* family is typical. Following the escape sequence +octets are encoded by an "embedded" encoding (which will be one +of the above types) until another escape sequence switches to +a different "embedded" encoding. + +These schemes are very flexible and can handle mixed languages but are +very complex to process (and have state). No escape encodings are +implemented for Perl yet. + +=back + +=head2 Specifying Encodings + +Encodings can be specified to the API described below in two ways: + +=over 4 + +=item 1. By name + +Encoding names are strings with characters taken from a restricted +repertoire. See L. + +=item 2. As an object + +Encoding objects are returned by C. +If the second parameter is true, Encode will refrain from loading external +modules for CJK encodings. + +=back + +=head2 Encoding Names + +Encoding names are case insensitive. White space in names is ignored. +In addition an encoding may have aliases. Each encoding has one +"canonical" name. The "canonical" name is chosen from the names of +the encoding by picking the first in the following sequence: + +=over 4 + +=item * The MIME name as defined in IETF RFCs. + +=item * The name in the IANA registry. + +=item * The name used by the organization that defined it. + +=back + +Because of all the alias issues, and because in the general case +encodings have state C uses the encoding object internally +once an operation is in progress. + +As of Perl 5.8.0, at least the following encodings are recognized +(the => marks aliases): + + ASCII + + US-ASCII => ASCII + +The Unicode: + + UTF-8 + UTF-16 + UCS-2 + + ISO 10646-1 => UCS-2 + +The ISO 8859 and KOI: + + ISO 8859-1 ISO 8859-6 ISO 8859-11 KOI8-F + ISO 8859-2 ISO 8859-7 (12 doesn't exist) KOI8-R + ISO 8859-3 ISO 8859-8 ISO 8859-13 KOI8-U + ISO 8859-4 ISO 8859-9 ISO 8859-14 + ISO 8859-5 ISO 8859-10 ISO 8859-15 + ISO 8859-16 + + Latin1 => 8859-1 Latin6 => 8859-10 + Latin2 => 8859-2 Latin7 => 8859-13 + Latin3 => 8859-3 Latin8 => 8859-14 + Latin4 => 8859-4 Latin9 => 8859-15 + Latin5 => 8859-9 Latin10 => 8859-16 + + Cyrillic => 8859-5 + Arabic => 8859-6 + Greek => 8859-7 + Hebrew => 8859-8 + Thai => 8859-11 + TIS620 => 8859-11 + +The CJKV: Chinese, Japanese, Korean, Vietnamese: + + ISO 2022 ISO 2022 JP-1 JIS 0201 GB 1988 Big5 EUC-CN + ISO 2022 CN ISO 2022 JP-2 JIS 0208 GB 2312 HZ EUC-JP + ISO 2022 JP ISO 2022 KR JIS 0210 GB 12345 CNS 11643 EUC-JP-0212 + Shift-JIS GBK Big5-HKSCS EUC-KR + VISCII ISO-IR-165 + +(Due to size concerns, additional Chinese encodings including C, +C and C are distributed separately on CPAN, under the name +L.) + +The PC codepages: + + CP37 CP852 CP861 CP866 CP949 CP1251 CP1256 + CP424 CP855 CP862 CP869 CP950 CP1252 CP1257 + CP737 CP856 CP863 CP874 CP1006 CP1253 CP1258 + CP775 CP857 CP864 CP932 CP1047 CP1254 + CP850 CP860 CP865 CP936 CP1250 CP1255 + + WinLatin1 => CP1252 + WinLatin2 => CP1250 + WinCyrillic => CP1251 + WinGreek => CP1253 + WinTurkiskh => CP1254 + WinHebrew => CP1255 + WinArabic => CP1256 + WinBaltic => CP1257 + WinVietnamese => CP1258 + +(All the CPI are available also as IBMI.) + +The Mac codepages: + + MacCentralEuropean MacJapanese + MacCroatian MacRoman + MacCyrillic MacRomanian + MacDingbats MacSami + MacGreek MacThai + MacIcelandic MacTurkish + MacUkraine + +Miscellaneous: + + 7bit-greek IR-197 + 7bit-kana NeXTstep + 7bit-latin1 POSIX-BC + DingBats Roman8 + GSM 0338 Symbol + +=head2 Encoding Classification + +Encodings + + US-ASCII UTF-8 KOI8-R ISO-8859-* + ISO-2022-CN ISO-2022-JP ISO-2022-KR Big5 + EUC-CN EUC-JP EUC-KR + +are L-registered +as preferred MIME names and may probably be used over the Internet. +So is + + Shift_JIS + +but despite its wide spread it bears the label of being +Microsft proprietary. + + UTF-16 KOI8-U ISO-2022-JP-2 + +are IANA-registered preferred MIME names but probably shoule +be avoided as encoding for web pages due to lack of browser +support. + + + ISO-2022 (http://www.ecma.ch/ecma1/STAND/ECMA-035.HTM) + ISO-2022-JP-1 (http://www.faqs.org/rfcs/rfc2237.html) + ISO-IR-165 (http://www.faqs.org/rfcs/rfc1345.html) + GBK + VISCII + GB 12345 (only plains 1 and 2 available) + GB 18030 + CNS 11643 + +are totally valid encodings but not registered at IANA. + + BIG5PLUS + EUC-JP-0212 (Encode::lib::Encode::Tcl::Extended) + +are a bit proprietary + +You may probably get some info on CJK encodings at + + brief description for most of the mentioned CJK encodings + http://www.debian.org.ru/doc/manuals/intro-i18n/ch-codes.html + + several years old, but still useful + http://www.oreilly.com/people/authors/lunde/cjk_inf.html + + and some in-depth reading for the heroes :-) + http://www.ecma.ch/ecma1/STAND/ECMA-035.HTM (eq ISO-2022) + http://www.faqs.org/rfcs/rfc1345.txt + + +=head1 PERL ENCODING API + +=head2 Generic Encoding Interface + +=over 4 + +=item * + + $bytes = encode(ENCODING, $string[, CHECK]) + +Encodes string from Perl's internal form into I and returns +a sequence of octets. For CHECK see L. + +For example to convert (internally UTF-8 encoded) Unicode data +to octets: + + $octets = encode("utf8", $unicode); + +=item * + + $string = decode(ENCODING, $bytes[, CHECK]) + +Decode sequence of octets assumed to be in I into Perl's +internal form and returns the resulting string. For CHECK see +L. + +For example to convert ISO-8859-1 data to UTF-8: + + $utf8 = decode("latin1", $latin1); + +=item * + + from_to($string, FROM_ENCODING, TO_ENCODING[, CHECK]) + +Convert B the data between two encodings. How did the data +in $string originally get to be in FROM_ENCODING? Either using +encode() or through PerlIO: See L. For CHECK +see L. + +For example to convert ISO-8859-1 data to UTF-8: + + from_to($data, "iso-8859-1", "utf-8"); + +and to convert it back: + + from_to($data, "utf-8", "iso-8859-1"); + +Note that because the conversion happens in place, the data to be +converted cannot be a string constant, it must be a scalar variable. + +=back + +=head2 Handling Malformed Data + +If CHECK is not set, C is returned. If the data is supposed to +be UTF-8, an optional lexical warning (category utf8) is given. If +CHECK is true but not a code reference, dies. + +It would desirable to have a way to indicate that transform should use +the encodings "replacement character" - no such mechanism is defined yet. + +It is also planned to allow I to be a code reference. + +This is not yet implemented as there are design issues with what its +arguments should be and how it returns its results. + +=over 4 + +=item Scheme 1 + +Passed remaining fragment of string being processed. +Modifies it in place to remove bytes/characters it can understand +and returns a string used to represent them. +e.g. + + sub fixup { + my $ch = substr($_[0],0,1,''); + return sprintf("\x{%02X}",ord($ch); + } + +This scheme is close to how underlying C code for Encode works, but gives +the fixup routine very little context. + +=item Scheme 2 + +Passed original string, and an index into it of the problem area, and +output string so far. Appends what it will to output string and +returns new index into original string. For example: + + sub fixup { + # my ($s,$i,$d) = @_; + my $ch = substr($_[0],$_[1],1); + $_[2] .= sprintf("\x{%02X}",ord($ch); + return $_[1]+1; + } + +This scheme gives maximal control to the fixup routine but is more +complicated to code, and may need internals of Encode to be tweaked to +keep original string intact. + +=item Other Schemes + +Hybrids of above. + +Multiple return values rather than in-place modifications. + +Index into the string could be pos($str) allowing s/\G...//. + +=back + +=head2 UTF-8 / utf8 + +The Unicode consortium defines the UTF-8 standard as a way of encoding +the entire Unicode repertiore as sequences of octets. This encoding is +expected to become very widespread. Perl can use this form internaly +to represent strings, so conversions to and from this form are +particularly efficient (as octets in memory do not have to change, +just the meta-data that tells Perl how to treat them). + +=over 4 + +=item * + + $bytes = encode_utf8($string); + +The characters that comprise string are encoded in Perl's superset of UTF-8 +and the resulting octets returned as a sequence of bytes. All possible +characters have a UTF-8 representation so this function cannot fail. + +=item * + + $string = decode_utf8($bytes [,CHECK]); + +The sequence of octets represented by $bytes is decoded from UTF-8 +into a sequence of logical characters. Not all sequences of octets +form valid UTF-8 encodings, so it is possible for this call to fail. +For CHECK see L. + +=back + +=head2 Other Encodings of Unicode + +UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. UCS-2 can only +represent 0..0xFFFF, while UTF-16 has a I scheme which +allows it to cover the whole Unicode range. + +Surrogates are code points set aside to encode the 0x01000..0x10FFFF +range of Unicode code points in pairs of 16-bit units. The I are the range 0xD800..0xDBFF, and the I +are the range 0xDC00..0xDFFFF. The surrogate encoding is + + $hi = ($uni - 0x10000) / 0x400 + 0xD800; + $lo = ($uni - 0x10000) % 0x400 + 0xDC00; + +and the decoding is + + $uni = 0x10000 + ($hi - 0xD8000) * 0x400 + ($lo - 0xDC00); + +Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that +happens to be the name used by that representation when used with X11 +fonts. + +UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters +can be considered as being in this form without encoding. An encoding +to transfer strings in this form (e.g. to write them to a file) would +need to + + pack('L*', unpack('U*', $string)); # native + or + pack('V*', unpack('U*', $string)); # little-endian + or + pack('N*', unpack('U*', $string)); # big-endian + +depending on the endianness required. + +No UTF-32 encodings are implemented yet. + +Both UCS-2 and UCS-4 style encodings can have "byte order marks" by +representing the code point 0xFFFE as the very first thing in a file. + +=head2 Listing available encodings + + use Encode qw(encodings); + @list = encodings(); + +Returns a list of the canonical names of the available encodings. + +=head2 Defining Aliases + + use Encode qw(define_alias); + define_alias( newName => ENCODING); + +Allows newName to be used as am alias for ENCODING. ENCODING may be +either the name of an encoding or and encoding object (as above). + +Currently I can be specified in the following ways: + +=over 4 + +=item As a simple string. + +=item As a qr// compiled regular expression, e.g.: + + define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' ); + +In this case if I is not a reference it is C-ed to +allow C<$1> etc. to be subsituted. The example is one way to names as +used in X11 font names to alias the MIME names for the iso-8859-* +family. Note the double quote inside the single quote. If you are +using regex here, y ou have to do so or it won't work in this case. + +=item As a code reference, e.g.: + + define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , ''); + +In this case C<$_> will be set to the name that is being looked up and +I is passed to the sub as its first argument. The example +is another way to names as used in X11 font names to alias the MIME +names for the iso-8859-* family. + +=back + +=head2 Defining Encodings + + use Encode qw(define_alias); + define_encoding( $object, 'canonicalName' [,alias...]); + +Causes I to be associated with I<$object>. The object +should provide the interface described in L +below. If more than two arguments are provided then additional +arguments are taken as aliases for I<$object> as for C. + +=head1 Encoding and IO + +It is very common to want to do encoding transformations when +reading or writing files, network connections, pipes etc. +If Perl is configured to use the new 'perlio' IO system then +C provides a "layer" (See L) which can transform +data as it is read or written. + +Here is how the blind poet would modernise the encoding: + + use Encode; + open(my $iliad,'<:encoding(iso-8859-7)','iliad.greek'); + open(my $utf8,'>:utf8','iliad.utf8'); + my @epic = <$iliad>; + print $utf8 @epic; + close($utf8); + close($illiad); + +In addition the new IO system can also be configured to read/write +UTF-8 encoded characters (as noted above this is efficient): + + open(my $fh,'>:utf8','anything'); + print $fh "Any \x{0021} string \N{SMILEY FACE}\n"; + +Either of the above forms of "layer" specifications can be made the default +for a lexical scope with the C pragma. See L. + +Once a handle is open is layers can be altered using C. + +Without any such configuration, or if Perl itself is built using +system's own IO, then write operations assume that file handle accepts +only I and will C if a character larger than 255 is +written to the handle. When reading, each octet from the handle +becomes a byte-in-a-character. Note that this default is the same +behaviour as bytes-only languages (including Perl before v5.6) would +have, and is sufficient to handle native 8-bit encodings +e.g. iso-8859-1, EBCDIC etc. and any legacy mechanisms for handling +other encodings and binary data. + +In other cases it is the programs responsibility to transform +characters into bytes using the API above before doing writes, and to +transform the bytes read from a handle into characters before doing +"character operations" (e.g. C, C, ...). + +You can also use PerlIO to convert larger amounts of data you don't +want to bring into memory. For example to convert between ISO-8859-1 +(Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines): + + open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!; + open(G, ">:utf8", "data.utf") or die $!; + while () { print G } + + # Could also do "print G " but that would pull + # the whole file into memory just to write it out again. + +More examples: + + open(my $f, "<:encoding(cp1252)") + open(my $g, ">:encoding(iso-8859-2)") + open(my $h, ">:encoding(latin9)") # iso-8859-15 + +See L for more information. + +See also L for how to change the default encoding of the +data in your script. + +=head1 Encoding How to ... + +To do: + +=over 4 + +=item * IO with mixed content (faking iso-2022-*) + +Encode::JP implements its own iso-2022 routines, however. + +=item * MIME's Content-Length: + +=item * UTF-8 strings in binary data. + +=item * Perl/Encode wrappers on non-Unicode XS modules. + +=back + +=head1 Messing with Perl's Internals + +The following API uses parts of Perl's internals in the current +implementation. As such they are efficient, but may change. + +=over 4 + +=item * is_utf8(STRING [, CHECK]) + +[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING. +If CHECK is true, also checks the data in STRING for being well-formed +UTF-8. Returns true if successful, false otherwise. + +=item * + + _utf8_on(STRING) + +[INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is +B checked for being well-formed UTF-8. Do not use unless you +B that the STRING is well-formed UTF-8. Returns the previous +state of the UTF-8 flag (so please don't test the return value as +I success or failure), or C if STRING is not a string. + +=item * + + _utf8_off(STRING) + +[INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously. +Returns the previous state of the UTF-8 flag (so please don't test the +return value as I success or failure), or C if STRING is +not a string. + +=back + +=head1 IMPLEMENTATION CLASSES + +As mentioned above encodings are (in the current implementation at least) +defined by objects. The mapping of encoding name to object is via the +C<%encodings> hash. + +The values of the hash can currently be either strings or objects. +The string form may go away in the future. The string form occurs +when C has scanned C<@INC> for loadable encodings but has +not actually loaded the encoding in question. This is because the +current "loading" process is all Perl and a bit slow. + +Once an encoding is loaded then value of the hash is object which +implements the encoding. The object should provide the following +interface: + +=over 4 + +=item -Ename + +Should return the string representing the canonical name of the encoding. + +=item -Enew_sequence + +This is a placeholder for encodings with state. It should return an +object which implements this interface, all current implementations +return the original object. + +=item -Eencode($string,$check) + +Should return the octet sequence representing I<$string>. If I<$check> +is true it should modify I<$string> in place to remove the converted +part (i.e. the whole string unless there is an error). If an error +occurs it should return the octet sequence for the fragment of string +that has been converted, and modify $string in-place to remove the +converted part leaving it starting with the problem fragment. + +If check is is false then C should make a "best effort" to +convert the string - for example by using a replacement character. + +=item -Edecode($octets,$check) + +Should return the string that I<$octets> represents. If I<$check> is +true it should modify I<$octets> in place to remove the converted part +(i.e. the whole sequence unless there is an error). If an error +occurs it should return the fragment of string that has been +converted, and modify $octets in-place to remove the converted part +leaving it starting with the problem fragment. + +If check is is false then C should make a "best effort" to +convert the string - for example by using Unicode's "\x{FFFD}" as a +replacement character. + +=back + +It should be noted that the check behaviour is different from the +outer public API. The logic is that the "unchecked" case is useful +when encoding is part of a stream which may be reporting errors +(e.g. STDERR). In such cases it is desirable to get everything +through somehow without causing additional errors which obscure the +original one. Also the encoding is best placed to know what the +correct replacement character is, so if that is the desired behaviour +then letting low level code do it is the most efficient. + +In contrast if check is true, the scheme above allows the encoding to +do as much as it can and tell layer above how much that was. What is +lacking at present is a mechanism to report what went wrong. The most +likely interface will be an additional method call to the object, or +perhaps (to avoid forcing per-stream objects on otherwise stateless +encodings) and additional parameter. + +It is also highly desirable that encoding classes inherit from +C as a base class. This allows that class to define +additional behaviour for all encoding objects. For example built in +Unicode, UCS-2 and UTF-8 classes use : + + package Encode::MyEncoding; + use base qw(Encode::Encoding); + + __PACKAGE__->Define(qw(myCanonical myAlias)); + +To create an object with bless {Name => ...},$class, and call +define_encoding. They inherit their C method from +C. + +=head2 Compiled Encodings + +F provides a class C which provides the +interface described above. It calls a generic octet-sequence to +octet-sequence "engine" that is driven by tables (defined in +F). The same engine is used for both encode and +decode. C's C forces Perl's characters to their +UTF-8 form and then treats them as just another multibyte +encoding. C's C transforms the sequence and then +turns the UTF-8-ness flag as that is the form that the tables are +defined to produce. For details of the engine see the comments in +F. + +The tables are produced by the Perl script F (the name needs +to change so we can eventually install it somewhere). F can +currently read two formats: + +=over 4 + +=item *.enc + +This is a coined format used by Tcl. It is documented in +Encode/EncodeFormat.pod. + +=item *.ucm + +This is the semi-standard format used by IBM's ICU package. + +=back + +F can write the following forms: + +=over 4 + +=item *.ucm + +See above - the F files provided with the distribution have +been created from the original Tcl .enc files using this approach. + +=item *.c + +Produces tables as C data structures - this is used to build in encodings +into F/F. + +=item *.xs + +In theory this allows encodings to be stand-alone loadable Perl +extensions. The process has not yet been tested. The plan is to use +this approach for large East Asian encodings. + +=back + +The set of encodings built-in to F/F is +determined by F. The current set is as follows: + +=over 4 + +=item ascii and iso-8859-* + +That is all the common 8-bit "western" encodings. + +=item IBM-1047 and two other variants of EBCDIC. + +These are the same variants that are supported by EBCDIC Perl as +"native" encodings. They are included to prove "reversibility" of +some constructs in EBCDIC Perl. + +=item symbol and dingbats as used by Tk on X11. + +(The reason Encode got started was to support Perl/Tk.) + +=back + +That set is rather ad hoc and has been driven by the needs of the +tests rather than the needs of typical applications. It is likely +to be rationalized. + +=head1 SEE ALSO + +L, L, L, L, L, +L, the Perl Unicode Mailing List Eperl-unicode@perl.orgE + + +=cut + diff --git a/ext/Encode/lib/Encode/Encoding.pm b/ext/Encode/lib/Encode/Encoding.pm index e03e22d..3327fa7 100644 --- a/ext/Encode/lib/Encode/Encoding.pm +++ b/ext/Encode/lib/Encode/Encoding.pm @@ -1,7 +1,7 @@ package Encode::Encoding; # Base class for classes which implement encodings use strict; -our $VERSION = do { my @r = (q$Revision: 0.92 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +our $VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; sub Define { @@ -24,3 +24,178 @@ sub DESTROY {} 1; __END__ + +=head1 NAME + +Encode::Encoding - Encode Implementation Base Class + +=head1 SYNOPSIS + + package Encode::MyEncoding; + use base qw(Encode::Encoding); + + __PACKAGE__->Define(qw(myCanonical myAlias)); + +=head 1 DESCRIPTION + +As mentioned in L, encodings are (in the current +implementation at least) defined by objects. The mapping of encoding +name to object is via the C<%encodings> hash. + +The values of the hash can currently be either strings or objects. +The string form may go away in the future. The string form occurs +when C has scanned C<@INC> for loadable encodings but has +not actually loaded the encoding in question. This is because the +current "loading" process is all Perl and a bit slow. + +Once an encoding is loaded then value of the hash is object which +implements the encoding. The object should provide the following +interface: + +=over 4 + +=item -Ename + +Should return the string representing the canonical name of the encoding. + +=item -Enew_sequence + +This is a placeholder for encodings with state. It should return an +object which implements this interface, all current implementations +return the original object. + +=item -Eencode($string,$check) + +Should return the octet sequence representing I<$string>. If I<$check> +is true it should modify I<$string> in place to remove the converted +part (i.e. the whole string unless there is an error). If an error +occurs it should return the octet sequence for the fragment of string +that has been converted, and modify $string in-place to remove the +converted part leaving it starting with the problem fragment. + +If check is is false then C should make a "best effort" to +convert the string - for example by using a replacement character. + +=item -Edecode($octets,$check) + +Should return the string that I<$octets> represents. If I<$check> is +true it should modify I<$octets> in place to remove the converted part +(i.e. the whole sequence unless there is an error). If an error +occurs it should return the fragment of string that has been +converted, and modify $octets in-place to remove the converted part +leaving it starting with the problem fragment. + +If check is is false then C should make a "best effort" to +convert the string - for example by using Unicode's "\x{FFFD}" as a +replacement character. + +=back + +It should be noted that the check behaviour is different from the +outer public API. The logic is that the "unchecked" case is useful +when encoding is part of a stream which may be reporting errors +(e.g. STDERR). In such cases it is desirable to get everything +through somehow without causing additional errors which obscure the +original one. Also the encoding is best placed to know what the +correct replacement character is, so if that is the desired behaviour +then letting low level code do it is the most efficient. + +In contrast if check is true, the scheme above allows the encoding to +do as much as it can and tell layer above how much that was. What is +lacking at present is a mechanism to report what went wrong. The most +likely interface will be an additional method call to the object, or +perhaps (to avoid forcing per-stream objects on otherwise stateless +encodings) and additional parameter. + +It is also highly desirable that encoding classes inherit from +C as a base class. This allows that class to define +additional behaviour for all encoding objects. For example built in +Unicode, UCS-2 and UTF-8 classes use : + + package Encode::MyEncoding; + use base qw(Encode::Encoding); + + __PACKAGE__->Define(qw(myCanonical myAlias)); + +To create an object with bless {Name => ...},$class, and call +define_encoding. They inherit their C method from +C. + +=head2 Compiled Encodings + +F provides a class C which provides the +interface described above. It calls a generic octet-sequence to +octet-sequence "engine" that is driven by tables (defined in +F). The same engine is used for both encode and +decode. C's C forces Perl's characters to their +UTF-8 form and then treats them as just another multibyte +encoding. C's C transforms the sequence and then +turns the UTF-8-ness flag as that is the form that the tables are +defined to produce. For details of the engine see the comments in +F. + +The tables are produced by the Perl script F (the name needs +to change so we can eventually install it somewhere). F can +currently read two formats: + +=over 4 + +=item *.enc + +This is a coined format used by Tcl. It is documented in +Encode/EncodeFormat.pod. + +=item *.ucm + +This is the semi-standard format used by IBM's ICU package. + +=back + +F can write the following forms: + +=over 4 + +=item *.ucm + +See above - the F files provided with the distribution have +been created from the original Tcl .enc files using this approach. + +=item *.c + +Produces tables as C data structures - this is used to build in encodings +into F/F. + +=item *.xs + +In theory this allows encodings to be stand-alone loadable Perl +extensions. The process has not yet been tested. The plan is to use +this approach for large East Asian encodings. + +=back + +The set of encodings built-in to F/F is +determined by F. The current set is as follows: + +=over 4 + +=item ascii and iso-8859-* + +That is all the common 8-bit "western" encodings. + +=item IBM-1047 and two other variants of EBCDIC. + +These are the same variants that are supported by EBCDIC Perl as +"native" encodings. They are included to prove "reversibility" of +some constructs in EBCDIC Perl. + +=item symbol and dingbats as used by Tk on X11. + +(The reason Encode got started was to support Perl/Tk.) + +=back + +That set is rather ad hoc and has been driven by the needs of the +tests rather than the needs of typical applications. It is likely +to be rationalized. + +=cut diff --git a/ext/Encode/lib/Encode/JP/ISO_2022_JP.pm b/ext/Encode/lib/Encode/JP/ISO_2022_JP.pm index 30b06b8..388be5f 100644 --- a/ext/Encode/lib/Encode/JP/ISO_2022_JP.pm +++ b/ext/Encode/lib/Encode/JP/ISO_2022_JP.pm @@ -5,12 +5,14 @@ use Encode::JP::H2Z; use base 'Encode::Encoding'; use vars qw($VERSION); -$VERSION = do { my @r = (q$Revision: 0.92 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +$VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; my $canon = 'iso-2022-jp'; my $obj = bless {name => $canon}, __PACKAGE__; $obj->Define($canon); +sub name { return $_[0]->{name}; } + # # decode is identical to 7bit-jis # diff --git a/ext/Encode/lib/Encode/JP/JIS.pm b/ext/Encode/lib/Encode/JP/JIS.pm index 005ca52..6e6dd0f 100644 --- a/ext/Encode/lib/Encode/JP/JIS.pm +++ b/ext/Encode/lib/Encode/JP/JIS.pm @@ -5,7 +5,7 @@ use base 'Encode::Encoding'; use strict; use vars qw($VERSION); -$VERSION = do { my @r = (q$Revision: 0.92 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +$VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; # Just for the time being, we implement jis-7bit # encoding via EUC @@ -14,6 +14,8 @@ my $canon = '7bit-jis'; my $obj = bless {name => $canon}, __PACKAGE__; $obj->Define($canon); +sub name { return $_[0]->{name}; } + sub decode { my ($obj,$str,$chk) = @_; diff --git a/ext/Encode/lib/Encode/Tcl.pm b/ext/Encode/lib/Encode/Tcl.pm index b5b93f9..d47ed11 100644 --- a/ext/Encode/lib/Encode/Tcl.pm +++ b/ext/Encode/lib/Encode/Tcl.pm @@ -5,7 +5,7 @@ BEGIN { } } use strict; -our $VERSION = do { my @r = (q$Revision: 0.92 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +our $VERSION = do { my @r = (q$Revision: 0.94 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; use Encode qw(find_encoding); use base 'Encode::Encoding'; @@ -158,6 +158,4 @@ L L -L - =cut diff --git a/ext/Encode/t/CJKalias.t b/ext/Encode/t/CJKalias.t index be4dd5e..160f5ab 100644 --- a/ext/Encode/t/CJKalias.t +++ b/ext/Encode/t/CJKalias.t @@ -1,5 +1,5 @@ use strict; -#use Test::More tests => 27; +#use Test::More tests => 29; use Test::More qw(no_plan); use Encode::CN; use Encode::JP; @@ -19,11 +19,13 @@ my %a2c = qw( Shift_JIS shiftjis x-sjis shiftjis jis 7bit-jis + big-5 big5 + zh_TW.Big5 big5 + big5-hk big5-hkscs ); foreach my $a (keys %a2c){ my $e = Encode::find_encoding($a); - my $n = $e->name || $e->{name}; - is($n, $a2c{$a}); + is($e->name, $a2c{$a}); } -- 1.8.3.1