This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Remove bad advice : -M doesn't work on the #! line
[perl5.git] / ext / Encode / encoding.pm
CommitLineData
3ef515df 1package encoding;
621b0f8d 2our $VERSION = do { my @r = (q$Revision: 1.35 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r };
3ef515df
JH
3
4use Encode;
046f36bf 5use strict;
3ef515df
JH
6
7BEGIN {
8 if (ord("A") == 193) {
9 require Carp;
10c5ecbb 10 Carp::croak("encoding pragma does not support EBCDIC platforms");
3ef515df
JH
11 }
12}
13
0ab8f81e
JH
14our $HAS_PERLIO = 0;
15eval { require PerlIO::encoding };
16unless ($@){
17 $HAS_PERLIO = (PerlIO::encoding->VERSION >= 0.02);
18}
b2704119 19
3ef515df
JH
20sub import {
21 my $class = shift;
22 my $name = shift;
23 my %arg = @_;
24 $name ||= $ENV{PERL_ENCODING};
25
26 my $enc = find_encoding($name);
27 unless (defined $enc) {
28 require Carp;
10c5ecbb 29 Carp::croak("Unknown encoding '$name'");
3ef515df 30 }
aae85ceb
DK
31 unless ($arg{Filter}){
32 ${^ENCODING} = $enc; # this is all you need, actually.
85982a32 33 $HAS_PERLIO or return 1;
aae85ceb
DK
34 for my $h (qw(STDIN STDOUT)){
35 if ($arg{$h}){
b2704119 36 unless (defined find_encoding($arg{$h})) {
aae85ceb 37 require Carp;
10c5ecbb 38 Carp::croak("Unknown encoding for $h, '$arg{$h}'");
aae85ceb 39 }
0ab8f81e 40 eval { binmode($h, ":encoding($arg{$h})") };
aae85ceb
DK
41 }else{
42 unless (exists $arg{$h}){
0ab8f81e
JH
43 eval {
44 no warnings 'uninitialized';
45 binmode($h, ":encoding($name)");
46 };
aae85ceb
DK
47 }
48 }
49 if ($@){
3ef515df 50 require Carp;
aae85ceb 51 Carp::croak($@);
3ef515df 52 }
3ef515df 53 }
aae85ceb
DK
54 }else{
55 defined(${^ENCODING}) and undef ${^ENCODING};
56 eval {
57 require Filter::Util::Call ;
58 Filter::Util::Call->import ;
b2704119
JH
59 binmode(STDIN);
60 binmode(STDOUT);
aae85ceb
DK
61 filter_add(sub{
62 my $status;
63 if (($status = filter_read()) > 0){
64 $_ = $enc->decode($_, 1);
65 # warn $_;
66 }
67 $status ;
68 });
69 };
70 # warn "Filter installed";
3ef515df
JH
71 }
72 return 1; # I doubt if we need it, though
73}
74
75sub unimport{
76 no warnings;
77 undef ${^ENCODING};
621b0f8d
DK
78 if ($HAS_PERLIO){
79 binmode(STDIN, ":raw");
80 binmode(STDOUT, ":raw");
81 }else{
b2704119
JH
82 binmode(STDIN);
83 binmode(STDOUT);
621b0f8d 84 }
aae85ceb
DK
85 if ($INC{"Filter/Util/Call.pm"}){
86 eval { filter_del() };
87 }
3ef515df
JH
88}
89
901;
91__END__
85982a32 92
3ef515df
JH
93=pod
94
95=head1 NAME
96
0ab8f81e 97encoding - allows you to write your script in non-ascii or non-utf8
3ef515df
JH
98
99=head1 SYNOPSIS
100
962111ca 101 use encoding "greek"; # Perl like Greek to you?
3ef515df
JH
102 use encoding "euc-jp"; # Jperl!
103
962111ca 104 # or you can even do this if your shell supports your native encoding
3ef515df 105
962111ca 106 perl -Mencoding=latin2 -e '...' # Feeling centrally European?
0ab8f81e 107 perl -Mencoding=euc-kr -e '...' # Or Korean?
3ef515df 108
3ef515df
JH
109 # more control
110
962111ca 111 # A simple euc-cn => utf-8 converter
6d1c0808 112 use encoding "euc-cn", STDOUT => "utf8"; while(<>){print};
3ef515df
JH
113
114 # "no encoding;" supported (but not scoped!)
115 no encoding;
116
aae85ceb
DK
117 # an alternate way, Filter
118 use encoding "euc-jp", Filter=>1;
119 use utf8;
120 # now you can use kanji identifiers -- in euc-jp!
121
3ef515df
JH
122=head1 ABSTRACT
123
962111ca
JH
124Let's start with a bit of history: Perl 5.6.0 introduced Unicode
125support. You could apply C<substr()> and regexes even to complex CJK
126characters -- so long as the script was written in UTF-8. But back
0ab8f81e
JH
127then, text editors that supported UTF-8 were still rare and many users
128instead chose to write scripts in legacy encodings, giving up a whole
129new feature of Perl 5.6.
3ef515df 130
0ab8f81e 131Rewind to the future: starting from perl 5.8.0 with the B<encoding>
962111ca
JH
132pragma, you can write your script in any encoding you like (so long
133as the C<Encode> module supports it) and still enjoy Unicode support.
0ab8f81e 134You can write code in EUC-JP as follows:
3ef515df
JH
135
136 my $Rakuda = "\xF1\xD1\xF1\xCC"; # Camel in Kanji
137 #<-char-><-char-> # 4 octets
138 s/\bCamel\b/$Rakuda/;
139
140And with C<use encoding "euc-jp"> in effect, it is the same thing as
962111ca 141the code in UTF-8:
3ef515df 142
32b9ed1f 143 my $Rakuda = "\x{99F1}\x{99DD}"; # two Unicode Characters
3ef515df
JH
144 s/\bCamel\b/$Rakuda/;
145
962111ca 146The B<encoding> pragma also modifies the filehandle disciplines of
3ef515df
JH
147STDIN, STDOUT, and STDERR to the specified encoding. Therefore,
148
149 use encoding "euc-jp";
150 my $message = "Camel is the symbol of perl.\n";
151 my $Rakuda = "\xF1\xD1\xF1\xCC"; # Camel in Kanji
152 $message =~ s/\bCamel\b/$Rakuda/;
153 print $message;
154
962111ca
JH
155Will print "\xF1\xD1\xF1\xCC is the symbol of perl.\n",
156not "\x{99F1}\x{99DD} is the symbol of perl.\n".
3ef515df 157
0ab8f81e 158You can override this by giving extra arguments; see below.
3ef515df
JH
159
160=head1 USAGE
161
162=over 4
163
164=item use encoding [I<ENCNAME>] ;
165
0ab8f81e
JH
166Sets the script encoding to I<ENCNAME>. Filehandle disciplines of
167STDIN and STDOUT are set to ":encoding(I<ENCNAME>)". Note that STDERR
168will not be changed.
3ef515df
JH
169
170If no encoding is specified, the environment variable L<PERL_ENCODING>
962111ca
JH
171is consulted. If no encoding can be found, the error C<Unknown encoding
172'I<ENCNAME>'> will be thrown.
3ef515df
JH
173
174Note that non-STD file handles remain unaffected. Use C<use open> or
175C<binmode> to change disciplines of those.
176
aae85ceb 177=item use encoding I<ENCNAME> [ STDIN =E<gt> I<ENCNAME_IN> ...] ;
3ef515df 178
0ab8f81e 179You can also individually set encodings of STDIN and STDOUT via the
32b9ed1f
A
180C<< STDIN => I<ENCNAME> >> form. In this case, you cannot omit the
181first I<ENCNAME>. C<< STDIN => undef >> turns the IO transcoding
aae85ceb 182completely off.
3ef515df
JH
183
184=item no encoding;
185
0ab8f81e 186Unsets the script encoding. The disciplines of STDIN, STDOUT are
962111ca 187reset to ":raw" (the default unprocessed raw stream of bytes).
3ef515df
JH
188
189=back
190
191=head1 CAVEATS
192
193=head2 NOT SCOPED
194
195The pragma is a per script, not a per block lexical. Only the last
621b0f8d
DK
196C<use encoding> or C<no encoding> matters, and it affects
197B<the whole script>. However, the <no encoding> pragma is supported and
198B<use encoding> can appear as many times as you want in a given script.
199The multiple use of this pragma is discouraged.
200
201Because of this nature, the use of this pragma inside the module is
202strongly discouraged (because the influence of this pragma lasts not
203only for the module but the script that uses). But if you have to,
204make sure you say C<no encoding> at the end of the module so you
205contain the influence of the pragma within the module.
3ef515df
JH
206
207=head2 DO NOT MIX MULTIPLE ENCODINGS
208
209Notice that only literals (string or regular expression) having only
210legacy code points are affected: if you mix data like this
211
212 \xDF\x{100}
213
214the data is assumed to be in (Latin 1 and) Unicode, not in your native
215encoding. In other words, this will match in "greek":
216
217 "\xDF" =~ /\x{3af}/
218
219but this will not
220
221 "\xDF\x{100}" =~ /\x{3af}\x{100}/
222
962111ca
JH
223since the C<\xDF> (ISO 8859-7 GREEK SMALL LETTER IOTA WITH TONOS) on
224the left will B<not> be upgraded to C<\x{3af}> (Unicode GREEK SMALL
225LETTER IOTA WITH TONOS) because of the C<\x{100}> on the left. You
226should not be mixing your legacy data and Unicode in the same string.
3ef515df
JH
227
228This pragma also affects encoding of the 0x80..0xFF code point range:
229normally characters in that range are left as eight-bit bytes (unless
230they are combined with characters with code points 0x100 or larger,
231in which case all characters need to become UTF-8 encoded), but if
232the C<encoding> pragma is present, even the 0x80..0xFF range always
233gets UTF-8 encoded.
234
235After all, the best thing about this pragma is that you don't have to
0ab8f81e
JH
236resort to \x{....} just to spell your name in a native encoding.
237So feel free to put your strings in your encoding in quotes and
238regexes.
3ef515df 239
962111ca 240=head1 Non-ASCII Identifiers and Filter option
aae85ceb 241
962111ca 242The magic of C<use encoding> is not applied to the names of
32b9ed1f 243identifiers. In order to make C<${"\x{4eba}"}++> ($human++, where human
962111ca
JH
244is a single Han ideograph) work, you still need to write your script
245in UTF-8 or use a source filter.
aae85ceb 246
0ab8f81e 247In other words, the same restriction as with Jperl applies.
aae85ceb 248
0ab8f81e 249If you dare to experiment, however, you can try the Filter option.
aae85ceb
DK
250
251=over 4
252
253=item use encoding I<ENCNAME> Filter=E<gt>1;
254
0ab8f81e 255This turns the encoding pragma into a source filter. While the default
aae85ceb 256approach just decodes interpolated literals (in qq() and qr()), this
0ab8f81e
JH
257will apply a source filter to the entire source code. In this case,
258STDIN and STDOUT remain untouched.
aae85ceb
DK
259
260=back
261
962111ca 262What does this mean? Your source code behaves as if it is written in
0ab8f81e
JH
263UTF-8. So even if your editor only supports Shift_JIS, for example,
264you can still try examples in Chapter 15 of C<Programming Perl, 3rd
265Ed.>. For instance, you can use UTF-8 identifiers.
aae85ceb
DK
266
267This option is significantly slower and (as of this writing) non-ASCII
268identifiers are not very stable WITHOUT this option and with the
269source code written in UTF-8.
270
962111ca
JH
271To make your script in legacy encoding work with minimum effort,
272do not use Filter=E<gt>1.
aae85ceb 273
3ef515df
JH
274=head1 EXAMPLE - Greekperl
275
276 use encoding "iso 8859-7";
277
0ab8f81e 278 # \xDF in ISO 8859-7 (Greek) is \x{3af} in Unicode.
3ef515df
JH
279
280 $a = "\xDF";
281 $b = "\x{100}";
282
283 printf "%#x\n", ord($a); # will print 0x3af, not 0xdf
284
285 $c = $a . $b;
286
287 # $c will be "\x{3af}\x{100}", not "\x{df}\x{100}".
288
289 # chr() is affected, and ...
290
291 print "mega\n" if ord(chr(0xdf)) == 0x3af;
292
293 # ... ord() is affected by the encoding pragma ...
294
295 print "tera\n" if ord(pack("C", 0xdf)) == 0x3af;
296
297 # ... as are eq and cmp ...
298
299 print "peta\n" if "\x{3af}" eq pack("C", 0xdf);
300 print "exa\n" if "\x{3af}" cmp pack("C", 0xdf) == 0;
301
302 # ... but pack/unpack C are not affected, in case you still
0ab8f81e 303 # want to go back to your native encoding
3ef515df
JH
304
305 print "zetta\n" if unpack("C", (pack("C", 0xdf))) == 0xdf;
306
307=head1 KNOWN PROBLEMS
308
0ab8f81e 309For native multibyte encodings (either fixed or variable length),
3ef515df 310the current implementation of the regular expressions may introduce
0ab8f81e 311recoding errors for regular expression literals longer than 127 bytes.
3ef515df
JH
312
313The encoding pragma is not supported on EBCDIC platforms.
0ab8f81e
JH
314(Porters who are willing and able to remove this limitation are
315welcome.)
3ef515df
JH
316
317=head1 SEE ALSO
318
aae85ceb
DK
319L<perlunicode>, L<Encode>, L<open>, L<Filter::Util::Call>,
320
321Ch. 15 of C<Programming Perl (3rd Edition)>
322by Larry Wall, Tom Christiansen, Jon Orwant;
323O'Reilly & Associates; ISBN 0-596-00027-8
3ef515df
JH
324
325=cut