This is a live mirror of the Perl 5 development currently hosted at https://github.com/perl/perl5
Switch Storable to IO::File from FileHandle, only load if needed
[perl5.git] / dist / Storable / Storable.pm
CommitLineData
25f64a11
AMS
1#
2# Copyright (c) 1995-2000, Raphael Manfredi
3#
4# You may redistribute only under the same terms as Perl 5, as specified
5# in the README file that comes with the distribution.
6#
7a6a85bf 7
71a48636 8require XSLoader;
7a6a85bf 9require Exporter;
71a48636 10package Storable; @ISA = qw(Exporter);
7a6a85bf
RG
11
12@EXPORT = qw(store retrieve);
13@EXPORT_OK = qw(
9e21b3d0 14 nstore store_fd nstore_fd fd_retrieve
7a6a85bf
RG
15 freeze nfreeze thaw
16 dclone
9e21b3d0 17 retrieve_fd
dd19458b 18 lock_store lock_nstore lock_retrieve
d4b9b6e4 19 file_magic read_magic
7a6a85bf
RG
20);
21
01d7b99e 22use vars qw($canonical $forgive_me $VERSION);
7a6a85bf 23
71a48636 24$VERSION = '2.27';
7a6a85bf 25
1541ec63
DL
26BEGIN {
27 if (eval { local $SIG{__DIE__}; require Log::Agent; 1 }) {
28 Log::Agent->import;
29 }
30 #
31 # Use of Log::Agent is optional. If it hasn't imported these subs then
32 # provide a fallback implementation.
33 #
34 else {
35 require Carp;
7a6a85bf 36
1541ec63
DL
37 *logcroak = sub {
38 Carp::croak(@_);
39 };
7a6a85bf 40
1541ec63
DL
41 *logcarp = sub {
42 Carp::carp(@_);
43 };
44 }
45}
7a6a85bf 46
dd19458b
JH
47#
48# They might miss :flock in Fcntl
49#
50
51BEGIN {
596596d5 52 if (eval { require Fcntl; 1 } && exists $Fcntl::EXPORT_TAGS{'flock'}) {
dd19458b
JH
53 Fcntl->import(':flock');
54 } else {
55 eval q{
56 sub LOCK_SH () {1}
57 sub LOCK_EX () {2}
58 };
59 }
60}
61
a8b7ef86
AMS
62sub CLONE {
63 # clone context under threads
64 Storable::init_perinterp();
65}
66
530b72ba
NC
67# By default restricted hashes are downgraded on earlier perls.
68
69$Storable::downgrade_restricted = 1;
e8189732 70$Storable::accept_future_minor = 1;
71a48636
DL
71
72XSLoader::load 'Storable', $Storable::VERSION;
b8778c7c 73
862382c7
JH
74#
75# Determine whether locking is possible, but only when needed.
76#
77
530b72ba 78sub CAN_FLOCK; my $CAN_FLOCK; sub CAN_FLOCK {
862382c7
JH
79 return $CAN_FLOCK if defined $CAN_FLOCK;
80 require Config; import Config;
81 return $CAN_FLOCK =
82 $Config{'d_flock'} ||
83 $Config{'d_fcntl_can_lock'} ||
84 $Config{'d_lockf'};
85}
86
0a0da639
JH
87sub show_file_magic {
88 print <<EOM;
89#
90# To recognize the data files of the Perl module Storable,
91# the following lines need to be added to the local magic(5) file,
92# usually either /usr/share/misc/magic or /etc/magic.
0a0da639
JH
93#
940 string perl-store perl Storable(v0.6) data
8b793558
JH
95>4 byte >0 (net-order %d)
96>>4 byte &01 (network-ordered)
97>>4 byte =3 (major 1)
98>>4 byte =2 (major 1)
99
0a0da639 1000 string pst0 perl Storable(v0.7) data
8b793558
JH
101>4 byte >0
102>>4 byte &01 (network-ordered)
103>>4 byte =5 (major 2)
104>>4 byte =4 (major 2)
105>>5 byte >0 (minor %d)
0a0da639
JH
106EOM
107}
108
d4b9b6e4 109sub file_magic {
ed0d1802
DL
110 require IO::File;
111
d4b9b6e4 112 my $file = shift;
ed0d1802 113 my $fh = IO::File->new;
ab6f8ca1 114 open($fh, "<". $file) || die "Can't open '$file': $!";
d4b9b6e4
GA
115 binmode($fh);
116 defined(sysread($fh, my $buf, 32)) || die "Can't read from '$file': $!";
117 close($fh);
118
119 $file = "./$file" unless $file; # ensure TRUE value
120
121 return read_magic($buf, $file);
122}
123
b8778c7c 124sub read_magic {
d4b9b6e4
GA
125 my($buf, $file) = @_;
126 my %info;
127
128 my $buflen = length($buf);
129 my $magic;
130 if ($buf =~ s/^(pst0|perl-store)//) {
131 $magic = $1;
132 $info{file} = $file || 1;
133 }
134 else {
135 return undef if $file;
136 $magic = "";
137 }
138
139 return undef unless length($buf);
140
141 my $net_order;
142 if ($magic eq "perl-store" && ord(substr($buf, 0, 1)) > 1) {
143 $info{version} = -1;
144 $net_order = 0;
145 }
146 else {
5e12106f
NC
147 $buf =~ s/(.)//s;
148 my $major = (ord $1) >> 1;
d4b9b6e4
GA
149 return undef if $major > 4; # sanity (assuming we never go that high)
150 $info{major} = $major;
5e12106f 151 $net_order = (ord $1) & 0x01;
d4b9b6e4 152 if ($major > 1) {
5e12106f
NC
153 return undef unless $buf =~ s/(.)//s;
154 my $minor = ord $1;
d4b9b6e4
GA
155 $info{minor} = $minor;
156 $info{version} = "$major.$minor";
157 $info{version_nv} = sprintf "%d.%03d", $major, $minor;
158 }
159 else {
160 $info{version} = $major;
161 }
162 }
163 $info{version_nv} ||= $info{version};
164 $info{netorder} = $net_order;
165
166 unless ($net_order) {
5e12106f
NC
167 return undef unless $buf =~ s/(.)//s;
168 my $len = ord $1;
d4b9b6e4
GA
169 return undef unless length($buf) >= $len;
170 return undef unless $len == 4 || $len == 8; # sanity
5e12106f
NC
171 @info{qw(byteorder intsize longsize ptrsize)}
172 = unpack "a${len}CCC", $buf;
173 (substr $buf, 0, $len + 3) = '';
d4b9b6e4 174 if ($info{version_nv} >= 2.002) {
5e12106f
NC
175 return undef unless $buf =~ s/(.)//s;
176 $info{nvsize} = ord $1;
d4b9b6e4
GA
177 }
178 }
179 $info{hdrsize} = $buflen - length($buf);
180
181 return \%info;
182}
183
184sub BIN_VERSION_NV {
185 sprintf "%d.%03d", BIN_MAJOR(), BIN_MINOR();
186}
187
188sub BIN_WRITE_VERSION_NV {
189 sprintf "%d.%03d", BIN_MAJOR(), BIN_WRITE_MINOR();
b8778c7c 190}
7a6a85bf
RG
191
192#
193# store
194#
195# Store target object hierarchy, identified by a reference to its root.
196# The stored object tree may later be retrieved to memory via retrieve.
197# Returns undef if an I/O error occurred, in which case the file is
198# removed.
199#
200sub store {
dd19458b 201 return _store(\&pstore, @_, 0);
7a6a85bf
RG
202}
203
204#
205# nstore
206#
207# Same as store, but in network order.
208#
209sub nstore {
dd19458b
JH
210 return _store(\&net_pstore, @_, 0);
211}
212
213#
214# lock_store
215#
216# Same as store, but flock the file first (advisory locking).
217#
218sub lock_store {
219 return _store(\&pstore, @_, 1);
220}
221
222#
223# lock_nstore
224#
225# Same as nstore, but flock the file first (advisory locking).
226#
227sub lock_nstore {
228 return _store(\&net_pstore, @_, 1);
7a6a85bf
RG
229}
230
231# Internal store to file routine
232sub _store {
233 my $xsptr = shift;
234 my $self = shift;
dd19458b 235 my ($file, $use_locking) = @_;
7a6a85bf 236 logcroak "not a reference" unless ref($self);
b12202d0 237 logcroak "wrong argument number" unless @_ == 2; # No @foo in arglist
7a6a85bf 238 local *FILE;
dd19458b 239 if ($use_locking) {
6e0ac6f5 240 open(FILE, ">>$file") || logcroak "can't write into $file: $!";
862382c7 241 unless (&CAN_FLOCK) {
b29b780f
RM
242 logcarp "Storable::lock_store: fcntl/flock emulation broken on $^O";
243 return undef;
f567092b 244 }
dd19458b
JH
245 flock(FILE, LOCK_EX) ||
246 logcroak "can't get exclusive lock on $file: $!";
247 truncate FILE, 0;
248 # Unlocking will happen when FILE is closed
6e0ac6f5
JH
249 } else {
250 open(FILE, ">$file") || logcroak "can't create $file: $!";
dd19458b 251 }
6e0ac6f5 252 binmode FILE; # Archaic systems...
7a6a85bf
RG
253 my $da = $@; # Don't mess if called from exception handler
254 my $ret;
255 # Call C routine nstore or pstore, depending on network order
256 eval { $ret = &$xsptr(*FILE, $self) };
9c09e8a7
NC
257 # close will return true on success, so the or short-circuits, the ()
258 # expression is true, and for that case the block will only be entered
259 # if $@ is true (ie eval failed)
260 # if close fails, it returns false, $ret is altered, *that* is (also)
261 # false, so the () expression is false, !() is true, and the block is
262 # entered.
cbc736f3 263 if (!(close(FILE) or undef $ret) || $@) {
9c09e8a7
NC
264 unlink($file) or warn "Can't unlink $file: $!\n";
265 }
7a6a85bf
RG
266 logcroak $@ if $@ =~ s/\.?\n$/,/;
267 $@ = $da;
cbc736f3 268 return $ret;
7a6a85bf
RG
269}
270
271#
272# store_fd
273#
274# Same as store, but perform on an already opened file descriptor instead.
275# Returns undef if an I/O error occurred.
276#
277sub store_fd {
278 return _store_fd(\&pstore, @_);
279}
280
281#
282# nstore_fd
283#
284# Same as store_fd, but in network order.
285#
286sub nstore_fd {
287 my ($self, $file) = @_;
288 return _store_fd(\&net_pstore, @_);
289}
290
291# Internal store routine on opened file descriptor
292sub _store_fd {
293 my $xsptr = shift;
294 my $self = shift;
295 my ($file) = @_;
296 logcroak "not a reference" unless ref($self);
297 logcroak "too many arguments" unless @_ == 1; # No @foo in arglist
298 my $fd = fileno($file);
299 logcroak "not a valid file descriptor" unless defined $fd;
300 my $da = $@; # Don't mess if called from exception handler
301 my $ret;
302 # Call C routine nstore or pstore, depending on network order
303 eval { $ret = &$xsptr($file, $self) };
304 logcroak $@ if $@ =~ s/\.?\n$/,/;
596596d5 305 local $\; print $file ''; # Autoflush the file if wanted
7a6a85bf 306 $@ = $da;
cbc736f3 307 return $ret;
7a6a85bf
RG
308}
309
310#
311# freeze
312#
313# Store oject and its hierarchy in memory and return a scalar
314# containing the result.
315#
316sub freeze {
317 _freeze(\&mstore, @_);
318}
319
320#
321# nfreeze
322#
323# Same as freeze but in network order.
324#
325sub nfreeze {
326 _freeze(\&net_mstore, @_);
327}
328
329# Internal freeze routine
330sub _freeze {
331 my $xsptr = shift;
332 my $self = shift;
333 logcroak "not a reference" unless ref($self);
334 logcroak "too many arguments" unless @_ == 0; # No @foo in arglist
335 my $da = $@; # Don't mess if called from exception handler
336 my $ret;
337 # Call C routine mstore or net_mstore, depending on network order
338 eval { $ret = &$xsptr($self) };
339 logcroak $@ if $@ =~ s/\.?\n$/,/;
340 $@ = $da;
341 return $ret ? $ret : undef;
342}
343
344#
345# retrieve
346#
347# Retrieve object hierarchy from disk, returning a reference to the root
348# object of that tree.
349#
350sub retrieve {
dd19458b
JH
351 _retrieve($_[0], 0);
352}
353
354#
355# lock_retrieve
356#
357# Same as retrieve, but with advisory locking.
358#
359sub lock_retrieve {
360 _retrieve($_[0], 1);
361}
362
363# Internal retrieve routine
364sub _retrieve {
365 my ($file, $use_locking) = @_;
7a6a85bf 366 local *FILE;
dd19458b 367 open(FILE, $file) || logcroak "can't open $file: $!";
7a6a85bf
RG
368 binmode FILE; # Archaic systems...
369 my $self;
370 my $da = $@; # Could be from exception handler
dd19458b 371 if ($use_locking) {
862382c7 372 unless (&CAN_FLOCK) {
8be2b38b 373 logcarp "Storable::lock_store: fcntl/flock emulation broken on $^O";
b29b780f
RM
374 return undef;
375 }
8be2b38b 376 flock(FILE, LOCK_SH) || logcroak "can't get shared lock on $file: $!";
dd19458b
JH
377 # Unlocking will happen when FILE is closed
378 }
7a6a85bf
RG
379 eval { $self = pretrieve(*FILE) }; # Call C routine
380 close(FILE);
381 logcroak $@ if $@ =~ s/\.?\n$/,/;
382 $@ = $da;
383 return $self;
384}
385
386#
9e21b3d0 387# fd_retrieve
7a6a85bf
RG
388#
389# Same as retrieve, but perform from an already opened file descriptor instead.
390#
9e21b3d0 391sub fd_retrieve {
7a6a85bf
RG
392 my ($file) = @_;
393 my $fd = fileno($file);
394 logcroak "not a valid file descriptor" unless defined $fd;
395 my $self;
396 my $da = $@; # Could be from exception handler
397 eval { $self = pretrieve($file) }; # Call C routine
398 logcroak $@ if $@ =~ s/\.?\n$/,/;
399 $@ = $da;
400 return $self;
401}
402
1541ec63
DL
403sub retrieve_fd { &fd_retrieve } # Backward compatibility
404
7a6a85bf
RG
405#
406# thaw
407#
408# Recreate objects in memory from an existing frozen image created
409# by freeze. If the frozen image passed is undef, return undef.
410#
411sub thaw {
412 my ($frozen) = @_;
413 return undef unless defined $frozen;
414 my $self;
415 my $da = $@; # Could be from exception handler
416 eval { $self = mretrieve($frozen) }; # Call C routine
417 logcroak $@ if $@ =~ s/\.?\n$/,/;
418 $@ = $da;
419 return $self;
420}
421
a2307be4
NC
4221;
423__END__
424
7a6a85bf
RG
425=head1 NAME
426
f062ea6c 427Storable - persistence for Perl data structures
7a6a85bf
RG
428
429=head1 SYNOPSIS
430
431 use Storable;
432 store \%table, 'file';
433 $hashref = retrieve('file');
434
435 use Storable qw(nstore store_fd nstore_fd freeze thaw dclone);
436
437 # Network order
438 nstore \%table, 'file';
439 $hashref = retrieve('file'); # There is NO nretrieve()
440
441 # Storing to and retrieving from an already opened file
442 store_fd \@array, \*STDOUT;
443 nstore_fd \%table, \*STDOUT;
9e21b3d0
JH
444 $aryref = fd_retrieve(\*SOCKET);
445 $hashref = fd_retrieve(\*SOCKET);
7a6a85bf
RG
446
447 # Serializing to memory
448 $serialized = freeze \%table;
449 %table_clone = %{ thaw($serialized) };
450
451 # Deep (recursive) cloning
452 $cloneref = dclone($ref);
453
dd19458b
JH
454 # Advisory locking
455 use Storable qw(lock_store lock_nstore lock_retrieve)
456 lock_store \%table, 'file';
457 lock_nstore \%table, 'file';
458 $hashref = lock_retrieve('file');
459
7a6a85bf
RG
460=head1 DESCRIPTION
461
f062ea6c 462The Storable package brings persistence to your Perl data structures
7a6a85bf 463containing SCALAR, ARRAY, HASH or REF objects, i.e. anything that can be
c261f00e 464conveniently stored to disk and retrieved at a later time.
7a6a85bf
RG
465
466It can be used in the regular procedural way by calling C<store> with
467a reference to the object to be stored, along with the file name where
468the image should be written.
775ecd75 469
7a6a85bf
RG
470The routine returns C<undef> for I/O problems or other internal error,
471a true value otherwise. Serious errors are propagated as a C<die> exception.
472
f062ea6c
PN
473To retrieve data stored to disk, use C<retrieve> with a file name.
474The objects stored into that file are recreated into memory for you,
475and a I<reference> to the root object is returned. In case an I/O error
7a6a85bf
RG
476occurs while reading, C<undef> is returned instead. Other serious
477errors are propagated via C<die>.
478
479Since storage is performed recursively, you might want to stuff references
480to objects that share a lot of common data into a single array or hash
481table, and then store that object. That way, when you retrieve back the
482whole thing, the objects will continue to share what they originally shared.
483
484At the cost of a slight header overhead, you may store to an already
485opened file descriptor using the C<store_fd> routine, and retrieve
9e21b3d0 486from a file via C<fd_retrieve>. Those names aren't imported by default,
c261f00e 487so you will have to do that explicitly if you need those routines.
7a6a85bf
RG
488The file descriptor you supply must be already opened, for read
489if you're going to retrieve and for write if you wish to store.
490
491 store_fd(\%table, *STDOUT) || die "can't store to stdout\n";
9e21b3d0 492 $hashref = fd_retrieve(*STDIN);
7a6a85bf
RG
493
494You can also store data in network order to allow easy sharing across
495multiple platforms, or when storing on a socket known to be remotely
496connected. The routines to call have an initial C<n> prefix for I<network>,
497as in C<nstore> and C<nstore_fd>. At retrieval time, your data will be
498correctly restored so you don't have to know whether you're restoring
dd19458b
JH
499from native or network ordered data. Double values are stored stringified
500to ensure portability as well, at the slight risk of loosing some precision
501in the last decimals.
7a6a85bf 502
9e21b3d0 503When using C<fd_retrieve>, objects are retrieved in sequence, one
7a6a85bf
RG
504object (i.e. one recursive tree) per associated C<store_fd>.
505
506If you're more from the object-oriented camp, you can inherit from
507Storable and directly store your objects by invoking C<store> as
508a method. The fact that the root of the to-be-stored tree is a
509blessed reference (i.e. an object) is special-cased so that the
510retrieve does not provide a reference to that object but rather the
511blessed object reference itself. (Otherwise, you'd get a reference
512to that blessed object).
513
514=head1 MEMORY STORE
515
516The Storable engine can also store data into a Perl scalar instead, to
517later retrieve them. This is mainly used to freeze a complex structure in
518some safe compact memory place (where it can possibly be sent to another
519process via some IPC, since freezing the structure also serializes it in
520effect). Later on, and maybe somewhere else, you can thaw the Perl scalar
521out and recreate the original complex structure in memory.
522
523Surprisingly, the routines to be called are named C<freeze> and C<thaw>.
524If you wish to send out the frozen scalar to another machine, use
525C<nfreeze> instead to get a portable image.
526
527Note that freezing an object structure and immediately thawing it
528actually achieves a deep cloning of that structure:
529
530 dclone(.) = thaw(freeze(.))
531
532Storable provides you with a C<dclone> interface which does not create
533that intermediary scalar but instead freezes the structure in some
c261f00e 534internal memory space and then immediately thaws it out.
7a6a85bf 535
dd19458b
JH
536=head1 ADVISORY LOCKING
537
f062ea6c
PN
538The C<lock_store> and C<lock_nstore> routine are equivalent to
539C<store> and C<nstore>, except that they get an exclusive lock on
540the file before writing. Likewise, C<lock_retrieve> does the same
541as C<retrieve>, but also gets a shared lock on the file before reading.
dd19458b 542
f062ea6c
PN
543As with any advisory locking scheme, the protection only works if you
544systematically use C<lock_store> and C<lock_retrieve>. If one side of
545your application uses C<store> whilst the other uses C<lock_retrieve>,
dd19458b
JH
546you will get no protection at all.
547
f062ea6c
PN
548The internal advisory locking is implemented using Perl's flock()
549routine. If your system does not support any form of flock(), or if
550you share your files across NFS, you might wish to use other forms
551of locking by using modules such as LockFile::Simple which lock a
552file using a filesystem entry, instead of locking the file descriptor.
dd19458b 553
7a6a85bf
RG
554=head1 SPEED
555
556The heart of Storable is written in C for decent speed. Extra low-level
4d3295e3
PN
557optimizations have been made when manipulating perl internals, to
558sacrifice encapsulation for the benefit of greater speed.
7a6a85bf
RG
559
560=head1 CANONICAL REPRESENTATION
561
f062ea6c 562Normally, Storable stores elements of hashes in the order they are
7a6a85bf
RG
563stored internally by Perl, i.e. pseudo-randomly. If you set
564C<$Storable::canonical> to some C<TRUE> value, Storable will store
565hashes with the elements sorted by their key. This allows you to
566compare data structures by comparing their frozen representations (or
567even the compressed frozen representations), which can be useful for
568creating lookup tables for complicated queries.
569
f062ea6c 570Canonical order does not imply network order; those are two orthogonal
7a6a85bf
RG
571settings.
572
d2b96869
SR
573=head1 CODE REFERENCES
574
575Since Storable version 2.05, CODE references may be serialized with
576the help of L<B::Deparse>. To enable this feature, set
3c4b39be 577C<$Storable::Deparse> to a true value. To enable deserialization,
d2b96869
SR
578C<$Storable::Eval> should be set to a true value. Be aware that
579deserialization is done through C<eval>, which is dangerous if the
580Storable file contains malicious data. You can set C<$Storable::Eval>
581to a subroutine reference which would be used instead of C<eval>. See
582below for an example using a L<Safe> compartment for deserialization
583of CODE references.
584
197b90bc
SR
585If C<$Storable::Deparse> and/or C<$Storable::Eval> are set to false
586values, then the value of C<$Storable::forgive_me> (see below) is
587respected while serializing and deserializing.
588
c261f00e
NC
589=head1 FORWARD COMPATIBILITY
590
591This release of Storable can be used on a newer version of Perl to
f062ea6c 592serialize data which is not supported by earlier Perls. By default,
c261f00e 593Storable will attempt to do the right thing, by C<croak()>ing if it
775ecd75 594encounters data that it cannot deserialize. However, the defaults
f062ea6c 595can be changed as follows:
c261f00e
NC
596
597=over 4
598
599=item utf8 data
600
601Perl 5.6 added support for Unicode characters with code points > 255,
602and Perl 5.8 has full support for Unicode characters in hash keys.
603Perl internally encodes strings with these characters using utf8, and
604Storable serializes them as utf8. By default, if an older version of
605Perl encounters a utf8 value it cannot represent, it will C<croak()>.
606To change this behaviour so that Storable deserializes utf8 encoded
607values as the string of bytes (effectively dropping the I<is_utf8> flag)
608set C<$Storable::drop_utf8> to some C<TRUE> value. This is a form of
609data loss, because with C<$drop_utf8> true, it becomes impossible to tell
610whether the original data was the Unicode string, or a series of bytes
611that happen to be valid utf8.
612
613=item restricted hashes
614
f062ea6c
PN
615Perl 5.8 adds support for restricted hashes, which have keys
616restricted to a given set, and can have values locked to be read only.
617By default, when Storable encounters a restricted hash on a perl
618that doesn't support them, it will deserialize it as a normal hash,
619silently discarding any placeholder keys and leaving the keys and
620all values unlocked. To make Storable C<croak()> instead, set
621C<$Storable::downgrade_restricted> to a C<FALSE> value. To restore
622the default set it back to some C<TRUE> value.
c261f00e 623
e8189732
NC
624=item files from future versions of Storable
625
626Earlier versions of Storable would immediately croak if they encountered
627a file with a higher internal version number than the reading Storable
628knew about. Internal version numbers are increased each time new data
629types (such as restricted hashes) are added to the vocabulary of the file
630format. This meant that a newer Storable module had no way of writing a
f062ea6c 631file readable by an older Storable, even if the writer didn't store newer
e8189732
NC
632data types.
633
634This version of Storable will defer croaking until it encounters a data
635type in the file that it does not recognize. This means that it will
636continue to read files generated by newer Storable modules which are careful
637in what they write out, making it easier to upgrade Storable modules in a
638mixed environment.
639
640The old behaviour of immediate croaking can be re-instated by setting
f062ea6c 641C<$Storable::accept_future_minor> to some C<FALSE> value.
e8189732 642
c261f00e
NC
643=back
644
f062ea6c 645All these variables have no effect on a newer Perl which supports the
c261f00e
NC
646relevant feature.
647
7a6a85bf
RG
648=head1 ERROR REPORTING
649
650Storable uses the "exception" paradigm, in that it does not try to workaround
651failures: if something bad happens, an exception is generated from the
652caller's perspective (see L<Carp> and C<croak()>). Use eval {} to trap
653those exceptions.
654
655When Storable croaks, it tries to report the error via the C<logcroak()>
656routine from the C<Log::Agent> package, if it is available.
657
212e9bde
JH
658Normal errors are reported by having store() or retrieve() return C<undef>.
659Such errors are usually I/O errors (or truncated stream errors at retrieval).
660
7a6a85bf
RG
661=head1 WIZARDS ONLY
662
663=head2 Hooks
664
665Any class may define hooks that will be called during the serialization
666and deserialization process on objects that are instances of that class.
667Those hooks can redefine the way serialization is performed (and therefore,
c261f00e 668how the symmetrical deserialization should be conducted).
7a6a85bf
RG
669
670Since we said earlier:
671
672 dclone(.) = thaw(freeze(.))
673
674everything we say about hooks should also hold for deep cloning. However,
675hooks get to know whether the operation is a mere serialization, or a cloning.
676
677Therefore, when serializing hooks are involved,
678
679 dclone(.) <> thaw(freeze(.))
680
681Well, you could keep them in sync, but there's no guarantee it will always
682hold on classes somebody else wrote. Besides, there is little to gain in
f062ea6c 683doing so: a serializing hook could keep only one attribute of an object,
7a6a85bf
RG
684which is probably not what should happen during a deep cloning of that
685same object.
686
687Here is the hooking interface:
688
bbc7dcd2 689=over 4
7a6a85bf
RG
690
691=item C<STORABLE_freeze> I<obj>, I<cloning>
692
693The serializing hook, called on the object during serialization. It can be
694inherited, or defined in the class itself, like any other method.
695
696Arguments: I<obj> is the object to serialize, I<cloning> is a flag indicating
697whether we're in a dclone() or a regular serialization via store() or freeze().
698
699Returned value: A LIST C<($serialized, $ref1, $ref2, ...)> where $serialized
700is the serialized form to be used, and the optional $ref1, $ref2, etc... are
701extra references that you wish to let the Storable engine serialize.
702
703At deserialization time, you will be given back the same LIST, but all the
704extra references will be pointing into the deserialized structure.
705
706The B<first time> the hook is hit in a serialization flow, you may have it
707return an empty list. That will signal the Storable engine to further
708discard that hook for this class and to therefore revert to the default
709serialization of the underlying Perl data. The hook will again be normally
710processed in the next serialization.
711
712Unless you know better, serializing hook should always say:
713
714 sub STORABLE_freeze {
715 my ($self, $cloning) = @_;
716 return if $cloning; # Regular default serialization
717 ....
718 }
719
720in order to keep reasonable dclone() semantics.
721
722=item C<STORABLE_thaw> I<obj>, I<cloning>, I<serialized>, ...
723
724The deserializing hook called on the object during deserialization.
f062ea6c 725But wait: if we're deserializing, there's no object yet... right?
7a6a85bf
RG
726
727Wrong: the Storable engine creates an empty one for you. If you know Eiffel,
728you can view C<STORABLE_thaw> as an alternate creation routine.
729
730This means the hook can be inherited like any other method, and that
731I<obj> is your blessed reference for this particular instance.
732
733The other arguments should look familiar if you know C<STORABLE_freeze>:
734I<cloning> is true when we're part of a deep clone operation, I<serialized>
735is the serialized string you returned to the engine in C<STORABLE_freeze>,
736and there may be an optional list of references, in the same order you gave
737them at serialization time, pointing to the deserialized objects (which
738have been processed courtesy of the Storable engine).
739
212e9bde
JH
740When the Storable engine does not find any C<STORABLE_thaw> hook routine,
741it tries to load the class by requiring the package dynamically (using
742the blessed package name), and then re-attempts the lookup. If at that
743time the hook cannot be located, the engine croaks. Note that this mechanism
c261f00e 744will fail if you define several classes in the same file, but L<perlmod>
212e9bde
JH
745warned you.
746
f062ea6c 747It is up to you to use this information to populate I<obj> the way you want.
7a6a85bf
RG
748
749Returned value: none.
750
2f796f32
AMS
751=item C<STORABLE_attach> I<class>, I<cloning>, I<serialized>
752
753While C<STORABLE_freeze> and C<STORABLE_thaw> are useful for classes where
3c4b39be 754each instance is independent, this mechanism has difficulty (or is
2f796f32
AMS
755incompatible) with objects that exist as common process-level or
756system-level resources, such as singleton objects, database pools, caches
757or memoized objects.
758
759The alternative C<STORABLE_attach> method provides a solution for these
1e2a0f0b
RGS
760shared objects. Instead of C<STORABLE_freeze> --E<gt> C<STORABLE_thaw>,
761you implement C<STORABLE_freeze> --E<gt> C<STORABLE_attach> instead.
2f796f32
AMS
762
763Arguments: I<class> is the class we are attaching to, I<cloning> is a flag
764indicating whether we're in a dclone() or a regular de-serialization via
765thaw(), and I<serialized> is the stored string for the resource object.
766
767Because these resource objects are considered to be owned by the entire
768process/system, and not the "property" of whatever is being serialized,
769no references underneath the object should be included in the serialized
770string. Thus, in any class that implements C<STORABLE_attach>, the
771C<STORABLE_freeze> method cannot return any references, and C<Storable>
772will throw an error if C<STORABLE_freeze> tries to return references.
773
774All information required to "attach" back to the shared resource object
775B<must> be contained B<only> in the C<STORABLE_freeze> return string.
776Otherwise, C<STORABLE_freeze> behaves as normal for C<STORABLE_attach>
777classes.
778
779Because C<STORABLE_attach> is passed the class (rather than an object),
780it also returns the object directly, rather than modifying the passed
781object.
782
783Returned value: object of type C<class>
784
7a6a85bf
RG
785=back
786
787=head2 Predicates
788
c261f00e 789Predicates are not exportable. They must be called by explicitly prefixing
7a6a85bf
RG
790them with the Storable package name.
791
bbc7dcd2 792=over 4
7a6a85bf
RG
793
794=item C<Storable::last_op_in_netorder>
795
796The C<Storable::last_op_in_netorder()> predicate will tell you whether
797network order was used in the last store or retrieve operation. If you
798don't know how to use this, just forget about it.
799
800=item C<Storable::is_storing>
801
802Returns true if within a store operation (via STORABLE_freeze hook).
803
804=item C<Storable::is_retrieving>
805
f062ea6c 806Returns true if within a retrieve operation (via STORABLE_thaw hook).
7a6a85bf
RG
807
808=back
809
810=head2 Recursion
811
f062ea6c
PN
812With hooks comes the ability to recurse back to the Storable engine.
813Indeed, hooks are regular Perl code, and Storable is convenient when
814it comes to serializing and deserializing things, so why not use it
815to handle the serialization string?
7a6a85bf 816
f062ea6c 817There are a few things you need to know, however:
7a6a85bf 818
bbc7dcd2 819=over 4
7a6a85bf
RG
820
821=item *
822
823You can create endless loops if the things you serialize via freeze()
f062ea6c
PN
824(for instance) point back to the object we're trying to serialize in
825the hook.
7a6a85bf
RG
826
827=item *
828
829Shared references among objects will not stay shared: if we're serializing
830the list of object [A, C] where both object A and C refer to the SAME object
831B, and if there is a serializing hook in A that says freeze(B), then when
832deserializing, we'll get [A', C'] where A' refers to B', but C' refers to D,
833a deep clone of B'. The topology was not preserved.
834
835=back
836
837That's why C<STORABLE_freeze> lets you provide a list of references
838to serialize. The engine guarantees that those will be serialized in the
839same context as the other objects, and therefore that shared objects will
840stay shared.
841
842In the above [A, C] example, the C<STORABLE_freeze> hook could return:
843
844 ("something", $self->{B})
845
846and the B part would be serialized by the engine. In C<STORABLE_thaw>, you
847would get back the reference to the B' object, deserialized for you.
848
849Therefore, recursion should normally be avoided, but is nonetheless supported.
850
851=head2 Deep Cloning
852
f062ea6c 853There is a Clone module available on CPAN which implements deep cloning
7a6a85bf
RG
854natively, i.e. without freezing to memory and thawing the result. It is
855aimed to replace Storable's dclone() some day. However, it does not currently
856support Storable hooks to redefine the way deep cloning is performed.
857
0a0da639
JH
858=head1 Storable magic
859
860Yes, there's a lot of that :-) But more precisely, in UNIX systems
861there's a utility called C<file>, which recognizes data files based on
862their contents (usually their first few bytes). For this to work,
8b793558 863a certain file called F<magic> needs to taught about the I<signature>
0a0da639 864of the data. Where that configuration file lives depends on the UNIX
f062ea6c 865flavour; often it's something like F</usr/share/misc/magic> or
8b793558
JH
866F</etc/magic>. Your system administrator needs to do the updating of
867the F<magic> file. The necessary signature information is output to
f062ea6c
PN
868STDOUT by invoking Storable::show_file_magic(). Note that the GNU
869implementation of the C<file> utility, version 3.38 or later,
870is expected to contain support for recognising Storable files
871out-of-the-box, in addition to other kinds of Perl files.
0a0da639 872
d4b9b6e4
GA
873You can also use the following functions to extract the file header
874information from Storable images:
875
876=over
877
878=item $info = Storable::file_magic( $filename )
879
880If the given file is a Storable image return a hash describing it. If
881the file is readable, but not a Storable image return C<undef>. If
882the file does not exist or is unreadable then croak.
883
884The hash returned has the following elements:
885
886=over
887
888=item C<version>
889
890This returns the file format version. It is a string like "2.7".
891
892Note that this version number is not the same as the version number of
893the Storable module itself. For instance Storable v0.7 create files
894in format v2.0 and Storable v2.15 create files in format v2.7. The
895file format version number only increment when additional features
896that would confuse older versions of the module are added.
897
898Files older than v2.0 will have the one of the version numbers "-1",
899"0" or "1". No minor number was used at that time.
900
901=item C<version_nv>
902
903This returns the file format version as number. It is a string like
904"2.007". This value is suitable for numeric comparisons.
905
906The constant function C<Storable::BIN_VERSION_NV> returns a comparable
907number that represent the highest file version number that this
908version of Storable fully support (but see discussion of
909C<$Storable::accept_future_minor> above). The constant
910C<Storable::BIN_WRITE_VERSION_NV> function returns what file version
911is written and might be less than C<Storable::BIN_VERSION_NV> in some
c4a6f826 912configurations.
d4b9b6e4
GA
913
914=item C<major>, C<minor>
915
916This also returns the file format version. If the version is "2.7"
917then major would be 2 and minor would be 7. The minor element is
918missing for when major is less than 2.
919
920=item C<hdrsize>
921
922The is the number of bytes that the Storable header occupies.
923
924=item C<netorder>
925
926This is TRUE if the image store data in network order. This means
927that it was created with nstore() or similar.
928
929=item C<byteorder>
930
931This is only present when C<netorder> is FALSE. It is the
932$Config{byteorder} string of the perl that created this image. It is
933a string like "1234" (32 bit little endian) or "87654321" (64 bit big
934endian). This must match the current perl for the image to be
935readable by Storable.
936
937=item C<intsize>, C<longsize>, C<ptrsize>, C<nvsize>
938
939These are only present when C<netorder> is FALSE. These are the sizes of
940various C datatypes of the perl that created this image. These must
941match the current perl for the image to be readable by Storable.
942
943The C<nvsize> element is only present for file format v2.2 and
944higher.
945
946=item C<file>
947
948The name of the file.
949
950=back
951
952=item $info = Storable::read_magic( $buffer )
953
954=item $info = Storable::read_magic( $buffer, $must_be_file )
955
956The $buffer should be a Storable image or the first few bytes of it.
957If $buffer starts with a Storable header, then a hash describing the
958image is returned, otherwise C<undef> is returned.
959
960The hash has the same structure as the one returned by
961Storable::file_magic(). The C<file> element is true if the image is a
962file image.
963
964If the $must_be_file argument is provided and is TRUE, then return
965C<undef> unless the image looks like it belongs to a file dump.
966
967The maximum size of a Storable header is currently 21 bytes. If the
968provided $buffer is only the first part of a Storable image it should
969at least be this long to ensure that read_magic() will recognize it as
970such.
971
972=back
973
7a6a85bf
RG
974=head1 EXAMPLES
975
976Here are some code samples showing a possible usage of Storable:
977
978 use Storable qw(store retrieve freeze thaw dclone);
979
980 %color = ('Blue' => 0.1, 'Red' => 0.8, 'Black' => 0, 'White' => 1);
981
2359510d 982 store(\%color, 'mycolors') or die "Can't store %a in mycolors!\n";
7a6a85bf 983
2359510d
SD
984 $colref = retrieve('mycolors');
985 die "Unable to retrieve from mycolors!\n" unless defined $colref;
7a6a85bf
RG
986 printf "Blue is still %lf\n", $colref->{'Blue'};
987
988 $colref2 = dclone(\%color);
989
990 $str = freeze(\%color);
991 printf "Serialization of %%color is %d bytes long.\n", length($str);
992 $colref3 = thaw($str);
993
994which prints (on my machine):
995
996 Blue is still 0.100000
997 Serialization of %color is 102 bytes long.
998
d2b96869
SR
999Serialization of CODE references and deserialization in a safe
1000compartment:
1001
197b90bc
SR
1002=for example begin
1003
d2b96869
SR
1004 use Storable qw(freeze thaw);
1005 use Safe;
1006 use strict;
1007 my $safe = new Safe;
197b90bc 1008 # because of opcodes used in "use strict":
d1e2299c 1009 $safe->permit(qw(:default require));
d2b96869
SR
1010 local $Storable::Deparse = 1;
1011 local $Storable::Eval = sub { $safe->reval($_[0]) };
197b90bc 1012 my $serialized = freeze(sub { 42 });
d2b96869 1013 my $code = thaw($serialized);
197b90bc
SR
1014 $code->() == 42;
1015
1016=for example end
1017
1018=for example_testing
1019 is( $code->(), 42 );
d2b96869 1020
7a6a85bf
RG
1021=head1 WARNING
1022
1023If you're using references as keys within your hash tables, you're bound
f062ea6c 1024to be disappointed when retrieving your data. Indeed, Perl stringifies
7a6a85bf
RG
1025references used as hash table keys. If you later wish to access the
1026items via another reference stringification (i.e. using the same
1027reference that was used for the key originally to record the value into
1028the hash table), it will work because both references stringify to the
1029same string.
1030
6fe6778b
PN
1031It won't work across a sequence of C<store> and C<retrieve> operations,
1032however, because the addresses in the retrieved objects, which are
1033part of the stringified references, will probably differ from the
1034original addresses. The topology of your structure is preserved,
1035but not hidden semantics like those.
7a6a85bf
RG
1036
1037On platforms where it matters, be sure to call C<binmode()> on the
1038descriptors that you pass to Storable functions.
1039
1040Storing data canonically that contains large hashes can be
1041significantly slower than storing the same data normally, as
c261f00e 1042temporary arrays to hold the keys for each hash have to be allocated,
7a6a85bf
RG
1043populated, sorted and freed. Some tests have shown a halving of the
1044speed of storing -- the exact penalty will depend on the complexity of
1045your data. There is no slowdown on retrieval.
1046
1047=head1 BUGS
1048
eda690a0 1049You can't store GLOB, FORMLINE, REGEXP, etc.... If you can define semantics
197b90bc
SR
1050for those operations, feel free to enhance Storable so that it can
1051deal with them.
7a6a85bf
RG
1052
1053The store functions will C<croak> if they run into such references
1054unless you set C<$Storable::forgive_me> to some C<TRUE> value. In that
1055case, the fatal message is turned in a warning and some
1056meaningless string is stored instead.
1057
1058Setting C<$Storable::canonical> may not yield frozen strings that
1059compare equal due to possible stringification of numbers. When the
f062ea6c 1060string version of a scalar exists, it is the form stored; therefore,
7a6a85bf
RG
1061if you happen to use your numbers as strings between two freezing
1062operations on the same data structures, you will get different
1063results.
1064
dd19458b
JH
1065When storing doubles in network order, their value is stored as text.
1066However, you should also not expect non-numeric floating-point values
1067such as infinity and "not a number" to pass successfully through a
1068nstore()/retrieve() pair.
1069
1070As Storable neither knows nor cares about character sets (although it
1071does know that characters may be more than eight bits wide), any difference
1072in the interpretation of character codes between a host and a target
1073system is your problem. In particular, if host and target use different
1074code points to represent the characters used in the text representation
1075of floating-point numbers, you will not be able be able to exchange
1076floating-point data, even with nstore().
1077
c261f00e
NC
1078C<Storable::drop_utf8> is a blunt tool. There is no facility either to
1079return B<all> strings as utf8 sequences, or to attempt to convert utf8
1080data back to 8 bit and C<croak()> if the conversion fails.
1081
ee0f7aac
NC
1082Prior to Storable 2.01, no distinction was made between signed and
1083unsigned integers on storing. By default Storable prefers to store a
1084scalars string representation (if it has one) so this would only cause
3c4b39be 1085problems when storing large unsigned integers that had never been converted
ee0f7aac
NC
1086to string or floating point. In other words values that had been generated
1087by integer operations such as logic ops and then not used in any string or
1088arithmetic context before storing.
1089
1090=head2 64 bit data in perl 5.6.0 and 5.6.1
1091
1092This section only applies to you if you have existing data written out
1093by Storable 2.02 or earlier on perl 5.6.0 or 5.6.1 on Unix or Linux which
1094has been configured with 64 bit integer support (not the default)
1095If you got a precompiled perl, rather than running Configure to build
1096your own perl from source, then it almost certainly does not affect you,
1097and you can stop reading now (unless you're curious). If you're using perl
1098on Windows it does not affect you.
1099
1100Storable writes a file header which contains the sizes of various C
1101language types for the C compiler that built Storable (when not writing in
1102network order), and will refuse to load files written by a Storable not
1103on the same (or compatible) architecture. This check and a check on
1104machine byteorder is needed because the size of various fields in the file
1105are given by the sizes of the C language types, and so files written on
1106different architectures are incompatible. This is done for increased speed.
1107(When writing in network order, all fields are written out as standard
1108lengths, which allows full interworking, but takes longer to read and write)
1109
1110Perl 5.6.x introduced the ability to optional configure the perl interpreter
1111to use C's C<long long> type to allow scalars to store 64 bit integers on 32
1112bit systems. However, due to the way the Perl configuration system
1113generated the C configuration files on non-Windows platforms, and the way
1114Storable generates its header, nothing in the Storable file header reflected
1115whether the perl writing was using 32 or 64 bit integers, despite the fact
1116that Storable was storing some data differently in the file. Hence Storable
1117running on perl with 64 bit integers will read the header from a file
1118written by a 32 bit perl, not realise that the data is actually in a subtly
1119incompatible format, and then go horribly wrong (possibly crashing) if it
1120encountered a stored integer. This is a design failure.
1121
1122Storable has now been changed to write out and read in a file header with
1123information about the size of integers. It's impossible to detect whether
1124an old file being read in was written with 32 or 64 bit integers (they have
1125the same header) so it's impossible to automatically switch to a correct
1126backwards compatibility mode. Hence this Storable defaults to the new,
1127correct behaviour.
1128
1129What this means is that if you have data written by Storable 1.x running
1130on perl 5.6.0 or 5.6.1 configured with 64 bit integers on Unix or Linux
1131then by default this Storable will refuse to read it, giving the error
1132I<Byte order is not compatible>. If you have such data then you you
1133should set C<$Storable::interwork_56_64bit> to a true value to make this
1134Storable read and write files with the old header. You should also
1135migrate your data, or any older perl you are communicating with, to this
1136current version of Storable.
1137
1138If you don't have data written with specific configuration of perl described
1139above, then you do not and should not do anything. Don't set the flag -
1140not only will Storable on an identically configured perl refuse to load them,
1141but Storable a differently configured perl will load them believing them
1142to be correct for it, and then may well fail or crash part way through
1143reading them.
1144
7a6a85bf
RG
1145=head1 CREDITS
1146
1147Thank you to (in chronological order):
1148
1149 Jarkko Hietaniemi <jhi@iki.fi>
1150 Ulrich Pfeifer <pfeifer@charly.informatik.uni-dortmund.de>
51f77169 1151 Benjamin A. Holzman <bholzman@earthlink.net>
7a6a85bf
RG
1152 Andrew Ford <A.Ford@ford-mason.co.uk>
1153 Gisle Aas <gisle@aas.no>
1154 Jeff Gresham <gresham_jeffrey@jpmorgan.com>
1155 Murray Nesbitt <murray@activestate.com>
1156 Marc Lehmann <pcg@opengroup.org>
9e21b3d0
JH
1157 Justin Banks <justinb@wamnet.com>
1158 Jarkko Hietaniemi <jhi@iki.fi> (AGAIN, as perl 5.7.0 Pumpkin!)
dd19458b
JH
1159 Salvador Ortiz Garcia <sog@msg.com.mx>
1160 Dominic Dunlop <domo@computer.org>
1161 Erik Haugan <erik@solbors.no>
51f77169 1162 Benjamin A. Holzman <ben.holzman@grantstreet.com>
7a6a85bf
RG
1163
1164for their bug reports, suggestions and contributions.
1165
1166Benjamin Holzman contributed the tied variable support, Andrew Ford
1167contributed the canonical order for hashes, and Gisle Aas fixed
f062ea6c 1168a few misunderstandings of mine regarding the perl internals,
7a6a85bf
RG
1169and optimized the emission of "tags" in the output streams by
1170simply counting the objects instead of tagging them (leading to
1171a binary incompatibility for the Storable image starting at version
f062ea6c 11720.6--older images are, of course, still properly understood).
7a6a85bf 1173Murray Nesbitt made Storable thread-safe. Marc Lehmann added overloading
51f77169
AMS
1174and references to tied items support. Benjamin Holzman added a performance
1175improvement for overloaded classes; thanks to Grant Street Group for footing
1176the bill.
7a6a85bf 1177
7a6a85bf
RG
1178=head1 AUTHOR
1179
0ba8809e 1180Storable was written by Raphael Manfredi F<E<lt>Raphael_Manfredi@pobox.comE<gt>>
775ecd75 1181Maintenance is now done by the perl5-porters F<E<lt>perl5-porters@perl.orgE<gt>>
0ba8809e
NC
1182
1183Please e-mail us with problems, bug fixes, comments and complaints,
d119459b 1184although if you have compliments you should send them to Raphael.
0ba8809e
NC
1185Please don't e-mail Raphael with problems, as he no longer works on
1186Storable, and your message will be delayed while he forwards it to us.
7a6a85bf
RG
1187
1188=head1 SEE ALSO
1189
c261f00e 1190L<Clone>.
7a6a85bf
RG
1191
1192=cut