3 # Convert users.v2 or .v3 to JSON .v4 format
5 # It is believed that this can be run at any time...
7 # Copyright (c) 2020 Dirk Koopman G1TLH
12 # make sure that modules are searched in the order local then perl
17 # root of directory tree for this system
19 $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'};
20 unshift @INC, "$root/perl"; # this IS the right way round!
21 unshift @INC, "$root/local";
29 use Data::Structure::Util qw(unbless);
30 use Time::HiRes qw(gettimeofday tv_interval);
41 my $json = JSON->new()->canonical(1);
42 my $ofn = localdata("$fn.v4");
50 if ( ! -e localdata("$fn.v3") && -e localdata("$fn.v2") ) {
53 LogDbg('',"the module Storable appears to be missing!!");
54 LogDbg('',"trying to continue in compatibility mode (this may fail)");
55 LogDbg('',"please install Storable from CPAN as soon as possible");
58 import Storable qw(nfreeze thaw);
59 $convert = 3 if -e localdata("users.v3") && !-e $ufn;
62 die "need to have a $fn.v2 or (preferably) a $fn.v3 file in /spider/data or /spider/local_data\n" unless $convert;
64 if (-e $ofn || -e "$ofn.n") {
65 my $nfn = localdata("$fn.v4.json");
66 say "You appear to have (or are using) $ofn, creating $nfn instead";
70 say "using $ofn.n for output";
74 # do a conversion if required
76 my ($key, $val, $action, $count, $err) = ('','',0,0,0);
77 my $ta = [gettimeofday];
78 my $ofh = IO::File->new(">$ofn") or die "cannot open $ofn ($!)\n";
79 my $oldfn = localdata("users.v$convert");
82 LogDbg('',"Converting the User File from V$convert to $fn.v4 ");
83 LogDbg('',"This will take a while, maybe as much as 10 secs");
84 my $odbm = tie (%oldu, 'DB_File', $oldfn, O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $oldfn ($!) [rebuild it from user_asc?]";
85 for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) {
88 eval { $ref = storable_decode($val) };
91 eval { $ref = asc_decode($val) };
96 $ofh->print("$ref->{call}\t" . $json->encode($ref) . "\n");
104 Log('err', "DXUser: error decoding $@");
109 my $t = _diffms($ta);
110 LogDbg('',"Conversion from $oldfn to $ofn completed $count records $err errors $t mS");
120 $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg;
123 LogDbg('err', "DXUser::asc_decode: on '$s' $@");
147 # find the correct local_data directory
148 # basically, if there is a local_data directory with this filename and it is younger than the
149 # equivalent one in the (system) data directory then return that name rather than the system one
153 my $ofn = "$root/local_data/$ifn";
156 if (-e "$root/local_data") {
157 $tfn = "$main::data/$ifn";
158 if ((-e $tfn) && (-e $ofn)) {
159 $ofn = $tfn if -M $ofn < -M $tfn;
162 $ofn = $tfn if -e $tfn;
168 # measure the time taken for something to happen; use Time::HiRes qw(gettimeofday tv_interval);
172 my $tb = shift || [gettimeofday];
173 my $a = int($ta->[0] * 1000) + int($ta->[1] / 1000);
174 my $b = int($tb->[0] * 1000) + int($tb->[1] / 1000);