From 0c6e2c3a5619ce9791db56632c1b22d10c60bc70 Mon Sep 17 00:00:00 2001 From: Dirk Koopman Date: Wed, 20 May 2020 14:23:11 +0100 Subject: [PATCH] include convert-users-v3-to-v4.pl READ THE UPGRADE.mojo file (especially things marked *IMPORTANT* --- Changes | 8 ++ UPGRADE.mojo | 14 +++- perl/convert-users-v3-to-v4.pl | 141 +++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 3 deletions(-) create mode 100755 perl/convert-users-v3-to-v4.pl diff --git a/Changes b/Changes index 745ea5ff..103343ed 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,11 @@ +20May20======================================================================= +1. Backport convert-users-v3-to-v4.pl to allow creation of the new json + formatted *text* based user file from the old v3 DB_File and Storable + version. This can be done either online (with the node running) or offline + with the node stopped. This is in preparation for the next update of the + mojo branch. Running this program just *BEFORE* doing your next update of + the mojo branch *should* ensure a seemless transition to the Storable & + DB_File free version of the users file. 17May20======================================================================= 1. Backport DXSubprocess to change serialisations. Currently the internals of Mojo::IOLoop::Subprocess defaults to diff --git a/UPGRADE.mojo b/UPGRADE.mojo index 05366658..5e5a402e 100644 --- a/UPGRADE.mojo +++ b/UPGRADE.mojo @@ -16,6 +16,9 @@ DXSpider node. But serious work on this won't start until we have a stable base on. Apart from anything else there will, almost certainly, need to be some internal data structure reorganisation before a decent web frontend could be constructed. +*IMPORTANT* There is an action needed to go from mojo build 228 and below. See items marked +*IMPORTANT* below. + Upgrading is not for the faint of heart. There is no installation script (but there will be) so, for the time being, you need to do some manual editing. Also, while there is a backward path, it will involve moving various files from their new home (/spider/local_data), @@ -53,7 +56,7 @@ You will need the following CPAN packages: If you are on a Debian based system (Devuan, Ubuntu, Mint etc) that is reasonably new (I use Ubuntu 18.04 and Debian 10) then you can simply do: - sudo apt-get install libev-perl libmojolicious-perl libjson-perl libjson-xs-perl + sudo apt-get install libev-perl libmojolicious-perl libjson-perl libjson-xs-perl libdata-structure-util-perl or on Redhat based systems you can install the very similarly (but not the same) named packages. I don't the exact names but using anything less than Centos 7 is likely to cause @@ -61,10 +64,11 @@ You will need the following CPAN packages: If in doubt or it is taking too long to find the packages you should build from CPAN. Note: you may need to install the essential packages to build some of these. At the very least you will need - to install 'make' (sudo apt-get install make) or just get everything you are likely to need with + to install 'make' (sudo apt-get install make) or just get everything you are likely to need with: + sudo apt-get install build-essential. - sudo cpanm EV Mojolicious JSON JSON::XS + sudo cpanm EV Mojolicious JSON JSON::XS Data::Structure::Util # just in case it's missing sudo apt-get install top @@ -162,6 +166,10 @@ if you have not already done this: sudo ln -s /spider/perl/console.pl /usr/local/bin/dx sudo ln -s /spider/perl/*dbg /usr/local/bin +*IMPORTANT* (for any build of dxspider) regardless of branch below build 229 run: + + /spider/perl/convert-users-v3-to-v4.pl + Now in another window run: watchdbg diff --git a/perl/convert-users-v3-to-v4.pl b/perl/convert-users-v3-to-v4.pl new file mode 100755 index 00000000..f8ac9bf4 --- /dev/null +++ b/perl/convert-users-v3-to-v4.pl @@ -0,0 +1,141 @@ +#!/usr/bin/env perl +# +# Convert users.v2 or .v3 to JSON .v4 format +# +# It is believed that this can be run at any time... +# +# Copyright (c) 2020 Dirk Koopman G1TLH +# +# +# + +# make sure that modules are searched in the order local then perl + +BEGIN { + # root of directory tree for this system + $root = "/spider"; + $root = $ENV{'DXSPIDER_ROOT'} if $ENV{'DXSPIDER_ROOT'}; + unshift @INC, "$root/perl"; # this IS the right way round! + unshift @INC, "$root/local"; +} + +use strict; + +use SysVar; +use DXUser; +use DXUtil; +use JSON; +use Data::Structure::Util qw(unbless); +use Time::HiRes qw(gettimeofday tv_interval); +use IO::File; +use Carp; +use DB_File; + +use 5.10.1; + +my $ufn; +my $fn = "users"; + +my $json = JSON->new()->canonical(1); +my $ofn = localdata("$fn.v4"); +my $convert; + +eval { + require Storable; +}; + +if ($@) { + if ( ! -e localdata("$fn.v3") && -e localdata("$fn.v2") ) { + $convert = 2; + } + LogDbg('',"the module Storable appears to be missing!!"); + LogDbg('',"trying to continue in compatibility mode (this may fail)"); + LogDbg('',"please install Storable from CPAN as soon as possible"); +} +else { + import Storable qw(nfreeze thaw); + $convert = 3 if -e localdata("users.v3") && !-e $ufn; +} + +die "need to have a $fn.v2 or (preferably) a $fn.v3 file in /spider/data or /spider/local_data\n" unless $convert; + +if (-e $ofn || -e "$ofn.n") { + my $nfn = localdata("$fn.v4.json"); + say "You appear to have (or are using) $ofn, creating $nfn instead"; + $ofn = $nfn; +} else { + say "using $ofn for output"; +} + + +# do a conversion if required +if ($convert) { + my ($key, $val, $action, $count, $err) = ('','',0,0,0); + my $ta = [gettimeofday]; + my $ofh = IO::File->new(">$ofn") or die "cannot open $ofn ($!)\n"; + + my %oldu; + LogDbg('',"Converting the User File from V$convert to $fn.v4 "); + LogDbg('',"This will take a while, maybe as much as 10 secs"); + my $odbm = tie (%oldu, 'DB_File', localdata("users.v$convert"), O_RDONLY, 0666, $DB_BTREE) or confess "can't open user file: $fn.v$convert ($!) [rebuild it from user_asc?]"; + for ($action = R_FIRST; !$odbm->seq($key, $val, $action); $action = R_NEXT) { + my $ref; + if ($convert == 3) { + eval { $ref = storable_decode($val) }; + } + else { + eval { $ref = asc_decode($val) }; + } + unless ($@) { + if ($ref) { + unbless $ref; + $ofh->print($json->encode($ref) . "\n"); + $count++; + } + else { + $err++ + } + } + else { + Log('err', "DXUser: error decoding $@"); + } + } + undef $odbm; + untie %oldu; + my $t = _diffms($ta); + LogDbg('',"Conversion from users.v$convert to $ofn completed $count records $err errors $t mS"); + $ofh->close; +} + +exit 0; + +sub asc_decode +{ + my $s = shift; + my $ref; + $s =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg; + eval '$ref = ' . $s; + if ($@) { + LogDbg('err', "DXUser::asc_decode: on '$s' $@"); + $ref = undef; + } + return $ref; +} + +sub storable_decode +{ + my $ref; + $ref = thaw(shift); + return $ref; +} + +sub LogDbg +{ + my (undef, $s) = @_; + say $s; +} + +sub Log +{ + say shift; +} -- 2.43.0