From: Dirk Koopman Date: Thu, 16 Dec 2021 19:30:10 +0000 (+0000) Subject: staging new anti spot spam code X-Git-Url: http://scm.dxcluster.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=e148ea6b5e9eb73af10a9ca4132bc257e4b9aac8;p=spider.git staging new anti spot spam code --- diff --git a/perl/DXCron.pm b/perl/DXCron.pm index b908af3c..4cfabc0e 100644 --- a/perl/DXCron.pm +++ b/perl/DXCron.pm @@ -68,7 +68,7 @@ sub cread $line++; chomp $l; next if $l =~ /^\s*#/o or $l =~ /^\s*$/o; - my ($min, $hour, $mday, $month, $wday, $cmd) = $l =~ /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/o; + my ($min, $hour, $mday, $month, $wday, $cmd) = $l =~ /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/; next unless defined $min; my $ref = bless {}; my $err = ''; @@ -110,7 +110,7 @@ sub parse # handle '*' values if ($val eq '*') { $ref->{$sort} = 0; - return; + return ''; } # handle comma delimited values @@ -131,7 +131,7 @@ sub parse } $ref->{$sort} = \@req; - return; + return ''; } # process the cronjobs diff --git a/perl/DXDebug.pm b/perl/DXDebug.pm index 57a8237a..c2e5f28f 100644 --- a/perl/DXDebug.pm +++ b/perl/DXDebug.pm @@ -104,7 +104,7 @@ sub dbgtrace last if $pkg =~ /Mojo/; # say "*** in dbgtrace $i after mojo"; $_isdbg = 'trace'; - dbg("Stack ($i): $pkg::$subr in $fn line: $l"); + dbg("Stack ($i): ${pkg}::$subr in $fn line: $l"); } $_isdbg = ''; } diff --git a/perl/DXProt.pm b/perl/DXProt.pm index 56c81042..95940e58 100644 --- a/perl/DXProt.pm +++ b/perl/DXProt.pm @@ -49,7 +49,7 @@ use vars qw($pc11_max_age $pc23_max_age $last_pc50 $eph_restime $eph_info_restim %pc92_find $pc92_find_timeout $pc92_short_update_period $next_pc92_obs_timeout $pc92_slug_changes $last_pc92_slug $pc92_extnode_update_period $pc50_interval - $pc92_keepalive_period + $pc92_keepalive_period $senderverify ); $pc11_max_age = 1*3600; # the maximum age for an incoming 'real-time' pc11 @@ -82,6 +82,8 @@ $pc92_extnode_update_period = 1*60*60; # the update period for external nodes $pc92_keepalive_period = 1*60*60; # frequency of PC92 K (keepalive) records %pc92_find = (); # outstanding pc92 find operations $pc92_find_timeout = 30; # maximum time to wait for a reply +$senderverify = 1; # 1 = check spotter is on node it says it is and check ip address if available + # 2 = do 1 and dump if check @checklist = diff --git a/perl/DXProtHandle.pm b/perl/DXProtHandle.pm index 6d3c09de..b22fea82 100644 --- a/perl/DXProtHandle.pm +++ b/perl/DXProtHandle.pm @@ -44,7 +44,7 @@ use vars qw($pc11_max_age $pc23_max_age $last_pc50 $eph_restime $eph_info_restim $eph_pc15_restime $pc9x_past_age $pc9x_dupe_age $pc10_dupe_age $pc92_slug_changes $last_pc92_slug $pc92Ain $pc92Cin $pc92Din $pc92Kin $pc9x_time_tolerance - $pc92filterdef + $pc92filterdef $senderverify ); $pc9x_dupe_age = 60; # catch loops of circular (usually) D records @@ -55,6 +55,9 @@ $pc9x_time_tolerance = 15*60; # the time on a pc9x is allowed to be out by this $pc9x_past_age = (122*60)+ # maximum age in the past of a px9x (a config record might be the only $pc9x_time_tolerance; # thing a node might send - once an hour and we allow an extra hour for luck) # this is actually the partition between "yesterday" and "today" but old. +$senderverify = 1; # 1 - check for forged PC11 or PC61. + # 2 - if forged, dump them. + $pc92filterdef = bless ([ # tag, sort, field, priv, special parser @@ -175,10 +178,6 @@ sub handle_11 dbg("PCPROT: Bad Spotter $pc->[6], dropped") if isdbg('chanerr'); return; } -# unless (is_ipaddr($pc->[8]) || DXUser::get_current($pc->[6])) { -# dbg("PCPROT: Unknown Spotter $pc->[6], dropped") if isdbg('chanerr'); -# return; -# } # convert the date to a unix date my $d = cltounix($pc->[3], $pc->[4]); @@ -208,6 +207,7 @@ sub handle_11 return; } } + my @spot = Spot::prepare($pc->[1], $pc->[2], $d, $pc->[5], $nossid, $pc->[7], $pc->[8]); @@ -220,59 +220,43 @@ sub handle_11 } } - # If is a new PC11, store it, releasing the one that is there (if any), - # if a PC61 comes along then dump the stored PC11 - # If there is a different PC11 stored, release that one and store this PC11 instead, - my $key = join '|', @spot[0..2,4,7]; - if (0) { - - if ($pc->[0] eq 'PC11') { - my $r = [$main::systime, $key, \@spot, $line, $origin, $pc]; - if (!$last) { - $last = [$main::systime, $key, \@spot, $line, $origin, $pc]; - dbg("PC11: $origin -> $key stored") if isdbg('pc11'); - return; - } elsif ($key eq $last->[1]) { # same as last one - dbg("PC11: $origin -> $key dupe dropped") if isdbg('pc11'); - return; - } else { - # it's a different PC11, kick the stored one onward and store this one instead, - dbg("PC11: PC11 new $origin -> $key stored, $last->[4] -> $last->[1] passed onward") if isdbg('pc11'); - @spot = @{$last->[2]}; - $line = $last->[3]; - $origin = $last->[4]; - $pc = $last->[5]; - $last = $r; - } - } elsif ($pc->[0] eq 'PC61') { - if ($last) { - if ($last->[1] eq $key) { - # dump $last and proceed with the PC61 - dbg("PC11: $origin -> $key dropped in favour of PC61") if isdbg('pc11'); - undef $last; - } else { - # it's a different spot send out stored pc11 - dbg("PC11: last $last->[4] -> $last->[1] different PC61 $origin -> $key, send PC11 first ") if isdbg('pc11'); - $last->[1] = 'new pc61'; - handle_11($self, 11, $last->[3], $last->[4], $last->[5]); - undef $last; - dbg("PC11: now process PC61 $origin -> $key") if isdbg('pc11'); - } - } - } else { - dbg("PC11: Unexpected line '$line' in bagging area (expecting PC61, PC11), ignored"); - return; - } - -} - # this goes after the input filtering, but before the add # so that if it is input filtered, it isn't added to the dup # list. This allows it to come in from a "legitimate" source - if (Spot::dup(@spot[0..4,5])) { + if (my $key = Spot::dup(@spot[0..4,7])) { dbg("PCPROT: Duplicate Spot $pc->[0] $key ignored\n") if isdbg('chanerr') || isdbg('dupespot'); return; } + + # here we verify the spotter is currently connected to the node it says it is one. AKA email sender verify + # but without the explicit probe to the node. We are relying on "historical" information, but it very likely + # to be current once we have seen the first PC92C from that node. + # + # As for spots generated from non-PC92 nodes, we'll see after about do_pc9x3h20m... + # + if ($senderverify) { + my $nroute = Route::Node::get($pc->[7]); + my $uroute = Route::Node::get($pc->[6]); + my $local = DXChannel::get($pc->[7]); + + if ($nroute && ($nroute->last_PC92C || ($local && !$local->do_pc9x))) { + my $s = ''; + my $ip = $pcno == 61 ? $pc->[8] : ''; +# $s .= "User $pc->[6] not logged in, " unless $uroute; + $s .= "User $pc->[6] not on node $pc->[7], " unless $nroute->is_user($pc->[6]); +# $s .= "Node $pc->[7] at '$ip' not on Node's IP " . $nroute->ip if $ip && $nroute && $nroute->ip && $nroute->ip ne $ip; + if ($s) { + my $action = $senderverify > 1 ? ", DUMPED" : ''; + $s =~ s/, $//; + dbg("PCProt: Suspicious Spot $pc->[2] on $pc->[1] by $pc->[6]($ip)\@$pc->[7] $s$action"); + return unless $senderverify < 2; + } + } + } + + # If is a new PC11, store it, releasing the one that is there (if any), + # if a PC61 comes along then dump the stored PC11 + # If there is a different PC11 stored, release that one and store this PC11 instead, # add it Spot::add(@spot); @@ -1551,6 +1535,14 @@ sub _add_thingy if ($call) { my $ncall = $parent->call; if ($ncall ne $call) { + my $user; + unless (DXChannel::get($call)) { # i.e. external entity - create an user entry for it - ALL entities will appear in ALL user files from now on. + $user = DXUser::get($call); + unless ($user) { + $user = DXUser->new($call); + dbg("PCProt::_add_thingy new user $call"); + } + } if ($is_node) { dbg("ROUTE: added node $call to $ncall") if isdbg('routelow'); @rout = $parent->add($call, $version, Route::here($here), $ip); @@ -1560,6 +1552,16 @@ sub _add_thingy $r->ip($ip); Log('DXProt', "PC92A $call -> $ip on $ncall"); } + if ($user && $user->sort eq 'U') { + if (defined $version) { + if ($version >= 5455 && $build > 0 || $version >= 3000 ) { + $user->sort('S'); + } else { + $user->sort('A'); + } + dbg("PCProt::_add_thingy node $call sort updated " . $user->sort15); + } + } } else { dbg("ROUTE: added user $call to $ncall") if isdbg('routelow'); my $user = check_add_node($call, 'U'); @@ -1575,7 +1577,8 @@ sub _add_thingy $things_add{$call} = Route::get($call); delete $things_del{$call}; } - } else { + $user->close($main::systime, $ip) if $user; # this just updates lastseen and the connlist list containing the IP address + } else { dbgprintring(10) if isdbg('nologchan'); dbg("DXProt::add_thingy: Trying to add parent $call to itself $ncall, ignored"); } @@ -1901,7 +1904,10 @@ sub handle_92 push @radd, $add if $add; $parent->reset_obs; $parent->version($ent[4]) if $ent[4]; - $parent->build($ent[5]) if $ent[5]; + if ($ent[5]) { + $ent[5] =~ s/^0.//; + $parent->build($ent[5]); + } dbg("ROUTE: reset obscount on $parent->{call} now " . $parent->obscount) if isdbg('obscount'); } diff --git a/perl/DXTimer.pm b/perl/DXTimer.pm new file mode 100644 index 00000000..9dd2607b --- /dev/null +++ b/perl/DXTimer.pm @@ -0,0 +1,67 @@ +# +# Polled Timer handling +# +# This uses callbacks. BE CAREFUL!!!! +# +# +# +# Copyright (c) 2001 Dirk Koopman G1TLH +# + +package DXTimer; + +use Mojo::IOLoop; + +use vars qw(@timerchain $notimers $lasttime); +use DXDebug; + +@timerchain = (); +$notimers = 0; + +$lasttime = 0; + +sub new +{ + my ($pkg, $time, $proc, $recur) = @_; + my $obj = ref($pkg); + my $class = $obj || $pkg; + my $self = bless { t=>$time + $main::systime, proc=>$proc }, $class; + $self->{interval} = $time if $recur; + push @timerchain, $self; + $notimers = @timerchain; + dbg("Timer created (notimers: $notimers)") if isdbg('connll'); + return $self; +} + +sub del +{ + my $self = shift; + delete $self->{proc}; + @timerchain = grep {$_ != $self} @timerchain; + $notimers = @timerchain; + dbg("Timer deleted (notimers: $notimers)") if isdbg('connll'); +} + +sub handler +{ + my $now = $main::systime; + + return unless $now != $lasttime; + + # handle things on the timer chain + my $t; + foreach $t (@timerchain) { + if ($now >= $t->{t}) { + &{$t->{proc}}(); + $t->{t} = $now + $t->{interval} if exists $t->{interval}; + } + } + + $lasttime = $now; +} + +sub DESTROY +{ + dbg("timer destroyed ($DXTimer::notimers)") if isdbg('connll'); +} +1; diff --git a/perl/DXUser.pm b/perl/DXUser.pm index 1af675ab..4af135e7 100644 --- a/perl/DXUser.pm +++ b/perl/DXUser.pm @@ -836,6 +836,16 @@ sub lastping $b->{$call} = shift if @_; return $b->{$call}; } + +sub registered +{ + my $self = shift; + if (exists $self->{registered}) { + return $self->{registered} || 0; + } + return ''; # to stop undef warnings +} + 1; __END__ diff --git a/perl/DXUtil.pm b/perl/DXUtil.pm index 8beb7e51..6d115a7b 100644 --- a/perl/DXUtil.pm +++ b/perl/DXUtil.pm @@ -14,6 +14,7 @@ use IO::File; use File::Copy; use Data::Dumper; use Time::HiRes qw(gettimeofday tv_interval); +use Text::Wrap; use strict; @@ -155,7 +156,7 @@ sub dd # format a prompt with its current value and return it with its privilege sub promptf { - my ($line, $value) = @_; + my ($line, $value, $promptl) = @_; my ($priv, $prompt, $action) = split ',', $line; # if there is an action treat it as a subroutine and replace $value @@ -165,7 +166,8 @@ sub promptf } elsif (ref $value) { $value = dd($value); } - $prompt = sprintf "%15s: %s", $prompt, $value; + $promptl ||= 15; + $prompt = sprintf "%${promptl}s: %s", $prompt, $value; return ($priv, $prompt); } @@ -193,7 +195,7 @@ sub ptimelist sub parray { my $ref = shift; - return ref $ref ? join(', ', @{$ref}) : $ref; + return ref $ref ? join(', ', sort @{$ref}) : $ref; } # take the arg as an array reference and print as a list of pairs @@ -219,11 +221,10 @@ sub phash my $ref = shift; my $out; - while (my ($k,$v) = each %$ref) { - $out .= "${k}=>$v, "; + while (my $k = sort keys %$ref) { + $out .= "${k}=>$ref->{$k}, "; } - chop $out; # remove last space - chop $out; # remove last comma + $out =~ s/, $// if $out; return $out; } @@ -248,23 +249,26 @@ sub print_all_fields my @fields = $ref->fields; my $field; my $width = $self->width - 1; + my $promptl = 0; $width ||= 80; + # find the maximum length of the prompt + foreach $field (@fields) { + if (defined $ref->{$field}) { + my (undef, $prompt, undef) = split ',', $ref->field_prompt($field); + $promptl = length $prompt if length $prompt > $promptl; + } + } + + # now do print foreach $field (sort {_sort_fields($ref, $a, $b)} @fields) { if (defined $ref->{$field}) { - my ($priv, $ans) = promptf($ref->field_prompt($field), $ref->{$field}); + my ($priv, $ans) = promptf($ref->field_prompt($field), $ref->{$field}, $promptl); my @tmp; if (length $ans > $width) { + $Text::Wrap::columns = $width-2; my ($p, $a) = split /: /, $ans, 2; - my $l = (length $p) + 2; - my $al = ($width - 1) - $l; - my $bit; - while (length $a > $al ) { - ($bit, $a) = unpack "A$al A*", $a; - push @tmp, "$p: $bit"; - $p = ' ' x ($l - 2); - } - push @tmp, "$p: $a" if length $a; + @tmp = split/\n/, Text::Wrap::wrap("$p: ", (' ' x $promptl) . ': ', $a); } else { push @tmp, $ans; } @@ -389,9 +393,10 @@ sub is_callsign [A-Z]{1,8} # callsign letters (required) (?:-(?:\d{1,2}))? # - nn possibly (eg G8BPQ-8) (?:/[0-9A-Z]{1,7})? # / another prefix, callsign or special label (including /MM, /P as well as /EURO or /LGT) possibly - $!x; + (?:/(?:AM?|MM?|P))? # finally /A /AM /M /MM /P + $!xo; - # longest callign allowed is 1X11/1Y11XXXXX-11/XXXXXXX + # longest callign allowed is 1X11/1Y11XXXXX-11/XXXXXXX/MM } sub is_prefix diff --git a/perl/EphMsg.pm b/perl/EphMsg.pm index bf43a334..3117aaba 100644 --- a/perl/EphMsg.pm +++ b/perl/EphMsg.pm @@ -18,6 +18,8 @@ use Msg; use DXVars; use DXUtil; use DXDebug; +use DXTimer; + use IO::File; use IO::Socket; use IPC::Open3; @@ -129,7 +131,7 @@ sub _dotimeout dbg("connect $conn->{cnum}: timeout set to $val") if isdbg('connect'); $conn->{timeout}->del if $conn->{timeout}; $conn->{timeval} = $val; - $conn->{timeout} = Timer->new($val, sub{ &_timedout($conn) }); + $conn->{timeout} = DXTimer->new($val, sub{ &_timedout($conn) }); } diff --git a/perl/ExtMsg.pm b/perl/ExtMsg.pm index 064bd90a..74a2cfcd 100644 --- a/perl/ExtMsg.pm +++ b/perl/ExtMsg.pm @@ -19,6 +19,8 @@ use Msg; use DXVars; use DXUtil; use DXDebug; +use DXTimer; + use IO::File; use IO::Socket; use IPC::Open3; @@ -302,7 +304,7 @@ sub _dotimeout dbg("connect $conn->{cnum}: timeout set to $val") if isdbg('connect'); $conn->{timeout}->del if $conn->{timeout}; $conn->{timeval} = $val; - $conn->{timeout} = Timer->new($val, sub{ &_timedout($conn) }); + $conn->{timeout} = DXTimer->new($val, sub{ &_timedout($conn) }); } sub _dolineend diff --git a/perl/Msg.pm b/perl/Msg.pm index 087c0f26..8992d3c1 100644 --- a/perl/Msg.pm +++ b/perl/Msg.pm @@ -18,7 +18,7 @@ use Mojo::IOLoop; use Mojo::IOLoop::Stream; use DXDebug; -use Timer; +use DXTimer; use vars qw($now %conns $noconns $cnum $total_in $total_out $total_lines_in $total_lines_out $connect_timeout $disc_waittime); diff --git a/perl/Route/Node.pm b/perl/Route/Node.pm index 02c8b341..76d98757 100644 --- a/perl/Route/Node.pm +++ b/perl/Route/Node.pm @@ -203,6 +203,14 @@ sub del_user return @out; } +# is a user on this node +sub is_user +{ + my $self = shift; + my $call = shift; + return scalar grep {$_ eq $call} @{$self->{users}}; +} + sub usercount { my $self = shift; diff --git a/perl/Route/User.pm b/perl/Route/User.pm index e99def15..8c1c824d 100644 --- a/perl/Route/User.pm +++ b/perl/Route/User.pm @@ -94,8 +94,6 @@ sub delparent return $self->_dellist('parent', @_); } - - # # generic AUTOLOAD for accessors # diff --git a/perl/Spot.pm b/perl/Spot.pm index 7f6265ac..92ce5249 100644 --- a/perl/Spot.pm +++ b/perl/Spot.pm @@ -477,7 +477,7 @@ sub formatl # enter the spot for dup checking and return true if it is already a dup sub dup { - my ($freq, $call, $d, $text, $by, $cty) = @_; + my ($freq, $call, $d, $text, $by, $node) = @_; # dump if too old return 2 if $d < $main::systime - $dupage; @@ -496,21 +496,16 @@ sub dup chomp $text; $text =~ s/\%([0-9A-F][0-9A-F])/chr(hex($1))/eg; $text = uc unpad($text); - if ($cty && $text && length $text <= 4) { - unless ($text =~ /^C?Q/ || $text =~ /^[\d\W]+$/) { - my @try = Prefix::cty_data($text); - $text = "" if $cty == $try[0]; - } - } my $otext = $text; # $text = Encode::encode("iso-8859-1", $text) if $main::can_encode && Encode::is_utf8($text, 1); $text =~ s/^\+\w+\s*//; # remove leading LoTW callsign $text =~ s/\s{2,}[\dA-Z]?[A-Z]\d?$// if length $text > 24; $text =~ s/[\W\x00-\x2F\x7B-\xFF]//g; # tautology, just to make quite sure! $text = substr($text, 0, $duplth) if length $text > $duplth; - my $ldupkey = "X$freq|$call|$by|$text"; + my $ldupkey = "X$|$call|$by|$node|$freq|$d||$text"; my $t = DXDupe::find($ldupkey); - return 1 if $t && $t - $main::systime > 0; + return $ldupkey if $t && $t - $main::systime > 0; + DXDupe::add($ldupkey, $main::systime+$dupage); $otext = substr($otext, 0, $duplth) if length $otext > $duplth; $otext =~ s/\s+$//; @@ -520,7 +515,7 @@ sub dup return 1 if $t && $t - $main::systime > 0; DXDupe::add($ldupkey, $main::systime+$dupage); } - return 0; + return undef; } sub listdups diff --git a/perl/Timer.pm b/perl/Timer.pm deleted file mode 100644 index e5ffa524..00000000 --- a/perl/Timer.pm +++ /dev/null @@ -1,67 +0,0 @@ -# -# Polled Timer handling -# -# This uses callbacks. BE CAREFUL!!!! -# -# -# -# Copyright (c) 2001 Dirk Koopman G1TLH -# - -package Timer; - -use Mojo::IOLoop; - -use vars qw(@timerchain $notimers $lasttime); -use DXDebug; - -@timerchain = (); -$notimers = 0; - -$lasttime = 0; - -sub new -{ - my ($pkg, $time, $proc, $recur) = @_; - my $obj = ref($pkg); - my $class = $obj || $pkg; - my $self = bless { t=>$time + $main::systime, proc=>$proc }, $class; - $self->{interval} = $time if $recur; - push @timerchain, $self; - $notimers = @timerchain; - dbg("Timer created (notimers: $notimers)") if isdbg('connll'); - return $self; -} - -sub del -{ - my $self = shift; - delete $self->{proc}; - @timerchain = grep {$_ != $self} @timerchain; - $notimers = @timerchain; - dbg("Timer deleted (notimers: $notimers)") if isdbg('connll'); -} - -sub handler -{ - my $now = $main::systime; - - return unless $now != $lasttime; - - # handle things on the timer chain - my $t; - foreach $t (@timerchain) { - if ($now >= $t->{t}) { - &{$t->{proc}}(); - $t->{t} = $now + $t->{interval} if exists $t->{interval}; - } - } - - $lasttime = $now; -} - -sub DESTROY -{ - dbg("timer destroyed ($Timer::notimers)") if isdbg('connll'); -} -1; diff --git a/perl/cluster.pl b/perl/cluster.pl index 919c59db..818718f8 100755 --- a/perl/cluster.pl +++ b/perl/cluster.pl @@ -138,7 +138,7 @@ use AnnTalk; use BBS; use WCY; use BadWords; -use Timer; +use DXTimer; use Route; use Route::Node; use Route::User; @@ -791,7 +791,7 @@ sub per_sec DXCron::process(); # do cron jobs RBN::process(); - Timer::handler(); + DXTimer::handler(); DXLog::flushall(); } diff --git a/perlbrew-dxspider b/perlbrew-dxspider new file mode 100755 index 00000000..cc2546fd --- /dev/null +++ b/perlbrew-dxspider @@ -0,0 +1,7 @@ +#!/bin/bash +. ~/perl5/perlbrew/etc/bashrc +cd /spider/perl +git reset --hard +git pull +~/perl5/perlbrew/perls/perl-5.32.1/bin/perl -w cluster.pl > /dev/null +