From: Mark Wooding <mdw@distorted.org.uk>
Date: Mon, 27 Mar 2006 11:14:49 +0000 (+0100)
Subject: inject: New posting system.
X-Git-Url: https://git.distorted.org.uk/~mdw/newsgate/commitdiff_plain/4aef1ddbe9819539d801aa1f8eabb3e1f62c35e5

inject: New posting system.

Rather than using POST (via a hacked copy of INN's mailpost script), we
inject the message directly using IHAVE.  This lets us hang onto more
headers and generally have a better experience of the mailing list.
---

diff --git a/bin/addrcheck b/bin/addrcheck
index 3a3c533..d693f3b 100755
--- a/bin/addrcheck
+++ b/bin/addrcheck
@@ -7,7 +7,7 @@ sender=${3?sender}
 
 ##bad () { echo "-$*"; exit 0; }
 bad () { 
-  echo "consider reject to $kind:$tail from $sender ($*)" >>addrcheck.log"
+  echo "consider reject to $kind:$tail from $sender ($*)" >>addrcheck.log
   echo "+just this once"
   exit 0
 }
@@ -20,9 +20,7 @@ case $kind in
       bad "invalid sender $sender"
     ;;
   t)
-    echo $sender
-    echo "$4"
     case "$sender" in $4) ;; *) bad "incorrect sender $sender" ;; esac
     ;;
 esac
-echo +
+echo +ok
diff --git a/bin/inject b/bin/inject
new file mode 100755
index 0000000..62a75fd
--- /dev/null
+++ b/bin/inject
@@ -0,0 +1,183 @@
+#! /usr/bin/python
+
+import sre as RX
+import os as OS
+import time as T
+import socket as S
+from getopt import getopt, GetoptError
+from sys import stdin, stdout, stderr, argv, exit
+from cStringIO import StringIO
+
+prog = argv[0]
+
+def bad(msg):
+  print >>stderr, '%s (fatal): %s' % (prog, msg)
+  exit(100)
+def die(msg):
+  print >>stderr, '%s: %s' % (prog, msg)
+  exit(111)
+def usage():
+  print >>stderr, \
+    ('Usage: %s [-d DIST] [-h HOST] [-r REMOTE] [-p PATH] GROUP <MESSAGE' %
+     prog)
+  exit(111)
+
+def headers(file):
+  h = None
+  while True:
+    line = file.next()
+    if line == '' or line == '\n':
+      break
+    if line[0].isspace():
+      if h is None:
+        bad('unexpected continuation')
+      h += line
+    else:
+      if h: yield h
+      h = line
+  if h: yield h
+
+def hdrsplit(h):
+  v = h.split(':', 1)
+  if len(v) != 2:
+    bad('failed to parse header')
+  return v[0].strip().lower(), v[1].strip()
+
+remote = ('localhost', 119)
+approved = None
+try:
+  host = OS.popen('hostname -f').read().strip()
+except:
+  host = 'localhost'
+dist = 'mail'
+path = 'newsgate'
+group = None
+
+def opts():
+  global approved, remote, host, dist, path, group
+  try:
+    opts, args = getopt(argv[1:], 'a:d:h:r:p:',
+                        ['approved=', 'distribution=',
+                         'hostname=', 'remote=', 'path='])
+  except GetoptError:
+    usage()
+  for o, a in opts:
+    if o in ('-a', '--approved'):
+      approved = a
+    elif o in ('-d', '--distribution'):
+      dist = a
+    elif o in ('-h', '--hostname'):
+      host = a
+    elif o in ('-r', '--remote'):
+      remote = (lambda addr, port = 119: (addr, int(port)))(*a.split(':'))
+  if len(args) != 1:
+    usage()
+  group, = args
+
+rx_msgid = RX.compile(r'^\<\S+@\S+\>$')
+
+class NNTP (object):
+  def __init__(me, addr):
+    me.sk = S.socket(S.AF_INET, S.SOCK_STREAM)
+    me.sk.connect(remote)
+    me.f = me.sk.makefile()
+    rc, msg = me.reply()
+    if rc != '200':
+      die('unable to contact server: %s %s' % (rc, msg))
+  def write(me, stuff):
+    me.f.write(stuff)
+  def flush(me):
+    me.f.flush()
+  def cmd(me, stuff):
+    me.f.write(stuff + '\r\n')
+    me.f.flush()
+  def reply(me):
+    rc, msg = (lambda rc, msg = '.': (rc, msg.strip())) \
+                (*me.f.readline().split(None, 1))
+    if rc.startswith('5'):
+      die('server hated me: %s %s' % (rc, msg))
+    return rc, msg.strip()
+
+def send():
+  hdr = StringIO()
+  hdr.write('Path: newsgate\r\n'
+            'Distribution: mail\r\n'
+            'Newsgroups: %s\r\n'
+            'Approved: %s\r\n'
+            % (group, approved or 'newsgate@%s' % host))
+  xify = {}
+  for h in '''
+    lines xref newsgroups path distribution approved received
+  '''.split():
+    xify[h] = 1
+  seen = {}
+  for h in headers(stdin):
+    n, c = hdrsplit(h)
+    if n in xify:
+      h = 'X-Newsgate-' + h
+    elif h.startswith('.'):
+      h = '.' + h
+    seen[n] = c
+    if h.endswith('\r\n'):
+      pass
+    elif h.endswith('\n'):
+      h = h[:-1] + '\r\n'
+    else:
+      h += '\r\n'
+    hdr.write(h)
+  if 'message-id' not in seen:
+    seen['message-id'] = ('<newsgate-%s@%s>'
+                          % (OS.popen('gorp 128').read().strip(),
+                             host))
+    hdr.write('Message-ID: %s\r\n' % seen['message-id'])
+  if 'date' not in seen:
+    hdr.write('Date: %s\r\n'
+              % (T.strftime('%a, %d %b %Y %H:%M:%S %Z')))
+  if 'subject' not in seen:
+    hdr.write('Subject: (no subject)\r\n')
+  hdr.write('\r\n')
+
+  msgid = seen['message-id']
+  if not rx_msgid.match(msgid):
+    bad('invalid message-id %s' % msgid)
+
+  nntp = NNTP(remote)
+  nntp.cmd('IHAVE %s' % msgid)
+  rc, msg = nntp.reply()
+  if rc == '335':
+    nntp.write(hdr.getvalue())
+    for i in stdin:
+      if i.startswith('.'):
+        i = '.' + i
+      if i.endswith('\r\n'):
+        pass
+      elif i.endswith('\n'):
+        i = i[:-1] + '\r\n'
+      else:
+        i = i + '\r\n'
+      nntp.write(i)
+    nntp.write('.\r\n')
+    nntp.flush()
+    rc, msg = nntp.reply()
+  if rc == '435':
+    ## doesn't want my article; pretend all is fine: I don't care
+    pass
+  elif rc == '436':
+    die('failed to send article: %s %s' % (rc, msg))
+  elif rc == '437':
+    bad('server rejected article: %s %s' % (rc, msg))
+  elif not rc.startswith('2'):
+    die('unexpected response from server: %s %s' % (rc, msg))
+  nntp.cmd('QUIT')
+  nntp.reply()
+
+def main():
+  try:
+    opts()
+    send()
+  except SystemExit:
+    raise
+#  except Exception, exc:
+#    die('unhandled exception: %s, %s' % (exc.__class__.__name__,
+#                                         exc.args))
+main()
diff --git a/bin/mailpost.newsgate b/bin/mailpost.newsgate
deleted file mode 100755
index 16794eb..0000000
--- a/bin/mailpost.newsgate
+++ /dev/null
@@ -1,464 +0,0 @@
-#! /usr/bin/perl
-require '/usr/lib/news/innshellvars.pl';
-
-# mailpost - yet another mail-to-news filter
-# 21feb00 [added "lc" to duplicate header fixer stmt to make it case-insensitive]
-# doka 11may99 [fixed duplicate headers problem]
-# brister 19oct98 cleaned up somewhat for perl v. 5. and made a little more robust.
-# vixie 29jan95 RCS'd [$Id: mailpost.in,v 1.3.2.1 2000/08/13 02:03:59 rra Exp $]
-# vixie 15jun93 [added -m]
-# vixie 30jun92 [added -a and -d]
-# vixie 17jun92 [attempt simple-minded fixup to $path]
-# vixie 14jun92 [original]
-
-use Getopt::Std ;
-use IPC::Open3;
-use IO::Select;
-use Sys::Syslog;
-use strict ;
-
-my $debugging = 0 ;
-my $tmpfile ;
-my $msg ;
-
-END {
-    unlink ($tmpfile) if $tmpfile ;		# incase we die()
-}
-
-my $LOCK_SH = 1;
-my $LOCK_EX = 2;
-my $LOCK_NB = 4;
-my $LOCK_UN = 8;
-
-my $usage = $0 ;
-$usage =~ s!.*/!! ;
-my $prog = $usage ;
-
-openlog $usage, "pid", $inn::syslog_facility ;
-
-$usage .= "[ -r addr ][ -f addr ][ -a approved ][ -d distribution ]" .
-    " [ -m mailing-list ][ -b database ][ -o output-path ] newsgroups" ;
-
-use vars qw($opt_r $opt_f $opt_a $opt_d $opt_m $opt_b $opt_n $opt_o $opt_h) ;
-getopts("hr:f:a:d:m:b:no:") || die "usage: $usage\n" ;
-die "usage: $usage\n" if $opt_h ;
-
-#
-# $Submit is a program which takes no arguments and whose stdin is supposed
-# to be a news article (without the #!rnews header but with the news hdr).
-#
-
-my $Sendmail = $inn::mta ;
-my $Submit = $inn::inews . " -S -h";
-my $Database = ($opt_b || $inn::pathtmp) . "/mailpost-msgid" ;
-my $Maintainer = $inn::newsmaster || "usenet" ; 
-my $WhereTo = $opt_o || $Submit ;
-my $Mailname = $inn::fromhost ;
-
-# can't use $inn::tmpdir as we're usually not running as news
-my $Tmpdir = "/var/tmp" ;	
-
-if ($debugging || $opt_n) {
-    $Sendmail = "cat" ;
-    $WhereTo = "cat" ;
-}
-
-chop ($Mailname = `/bin/hostname`) if ! $Mailname ;
-
-
-#
-# our command-line argument(s) are the list of newsgroups to post to.
-#
-# there may be a "-r sender" or "-f sender" which becomes the $path
-# (which is in turn overridden below by various optional headers.)
-#
-# -d (distribution) and -a (approved) are also supported to supply
-# or override the mail headers by those names.
-#
-
-my $path = 'nobody';
-my $newsgroups = undef;
-my $approved = undef;
-my $distribution = undef;
-my $mailing_list = undef;
-my $references = undef;
-my @errorText = ();
-
-if ($opt_r || $opt_f) {
-    $path = $opt_r || $opt_f ;
-    push @errorText, "((path: $path))\n" ;
-}
-
-if ($opt_a) {
-    $approved = &fix_sender_addr($opt_a);
-    push @errorText, "((approved: $approved))\n";
-}
-
-if ($opt_d) {
-    $distribution = $opt_d ;
-    push @errorText, "((distribution: $distribution))\n";
-}
-
-if ($opt_m) {
-    $mailing_list = "<" . $opt_m . "> /dev/null";
-    push @errorText, "((mailing_list: $mailing_list))\n";
-}
-
-$newsgroups = join ", ", @ARGV ;
-
-die "usage:  $0 newsgroup [newsgroup]\n" unless $newsgroups;
-
-
-#
-# do the header.  our input is a mail message, with or without the From_
-#
-
-#$message_id = sprintf("<mailpost.%d.%d@%s>", time, $$, $Hostname);
-my $real_news_hdrs = '';
-my $weird_mail_hdrs = '';
-my $fromHdr = "MAILPOST-UNKNOWN-FROM" ;
-my $dateHdr= "MAILPOST-UNKNOWN-DATE" ;
-my $msgIdHdr = "MAILPOST-UNKNOWN-MESSAGE-ID" ;
-my $from = undef;
-my $date = undef;
-my $hdr = undef;
-my $txt = undef;
-my $message_id ;
-my $subject = "(NONE)";
-
-$_ = <STDIN>;
-if (!$_) {
-    if ( $debugging || -t STDERR ) {
-	die "empty input" ;
-    } else {
-	syslog "err", "empty input" ;
-	exit (0) ;
-    }
-}
-
-chomp $_;
-
-my $line = undef;
-if (/^From\s+([^\s]+)\s+/) {
-    $path = $1;
-    push @errorText, "((path: $path))\n";
-    $_ = $';
-    if (/ remote from /) {
-	$path = $' . '!' . $path;
-	$_ = $`;
-    }
-    $date = $_;
-} else {
-    $line = $_;
-}
-
-for (;;) {
-    last if defined($line) && ($line =~ /^$/) ;
-
-    $_ = <STDIN> ;
-    chomp ;
-
-    # gather up a single header with possible continuation lines into $line
-    if (/^\s+/) {
-	if (! $line) {
-	    $msg = "First line with leading whitespace!" ;
-	    syslog "err", $msg unless -t STDERR ;
-	    die "$msg\n" ;
-	}	    
-
-	$line .= "\n" . $_ ;
-	next ;
-    }
-
-    # On the first header $line will be undefined.
-    ($_, $line) = ($line, $_) ; # swap $line and $_ ;
-
-    last if defined($_) && /^$/ ;
-    next if /^$/ ;		# only on first header will this happen
-
-    push @errorText, "($_)\n";
-
-    next if /^Approved:\s/sio && defined($approved);
-    next if /^Distribution:\s/sio && defined($distribution);
-
-    if (/^(Organization|Distribution):\s*/sio) {
-	$real_news_hdrs .= "$_\n";
-	next;
-    }
-
-    if (/^Subject:\s*/sio) {
-	$subject = $';
-	next;
-    }
-
-    if (/^Message-ID:\s*/sio) {
-	$message_id = $';
-	next;
-    }
-
-    if (/^Mailing-List:\s*/sio) {
-	$mailing_list = $';
-	next;
-    }
-
-    if (/^(Sender|Approved):\s*/sio) {
-	$real_news_hdrs .= "$&" . fix_sender_addr($') . "\n";
-	next;
-    }
-
-    if (/^Return-Path:\s*/sio) {
-	$path = $';
-	$path = $1 if ($path =~ /\<([^\>]*)\>/);
-	push@errorText, "((path: $path))\n";
-	next;
-    }
-
-    if (/^Date:\s*/sio) {
-	$date = $';
-	next;
-    }
-
-    if (/^From:\s*/sio) {
-	$from = &fix_sender_addr($');
-	next;
-    }
-
-    if (/^References:\s*/sio) {
-	$references = $';
-	next;
-    }
-
-    if (!defined($references) && /^In-Reply-To:[^\<]*\<([^\>]+)\>/sio) {
-	$references = "<$1>";
-	# FALLTHROUGH
-    }
-
-    if (/^(MIME|Content)-[^:]+:\s*/sio) {
-	$real_news_hdrs .= $_ . "\n" ;
-	next ;
-    }
-
-    # strip out news trace headers since otherwise posting may fail.  other
-    # trace headers will be renamed to add 'X-' so we don't have to worry
-    # about them.
-    if (/^X-(Trace|Complaints-To):\s*/sio) {
-        next ;
-    }
-
-    # strip out Received headers since otherwise posting may fail
-    # due to too large header size.
-    if (/^(Received):\s*/sio) {
-	next ;
-    }
-
-    # random unknown header.  prepend 'X-' if it's not already there.
-    $_ = "X-$_" unless /^X-/sio ;
-    $weird_mail_hdrs .= "$_\n";
-}
-
-
-$msgIdHdr = $message_id if $message_id ;
-$fromHdr = $from if $from ;
-$dateHdr = $date if $date ;
-
-if ($path !~ /\!/) {
-    $path = "$'!$`" if ($path =~ /\@/);
-}
-
-$real_news_hdrs .= "Subject: ${subject}\n";
-$real_news_hdrs .= "Message-ID: ${msgIdHdr}\n"     if defined($message_id);
-$real_news_hdrs .= "Mailing-List: ${mailing_list}\n" if defined($mailing_list);
-$real_news_hdrs .= "Distribution: ${distribution}\n" if defined($distribution);
-$real_news_hdrs .= "Approved: ${approved}\n"         if defined($approved);
-$real_news_hdrs .= "References: ${references}\n"     if defined($references);
-
-# Remove duplicate headers.
-my %headers = ();
-$real_news_hdrs =~ s/(.*?:)[ \t].*?($|\n)([ \t]+.*?($|\n))*/$headers{lc$1}++?"":$&/ge;
-
-# Inews writes error messages to stdout. We want to capture those and mail
-# them back to the newsmaster. Trying to write and read from a subprocess is 
-# ugly and prone to deadlock, so we use a temp file.
-$tmpfile = sprintf "%s/mailpost.%d.%d", $Tmpdir, time, $$ ;
-
-if (!open TMPFILE,">$tmpfile") {
-    $msg = "cant open temp file ($tmpfile): $!" ;
-    $tmpfile = undef ;
-    syslog "err", "$msg\n" unless $debugging || -t STDERR ;
-    open TMPFILE, "|" . sprintf ($Sendmail, $Maintainer) ||
-	die "die(no tmpfile): sendmail: $!\n" ;
-    print TMPFILE <<"EOF";
-To: $Maintainer
-Subject: mailpost failure ($newsgroups): $msg
-
--------- Article Contents
-
-EOF
-}
-	     
-print TMPFILE <<"EOF";
-Path: ${path}
-From: ${fromHdr}
-Newsgroups: ${newsgroups}
-${real_news_hdrs}Date: ${dateHdr}
-${weird_mail_hdrs}
-EOF
-    
-my $rest;
-$rest .= $_ while (<STDIN>);
-$rest =~ s/\n*$/\n/g;		# Remove trailing \n except very last
-
-print TMPFILE $rest;
-close TMPFILE ;
-
-if ( ! $tmpfile ) {
-    # we had to bail and mail the article to the admin.
-    exit (0) ;
-}
-
-
-##
-## We've got the article in a temp file and now we validate some of the 
-## data we found and update our message-id database.
-##
-
-mailArtAndDie ("no From: found") unless $from;
-mailArtAndDie ("no Date: found") unless $date;
-mailArtAndDie ("no Message-ID: found") unless $message_id;
-mailArtAndDie ("Malformed message ID ($message_id)") 
-    if ($message_id !~ /\<(\S+)\@(\S+)\>/);
-
-
-# update (with locking) our message-id database.  this is used to make sure we
-# don't loop our own gatewayed articles back through the mailing list.
-
-my ($lhs, $rhs) = ($1, $2);	# of message_id match above.
-$rhs =~ tr/A-Z/a-z/;
-
-$message_id = "${lhs}\@${rhs}";
-
-push @errorText, "(TAS message-id database for $message_id)\n";
-
-my $lockfile = sprintf("%s.lock", $Database);
-
-open LOCKFILE, "<$lockfile" || 
-    open LOCKFILE, ">$lockfile" ||
-    mailArtAndDie ("can't open $lockfile: $!") ;
-
-my $i ;
-for ($i = 0 ; $i < 5 ; $i++) {
-    flock LOCKFILE, $LOCK_EX && last ;
-    sleep 1 ;
-}
-
-mailArtAndDie ("can't lock $lockfile: $!") if ($i == 5) ;
-
-my %DATABASE ;
-dbmopen %DATABASE, $Database, 0666 || mailArtAndDie ("can't dbmopen $lockfile: $!");
-
-exit 0	if defined $DATABASE{$message_id}; # already seen.
-
-$DATABASE{$message_id} = sprintf "%d.%s", time, 'mailpost' ;
-
-mailArtAndDie ("TAS didn't set $message_id") unless defined $DATABASE{$message_id};
-
-dbmclose %DATABASE || mailArtAndDie ("can't dbmclose $lockfile: $!") ;
-
-flock LOCKFILE, $LOCK_UN || mailArtAndDie ("can't unlock $lockfile: $!");
-close LOCKFILE ;
-
-if (!open INEWS, "$WhereTo < $tmpfile 2>&1 |") {
-    mailArtAndDie ("cant start: $WhereTo: $!") ;
-}
-
-my @inews = <INEWS> ;
-close INEWS ;
-my $status = $? ;
-
-if (@inews) {
-    chomp @inews ;
-    mailArtAndDie ("inews failed: @inews") ;
-}
-
-unlink $tmpfile ;
-
-exit $status;
-
-sub mailArtAndDie {
-    my ($msg) = @_ ;
-    
-    print STDERR $msg,"\n" if -t STDERR ;
-    
-    open SENDMAIL, "|" . sprintf ($Sendmail,$Maintainer) ||
-	die "die($msg): sendmail: $!\n" ;
-    print SENDMAIL <<"EOF" ;
-To: $Maintainer
-Subject: mailpost failure ($newsgroups): $msg
-     
-$msg
-EOF
-	     
-    if ($tmpfile && -f $tmpfile) {
-	print SENDMAIL "\n-------- Article Contents\n\n" ;
-	open FILE, "<$tmpfile" || die "open($tmpfile): $!\n" ;
-	print SENDMAIL while <FILE> ;
-	close FILE ;
-    } else {
-	print "No article left to send back.\n" ;
-    }
-    close SENDMAIL ;
-    
-#    unlink $tmpfile ;
-    
-    exit (0) ;			# using a non-zero exit may cause problems.
-}
-
-
-#
-# take 822-format name (either "comment <addr> comment" or "addr (comment)")
-# and return in always-qualified 974-format ("addr (comment)").
-#
-sub fix_sender_addr {
-    my ($address) = @_;
-    my ($lcomment, $addr, $rcomment, $comment);
-    local ($',$`,$_) ;
-
-    if ($address =~ /\<([^\>]*)\>/) {
-	($lcomment, $addr, $rcomment) = (&dltb($`), &dltb($1), &dltb($'));
-    } elsif ($address =~ /\(([^\)]*)\)/) {
-	($lcomment, $addr, $rcomment) = ('', &dltb($`.$'), &dltb($1));
-    } else {
-	($lcomment, $addr, $rcomment) = ('', &dltb($address), '');
-    }
-    
-    #print STDERR "fix_sender_addr($address) == ($lcomment, $addr, $rcomment)\n";
-    
-    $addr .= "\@$Mailname" unless ($addr =~ /\@/);
-    
-    if ($lcomment && $rcomment) {
-	$comment = $lcomment . ' ' . $rcomment;
-    } else {
-	$comment = $lcomment . $rcomment;
-    }
-    
-    $_ = $addr;
-    $_ .= " ($comment)" if $comment;
-    
-    #print STDERR "\t-> $_\n";
-    
-    return $_;
-}
-
-#
-# delete leading and trailing blanks
-#
-
-sub dltb {
-    my ($str) = @_;
-    
-    $str =~ s/^\s+//o;
-    $str =~ s/\s+$//o;
-    
-    return $str;
-}
-
diff --git a/bin/post b/bin/post
index fbcea27..43e5f32 100755
--- a/bin/post
+++ b/bin/post
@@ -7,4 +7,5 @@ set -e
 tag=$1
 entry=`cdb -q groups.cdb "t:$tag"` || bad "unknown tag $tag"
 set -- $entry
-exec mailpost.newsgate -a $USER@`cat $QMAIL/control/me` -d mail mail.$2
+case "$SENDER" in $4) ;; *) bad "incorrect sender $SENDER" ;; esac
+exec bin/inject mail.$2
diff --git a/config b/config
index 1e1d271..ec2d0df 100644
--- a/config
+++ b/config
@@ -15,7 +15,7 @@ admin: root
 
 [post-default]
 #? addrcheck
-|preline bin/post "$DEFAULT"
+|bin/post "$DEFAULT"
 
 [mail-default]
 #? addrcheck