From a682e5d7bf69519110ced2e9502ad1d01f70314f Mon Sep 17 00:00:00 2001 From: Mark Wooding Date: Wed, 1 Mar 2006 16:53:25 +0000 Subject: [PATCH] Initial check-in. --- .gitignore | 8 + Makefile | 16 ++ README | 40 +++++ bin/mail | 15 ++ bin/mailpost.newsgate | 464 ++++++++++++++++++++++++++++++++++++++++++++++++++ bin/mkcdb | 20 +++ bin/mkgroups | 14 ++ bin/post | 10 ++ config | 17 ++ defs | 12 ++ 10 files changed, 616 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README create mode 100755 bin/mail create mode 100755 bin/mailpost.newsgate create mode 100755 bin/mkcdb create mode 100755 bin/mkgroups create mode 100755 bin/post create mode 100644 config create mode 100644 defs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..53f0420 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*~ +*.cdb +auth +groups +*.files +.qmail* +*.stamp +dead.article diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..adc9708 --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +## makefile for newsgate + +all: config.files auth.cdb groups.cdb newsconf.stamp + +config.files: config + splitconf config + +newsconf.stamp: groups + bin/mkgroups + touch newsconf.stamp + +auth.cdb: config.files + cdb-list auth.cdb auth + +groups.cdb: groups + bin/mkcdb groups diff --git a/README b/README new file mode 100644 index 0000000..6ceec51 --- /dev/null +++ b/README @@ -0,0 +1,40 @@ +REQUIRED STUFF + + INN 2 + qmail 1.03 + splitconf, tinycdb, libfile-cdb-perl (CDB_File.pm) + + +INITIAL INSTALLATION + + 0. Create a user `newsgate', and put this stuff in his home + directory. + + 1. Add a line to your /etc/news/moderators file which says + + mail.*:newsgate-%s@YOUR-DOMAIN + + 2. Edit the `config' file. The `auth' section will want to list + the domains who ought to be able to send to gatewayed mailing + lists through this server. This is effectively a relay: + don't leave it open. + + 3. Add mailing lists. Run `make'. + + +ADDING NEW MAILING LISTS TO NEWSGATE + + Add a line to the `groups' file containing + + TAG GROUP ADDR + + where newsgate-TAG@tux.nsict.org is subscribed to the list, + mail.GROUP is the newsgroup, and ADDR is the mailing list + posting address. + + Say `make' as root. + + +Local variables: +mode: text +End: diff --git a/bin/mail b/bin/mail new file mode 100755 index 0000000..20b2d66 --- /dev/null +++ b/bin/mail @@ -0,0 +1,15 @@ +#! /bin/sh + +set -e +. defs + +[ $# = 1 ] || fail "usage: $0 GROUP" +group=$1 +entry=`cdbget "g:$group" /dev/null"; + push @errorText, "((mailing_list: $mailing_list))\n"; +} + +$newsgroups = join ", ", @ARGV ; + +die "usage: $0 newsgroup [newsgroup]\n" unless $newsgroups; + + +# +# do the header. our input is a mail message, with or without the From_ +# + +#$message_id = sprintf("", time, $$, $Hostname); +my $real_news_hdrs = ''; +my $weird_mail_hdrs = ''; +my $fromHdr = "MAILPOST-UNKNOWN-FROM" ; +my $dateHdr= "MAILPOST-UNKNOWN-DATE" ; +my $msgIdHdr = "MAILPOST-UNKNOWN-MESSAGE-ID" ; +my $from = undef; +my $date = undef; +my $hdr = undef; +my $txt = undef; +my $message_id ; +my $subject = "(NONE)"; + +$_ = ; +if (!$_) { + if ( $debugging || -t STDERR ) { + die "empty input" ; + } else { + syslog "err", "empty input" ; + exit (0) ; + } +} + +chomp $_; + +my $line = undef; +if (/^From\s+([^\s]+)\s+/) { + $path = $1; + push @errorText, "((path: $path))\n"; + $_ = $'; + if (/ remote from /) { + $path = $' . '!' . $path; + $_ = $`; + } + $date = $_; +} else { + $line = $_; +} + +for (;;) { + last if defined($line) && ($line =~ /^$/) ; + + $_ = ; + chomp ; + + # gather up a single header with possible continuation lines into $line + if (/^\s+/) { + if (! $line) { + $msg = "First line with leading whitespace!" ; + syslog "err", $msg unless -t STDERR ; + die "$msg\n" ; + } + + $line .= "\n" . $_ ; + next ; + } + + # On the first header $line will be undefined. + ($_, $line) = ($line, $_) ; # swap $line and $_ ; + + last if defined($_) && /^$/ ; + next if /^$/ ; # only on first header will this happen + + push @errorText, "($_)\n"; + + next if /^Approved:\s/sio && defined($approved); + next if /^Distribution:\s/sio && defined($distribution); + + if (/^(Organization|Distribution):\s*/sio) { + $real_news_hdrs .= "$_\n"; + next; + } + + if (/^Subject:\s*/sio) { + $subject = $'; + next; + } + + if (/^Message-ID:\s*/sio) { + $message_id = $'; + next; + } + + if (/^Mailing-List:\s*/sio) { + $mailing_list = $'; + next; + } + + if (/^(Sender|Approved):\s*/sio) { + $real_news_hdrs .= "$&" . fix_sender_addr($') . "\n"; + next; + } + + if (/^Return-Path:\s*/sio) { + $path = $'; + $path = $1 if ($path =~ /\<([^\>]*)\>/); + push@errorText, "((path: $path))\n"; + next; + } + + if (/^Date:\s*/sio) { + $date = $'; + next; + } + + if (/^From:\s*/sio) { + $from = &fix_sender_addr($'); + next; + } + + if (/^References:\s*/sio) { + $references = $'; + next; + } + + if (!defined($references) && /^In-Reply-To:[^\<]*\<([^\>]+)\>/sio) { + $references = "<$1>"; + # FALLTHROUGH + } + + if (/^(MIME|Content)-[^:]+:\s*/sio) { + $real_news_hdrs .= $_ . "\n" ; + next ; + } + + # strip out news trace headers since otherwise posting may fail. other + # trace headers will be renamed to add 'X-' so we don't have to worry + # about them. + if (/^X-(Trace|Complaints-To):\s*/sio) { + next ; + } + + # strip out Received headers since otherwise posting may fail + # due to too large header size. + if (/^(Received):\s*/sio) { + next ; + } + + # random unknown header. prepend 'X-' if it's not already there. + $_ = "X-$_" unless /^X-/sio ; + $weird_mail_hdrs .= "$_\n"; +} + + +$msgIdHdr = $message_id if $message_id ; +$fromHdr = $from if $from ; +$dateHdr = $date if $date ; + +if ($path !~ /\!/) { + $path = "$'!$`" if ($path =~ /\@/); +} + +$real_news_hdrs .= "Subject: ${subject}\n"; +$real_news_hdrs .= "Message-ID: ${msgIdHdr}\n" if defined($message_id); +$real_news_hdrs .= "Mailing-List: ${mailing_list}\n" if defined($mailing_list); +$real_news_hdrs .= "Distribution: ${distribution}\n" if defined($distribution); +$real_news_hdrs .= "Approved: ${approved}\n" if defined($approved); +$real_news_hdrs .= "References: ${references}\n" if defined($references); + +# Remove duplicate headers. +my %headers = (); +$real_news_hdrs =~ s/(.*?:)[ \t].*?($|\n)([ \t]+.*?($|\n))*/$headers{lc$1}++?"":$&/ge; + +# Inews writes error messages to stdout. We want to capture those and mail +# them back to the newsmaster. Trying to write and read from a subprocess is +# ugly and prone to deadlock, so we use a temp file. +$tmpfile = sprintf "%s/mailpost.%d.%d", $Tmpdir, time, $$ ; + +if (!open TMPFILE,">$tmpfile") { + $msg = "cant open temp file ($tmpfile): $!" ; + $tmpfile = undef ; + syslog "err", "$msg\n" unless $debugging || -t STDERR ; + open TMPFILE, "|" . sprintf ($Sendmail, $Maintainer) || + die "die(no tmpfile): sendmail: $!\n" ; + print TMPFILE <<"EOF"; +To: $Maintainer +Subject: mailpost failure ($newsgroups): $msg + +-------- Article Contents + +EOF +} + +print TMPFILE <<"EOF"; +Path: ${path} +From: ${fromHdr} +Newsgroups: ${newsgroups} +${real_news_hdrs}Date: ${dateHdr} +${weird_mail_hdrs} +EOF + +my $rest; +$rest .= $_ while (); +$rest =~ s/\n*$/\n/g; # Remove trailing \n except very last + +print TMPFILE $rest; +close TMPFILE ; + +if ( ! $tmpfile ) { + # we had to bail and mail the article to the admin. + exit (0) ; +} + + +## +## We've got the article in a temp file and now we validate some of the +## data we found and update our message-id database. +## + +mailArtAndDie ("no From: found") unless $from; +mailArtAndDie ("no Date: found") unless $date; +mailArtAndDie ("no Message-ID: found") unless $message_id; +mailArtAndDie ("Malformed message ID ($message_id)") + if ($message_id !~ /\<(\S+)\@(\S+)\>/); + + +# update (with locking) our message-id database. this is used to make sure we +# don't loop our own gatewayed articles back through the mailing list. + +my ($lhs, $rhs) = ($1, $2); # of message_id match above. +$rhs =~ tr/A-Z/a-z/; + +$message_id = "${lhs}\@${rhs}"; + +push @errorText, "(TAS message-id database for $message_id)\n"; + +my $lockfile = sprintf("%s.lock", $Database); + +open LOCKFILE, "<$lockfile" || + open LOCKFILE, ">$lockfile" || + mailArtAndDie ("can't open $lockfile: $!") ; + +my $i ; +for ($i = 0 ; $i < 5 ; $i++) { + flock LOCKFILE, $LOCK_EX && last ; + sleep 1 ; +} + +mailArtAndDie ("can't lock $lockfile: $!") if ($i == 5) ; + +my %DATABASE ; +dbmopen %DATABASE, $Database, 0666 || mailArtAndDie ("can't dbmopen $lockfile: $!"); + +exit 0 if defined $DATABASE{$message_id}; # already seen. + +$DATABASE{$message_id} = sprintf "%d.%s", time, 'mailpost' ; + +mailArtAndDie ("TAS didn't set $message_id") unless defined $DATABASE{$message_id}; + +dbmclose %DATABASE || mailArtAndDie ("can't dbmclose $lockfile: $!") ; + +flock LOCKFILE, $LOCK_UN || mailArtAndDie ("can't unlock $lockfile: $!"); +close LOCKFILE ; + +if (!open INEWS, "$WhereTo < $tmpfile 2>&1 |") { + mailArtAndDie ("cant start: $WhereTo: $!") ; +} + +my @inews = ; +close INEWS ; +my $status = $? ; + +if (@inews) { + chomp @inews ; + mailArtAndDie ("inews failed: @inews") ; +} + +unlink $tmpfile ; + +exit $status; + +sub mailArtAndDie { + my ($msg) = @_ ; + + print STDERR $msg,"\n" if -t STDERR ; + + open SENDMAIL, "|" . sprintf ($Sendmail,$Maintainer) || + die "die($msg): sendmail: $!\n" ; + print SENDMAIL <<"EOF" ; +To: $Maintainer +Subject: mailpost failure ($newsgroups): $msg + +$msg +EOF + + if ($tmpfile && -f $tmpfile) { + print SENDMAIL "\n-------- Article Contents\n\n" ; + open FILE, "<$tmpfile" || die "open($tmpfile): $!\n" ; + print SENDMAIL while ; + close FILE ; + } else { + print "No article left to send back.\n" ; + } + close SENDMAIL ; + +# unlink $tmpfile ; + + exit (0) ; # using a non-zero exit may cause problems. +} + + +# +# take 822-format name (either "comment comment" or "addr (comment)") +# and return in always-qualified 974-format ("addr (comment)"). +# +sub fix_sender_addr { + my ($address) = @_; + my ($lcomment, $addr, $rcomment, $comment); + local ($',$`,$_) ; + + if ($address =~ /\<([^\>]*)\>/) { + ($lcomment, $addr, $rcomment) = (&dltb($`), &dltb($1), &dltb($')); + } elsif ($address =~ /\(([^\)]*)\)/) { + ($lcomment, $addr, $rcomment) = ('', &dltb($`.$'), &dltb($1)); + } else { + ($lcomment, $addr, $rcomment) = ('', &dltb($address), ''); + } + + #print STDERR "fix_sender_addr($address) == ($lcomment, $addr, $rcomment)\n"; + + $addr .= "\@$Mailname" unless ($addr =~ /\@/); + + if ($lcomment && $rcomment) { + $comment = $lcomment . ' ' . $rcomment; + } else { + $comment = $lcomment . $rcomment; + } + + $_ = $addr; + $_ .= " ($comment)" if $comment; + + #print STDERR "\t-> $_\n"; + + return $_; +} + +# +# delete leading and trailing blanks +# + +sub dltb { + my ($str) = @_; + + $str =~ s/^\s+//o; + $str =~ s/\s+$//o; + + return $str; +} + diff --git a/bin/mkcdb b/bin/mkcdb new file mode 100755 index 0000000..7cea767 --- /dev/null +++ b/bin/mkcdb @@ -0,0 +1,20 @@ +#! /usr/bin/perl + +use CDB_File; + +@ARGV == 1 or die "usage: $0 GROUPFILE\n"; +$g = shift; +open GROUPS, $g or die "$0: open($g): $!\n"; +$c = CDB_File->new("$g.cdb", "$g.new") or die "$0: cdbmake($g.cdb): $!\n"; +while () { + next if m'^\s*(\#|$)'; + my @f = split; + @f == 3 or die "$0: bad info line $.\n"; + my $l = join(" ", @f); + my ($tag, $group, $addr) = @f; + $group =~ tr/./-/; + $c->insert("t:$tag" => $l); + $c->insert("g:$group" => $l); +} +$c->finish() or die "$0: cdbfinish($g.cdb): $!\n"; +exit 0; diff --git a/bin/mkgroups b/bin/mkgroups new file mode 100755 index 0000000..49b79ff --- /dev/null +++ b/bin/mkgroups @@ -0,0 +1,14 @@ +#! /bin/sh + +set -e +ACTIVE=/var/lib/news/active +run () { echo -n "$@: "; "$@"; } +sed '/^[ ]*\(#\|$\)/ d; s/[ ]*=[ ]*/ /' groups | +while read tag group addr; do + set -- `grep "mail.$group" $ACTIVE` + if [ $# -gt 0 ]; then + [ m = $4 ] || run ctlinnd changegroup mail.$group m + else + run ctlinnd newgroup mail.$group m newsgate + fi +done \ No newline at end of file diff --git a/bin/post b/bin/post new file mode 100755 index 0000000..fbcea27 --- /dev/null +++ b/bin/post @@ -0,0 +1,10 @@ +#! /bin/sh + +set -e +. defs + +[ $# = 1 ] || fail "usage: $0 TAG" +tag=$1 +entry=`cdb -q groups.cdb "t:$tag"` || bad "unknown tag $tag" +set -- $entry +exec mailpost.newsgate -a $USER@`cat $QMAIL/control/me` -d mail mail.$2 diff --git a/config b/config new file mode 100644 index 0000000..5fac6c3 --- /dev/null +++ b/config @@ -0,0 +1,17 @@ +## qmail configuration for newsgate + +before = chmod +t . +after = chmod -t . + +.qmail-admin: root +.qmail-default: |preline bin/post "$EXT" +.qmail-in-default: |preline bin/post "$EXT2" + +[.qmail-mail-default] +|check-sender &2 "newsgate: fatal: $@"; exit 100; } +fail () { echo >&2 "newsgate: $@"; exit 111; } + -- 2.11.0