Merge branch 'master' into distorted
authorMark Wooding <mdw@distorted.org.uk>
Mon, 10 Mar 2014 10:49:15 +0000 (10:49 +0000)
committerMark Wooding <mdw@distorted.org.uk>
Mon, 10 Mar 2014 10:49:15 +0000 (10:49 +0000)
* master:
  New ftpsync upstream.
  bin/fetch-ftpsync: Add a script to update the `ftpsync' machinery.
  lib/functions.sh: Tweak rsync options somewhat.
  Isolate ftpsync into its own branch.

13 files changed:
.userv/rc
bin/run-mirrors
etc/ftpsync-common.conf
etc/ftpsync-debian-security.conf
etc/ftpsync-debian.conf
etc/logrotate.conf
etc/mirror.packages
etc/mirrors.d/50-debian [new file with mode: 0755]
etc/mirrors.d/50-distorted [new file with mode: 0755]
etc/mirrors.d/50-eprint-bib [new file with mode: 0755]
etc/mirrors.d/50-mirror [new file with mode: 0755]
etc/mirrors.d/50-rfc [new file with mode: 0755]
lib/rfcxmlindex2bibtex.xslt [new file with mode: 0644]

index b86f0c0..daff920 100644 (file)
--- a/.userv/rc
+++ b/.userv/rc
@@ -1,7 +1,9 @@
 ### -*-conf-*-
 
 if ( glob service run
-   & glob calling-group ftpadmin
+   & ( glob calling-group ftpadmin
+     | glob calling-user root
+     )
    )
        no-suppress-args
        execute bin/run-mirrors
index 9c24c7d..8fb9ccc 100755 (executable)
@@ -9,8 +9,8 @@ case $(id -un) in
 esac
 
 ## Set up a plausible environment.
-HOME=/var/lib/mirror-admin; export HOME; cd
-MIRRORS=/mnt/mirrors; export MIRRORS
+HOME=/var/lib/mirror; export HOME; cd
+MIRRORS=/mnt/ftp/pub/mirrors; export MIRRORS
 PATH=$HOME/bin:/usr/bin:/usr/sbin:/bin:/sbin:/usr/local/bin:/usr/local/sbin
 export PATH
 umask 002
index 5af48ee..19cff88 100644 (file)
@@ -21,7 +21,7 @@
 : ${RSYNC_PATH="$LABEL"}
 
 ## The host we mirror from
-: ${RSYNC_HOST=ftp.de.debian.org}
+: ${RSYNC_HOST=ftp.uk.debian.org}
 
 ## In case we need a user to access the rsync share at our upstream host
 : ${RSYNC_USER=}
@@ -30,7 +30,7 @@
 : ${RSYNC_PASSWORD=}
 
 ## Architectures we want to carry.
-: ${WANT_ARCH="i386:amd64"}
+: ${WANT_ARCH="i386:amd64:armel"}
 
 ## Distributions to inspect for available architectures.
 : ${SEARCH_DISTS="oldstable stable testing unstable experimental"}
@@ -83,7 +83,6 @@
 ## want.  Crazy, no?
 ARCH_EXCLUDE=$(
        bin/debian-excludes $RSYNC_HOST $RSYNC_PATH $WANT_ARCH $SEARCH_DISTS
-       echo source
 )
 
 ## Do we have leaf mirror to signal we are done and they should sync?
index 17aaf8d..7069fa8 100644 (file)
@@ -5,7 +5,7 @@
 LABEL=debian-security
 RSYNC_HOST=security.debian.org
 SEARCH_DISTS=$(
-       for i in oldstable stable testing; do
+       for i in stable testing; do
          echo $i/updates
        done
 )
index 3417c94..db7a9fd 100644 (file)
@@ -3,7 +3,7 @@
 ### ftpsync configuration for the Debian main archive.
 
 LABEL=debian
-RSYNC_HOST=ftp.de.debian.org
-SEARCH_DISTS="oldstable stable testing unstable experimental"
+RSYNC_HOST=ftp.uk.debian.org
+SEARCH_DISTS="stable testing unstable experimental"
 
 . etc/ftpsync-common.conf
index 601aeb5..ce3c186 100644 (file)
@@ -2,7 +2,7 @@
 ###
 ### Log rotation configuration for mirrors.
 
-/var/lib/mirror-admin/log/*.log {
+/var/lib/mirror/log/*.log {
        compress
        rotate 8
        notifempty
index a790722..a3c951c 100644 (file)
@@ -24,4 +24,14 @@ package=defaults
        do_deletes=true
        max_delete_files=80%
 
+###--------------------------------------------------------------------------
+### Nelson Beebe's BibTeX database archive.
+
+package=utah-bib
+       site=ftp.math.utah.edu
+       remote_dir=/pub/tex/bib/
+       local_dir=utah-bib
+       recursive=false
+       get_patt=\.bib$
+
 ###----- That's all, folks --------------------------------------------------
diff --git a/etc/mirrors.d/50-debian b/etc/mirrors.d/50-debian
new file mode 100755 (executable)
index 0000000..c17194d
--- /dev/null
@@ -0,0 +1,7 @@
+#! /bin/sh -e
+###
+### Mirror Debian archives.
+
+for archive in debian-security debian; do
+  RSYNC="rsync -4" bin/ftpsync sync:all sync:archive:$archive
+done
diff --git a/etc/mirrors.d/50-distorted b/etc/mirrors.d/50-distorted
new file mode 100755 (executable)
index 0000000..1a3316e
--- /dev/null
@@ -0,0 +1,13 @@
+#! /bin/sh -e
+###
+### Mirror `distorted.org.uk' FTP archive from telecaster.
+
+. lib/functions.sh
+cd $MIRRORS
+
+## Fetch the RFC collection.
+standard_rsync "$@" \
+  --exclude="/pub/debian/mini-dinstall/" \
+  ftp.distorted.org.uk::ftp/pub \
+  ftp.distorted.org.uk::ftp/u \
+  distorted
diff --git a/etc/mirrors.d/50-eprint-bib b/etc/mirrors.d/50-eprint-bib
new file mode 100755 (executable)
index 0000000..8065424
--- /dev/null
@@ -0,0 +1,42 @@
+#! /bin/bash
+
+set -e
+
+## Initial setup.
+mkdir -p $MIRRORS/eprint-bib; cd $MIRRORS/eprint-bib
+eprint=http://eprint.iacr.org
+rm -f list tags
+
+## Fetch the master index.
+curl -sS -o list -z '1 hour ago' $eprint/complete/
+[ -r list ] || exit 0
+
+## Pick out the article tags.
+sed -n '/^<a href="\/\([0-9]\+\/[0-9]\+\)">.*$/ s::\1:p' list >tags
+
+## For each tag, make sure we have the article information.
+while read tag; do
+  dir=${tag%/*}
+  mkdir -p $dir
+  file=$dir/eprint-${tag/\//-}.bib
+  [ -r $file ] || {
+    curl -sS -o $file.in "$eprint/cgi-bin/cite.pl?entry=$tag"
+    sed -n "
+    /^<PRE>/,/<\/PRE>/ {
+      /^<PRE>/ d
+      s:</PRE>.*::
+      s%note =.*\$%url = {http://eprint.iacr.org/$tag},%
+      p
+    }" $file.in >$file.sed
+    mv $file.sed $file
+    rm -f $file.in
+    echo $tag
+  }
+done <tags
+
+## Build the main bibliography.
+{
+  cat header
+  find . -name 'eprint-*.bib' -print | sort | xargs cat
+} >eprint.bib.new
+mv eprint.bib.new eprint.bib
diff --git a/etc/mirrors.d/50-mirror b/etc/mirrors.d/50-mirror
new file mode 100755 (executable)
index 0000000..a65823d
--- /dev/null
@@ -0,0 +1,7 @@
+#! /bin/sh -e
+###
+### Lee MacLoughlin's mirror scripts.
+
+here=$(pwd)
+cd $MIRRORS
+mirror -d "$@" "$here"/etc/mirror.packages
diff --git a/etc/mirrors.d/50-rfc b/etc/mirrors.d/50-rfc
new file mode 100755 (executable)
index 0000000..29dcca4
--- /dev/null
@@ -0,0 +1,29 @@
+#! /bin/sh -e
+###
+### Mirror RFCs and Internet Drafts from mirrorservice.org
+
+. lib/functions.sh
+cd $MIRRORS
+
+## Fetch the RFC collection.
+standard_rsync "$@" \
+  --exclude=/tar/ \
+  ftp.rfc-editor.org::rfc-ed-all/ \
+  rfc/
+
+## Fetch the Internet Draft collection.
+standard_rsync "$@" \
+  ftp.rfc-editor.org::internet-drafts/ \
+  internet-drafts/
+
+## Build a usable BibTeX database of RFCs.  XSLT runes are by Roland
+## Bless, taken from http://www.tm.uka.de/~bless/bibrfcindex.html
+mkdir -p rfc-bib
+if [ rfc/rfc-index.xml -nt rfc-bib/rfc.bib ]; then
+  xsltproc $HOME/lib/rfcxmlindex2bibtex.xslt rfc/rfc-index.xml |
+  sed '
+    s/\([_&%#$]\)/\\\1/g
+    /author=/s/\([^ ]*\) \(3rd\|Jr\.\)/\{\1 \2\}/g
+  ' >rfc-bib/rfc-bib.new
+  mv rfc-bib/rfc-bib.new rfc-bib/rfc.bib
+fi
diff --git a/lib/rfcxmlindex2bibtex.xslt b/lib/rfcxmlindex2bibtex.xslt
new file mode 100644 (file)
index 0000000..3006969
--- /dev/null
@@ -0,0 +1,233 @@
+<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0" 
+               xmlns:rfcidx="http://www.rfc-editor.org/rfc-index" >
+<xsl:output method="text"/>
+
+<!-- $Id: rfcxmlindex2bibtex.xslt,v 1.7 2004/12/10 22:47:36 bless Exp $ -->
+
+<!-- rfcxmlindex2bibtex.xslt written by Roland Bless  -->
+<!-- This is my very first xslt script, so please don't blame me for bad coding style -->
+<!-- Get rfc-index in xml format from ftp://ftp.isi.edu/in-notes/rfc-index.xml
+     then translate it to bibtex format by invoking e.g.: 
+     xsltproc  rfcxmlindex2bibtex rfc-index.xml | sed 's/_/\\_/'
+     or 
+     java org.apache.xalan.xslt.Process -in rfc-index.xml -xsl rfcxmlindex2bibtex.xslt -out -  | sed 's/_/\\_/'
+     -->
+
+<!-- customizable variables -->
+
+<xsl:variable name="entrytype">
+  <xsl:text>@misc</xsl:text>
+</xsl:variable>
+
+<xsl:variable name="rfcname">
+<xsl:text>RFC</xsl:text>
+</xsl:variable>
+
+<xsl:variable name="urlrfcretrieval">
+<xsl:text>http://www.ietf.org/rfc/</xsl:text>
+</xsl:variable>
+
+<xsl:variable name="obsoletedby">
+<xsl:text>Obsoleted by </xsl:text>
+</xsl:variable>
+
+<xsl:variable name="updatedby">
+<xsl:text>Updated by </xsl:text>
+</xsl:variable>
+
+<xsl:variable name="statFS">STANDARD</xsl:variable>
+<xsl:variable name="statDS">DRAFT STANDARD</xsl:variable>
+<xsl:variable name="statPS">PROPOSED STANDARD</xsl:variable>
+<xsl:variable name="statU">UNKNOWN</xsl:variable>
+<xsl:variable name="statBCP">BEST CURRENT PRACTICE</xsl:variable>
+<xsl:variable name="statFYI">FOR YOUR INFORMATION</xsl:variable>
+<xsl:variable name="statEXP">EXPERIMENTAL</xsl:variable>
+<xsl:variable name="statH">HISTORIC</xsl:variable>
+<xsl:variable name="statI">INFORMATIONAL</xsl:variable>
+
+<xsl:variable name="lstatFS">Standard</xsl:variable>
+<xsl:variable name="lstatDS">Draft Standard</xsl:variable>
+<xsl:variable name="lstatPS">Proposed Standard</xsl:variable>
+<xsl:variable name="lstatU">Unknown</xsl:variable>
+<xsl:variable name="lstatBCP">Best Current Practice</xsl:variable>
+<xsl:variable name="lstatFYI">For Your Information</xsl:variable>
+<xsl:variable name="lstatEXP">Experimental</xsl:variable>
+<xsl:variable name="lstatH">Historic</xsl:variable>
+<xsl:variable name="lstatI">Informational</xsl:variable>
+
+<!-- internal variables -->
+
+<xsl:variable name="newline">
+<xsl:text>
+</xsl:text>
+</xsl:variable>
+
+<xsl:variable name="escapesyms">
+  <xsl:text>_</xsl:text>
+</xsl:variable>
+
+<!-- template matches -->
+
+<xsl:template match="/">
+  <xsl:apply-templates select="rfcidx:rfc-index"/>
+</xsl:template>
+
+<xsl:template match="rfcidx:rfc-index">
+  <xsl:apply-templates select="rfcidx:rfc-entry"/>
+</xsl:template>
+
+<xsl:template match="rfcidx:rfc-entry">
+  <xsl:value-of select="$entrytype" /><xsl:text>{rfc</xsl:text>
+  <xsl:variable name="rfcno" select="number(substring-after(rfcidx:doc-id/text(),$rfcname))"/>
+  <xsl:value-of select="$rfcno"/>,
+  <xsl:text>author="</xsl:text>
+  <xsl:value-of select="rfcidx:author/rfcidx:name"/>
+  <xsl:for-each select="rfcidx:author/rfcidx:name">
+  <xsl:choose>
+   <xsl:when test="position()=1"><xsl:value-of select="rfcidx:author/rfcidx:name"/></xsl:when>
+   <xsl:when test="position()>1"><xsl:text> and </xsl:text><xsl:value-of select="."/></xsl:when>
+  </xsl:choose>
+  </xsl:for-each>",
+  <xsl:apply-templates  select="rfcidx:title" mode="plain"/>,
+  <xsl:text>series="Request for Comments"</xsl:text>,
+  <xsl:text>number="</xsl:text><xsl:value-of select="$rfcno"/>",
+  <xsl:text>howpublished="RFC </xsl:text><xsl:value-of select="$rfcno"/>
+  <xsl:variable name="rstatus" select="rfcidx:current-status"/>                            
+  <xsl:variable name="cstatus">
+  <xsl:choose>
+      <xsl:when test="$rstatus=$statFS" ><xsl:value-of select="$lstatFS"/></xsl:when>
+      <xsl:when test="$rstatus=$statDS" ><xsl:value-of select="$lstatDS"/></xsl:when>
+      <xsl:when test="$rstatus=$statPS" ><xsl:value-of select="$lstatPS"/></xsl:when>
+      <xsl:when test="$rstatus=$statU"  ><xsl:value-of select="$lstatU"/></xsl:when>
+      <xsl:when test="$rstatus=$statBCP"><xsl:value-of select="$lstatBCP"/></xsl:when>
+      <xsl:when test="$rstatus=$statFYI"><xsl:value-of select="$lstatFYI"/></xsl:when>
+      <xsl:when test="$rstatus=$statEXP"><xsl:value-of select="$lstatEXP"/></xsl:when>
+      <xsl:when test="$rstatus=$statH"  ><xsl:value-of select="$lstatH"/></xsl:when>
+      <xsl:when test="$rstatus=$statI"><xsl:value-of select="$lstatI"/></xsl:when>
+  <xsl:otherwise>
+      <xsl:value-of select="rfcidx:current-status"/>
+  </xsl:otherwise>
+  </xsl:choose>
+  </xsl:variable>
+  <xsl:if test="not(rfcidx:current-status='UNKNOWN')"> (<xsl:value-of select="$cstatus"/>)</xsl:if>",
+  <xsl:text>publisher="IETF"</xsl:text>,
+  <xsl:text>organisation="Internet Engineering Task Force"</xsl:text>,
+  <xsl:text>year=</xsl:text><xsl:value-of select="rfcidx:date/rfcidx:year"/>,
+  <xsl:text>month=</xsl:text><xsl:value-of select="translate(substring(rfcidx:date/rfcidx:month,1,3),'ABCDEFGHIJKMLNOPQRSTUVWXYZ','abcdefghijkmlnopqrstuvwxyz')"/>,
+  <xsl:apply-templates  select="rfcidx:date/rfcidx:day"/>
+  <xsl:if test="count(rfcidx:obsoleted-by|rfcidx:updated-by) &gt; 0">
+  <xsl:text>  note="</xsl:text>
+  </xsl:if>
+  <xsl:apply-templates  select="rfcidx:obsoleted-by"/>
+  <xsl:if test="count(rfcidx:updated-by) &gt; 0">
+  <xsl:choose>
+  <xsl:when test="count(rfcidx:obsoleted-by) &gt; 0">
+    <xsl:text>, </xsl:text><xsl:value-of select="translate(substring($updatedby,1,1),'ABCDEFGHIJKMLNOPQRSTUVWXYZ','abcdefghijkmlnopqrstuvwxyz')"/><xsl:value-of select="substring($updatedby,2)"/>
+  </xsl:when>
+  <xsl:otherwise><xsl:value-of select="$updatedby"/></xsl:otherwise>
+  </xsl:choose>
+  <xsl:apply-templates  select="rfcidx:updated-by"/>
+  </xsl:if>
+  <xsl:if test="count(rfcidx:obsoleted-by|rfcidx:updated-by) &gt; 0"><xsl:text>",</xsl:text><xsl:value-of select="$newline"/></xsl:if>
+  <xsl:text>  url="</xsl:text><xsl:value-of select="$urlrfcretrieval"/><xsl:text>rfc</xsl:text><xsl:value-of select="$rfcno"/>.txt",
+<xsl:text>}</xsl:text><xsl:value-of select="$newline"/>
+<xsl:value-of select="$newline"/>
+</xsl:template>
+
+
+<xsl:template match="rfcidx:title" mode="plain">
+  <xsl:text>title="{</xsl:text><xsl:value-of select="."/><xsl:text>}"</xsl:text>
+</xsl:template>
+
+<!-- replaces _ with \_ , but it is very slow -> better use sed -->
+<xsl:template match="rfcidx:title" mode="replace">
+  <xsl:text>title="{</xsl:text>
+      <xsl:variable name="originaltitle" select="."/>
+      <xsl:variable name="return">
+      <xsl:choose>
+        <xsl:when test="contains($originaltitle,$escapesyms)">
+            <xsl:call-template name="escapechars">
+              <xsl:with-param name="inputstr"><xsl:value-of select="$originaltitle"/></xsl:with-param>
+            </xsl:call-template>
+        </xsl:when>
+        <xsl:otherwise>
+         <xsl:value-of select="$originaltitle"/>
+        </xsl:otherwise>
+      </xsl:choose>
+      </xsl:variable>
+      <xsl:value-of select="$return"/>
+   <xsl:text>}"</xsl:text>
+</xsl:template>
+
+
+<!-- replaces escapechar with \escapechar -->
+<xsl:template name="escapechars">
+ <xsl:param name="inputstr"/>
+  <xsl:variable name="first">
+      <xsl:choose>
+        <xsl:when test="contains($inputstr, $escapesyms)">
+          <xsl:value-of select="substring-before($inputstr, $escapesyms)"/>
+        </xsl:when>
+        <xsl:otherwise>
+          <xsl:value-of select="$inputstr"/>
+        </xsl:otherwise>
+      </xsl:choose>
+  </xsl:variable>
+    <xsl:variable name="middle">
+      <xsl:choose>
+        <xsl:when test="contains($inputstr, $escapesyms)">
+          <xsl:text>\</xsl:text><xsl:value-of select="$escapesyms"/>
+        </xsl:when>
+        <xsl:otherwise>
+          <xsl:text></xsl:text>
+        </xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+  <xsl:variable name="last">
+      <xsl:choose>
+        <xsl:when test="contains($inputstr, $escapesyms)">
+          <xsl:choose>
+            <xsl:when test="contains(substring-after($inputstr, $escapesyms), $escapesyms)">
+              <xsl:call-template name="escapechars">
+                <xsl:with-param name="inputstr"><xsl:value-of select="substring-after($inputstr, $escapesyms)"/></xsl:with-param>
+              </xsl:call-template>
+            </xsl:when>
+            <xsl:otherwise>
+              <xsl:value-of select="substring-after($inputstr, $escapesyms)"/>
+            </xsl:otherwise>
+          </xsl:choose>
+        </xsl:when>
+        <xsl:otherwise>
+          <xsl:text></xsl:text>
+        </xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+  <xsl:value-of select="concat($first, $middle, $last)"/>
+</xsl:template>
+
+<xsl:template match="rfcidx:date/rfcidx:day">
+<xsl:text>day="</xsl:text><xsl:value-of select="number(.)"/>",  
+</xsl:template>
+
+<xsl:template match="rfcidx:obsoleted-by">
+  <xsl:value-of select="$obsoletedby"/>
+  <xsl:text>RFC</xsl:text><xsl:if test="count(rfcidx:doc-id) &gt; 1">s</xsl:if><xsl:text> </xsl:text>
+  <xsl:for-each select="rfcidx:doc-id">
+  <xsl:choose>
+   <xsl:when test="position()=1"><xsl:value-of select="number(substring-after(text(),$rfcname))"/></xsl:when>
+   <xsl:when test="position()>1"><xsl:text>, </xsl:text><xsl:value-of select="number(substring-after(text(),$rfcname))"/></xsl:when>
+  </xsl:choose>
+  </xsl:for-each>
+</xsl:template>
+
+<xsl:template match="rfcidx:updated-by">
+  <xsl:text>RFC</xsl:text><xsl:if test="count(rfcidx:doc-id) &gt; 1">s</xsl:if><xsl:text> </xsl:text>
+  <xsl:for-each select="rfcidx:doc-id">
+  <xsl:choose>
+   <xsl:when test="position()=1"><xsl:value-of select="number(substring-after(text(),$rfcname))"/></xsl:when>
+   <xsl:when test="position()>1"><xsl:text>, </xsl:text><xsl:value-of select="number(substring-after(text(),$rfcname))"/></xsl:when>
+  </xsl:choose>
+  </xsl:for-each>
+</xsl:template>
+
+</xsl:transform>