From c8292b34485a2e00e676023d4164dd5841e4659f Mon Sep 17 00:00:00 2001 From: Mark Wooding Date: Sun, 3 Jun 2018 13:48:30 +0100 Subject: [PATCH] rough work in progress; may not build --- Makefile | 14 +- admin.scala | 27 ++-- jni.c | 72 ++++++++-- keys.scala | 347 +++++++++++++++++++++++++++++++++++--------- peers.scala | 44 +----- progress.scala | 90 ++++++++++++ sys.scala | 331 ++++++++++++++++++++++++++++++++++-------- tar.scala | 446 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ util.scala | 182 +++++++++++++++++++---- 9 files changed, 1323 insertions(+), 230 deletions(-) create mode 100644 progress.scala create mode 100644 tar.scala diff --git a/Makefile b/Makefile index fb3c83a..ff2a65c 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,8 @@ PKGS_CFLAGS := $(foreach p,$(PKGS),$(shell pkg-config --cflags $p)) PKGS_LIBS := $(foreach p,$(PKGS),$(shell pkg-config --libs $p)) CC = gcc -CFLAGS = -O2 -g -Wall -fPIC $(addprefix -I,$(INCLUDES)) +CFLAGS = -O2 -g -Wall -pedantic -Werror \ + -fPIC $(addprefix -I,$(INCLUDES)) CFLAGS += $(PKGS_CFLAGS) LD = gcc @@ -26,7 +27,8 @@ JAVAC = javac JAVAFLAGS = SCALAC = fsc -SCALAFLAGS = -optimise +SCALAFLAGS = -optimise -Xlint -Xlint:-package-object-classes \ + -Yinline-warnings:false ## Hack around https://issues.scala-lang.org/browse/SI-9689 SCALAFLAGS += -Yno-load-impl-class @@ -64,7 +66,13 @@ TARGETS += sys.stamp sys.stamp: util.stamp TARGETS += admin.stamp -admin.stamp: util.stamp sys.stamp +admin.stamp: sys.stamp util.stamp + +TARGETS += tar.stamp +tar.stamp: sys.stamp util.stamp + +TARGETS += keys.stamp +keys.stamp: tar.stamp sys.stamp util.stamp TARGETS += main.stamp main.stamp: sys.stamp diff --git a/admin.scala b/admin.scala index 01d7995..56f1c30 100644 --- a/admin.scala +++ b/admin.scala @@ -42,33 +42,30 @@ sealed abstract class Message; sealed abstract class JobMessage extends Message; case object JobOK extends JobMessage; -case class JobInfo(info: Seq[String]) extends JobMessage; -case class JobFail(err: Seq[String]) extends JobMessage; +final case class JobInfo(info: Seq[String]) extends JobMessage; +final case class JobFail(err: Seq[String]) extends JobMessage; case object JobLostConnection extends JobMessage; -case class BackgroundJobMessage(tag: String, msg: JobMessage) +final case class BackgroundJobMessage(tag: String, msg: JobMessage) extends Message; -case class JobDetached(tag: String) extends Message; +final case class JobDetached(tag: String) extends Message; sealed abstract class AsyncMessage extends Message; -case class Trace(msg: String) extends AsyncMessage; -case class Warning(err: Seq[String]) extends AsyncMessage; -case class Notify(note: Seq[String]) extends AsyncMessage; +final case class Trace(msg: String) extends AsyncMessage; +final case class Warning(err: Seq[String]) extends AsyncMessage; +final case class Notify(note: Seq[String]) extends AsyncMessage; case object ConnectionLost extends AsyncMessage; sealed abstract class ServiceMessage extends Message; -case class ServiceCancel(jobid: String) extends ServiceMessage; -case class ServiceClaim(svc: String, version: String) +final case class ServiceCancel(jobid: String) extends ServiceMessage; +final case class ServiceClaim(svc: String, version: String) extends ServiceMessage; -case class ServiceJob(jobid: String, svc: String, +final case class ServiceJob(jobid: String, svc: String, cmd: String, args: Seq[String]) extends ServiceMessage; /*----- Main code ---------------------------------------------------------*/ -object Connection { -} - class ConnectionClosed extends Exception; class ServerFailed(msg: String) extends Exception(msg); @@ -250,7 +247,7 @@ println(s";; line: $line"); case msg: AsyncMessage => publish(msg); case _: ServiceMessage => - (); + ok; } } } catch { @@ -264,7 +261,7 @@ println(s";; line: $line"); j.ch.write(JobLostConnection); fgjob = None; notifyAll(); - case None => (); + case None => ok; } } publish(ConnectionLost); diff --git a/jni.c b/jni.c index 934be91..f65e464 100644 --- a/jni.c +++ b/jni.c @@ -319,17 +319,17 @@ static const struct errtab { const char *tag; int err; } errtab[] = { ECANCELED ENOKEY EKEYEXPIRED EKEYREVOKED EKEYREJECTED EOWNERDEAD ENOTRECOVERABLE ERFKILL EHWPOISON))) (save-excursion - (goto-char (point-min)) - (search-forward (concat "***" "BEGIN errtab" "***")) - (beginning-of-line 2) - (delete-region (point) - (progn - (search-forward "***END***") - (beginning-of-line) - (point))) - (dolist (err errors) - (insert (format "#ifdef %s\n { \"%s\", %s },\n#endif\n" - err err err))))) + (goto-char (point-min)) + (search-forward (concat "***" "BEGIN errtab" "***")) + (beginning-of-line 2) + (delete-region (point) + (progn + (search-forward "***END***") + (beginning-of-line) + (point))) + (dolist (err errors) + (insert (format "#ifdef %s\n { \"%s\", %s },\n#endif\n" + err err err))))) */ /***BEGIN errtab***/ #ifdef EPERM @@ -762,6 +762,48 @@ JNIEXPORT jobject JNIFUNC(errtab)(JNIEnv *jni, jobject cls) JNIEXPORT jobject JNIFUNC(strerror)(JNIEnv *jni, jobject cls, jint err) { return (wrap_cstring(jni, strerror(err))); } +/*----- Messing with file descriptors -------------------------------------*/ + +static void fdguts(JNIEnv *jni, jclass *cls, jfieldID *fid) +{ + *cls = (*jni)->FindClass(jni, "java/io/FileDescriptor"); assert(cls); + *fid = (*jni)->GetFieldID(jni, *cls, "fd", "I"); // OpenJDK + if (!*fid) *fid = (*jni)->GetFieldID(jni, *cls, "descriptor", "I"); // Android + assert(*fid); +} + +static int fdint(JNIEnv *jni, jobject jfd) +{ + jclass cls; + jfieldID fid; + + fdguts(jni, &cls, &fid); + return ((*jni)->GetIntField(jni, jfd, fid)); +} + +static jobject newfd(JNIEnv *jni, int fd) +{ + jobject jfd; + jclass cls; + jmethodID init; + jfieldID fid; + + fdguts(jni, &cls, &fid); + init = (*jni)->GetMethodID(jni, cls, "", "()V"); assert(init); + jfd = (*jni)->NewObject(jni, cls, init); + (*jni)->SetIntField(jni, jfd, fid, fd); + return (jfd); +} + +JNIEXPORT jint JNIFUNC(fdint)(JNIEnv *jni, jobject cls, jobject jfd) + { return (fdint(jni, jfd)); } + +JNIEXPORT jobject JNIFUNC(newfd)(JNIEnv *jni, jobject cls, jint fd) + { return (newfd(jni, fd)); } + +JNIEXPORT jboolean JNIFUNC(isatty)(JNIEnv *jni, jobject cls, jobject jfd) + { return (isatty(fdint(jni, jfd))); } + /*----- Low-level file operations -----------------------------------------*/ /* Java has these already, as methods on `java.io.File' objects. Alas, these @@ -853,8 +895,8 @@ end: put_cstring(jni, to, tostr); } -#define LKF_EXCL 1u -#define LKF_WAIT 2u +#define LKF_EXCL 0x1000u +#define LKF_WAIT 0x2000u struct lockf { struct native_base _base; int fd; @@ -875,7 +917,7 @@ JNIEXPORT wrapper JNIFUNC(lock)(JNIEnv *jni, jobject cls, pathstr = get_cstring(jni, path); if (!pathstr) goto end; again: - fd = open(pathstr, O_RDWR | O_CREAT); if (fd < 0) goto err; + fd = open(pathstr, O_RDWR | O_CREAT, flags&07777); if (fd < 0) goto err; if (fstat(fd, &st0)) goto err; f = fcntl(fd, F_GETFD); if (f < 0) goto err; if (fcntl(fd, F_SETFD, f | FD_CLOEXEC)) goto err; @@ -927,7 +969,7 @@ static jobject xltstat(JNIEnv *jni, const struct stat *st) jclass cls; jmethodID init; jint modehack; - + modehack = st->st_mode&07777; if (S_ISFIFO(st->st_mode)) modehack |= 0010000; else if (S_ISCHR(st->st_mode)) modehack |= 0020000; diff --git a/keys.scala b/keys.scala index f075159..b49e334 100644 --- a/keys.scala +++ b/keys.scala @@ -27,19 +27,25 @@ package uk.org.distorted.tripe; package object keys { /*----- Imports -----------------------------------------------------------*/ -import java.io.{Closeable, File, FileOutputStream, FileReader, IOException}; - import scala.collection.mutable.HashMap; +import java.io.{Closeable, File}; +import java.net.{URL, URLConnection}; +import java.util.zip.GZIPInputStream; + +import sys.{SystemError, hashsz, runCommand}; +import sys.Errno.EEXIST; +import sys.FileImplicits._; + /*----- Useful regular expressions ----------------------------------------*/ -val RX_COMMENT = """(?x) ^ \s* (?: \# .* )? $""".r; -val RX_KEYVAL = """(?x) ^ \s* +private val RX_COMMENT = """(?x) ^ \s* (?: \# .* )? $""".r; +private val RX_KEYVAL = """(?x) ^ \s* ([-\w]+) (?:\s+(?!=)|\s*=\s*) (|\S|\S.*\S) \s* $""".r; -val RX_DOLLARSUBST = """(?x) \$ \{ ([-\w]+) \}""".r; +private val RX_DOLLARSUBST = """(?x) \$ \{ ([-\w]+) \}""".r; /*----- Things that go wrong ----------------------------------------------*/ @@ -62,7 +68,7 @@ class DefaultFailed(val key: String) extends Exception; type Config = scala.collection.Map[String, String]; -val DEFAULTS: Seq[(String, Config => String)] = +private val DEFAULTS: Seq[(String, Config => String)] = Seq("repos-base" -> { _ => "tripe-keys.tar.gz" }, "sig-base" -> { _ => "tripe-keys.sig-" }, "repos-url" -> { conf => conf("base-url") + conf("repos-base") }, @@ -85,7 +91,7 @@ val DEFAULTS: Seq[(String, Config => String)] = case "naclbox" => "poly1305/128" case _ => val h = conf("hash"); - JNI.hashsz(h) match { + hashsz(h) match { case -1 => throw new DefaultFailed("hash") case hsz => s"${h}-hmac/${4*hsz}" } @@ -100,36 +106,28 @@ val DEFAULTS: Seq[(String, Config => String)] = "sig-fresh" -> { _ => "always" }, "fingerprint-hash" -> { _("hash") }); -def readConfig(path: String): Config = { - var m = HashMap[String, String](); - withCleaner { clean => - var in = new FileReader(path); clean { in.close(); } - var lno = 1; - for (line <- lines(in)) { - line match { - case RX_COMMENT() => (); - case RX_KEYVAL(key, value) => m += key -> value; - case _ => - throw new ConfigSyntaxError(path, lno, "failed to parse line"); - } - lno += 1; - } - } +/*----- Managing a key repository -----------------------------------------*/ - for ((key, dflt) <- DEFAULTS) { - if (!(m contains key)) { - try { m += key -> dflt(m); } - catch { - case e: DefaultFailed => - throw new ConfigDefaultFailed(path, key, e.key, m(e.key)); - } +def downloadToFile(file: File, url: URL, maxlen: Long = Long.MaxValue) { + fetchURL(url, new URLFetchCallbacks { + val out = file.openForOutput(); + private def toobig() { + throw new KeyConfigException(s"remote file `$url' is " + + "suspiciously large"); } - } - m + var totlen: Long = 0; + override def preflight(conn: URLConnection) { + totlen = conn.getContentLength; + if (totlen > maxlen) toobig(); + } + override def done(win: Boolean) { out.close(); } + def write(buf: Array[Byte], n: Int, len: Long) { + if (len + n > maxlen) toobig(); + out.write(buf, 0, n); + } + }); } -/*----- Managing a key repository -----------------------------------------*/ - /* Lifecycle notes * * -> empty @@ -167,55 +165,266 @@ object Repository { object State extends Enumeration { val Empty, Pending, Confirmed, Updating, Committing, Live = Value; } - } +class RepositoryStateException(val state: Repository.State.Value, + msg: String) + extends Exception(msg); + +class KeyConfigException(msg: String) extends Exception(msg); + class Repository(val root: File) extends Closeable { import Repository.State.{Value => State, _}; - val livedir = new File(root, "live"); - val livereposdir = new File(livedir, "repos"); - val newdir = new File(root, "new"); - val olddir = new File(root, "old"); - val pendingdir = new File(root, "pending"); - val tmpdir = new File(root, "tmp"); - - val lock = { - if (!root.isDirectory && !root.mkdir()) ???; - val chan = new FileOutputStream(new File(root, "lk")).getChannel; - chan.tryLock() match { - case null => - throw new IOException(s"repository `${root.getPath}' locked") - case lk => lk + /* Important directories and files. */ + private[this] val livedir = root + "live"; + private[this] val livereposdir = livedir + "repos"; + private[this] val newdir = root + "new"; + private[this] val olddir = root + "old"; + private[this] val pendingdir = root + "pending"; + private[this] val tmpdir = root + "tmp"; + + /* Take out a lock in case of other instances. */ + private[this] val lock = { + try { root.mkdir_!(); } + catch { case SystemError(EEXIST, _) => ok; } + (root + "lk").lock_!() + } + def close() { lock.close(); } + + /* Maintain a cache of some repository state. */ + private var _state: State = null; + private var _config: Config = null; + private def invalidate() { + _state = null; + _config = null; + } + + def state: State = { + /* Determine the current repository state. */ + + if (_state == null) + _state = if (livedir.isdir_!) { + if (!livereposdir.isdir_!) Confirmed + else if (newdir.isdir_!) Updating + else Live + } else { + if (newdir.isdir_!) Committing + else if (pendingdir.isdir_!) Pending + else Empty + } + + _state + } + + def checkState(wanted: State*) { + /* Ensure we're in a particular state. */ + val st = state; + if (wanted.forall(_ != st)) { + throw new RepositoryStateException(st, s"Repository is $st, not " + + oxford("or", + wanted.map(_.toString))); + } + } + + def cleanup() { + + /* If we're part-way through an update then back out or press forward. */ + state match { + + case Updating => + /* We have a new tree allegedly ready, but the current one is still + * in place. It seems safer to zap the new one here, but we could go + * either way. + */ + + newdir.rmTree(); + invalidate(); // should move back to `Live' or `Confirmed' + + case Committing => + /* We have a new tree ready, and an old one moved aside. We're going + * to have to move one of them. Let's try committing the new tree. + */ + + newdir.rename_!(livedir); // should move on to `Live' + invalidate(); + + case _ => + /* Other states are stable. */ + ok; } + + /* Now work through the things in our area of the filesystem and zap the + * ones which don't belong. In particular, this will always erase + * `tmpdir'. + */ + val st = state; + root.foreachFile { f => (f.getName, st) match { + case ("lk", _) => ok; + case ("live", Live | Confirmed) => ok; + case ("pending", Pending) => ok; + case (_, Updating | Committing) => + unreachable(s"unexpectedly still in `$st' state"); + case _ => f.rmTree(); + } + } } + + def destroy() { + /* Clear out the entire repository. Everything. It's all gone. */ + root.foreachFile { f => if (f.getName != "lk") f.rmTree(); } + } + + def clearTmp() { + /* Arrange to have an empty `tmpdir'. */ + tmpdir.rmTree(); + tmpdir.mkdir_!(); + } + + def config: Config = { + /* Return the repository configuration. */ + + if (_config == null) { + + /* Firstly, decide where to find the configuration file. */ + cleanup(); + val dir = state match { + case Live | Confirmed => livedir + case Pending => pendingdir + case Empty => + throw new RepositoryStateException(Empty, "repository is Empty"); + } + val file = dir + "tripe-keys.conf"; + + /* Build the new configuration in a temporary place. */ + var m = HashMap[String, String](); + + /* Read the config file into our map. */ + file.withReader { in => + var lno = 1; + for (line <- lines(in)) { + line match { + case RX_COMMENT() => ok; + case RX_KEYVAL(key, value) => m += key -> value; + case _ => + throw new ConfigSyntaxError(file.getPath, lno, + "failed to parse line"); + } + lno += 1; + } + } + + /* Fill in defaults where things have been missed out. */ + for ((key, dflt) <- DEFAULTS) { + if (!(m contains key)) { + try { m += key -> dflt(m); } + catch { + case e: DefaultFailed => + throw new ConfigDefaultFailed(file.getPath, key, + e.key, m(e.key)); + } + } + } + + /* All done. */ + _config = m; + } + + _config } - def close() { - lock.release(); - lock.channel.close(); + def fetchConfig(url: URL) { + /* Fetch an initial configuration file from a given URL. */ + + checkState(Empty); + clearTmp(); + downloadToFile(tmpdir + "tripe-keys.conf", url); + tmpdir.rename_!(pendingdir); + invalidate(); // should move to `Pending' } - def state: State = - if (livedir.isDirectory) { - if (!livereposdir.isDirectory) Confirmed - else if (newdir.isDirectory && olddir.isDirectory) Committing - else Live - } else { - if (newdir.isDirectory) Updating - else if (pendingdir.isDirectory) Pending - else Empty + def confirm() { + /* The user has approved the master key fingerprint in the `Pending' + * configuration. Advance to `Confirmed'. + */ + + checkState(Pending); + pendingdir.rename_!(livedir); + invalidate(); // should move to `Confirmed' + } + + def update() { + /* Update the repository from the master. + * + * Fetch a (possibly new) archive; unpack it; verify the master key + * against the known fingerprint; and check the signature on the bundle. + */ + + checkState(Confirmed, Live); + val conf = config; + clearTmp(); + + /* First thing is to download the tarball and signature. */ + val tarfile = tmpdir + "tripe-keys.tar.gz"; + downloadToFile(tarfile, new URL(conf("repos-url"))); + val sigfile = tmpdir + "tripe-keys.sig"; + val seq = conf("master-sequence"); + downloadToFile(sigfile, + new URL(conf("sig-url").replaceAllLiterally("", + seq))); + + /* Unpack the tarball. Carefully. */ + val unpkdir = tmpdir + "unpk"; + unpkdir.mkdir_!(); + withCleaner { clean => + val tar = new TarFile(new GZIPInputStream(tarfile.open())); + clean { tar.close(); } + for (e <- tar) { + + /* Check the filename to make sure it's not evil. */ + if (e.name.split('/').exists { _ == ".." }) + throw new KeyConfigException("invalid path in tarball"); + + /* Find out where this file points. */ + val f = unpkdir + e.name; + + /* Unpack it. */ + if (e.isdir) { + /* A directory. Create it if it doesn't exist already. */ + + try { f.mkdir_!(); } + catch { case SystemError(EEXIST, _) => ok; } + } else if (e.isreg) { + /* A regular file. Write stuff to it. */ + + e.withStream { in => + f.withOutput { out => + for ((b, n) <- blocks(in)) out.write(b, 0, n); + } + } + } else { + /* Something else. Be paranoid and reject it. */ + + throw new KeyConfigException("unexpected object type in tarball"); + } + } } - def commitState(): State = state match { - case Updating => rmTree(newdir); state - case Committing => - if (!newdir.renameTo(livedir) && !olddir.renameTo(livedir)) - throw new IOException("failed to commit update"); - state - case st => st; + /* There ought to be a file in here called `repos/master.pub'. */ + val reposdir = unpkdir + "repos"; + if (!reposdir.isdir_!) + throw new KeyConfigException("missing `repos/' directory"); + val masterfile = reposdir + "master.pub"; + if (!masterfile.isreg_!) + throw new KeyConfigException("missing `repos/master.pub' file"); - def clean() { - + /* Fetch the master key's fingerprint. */ + val (out, _) = runCommand("key", "-k", masterfile.getPath, + "fingerprint", + "-f", "-secret", + "-a", conf("fingerprint-hash"), + s"master-$seq"); + println(s";; $out"); + } } /*----- That's all, folks -------------------------------------------------*/ diff --git a/peers.scala b/peers.scala index 931c8f4..76326be 100644 --- a/peers.scala +++ b/peers.scala @@ -23,36 +23,6 @@ val RX_REF = """(?x) \$ \( ([^)]+) \)""".r; val RX_RESOLVE = """(?x) \$ ([46*]*) \[ ([^\]]+) \]""".r; val RX_PARENT = """(?x) [^\s,]+""".r -def with_cleaner[T](body: Cleaner => T): T = { - val cleaner = new Cleaner; - try { body(cleaner) } - finally { cleaner.cleanup(); } -} - -class Cleaner { - var cleanups: List[() => Unit] = Nil; - def apply(cleanup: => Unit) { cleanups +:= { () => cleanup; } } - def cleanup() { cleanups foreach { _() } } -} - -def lines(r: Reader) = new Traversable[String] { - val in: BufferedReader = new BufferedReader(r); - override def foreach[T](f: String => T) { - while (true) in.readLine match { - case null => return; - case line => f(line); - } - } -} - -def thread(name: String, run: Boolean = true, daemon: Boolean = true) - (body: => Unit): Thread = { - val t = new Thread(new Runnable { override def run() { body } }, name); - t.setDaemon(daemon); - if (run) t.start(); - t -} - object BulkResolver { val BREAK = new Breaks; } @@ -93,14 +63,12 @@ class BulkResolver(val nthreads: Int = 8) { val workers = Array.tabulate(nthreads) { i => thread(s"resolver worker #$i") { - breakable { - while (true) { - val host = ch.read; if (host == null) break; + loopUnit { exit => + val host = ch.read; if (host == null) exit; println(s";; ${Thread.currentThread.getName} resolving `${host.name}'"); - try { - for (a <- InetAddress.getAllByName(host.name)) host.addaddr(a); - } catch { case e: UnknownHostException => () } - } + try { + for (a <- InetAddress.getAllByName(host.name)) host.addaddr(a); + } catch { case e: UnknownHostException => () } } println(s";; ${Thread.currentThread.getName} done'"); ch.write(null); @@ -274,7 +242,7 @@ class Config { conf => def parseFile(path: File): this.type = { println(s";; parse ${path.getPath}"); - with_cleaner { clean => + withCleaner { clean => val in = new FileReader(path); clean { in.close(); } val lno = 1; diff --git a/progress.scala b/progress.scala new file mode 100644 index 0000000..ddd2b6a --- /dev/null +++ b/progress.scala @@ -0,0 +1,90 @@ +/* -*-scala-*- + * + * Reporting progress for long-running jobs + * + * (c) 2018 Straylight/Edgeware + */ + +/*----- Licensing notice --------------------------------------------------* + * + * This file is part of the Trivial IP Encryption (TrIPE) Android app. + * + * TrIPE is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your + * option) any later version. + * + * TrIPE is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * along with TrIPE. If not, see . + */ + +package uk.org.distorted.tripe; package object progress; + +/*----- Imports -----------------------------------------------------------*/ + +import Math.ceil; +import System.currentTimeMillis; +import System.{err => stderr}; // FIXME: split out terminal progress + +/*----- Main code ---------------------------------------------------------*/ + +def formatTime(t: Int): String = + if (t < -1) "???" + else { + val (s, t1) = (t%60, t/60); + val (m, h) = (t1%60, t1/60); + if (h > 0) f"$h%d:$m%02d:$s%02d" + else f"$m%02d:$s%02d" + } + +private val UDATA = Seq("kB", "MB", "GB", "PB", "EB"); +def formatBytes(n: Long): String = { + val (x, u) = (n.toDouble, "B ") /: UDATA { + case ((x, u), name) if x >= 1024.0 => (x/1024.0, name) + case (xu, _) => xu + } + f"$x%6.1f$u%s" +} + +trait Eyecandy { + def set(line: String); + def clear(); + def commit(); + def commit(line: String) { commit(); set(line); commit(); } + + def begin(job: Job); +} + + +trait Job with Publisher[ { + def what: String; // imperative for what we're doing + def cur: Long; // current position in work + def max: Long; // maximum work to do + def format: String; // describe progress in useful terms + + private[this] val t0 = currentTimeMillis; + + def eta: Int = + /* Report the estimated time remaining in seconds, or -1 if no idea. + * + * The model here is very stupid. Weird jobs should override this and do + * something more sensible. + */ + + if (max < 0 || cur <= 0) -1 + else ceil((currentTimeMillis - t0)/1000.0 * + (max - cur)/cur.toDouble).toInt; +} + +object TerminalEyecandy extends Eyecandy { + private var last = ""; + var eyecandyp = + +} + +/*----- That's all, folks -------------------------------------------------*/ diff --git a/sys.scala b/sys.scala index c012d1f..cf0a72d 100644 --- a/sys.scala +++ b/sys.scala @@ -27,9 +27,13 @@ package uk.org.distorted.tripe; package object sys { /*----- Imports -----------------------------------------------------------*/ +import scala.collection.convert.decorateAsJava._; import scala.collection.mutable.HashSet; -import java.io.{Closeable, File}; +import java.io.{BufferedReader, BufferedWriter, Closeable, File, + FileDescriptor, FileInputStream, FileOutputStream, + InputStream, InputStreamReader, + OutputStream, OutputStreamWriter}; import java.nio.{ByteBuffer, CharBuffer}; import java.nio.charset.Charset; import java.util.Date; @@ -58,7 +62,7 @@ object StringImplicits { var sz: Int = (s.length*enc.averageBytesPerChar + 1).toInt; var out = ByteBuffer.allocate(sz); - while (true) { + loop[CString] { exit => /* If there's still stuff to encode, then encode it. Otherwise, * there must be some dregs left in the encoder, so flush them out. */ @@ -86,12 +90,9 @@ object StringImplicits { val v = new Array[Byte](n + 1); out.array.copyToArray(v, 0, n); v(n) = 0; - return v; + exit(v); } } - - /* Placate the type checker. */ - unreachable("unreachable"); } } @@ -125,37 +126,80 @@ import StringImplicits._; /* Import the native code library. */ System.loadLibrary("toy"); -/* Exception indicating that a wrapped native object has been clobbered. */ -class NativeObjectTypeException(msg: String) extends RuntimeException(msg); +/* Native types. + * + * See `jni.c'. There's no good way to hand a C pointer into Java, so we + * just copy whole structures into Java byte arrays and hope. Well, also we + * tag them so we can detect mixups. + */ type Wrapper = Array[Byte]; +class NativeObjectTypeException(msg: String) extends RuntimeException(msg); /*----- Error codes -------------------------------------------------------*/ +/* Machinery for collecting error information from C. */ protected case class ErrorEntry(val tag: String, val err: Int); @native protected def errtab: Array[ErrorEntry]; @native protected def strerror(err: Int): CString; object Errno extends Enumeration { - private val tagmap = { + /* System errors. + * + * There are two slight difficulties here. + * + * * Not all target systems have the same errors. C has a preprocessor + * to deal with this, but we don't; so instead we'll define all of the + * errors we'll ever need, but maybe with bogus values. + * + * * Some systems reuse code numbers for two different error names, e.g., + * both `EAGAIN' and `EWOULDBLOCK' are the same on Linux -- but not + * necessarily on other systems. Scala's `Enumeration' machinery + * doesn't like sharing `id' numbers between values. + * + * We augment the value type with an additional `code' value which is the + * actual system error code; we arbitrarily pick one error symbol with a + * given code to be `canonical', i.e., it has E.id == E.code; the others + * have synthetic `id' values. And symbols which don't correspond to any + * error on the target system have synthetic `id' /and/ `code', so that + * they can still be spoken about, but won't match any real error. + */ + + private val tagmap = { // map names to numbers based on what C reports val b = Map.newBuilder[String, Int]; for (ErrorEntry(tag, err) <- errtab) b += tag -> err; b.result } - private var wrong = -255; - private val seen = HashSet[Int](); - class ErrnoVal private[Errno](tag: String, val code: Int, id: Int) - extends Val(id, tag) { + private val seen = HashSet[Int](); // which error codes have been taken + + private var wrong = -256; // next synthetic code + private def nextwrong: Int = { val w = wrong; wrong -= 1; w } + + class Type private[Errno](tag: String, val code: Int, id: Int) + extends Val(id, tag) { + /* Our augmented error type. */ + def message: String = strerror(code).toJString; } + private class UnknownError(code: Int) + extends Type("", code, code); - private def err(tag: String, code: Int): ErrnoVal = { - if (seen contains code) { wrong -= 1; new ErrnoVal(tag, code, wrong) } - else { seen += code; new ErrnoVal(tag, code, code) } + private def err(tag: String, code: Int): Type = { + /* Construct an error symbol given its tag string and a code number. */ + + if (code < 0) new Type(tag, code, code) + else if (seen contains code) new Type(tag, code, nextwrong) + else { seen += code; new Type(tag, code, code) } + } + private def err(tag: String): Type = + err(tag, tagmap.getOrElse(tag, nextwrong)); + + def byid(id: Int): Value = { + if (seen contains id) apply(id) + else new UnknownError(id) } - private def err(tag: String): ErrnoVal = err(tag, tagmap(tag)); - val OK = err("OK", 0); + val OK = err("OK", 0); // `errno' zero is a real thing /* ;;; The errno name table is very boring to type. To make life less @@ -333,23 +377,25 @@ object Errno extends Enumeration { val EHWPOISON = err("EHWPOISON"); /***end***/ } -import Errno.{Value => _, _}; +import Errno.{Type => Errno, EEXIST, EISDIR, ENOENT, ENOTDIR}; object SystemError { - def apply(err: Errno.Value, what: String): SystemError = + /* Pattern matching for `SystemError', below. */ + + def apply(err: Errno, what: String): SystemError = new SystemError(err, what); - def unapply(e: Exception): Option[(Errno.Value, String)] = e match { + def unapply(e: Exception): Option[(Errno, String)] = e match { case e: SystemError => Some((e.err, e.what)) case _ => None } } +class SystemError (val err: Errno, val what: String) extends Exception { + /* An error from a syscall or similar, usually from native code. */ -class SystemError private[this](val err: Errno.ErrnoVal, val what: String) - extends Exception { - def this(err: Errno.Value, what: String) - { this(err.asInstanceOf[Errno.ErrnoVal], what); } + /* A constructor which takes an error number, for easier access from C. */ private def this(err: Int, what: CString) - { this(Errno(err), what.toJString); } + { this(Errno.byid(err).asInstanceOf[Errno], what.toJString); } + override def getMessage(): String = s"$what: ${err.message}"; } @@ -367,32 +413,37 @@ def mkfile(path: String, mode: Int) { mkfile(path.toCString, mode); } def rename(from: String, to: String) { rename(from.toCString, to.toCString); } +@native def fdint(fd: FileDescriptor): Int; +@native def newfd(fd: Int): FileDescriptor; +@native def isatty(fd: FileDescriptor): Boolean; + /*----- File status information -------------------------------------------*/ /* These are the traditional values, but the C code carefully arranges to * return them regardless of what your kernel actually thinks. */ -val S_IFMT = 0xf000; -val S_IFIFO = 0x1000; -val S_IFCHR = 0x2000; -val S_IFDIR = 0x4000; -val S_IFBLK = 0x6000; -val S_IFREG = 0x8000; -val S_IFLNK = 0xa000; -val S_IFSOCK = 0xc000; - +final val S_IFMT = 0xf000; +final val S_IFIFO = 0x1000; +final val S_IFCHR = 0x2000; +final val S_IFDIR = 0x4000; +final val S_IFBLK = 0x6000; +final val S_IFREG = 0x8000; +final val S_IFLNK = 0xa000; +final val S_IFSOCK = 0xc000; + +/* Primitive read-the-file-status calls. */ @native protected def stat(path: CString): sys.FileInfo; def stat(path: String): sys.FileInfo = stat(path.toCString); @native protected def lstat(path: CString): sys.FileInfo; def lstat(path: String): sys.FileInfo = lstat(path.toCString); object FileInfo extends Enumeration { + /* A simple enumeration of things a file might be. */ val FIFO, CHR, DIR, BLK, REG, LNK, SOCK, UNK = Value; type Type = Value; } import FileInfo._; - class FileInfo private[this](val devMajor: Int, val devMinor: Int, val ino: Long, val mode: Int, val nlink: Int, val uid: Int, val gid: Int, @@ -401,18 +452,26 @@ class FileInfo private[this](val devMajor: Int, val devMinor: Int, val blksize: Int, val blocks: Long, val atime: Date, val mtime: Date, val ctime: Date) { + /* Information about a file. This is constructed directly from native + * code. + */ + private def this(devMajor: Int, devMinor: Int, ino: Long, mode: Int, nlink: Int, uid: Int, gid: Int, rdevMinor: Int, rdevMajor: Int, size: Long, blksize: Int, blocks: Long, atime: Long, mtime: Long, ctime: Long) { + /* Lightly cook the values from the underlying `struct stat'. */ + this(devMajor, devMinor, ino, mode, nlink, uid, gid, rdevMajor, rdevMinor, size, blksize, blocks, new Date(atime), new Date(mtime), new Date(ctime)); } + /* Return the file permissions only. */ def perms: Int = mode&0xfff; + /* Return the filetype, as a `FileInfo.Type'. */ def ftype: Type = (mode&S_IFMT) match { case S_IFIFO => FIFO case S_IFCHR => CHR @@ -425,29 +484,48 @@ class FileInfo private[this](val devMajor: Int, val devMinor: Int, } private[this] def mustBeDevice() { + /* Insist that you only ask for `rdev' fields on actual device nodes. */ ftype match { - case CHR | BLK => (); + case CHR | BLK => ok; case _ => throw new IllegalArgumentException("Object is not a device"); } } + + /* Query the device-node numbers. */ def rdevMajor: Int = { mustBeDevice(); _rdevMajor } def rdevMinor: Int = { mustBeDevice(); _rdevMinor } } /*----- Listing directories -----------------------------------------------*/ +/* Primitive operations. */ @native protected def opendir(path: CString): Wrapper; @native protected def readdir(path: CString, dir: Wrapper): CString; @native protected def closedir(path: CString, dir: Wrapper); protected abstract class BaseDirIterator[T](cpath: CString) extends LookaheadIterator[T] with Closeable { + /* The underlying machinery for directory iterators. + * + * Subclasses must define `mangle' to convert raw filenames into a T. + * We keep track of the path C-string, because we need to keep passing that + * back to C for inclusion in error messages. Recording higher-level + * things is left for subclasses. + */ + + /* Constructors from more convenient types. */ def this(path: String) { this(path.toCString); } def this(dir: File) { this(dir.getPath); } + + /* Cleaning up after ourselves. */ override def close() { closedir(cpath, dir); } override protected def finalize() { super.finalize(); close(); } - private[this] val dir = opendir(cpath); + + /* Subclass responsibility. */ protected def mangle(file: String): T; + + /* Main machinery. */ + private[this] val dir = opendir(cpath); override protected def fetch(): Option[T] = readdir(cpath, dir) match { case null => None case f => f.toJString match { @@ -458,29 +536,47 @@ protected abstract class BaseDirIterator[T](cpath: CString) } class DirIterator(val path: String) extends BaseDirIterator[String](path) { + /* Iterator over the basenames of files in a directory. */ + def this(dir: File) { this(dir.getPath); } + override protected def mangle(file: String): String = file; } class DirFilesIterator private[this](val dir: File, cpath: CString) extends BaseDirIterator[File](cpath) { + /* Iterator over full `File' objects in a directory. */ + def this(dir: File) { this(dir, dir.getPath.toCString); } def this(path: String) { this(new File(path), path.toCString); } + override protected def mangle(file: String): File = new File(dir, file); } /*----- File locking ------------------------------------------------------*/ -val LKF_EXCL = 1; -val LKF_WAIT = 2; -@native protected def lock(path: CString, flags: Int): Wrapper; +/* Primitive operations. The low `mode' bits are for the lock file if we + * have to create it. + */ +final val LKF_EXCL = 0x1000; +final val LKF_WAIT = 0x2000; +@native protected def lock(path: CString, mode: Int): Wrapper; @native protected def unlock(lock: Wrapper); class FileLock(path: String, flags: Int) extends Closeable { + /* A class which represents a held lock on a file. */ + + /* Constructors. The default is to take an exclusive lock or fail + * immediately. + */ def this(file: File, flags: Int) { this(file.getPath, flags); } - def this(path: String) { this(path, LKF_EXCL); } - def this(file: File) { this(file.getPath, LKF_EXCL); } + def this(path: String) { this(path, LKF_EXCL | 0x1b6); } + def this(file: File) { this(file.getPath, LKF_EXCL | 0x1b6); } + + /* The low-level lock object, actually a file descriptor. */ private[this] val lk = lock(path.toCString, flags); + + /* Making sure things get cleaned up. */ override def close() { unlock(lk); } override protected def finalize() { super.finalize(); close(); } } @@ -489,23 +585,41 @@ class FileLock(path: String, flags: Int) extends Closeable { object FileImplicits { implicit class FileOps(file: File) { + /* Augment `File' with operations which throw informative (if low-level + * and system-specific) exceptions rather than returning unhelpful + * win/lose booleans. These have names ending with `_!' because they + * might explode. + * + * And some other useful methods. + */ + /* Constructing names of files in a directory. Honestly, I'm surprised + * there isn't a method for this already. + */ + def +(sub: String): File = new File(file, sub); + + /* Simple file operations. */ def unlink_!() { unlink(file.getPath); } def rmdir_!() { rmdir(file.getPath); } def mkdir_!(mode: Int) { mkdir(file.getPath, mode); } def mkdir_!() { mkdir_!(0x1ff); } def mkfile_!(mode: Int) { mkfile(file.getPath, mode); } def mkfile_!() { mkfile_!(0x1b6); } + def rename_!(to: File) { rename(file.getPath, to.getPath); } + /* Listing directories. */ def withFilesIterator[T](body: DirFilesIterator => T): T = { val iter = new DirFilesIterator(file.getPath); try { body(iter) } finally { iter.close(); } } + def foreachFile(fn: File => Unit) { withFilesIterator(_.foreach(fn)) } def files_! : Seq[File] = withFilesIterator { _.toSeq }; + /* Low-level lFile information. */ def stat_! : FileInfo = stat(file.getPath); def lstat_! : FileInfo = lstat(file.getPath); + /* Specific file-status queries. */ private[this] def statish[T](statfn: String => FileInfo, ifexists: FileInfo => T, ifmissing: => T): T = @@ -524,33 +638,63 @@ object FileImplicits { def issock_! : Boolean = statish(stat _, _.ftype == SOCK, false); def remove_!() { + /* Delete a file, or directory, whatever it is. */ while (true) { - try { unlink_!(); return } + try { unlink_!(); return; } catch { case SystemError(ENOENT, _) => return; - case SystemError(EISDIR, _) => (); + case SystemError(EISDIR, _) => ok; } - try { rmdir_!(); return } + try { rmdir_!(); return; } catch { case SystemError(ENOENT, _) => return; - case SystemError(ENOTDIR, _) => (); + case SystemError(ENOTDIR, _) => ok; } } } def rmTree() { + /* Delete a thing recursively. */ def walk(f: File) { - if (f.isdir_!) f.withFilesIterator { _ foreach(walk _) }; + if (f.isdir_!) f.foreachFile(walk _); f.remove_!(); } walk(file); } + /* File locking. */ + def lock_!(flags: Int): FileLock = new FileLock(file.getPath, flags); + def lock_!(): FileLock = lock_!(LKF_EXCL | 0x1b6); def withLock[T](flags: Int)(body: => T): T = { - val lk = new FileLock(file.getPath, flags); + val lk = lock_!(flags); try { body } finally { lk.close(); } } - def withLock[T](body: => T): T = withLock(LKF_EXCL) { body }; + def withLock[T](body: => T): T = withLock(LKF_EXCL | 0x1b6) { body }; + + /* Opening files. Again, I'm surprised this isn't here already. */ + def open(): FileInputStream = new FileInputStream(file); + def openForOutput(): FileOutputStream = new FileOutputStream(file); + def reader(): BufferedReader = + new BufferedReader(new InputStreamReader(open())); + def writer(): BufferedWriter = + new BufferedWriter(new OutputStreamWriter(openForOutput())); + def withInput[T](body: FileInputStream => T): T = { + val in = open(); + try { body(in) } + finally { in.close(); } + } + def withOutput[T](body: FileOutputStream => T): T = { + val out = openForOutput(); + try { body(out) } finally { out.close(); } + } + def withReader[T](body: BufferedReader => T): T = withInput { in => + body(new BufferedReader(new InputStreamReader(in))) + }; + def withWriter[T](body: BufferedWriter => T): T = withOutput { out => + val w = new BufferedWriter(new OutputStreamWriter(out)); + /* Do this the hard way, so that we flush the `BufferedWriter'. */ + try { body(w) } finally { w.close(); } + } } } import FileImplicits._; @@ -563,7 +707,7 @@ def freshFile(d: File): File = { val buf = new Array[Byte](6); val b = new StringBuilder; - while (true) { + loop[File] { exit => /* Keep going until we find a fresh one. */ /* Provide a prefix. Mostly this is to prevent the file starting with @@ -596,19 +740,70 @@ def freshFile(d: File): File = { * win. */ val f = new File(d, b.result); b.clear(); - try { f.mkfile_!(); return f; } - catch { case SystemError(EEXIST, _) => (); } + try { f.mkfile_!(); exit(f); } + catch { case SystemError(EEXIST, _) => ok; } } +} + +/*----- Running a command -------------------------------------------------*/ - /* We shouldn't get here, but the type checker needs placating. */ - unreachable("unreachable"); +private val devnull = new File("/dev/null"); + +private def captureStream(in: InputStream, out: StringBuilder) { + /* Capture the INSTREAM's contents in a string. */ + + for ((buf, n) <- blocks(new InputStreamReader(in))) + out.appendAll(buf, 0, n); +} + +class SubprocessFailed(val cmd: Seq[String], rc: Int, stderr: String) + extends Exception { + override def getMessage(): String = + s"process (${quoteTokens(cmd)}) failed (rc = $rc):\n" + stderr +} + +def runCommand(cmd: String*): (String, String) = { + /* Run a command, returning its stdout and stderr. */ + + withCleaner { clean => + + /* Create the child process and pick up the ends of its streams. */ + val pb = new ProcessBuilder(cmd.asJava).redirectInput(devnull); + val kid = pb.start(); clean { kid.destroy(); } + val out = kid.getInputStream(); clean { out.close(); } + val err = kid.getErrorStream(); clean { err.close(); } + + /* Capture the output in threads, so we don't block. Also, wait for the + * child to complete. Amazingly, messing with threads here isn't too + * much of a disaster. + */ + val bout, berr = new StringBuilder; + val rdout = thread("capture process stdout", daemon = false) { + captureStream(out, bout); + } + val rderr = thread("capture process stderr", daemon = false) { + captureStream(err, berr); + } + val wait = thread("await process exit", daemon = false) { + kid.waitFor(); + } + rdout.join(); rderr.join(); wait.join(); + + /* Check the exit status. */ + val rc = kid.exitValue; + if (rc != 0) throw new SubprocessFailed(cmd, rc, berr.result); + + /* We're all done. */ + return (bout.result, berr.result); + } } /*----- Connecting to a server --------------------------------------------*/ -val CF_CLOSERD = 1; -val CF_CLOSEWR = 2; -val CF_CLOSEMASK = CF_CLOSERD | CF_CLOSEWR; +/* Primitive operations. */ +final val CF_CLOSERD = 1; +final val CF_CLOSEWR = 2; +final val CF_CLOSEMASK = CF_CLOSERD | CF_CLOSEWR; @native protected def connect(path: CString): Wrapper; @native protected def send(conn: Wrapper, buf: CString, start: Int, len: Int); @@ -617,12 +812,20 @@ val CF_CLOSEMASK = CF_CLOSERD | CF_CLOSEWR; @native def closeconn(conn: Wrapper, how: Int); class Connection(path: String) extends Closeable { - def this(file: File) { this(file.getPath); } + + /* The underlying primitive connection. */ private[this] val conn = connect(path.toCString); + + /* Alternative constructors. */ + def this(file: File) { this(file.getPath); } + + /* Cleanup.*/ override def close() { closeconn(conn, CF_CLOSEMASK); } override protected def finalize() { super.finalize(); close(); } - class InputStream private[Connection] extends java.io.InputStream { + class Input private[Connection] extends InputStream { + /* An input stream which reads from the connection. */ + override def read(): Int = { val buf = new Array[Byte](1); val n = read(buf, 0, 1); @@ -634,16 +837,18 @@ class Connection(path: String) extends Closeable { recv(conn, buf, start, len); override def close() { closeconn(conn, CF_CLOSERD); } } - lazy val input = new InputStream; + lazy val input = new Input; + + class Output private[Connection] extends OutputStream { + /* An output stream which writes to the connection. */ - class OutputStream private[Connection] extends java.io.OutputStream { override def write(b: Int) { write(Array[Byte](b.toByte), 0, 1); } override def write(buf: Array[Byte]) { write(buf, 0, buf.length); } override def write(buf: Array[Byte], start: Int, len: Int) { send(conn, buf, start, len); } override def close() { closeconn(conn, CF_CLOSEWR); } } - lazy val output = new OutputStream; + lazy val output = new Output; } /*----- Crypto-library hacks ----------------------------------------------*/ diff --git a/tar.scala b/tar.scala new file mode 100644 index 0000000..5f20b0a --- /dev/null +++ b/tar.scala @@ -0,0 +1,446 @@ +/* -*-scala-*- + * + * Extract data from `tar' archives + * + * (c) 2018 Straylight/Edgeware + */ + +/*----- Licensing notice --------------------------------------------------* + * + * This file is part of the Trivial IP Encryption (TrIPE) Android app. + * + * TrIPE is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your + * option) any later version. + * + * TrIPE is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * along with TrIPE. If not, see . + */ + +package uk.org.distorted.tripe; + +/*----- Imports -----------------------------------------------------------*/ + +import java.io.{Closeable, InputStream}; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.Date; + +/*----- Main code ---------------------------------------------------------*/ + +class TarFormatError(msg: String) extends Exception(msg); + +trait TarEntry { + /* Honestly, I'd rather just have `TarFile#Entry', but Scala doesn't permit + * the trait inheritance circularity. So this is a cardboard cutout + * version of `Entry'. + */ + + /* Basic facts about the entry. */ + def name: String; + def size: Long; + def typ: Char; + def mode: Int; + def mtime: Date; + def uid: Int; + def gid: Int; + def link: String; + + /* Type predicates (intentionally like `FileInfo'). */ + def isfifo: Boolean = typ == '6'; + def ischr: Boolean = typ == '3'; + def isdir: Boolean = typ == '5'; + def isblk: Boolean = typ == '4'; + def isreg: Boolean = typ match { + case 0 | '0' | '7' => true + case _ => false + } + def islnk: Boolean = typ == '2'; + def issock: Boolean = false; + def ishardlink: Boolean = typ == '1'; + + def verbose: String = { + /* Encode information about this tar header as a string. */ + + val sb = new StringBuilder; + + /* First, the type code. */ + sb += (typ match { + case 0 | '0' | '7' => '-' + case '1' => 'L' + case '2' => 'l' + case '3' => 'c' + case '4' => 'b' + case '5' => 'd' + case '6' => '|' + case _ => '?' + }) + + /* Then the permissions bits. Ugh, the permissions bits. */ + def perm(s: Int, r: Int, w: Int, x: Int, schar: Char, Schar: Char) { + sb += (if ((mode&r) != 0) 'r' else '-'); + sb += (if ((mode&w) != 0) 'w' else '-'); + sb += (if ((mode&s) != 0) + if ((mode&x) != 0) schar else Schar; + else + if ((mode&x) != 0) 'x' else '-'); + } + perm(0x800, 0x100, 0x080, 0x040, 's', 'S'); + perm(0x400, 0x020, 0x010, 0x008, 's', 'S'); + perm(0x200, 0x004, 0x002, 0x001, 't', 'T'); + + /* And the rest, which is easy. */ + sb ++= f" $uid%8d $gid%8d $size%12d $mtime%tFT% T): T = { + val s = stream; + try { body(s) } + finally { s.close(); } + } +} + +class TarFile(in: InputStream) + extends LookaheadIterator[TarEntry] with Closeable { tar => + + /* Tokens are just objects, meaningful only for their identity. */ + private[TarFile] class Token; + + /* Some useful state. */ + private[TarFile] var offset: Long = 0; // current byte offset + private[this] var lockp = false; // locked by open entry? + private[this] var locktok = new Token; // active lock token + private[this] var nexthdr: Long = 0; // byte offset of next header + private[this] val hdr = new Array[Byte](512); // header under consideration + + /* Making sure we clean up properly. */ + override def close() { in.close(); } + override protected def finalize() { super.finalize(); close(); } + + private[this] def eoferr() + { throw new TarFormatError(s"unexpected EOF (at $offset)"); } + + /* Locking machinery. + * + * We work from a primitive `InputStream' which we can't seek. From this, + * we must be able to extract file contents, as an `InputStream', and parse + * file headers. We'll be badly lost if we lose track of where we are in + * the archive. + * + * So, there's a lock, which can be held by at most one actor at a time: + * either the `TarFile' itself, while it's (hopefully) reading a header + * block, or by the `Stream' object which lets the caller read an + * individual entry's content. Furthermore, if we start activating the + * per-entry streams out of order, we'll get confused about where we're + * meant to be, so there's also a `token' which represents a participant's + * right to claim the lock. The `TarFile' itself has special privileges + * and doesn't need a token, but the per-entry streams do, and only the + * stream associated with the most recently-read header is allowed to claim + * the lock. + */ + + private[this] def lock() { + /* Claim exclusive use of the input stream. */ + + if (lockp) throw new IllegalArgumentException("tarfile lock still held"); + lockp = true; + } + + private[TarFile] def lock(tok: Token) { + /* Claim exclusive use of the input stream, passing a token. */ + + if (tok ne locktok) + throw new IllegalArgumentException("stale lock token"); + lock(); + } + + private[TarFile] def unlock() { + /* Release the input stream so someone else can have a go. */ + + assert(lockp); + lockp = false; + locktok = new Token; + } + + /* Doing I/O on the input stream. + * + * Our `Stream' object sneakily grabs single bytes from the input. Given + * the way Scala works, we can't prevent that, so roll with it. + */ + + private[TarFile] def read(buf: Array[Byte], start: Int, len: Int) { + /* Read input data into the indicated region of the buffer. Short reads + * are diagnosed as errors. Advances the cursor. + */ + + var pos = start; + val limit = start + len; + while (pos < len) { + val n = in.read(buf, pos, limit - pos); + if (n < 0) eoferr(); + pos += n; offset += n; + } + } + + private[TarFile] def skip(len: Long) { + /* Skip ahead LEN bytes in the archive. (The int/long discrepancy + * matches Java's bizarre `InputStream' protocol.) + */ + + var remain = len; + while (remain > 0) { + val n = in.skip(remain); + + if (n > 0) { remain -= n; offset += n; } + else { + /* It's hard to work out whether this means we've hit EOF or not. It + * seems best to check. We must have at least one byte left to skip + * or we wouldn't have started this iteration, so try to read that. + * If that works, then there's more stuff available and skipping + * isn't working, so start to read buffers and discard them. + */ + + if (in.read() == -1) eoferr(); + remain -= 1; offset += 1; + + /* Ugh. So, buffers it is then. */ + val buf = new Array[Byte]((remain min 4096).toInt); + while (remain >= buf.length) { + val n = (remain min buf.length).toInt; + read(buf, 0, n); + remain -= n; + } + } + } + } + + private[TarFile] class Stream(end: Long, tok: Token) extends InputStream { + /* An input stream for a single archive entry's content. */ + + /* Claim the lock. If we're stale, this won't work. */ + lock(tok); + private[this] var open = true; + + private[this] def checkopen() { + /* Complain if the stream is closed. */ + + if (!lockp) throw new IllegalArgumentException("stream is closed"); + } + + override def read(): Int = { + /* Read one byte. Don't know why there isn't a default implementation + * of this. + */ + + checkopen(); + if (offset >= end) -1 + else { + val b = in.read(); + if (b == -1) eoferr(); + offset += 1; + b + } + } + + override def read(buf: Array[Byte], start: Int, len: Int): Int = { + /* Read a block. */ + + checkopen(); + if (offset >= end) -1 + else { + var n = (len.toLong min (end - offset)).toInt; + tar.read(buf, start, n); + n + } + } + + override def close() { + /* Release the lock. */ + + if (open) { unlock(); open = false; } + } + } + + private[this] class Entry(val name: String, val size: Long, + val typ: Char, val mode: Int, + val mtime: Date, + val uid: Int, val gid: Int, + val link: String, + end: Long, tok: Token) + extends TarEntry{ + /* See `TarEntry' for why we have this silliness. Most of the work is in + * the constructor above. + */ + + lazy val stream: InputStream = new Stream(end, tok); + } + + /* Utilities for parsing archive-entry header blocks. */ + + private[this] def string(off: Int, len: Int): String = { + /* Parse a string from the block header. POSIX.1-2008 says that header + * fields should be ISO/IEC 646, but strange things can turn up + * especially in filenames. I'm going to translate strings according to + * the local character set, because that will map most easily if a + * program tries to write out files from the archive with their + * associated names. + */ + + /* First, find the null terminator, if there is one. Scala doesn't make + * this especially easy. Rustle up a view to limit the search. + */ + val bview = hdr.view(off, off + len); + val n = bview.indexOf(0) match { + case -1 => len + case nul => nul + }; + + /* And then decode the relevant portion of the orignal buffer. */ + val dec = Charset.defaultCharset.newDecoder; + val in = ByteBuffer.wrap(hdr, off, n); + dec.decode(in).toString + } + + private[this] def number(off: Int, len: Int, max: Long): Long = { + /* Parse a number from the block header. POSIX.1-2008 says that numbers + * are in octal and terminated by space or nul. + */ + + var n = 0l; // accumulate the value + for (i <- off until off + len) { + val b = hdr(i); + + /* See if we're done now. */ + if (b == ' ' || b == 0) return n; + else if (b < '0' || b > '7') + throw new TarFormatError(s"bad octal digit (at ${offset + off + i})"); + + /* Convert to a digit. */ + val m = b - '0'; + + /* Check for overflow -- without overflowing. + * + * Write max 8 N + M. We overflow if 8 n + m > 8 N + M, i.e., 8 n > + * 8 N + (M - m), so n > N + (M - m)/8. This last calculation is a + * little fraught because Scala has the wrong semantics when dividing + * negative integers. + */ + if (n > max/8 + (8 + max%8 - m)/8 - 1) + throw new TarFormatError(s"number out of range (at ${offset + off})"); + + /* Accumulate and go round again. */ + n = 8*n + (b - '0'); + } + unreachable; + } + + override protected def fetch(): Option[TarEntry] = { + /* Collect the next archive header and return it as a file entry. */ + + /* Make sure that we can actually do this. */ + withCleaner { clean => + lock(); clean { unlock(); } + + /* Skip ahead to the next header. */ + skip(nexthdr - offset); + + /* Read the header. The end of the archive is marked by two zero + * blocks, so the archive is broken if there isn't at least one here. + */ + read(hdr, 0, 512); + } + + /* If the block is entirely zero-filled then declare this file at an + * end. No good can come from checking the next block. + */ + if (hdr.forall(_ == 0)) return None; + + /* Verify the checksum. Pay attention because Java's bytes are + * (idiotically) signed. + */ + var ck: Int = 8*' '; // pretend chksum field is spaces + for (i <- 0 until 148) ck += hdr(i)&0xff; + for (i <- 156 until 512) ck += hdr(i)&0xff; + val wantck = number(148, 8, 0x20000); + if (ck != wantck) { + throw new TarFormatError( + s"invalid checksum $ck /= $wantck (at $nexthdr)"); + } + + /* Fetch the `magic' and `version' fields. If this is a proper POSIX + * `ustar' file then special processing will apply. + */ + val magic = string(257, 6); + val version = string(263, 2); + val posixp = magic == "ustar" && version == "00"; + + /* Figure out this entry's name. If this is a POSIX archive, then part + * of the name is stashed at the end of the header because of old, bad + * decisions. But don't look there unless we're sure because old GNU + * `tar' used that space for other things. + */ + val name = { + val tail = string(0, 100); + if (!posixp || hdr(345) == 0) tail + else { + val prefix = string(345, 155); + prefix + '/' + tail + } + } + + /* Read some other easy stuff. */ + val mode = number(100, 8, 0xfff).toInt; + val uid = number(108, 8, Int.MaxValue).toInt; + val gid = number(116, 8, Int.MaxValue).toInt; + val typ = hdr(156).toChar; + val mtime = number(136, 12, Long.MaxValue); + + /* The size is irrelevant, and possibly even misleading, for some entry + * types. We're not interested, for example, in systems where + * directories need to be preallocated. + */ + val size = typ match { + case '1' | '2' | '3' | '4' | '5' | '6' => 0 + case _ => number(124, 12, Long.MaxValue) + } + + /* Maybe fetch the link name. */ + val link = typ match { + case '1' | '2' => string(157, 100) + case _ => "" + } + + /* Figure out where the next header ought to be. */ + nexthdr = (offset + size + 511)& -512; + + /* Return the finished archive entry. */ + Some(new Entry(name, size, typ, mode, + new Date(1000*mtime), uid, gid, link, + offset + size, locktok)); + } +} + +/* Example: + * + * for (e <- TarFile(new GZIPInputStream(tarball.open())); if e.isreg) + * e withStream { in => + * val h = java.security.MessageDigest.getInstance("SHA-256"); + * for ((buf, n) <- in.blocks) h.update(b, 0, n); + * val hex = new String(h.digest flatMap { _.formatted("%02x") }); + * println("s$hex ${e.name}"); + * } + */ + +/*----- That's all, folks -------------------------------------------------*/ diff --git a/util.scala b/util.scala index 79ac861..75b3677 100644 --- a/util.scala +++ b/util.scala @@ -28,9 +28,9 @@ package uk.org.distorted; package object tripe { /*----- Imports -----------------------------------------------------------*/ import scala.concurrent.duration.{Deadline, Duration}; -import scala.util.control.Breaks; +import scala.util.control.{Breaks, ControlThrowable}; -import java.io.{BufferedReader, Closeable, File, Reader}; +import java.io.{BufferedReader, Closeable, File, InputStream, Reader}; import java.net.{URL, URLConnection}; import java.nio.{ByteBuffer, CharBuffer}; import java.nio.charset.Charset; @@ -41,6 +41,9 @@ import java.util.concurrent.locks.{Lock, ReentrantLock}; val rng = new java.security.SecureRandom; def unreachable(msg: String): Nothing = throw new AssertionError(msg); +def unreachable(): Nothing = unreachable("unreachable"); +final val ok = (); +final class Brand; /*----- Various pieces of implicit magic ----------------------------------*/ @@ -134,13 +137,70 @@ def closing[T, U <: Closeable](thing: U)(body: U => T): T = try { body(thing) } finally { thing.close(); } +/*----- Control structures ------------------------------------------------*/ + +private case class ExitBlock[T](brand: Brand, result: T) + extends ControlThrowable; + +def block[T](body: (T => Nothing) => T): T = { + /* block { exit[T] => ...; exit(x); ... } + * + * Execute the body until it calls the `exit' function or finishes. + * Annoyingly, Scala isn't clever enough to infer the return type, so + * you'll have to write it explicitly. + */ + + val mybrand = new Brand; + try { body { result => throw new ExitBlock(mybrand, result) } } + catch { + case ExitBlock(brand, result) if brand eq mybrand => + result.asInstanceOf[T] + } +} + +def blockUnit(body: (=> Nothing) => Unit) { + /* blockUnit { exit => ...; exit; ... } + * + * Like `block'; it just saves you having to write `exit[Unit] => ...; + * exit(ok); ...'. + */ + + val mybrand = new Brand; + try { body { throw new ExitBlock(mybrand, null) }; } + catch { case ExitBlock(brand, result) if brand eq mybrand => ok; } +} + +def loop[T](body: (T => Nothing) => Unit): T = { + /* loop { exit[T] => ...; exit(x); ... } + * + * Repeatedly execute the body until it calls the `exit' function. + * Annoyingly, Scala isn't clever enough to infer the return type, so + * you'll have to write it explicitly. + */ + + block { exit => while (true) body(exit); unreachable } +} + +def loopUnit(body: (=> Nothing) => Unit): Unit = { + /* loopUnit { exit => ...; exit; ... } + * + * Like `loop'; it just saves you having to write `exit[Unit] => ...; + * exit(()); ...'. + */ + + blockUnit { exit => while (true) body(exit); } +} + +val BREAKS = new Breaks; +import BREAKS.{breakable, break}; + /*----- A gadget for fetching URLs ----------------------------------------*/ class URLFetchException(msg: String) extends Exception(msg); trait URLFetchCallbacks { def preflight(conn: URLConnection) { } - def write(buf: Array[Byte], n: Int, len: Int): Unit; + def write(buf: Array[Byte], n: Int, len: Long): Unit; def done(win: Boolean) { } } @@ -157,17 +217,18 @@ def fetchURL(url: URL, cb: URLFetchCallbacks) { /* Start fetching data. */ val in = c.getInputStream; clean { in.close(); } - val explen = c.getContentLength(); + val explen = c.getContentLength; /* Read a buffer at a time, and give it to the callback. Maintain a * running total. */ - val buf = new Array[Byte](4096); - var n = 0; - var len = 0; - while ({n = in.read(buf); n >= 0 && (explen == -1 || len <= explen)}) { - cb.write(buf, n, len); - len += n; + var len: Long = 0; + blockUnit { exit => + for ((buf, n) <- blocks(in)) { + cb.write(buf, n, len); + len += n; + if (explen != -1 && len > explen) exit; + } } /* I can't find it documented anywhere that the existing machinery @@ -184,10 +245,6 @@ def fetchURL(url: URL, cb: URLFetchCallbacks) { } } -/*----- Running processes -------------------------------------------------*/ - -//def runProgram( - /*----- Threading things --------------------------------------------------*/ def thread[T](name: String, run: Boolean = true, daemon: Boolean = true) @@ -294,36 +351,107 @@ def splitTokens(s: String, pos: Int = 0): Seq[String] = { val b = List.newBuilder[String]; var i = pos; - while (nextToken(s, i) match { - case Some((w, j)) => b += w; i = j; true - case None => false - }) (); + loopUnit { exit => nextToken(s, i) match { + case Some((w, j)) => b += w; i = j; + case None => exit; + } } b.result } +/*----- Other random things -----------------------------------------------*/ + trait LookaheadIterator[T] extends BufferedIterator[T] { - private[this] var st: Option[T] = None; + /* An iterator in terms of a single `maybe there's another item' function. + * + * It seems like every time I write an iterator in Scala, the only way to + * find out whether there's a next item, for `hasNext', is to actually try + * to fetch it. So here's an iterator in terms of a function which goes + * off and maybe returns a next thing. It turns out to be easy to satisfy + * the additional requirements for `BufferedIterator', so why not? + */ + + /* Subclass responsibility. */ protected def fetch(): Option[T]; + + /* The machinery. `st' is `None' if there's no current item, null if we've + * actually hit the end, or `Some(x)' if the current item is x. + */ + private[this] var st: Option[T] = None; private[this] def peek() { + /* Arrange to have a current item. */ if (st == None) fetch() match { case None => st = null; case x@Some(_) => st = x; } } + + /* The `BufferedIterator' protocol. */ override def hasNext: Boolean = { peek(); st != null } - override def head(): T = + override def head: T = { peek(); if (st == null) throw new NoSuchElementException; st.get } - override def next(): T = { val it = head(); st = None; it } + override def next(): T = { val it = head; st = None; it } } -def lines(r: Reader) = new LookaheadIterator[String] { - /* Iterates over the lines of text in a `Reader' object. */ +def bufferedReader(r: Reader): BufferedReader = r match { + case br: BufferedReader => br + case _ => new BufferedReader(r) +} - private[this] val in = r match { - case br: BufferedReader => br; - case _ => new BufferedReader(r); +def lines(r: BufferedReader): BufferedIterator[String] = + new LookaheadIterator[String] { + /* Iterates over the lines of text in a `Reader' object. */ + override protected def fetch() = Option(r.readLine()); + } +def lines(r: Reader): BufferedIterator[String] = lines(bufferedReader(r)); + +def blocks(in: InputStream, blksz: Int): + BufferedIterator[(Array[Byte], Int)] = + /* Iterates over (possibly irregularly sized) blocks in a stream. */ + new LookaheadIterator[(Array[Byte], Int)] { + val buf = new Array[Byte](blksz) + override protected def fetch() = { + val n = in.read(buf); + if (n < 0) None + else Some((buf, n)) + } + } +def blocks(in: InputStream): + BufferedIterator[(Array[Byte], Int)] = blocks(in, 4096); + +def blocks(in: BufferedReader, blksz: Int): + BufferedIterator[(Array[Char], Int)] = + /* Iterates over (possibly irregularly sized) blocks in a reader. */ + new LookaheadIterator[(Array[Char], Int)] { + val buf = new Array[Char](blksz) + override protected def fetch() = { + val n = in.read(buf); + if (n < 0) None + else Some((buf, n)) + } } - protected override def fetch(): Option[String] = Option(in.readLine); +def blocks(in: BufferedReader): + BufferedIterator[(Array[Char], Int)] = blocks(in, 4096); +def blocks(r: Reader, blksz: Int): BufferedIterator[(Array[Char], Int)] = + blocks(bufferedReader(r), blksz); +def blocks(r: Reader): BufferedIterator[(Array[Char], Int)] = + blocks(bufferedReader(r)); + +def oxford(conj: String, things: Seq[String]): String = things match { + case Seq() => "" + case Seq(a) => a + case Seq(a, b) => s"$a $conj $b" + case Seq(a, tail@_*) => + val sb = new StringBuilder; + sb ++= a; sb ++= ", "; + def iter(rest: Seq[String]) { + rest match { + case Seq() => unreachable; + case Seq(a) => sb ++= conj; sb += ' '; sb ++= a; + case Seq(a, tail@_*) => sb ++= a; sb ++= ", "; iter(tail); + } + } + iter(tail); + sb.result } /*----- That's all, folks -------------------------------------------------*/ -- 2.11.0