PKGS_LIBS := $(foreach p,$(PKGS),$(shell pkg-config --libs $p))
CC = gcc
-CFLAGS = -O2 -g -Wall -fPIC $(addprefix -I,$(INCLUDES))
+CFLAGS = -O2 -g -Wall -pedantic -Werror \
+ -fPIC $(addprefix -I,$(INCLUDES))
CFLAGS += $(PKGS_CFLAGS)
LD = gcc
JAVAFLAGS =
SCALAC = fsc
-SCALAFLAGS = -optimise
+SCALAFLAGS = -optimise -Xlint -Xlint:-package-object-classes \
+ -Yinline-warnings:false
## Hack around https://issues.scala-lang.org/browse/SI-9689
SCALAFLAGS += -Yno-load-impl-class
sys.stamp: util.stamp
TARGETS += admin.stamp
-admin.stamp: util.stamp sys.stamp
+admin.stamp: sys.stamp util.stamp
+
+TARGETS += tar.stamp
+tar.stamp: sys.stamp util.stamp
+
+TARGETS += keys.stamp
+keys.stamp: tar.stamp sys.stamp util.stamp
TARGETS += main.stamp
main.stamp: sys.stamp
sealed abstract class JobMessage extends Message;
case object JobOK extends JobMessage;
-case class JobInfo(info: Seq[String]) extends JobMessage;
-case class JobFail(err: Seq[String]) extends JobMessage;
+final case class JobInfo(info: Seq[String]) extends JobMessage;
+final case class JobFail(err: Seq[String]) extends JobMessage;
case object JobLostConnection extends JobMessage;
-case class BackgroundJobMessage(tag: String, msg: JobMessage)
+final case class BackgroundJobMessage(tag: String, msg: JobMessage)
extends Message;
-case class JobDetached(tag: String) extends Message;
+final case class JobDetached(tag: String) extends Message;
sealed abstract class AsyncMessage extends Message;
-case class Trace(msg: String) extends AsyncMessage;
-case class Warning(err: Seq[String]) extends AsyncMessage;
-case class Notify(note: Seq[String]) extends AsyncMessage;
+final case class Trace(msg: String) extends AsyncMessage;
+final case class Warning(err: Seq[String]) extends AsyncMessage;
+final case class Notify(note: Seq[String]) extends AsyncMessage;
case object ConnectionLost extends AsyncMessage;
sealed abstract class ServiceMessage extends Message;
-case class ServiceCancel(jobid: String) extends ServiceMessage;
-case class ServiceClaim(svc: String, version: String)
+final case class ServiceCancel(jobid: String) extends ServiceMessage;
+final case class ServiceClaim(svc: String, version: String)
extends ServiceMessage;
-case class ServiceJob(jobid: String, svc: String,
+final case class ServiceJob(jobid: String, svc: String,
cmd: String, args: Seq[String])
extends ServiceMessage;
/*----- Main code ---------------------------------------------------------*/
-object Connection {
-}
-
class ConnectionClosed extends Exception;
class ServerFailed(msg: String) extends Exception(msg);
case msg: AsyncMessage =>
publish(msg);
case _: ServiceMessage =>
- ();
+ ok;
}
}
} catch {
j.ch.write(JobLostConnection);
fgjob = None;
notifyAll();
- case None => ();
+ case None => ok;
}
}
publish(ConnectionLost);
ECANCELED ENOKEY EKEYEXPIRED EKEYREVOKED EKEYREJECTED
EOWNERDEAD ENOTRECOVERABLE ERFKILL EHWPOISON)))
(save-excursion
- (goto-char (point-min))
- (search-forward (concat "***" "BEGIN errtab" "***"))
- (beginning-of-line 2)
- (delete-region (point)
- (progn
- (search-forward "***END***")
- (beginning-of-line)
- (point)))
- (dolist (err errors)
- (insert (format "#ifdef %s\n { \"%s\", %s },\n#endif\n"
- err err err)))))
+ (goto-char (point-min))
+ (search-forward (concat "***" "BEGIN errtab" "***"))
+ (beginning-of-line 2)
+ (delete-region (point)
+ (progn
+ (search-forward "***END***")
+ (beginning-of-line)
+ (point)))
+ (dolist (err errors)
+ (insert (format "#ifdef %s\n { \"%s\", %s },\n#endif\n"
+ err err err)))))
*/
/***BEGIN errtab***/
#ifdef EPERM
JNIEXPORT jobject JNIFUNC(strerror)(JNIEnv *jni, jobject cls, jint err)
{ return (wrap_cstring(jni, strerror(err))); }
+/*----- Messing with file descriptors -------------------------------------*/
+
+static void fdguts(JNIEnv *jni, jclass *cls, jfieldID *fid)
+{
+ *cls = (*jni)->FindClass(jni, "java/io/FileDescriptor"); assert(cls);
+ *fid = (*jni)->GetFieldID(jni, *cls, "fd", "I"); // OpenJDK
+ if (!*fid) *fid = (*jni)->GetFieldID(jni, *cls, "descriptor", "I"); // Android
+ assert(*fid);
+}
+
+static int fdint(JNIEnv *jni, jobject jfd)
+{
+ jclass cls;
+ jfieldID fid;
+
+ fdguts(jni, &cls, &fid);
+ return ((*jni)->GetIntField(jni, jfd, fid));
+}
+
+static jobject newfd(JNIEnv *jni, int fd)
+{
+ jobject jfd;
+ jclass cls;
+ jmethodID init;
+ jfieldID fid;
+
+ fdguts(jni, &cls, &fid);
+ init = (*jni)->GetMethodID(jni, cls, "<init>", "()V"); assert(init);
+ jfd = (*jni)->NewObject(jni, cls, init);
+ (*jni)->SetIntField(jni, jfd, fid, fd);
+ return (jfd);
+}
+
+JNIEXPORT jint JNIFUNC(fdint)(JNIEnv *jni, jobject cls, jobject jfd)
+ { return (fdint(jni, jfd)); }
+
+JNIEXPORT jobject JNIFUNC(newfd)(JNIEnv *jni, jobject cls, jint fd)
+ { return (newfd(jni, fd)); }
+
+JNIEXPORT jboolean JNIFUNC(isatty)(JNIEnv *jni, jobject cls, jobject jfd)
+ { return (isatty(fdint(jni, jfd))); }
+
/*----- Low-level file operations -----------------------------------------*/
/* Java has these already, as methods on `java.io.File' objects. Alas, these
put_cstring(jni, to, tostr);
}
-#define LKF_EXCL 1u
-#define LKF_WAIT 2u
+#define LKF_EXCL 0x1000u
+#define LKF_WAIT 0x2000u
struct lockf {
struct native_base _base;
int fd;
pathstr = get_cstring(jni, path); if (!pathstr) goto end;
again:
- fd = open(pathstr, O_RDWR | O_CREAT); if (fd < 0) goto err;
+ fd = open(pathstr, O_RDWR | O_CREAT, flags&07777); if (fd < 0) goto err;
if (fstat(fd, &st0)) goto err;
f = fcntl(fd, F_GETFD); if (f < 0) goto err;
if (fcntl(fd, F_SETFD, f | FD_CLOEXEC)) goto err;
jclass cls;
jmethodID init;
jint modehack;
-
+
modehack = st->st_mode&07777;
if (S_ISFIFO(st->st_mode)) modehack |= 0010000;
else if (S_ISCHR(st->st_mode)) modehack |= 0020000;
/*----- Imports -----------------------------------------------------------*/
-import java.io.{Closeable, File, FileOutputStream, FileReader, IOException};
-
import scala.collection.mutable.HashMap;
+import java.io.{Closeable, File};
+import java.net.{URL, URLConnection};
+import java.util.zip.GZIPInputStream;
+
+import sys.{SystemError, hashsz, runCommand};
+import sys.Errno.EEXIST;
+import sys.FileImplicits._;
+
/*----- Useful regular expressions ----------------------------------------*/
-val RX_COMMENT = """(?x) ^ \s* (?: \# .* )? $""".r;
-val RX_KEYVAL = """(?x) ^ \s*
+private val RX_COMMENT = """(?x) ^ \s* (?: \# .* )? $""".r;
+private val RX_KEYVAL = """(?x) ^ \s*
([-\w]+)
(?:\s+(?!=)|\s*=\s*)
(|\S|\S.*\S)
\s* $""".r;
-val RX_DOLLARSUBST = """(?x) \$ \{ ([-\w]+) \}""".r;
+private val RX_DOLLARSUBST = """(?x) \$ \{ ([-\w]+) \}""".r;
/*----- Things that go wrong ----------------------------------------------*/
type Config = scala.collection.Map[String, String];
-val DEFAULTS: Seq[(String, Config => String)] =
+private val DEFAULTS: Seq[(String, Config => String)] =
Seq("repos-base" -> { _ => "tripe-keys.tar.gz" },
"sig-base" -> { _ => "tripe-keys.sig-<SEQ>" },
"repos-url" -> { conf => conf("base-url") + conf("repos-base") },
case "naclbox" => "poly1305/128"
case _ =>
val h = conf("hash");
- JNI.hashsz(h) match {
+ hashsz(h) match {
case -1 => throw new DefaultFailed("hash")
case hsz => s"${h}-hmac/${4*hsz}"
}
"sig-fresh" -> { _ => "always" },
"fingerprint-hash" -> { _("hash") });
-def readConfig(path: String): Config = {
- var m = HashMap[String, String]();
- withCleaner { clean =>
- var in = new FileReader(path); clean { in.close(); }
- var lno = 1;
- for (line <- lines(in)) {
- line match {
- case RX_COMMENT() => ();
- case RX_KEYVAL(key, value) => m += key -> value;
- case _ =>
- throw new ConfigSyntaxError(path, lno, "failed to parse line");
- }
- lno += 1;
- }
- }
+/*----- Managing a key repository -----------------------------------------*/
- for ((key, dflt) <- DEFAULTS) {
- if (!(m contains key)) {
- try { m += key -> dflt(m); }
- catch {
- case e: DefaultFailed =>
- throw new ConfigDefaultFailed(path, key, e.key, m(e.key));
- }
+def downloadToFile(file: File, url: URL, maxlen: Long = Long.MaxValue) {
+ fetchURL(url, new URLFetchCallbacks {
+ val out = file.openForOutput();
+ private def toobig() {
+ throw new KeyConfigException(s"remote file `$url' is " +
+ "suspiciously large");
}
- }
- m
+ var totlen: Long = 0;
+ override def preflight(conn: URLConnection) {
+ totlen = conn.getContentLength;
+ if (totlen > maxlen) toobig();
+ }
+ override def done(win: Boolean) { out.close(); }
+ def write(buf: Array[Byte], n: Int, len: Long) {
+ if (len + n > maxlen) toobig();
+ out.write(buf, 0, n);
+ }
+ });
}
-/*----- Managing a key repository -----------------------------------------*/
-
/* Lifecycle notes
*
* -> empty
object State extends Enumeration {
val Empty, Pending, Confirmed, Updating, Committing, Live = Value;
}
-
}
+class RepositoryStateException(val state: Repository.State.Value,
+ msg: String)
+ extends Exception(msg);
+
+class KeyConfigException(msg: String) extends Exception(msg);
+
class Repository(val root: File) extends Closeable {
import Repository.State.{Value => State, _};
- val livedir = new File(root, "live");
- val livereposdir = new File(livedir, "repos");
- val newdir = new File(root, "new");
- val olddir = new File(root, "old");
- val pendingdir = new File(root, "pending");
- val tmpdir = new File(root, "tmp");
-
- val lock = {
- if (!root.isDirectory && !root.mkdir()) ???;
- val chan = new FileOutputStream(new File(root, "lk")).getChannel;
- chan.tryLock() match {
- case null =>
- throw new IOException(s"repository `${root.getPath}' locked")
- case lk => lk
+ /* Important directories and files. */
+ private[this] val livedir = root + "live";
+ private[this] val livereposdir = livedir + "repos";
+ private[this] val newdir = root + "new";
+ private[this] val olddir = root + "old";
+ private[this] val pendingdir = root + "pending";
+ private[this] val tmpdir = root + "tmp";
+
+ /* Take out a lock in case of other instances. */
+ private[this] val lock = {
+ try { root.mkdir_!(); }
+ catch { case SystemError(EEXIST, _) => ok; }
+ (root + "lk").lock_!()
+ }
+ def close() { lock.close(); }
+
+ /* Maintain a cache of some repository state. */
+ private var _state: State = null;
+ private var _config: Config = null;
+ private def invalidate() {
+ _state = null;
+ _config = null;
+ }
+
+ def state: State = {
+ /* Determine the current repository state. */
+
+ if (_state == null)
+ _state = if (livedir.isdir_!) {
+ if (!livereposdir.isdir_!) Confirmed
+ else if (newdir.isdir_!) Updating
+ else Live
+ } else {
+ if (newdir.isdir_!) Committing
+ else if (pendingdir.isdir_!) Pending
+ else Empty
+ }
+
+ _state
+ }
+
+ def checkState(wanted: State*) {
+ /* Ensure we're in a particular state. */
+ val st = state;
+ if (wanted.forall(_ != st)) {
+ throw new RepositoryStateException(st, s"Repository is $st, not " +
+ oxford("or",
+ wanted.map(_.toString)));
+ }
+ }
+
+ def cleanup() {
+
+ /* If we're part-way through an update then back out or press forward. */
+ state match {
+
+ case Updating =>
+ /* We have a new tree allegedly ready, but the current one is still
+ * in place. It seems safer to zap the new one here, but we could go
+ * either way.
+ */
+
+ newdir.rmTree();
+ invalidate(); // should move back to `Live' or `Confirmed'
+
+ case Committing =>
+ /* We have a new tree ready, and an old one moved aside. We're going
+ * to have to move one of them. Let's try committing the new tree.
+ */
+
+ newdir.rename_!(livedir); // should move on to `Live'
+ invalidate();
+
+ case _ =>
+ /* Other states are stable. */
+ ok;
}
+
+ /* Now work through the things in our area of the filesystem and zap the
+ * ones which don't belong. In particular, this will always erase
+ * `tmpdir'.
+ */
+ val st = state;
+ root.foreachFile { f => (f.getName, st) match {
+ case ("lk", _) => ok;
+ case ("live", Live | Confirmed) => ok;
+ case ("pending", Pending) => ok;
+ case (_, Updating | Committing) =>
+ unreachable(s"unexpectedly still in `$st' state");
+ case _ => f.rmTree();
+ }
+ } }
+
+ def destroy() {
+ /* Clear out the entire repository. Everything. It's all gone. */
+ root.foreachFile { f => if (f.getName != "lk") f.rmTree(); }
+ }
+
+ def clearTmp() {
+ /* Arrange to have an empty `tmpdir'. */
+ tmpdir.rmTree();
+ tmpdir.mkdir_!();
+ }
+
+ def config: Config = {
+ /* Return the repository configuration. */
+
+ if (_config == null) {
+
+ /* Firstly, decide where to find the configuration file. */
+ cleanup();
+ val dir = state match {
+ case Live | Confirmed => livedir
+ case Pending => pendingdir
+ case Empty =>
+ throw new RepositoryStateException(Empty, "repository is Empty");
+ }
+ val file = dir + "tripe-keys.conf";
+
+ /* Build the new configuration in a temporary place. */
+ var m = HashMap[String, String]();
+
+ /* Read the config file into our map. */
+ file.withReader { in =>
+ var lno = 1;
+ for (line <- lines(in)) {
+ line match {
+ case RX_COMMENT() => ok;
+ case RX_KEYVAL(key, value) => m += key -> value;
+ case _ =>
+ throw new ConfigSyntaxError(file.getPath, lno,
+ "failed to parse line");
+ }
+ lno += 1;
+ }
+ }
+
+ /* Fill in defaults where things have been missed out. */
+ for ((key, dflt) <- DEFAULTS) {
+ if (!(m contains key)) {
+ try { m += key -> dflt(m); }
+ catch {
+ case e: DefaultFailed =>
+ throw new ConfigDefaultFailed(file.getPath, key,
+ e.key, m(e.key));
+ }
+ }
+ }
+
+ /* All done. */
+ _config = m;
+ }
+
+ _config
}
- def close() {
- lock.release();
- lock.channel.close();
+ def fetchConfig(url: URL) {
+ /* Fetch an initial configuration file from a given URL. */
+
+ checkState(Empty);
+ clearTmp();
+ downloadToFile(tmpdir + "tripe-keys.conf", url);
+ tmpdir.rename_!(pendingdir);
+ invalidate(); // should move to `Pending'
}
- def state: State =
- if (livedir.isDirectory) {
- if (!livereposdir.isDirectory) Confirmed
- else if (newdir.isDirectory && olddir.isDirectory) Committing
- else Live
- } else {
- if (newdir.isDirectory) Updating
- else if (pendingdir.isDirectory) Pending
- else Empty
+ def confirm() {
+ /* The user has approved the master key fingerprint in the `Pending'
+ * configuration. Advance to `Confirmed'.
+ */
+
+ checkState(Pending);
+ pendingdir.rename_!(livedir);
+ invalidate(); // should move to `Confirmed'
+ }
+
+ def update() {
+ /* Update the repository from the master.
+ *
+ * Fetch a (possibly new) archive; unpack it; verify the master key
+ * against the known fingerprint; and check the signature on the bundle.
+ */
+
+ checkState(Confirmed, Live);
+ val conf = config;
+ clearTmp();
+
+ /* First thing is to download the tarball and signature. */
+ val tarfile = tmpdir + "tripe-keys.tar.gz";
+ downloadToFile(tarfile, new URL(conf("repos-url")));
+ val sigfile = tmpdir + "tripe-keys.sig";
+ val seq = conf("master-sequence");
+ downloadToFile(sigfile,
+ new URL(conf("sig-url").replaceAllLiterally("<SEQ>",
+ seq)));
+
+ /* Unpack the tarball. Carefully. */
+ val unpkdir = tmpdir + "unpk";
+ unpkdir.mkdir_!();
+ withCleaner { clean =>
+ val tar = new TarFile(new GZIPInputStream(tarfile.open()));
+ clean { tar.close(); }
+ for (e <- tar) {
+
+ /* Check the filename to make sure it's not evil. */
+ if (e.name.split('/').exists { _ == ".." })
+ throw new KeyConfigException("invalid path in tarball");
+
+ /* Find out where this file points. */
+ val f = unpkdir + e.name;
+
+ /* Unpack it. */
+ if (e.isdir) {
+ /* A directory. Create it if it doesn't exist already. */
+
+ try { f.mkdir_!(); }
+ catch { case SystemError(EEXIST, _) => ok; }
+ } else if (e.isreg) {
+ /* A regular file. Write stuff to it. */
+
+ e.withStream { in =>
+ f.withOutput { out =>
+ for ((b, n) <- blocks(in)) out.write(b, 0, n);
+ }
+ }
+ } else {
+ /* Something else. Be paranoid and reject it. */
+
+ throw new KeyConfigException("unexpected object type in tarball");
+ }
+ }
}
- def commitState(): State = state match {
- case Updating => rmTree(newdir); state
- case Committing =>
- if (!newdir.renameTo(livedir) && !olddir.renameTo(livedir))
- throw new IOException("failed to commit update");
- state
- case st => st;
+ /* There ought to be a file in here called `repos/master.pub'. */
+ val reposdir = unpkdir + "repos";
+ if (!reposdir.isdir_!)
+ throw new KeyConfigException("missing `repos/' directory");
+ val masterfile = reposdir + "master.pub";
+ if (!masterfile.isreg_!)
+ throw new KeyConfigException("missing `repos/master.pub' file");
- def clean() {
-
+ /* Fetch the master key's fingerprint. */
+ val (out, _) = runCommand("key", "-k", masterfile.getPath,
+ "fingerprint",
+ "-f", "-secret",
+ "-a", conf("fingerprint-hash"),
+ s"master-$seq");
+ println(s";; $out");
+ }
}
/*----- That's all, folks -------------------------------------------------*/
val RX_RESOLVE = """(?x) \$ ([46*]*) \[ ([^\]]+) \]""".r;
val RX_PARENT = """(?x) [^\s,]+""".r
-def with_cleaner[T](body: Cleaner => T): T = {
- val cleaner = new Cleaner;
- try { body(cleaner) }
- finally { cleaner.cleanup(); }
-}
-
-class Cleaner {
- var cleanups: List[() => Unit] = Nil;
- def apply(cleanup: => Unit) { cleanups +:= { () => cleanup; } }
- def cleanup() { cleanups foreach { _() } }
-}
-
-def lines(r: Reader) = new Traversable[String] {
- val in: BufferedReader = new BufferedReader(r);
- override def foreach[T](f: String => T) {
- while (true) in.readLine match {
- case null => return;
- case line => f(line);
- }
- }
-}
-
-def thread(name: String, run: Boolean = true, daemon: Boolean = true)
- (body: => Unit): Thread = {
- val t = new Thread(new Runnable { override def run() { body } }, name);
- t.setDaemon(daemon);
- if (run) t.start();
- t
-}
-
object BulkResolver {
val BREAK = new Breaks;
}
val workers = Array.tabulate(nthreads) { i =>
thread(s"resolver worker #$i") {
- breakable {
- while (true) {
- val host = ch.read; if (host == null) break;
+ loopUnit { exit =>
+ val host = ch.read; if (host == null) exit;
println(s";; ${Thread.currentThread.getName} resolving `${host.name}'");
- try {
- for (a <- InetAddress.getAllByName(host.name)) host.addaddr(a);
- } catch { case e: UnknownHostException => () }
- }
+ try {
+ for (a <- InetAddress.getAllByName(host.name)) host.addaddr(a);
+ } catch { case e: UnknownHostException => () }
}
println(s";; ${Thread.currentThread.getName} done'");
ch.write(null);
def parseFile(path: File): this.type = {
println(s";; parse ${path.getPath}");
- with_cleaner { clean =>
+ withCleaner { clean =>
val in = new FileReader(path); clean { in.close(); }
val lno = 1;
--- /dev/null
+/* -*-scala-*-
+ *
+ * Reporting progress for long-running jobs
+ *
+ * (c) 2018 Straylight/Edgeware
+ */
+
+/*----- Licensing notice --------------------------------------------------*
+ *
+ * This file is part of the Trivial IP Encryption (TrIPE) Android app.
+ *
+ * TrIPE is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * TrIPE is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package uk.org.distorted.tripe; package object progress;
+
+/*----- Imports -----------------------------------------------------------*/
+
+import Math.ceil;
+import System.currentTimeMillis;
+import System.{err => stderr}; // FIXME: split out terminal progress
+
+/*----- Main code ---------------------------------------------------------*/
+
+def formatTime(t: Int): String =
+ if (t < -1) "???"
+ else {
+ val (s, t1) = (t%60, t/60);
+ val (m, h) = (t1%60, t1/60);
+ if (h > 0) f"$h%d:$m%02d:$s%02d"
+ else f"$m%02d:$s%02d"
+ }
+
+private val UDATA = Seq("kB", "MB", "GB", "PB", "EB");
+def formatBytes(n: Long): String = {
+ val (x, u) = (n.toDouble, "B ") /: UDATA {
+ case ((x, u), name) if x >= 1024.0 => (x/1024.0, name)
+ case (xu, _) => xu
+ }
+ f"$x%6.1f$u%s"
+}
+
+trait Eyecandy {
+ def set(line: String);
+ def clear();
+ def commit();
+ def commit(line: String) { commit(); set(line); commit(); }
+
+ def begin(job: Job);
+}
+
+
+trait Job with Publisher[ {
+ def what: String; // imperative for what we're doing
+ def cur: Long; // current position in work
+ def max: Long; // maximum work to do
+ def format: String; // describe progress in useful terms
+
+ private[this] val t0 = currentTimeMillis;
+
+ def eta: Int =
+ /* Report the estimated time remaining in seconds, or -1 if no idea.
+ *
+ * The model here is very stupid. Weird jobs should override this and do
+ * something more sensible.
+ */
+
+ if (max < 0 || cur <= 0) -1
+ else ceil((currentTimeMillis - t0)/1000.0 *
+ (max - cur)/cur.toDouble).toInt;
+}
+
+object TerminalEyecandy extends Eyecandy {
+ private var last = "";
+ var eyecandyp =
+
+}
+
+/*----- That's all, folks -------------------------------------------------*/
/*----- Imports -----------------------------------------------------------*/
+import scala.collection.convert.decorateAsJava._;
import scala.collection.mutable.HashSet;
-import java.io.{Closeable, File};
+import java.io.{BufferedReader, BufferedWriter, Closeable, File,
+ FileDescriptor, FileInputStream, FileOutputStream,
+ InputStream, InputStreamReader,
+ OutputStream, OutputStreamWriter};
import java.nio.{ByteBuffer, CharBuffer};
import java.nio.charset.Charset;
import java.util.Date;
var sz: Int = (s.length*enc.averageBytesPerChar + 1).toInt;
var out = ByteBuffer.allocate(sz);
- while (true) {
+ loop[CString] { exit =>
/* If there's still stuff to encode, then encode it. Otherwise,
* there must be some dregs left in the encoder, so flush them out.
*/
val v = new Array[Byte](n + 1);
out.array.copyToArray(v, 0, n);
v(n) = 0;
- return v;
+ exit(v);
}
}
-
- /* Placate the type checker. */
- unreachable("unreachable");
}
}
/* Import the native code library. */
System.loadLibrary("toy");
-/* Exception indicating that a wrapped native object has been clobbered. */
-class NativeObjectTypeException(msg: String) extends RuntimeException(msg);
+/* Native types.
+ *
+ * See `jni.c'. There's no good way to hand a C pointer into Java, so we
+ * just copy whole structures into Java byte arrays and hope. Well, also we
+ * tag them so we can detect mixups.
+ */
type Wrapper = Array[Byte];
+class NativeObjectTypeException(msg: String) extends RuntimeException(msg);
/*----- Error codes -------------------------------------------------------*/
+/* Machinery for collecting error information from C. */
protected case class ErrorEntry(val tag: String, val err: Int);
@native protected def errtab: Array[ErrorEntry];
@native protected def strerror(err: Int): CString;
object Errno extends Enumeration {
- private val tagmap = {
+ /* System errors.
+ *
+ * There are two slight difficulties here.
+ *
+ * * Not all target systems have the same errors. C has a preprocessor
+ * to deal with this, but we don't; so instead we'll define all of the
+ * errors we'll ever need, but maybe with bogus values.
+ *
+ * * Some systems reuse code numbers for two different error names, e.g.,
+ * both `EAGAIN' and `EWOULDBLOCK' are the same on Linux -- but not
+ * necessarily on other systems. Scala's `Enumeration' machinery
+ * doesn't like sharing `id' numbers between values.
+ *
+ * We augment the value type with an additional `code' value which is the
+ * actual system error code; we arbitrarily pick one error symbol with a
+ * given code to be `canonical', i.e., it has E.id == E.code; the others
+ * have synthetic `id' values. And symbols which don't correspond to any
+ * error on the target system have synthetic `id' /and/ `code', so that
+ * they can still be spoken about, but won't match any real error.
+ */
+
+ private val tagmap = { // map names to numbers based on what C reports
val b = Map.newBuilder[String, Int];
for (ErrorEntry(tag, err) <- errtab) b += tag -> err;
b.result
}
- private var wrong = -255;
- private val seen = HashSet[Int]();
- class ErrnoVal private[Errno](tag: String, val code: Int, id: Int)
- extends Val(id, tag) {
+ private val seen = HashSet[Int](); // which error codes have been taken
+
+ private var wrong = -256; // next synthetic code
+ private def nextwrong: Int = { val w = wrong; wrong -= 1; w }
+
+ class Type private[Errno](tag: String, val code: Int, id: Int)
+ extends Val(id, tag) {
+ /* Our augmented error type. */
+
def message: String = strerror(code).toJString;
}
+ private class UnknownError(code: Int)
+ extends Type("<unknown>", code, code);
- private def err(tag: String, code: Int): ErrnoVal = {
- if (seen contains code) { wrong -= 1; new ErrnoVal(tag, code, wrong) }
- else { seen += code; new ErrnoVal(tag, code, code) }
+ private def err(tag: String, code: Int): Type = {
+ /* Construct an error symbol given its tag string and a code number. */
+
+ if (code < 0) new Type(tag, code, code)
+ else if (seen contains code) new Type(tag, code, nextwrong)
+ else { seen += code; new Type(tag, code, code) }
+ }
+ private def err(tag: String): Type =
+ err(tag, tagmap.getOrElse(tag, nextwrong));
+
+ def byid(id: Int): Value = {
+ if (seen contains id) apply(id)
+ else new UnknownError(id)
}
- private def err(tag: String): ErrnoVal = err(tag, tagmap(tag));
- val OK = err("OK", 0);
+ val OK = err("OK", 0); // `errno' zero is a real thing
/*
;;; The errno name table is very boring to type. To make life less
val EHWPOISON = err("EHWPOISON");
/***end***/
}
-import Errno.{Value => _, _};
+import Errno.{Type => Errno, EEXIST, EISDIR, ENOENT, ENOTDIR};
object SystemError {
- def apply(err: Errno.Value, what: String): SystemError =
+ /* Pattern matching for `SystemError', below. */
+
+ def apply(err: Errno, what: String): SystemError =
new SystemError(err, what);
- def unapply(e: Exception): Option[(Errno.Value, String)] = e match {
+ def unapply(e: Exception): Option[(Errno, String)] = e match {
case e: SystemError => Some((e.err, e.what))
case _ => None
}
}
+class SystemError (val err: Errno, val what: String) extends Exception {
+ /* An error from a syscall or similar, usually from native code. */
-class SystemError private[this](val err: Errno.ErrnoVal, val what: String)
- extends Exception {
- def this(err: Errno.Value, what: String)
- { this(err.asInstanceOf[Errno.ErrnoVal], what); }
+ /* A constructor which takes an error number, for easier access from C. */
private def this(err: Int, what: CString)
- { this(Errno(err), what.toJString); }
+ { this(Errno.byid(err).asInstanceOf[Errno], what.toJString); }
+
override def getMessage(): String = s"$what: ${err.message}";
}
def rename(from: String, to: String)
{ rename(from.toCString, to.toCString); }
+@native def fdint(fd: FileDescriptor): Int;
+@native def newfd(fd: Int): FileDescriptor;
+@native def isatty(fd: FileDescriptor): Boolean;
+
/*----- File status information -------------------------------------------*/
/* These are the traditional values, but the C code carefully arranges to
* return them regardless of what your kernel actually thinks.
*/
-val S_IFMT = 0xf000;
-val S_IFIFO = 0x1000;
-val S_IFCHR = 0x2000;
-val S_IFDIR = 0x4000;
-val S_IFBLK = 0x6000;
-val S_IFREG = 0x8000;
-val S_IFLNK = 0xa000;
-val S_IFSOCK = 0xc000;
-
+final val S_IFMT = 0xf000;
+final val S_IFIFO = 0x1000;
+final val S_IFCHR = 0x2000;
+final val S_IFDIR = 0x4000;
+final val S_IFBLK = 0x6000;
+final val S_IFREG = 0x8000;
+final val S_IFLNK = 0xa000;
+final val S_IFSOCK = 0xc000;
+
+/* Primitive read-the-file-status calls. */
@native protected def stat(path: CString): sys.FileInfo;
def stat(path: String): sys.FileInfo = stat(path.toCString);
@native protected def lstat(path: CString): sys.FileInfo;
def lstat(path: String): sys.FileInfo = lstat(path.toCString);
object FileInfo extends Enumeration {
+ /* A simple enumeration of things a file might be. */
val FIFO, CHR, DIR, BLK, REG, LNK, SOCK, UNK = Value;
type Type = Value;
}
import FileInfo._;
-
class FileInfo private[this](val devMajor: Int, val devMinor: Int,
val ino: Long, val mode: Int, val nlink: Int,
val uid: Int, val gid: Int,
val blksize: Int, val blocks: Long,
val atime: Date, val mtime: Date,
val ctime: Date) {
+ /* Information about a file. This is constructed directly from native
+ * code.
+ */
+
private def this(devMajor: Int, devMinor: Int, ino: Long,
mode: Int, nlink: Int, uid: Int, gid: Int,
rdevMinor: Int, rdevMajor: Int,
size: Long, blksize: Int, blocks: Long,
atime: Long, mtime: Long, ctime: Long) {
+ /* Lightly cook the values from the underlying `struct stat'. */
+
this(devMajor, devMinor, ino, mode, nlink, uid, gid,
rdevMajor, rdevMinor, size, blksize, blocks,
new Date(atime), new Date(mtime), new Date(ctime));
}
+ /* Return the file permissions only. */
def perms: Int = mode&0xfff;
+ /* Return the filetype, as a `FileInfo.Type'. */
def ftype: Type = (mode&S_IFMT) match {
case S_IFIFO => FIFO
case S_IFCHR => CHR
}
private[this] def mustBeDevice() {
+ /* Insist that you only ask for `rdev' fields on actual device nodes. */
ftype match {
- case CHR | BLK => ();
+ case CHR | BLK => ok;
case _ => throw new IllegalArgumentException("Object is not a device");
}
}
+
+ /* Query the device-node numbers. */
def rdevMajor: Int = { mustBeDevice(); _rdevMajor }
def rdevMinor: Int = { mustBeDevice(); _rdevMinor }
}
/*----- Listing directories -----------------------------------------------*/
+/* Primitive operations. */
@native protected def opendir(path: CString): Wrapper;
@native protected def readdir(path: CString, dir: Wrapper): CString;
@native protected def closedir(path: CString, dir: Wrapper);
protected abstract class BaseDirIterator[T](cpath: CString)
extends LookaheadIterator[T] with Closeable {
+ /* The underlying machinery for directory iterators.
+ *
+ * Subclasses must define `mangle' to convert raw filenames into a T.
+ * We keep track of the path C-string, because we need to keep passing that
+ * back to C for inclusion in error messages. Recording higher-level
+ * things is left for subclasses.
+ */
+
+ /* Constructors from more convenient types. */
def this(path: String) { this(path.toCString); }
def this(dir: File) { this(dir.getPath); }
+
+ /* Cleaning up after ourselves. */
override def close() { closedir(cpath, dir); }
override protected def finalize() { super.finalize(); close(); }
- private[this] val dir = opendir(cpath);
+
+ /* Subclass responsibility. */
protected def mangle(file: String): T;
+
+ /* Main machinery. */
+ private[this] val dir = opendir(cpath);
override protected def fetch(): Option[T] = readdir(cpath, dir) match {
case null => None
case f => f.toJString match {
}
class DirIterator(val path: String) extends BaseDirIterator[String](path) {
+ /* Iterator over the basenames of files in a directory. */
+
def this(dir: File) { this(dir.getPath); }
+
override protected def mangle(file: String): String = file;
}
class DirFilesIterator private[this](val dir: File, cpath: CString)
extends BaseDirIterator[File](cpath) {
+ /* Iterator over full `File' objects in a directory. */
+
def this(dir: File) { this(dir, dir.getPath.toCString); }
def this(path: String) { this(new File(path), path.toCString); }
+
override protected def mangle(file: String): File = new File(dir, file);
}
/*----- File locking ------------------------------------------------------*/
-val LKF_EXCL = 1;
-val LKF_WAIT = 2;
-@native protected def lock(path: CString, flags: Int): Wrapper;
+/* Primitive operations. The low `mode' bits are for the lock file if we
+ * have to create it.
+ */
+final val LKF_EXCL = 0x1000;
+final val LKF_WAIT = 0x2000;
+@native protected def lock(path: CString, mode: Int): Wrapper;
@native protected def unlock(lock: Wrapper);
class FileLock(path: String, flags: Int) extends Closeable {
+ /* A class which represents a held lock on a file. */
+
+ /* Constructors. The default is to take an exclusive lock or fail
+ * immediately.
+ */
def this(file: File, flags: Int) { this(file.getPath, flags); }
- def this(path: String) { this(path, LKF_EXCL); }
- def this(file: File) { this(file.getPath, LKF_EXCL); }
+ def this(path: String) { this(path, LKF_EXCL | 0x1b6); }
+ def this(file: File) { this(file.getPath, LKF_EXCL | 0x1b6); }
+
+ /* The low-level lock object, actually a file descriptor. */
private[this] val lk = lock(path.toCString, flags);
+
+ /* Making sure things get cleaned up. */
override def close() { unlock(lk); }
override protected def finalize() { super.finalize(); close(); }
}
object FileImplicits {
implicit class FileOps(file: File) {
+ /* Augment `File' with operations which throw informative (if low-level
+ * and system-specific) exceptions rather than returning unhelpful
+ * win/lose booleans. These have names ending with `_!' because they
+ * might explode.
+ *
+ * And some other useful methods.
+ */
+ /* Constructing names of files in a directory. Honestly, I'm surprised
+ * there isn't a method for this already.
+ */
+ def +(sub: String): File = new File(file, sub);
+
+ /* Simple file operations. */
def unlink_!() { unlink(file.getPath); }
def rmdir_!() { rmdir(file.getPath); }
def mkdir_!(mode: Int) { mkdir(file.getPath, mode); }
def mkdir_!() { mkdir_!(0x1ff); }
def mkfile_!(mode: Int) { mkfile(file.getPath, mode); }
def mkfile_!() { mkfile_!(0x1b6); }
+ def rename_!(to: File) { rename(file.getPath, to.getPath); }
+ /* Listing directories. */
def withFilesIterator[T](body: DirFilesIterator => T): T = {
val iter = new DirFilesIterator(file.getPath);
try { body(iter) } finally { iter.close(); }
}
+ def foreachFile(fn: File => Unit) { withFilesIterator(_.foreach(fn)) }
def files_! : Seq[File] = withFilesIterator { _.toSeq };
+ /* Low-level lFile information. */
def stat_! : FileInfo = stat(file.getPath);
def lstat_! : FileInfo = lstat(file.getPath);
+ /* Specific file-status queries. */
private[this] def statish[T](statfn: String => FileInfo,
ifexists: FileInfo => T,
ifmissing: => T): T =
def issock_! : Boolean = statish(stat _, _.ftype == SOCK, false);
def remove_!() {
+ /* Delete a file, or directory, whatever it is. */
while (true) {
- try { unlink_!(); return }
+ try { unlink_!(); return; }
catch {
case SystemError(ENOENT, _) => return;
- case SystemError(EISDIR, _) => ();
+ case SystemError(EISDIR, _) => ok;
}
- try { rmdir_!(); return }
+ try { rmdir_!(); return; }
catch {
case SystemError(ENOENT, _) => return;
- case SystemError(ENOTDIR, _) => ();
+ case SystemError(ENOTDIR, _) => ok;
}
}
}
def rmTree() {
+ /* Delete a thing recursively. */
def walk(f: File) {
- if (f.isdir_!) f.withFilesIterator { _ foreach(walk _) };
+ if (f.isdir_!) f.foreachFile(walk _);
f.remove_!();
}
walk(file);
}
+ /* File locking. */
+ def lock_!(flags: Int): FileLock = new FileLock(file.getPath, flags);
+ def lock_!(): FileLock = lock_!(LKF_EXCL | 0x1b6);
def withLock[T](flags: Int)(body: => T): T = {
- val lk = new FileLock(file.getPath, flags);
+ val lk = lock_!(flags);
try { body } finally { lk.close(); }
}
- def withLock[T](body: => T): T = withLock(LKF_EXCL) { body };
+ def withLock[T](body: => T): T = withLock(LKF_EXCL | 0x1b6) { body };
+
+ /* Opening files. Again, I'm surprised this isn't here already. */
+ def open(): FileInputStream = new FileInputStream(file);
+ def openForOutput(): FileOutputStream = new FileOutputStream(file);
+ def reader(): BufferedReader =
+ new BufferedReader(new InputStreamReader(open()));
+ def writer(): BufferedWriter =
+ new BufferedWriter(new OutputStreamWriter(openForOutput()));
+ def withInput[T](body: FileInputStream => T): T = {
+ val in = open();
+ try { body(in) }
+ finally { in.close(); }
+ }
+ def withOutput[T](body: FileOutputStream => T): T = {
+ val out = openForOutput();
+ try { body(out) } finally { out.close(); }
+ }
+ def withReader[T](body: BufferedReader => T): T = withInput { in =>
+ body(new BufferedReader(new InputStreamReader(in)))
+ };
+ def withWriter[T](body: BufferedWriter => T): T = withOutput { out =>
+ val w = new BufferedWriter(new OutputStreamWriter(out));
+ /* Do this the hard way, so that we flush the `BufferedWriter'. */
+ try { body(w) } finally { w.close(); }
+ }
}
}
import FileImplicits._;
val buf = new Array[Byte](6);
val b = new StringBuilder;
- while (true) {
+ loop[File] { exit =>
/* Keep going until we find a fresh one. */
/* Provide a prefix. Mostly this is to prevent the file starting with
* win.
*/
val f = new File(d, b.result); b.clear();
- try { f.mkfile_!(); return f; }
- catch { case SystemError(EEXIST, _) => (); }
+ try { f.mkfile_!(); exit(f); }
+ catch { case SystemError(EEXIST, _) => ok; }
}
+}
+
+/*----- Running a command -------------------------------------------------*/
- /* We shouldn't get here, but the type checker needs placating. */
- unreachable("unreachable");
+private val devnull = new File("/dev/null");
+
+private def captureStream(in: InputStream, out: StringBuilder) {
+ /* Capture the INSTREAM's contents in a string. */
+
+ for ((buf, n) <- blocks(new InputStreamReader(in)))
+ out.appendAll(buf, 0, n);
+}
+
+class SubprocessFailed(val cmd: Seq[String], rc: Int, stderr: String)
+ extends Exception {
+ override def getMessage(): String =
+ s"process (${quoteTokens(cmd)}) failed (rc = $rc):\n" + stderr
+}
+
+def runCommand(cmd: String*): (String, String) = {
+ /* Run a command, returning its stdout and stderr. */
+
+ withCleaner { clean =>
+
+ /* Create the child process and pick up the ends of its streams. */
+ val pb = new ProcessBuilder(cmd.asJava).redirectInput(devnull);
+ val kid = pb.start(); clean { kid.destroy(); }
+ val out = kid.getInputStream(); clean { out.close(); }
+ val err = kid.getErrorStream(); clean { err.close(); }
+
+ /* Capture the output in threads, so we don't block. Also, wait for the
+ * child to complete. Amazingly, messing with threads here isn't too
+ * much of a disaster.
+ */
+ val bout, berr = new StringBuilder;
+ val rdout = thread("capture process stdout", daemon = false) {
+ captureStream(out, bout);
+ }
+ val rderr = thread("capture process stderr", daemon = false) {
+ captureStream(err, berr);
+ }
+ val wait = thread("await process exit", daemon = false) {
+ kid.waitFor();
+ }
+ rdout.join(); rderr.join(); wait.join();
+
+ /* Check the exit status. */
+ val rc = kid.exitValue;
+ if (rc != 0) throw new SubprocessFailed(cmd, rc, berr.result);
+
+ /* We're all done. */
+ return (bout.result, berr.result);
+ }
}
/*----- Connecting to a server --------------------------------------------*/
-val CF_CLOSERD = 1;
-val CF_CLOSEWR = 2;
-val CF_CLOSEMASK = CF_CLOSERD | CF_CLOSEWR;
+/* Primitive operations. */
+final val CF_CLOSERD = 1;
+final val CF_CLOSEWR = 2;
+final val CF_CLOSEMASK = CF_CLOSERD | CF_CLOSEWR;
@native protected def connect(path: CString): Wrapper;
@native protected def send(conn: Wrapper, buf: CString,
start: Int, len: Int);
@native def closeconn(conn: Wrapper, how: Int);
class Connection(path: String) extends Closeable {
- def this(file: File) { this(file.getPath); }
+
+ /* The underlying primitive connection. */
private[this] val conn = connect(path.toCString);
+
+ /* Alternative constructors. */
+ def this(file: File) { this(file.getPath); }
+
+ /* Cleanup.*/
override def close() { closeconn(conn, CF_CLOSEMASK); }
override protected def finalize() { super.finalize(); close(); }
- class InputStream private[Connection] extends java.io.InputStream {
+ class Input private[Connection] extends InputStream {
+ /* An input stream which reads from the connection. */
+
override def read(): Int = {
val buf = new Array[Byte](1);
val n = read(buf, 0, 1);
recv(conn, buf, start, len);
override def close() { closeconn(conn, CF_CLOSERD); }
}
- lazy val input = new InputStream;
+ lazy val input = new Input;
+
+ class Output private[Connection] extends OutputStream {
+ /* An output stream which writes to the connection. */
- class OutputStream private[Connection] extends java.io.OutputStream {
override def write(b: Int) { write(Array[Byte](b.toByte), 0, 1); }
override def write(buf: Array[Byte]) { write(buf, 0, buf.length); }
override def write(buf: Array[Byte], start: Int, len: Int)
{ send(conn, buf, start, len); }
override def close() { closeconn(conn, CF_CLOSEWR); }
}
- lazy val output = new OutputStream;
+ lazy val output = new Output;
}
/*----- Crypto-library hacks ----------------------------------------------*/
--- /dev/null
+/* -*-scala-*-
+ *
+ * Extract data from `tar' archives
+ *
+ * (c) 2018 Straylight/Edgeware
+ */
+
+/*----- Licensing notice --------------------------------------------------*
+ *
+ * This file is part of the Trivial IP Encryption (TrIPE) Android app.
+ *
+ * TrIPE is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * TrIPE is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with TrIPE. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package uk.org.distorted.tripe;
+
+/*----- Imports -----------------------------------------------------------*/
+
+import java.io.{Closeable, InputStream};
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Date;
+
+/*----- Main code ---------------------------------------------------------*/
+
+class TarFormatError(msg: String) extends Exception(msg);
+
+trait TarEntry {
+ /* Honestly, I'd rather just have `TarFile#Entry', but Scala doesn't permit
+ * the trait inheritance circularity. So this is a cardboard cutout
+ * version of `Entry'.
+ */
+
+ /* Basic facts about the entry. */
+ def name: String;
+ def size: Long;
+ def typ: Char;
+ def mode: Int;
+ def mtime: Date;
+ def uid: Int;
+ def gid: Int;
+ def link: String;
+
+ /* Type predicates (intentionally like `FileInfo'). */
+ def isfifo: Boolean = typ == '6';
+ def ischr: Boolean = typ == '3';
+ def isdir: Boolean = typ == '5';
+ def isblk: Boolean = typ == '4';
+ def isreg: Boolean = typ match {
+ case 0 | '0' | '7' => true
+ case _ => false
+ }
+ def islnk: Boolean = typ == '2';
+ def issock: Boolean = false;
+ def ishardlink: Boolean = typ == '1';
+
+ def verbose: String = {
+ /* Encode information about this tar header as a string. */
+
+ val sb = new StringBuilder;
+
+ /* First, the type code. */
+ sb += (typ match {
+ case 0 | '0' | '7' => '-'
+ case '1' => 'L'
+ case '2' => 'l'
+ case '3' => 'c'
+ case '4' => 'b'
+ case '5' => 'd'
+ case '6' => '|'
+ case _ => '?'
+ })
+
+ /* Then the permissions bits. Ugh, the permissions bits. */
+ def perm(s: Int, r: Int, w: Int, x: Int, schar: Char, Schar: Char) {
+ sb += (if ((mode&r) != 0) 'r' else '-');
+ sb += (if ((mode&w) != 0) 'w' else '-');
+ sb += (if ((mode&s) != 0)
+ if ((mode&x) != 0) schar else Schar;
+ else
+ if ((mode&x) != 0) 'x' else '-');
+ }
+ perm(0x800, 0x100, 0x080, 0x040, 's', 'S');
+ perm(0x400, 0x020, 0x010, 0x008, 's', 'S');
+ perm(0x200, 0x004, 0x002, 0x001, 't', 'T');
+
+ /* And the rest, which is easy. */
+ sb ++= f" $uid%8d $gid%8d $size%12d $mtime%tFT%<tT%<tz $name%s";
+
+ /* Done. */
+ sb.result
+ }
+
+ override def toString(): String = s"${getClass.getName}($verbose)";
+
+ def stream: InputStream;
+ def withStream[T](body: InputStream => T): T = {
+ val s = stream;
+ try { body(s) }
+ finally { s.close(); }
+ }
+}
+
+class TarFile(in: InputStream)
+ extends LookaheadIterator[TarEntry] with Closeable { tar =>
+
+ /* Tokens are just objects, meaningful only for their identity. */
+ private[TarFile] class Token;
+
+ /* Some useful state. */
+ private[TarFile] var offset: Long = 0; // current byte offset
+ private[this] var lockp = false; // locked by open entry?
+ private[this] var locktok = new Token; // active lock token
+ private[this] var nexthdr: Long = 0; // byte offset of next header
+ private[this] val hdr = new Array[Byte](512); // header under consideration
+
+ /* Making sure we clean up properly. */
+ override def close() { in.close(); }
+ override protected def finalize() { super.finalize(); close(); }
+
+ private[this] def eoferr()
+ { throw new TarFormatError(s"unexpected EOF (at $offset)"); }
+
+ /* Locking machinery.
+ *
+ * We work from a primitive `InputStream' which we can't seek. From this,
+ * we must be able to extract file contents, as an `InputStream', and parse
+ * file headers. We'll be badly lost if we lose track of where we are in
+ * the archive.
+ *
+ * So, there's a lock, which can be held by at most one actor at a time:
+ * either the `TarFile' itself, while it's (hopefully) reading a header
+ * block, or by the `Stream' object which lets the caller read an
+ * individual entry's content. Furthermore, if we start activating the
+ * per-entry streams out of order, we'll get confused about where we're
+ * meant to be, so there's also a `token' which represents a participant's
+ * right to claim the lock. The `TarFile' itself has special privileges
+ * and doesn't need a token, but the per-entry streams do, and only the
+ * stream associated with the most recently-read header is allowed to claim
+ * the lock.
+ */
+
+ private[this] def lock() {
+ /* Claim exclusive use of the input stream. */
+
+ if (lockp) throw new IllegalArgumentException("tarfile lock still held");
+ lockp = true;
+ }
+
+ private[TarFile] def lock(tok: Token) {
+ /* Claim exclusive use of the input stream, passing a token. */
+
+ if (tok ne locktok)
+ throw new IllegalArgumentException("stale lock token");
+ lock();
+ }
+
+ private[TarFile] def unlock() {
+ /* Release the input stream so someone else can have a go. */
+
+ assert(lockp);
+ lockp = false;
+ locktok = new Token;
+ }
+
+ /* Doing I/O on the input stream.
+ *
+ * Our `Stream' object sneakily grabs single bytes from the input. Given
+ * the way Scala works, we can't prevent that, so roll with it.
+ */
+
+ private[TarFile] def read(buf: Array[Byte], start: Int, len: Int) {
+ /* Read input data into the indicated region of the buffer. Short reads
+ * are diagnosed as errors. Advances the cursor.
+ */
+
+ var pos = start;
+ val limit = start + len;
+ while (pos < len) {
+ val n = in.read(buf, pos, limit - pos);
+ if (n < 0) eoferr();
+ pos += n; offset += n;
+ }
+ }
+
+ private[TarFile] def skip(len: Long) {
+ /* Skip ahead LEN bytes in the archive. (The int/long discrepancy
+ * matches Java's bizarre `InputStream' protocol.)
+ */
+
+ var remain = len;
+ while (remain > 0) {
+ val n = in.skip(remain);
+
+ if (n > 0) { remain -= n; offset += n; }
+ else {
+ /* It's hard to work out whether this means we've hit EOF or not. It
+ * seems best to check. We must have at least one byte left to skip
+ * or we wouldn't have started this iteration, so try to read that.
+ * If that works, then there's more stuff available and skipping
+ * isn't working, so start to read buffers and discard them.
+ */
+
+ if (in.read() == -1) eoferr();
+ remain -= 1; offset += 1;
+
+ /* Ugh. So, buffers it is then. */
+ val buf = new Array[Byte]((remain min 4096).toInt);
+ while (remain >= buf.length) {
+ val n = (remain min buf.length).toInt;
+ read(buf, 0, n);
+ remain -= n;
+ }
+ }
+ }
+ }
+
+ private[TarFile] class Stream(end: Long, tok: Token) extends InputStream {
+ /* An input stream for a single archive entry's content. */
+
+ /* Claim the lock. If we're stale, this won't work. */
+ lock(tok);
+ private[this] var open = true;
+
+ private[this] def checkopen() {
+ /* Complain if the stream is closed. */
+
+ if (!lockp) throw new IllegalArgumentException("stream is closed");
+ }
+
+ override def read(): Int = {
+ /* Read one byte. Don't know why there isn't a default implementation
+ * of this.
+ */
+
+ checkopen();
+ if (offset >= end) -1
+ else {
+ val b = in.read();
+ if (b == -1) eoferr();
+ offset += 1;
+ b
+ }
+ }
+
+ override def read(buf: Array[Byte], start: Int, len: Int): Int = {
+ /* Read a block. */
+
+ checkopen();
+ if (offset >= end) -1
+ else {
+ var n = (len.toLong min (end - offset)).toInt;
+ tar.read(buf, start, n);
+ n
+ }
+ }
+
+ override def close() {
+ /* Release the lock. */
+
+ if (open) { unlock(); open = false; }
+ }
+ }
+
+ private[this] class Entry(val name: String, val size: Long,
+ val typ: Char, val mode: Int,
+ val mtime: Date,
+ val uid: Int, val gid: Int,
+ val link: String,
+ end: Long, tok: Token)
+ extends TarEntry{
+ /* See `TarEntry' for why we have this silliness. Most of the work is in
+ * the constructor above.
+ */
+
+ lazy val stream: InputStream = new Stream(end, tok);
+ }
+
+ /* Utilities for parsing archive-entry header blocks. */
+
+ private[this] def string(off: Int, len: Int): String = {
+ /* Parse a string from the block header. POSIX.1-2008 says that header
+ * fields should be ISO/IEC 646, but strange things can turn up
+ * especially in filenames. I'm going to translate strings according to
+ * the local character set, because that will map most easily if a
+ * program tries to write out files from the archive with their
+ * associated names.
+ */
+
+ /* First, find the null terminator, if there is one. Scala doesn't make
+ * this especially easy. Rustle up a view to limit the search.
+ */
+ val bview = hdr.view(off, off + len);
+ val n = bview.indexOf(0) match {
+ case -1 => len
+ case nul => nul
+ };
+
+ /* And then decode the relevant portion of the orignal buffer. */
+ val dec = Charset.defaultCharset.newDecoder;
+ val in = ByteBuffer.wrap(hdr, off, n);
+ dec.decode(in).toString
+ }
+
+ private[this] def number(off: Int, len: Int, max: Long): Long = {
+ /* Parse a number from the block header. POSIX.1-2008 says that numbers
+ * are in octal and terminated by space or nul.
+ */
+
+ var n = 0l; // accumulate the value
+ for (i <- off until off + len) {
+ val b = hdr(i);
+
+ /* See if we're done now. */
+ if (b == ' ' || b == 0) return n;
+ else if (b < '0' || b > '7')
+ throw new TarFormatError(s"bad octal digit (at ${offset + off + i})");
+
+ /* Convert to a digit. */
+ val m = b - '0';
+
+ /* Check for overflow -- without overflowing.
+ *
+ * Write max 8 N + M. We overflow if 8 n + m > 8 N + M, i.e., 8 n >
+ * 8 N + (M - m), so n > N + (M - m)/8. This last calculation is a
+ * little fraught because Scala has the wrong semantics when dividing
+ * negative integers.
+ */
+ if (n > max/8 + (8 + max%8 - m)/8 - 1)
+ throw new TarFormatError(s"number out of range (at ${offset + off})");
+
+ /* Accumulate and go round again. */
+ n = 8*n + (b - '0');
+ }
+ unreachable;
+ }
+
+ override protected def fetch(): Option[TarEntry] = {
+ /* Collect the next archive header and return it as a file entry. */
+
+ /* Make sure that we can actually do this. */
+ withCleaner { clean =>
+ lock(); clean { unlock(); }
+
+ /* Skip ahead to the next header. */
+ skip(nexthdr - offset);
+
+ /* Read the header. The end of the archive is marked by two zero
+ * blocks, so the archive is broken if there isn't at least one here.
+ */
+ read(hdr, 0, 512);
+ }
+
+ /* If the block is entirely zero-filled then declare this file at an
+ * end. No good can come from checking the next block.
+ */
+ if (hdr.forall(_ == 0)) return None;
+
+ /* Verify the checksum. Pay attention because Java's bytes are
+ * (idiotically) signed.
+ */
+ var ck: Int = 8*' '; // pretend chksum field is spaces
+ for (i <- 0 until 148) ck += hdr(i)&0xff;
+ for (i <- 156 until 512) ck += hdr(i)&0xff;
+ val wantck = number(148, 8, 0x20000);
+ if (ck != wantck) {
+ throw new TarFormatError(
+ s"invalid checksum $ck /= $wantck (at $nexthdr)");
+ }
+
+ /* Fetch the `magic' and `version' fields. If this is a proper POSIX
+ * `ustar' file then special processing will apply.
+ */
+ val magic = string(257, 6);
+ val version = string(263, 2);
+ val posixp = magic == "ustar" && version == "00";
+
+ /* Figure out this entry's name. If this is a POSIX archive, then part
+ * of the name is stashed at the end of the header because of old, bad
+ * decisions. But don't look there unless we're sure because old GNU
+ * `tar' used that space for other things.
+ */
+ val name = {
+ val tail = string(0, 100);
+ if (!posixp || hdr(345) == 0) tail
+ else {
+ val prefix = string(345, 155);
+ prefix + '/' + tail
+ }
+ }
+
+ /* Read some other easy stuff. */
+ val mode = number(100, 8, 0xfff).toInt;
+ val uid = number(108, 8, Int.MaxValue).toInt;
+ val gid = number(116, 8, Int.MaxValue).toInt;
+ val typ = hdr(156).toChar;
+ val mtime = number(136, 12, Long.MaxValue);
+
+ /* The size is irrelevant, and possibly even misleading, for some entry
+ * types. We're not interested, for example, in systems where
+ * directories need to be preallocated.
+ */
+ val size = typ match {
+ case '1' | '2' | '3' | '4' | '5' | '6' => 0
+ case _ => number(124, 12, Long.MaxValue)
+ }
+
+ /* Maybe fetch the link name. */
+ val link = typ match {
+ case '1' | '2' => string(157, 100)
+ case _ => ""
+ }
+
+ /* Figure out where the next header ought to be. */
+ nexthdr = (offset + size + 511)& -512;
+
+ /* Return the finished archive entry. */
+ Some(new Entry(name, size, typ, mode,
+ new Date(1000*mtime), uid, gid, link,
+ offset + size, locktok));
+ }
+}
+
+/* Example:
+ *
+ * for (e <- TarFile(new GZIPInputStream(tarball.open())); if e.isreg)
+ * e withStream { in =>
+ * val h = java.security.MessageDigest.getInstance("SHA-256");
+ * for ((buf, n) <- in.blocks) h.update(b, 0, n);
+ * val hex = new String(h.digest flatMap { _.formatted("%02x") });
+ * println("s$hex ${e.name}");
+ * }
+ */
+
+/*----- That's all, folks -------------------------------------------------*/
/*----- Imports -----------------------------------------------------------*/
import scala.concurrent.duration.{Deadline, Duration};
-import scala.util.control.Breaks;
+import scala.util.control.{Breaks, ControlThrowable};
-import java.io.{BufferedReader, Closeable, File, Reader};
+import java.io.{BufferedReader, Closeable, File, InputStream, Reader};
import java.net.{URL, URLConnection};
import java.nio.{ByteBuffer, CharBuffer};
import java.nio.charset.Charset;
val rng = new java.security.SecureRandom;
def unreachable(msg: String): Nothing = throw new AssertionError(msg);
+def unreachable(): Nothing = unreachable("unreachable");
+final val ok = ();
+final class Brand;
/*----- Various pieces of implicit magic ----------------------------------*/
try { body(thing) }
finally { thing.close(); }
+/*----- Control structures ------------------------------------------------*/
+
+private case class ExitBlock[T](brand: Brand, result: T)
+ extends ControlThrowable;
+
+def block[T](body: (T => Nothing) => T): T = {
+ /* block { exit[T] => ...; exit(x); ... }
+ *
+ * Execute the body until it calls the `exit' function or finishes.
+ * Annoyingly, Scala isn't clever enough to infer the return type, so
+ * you'll have to write it explicitly.
+ */
+
+ val mybrand = new Brand;
+ try { body { result => throw new ExitBlock(mybrand, result) } }
+ catch {
+ case ExitBlock(brand, result) if brand eq mybrand =>
+ result.asInstanceOf[T]
+ }
+}
+
+def blockUnit(body: (=> Nothing) => Unit) {
+ /* blockUnit { exit => ...; exit; ... }
+ *
+ * Like `block'; it just saves you having to write `exit[Unit] => ...;
+ * exit(ok); ...'.
+ */
+
+ val mybrand = new Brand;
+ try { body { throw new ExitBlock(mybrand, null) }; }
+ catch { case ExitBlock(brand, result) if brand eq mybrand => ok; }
+}
+
+def loop[T](body: (T => Nothing) => Unit): T = {
+ /* loop { exit[T] => ...; exit(x); ... }
+ *
+ * Repeatedly execute the body until it calls the `exit' function.
+ * Annoyingly, Scala isn't clever enough to infer the return type, so
+ * you'll have to write it explicitly.
+ */
+
+ block { exit => while (true) body(exit); unreachable }
+}
+
+def loopUnit(body: (=> Nothing) => Unit): Unit = {
+ /* loopUnit { exit => ...; exit; ... }
+ *
+ * Like `loop'; it just saves you having to write `exit[Unit] => ...;
+ * exit(()); ...'.
+ */
+
+ blockUnit { exit => while (true) body(exit); }
+}
+
+val BREAKS = new Breaks;
+import BREAKS.{breakable, break};
+
/*----- A gadget for fetching URLs ----------------------------------------*/
class URLFetchException(msg: String) extends Exception(msg);
trait URLFetchCallbacks {
def preflight(conn: URLConnection) { }
- def write(buf: Array[Byte], n: Int, len: Int): Unit;
+ def write(buf: Array[Byte], n: Int, len: Long): Unit;
def done(win: Boolean) { }
}
/* Start fetching data. */
val in = c.getInputStream; clean { in.close(); }
- val explen = c.getContentLength();
+ val explen = c.getContentLength;
/* Read a buffer at a time, and give it to the callback. Maintain a
* running total.
*/
- val buf = new Array[Byte](4096);
- var n = 0;
- var len = 0;
- while ({n = in.read(buf); n >= 0 && (explen == -1 || len <= explen)}) {
- cb.write(buf, n, len);
- len += n;
+ var len: Long = 0;
+ blockUnit { exit =>
+ for ((buf, n) <- blocks(in)) {
+ cb.write(buf, n, len);
+ len += n;
+ if (explen != -1 && len > explen) exit;
+ }
}
/* I can't find it documented anywhere that the existing machinery
}
}
-/*----- Running processes -------------------------------------------------*/
-
-//def runProgram(
-
/*----- Threading things --------------------------------------------------*/
def thread[T](name: String, run: Boolean = true, daemon: Boolean = true)
val b = List.newBuilder[String];
var i = pos;
- while (nextToken(s, i) match {
- case Some((w, j)) => b += w; i = j; true
- case None => false
- }) ();
+ loopUnit { exit => nextToken(s, i) match {
+ case Some((w, j)) => b += w; i = j;
+ case None => exit;
+ } }
b.result
}
+/*----- Other random things -----------------------------------------------*/
+
trait LookaheadIterator[T] extends BufferedIterator[T] {
- private[this] var st: Option[T] = None;
+ /* An iterator in terms of a single `maybe there's another item' function.
+ *
+ * It seems like every time I write an iterator in Scala, the only way to
+ * find out whether there's a next item, for `hasNext', is to actually try
+ * to fetch it. So here's an iterator in terms of a function which goes
+ * off and maybe returns a next thing. It turns out to be easy to satisfy
+ * the additional requirements for `BufferedIterator', so why not?
+ */
+
+ /* Subclass responsibility. */
protected def fetch(): Option[T];
+
+ /* The machinery. `st' is `None' if there's no current item, null if we've
+ * actually hit the end, or `Some(x)' if the current item is x.
+ */
+ private[this] var st: Option[T] = None;
private[this] def peek() {
+ /* Arrange to have a current item. */
if (st == None) fetch() match {
case None => st = null;
case x@Some(_) => st = x;
}
}
+
+ /* The `BufferedIterator' protocol. */
override def hasNext: Boolean = { peek(); st != null }
- override def head(): T =
+ override def head: T =
{ peek(); if (st == null) throw new NoSuchElementException; st.get }
- override def next(): T = { val it = head(); st = None; it }
+ override def next(): T = { val it = head; st = None; it }
}
-def lines(r: Reader) = new LookaheadIterator[String] {
- /* Iterates over the lines of text in a `Reader' object. */
+def bufferedReader(r: Reader): BufferedReader = r match {
+ case br: BufferedReader => br
+ case _ => new BufferedReader(r)
+}
- private[this] val in = r match {
- case br: BufferedReader => br;
- case _ => new BufferedReader(r);
+def lines(r: BufferedReader): BufferedIterator[String] =
+ new LookaheadIterator[String] {
+ /* Iterates over the lines of text in a `Reader' object. */
+ override protected def fetch() = Option(r.readLine());
+ }
+def lines(r: Reader): BufferedIterator[String] = lines(bufferedReader(r));
+
+def blocks(in: InputStream, blksz: Int):
+ BufferedIterator[(Array[Byte], Int)] =
+ /* Iterates over (possibly irregularly sized) blocks in a stream. */
+ new LookaheadIterator[(Array[Byte], Int)] {
+ val buf = new Array[Byte](blksz)
+ override protected def fetch() = {
+ val n = in.read(buf);
+ if (n < 0) None
+ else Some((buf, n))
+ }
+ }
+def blocks(in: InputStream):
+ BufferedIterator[(Array[Byte], Int)] = blocks(in, 4096);
+
+def blocks(in: BufferedReader, blksz: Int):
+ BufferedIterator[(Array[Char], Int)] =
+ /* Iterates over (possibly irregularly sized) blocks in a reader. */
+ new LookaheadIterator[(Array[Char], Int)] {
+ val buf = new Array[Char](blksz)
+ override protected def fetch() = {
+ val n = in.read(buf);
+ if (n < 0) None
+ else Some((buf, n))
+ }
}
- protected override def fetch(): Option[String] = Option(in.readLine);
+def blocks(in: BufferedReader):
+ BufferedIterator[(Array[Char], Int)] = blocks(in, 4096);
+def blocks(r: Reader, blksz: Int): BufferedIterator[(Array[Char], Int)] =
+ blocks(bufferedReader(r), blksz);
+def blocks(r: Reader): BufferedIterator[(Array[Char], Int)] =
+ blocks(bufferedReader(r));
+
+def oxford(conj: String, things: Seq[String]): String = things match {
+ case Seq() => "<nothing>"
+ case Seq(a) => a
+ case Seq(a, b) => s"$a $conj $b"
+ case Seq(a, tail@_*) =>
+ val sb = new StringBuilder;
+ sb ++= a; sb ++= ", ";
+ def iter(rest: Seq[String]) {
+ rest match {
+ case Seq() => unreachable;
+ case Seq(a) => sb ++= conj; sb += ' '; sb ++= a;
+ case Seq(a, tail@_*) => sb ++= a; sb ++= ", "; iter(tail);
+ }
+ }
+ iter(tail);
+ sb.result
}
/*----- That's all, folks -------------------------------------------------*/
~mdw / tripe-android / commitdiff