Read several objects at once with git cat-file --batch

Instead of spawning a separate cat-file process for every blob and
commit we want to read. This speeds things up slightly: about 6-8%
when uncommitting and rebasing 1470 linux-kernel patches (perftest.py
rebase-newrebase-add-file-linux).

Signed-off-by: Karl Wiberg <kha@treskal.com>

diff --git a/INSTALL b/INSTALL
index 3da4efd..d728441 100644 (file)
--- a/INSTALL
+++ b/INSTALL
@@ -12,7 +12,9 @@ prefix option.
 
 Issues of note:
 
+- StGit requires git version 1.5.6 or later. (Specifically, it needs a
+  git that includes commit a8128ed6: "git-cat-file: Add --batch option".)
+
 - To build and install the documentation, you need to have the
   asciidoc/xmlto toolchain.  The default build target ("make all")
   does _not_ build them.
-

diff --git a/stgit/lib/git.py b/stgit/lib/git.py
index 899c1a2..d60e1d2 100644 (file)
--- a/stgit/lib/git.py
+++ b/stgit/lib/git.py
@@ -1,7 +1,7 @@
 """A Python class hierarchy wrapping a git repository and its
 contents."""
 
-import os, os.path, re
+import atexit, os, os.path, re, signal
 from datetime import datetime, timedelta, tzinfo
 
 from stgit import exception, run, utils
@@ -520,6 +520,43 @@ class RunWithEnvCwd(RunWithEnv):
         @param args: Command and argument vector"""
         return RunWithEnv.run(self, args, self.env_in_cwd)
 
+class CatFileProcess(object):
+    def __init__(self, repo):
+        self.__repo = repo
+        self.__proc = None
+        atexit.register(self.__shutdown)
+    def __get_process(self):
+        if not self.__proc:
+            self.__proc = self.__repo.run(['git', 'cat-file', '--batch']
+                                          ).run_background()
+        return self.__proc
+    def __shutdown(self):
+        p = self.__proc
+        if p:
+            os.kill(p.pid(), signal.SIGTERM)
+            p.wait()
+    def cat_file(self, sha1):
+        p = self.__get_process()
+        p.stdin.write('%s\n' % sha1)
+        p.stdin.flush()
+
+        # Read until we have the entire status line.
+        s = ''
+        while not '\n' in s:
+            s += os.read(p.stdout.fileno(), 4096)
+        h, b = s.split('\n', 1)
+        if h == '%s missing' % sha1:
+            raise SomeException()
+        hash, type, length = h.split()
+        assert hash == sha1
+        length = int(length)
+
+        # Read until we have the entire object plus the trailing
+        # newline.
+        while len(b) < length + 1:
+            b += os.read(p.stdout.fileno(), 4096)
+        return type, b[:-1]
+
 class Repository(RunWithEnv):
     """Represents a git repository."""
     def __init__(self, directory):
@@ -531,6 +568,7 @@ class Repository(RunWithEnv):
         self.__default_index = None
         self.__default_worktree = None
         self.__default_iw = None
+        self.__catfile = CatFileProcess(self)
     env = property(lambda self: { 'GIT_DIR': self.__git_dir })
     @classmethod
     def default(cls):
@@ -580,7 +618,7 @@ class Repository(RunWithEnv):
     directory = property(lambda self: self.__git_dir)
     refs = property(lambda self: self.__refs)
     def cat_object(self, sha1):
-        return self.run(['git', 'cat-file', '-p', sha1]).raw_output()
+        return self.__catfile.cat_file(sha1)[1]
     def rev_parse(self, rev, discard_stderr = False, object_type = 'commit'):
         assert object_type in ('commit', 'tree', 'blob')
         getter = getattr(self, 'get_' + object_type)

diff --git a/stgit/run.py b/stgit/run.py
index 2d8ed34..b36b6f4 100644 (file)
--- a/stgit/run.py
+++ b/stgit/run.py
@@ -134,6 +134,21 @@ class Run:
             raise self.exc('%s failed: %s' % (self.__cmd[0], e))
         self.__log_end(self.exitcode)
         self.__check_exitcode()
+    def __run_background(self):
+        """Run in background."""
+        assert self.__indata == None
+        try:
+            p = subprocess.Popen(self.__cmd, env = self.__env, cwd = self.__cwd,
+                                 stdin = subprocess.PIPE,
+                                 stdout = subprocess.PIPE,
+                                 stderr = subprocess.PIPE)
+        except OSError, e:
+            raise self.exc('%s failed: %s' % (self.__cmd[0], e))
+        self.stdin = p.stdin
+        self.stdout = p.stdout
+        self.stderr = p.stderr
+        self.wait = p.wait
+        self.pid = lambda: p.pid
     def returns(self, retvals):
         self.__good_retvals = retvals
         return self
@@ -185,6 +200,10 @@ class Run:
     def run(self):
         """Just run, with no IO redirection."""
         self.__run_noio()
+    def run_background(self):
+        """Run as a background process."""
+        self.__run_background()
+        return self
     def xargs(self, xargs):
         """Just run, with no IO redirection. The extra arguments are
         appended to the command line a few at a time; the command is