Print progress message to stderr, not stdout
[stgit] / stgit / git.py
1 """Python GIT interface
2 """
3
4 __copyright__ = """
5 Copyright (C) 2005, Catalin Marinas <catalin.marinas@gmail.com>
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License version 2 as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 """
20
21 import sys, os, popen2, re, gitmergeonefile
22
23 from stgit import basedir
24 from stgit.utils import *
25 from stgit.config import config
26
27 # git exception class
28 class GitException(Exception):
29 pass
30
31
32
33 #
34 # Classes
35 #
36 class Commit:
37 """Handle the commit objects
38 """
39 def __init__(self, id_hash):
40 self.__id_hash = id_hash
41
42 lines = _output_lines('git-cat-file commit %s' % id_hash)
43 for i in range(len(lines)):
44 line = lines[i]
45 if line == '\n':
46 break
47 field = line.strip().split(' ', 1)
48 if field[0] == 'tree':
49 self.__tree = field[1]
50 if field[0] == 'author':
51 self.__author = field[1]
52 if field[0] == 'committer':
53 self.__committer = field[1]
54 self.__log = ''.join(lines[i+1:])
55
56 def get_id_hash(self):
57 return self.__id_hash
58
59 def get_tree(self):
60 return self.__tree
61
62 def get_parent(self):
63 parents = self.get_parents()
64 if parents:
65 return parents[0]
66 else:
67 return None
68
69 def get_parents(self):
70 return _output_lines('git-rev-list --parents --max-count=1 %s'
71 % self.__id_hash)[0].split()[1:]
72
73 def get_author(self):
74 return self.__author
75
76 def get_committer(self):
77 return self.__committer
78
79 def get_log(self):
80 return self.__log
81
82 def __str__(self):
83 return self.get_id_hash()
84
85 # dictionary of Commit objects, used to avoid multiple calls to git
86 __commits = dict()
87
88 #
89 # Functions
90 #
91
92 def get_commit(id_hash):
93 """Commit objects factory. Save/look-up them in the __commits
94 dictionary
95 """
96 global __commits
97
98 if id_hash in __commits:
99 return __commits[id_hash]
100 else:
101 commit = Commit(id_hash)
102 __commits[id_hash] = commit
103 return commit
104
105 def get_conflicts():
106 """Return the list of file conflicts
107 """
108 conflicts_file = os.path.join(basedir.get(), 'conflicts')
109 if os.path.isfile(conflicts_file):
110 f = file(conflicts_file)
111 names = [line.strip() for line in f.readlines()]
112 f.close()
113 return names
114 else:
115 return None
116
117 def _input(cmd, file_desc):
118 p = popen2.Popen3(cmd, True)
119 while True:
120 line = file_desc.readline()
121 if not line:
122 break
123 p.tochild.write(line)
124 p.tochild.close()
125 if p.wait():
126 raise GitException, '%s failed (%s)' % (str(cmd),
127 p.childerr.read().strip())
128
129 def _input_str(cmd, string):
130 p = popen2.Popen3(cmd, True)
131 p.tochild.write(string)
132 p.tochild.close()
133 if p.wait():
134 raise GitException, '%s failed (%s)' % (str(cmd),
135 p.childerr.read().strip())
136
137 def _output(cmd):
138 p=popen2.Popen3(cmd, True)
139 output = p.fromchild.read()
140 if p.wait():
141 raise GitException, '%s failed (%s)' % (str(cmd),
142 p.childerr.read().strip())
143 return output
144
145 def _output_one_line(cmd, file_desc = None):
146 p=popen2.Popen3(cmd, True)
147 if file_desc != None:
148 for line in file_desc:
149 p.tochild.write(line)
150 p.tochild.close()
151 output = p.fromchild.readline().strip()
152 if p.wait():
153 raise GitException, '%s failed (%s)' % (str(cmd),
154 p.childerr.read().strip())
155 return output
156
157 def _output_lines(cmd):
158 p=popen2.Popen3(cmd, True)
159 lines = p.fromchild.readlines()
160 if p.wait():
161 raise GitException, '%s failed (%s)' % (str(cmd),
162 p.childerr.read().strip())
163 return lines
164
165 def __run(cmd, args=None):
166 """__run: runs cmd using spawnvp.
167
168 Runs cmd using spawnvp. The shell is avoided so it won't mess up
169 our arguments. If args is very large, the command is run multiple
170 times; args is split xargs style: cmd is passed on each
171 invocation. Unlike xargs, returns immediately if any non-zero
172 return code is received.
173 """
174
175 args_l=cmd.split()
176 if args is None:
177 args = []
178 for i in range(0, len(args)+1, 100):
179 r=os.spawnvp(os.P_WAIT, args_l[0], args_l + args[i:min(i+100, len(args))])
180 if r:
181 return r
182 return 0
183
184 def __tree_status(files = None, tree_id = 'HEAD', unknown = False,
185 noexclude = True, verbose = False):
186 """Returns a list of pairs - [status, filename]
187 """
188 if verbose:
189 print >> sys.stderr, \
190 'Checking for changes in the working directory...',
191 sys.stderr.flush()
192
193 refresh_index()
194
195 if not files:
196 files = []
197 cache_files = []
198
199 # unknown files
200 if unknown:
201 exclude_file = os.path.join(basedir.get(), 'info', 'exclude')
202 base_exclude = ['--exclude=%s' % s for s in
203 ['*.[ao]', '*.pyc', '.*', '*~', '#*', 'TAGS', 'tags']]
204 base_exclude.append('--exclude-per-directory=.gitignore')
205
206 if os.path.exists(exclude_file):
207 extra_exclude = ['--exclude-from=%s' % exclude_file]
208 else:
209 extra_exclude = []
210 if noexclude:
211 extra_exclude = base_exclude = []
212
213 lines = _output_lines(['git-ls-files', '--others', '--directory']
214 + base_exclude + extra_exclude)
215 cache_files += [('?', line.strip()) for line in lines]
216
217 # conflicted files
218 conflicts = get_conflicts()
219 if not conflicts:
220 conflicts = []
221 cache_files += [('C', filename) for filename in conflicts]
222
223 # the rest
224 for line in _output_lines(['git-diff-index', tree_id, '--'] + files):
225 fs = tuple(line.rstrip().split(' ',4)[-1].split('\t',1))
226 if fs[1] not in conflicts:
227 cache_files.append(fs)
228
229 if verbose:
230 print >> sys.stderr, 'done'
231
232 return cache_files
233
234 def local_changes():
235 """Return true if there are local changes in the tree
236 """
237 return len(__tree_status(verbose = True)) != 0
238
239 # HEAD value cached
240 __head = None
241
242 def get_head():
243 """Verifies the HEAD and returns the SHA1 id that represents it
244 """
245 global __head
246
247 if not __head:
248 __head = rev_parse('HEAD')
249 return __head
250
251 def get_head_file():
252 """Returns the name of the file pointed to by the HEAD link
253 """
254 return strip_prefix('refs/heads/',
255 _output_one_line('git-symbolic-ref HEAD'))
256
257 def set_head_file(ref):
258 """Resets HEAD to point to a new ref
259 """
260 # head cache flushing is needed since we might have a different value
261 # in the new head
262 __clear_head_cache()
263 if __run('git-symbolic-ref HEAD',
264 [os.path.join('refs', 'heads', ref)]) != 0:
265 raise GitException, 'Could not set head to "%s"' % ref
266
267 def __set_head(val):
268 """Sets the HEAD value
269 """
270 global __head
271
272 if not __head or __head != val:
273 if __run('git-update-ref HEAD', [val]) != 0:
274 raise GitException, 'Could not update HEAD to "%s".' % val
275 __head = val
276
277 # only allow SHA1 hashes
278 assert(len(__head) == 40)
279
280 def __clear_head_cache():
281 """Sets the __head to None so that a re-read is forced
282 """
283 global __head
284
285 __head = None
286
287 def refresh_index():
288 """Refresh index with stat() information from the working directory.
289 """
290 __run('git-update-index -q --unmerged --refresh')
291
292 def rev_parse(git_id):
293 """Parse the string and return a verified SHA1 id
294 """
295 try:
296 return _output_one_line(['git-rev-parse', '--verify', git_id])
297 except GitException:
298 raise GitException, 'Unknown revision: %s' % git_id
299
300 def branch_exists(branch):
301 """Existence check for the named branch
302 """
303 branch = os.path.join('refs', 'heads', branch)
304 for line in _output_lines('git-rev-parse --symbolic --all 2>&1'):
305 if line.strip() == branch:
306 return True
307 if re.compile('[ |/]'+branch+' ').search(line):
308 raise GitException, 'Bogus branch: %s' % line
309 return False
310
311 def create_branch(new_branch, tree_id = None):
312 """Create a new branch in the git repository
313 """
314 if branch_exists(new_branch):
315 raise GitException, 'Branch "%s" already exists' % new_branch
316
317 current_head = get_head()
318 set_head_file(new_branch)
319 __set_head(current_head)
320
321 # a checkout isn't needed if new branch points to the current head
322 if tree_id:
323 switch(tree_id)
324
325 if os.path.isfile(os.path.join(basedir.get(), 'MERGE_HEAD')):
326 os.remove(os.path.join(basedir.get(), 'MERGE_HEAD'))
327
328 def switch_branch(new_branch):
329 """Switch to a git branch
330 """
331 global __head
332
333 if not branch_exists(new_branch):
334 raise GitException, 'Branch "%s" does not exist' % new_branch
335
336 tree_id = rev_parse(os.path.join('refs', 'heads', new_branch)
337 + '^{commit}')
338 if tree_id != get_head():
339 refresh_index()
340 if __run('git-read-tree -u -m', [get_head(), tree_id]) != 0:
341 raise GitException, 'git-read-tree failed (local changes maybe?)'
342 __head = tree_id
343 set_head_file(new_branch)
344
345 if os.path.isfile(os.path.join(basedir.get(), 'MERGE_HEAD')):
346 os.remove(os.path.join(basedir.get(), 'MERGE_HEAD'))
347
348 def delete_branch(name):
349 """Delete a git branch
350 """
351 if not branch_exists(name):
352 raise GitException, 'Branch "%s" does not exist' % name
353 remove_file_and_dirs(os.path.join(basedir.get(), 'refs', 'heads'),
354 name)
355
356 def rename_branch(from_name, to_name):
357 """Rename a git branch
358 """
359 if not branch_exists(from_name):
360 raise GitException, 'Branch "%s" does not exist' % from_name
361 if branch_exists(to_name):
362 raise GitException, 'Branch "%s" already exists' % to_name
363
364 if get_head_file() == from_name:
365 set_head_file(to_name)
366 rename(os.path.join(basedir.get(), 'refs', 'heads'),
367 from_name, to_name)
368
369 def add(names):
370 """Add the files or recursively add the directory contents
371 """
372 # generate the file list
373 files = []
374 for i in names:
375 if not os.path.exists(i):
376 raise GitException, 'Unknown file or directory: %s' % i
377
378 if os.path.isdir(i):
379 # recursive search. We only add files
380 for root, dirs, local_files in os.walk(i):
381 for name in [os.path.join(root, f) for f in local_files]:
382 if os.path.isfile(name):
383 files.append(os.path.normpath(name))
384 elif os.path.isfile(i):
385 files.append(os.path.normpath(i))
386 else:
387 raise GitException, '%s is not a file or directory' % i
388
389 if files:
390 if __run('git-update-index --add --', files):
391 raise GitException, 'Unable to add file'
392
393 def rm(files, force = False):
394 """Remove a file from the repository
395 """
396 if not force:
397 for f in files:
398 if os.path.exists(f):
399 raise GitException, '%s exists. Remove it first' %f
400 if files:
401 __run('git-update-index --remove --', files)
402 else:
403 if files:
404 __run('git-update-index --force-remove --', files)
405
406 def update_cache(files = None, force = False):
407 """Update the cache information for the given files
408 """
409 if not files:
410 files = []
411
412 cache_files = __tree_status(files, verbose = False)
413
414 # everything is up-to-date
415 if len(cache_files) == 0:
416 return False
417
418 # check for unresolved conflicts
419 if not force and [x for x in cache_files
420 if x[0] not in ['M', 'N', 'A', 'D']]:
421 raise GitException, 'Updating cache failed: unresolved conflicts'
422
423 # update the cache
424 add_files = [x[1] for x in cache_files if x[0] in ['N', 'A']]
425 rm_files = [x[1] for x in cache_files if x[0] in ['D']]
426 m_files = [x[1] for x in cache_files if x[0] in ['M']]
427
428 if add_files and __run('git-update-index --add --', add_files) != 0:
429 raise GitException, 'Failed git-update-index --add'
430 if rm_files and __run('git-update-index --force-remove --', rm_files) != 0:
431 raise GitException, 'Failed git-update-index --rm'
432 if m_files and __run('git-update-index --', m_files) != 0:
433 raise GitException, 'Failed git-update-index'
434
435 return True
436
437 def commit(message, files = None, parents = None, allowempty = False,
438 cache_update = True, tree_id = None,
439 author_name = None, author_email = None, author_date = None,
440 committer_name = None, committer_email = None):
441 """Commit the current tree to repository
442 """
443 if not files:
444 files = []
445 if not parents:
446 parents = []
447
448 # Get the tree status
449 if cache_update and parents != []:
450 changes = update_cache(files)
451 if not changes and not allowempty:
452 raise GitException, 'No changes to commit'
453
454 # get the commit message
455 if not message:
456 message = '\n'
457 elif message[-1:] != '\n':
458 message += '\n'
459
460 must_switch = True
461 # write the index to repository
462 if tree_id == None:
463 tree_id = _output_one_line('git-write-tree')
464 else:
465 must_switch = False
466
467 # the commit
468 cmd = ''
469 if author_name:
470 cmd += 'GIT_AUTHOR_NAME="%s" ' % author_name
471 if author_email:
472 cmd += 'GIT_AUTHOR_EMAIL="%s" ' % author_email
473 if author_date:
474 cmd += 'GIT_AUTHOR_DATE="%s" ' % author_date
475 if committer_name:
476 cmd += 'GIT_COMMITTER_NAME="%s" ' % committer_name
477 if committer_email:
478 cmd += 'GIT_COMMITTER_EMAIL="%s" ' % committer_email
479 cmd += 'git-commit-tree %s' % tree_id
480
481 # get the parents
482 for p in parents:
483 cmd += ' -p %s' % p
484
485 commit_id = _output_one_line(cmd, message)
486 if must_switch:
487 __set_head(commit_id)
488
489 return commit_id
490
491 def apply_diff(rev1, rev2, check_index = True, files = None):
492 """Apply the diff between rev1 and rev2 onto the current
493 index. This function doesn't need to raise an exception since it
494 is only used for fast-pushing a patch. If this operation fails,
495 the pushing would fall back to the three-way merge.
496 """
497 if check_index:
498 index_opt = '--index'
499 else:
500 index_opt = ''
501
502 if not files:
503 files = []
504
505 diff_str = diff(files, rev1, rev2)
506 if diff_str:
507 try:
508 _input_str('git-apply %s' % index_opt, diff_str)
509 except GitException:
510 return False
511
512 return True
513
514 def merge(base, head1, head2):
515 """Perform a 3-way merge between base, head1 and head2 into the
516 local tree
517 """
518 refresh_index()
519
520 try:
521 # use _output() to mask the verbose prints of the tool
522 _output('git-merge-recursive %s -- %s %s' % (base, head1, head2))
523 except GitException:
524 pass
525
526 # check the index for unmerged entries
527 files = {}
528 stages_re = re.compile('^([0-7]+) ([0-9a-f]{40}) ([1-3])\t(.*)$', re.S)
529
530 for line in _output('git-ls-files --unmerged --stage -z').split('\0'):
531 if not line:
532 continue
533
534 mode, hash, stage, path = stages_re.findall(line)[0]
535
536 if not path in files:
537 files[path] = {}
538 files[path]['1'] = ('', '')
539 files[path]['2'] = ('', '')
540 files[path]['3'] = ('', '')
541
542 files[path][stage] = (mode, hash)
543
544 # merge the unmerged files
545 errors = False
546 for path in files:
547 stages = files[path]
548 if gitmergeonefile.merge(stages['1'][1], stages['2'][1],
549 stages['3'][1], path, stages['1'][0],
550 stages['2'][0], stages['3'][0]) != 0:
551 errors = True
552
553 if errors:
554 raise GitException, 'GIT index merging failed (possible conflicts)'
555
556 def status(files = None, modified = False, new = False, deleted = False,
557 conflict = False, unknown = False, noexclude = False):
558 """Show the tree status
559 """
560 if not files:
561 files = []
562
563 cache_files = __tree_status(files, unknown = True, noexclude = noexclude)
564 all = not (modified or new or deleted or conflict or unknown)
565
566 if not all:
567 filestat = []
568 if modified:
569 filestat.append('M')
570 if new:
571 filestat.append('A')
572 filestat.append('N')
573 if deleted:
574 filestat.append('D')
575 if conflict:
576 filestat.append('C')
577 if unknown:
578 filestat.append('?')
579 cache_files = [x for x in cache_files if x[0] in filestat]
580
581 for fs in cache_files:
582 if all:
583 print '%s %s' % (fs[0], fs[1])
584 else:
585 print '%s' % fs[1]
586
587 def diff(files = None, rev1 = 'HEAD', rev2 = None, out_fd = None):
588 """Show the diff between rev1 and rev2
589 """
590 if not files:
591 files = []
592
593 if rev1 and rev2:
594 diff_str = _output(['git-diff-tree', '-p', rev1, rev2, '--'] + files)
595 elif rev1 or rev2:
596 refresh_index()
597 if rev2:
598 diff_str = _output(['git-diff-index', '-p', '-R', rev2, '--'] + files)
599 else:
600 diff_str = _output(['git-diff-index', '-p', rev1, '--'] + files)
601 else:
602 diff_str = ''
603
604 if out_fd:
605 out_fd.write(diff_str)
606 else:
607 return diff_str
608
609 def diffstat(files = None, rev1 = 'HEAD', rev2 = None):
610 """Return the diffstat between rev1 and rev2
611 """
612 if not files:
613 files = []
614
615 p=popen2.Popen3('git-apply --stat')
616 diff(files, rev1, rev2, p.tochild)
617 p.tochild.close()
618 diff_str = p.fromchild.read().rstrip()
619 if p.wait():
620 raise GitException, 'git.diffstat failed'
621 return diff_str
622
623 def files(rev1, rev2):
624 """Return the files modified between rev1 and rev2
625 """
626
627 result = ''
628 for line in _output_lines('git-diff-tree -r %s %s' % (rev1, rev2)):
629 result += '%s %s\n' % tuple(line.rstrip().split(' ',4)[-1].split('\t',1))
630
631 return result.rstrip()
632
633 def barefiles(rev1, rev2):
634 """Return the files modified between rev1 and rev2, without status info
635 """
636
637 result = ''
638 for line in _output_lines('git-diff-tree -r %s %s' % (rev1, rev2)):
639 result += '%s\n' % line.rstrip().split(' ',4)[-1].split('\t',1)[-1]
640
641 return result.rstrip()
642
643 def pretty_commit(commit_id = 'HEAD'):
644 """Return a given commit (log + diff)
645 """
646 return _output(['git-diff-tree', '--cc', '--always', '--pretty', '-r',
647 commit_id])
648
649 def checkout(files = None, tree_id = None, force = False):
650 """Check out the given or all files
651 """
652 if not files:
653 files = []
654
655 if tree_id and __run('git-read-tree --reset', [tree_id]) != 0:
656 raise GitException, 'Failed git-read-tree --reset %s' % tree_id
657
658 checkout_cmd = 'git-checkout-index -q -u'
659 if force:
660 checkout_cmd += ' -f'
661 if len(files) == 0:
662 checkout_cmd += ' -a'
663 else:
664 checkout_cmd += ' --'
665
666 if __run(checkout_cmd, files) != 0:
667 raise GitException, 'Failed git-checkout-index'
668
669 def switch(tree_id, keep = False):
670 """Switch the tree to the given id
671 """
672 if not keep:
673 refresh_index()
674 if __run('git-read-tree -u -m', [get_head(), tree_id]) != 0:
675 raise GitException, 'git-read-tree failed (local changes maybe?)'
676
677 __set_head(tree_id)
678
679 def reset(files = None, tree_id = None, check_out = True):
680 """Revert the tree changes relative to the given tree_id. It removes
681 any local changes
682 """
683 if not tree_id:
684 tree_id = get_head()
685
686 if check_out:
687 cache_files = __tree_status(files, tree_id)
688 # files which were added but need to be removed
689 rm_files = [x[1] for x in cache_files if x[0] in ['A']]
690
691 checkout(files, tree_id, True)
692 # checkout doesn't remove files
693 map(os.remove, rm_files)
694
695 # if the reset refers to the whole tree, switch the HEAD as well
696 if not files:
697 __set_head(tree_id)
698
699 def pull(repository = 'origin', refspec = None):
700 """Pull changes from the remote repository. At the moment, just
701 use the 'git-pull' command
702 """
703 # 'git-pull' updates the HEAD
704 __clear_head_cache()
705
706 args = [repository]
707 if refspec:
708 args.append(refspec)
709
710 if __run(config.get('stgit', 'pullcmd'), args) != 0:
711 raise GitException, 'Failed "git-pull %s"' % repository
712
713 def repack():
714 """Repack all objects into a single pack
715 """
716 __run('git-repack -a -d -f')
717
718 def apply_patch(filename = None, diff = None, base = None,
719 fail_dump = True):
720 """Apply a patch onto the current or given index. There must not
721 be any local changes in the tree, otherwise the command fails
722 """
723 if base:
724 orig_head = get_head()
725 switch(base)
726 else:
727 refresh_index()
728
729 if diff is None:
730 if filename:
731 f = file(filename)
732 else:
733 f = sys.stdin
734 diff = f.read()
735 if filename:
736 f.close()
737
738 try:
739 _input_str('git-apply --index', diff)
740 except GitException:
741 if base:
742 switch(orig_head)
743 if fail_dump:
744 # write the failed diff to a file
745 f = file('.stgit-failed.patch', 'w+')
746 f.write(diff)
747 f.close()
748
749 raise
750
751 if base:
752 top = commit(message = 'temporary commit used for applying a patch',
753 parents = [base])
754 switch(orig_head)
755 merge(base, orig_head, top)
756
757 def clone(repository, local_dir):
758 """Clone a remote repository. At the moment, just use the
759 'git-clone' script
760 """
761 if __run('git-clone', [repository, local_dir]) != 0:
762 raise GitException, 'Failed "git-clone %s %s"' \
763 % (repository, local_dir)
764
765 def modifying_revs(files, base_rev):
766 """Return the revisions from the list modifying the given files
767 """
768 cmd = ['git-rev-list', '%s..' % base_rev, '--']
769 revs = [line.strip() for line in _output_lines(cmd + files)]
770
771 return revs