bin/run-mirrors: Run post scripts after mirror jobs.
[mirror-admin] / bin / typicalsync
1 #!/usr/bin/perl -wT
2
3 # Copyright (c) 2006 Anthony Towns <ajt@debian.org>
4 #
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14
15 use strict;
16 use Fcntl ':flock';
17 use File::Find;
18 use POSIX qw(strftime);
19
20 # configuration:
21
22 my $local_dir = "/srv/ftp.debian.org/mirror";
23 my $rsync_host = undef; #"merkel.debian.org";
24 my $rsync_dir = undef; #"debian";
25
26 my $dest = "/srv/ftp.debian.org/rsync/typical";
27 my $max_del = 1000;
28
29 $ENV{"PATH"} = "/bin:/usr/bin";
30
31 # program
32
33 my $hostname = `/bin/hostname -f`;
34 die "bad hostname" unless $hostname =~ m/^([a-zA-Z0-9._-]+)/;
35 $hostname = $1;
36
37 my $lockfile = "./Archive-Update-in-Progress-$hostname";
38
39 unless (open LKFILE, "> $dest/$lockfile" and flock(LKFILE, LOCK_EX)) {
40 print "$hostname is unable to start sync, lock file exists\n";
41 exit(1);
42 }
43
44 if (defined $rsync_host && defined $rsync_dir) {
45 system("rsync --links --hard-links --times --verbose --recursive"
46 ." --delay-updates --files-from :indices/files/typical.files"
47 ." rsync://$rsync_host/$rsync_dir/ $dest/");
48 } else {
49 open FILELIST, "< $local_dir/indices/files/typical.files"
50 or die "typical.files index not found";
51 while (<FILELIST>) {
52 chomp;
53 m/^(.*)$/; $_ = $1;
54 my @l = lstat("$local_dir/$_");
55 next unless (@l);
56
57 if (-l _) {
58 my $lpath = readlink("$local_dir/$_");
59 $lpath =~ m/^(.*)$/; $lpath = $1;
60 if (-l "$dest/$_") {
61 next if ($lpath eq readlink("$dest/$_"));
62 }
63
64 unless (mk_dirname_as_dirs($dest, $_)) {
65 print "E: couldn't create path for $_\n";
66 next;
67 }
68
69 if (-d "$dest/$_") {
70 rename "$dest/$_", "$dest/$_.remove" or print "E: couldn't rename old dir $_ out of the way\n";
71 } elsif (-e "$dest/$_") {
72 unlink("$dest/$_") or print "E: couldn't unlink $_\n";
73 }
74 symlink($lpath, "$dest/$_") or print "E: couldn't create $_ as symlink to $lpath\n";
75 next;
76 }
77
78 next if (-d _);
79
80 unless (mk_dirname_as_dirs($dest, $_)) {
81 print "E: couldn't create path for $_\n";
82 next;
83 }
84
85 my @d = lstat("$dest/$_");
86 if (@d) {
87 if (-d _) {
88 rename("$dest/$_", "$dest/$_.remove") or print "E: couldn't rename old dir $_ out of the way\n";
89 } else {
90 next if (@l and @d and $l[0] == $d[0] and $l[1] == $d[1]);
91 #next if (@l and @d and $l[7] == $d[7]);
92 print "I: updating $_\n";
93 unlink("$dest/$_");
94 }
95 }
96
97 link("$local_dir/$_", "$dest/$_") or print "E: couldn't link $_\n";
98 }
99 close(FILELIST);
100 }
101
102 print "Files synced, now deleting any unnecessary files\n";
103
104 my %expected_files = ();
105 open FILES, "< $dest/indices/files/typical.files"
106 or die "typical.files index not found";
107 while (<FILES>) {
108 chomp;
109 $expected_files{$_} = 1;
110 }
111 close(FILES);
112
113 chdir($dest);
114
115 my $del_count = 0;
116 my $last = '';
117 finddepth({wanted => \&wanted, no_chdir => 1}, ".");
118
119 open TRACE, "> $dest/project/trace/$hostname" or die "couldn't open trace";
120 print TRACE strftime("%a %b %e %H:%M:%S UTC %Y", gmtime) . "\n";
121 close TRACE;
122
123 close LKFILE;
124 unlink("$dest/$lockfile");
125 exit(0);
126
127 sub wanted {
128 my ($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_);
129 if (-d _) {
130 if (substr($last, 0, length($_) + 1) ne "$_/") {
131 print "Deleting empty directory: $_\n";
132 $_ = m/^(.*)$/;
133 my $f = $1;
134 rmdir($f);
135 } else {
136 $last = $_;
137 }
138 } elsif ($_ =~ m|^\./project/trace/| or $_ eq $lockfile) {
139 $last = $_;
140 } elsif (defined $expected_files{$_}) {
141 $last = $_;
142 } elsif ($del_count < $max_del) {
143 $del_count++;
144 print "Deleting file: $_\n";
145 $_ = m/^(.*)$/;
146 my $f = $1;
147 unlink($f);
148 }
149 }
150
151 sub mk_dirname_as_dirs {
152 my ($base, $file) = @_;
153 while ($file =~ m,^/*([^/]+)/+([^/].*)$,) {
154 $file = $2;
155 $base = "$base/$1";
156 my @blah = lstat($base);
157 if (!@blah) {
158 mkdir($base, 0777);
159 } elsif (-l _ or ! -d _) {
160 print "SHOULD BE A DIRECTORY: $base\n";
161 unlink($base);
162 mkdir($base, 0777);
163 }
164 }
165 1;
166 }
167
168