2 * This file is part of DisOrder
3 * Copyright (C) 2007-2009 Richard Kettlewell
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 /** @file server/dbupgrade.c
19 * @brief Database upgrade utility
21 * Invoked by the server when necessary.
23 #include "disorder-server.h"
25 static DB_TXN
*global_tid
;
29 #define BADKEY_DELETE 2
31 /** @brief Bad key behavior */
32 static int badkey
= BADKEY_WARN
;
34 static long aliases_removed
, keys_normalized
, values_normalized
, renoticed
;
35 static long keys_already_ok
, values_already_ok
;
37 static const struct option options
[] = {
38 { "help", no_argument
, 0, 'h' },
39 { "version", no_argument
, 0, 'V' },
40 { "config", required_argument
, 0, 'c' },
41 { "debug", no_argument
, 0, 'd' },
42 { "no-debug", no_argument
, 0, 'D' },
43 { "delete-bad-keys", no_argument
, 0, 'x' },
44 { "fail-bad-keys", no_argument
, 0, 'X' },
45 { "syslog", no_argument
, 0, 's' },
46 { "no-syslog", no_argument
, 0, 'S' },
50 /* display usage message and terminate */
51 static void attribute((noreturn
)) help(void) {
53 " disorder-dbupgrade [OPTIONS]\n"
55 " --help, -h Display usage message\n"
56 " --version, -V Display version number\n"
57 " --config PATH, -c PATH Set configuration file\n"
58 " --debug, -d Turn on debugging\n"
59 " --[no-]syslog Force logging\n"
60 " --delete-bad-keys, -x Delete unconvertible keys\n"
61 " --fail-bad-keys, -X Fail if bad keys are found\n"
63 "Database upgrader for DisOrder. Not intended to be run\n"
69 /** @brief Visit each key in a database and call @p callback
70 * @return 0 or DB_LOCK_DEADLOCK
72 * @p global_tid must be set. @p callback should return 0 or DB_LOCK_DEADLOCK.
74 static int scan_core(const char *name
, DB
*db
,
75 int (*callback
)(const char *name
, DB
*db
, DBC
*c
,
78 DBC
*c
= trackdb_opencursor(db
, global_tid
);
82 values_normalized
= 0;
87 values_already_ok
= 0;
88 memset(k
, 0, sizeof k
);
89 memset(d
, 0, sizeof d
);
90 while((err
= c
->c_get(c
, k
, d
, DB_NEXT
)) == 0) {
91 if((err
= callback(name
, db
, c
, k
, d
)))
95 disorder_info("scanning %s, %ld so far", name
, count
);
97 if(err
&& err
!= DB_NOTFOUND
&& err
!= DB_LOCK_DEADLOCK
)
98 disorder_fatal(0, "%s: error scanning database: %s", name
, db_strerror(err
));
99 r
= (err
== DB_LOCK_DEADLOCK ? err
: 0);
100 if((err
= c
->c_close(c
)))
101 disorder_fatal(0, "%s: error closing cursor: %s", name
, db_strerror(err
));
102 disorder_info("%s: %ld entries scanned", name
, count
);
103 if(values_normalized
|| values_already_ok
)
104 disorder_info("%s: %ld values converted, %ld already ok", name
,
105 values_normalized
, values_already_ok
);
106 if(keys_normalized
|| keys_already_ok
)
107 disorder_info("%s: %ld keys converted, %ld already OK", name
,
108 keys_normalized
, keys_already_ok
);
110 disorder_info("%s: %ld aliases removed", name
, aliases_removed
);
112 disorder_info("%s: %ld tracks re-noticed", name
, renoticed
);
116 /** @brief Visit each key in a database and call @p callback
118 * Everything happens inside the @p global_tid tranasction. @p callback
119 * should return 0 or DB_LOCK_DEADLOCK.
121 static void scandb(const char *name
, DB
*db
,
122 int (*callback
)(const char *name
, DB
*db
, DBC
*c
,
124 disorder_info("scanning %s", name
);
126 global_tid
= trackdb_begin_transaction();
127 if(scan_core(name
, db
, callback
)) {
128 trackdb_abort_transaction(global_tid
);
130 disorder_error(0, "detected deadlock, restarting scan");
133 trackdb_commit_transaction(global_tid
);
140 /** @brief Truncate database @p db */
141 static void truncate_database(const char *name
, DB
*db
) {
146 err
= db
->truncate(db
, 0, &count
, DB_AUTO_COMMIT
);
147 } while(err
== DB_LOCK_DEADLOCK
);
149 disorder_fatal(0, "error truncating %s: %s", name
, db_strerror(err
));
154 static int normalize_keys(const char *name
, DB
*db
, DBC
*c
,
160 /* Find the normalized form of the key */
161 knfc
= utf8_compose_canon(k
->data
, k
->size
, &nknfc
);
165 disorder_error(0, "%s: invalid key: %.*s", name
,
166 (int)k
->size
, (const char *)k
->data
);
169 disorder_error(0, "%s: deleting invalid key: %.*s", name
,
170 (int)k
->size
, (const char *)k
->data
);
171 if((err
= c
->c_del(c
, 0))) {
172 if(err
!= DB_LOCK_DEADLOCK
)
173 disorder_fatal(0, "%s: error removing denormalized key: %s",
174 name
, db_strerror(err
));
179 disorder_fatal(0, "%s: invalid key: %.*s", name
,
180 (int)k
->size
, (const char *)k
->data
);
184 /* If the key is already in NFC then do nothing */
185 if(nknfc
== k
->size
&& !memcmp(k
->data
, knfc
, nknfc
)) {
189 /* To rename the key we must delete the old one and insert a new one */
190 if((err
= c
->c_del(c
, 0))) {
191 if(err
!= DB_LOCK_DEADLOCK
)
192 disorder_fatal(0, "%s: error removing denormalized key: %s",
193 name
, db_strerror(err
));
198 if((err
= db
->put(db
, global_tid
, k
, d
, DB_NOOVERWRITE
))) {
199 if(err
!= DB_LOCK_DEADLOCK
)
200 disorder_fatal(0, "%s: error storing normalized key: %s",
201 name
, db_strerror(err
));
208 static int normalize_values(const char *name
, DB
*db
,
209 DBC
attribute((unused
)) *c
,
215 /* Find the normalized form of the value */
216 dnfc
= utf8_compose_canon(d
->data
, d
->size
, &ndnfc
);
218 disorder_fatal(0, "%s: cannot convert data to NFC: %.*s", name
,
219 (int)d
->size
, (const char *)d
->data
);
220 /* If the key is already in NFC then do nothing */
221 if(ndnfc
== d
->size
&& !memcmp(d
->data
, dnfc
, ndnfc
)) {
227 if((err
= db
->put(db
, global_tid
, k
, d
, 0))) {
228 if(err
!= DB_LOCK_DEADLOCK
)
229 disorder_fatal(0, "%s: error storing normalized data: %s",
230 name
, db_strerror(err
));
237 static int renotice(const char *name
, DB
attribute((unused
)) *db
,
238 DBC
attribute((unused
)) *c
,
240 const struct kvp
*const t
= kvp_urldecode(d
->data
, d
->size
);
241 const char *const track
= xstrndup(k
->data
, k
->size
);
242 const char *const path
= kvp_get(t
, "_path");
246 /* If an alias sorts later than the actual filename then it'll appear
248 if(kvp_get(t
, "_alias_for"))
250 disorder_fatal(0, "%s: no '_path' for %.*s", name
,
251 (int)k
->size
, (const char *)k
->data
);
253 switch(err
= trackdb_notice_tid(track
, path
, global_tid
)) {
257 case DB_LOCK_DEADLOCK
:
260 disorder_fatal(0, "%s: unexpected return from trackdb_notice_tid: %s",
261 name
, db_strerror(err
));
265 static int remove_aliases_normalize_keys(const char *name
, DB
*db
, DBC
*c
,
267 const struct kvp
*const t
= kvp_urldecode(d
->data
, d
->size
);
270 if(kvp_get(t
, "_alias_for")) {
271 /* This is an alias. We remove all the alias entries. */
272 if((err
= c
->c_del(c
, 0))) {
273 if(err
!= DB_LOCK_DEADLOCK
)
274 disorder_fatal(0, "%s: error removing alias: %s", name
, db_strerror(err
));
279 } else if(!kvp_get(t
, "_path"))
280 disorder_error(0, "%s: %.*s has neither _alias_for nor _path", name
,
281 (int)k
->size
, (const char *)k
->data
);
282 return normalize_keys(name
, db
, c
, k
, d
);
285 /** @brief Upgrade the database to the current version
287 * This function is supposed to be idempotent, so if it is interrupted
288 * half way through it is safe to restart.
290 static void upgrade(void) {
293 disorder_info("upgrading database to dbversion %ld", config
->dbversion
);
294 /* Normalize keys and values as required. We will also remove aliases as
295 * they will be regenerated when we re-noticed the tracks. */
296 disorder_info("renormalizing keys");
297 scandb("tracks.db", trackdb_tracksdb
, remove_aliases_normalize_keys
);
298 scandb("prefs.db", trackdb_prefsdb
, normalize_keys
);
299 scandb("global.db", trackdb_globaldb
, normalize_keys
);
300 scandb("noticed.db", trackdb_noticeddb
, normalize_values
);
301 /* search.db and tags.db we will rebuild */
302 disorder_info("regenerating search database and aliases");
303 truncate_database("search.db", trackdb_searchdb
);
304 truncate_database("tags.db", trackdb_tagsdb
);
305 /* Regenerate the search database and aliases */
306 scandb("tracks.db", trackdb_tracksdb
, renotice
);
307 /* Finally update the database version */
308 snprintf(buf
, sizeof buf
, "%ld", config
->dbversion
);
309 trackdb_set_global("_dbversion", buf
, 0);
310 disorder_info("completed database upgrade");
313 int main(int argc
, char **argv
) {
314 int n
, logsyslog
= !isatty(2);
318 if(!setlocale(LC_CTYPE
, "")) disorder_fatal(errno
, "error calling setlocale");
319 while((n
= getopt_long(argc
, argv
, "hVc:dDSsxX", options
, 0)) >= 0) {
322 case 'V': version("disorder-dbupgrade");
323 case 'c': configfile
= optarg
; break;
324 case 'd': debugging
= 1; break;
325 case 'D': debugging
= 0; break;
326 case 'S': logsyslog
= 0; break;
327 case 's': logsyslog
= 1; break;
328 case 'x': badkey
= BADKEY_DELETE
; break;
329 case 'X': badkey
= BADKEY_FAIL
; break;
330 default: disorder_fatal(0, "invalid option");
333 /* If stderr is a TTY then log there, otherwise to syslog. */
335 openlog(progname
, LOG_PID
, LOG_DAEMON
);
336 log_default
= &log_syslog
;
338 if(config_read(0, NULL
)) disorder_fatal(0, "cannot read configuration");
339 /* Open the database */
340 trackdb_init(TRACKDB_NO_RECOVER
);
341 trackdb_open(TRACKDB_OPEN_FOR_UPGRADE
);