| 1 | /** |
| 2 | python-cdb 0.32 |
| 3 | Copyright 2001, 2002 Michael J. Pomraning <mjp@pilcrow.madison.wi.us> |
| 4 | |
| 5 | This program is free software; you can redistribute it and/or modify |
| 6 | it under the terms of the GNU General Public License as published by |
| 7 | the Free Software Foundation; either version 2 of the License, or |
| 8 | (at your option) any later version. |
| 9 | |
| 10 | This program is distributed in the hope that it will be useful, |
| 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | GNU General Public License for more details. |
| 14 | |
| 15 | You should have received a copy of the GNU General Public License |
| 16 | along with this program; if not, write to the Free Software |
| 17 | Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| 18 | |
| 19 | **/ |
| 20 | |
| 21 | #include <Python.h> |
| 22 | #include <unistd.h> |
| 23 | #include <sys/types.h> |
| 24 | #include <fcntl.h> |
| 25 | #include "cdb.h" |
| 26 | #include "cdb_make.h" |
| 27 | |
| 28 | #define open_read(x) (open((x),O_RDONLY|O_NDELAY)) |
| 29 | /* ala djb's open_foo */ |
| 30 | |
| 31 | #define VERSION "0.32" |
| 32 | #define CDBVERSION "0.75" |
| 33 | |
| 34 | /* ------------------- cdb object -------------------- */ |
| 35 | |
| 36 | static char cdbo_object_doc[] = "\ |
| 37 | This object represents a CDB database: a reliable, constant\n\ |
| 38 | database mapping strings of bytes (\"keys\") to strings of bytes\n\ |
| 39 | (\"data\"), and designed for fast lookups.\n\ |
| 40 | \n\ |
| 41 | Unlike a conventional mapping, CDBs can meaningfully store multiple\n\ |
| 42 | records under one key (though this feature is not often used).\n\ |
| 43 | \n\ |
| 44 | A CDB object 'cdb_o' offers the following interesting attributes:\n\ |
| 45 | \n\ |
| 46 | Dict-like Lookup Methods:\n\ |
| 47 | cdb_o[key], get(key), getnext(), getall(key)\n\ |
| 48 | \n\ |
| 49 | Key-based Iteration Methods:\n\ |
| 50 | keys(), firstkey(), nextkey()\n\ |
| 51 | (Key-based iteration returns only distinct keys.)\n\ |
| 52 | \n\ |
| 53 | Raw Iteration Method:\n\ |
| 54 | each()\n\ |
| 55 | (\"Dumping\" may return the same key more than once.)\n\ |
| 56 | \n\ |
| 57 | __members__:\n\ |
| 58 | fd - File descriptor of the underlying cdb.\n\ |
| 59 | name - Name of the cdb, or None if not known.\n\ |
| 60 | size - Size of the cdb, or None if not mmap()d.\n\ |
| 61 | \n\ |
| 62 | __length__:\n\ |
| 63 | len(cdb_o) returns the total number of items in a cdb,\n\ |
| 64 | which may or may not exceed the number of distinct keys.\n"; |
| 65 | |
| 66 | |
| 67 | typedef struct { |
| 68 | PyObject_HEAD |
| 69 | struct cdb c; |
| 70 | PyObject * name_py; /* 'filename' or Py_None */ |
| 71 | PyObject * getkey; /* squirreled away for getnext() */ |
| 72 | uint32 eod; /* as in cdbdump */ |
| 73 | uint32 iter_pos; |
| 74 | uint32 each_pos; |
| 75 | uint32 numrecords; |
| 76 | } CdbObject; |
| 77 | |
| 78 | staticforward PyTypeObject CdbType; |
| 79 | |
| 80 | PyObject * CDBError; |
| 81 | #define CDBerr PyErr_SetFromErrno(CDBError) |
| 82 | |
| 83 | static PyObject * |
| 84 | cdb_pyread(CdbObject *cdb_o, unsigned int len, uint32 pos) { |
| 85 | struct cdb *c; |
| 86 | PyObject *s = NULL; |
| 87 | |
| 88 | c = &cdb_o->c; |
| 89 | |
| 90 | if (c->map) { |
| 91 | if ((pos > c->size) || (c->size - pos < len)) |
| 92 | goto FORMAT; |
| 93 | s = PyString_FromStringAndSize(c->map + pos, len); |
| 94 | } else { |
| 95 | s = PyString_FromStringAndSize(NULL, len); |
| 96 | if (s == NULL) |
| 97 | return NULL; |
| 98 | if (lseek(c->fd,pos,SEEK_SET) == -1) goto ERRNO; |
| 99 | while (len > 0) { |
| 100 | int r; |
| 101 | char * buf = PyString_AsString(s); |
| 102 | |
| 103 | do { |
| 104 | Py_BEGIN_ALLOW_THREADS |
| 105 | r = read(c->fd,buf,len); |
| 106 | Py_END_ALLOW_THREADS |
| 107 | } |
| 108 | while ((r == -1) && (errno == EINTR)); |
| 109 | if (r == -1) goto ERRNO; |
| 110 | if (r == 0) goto FORMAT; |
| 111 | buf += r; |
| 112 | len -= r; |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | return s; |
| 117 | |
| 118 | FORMAT: |
| 119 | Py_XDECREF(s); |
| 120 | PyErr_SetFromErrno(PyExc_RuntimeError); |
| 121 | return NULL; |
| 122 | |
| 123 | ERRNO: |
| 124 | Py_XDECREF(s); |
| 125 | return CDBerr; |
| 126 | |
| 127 | } |
| 128 | |
| 129 | |
| 130 | #define CDBO_CURDATA(x) (cdb_pyread(x, x->c.dlen, x->c.dpos)) |
| 131 | |
| 132 | |
| 133 | /* ------------------- CdbObject methods -------------------- */ |
| 134 | |
| 135 | static char cdbo_has_key_doc[] = |
| 136 | "cdb_o.has_key(k) -> 1 (or 0)\n\ |
| 137 | \n\ |
| 138 | Returns true if the CDB contains key k."; |
| 139 | |
| 140 | static PyObject * |
| 141 | cdbo_has_key(CdbObject *self, PyObject *args) { |
| 142 | |
| 143 | char * key; |
| 144 | unsigned int klen; |
| 145 | int r; |
| 146 | |
| 147 | if (!PyArg_ParseTuple(args, "s#", &key, &klen)) |
| 148 | return NULL; |
| 149 | |
| 150 | r = cdb_find(&self->c, key, klen); |
| 151 | if (r == -1) |
| 152 | return CDBerr; |
| 153 | |
| 154 | return Py_BuildValue("i", r); |
| 155 | |
| 156 | } |
| 157 | |
| 158 | static char cdbo_get_doc[] = |
| 159 | "cdb_o.get(k [, i]) -> data (or None)\n\ |
| 160 | \n\ |
| 161 | Fetches the record stored under key k, skipping past the first i\n\ |
| 162 | records under that key (default: 0). Prepares the next call to\n\ |
| 163 | getnext().\n\ |
| 164 | \n\ |
| 165 | Assuming cdb_o.has_key(k) == 1, then all of the following return:\n\ |
| 166 | the first record stored under key k:\n\ |
| 167 | \n\ |
| 168 | cdb_o.get(k) == cdb_o[k] == cdb_o.getall(k)[0]\n"; |
| 169 | |
| 170 | static PyObject * |
| 171 | cdbo_get(CdbObject *self, PyObject *args) { |
| 172 | |
| 173 | char * key; |
| 174 | unsigned int klen; |
| 175 | int r; |
| 176 | int i = 0; |
| 177 | |
| 178 | if (!PyArg_ParseTuple(args, "s#|i:get", &key, &klen, &i)) |
| 179 | return NULL; |
| 180 | |
| 181 | cdb_findstart(&self->c); |
| 182 | |
| 183 | for (;;) { |
| 184 | r = cdb_findnext(&self->c, key, klen); |
| 185 | if (r == -1) return CDBerr; |
| 186 | if (!r) return Py_BuildValue(""); |
| 187 | if (!i) break; |
| 188 | --i; |
| 189 | } |
| 190 | |
| 191 | /* prep. possibly ensuing call to getnext() */ |
| 192 | Py_XDECREF(self->getkey); |
| 193 | self->getkey = PyString_FromStringAndSize(key, klen); |
| 194 | if (self->getkey == NULL) |
| 195 | return NULL; |
| 196 | |
| 197 | return CDBO_CURDATA(self); |
| 198 | } |
| 199 | |
| 200 | static char cdbo_getall_doc[] = |
| 201 | "cdb_o.getall(k) -> ['data', ... ]\n\ |
| 202 | \n\ |
| 203 | Return a list of all records stored under key k."; |
| 204 | |
| 205 | static PyObject * |
| 206 | cdbo_getall(CdbObject *self, PyObject *args) { |
| 207 | |
| 208 | PyObject * list, * data; |
| 209 | char * key; |
| 210 | unsigned int klen; |
| 211 | int r, err; |
| 212 | |
| 213 | if (!PyArg_ParseTuple(args, "s#:getall", &key, &klen)) |
| 214 | return NULL; |
| 215 | |
| 216 | list = PyList_New(0); |
| 217 | |
| 218 | if (list == NULL) return NULL; |
| 219 | |
| 220 | cdb_findstart(&self->c); |
| 221 | |
| 222 | while ((r = cdb_findnext(&self->c, key, klen))) { |
| 223 | if (r == -1) { |
| 224 | Py_DECREF(list); |
| 225 | return CDBerr; |
| 226 | } |
| 227 | data = CDBO_CURDATA(self); |
| 228 | if (data == NULL) { |
| 229 | Py_DECREF(list); |
| 230 | return NULL; |
| 231 | } |
| 232 | err = PyList_Append(list, data); |
| 233 | Py_DECREF(data); |
| 234 | if (err != 0) { |
| 235 | Py_DECREF(list); |
| 236 | return NULL; |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | return list; |
| 241 | |
| 242 | } |
| 243 | |
| 244 | static char cdbo_getnext_doc[] = |
| 245 | "cdb_o.getnext() -> 'data' (or None)\n\ |
| 246 | \n\ |
| 247 | For iteration over the records stored under one key, avoiding loading\n\ |
| 248 | all items into memory). The \"current key\" is determined by the most\n\ |
| 249 | recent call to get().\n\ |
| 250 | \n\ |
| 251 | The following loops through all items stored under key k:\n\ |
| 252 | \n\ |
| 253 | ## cdb_o.getall(k) possibly too big for memory\n\ |
| 254 | rec = cdb_o.get(k)\n\ |
| 255 | while rec is not None:\n\ |
| 256 | do_something(rec)\n\ |
| 257 | rec = cdb_o.getnext()\n"; |
| 258 | |
| 259 | static PyObject * |
| 260 | cdbo_getnext(CdbObject *self, PyObject *args) { |
| 261 | |
| 262 | if (!PyArg_ParseTuple(args, ":getnext")) |
| 263 | return NULL; |
| 264 | |
| 265 | if (self->getkey == NULL) { |
| 266 | PyErr_SetString(PyExc_TypeError, |
| 267 | "getnext() called without first calling get()"); |
| 268 | return NULL; |
| 269 | } |
| 270 | |
| 271 | switch(cdb_findnext(&self->c, |
| 272 | PyString_AsString(self->getkey), |
| 273 | PyString_Size(self->getkey))) { |
| 274 | case -1: |
| 275 | return CDBerr; |
| 276 | case 0: |
| 277 | Py_DECREF(self->getkey); |
| 278 | self->getkey = NULL; |
| 279 | return Py_BuildValue(""); |
| 280 | default: |
| 281 | return CDBO_CURDATA(self); |
| 282 | } |
| 283 | /* not reached */ |
| 284 | } |
| 285 | |
| 286 | uint32 |
| 287 | _cdbo_init_eod(CdbObject *self) { |
| 288 | |
| 289 | char nonce[4]; |
| 290 | |
| 291 | if (cdb_read(&self->c, nonce, 4, 0) == -1) |
| 292 | return 0; |
| 293 | |
| 294 | uint32_unpack(nonce, &self->eod); |
| 295 | |
| 296 | return self->eod; |
| 297 | |
| 298 | } |
| 299 | |
| 300 | /* |
| 301 | * _cdbo_keyiter(cdb_o) |
| 302 | * |
| 303 | * Whiz-bang all-in-one: |
| 304 | * extract current record |
| 305 | * compare current pos to pos implied by cdb_find(current_key) |
| 306 | * (Different? adv. iter cursor, loop and try again) |
| 307 | * advance iteration cursor |
| 308 | * return key |
| 309 | */ |
| 310 | |
| 311 | static PyObject * |
| 312 | _cdbo_keyiter(CdbObject *self) { |
| 313 | |
| 314 | PyObject *key; |
| 315 | char buf[8]; |
| 316 | uint32 klen, dlen; |
| 317 | |
| 318 | if (! self->eod) |
| 319 | _cdbo_init_eod(self); |
| 320 | |
| 321 | while (self->iter_pos < self->eod) { |
| 322 | if (cdb_read(&self->c, buf, 8, self->iter_pos) == -1) |
| 323 | return CDBerr; |
| 324 | |
| 325 | uint32_unpack(buf, &klen); |
| 326 | uint32_unpack(buf+4, &dlen); |
| 327 | |
| 328 | key = cdb_pyread(self, klen, self->iter_pos + 8); |
| 329 | |
| 330 | if (key == NULL) |
| 331 | return NULL; |
| 332 | |
| 333 | switch(cdb_find(&self->c,PyString_AsString(key),PyString_Size(key))) { |
| 334 | case -1: |
| 335 | Py_DECREF(key); |
| 336 | key = NULL; |
| 337 | return CDBerr; |
| 338 | case 0: |
| 339 | /* bizarre, impossible? PyExc_RuntimeError? */ |
| 340 | PyErr_SetString(PyExc_KeyError, |
| 341 | PyString_AS_STRING((PyStringObject *) key)); |
| 342 | Py_DECREF(key); |
| 343 | key = NULL; |
| 344 | default: |
| 345 | if (key == NULL) /* already raised error */ |
| 346 | return NULL; |
| 347 | |
| 348 | if (cdb_datapos(&self->c) == self->iter_pos + klen + 8) { |
| 349 | /** first occurrence of key in the cdb **/ |
| 350 | self->iter_pos += 8 + klen + dlen; |
| 351 | return key; |
| 352 | } |
| 353 | Py_DECREF(key); /* better luck next time around */ |
| 354 | self->iter_pos += 8 + klen + dlen; |
| 355 | } |
| 356 | } |
| 357 | |
| 358 | return Py_BuildValue(""); /* iter_pos >= eod; we're done */ |
| 359 | |
| 360 | } |
| 361 | |
| 362 | static char cdbo_keys_doc[] = |
| 363 | "cdb_o.keys() -> ['k1', 'k2', ... ]\n\ |
| 364 | \n\ |
| 365 | Returns a list of all (distinct) keys in the database."; |
| 366 | |
| 367 | static PyObject * |
| 368 | cdbo_keys(CdbObject *self, PyObject *args) { |
| 369 | |
| 370 | PyObject *r, *key; |
| 371 | uint32 pos; |
| 372 | int err; |
| 373 | |
| 374 | if (! PyArg_ParseTuple(args, "")) |
| 375 | return NULL; |
| 376 | |
| 377 | r = PyList_New(0); |
| 378 | if (r == NULL) |
| 379 | return NULL; |
| 380 | |
| 381 | pos = self->iter_pos; /* don't interrupt a manual iteration */ |
| 382 | |
| 383 | self->iter_pos = 2048; |
| 384 | |
| 385 | key = _cdbo_keyiter(self); |
| 386 | while (key != Py_None) { |
| 387 | err = PyList_Append(r, key); |
| 388 | Py_DECREF(key); |
| 389 | if (err != 0) { |
| 390 | Py_DECREF(r); |
| 391 | self->iter_pos = pos; |
| 392 | return NULL; |
| 393 | } |
| 394 | key = _cdbo_keyiter(self); |
| 395 | } |
| 396 | Py_DECREF(key); |
| 397 | |
| 398 | self->iter_pos = pos; |
| 399 | |
| 400 | return r; |
| 401 | |
| 402 | } |
| 403 | |
| 404 | static char cdbo_firstkey_doc[] = |
| 405 | "cdb_o.firstkey() -> key (or None)\n\ |
| 406 | \n\ |
| 407 | Return the first key in the database, resetting the internal\n\ |
| 408 | iteration cursor. firstkey() and nextkey() may be used to\n\ |
| 409 | traverse all distinct keys in the cdb. See each() for raw\n\ |
| 410 | iteration."; |
| 411 | |
| 412 | static PyObject * |
| 413 | cdbo_firstkey(CdbObject *self, PyObject *args) { |
| 414 | |
| 415 | if (! PyArg_ParseTuple(args, ":firstkey")) |
| 416 | return NULL; |
| 417 | |
| 418 | self->iter_pos = 2048; |
| 419 | |
| 420 | return _cdbo_keyiter(self); |
| 421 | |
| 422 | } |
| 423 | |
| 424 | static char cdbo_nextkey_doc[] = |
| 425 | "cdb_o.nextkey() -> key (or None)\n\ |
| 426 | \n\ |
| 427 | Return the next distinct key in the cdb.\n\ |
| 428 | \n\ |
| 429 | The following code walks the CDB one key at a time:\n\ |
| 430 | \n\ |
| 431 | key = cdb_o.firstkey()\n\ |
| 432 | while key is not None:\n\ |
| 433 | print key\n\ |
| 434 | key = cdb_o.nextkey()\n"; |
| 435 | |
| 436 | static PyObject * |
| 437 | cdbo_nextkey(CdbObject *self, PyObject *args) { |
| 438 | |
| 439 | if (! PyArg_ParseTuple(args, ":nextkey")) |
| 440 | return NULL; |
| 441 | |
| 442 | return _cdbo_keyiter(self); |
| 443 | |
| 444 | } |
| 445 | |
| 446 | static char cdbo_each_doc[] = |
| 447 | "cdb_o.each() -> (key, data) (or None)\n\ |
| 448 | \n\ |
| 449 | Fetch the next ('key', 'data') record from the underlying cdb file,\n\ |
| 450 | returning None and resetting the iteration cursor when all records\n\ |
| 451 | have been fetched.\n\ |
| 452 | \n\ |
| 453 | Keys appear with each item under them -- e.g., (key,foo), (key2,bar),\n\ |
| 454 | (key,baz) -- order of records is determined by actual position on\n\ |
| 455 | disk. Both keys() and (for GDBM fanciers) firstkey()/nextkey()-style\n\ |
| 456 | iteration go to pains to present the user with only distinct keys."; |
| 457 | |
| 458 | static PyObject * |
| 459 | cdbo_each(CdbObject *self, PyObject *args) { |
| 460 | |
| 461 | PyObject *tup, *key, *dat; |
| 462 | char buf[8]; |
| 463 | uint32 klen, dlen; |
| 464 | |
| 465 | if (! PyArg_ParseTuple(args, ":each")) |
| 466 | return NULL; |
| 467 | |
| 468 | tup = PyTuple_New(2); |
| 469 | if (tup == NULL) |
| 470 | return NULL; |
| 471 | |
| 472 | if (! self->eod) |
| 473 | (void) _cdbo_init_eod(self); |
| 474 | |
| 475 | if (self->each_pos >= self->eod) { /* all done, reset cursor */ |
| 476 | self->each_pos = 2048; |
| 477 | Py_INCREF(Py_None); |
| 478 | return Py_None; |
| 479 | } |
| 480 | |
| 481 | if (cdb_read(&self->c, buf, 8, self->each_pos) == -1) |
| 482 | return CDBerr; |
| 483 | |
| 484 | uint32_unpack(buf, &klen); |
| 485 | uint32_unpack(buf+4, &dlen); |
| 486 | |
| 487 | key = cdb_pyread(self, klen, self->each_pos + 8); |
| 488 | dat = cdb_pyread(self, dlen, self->each_pos + 8 + klen); |
| 489 | |
| 490 | self->each_pos += klen + dlen + 8; |
| 491 | |
| 492 | if (key == NULL || dat == NULL) { |
| 493 | Py_XDECREF(key); Py_XDECREF(dat); |
| 494 | Py_DECREF(tup); |
| 495 | return NULL; |
| 496 | } |
| 497 | |
| 498 | if (PyTuple_SetItem(tup, 0, key) || PyTuple_SetItem(tup, 1, dat)) { |
| 499 | Py_DECREF(key); Py_DECREF(dat); Py_DECREF(tup); |
| 500 | return NULL; |
| 501 | } |
| 502 | |
| 503 | return tup; |
| 504 | } |
| 505 | |
| 506 | /*** cdb object as mapping ***/ |
| 507 | |
| 508 | static int |
| 509 | cdbo_length(CdbObject *self) { |
| 510 | |
| 511 | if (! self->numrecords) { |
| 512 | char buf[8]; |
| 513 | uint32 pos, klen, dlen; |
| 514 | |
| 515 | pos = 2048; |
| 516 | |
| 517 | if (! self->eod) |
| 518 | (void) _cdbo_init_eod(self); |
| 519 | |
| 520 | while (pos < self->eod) { |
| 521 | if (cdb_read(&self->c, buf, 8, pos) == -1) |
| 522 | return -1; |
| 523 | uint32_unpack(buf, &klen); |
| 524 | uint32_unpack(buf + 4, &dlen); |
| 525 | pos += 8 + klen + dlen; |
| 526 | self->numrecords++; |
| 527 | } |
| 528 | } |
| 529 | return (int) self->numrecords; |
| 530 | } |
| 531 | |
| 532 | static PyObject * |
| 533 | cdbo_subscript(CdbObject *self, PyObject *k) { |
| 534 | char * key; |
| 535 | int klen; |
| 536 | |
| 537 | if (! PyArg_Parse(k, "s#", &key, &klen)) |
| 538 | return NULL; |
| 539 | |
| 540 | switch(cdb_find(&self->c, key, (unsigned int)klen)) { |
| 541 | case -1: |
| 542 | return CDBerr; |
| 543 | case 0: |
| 544 | PyErr_SetString(PyExc_KeyError, |
| 545 | PyString_AS_STRING((PyStringObject *) k)); |
| 546 | return NULL; |
| 547 | default: |
| 548 | return CDBO_CURDATA(self); |
| 549 | } |
| 550 | /* not reached */ |
| 551 | } |
| 552 | |
| 553 | static PyMappingMethods cdbo_as_mapping = { |
| 554 | (inquiry)cdbo_length, |
| 555 | (binaryfunc)cdbo_subscript, |
| 556 | (objobjargproc)0 |
| 557 | }; |
| 558 | |
| 559 | static PyMethodDef cdb_methods[] = { |
| 560 | |
| 561 | {"get", (PyCFunction)cdbo_get, METH_VARARGS, |
| 562 | cdbo_get_doc }, |
| 563 | {"getnext", (PyCFunction)cdbo_getnext, METH_VARARGS, |
| 564 | cdbo_getnext_doc }, |
| 565 | {"getall", (PyCFunction)cdbo_getall, METH_VARARGS, |
| 566 | cdbo_getall_doc }, |
| 567 | {"has_key", (PyCFunction)cdbo_has_key, METH_VARARGS, |
| 568 | cdbo_has_key_doc }, |
| 569 | {"keys", (PyCFunction)cdbo_keys, METH_VARARGS, |
| 570 | cdbo_keys_doc }, |
| 571 | {"firstkey", (PyCFunction)cdbo_firstkey, METH_VARARGS, |
| 572 | cdbo_firstkey_doc }, |
| 573 | {"nextkey", (PyCFunction)cdbo_nextkey, METH_VARARGS, |
| 574 | cdbo_nextkey_doc }, |
| 575 | {"each", (PyCFunction)cdbo_each, METH_VARARGS, |
| 576 | cdbo_each_doc }, |
| 577 | { NULL, NULL } |
| 578 | }; |
| 579 | |
| 580 | /* ------------------- cdb operations -------------------- */ |
| 581 | |
| 582 | static PyObject * |
| 583 | _wrap_cdb_init(int fd) { /* constructor implementation */ |
| 584 | |
| 585 | CdbObject *self; |
| 586 | |
| 587 | self = PyObject_NEW(CdbObject, &CdbType); |
| 588 | if (self == NULL) return NULL; |
| 589 | |
| 590 | self->c.map = 0; /* break encapsulation -- cdb struct init'd to zero */ |
| 591 | cdb_init(&self->c, fd); |
| 592 | |
| 593 | self->iter_pos = 2048; |
| 594 | self->each_pos = 2048; |
| 595 | self->numrecords = 0; |
| 596 | self->eod = 0; |
| 597 | self->getkey = NULL; |
| 598 | |
| 599 | return (PyObject *) self; |
| 600 | } |
| 601 | |
| 602 | |
| 603 | static PyObject * |
| 604 | cdbo_constructor(PyObject *ignore, PyObject *args) { |
| 605 | |
| 606 | PyObject *self; |
| 607 | PyObject *f; |
| 608 | PyObject *name_attr = Py_None; |
| 609 | int fd; |
| 610 | |
| 611 | if (! PyArg_ParseTuple(args, "O:new", &f)) |
| 612 | return NULL; |
| 613 | |
| 614 | if (PyString_Check(f)) { |
| 615 | |
| 616 | if ((fd = open_read(PyString_AsString(f))) == -1) |
| 617 | return CDBerr; |
| 618 | |
| 619 | name_attr = f; |
| 620 | |
| 621 | } else if (PyInt_Check(f)) { |
| 622 | |
| 623 | fd = (int) PyInt_AsLong(f); |
| 624 | |
| 625 | } else { |
| 626 | |
| 627 | PyErr_SetString(PyExc_TypeError, |
| 628 | "expected filename or file descriptor"); |
| 629 | return NULL; |
| 630 | |
| 631 | } |
| 632 | |
| 633 | self = _wrap_cdb_init(fd); |
| 634 | if (self == NULL) return NULL; |
| 635 | |
| 636 | ((CdbObject *)self)->name_py = name_attr; |
| 637 | Py_INCREF(name_attr); |
| 638 | |
| 639 | return self; |
| 640 | } |
| 641 | |
| 642 | static void |
| 643 | cdbo_dealloc(CdbObject *self) { /* del(cdb_o) */ |
| 644 | |
| 645 | if (self->name_py != NULL) { |
| 646 | |
| 647 | /* if cdb_o.name is not None: we open()d it ourselves, so close it */ |
| 648 | if (PyString_Check(self->name_py)) |
| 649 | close(self->c.fd); |
| 650 | |
| 651 | Py_DECREF(self->name_py); |
| 652 | } |
| 653 | |
| 654 | Py_XDECREF(self->getkey); |
| 655 | |
| 656 | cdb_free(&self->c); |
| 657 | |
| 658 | PyMem_DEL(self); |
| 659 | } |
| 660 | |
| 661 | static PyObject * |
| 662 | cdbo_getattr(CdbObject *self, char *name) { |
| 663 | |
| 664 | PyObject * r; |
| 665 | |
| 666 | r = Py_FindMethod(cdb_methods, (PyObject *) self, name); |
| 667 | |
| 668 | if (r != NULL) |
| 669 | return r; |
| 670 | |
| 671 | PyErr_Clear(); |
| 672 | |
| 673 | if (!strcmp(name,"__members__")) |
| 674 | return Py_BuildValue("[sss]", "fd", "name", "size"); |
| 675 | |
| 676 | if (!strcmp(name,"fd")) { |
| 677 | return Py_BuildValue("i", self->c.fd); /* cdb_o.fd */ |
| 678 | } |
| 679 | |
| 680 | if (!strcmp(name,"name")) { |
| 681 | Py_INCREF(self->name_py); |
| 682 | return self->name_py; /* cdb_o.name */ |
| 683 | } |
| 684 | |
| 685 | if (!strcmp(name,"size")) { /* cdb_o.size */ |
| 686 | return self->c.map ? /** mmap()d ? stat.st_size : None **/ |
| 687 | Py_BuildValue("l", (long) self->c.size) : |
| 688 | Py_BuildValue(""); |
| 689 | } |
| 690 | |
| 691 | PyErr_SetString(PyExc_AttributeError, name); |
| 692 | return NULL; |
| 693 | } |
| 694 | |
| 695 | |
| 696 | /* ----------------- cdbmake object ------------------ */ |
| 697 | |
| 698 | static char cdbmake_object_doc[] = |
| 699 | "cdbmake objects resemble the struct cdb_make interface:\n\ |
| 700 | \n\ |
| 701 | CDB Construction Methods:\n\ |
| 702 | add(k, v), finish()\n\ |
| 703 | \n\ |
| 704 | __members__:\n\ |
| 705 | fd - fd of underlying CDB, or -1 if finish()ed\n\ |
| 706 | fn, fntmp - as from the cdb package's cdbmake utility\n\ |
| 707 | numentries - current number of records add()ed\n"; |
| 708 | |
| 709 | typedef struct { |
| 710 | PyObject_HEAD |
| 711 | struct cdb_make cm; |
| 712 | PyObject * fn; |
| 713 | PyObject * fntmp; |
| 714 | } cdbmakeobject; |
| 715 | |
| 716 | staticforward PyTypeObject CdbMakeType; |
| 717 | |
| 718 | #define CDBMAKEerr PyErr_SetFromErrno(PyExc_IOError) |
| 719 | |
| 720 | |
| 721 | /* ----------------- CdbMake methods ------------------ */ |
| 722 | |
| 723 | static PyObject * |
| 724 | CdbMake_add(cdbmakeobject *self, PyObject *args) { |
| 725 | |
| 726 | char * key, * dat; |
| 727 | unsigned int klen, dlen; |
| 728 | |
| 729 | if (!PyArg_ParseTuple(args,"s#s#:add",&key,&klen,&dat,&dlen)) |
| 730 | return NULL; |
| 731 | |
| 732 | if (cdb_make_add(&self->cm, key, klen, dat, dlen) == -1) |
| 733 | return CDBMAKEerr; |
| 734 | |
| 735 | return Py_BuildValue(""); |
| 736 | |
| 737 | } |
| 738 | |
| 739 | static PyObject * |
| 740 | CdbMake_finish(cdbmakeobject *self, PyObject *args) { |
| 741 | |
| 742 | if (!PyArg_ParseTuple(args, ":finish")) |
| 743 | return NULL; |
| 744 | |
| 745 | if (cdb_make_finish(&self->cm) == -1) |
| 746 | return CDBMAKEerr; |
| 747 | |
| 748 | /* cleanup as in cdb dist's cdbmake */ |
| 749 | |
| 750 | if (fsync(fileno(self->cm.fp)) == -1) |
| 751 | return CDBMAKEerr; |
| 752 | |
| 753 | if (fclose(self->cm.fp) != 0) |
| 754 | return CDBMAKEerr; |
| 755 | |
| 756 | self->cm.fp = NULL; |
| 757 | |
| 758 | if (rename(PyString_AsString(self->fntmp), |
| 759 | PyString_AsString(self->fn)) == -1) |
| 760 | return CDBMAKEerr; |
| 761 | |
| 762 | return Py_BuildValue(""); |
| 763 | } |
| 764 | |
| 765 | static PyMethodDef cdbmake_methods[] = { |
| 766 | {"add", (PyCFunction)CdbMake_add, METH_VARARGS, |
| 767 | "cm.add(key, data) -> None\n\ |
| 768 | \n\ |
| 769 | Add 'key' -> 'data' pair to the underlying CDB." }, |
| 770 | {"finish", (PyCFunction)CdbMake_finish, METH_VARARGS, |
| 771 | "cm.finish() -> None\n\ |
| 772 | \n\ |
| 773 | Finish safely composing a new CDB, renaming cm.fntmp to\n\ |
| 774 | cm.fn." }, |
| 775 | { NULL, NULL } |
| 776 | }; |
| 777 | |
| 778 | /* ----------------- cdbmake operations ------------------ */ |
| 779 | |
| 780 | static PyObject * |
| 781 | new_cdbmake(PyObject *ignore, PyObject *args) { |
| 782 | |
| 783 | cdbmakeobject *self; |
| 784 | PyObject *fn, *fntmp; |
| 785 | FILE * f; |
| 786 | |
| 787 | if (! PyArg_ParseTuple(args, "SS|i", &fn, &fntmp)) |
| 788 | return NULL; |
| 789 | |
| 790 | f = fopen(PyString_AsString(fntmp), "w+b"); |
| 791 | if (f == NULL) { |
| 792 | return CDBMAKEerr; |
| 793 | } |
| 794 | |
| 795 | self = PyObject_NEW(cdbmakeobject, &CdbMakeType); |
| 796 | if (self == NULL) return NULL; |
| 797 | |
| 798 | self->fn = fn; |
| 799 | Py_INCREF(self->fn); |
| 800 | |
| 801 | self->fntmp = fntmp; |
| 802 | Py_INCREF(fntmp); |
| 803 | |
| 804 | if (cdb_make_start(&self->cm, f) == -1) { |
| 805 | Py_DECREF(self); |
| 806 | CDBMAKEerr; |
| 807 | return NULL; |
| 808 | } |
| 809 | |
| 810 | return (PyObject *) self; |
| 811 | } |
| 812 | |
| 813 | static void |
| 814 | cdbmake_dealloc(cdbmakeobject *self) { |
| 815 | |
| 816 | Py_XDECREF(self->fn); |
| 817 | |
| 818 | if (self->fntmp != NULL) { |
| 819 | if (self->cm.fp != NULL) { |
| 820 | fclose(self->cm.fp); |
| 821 | unlink(PyString_AsString(self->fntmp)); |
| 822 | } |
| 823 | Py_DECREF(self->fntmp); |
| 824 | } |
| 825 | |
| 826 | PyMem_DEL(self); |
| 827 | } |
| 828 | |
| 829 | static PyObject * |
| 830 | cdbmake_getattr(cdbmakeobject *self, char *name) { |
| 831 | |
| 832 | if (!strcmp(name,"__members__")) |
| 833 | return Py_BuildValue("[ssss]", "fd", "fn", "fntmp", "numentries"); |
| 834 | |
| 835 | if (!strcmp(name,"fd")) |
| 836 | return Py_BuildValue("i", fileno(self->cm.fp)); /* self.fd */ |
| 837 | |
| 838 | if (!strcmp(name,"fn")) { |
| 839 | Py_INCREF(self->fn); |
| 840 | return self->fn; /* self.fn */ |
| 841 | } |
| 842 | |
| 843 | if (!strcmp(name,"fntmp")) { |
| 844 | Py_INCREF(self->fntmp); |
| 845 | return self->fntmp; /* self.fntmp */ |
| 846 | } |
| 847 | |
| 848 | if (!strcmp(name,"numentries")) |
| 849 | return Py_BuildValue("l", self->cm.numentries); /* self.numentries */ |
| 850 | |
| 851 | return Py_FindMethod(cdbmake_methods, (PyObject *) self, name); |
| 852 | } |
| 853 | |
| 854 | /* ---------------- Type delineation -------------------- */ |
| 855 | |
| 856 | statichere PyTypeObject CdbType = { |
| 857 | /* The ob_type field must be initialized in the module init function |
| 858 | * to be portable to Windows without using C++. */ |
| 859 | PyObject_HEAD_INIT(NULL) |
| 860 | 0, /*ob_size*/ |
| 861 | "cdb", /*tp_name*/ |
| 862 | sizeof(CdbObject), /*tp_basicsize*/ |
| 863 | 0, /*tp_itemsize*/ |
| 864 | /* methods */ |
| 865 | (destructor)cdbo_dealloc, /*tp_dealloc*/ |
| 866 | 0, /*tp_print*/ |
| 867 | (getattrfunc)cdbo_getattr, /*tp_getattr*/ |
| 868 | 0, /*tp_setattr*/ |
| 869 | 0, /*tp_compare*/ |
| 870 | 0, /*tp_repr*/ |
| 871 | 0, /*tp_as_number*/ |
| 872 | 0, /*tp_as_sequence*/ |
| 873 | &cdbo_as_mapping, /*tp_as_mapping*/ |
| 874 | 0, /*tp_hash*/ |
| 875 | 0, /*tp_call*/ |
| 876 | 0, /*tp_str*/ |
| 877 | 0, /*tp_getattro*/ |
| 878 | 0, /*tp_setattro*/ |
| 879 | 0, /*tp_as_buffer*/ |
| 880 | 0, /*tp_xxx4*/ |
| 881 | cdbo_object_doc, /*tp_doc*/ |
| 882 | }; |
| 883 | |
| 884 | statichere PyTypeObject CdbMakeType = { |
| 885 | /* The ob_type field must be initialized in the module init function |
| 886 | * to be portable to Windows without using C++. */ |
| 887 | PyObject_HEAD_INIT(NULL) |
| 888 | 0, /*ob_size*/ |
| 889 | "cdbmake", /*tp_name*/ |
| 890 | sizeof(cdbmakeobject), /*tp_basicsize*/ |
| 891 | 0, /*tp_itemsize*/ |
| 892 | /* methods */ |
| 893 | (destructor)cdbmake_dealloc, /*tp_dealloc*/ |
| 894 | 0, /*tp_print*/ |
| 895 | (getattrfunc)cdbmake_getattr, /*tp_getattr*/ |
| 896 | 0, /*tp_setattr*/ |
| 897 | 0, /*tp_compare*/ |
| 898 | 0, /*tp_repr*/ |
| 899 | 0, /*tp_as_number*/ |
| 900 | 0, /*tp_as_sequence*/ |
| 901 | 0, /*tp_as_mapping*/ |
| 902 | 0, /*tp_hash*/ |
| 903 | 0, /*tp_call*/ |
| 904 | 0, /*tp_str*/ |
| 905 | 0, /*tp_getattro*/ |
| 906 | 0, /*tp_setattro*/ |
| 907 | 0, /*tp_as_buffer*/ |
| 908 | 0, /*tp_xxx4*/ |
| 909 | cdbmake_object_doc, /*tp_doc*/ |
| 910 | }; |
| 911 | |
| 912 | /* ---------------- exported functions ------------------ */ |
| 913 | static PyObject * |
| 914 | _wrap_cdb_hash(PyObject *ignore, PyObject *args) { |
| 915 | |
| 916 | char *s; |
| 917 | int sz; |
| 918 | |
| 919 | if (! PyArg_ParseTuple(args, "s#:hash", &s, &sz)) |
| 920 | return NULL; |
| 921 | |
| 922 | return Py_BuildValue("l", cdb_hash(s, (unsigned int) sz)); |
| 923 | |
| 924 | } |
| 925 | |
| 926 | /* ---------------- cdb Module -------------------- */ |
| 927 | |
| 928 | static PyMethodDef module_functions[] = { |
| 929 | {"init", cdbo_constructor, METH_VARARGS, |
| 930 | "cdb.init(f) -> cdb_object\n\ |
| 931 | \n\ |
| 932 | Open a CDB specified by f and return a cdb object.\n\ |
| 933 | f may be a filename or an integral file descriptor\n\ |
| 934 | (e.g., init( sys.stdin.fileno() )...)."}, |
| 935 | {"cdbmake", new_cdbmake, METH_VARARGS, |
| 936 | "cdb.cdbmake(cdb, tmp) -> cdbmake_object\n\ |
| 937 | \n\ |
| 938 | Interface to the creation of a new CDB file \"cdb\".\n\ |
| 939 | \n\ |
| 940 | The cdbmake object first writes records to the temporary file\n\ |
| 941 | \"tmp\" (records are inserted via the object's add() method).\n\ |
| 942 | The finish() method then atomically renames \"tmp\" to \"cdb\",\n\ |
| 943 | ensuring that readers of \"cdb\" need never wait for updates to\n\ |
| 944 | complete." |
| 945 | }, |
| 946 | {"hash", _wrap_cdb_hash, METH_VARARGS, |
| 947 | "hash(s) -> hashval\n\ |
| 948 | \n\ |
| 949 | Compute the 32-bit hash value of some sequence of bytes s."}, |
| 950 | {NULL, NULL} |
| 951 | }; |
| 952 | |
| 953 | static char module_doc[] = |
| 954 | "Python adaptation of D. J. Bernstein's constant database (CDB)\n\ |
| 955 | package. See <http://cr.yp.to/cdb.html>\n\ |
| 956 | \n\ |
| 957 | CDB objects, created by init(), provide read-only, dict-like\n\ |
| 958 | access to cdb files, as well as iterative methods.\n\ |
| 959 | \n\ |
| 960 | CDBMake objects, created by cdbmake(), allow for creation and\n\ |
| 961 | atomic replacement of CDBs.\n\ |
| 962 | \n\ |
| 963 | This module defines a new Exception \"error\"."; |
| 964 | |
| 965 | DL_EXPORT(void) |
| 966 | initcdb() { |
| 967 | PyObject *m, *d, *v; |
| 968 | |
| 969 | CdbType.ob_type = &PyType_Type; |
| 970 | CdbMakeType.ob_type = &PyType_Type; |
| 971 | |
| 972 | m = Py_InitModule3("cdb", module_functions, module_doc); |
| 973 | |
| 974 | d = PyModule_GetDict(m); |
| 975 | |
| 976 | CDBError = PyErr_NewException("cdb.error", NULL, NULL); |
| 977 | PyDict_SetItemString(d, "error", CDBError); |
| 978 | |
| 979 | PyDict_SetItemString(d, "__version__", |
| 980 | v = PyString_FromString(VERSION)); |
| 981 | PyDict_SetItemString(d, "__cdb_version__", |
| 982 | v = PyString_FromString(CDBVERSION)); |
| 983 | Py_XDECREF(v); |
| 984 | |
| 985 | } |