From efb0bf0ea231f3c53c4ac9c68405d511ec582ccb Mon Sep 17 00:00:00 2001 From: Mark Wooding Date: Sat, 5 Oct 2019 23:41:24 +0100 Subject: [PATCH] @@@ cython and python3 --- .gitignore | 1 + assoc.pyx | 35 ++++++-------- atom-base.c | 40 +++++++++------- crc32.pyx | 21 ++++---- debian/compat | 2 +- debian/control | 4 +- defs.pxi | 20 ++++---- grim.h | 98 +++++++++++++++++++++++++++++++++++++ mLib.pyx | 56 ++++++++++++---------- mapping.pyx | 149 ++++++++++++++++++++++++++++----------------------------- setup.py | 40 +++++++++++----- sym.pyx | 33 +++++++------ test.py | 70 +++++++++++++++++++++++++++ unihash.pyx | 54 +++++++++++++-------- utils.pyx | 52 ++++++++------------ 15 files changed, 435 insertions(+), 240 deletions(-) create mode 100644 test.py diff --git a/.gitignore b/.gitignore index fd89164..ca7ed0f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build MANIFEST dist mLib.c +mLib-py*.c COPYING mdwsetup.py *.pyc diff --git a/assoc.pyx b/assoc.pyx index 7685dd6..01f04de 100644 --- a/assoc.pyx +++ b/assoc.pyx @@ -27,43 +27,38 @@ cdef struct _assoc_entry: sym_base _b PyObject *v -cdef class AssocTable (Mapping): +cdef class AssocTable (_Mapping): cdef assoc_table _t - cdef int _init(me) except -1: + def __cinit__(me): assoc_create(&me._t) + cdef int _init(me) except -1: return 0 cdef void *_find(me, object key, unsigned *f) except NULL: - cdef void *p - cdef Py_ssize_t n - cdef _assoc_entry *e - cdef atom *a - a = ATOM_A(atom_pyintern(key)) - PyObject_AsReadBuffer(key, &p, &n) - if f: - f[0] = 0 - e = <_assoc_entry *>assoc_find(&me._t, a, PSIZEOF(e), f) + cdef _assoc_entry *e = NULL + cdef object a = atom_pyintern(key) + if not f: + e = <_assoc_entry *>assoc_find(&me._t, ATOM_A(a), 0, NULL) + if not e: + raise KeyError(a) + else: + e = <_assoc_entry *>assoc_find(&me._t, ATOM_A(a), PSIZEOF(e), f) if not f[0]: e.v = NULL - else: - e = <_assoc_entry *>assoc_find(&me._t, a, 0, NULL) return e cdef object _key(me, void *e): return atom_pywrap(ASSOC_ATOM(e)) cdef object _value(me, void *e): - cdef _assoc_entry *ee - ee = <_assoc_entry *>e + cdef _assoc_entry *ee = <_assoc_entry *>e Py_INCREF(ee.v) return ee.v cdef void _setval(me, void *e, object val): - cdef _assoc_entry *ee - ee = <_assoc_entry *>e + cdef _assoc_entry *ee = <_assoc_entry *>e if ee.v: Py_DECREF(ee.v) - ee.v = v + ee.v = val Py_INCREF(ee.v) cdef void _del(me, void *e): - cdef _assoc_entry *ee - ee = <_assoc_entry *>e + cdef _assoc_entry *ee = <_assoc_entry *>e if ee.v: Py_DECREF(ee.v) assoc_remove(&me._t, ee) diff --git a/atom-base.c b/atom-base.c index f57eaee..1078e2e 100644 --- a/atom-base.c +++ b/atom-base.c @@ -50,15 +50,20 @@ static assoc_table obarray; PyObject *atom_pywrap(atom *a) { - entry *e; + entry *e = 0; + atom_pyobj *ao; unsigned f = 0; e = assoc_find(&obarray, a, sizeof(entry), &f); if (!f) { - atom_pyobj *ao = PyObject_NEW(atom_pyobj, &atom_pytype); - ao->a = a; + ao = PyObject_NEW(atom_pyobj, &atom_pytype); ao->a = a; e->a = (PyObject *)ao; } + + /* If we just created the new `Atom' object, we get a reference, which + * belongs to the `obarray' table. So when we return it to the caller, we + * always need to increment it. + */ RETURN_OBJ(e->a); } @@ -72,8 +77,10 @@ PyObject *atom_pyintern(PyObject *x) RETURN_OBJ(x); if (x == Py_None) a = atom_gensym(ATOM_GLOBAL); + else if (!TEXT_CHECK(x)) + { PyErr_SetString(PyExc_TypeError, "string wanted"); return (0); } else { - if (PyObject_AsReadBuffer(x, &p, &n)) return (0); + TEXT_PTRLEN(x, &p, &n); if (!p) return (0); a = atom_nintern(ATOM_GLOBAL, p, n); } return (atom_pywrap(a)); @@ -84,19 +91,16 @@ static void atom_pydealloc(PyObject *me) static PyObject *atom_pynew(PyTypeObject *ty, PyObject *arg, PyObject *kw) { - PyObject *name; - static char *kwlist[] = { "name", 0 }; + PyObject *name = Py_None; + static const char *const kwlist[] = { "name", 0 }; - if (!PyArg_ParseTupleAndKeywords(arg, kw, "O:new", kwlist, &name)) + if (!PyArg_ParseTupleAndKeywords(arg, kw, "|O:new", KWLIST, &name)) return (0); return (atom_pyintern(name)); } static PyObject *aget_name(PyObject *me, void *hunoz) -{ - return (PyString_FromStringAndSize(ATOM_NAME(ATOM_A(me)), - ATOM_LEN(ATOM_A(me)))); -} + { return (TEXT_FROMSTRLEN(ATOM_NAME(ATOM_A(me)), ATOM_LEN(ATOM_A(me)))); } static PyObject *aget_internedp(PyObject *me, void *hunoz) { @@ -135,13 +139,15 @@ static PyObject *atom_pyrepr(PyObject *me) dstr d = DSTR_INIT; if ((s = aget_name(me, 0)) == 0 || - (sr = PyObject_Repr(s)) == 0 || - PyString_AsStringAndSize(sr, &p, &n)) + (sr = PyObject_Repr(s)) == 0) goto done; + if (!TEXT_CHECK(sr)) + { PyErr_SetString(PyExc_TypeError, "string wanted"); goto done; } + TEXT_PTRLEN(sr, &p, &n); if (!p) goto done; dstr_puts(&d, "Atom("); dstr_putm(&d, p, n); dstr_puts(&d, ")"); - rc = PyString_FromStringAndSize(d.buf, d.len); + rc = TEXT_FROMSTRLEN(d.buf, d.len); done: Py_XDECREF(s); Py_XDECREF(sr); @@ -149,11 +155,11 @@ done: return (rc); } -static long atom_pyhash(PyObject *me) - { long h = ATOM_HASH(ATOM_A(me)); if (h == -1) h = -2; return (h); } +static Py_hash_t atom_pyhash(PyObject *me) + { Py_hash_t h = ATOM_HASH(ATOM_A(me)); if (h == -1) h = -2; return (h); } PyTypeObject atom_pytype = { - PyObject_HEAD_INIT(0) 0, /* Header */ + PyVarObject_HEAD_INIT(0, 0) /* Header */ "mLib.Atom", /* @tp_name@ */ sizeof(atom_pyobj), /* @tp_basicsize@ */ 0, /* @tp_itemsize@ */ diff --git a/crc32.pyx b/crc32.pyx index f4072a0..30db336 100644 --- a/crc32.pyx +++ b/crc32.pyx @@ -26,28 +26,27 @@ cdef class CRC32: """CRC32(): calculate CRC32 of a stream""" cdef uint32 _a - def __cinit__(me, *hunoz, **hukairz): + def __cinit__(me): me._a = 0 def __init__(me): pass - def chunk(me, data): + def chunk(me, object data): """C.chunk(STR): process another chunk of input""" - cdef void *p + cdef const void *p cdef Py_ssize_t n - PyObject_AsReadBuffer(data, &p, &n) - me._a = c_crc32(me._a, p, n) + PyObject_AsReadBuffer(data, &p, &n) + me._a = _crc32(me._a, p, n) return me def done(me): """C.done() -> INT: return CRC of data""" - return _u32(me._a) + return me._a -def crc32(data): +def crc32(object data): """crc32(STR) -> INT""" - cdef void *p + cdef const void *p cdef Py_ssize_t n cdef uint32 c - PyObject_AsReadBuffer(data, &p, &n) - c = c_crc32(0, p, n) - return _u32(c) + PyObject_AsReadBuffer(data, &p, &n) + return _crc32(0, p, n) ###----- That's all, folks -------------------------------------------------- diff --git a/debian/compat b/debian/compat index ec63514..f599e28 100644 --- a/debian/compat +++ b/debian/compat @@ -1 +1 @@ -9 +10 diff --git a/debian/control b/debian/control index 335e071..4ee1960 100644 --- a/debian/control +++ b/debian/control @@ -2,9 +2,9 @@ Source: mlib-python Section: python Priority: extra XS-Python-Version: >= 2.6, << 2.8 -Build-Depends: debhelper (>= 9), dh-python, pkg-config, +Build-Depends: debhelper (>= 10), dh-python, pkg-config, python (>= 2.6.6-3~), python-all-dev, python-pyrex, - mlib-dev (>= 2.2.2.1) + mlib-dev (>= 2.4.99~) Maintainer: Mark Wooding Standards-Version: 3.8.0 diff --git a/defs.pxi b/defs.pxi index b14b855..1ed4cb7 100644 --- a/defs.pxi +++ b/defs.pxi @@ -95,18 +95,18 @@ cdef extern from 'Python.h': ctypedef struct PyTypeObject: pass - object PyString_FromStringAndSize(char *p, Py_ssize_t len) - int PyString_AsStringAndSize(obj, char **p, Py_ssize_t *len) except -1 int PyObject_AsReadBuffer(obj, void **buf, Py_ssize_t *len) except -1 object PyInt_FromLong(long i) object PyLong_FromUnsignedLong(unsigned long i) - char *PyString_AS_STRING(string) - int _PyString_Resize(PyObject **string, int size) except -1 void PyErr_Clear() void Py_INCREF(PyObject *obj) void Py_DECREF(PyObject *obj) + PyTypeObject *Py_TYPE(PyObject *obj) + Py_ssize_t Py_SIZE(PyObject *obj) + Py_ssize_t Py_REFCNT(PyObject *obj) + ###-------------------------------------------------------------------------- ### mLib basic stuff. @@ -129,7 +129,7 @@ cdef extern from 'mLib/dstr.h': ### CRC32. cdef extern from 'mLib/crc32.h': - uint32 c_crc32 "crc32" (uint32 a, void *p, int sz) + uint32 _crc32 "crc32" (uint32 a, void *p, int sz) ###-------------------------------------------------------------------------- ### Universal hashing. @@ -174,7 +174,7 @@ cdef extern from 'mLib/str.h': void str_sanitize(char *d, char *p, size_t sz) cdef extern from 'mLib/versioncmp.h': - int _versioncmp "versioncmp"(char *va, char *vb) + int _versioncmp "versioncmp" (char *va, char *vb) ###-------------------------------------------------------------------------- ### Form-urlencoding functions. @@ -211,9 +211,9 @@ cdef extern from 'atom.h': void atom_mkiter(atom_iter *i, atom_table *t) atom *atom_next(atom_iter *) void atom_pysetup() - atom_pywrap(atom *a) - atom_pyintern(obj) - atom *ATOM_A(obj) + object atom_pywrap(atom *a) + object atom_pyintern(object obj) + atom *ATOM_A(object obj) PyTypeObject atom_pytype ## Association tables. @@ -439,6 +439,8 @@ cdef extern from 'mLib/daemonize.h': cdef extern from 'grim.h': int PSIZEOF(void *x) + void TEXT_PTRLEN(str s, const char **p, Py_ssize_t *sz) + object TEXT_FROMSTRLEN(const char *p, Py_ssize_t sz) ctypedef void *cvp ###----- That's all, folks -------------------------------------------------- diff --git a/grim.h b/grim.h index 85ea681..016e2ef 100644 --- a/grim.h +++ b/grim.h @@ -64,6 +64,104 @@ typedef const void *cvp; #define FREEOBJ(obj) \ (((PyObject *)(obj))->ob_type->tp_free((PyObject *)(obj))) +#define KWLIST ((char **)kwlist) + +/*----- Compatibility hacks -----------------------------------------------*/ +#if PY_VERSION_HEX >= 0x03000000 + +/*#define PyInt_Check PyLong_Check*/ +#define PyInt_FromLong PyLong_FromLong +#define PyInt_AS_LONG PyLong_AS_LONG +#define PyInt_AsLong PyLong_AsLong +#define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask +#define PyNumber_Int PyNumber_Long + +#define BIN_CHECK(obj) PyBytes_Check(obj) +#define BIN_PTR(obj) PyBytes_AS_STRING(obj) +#define BIN_LEN(obj) PyBytes_GET_SIZE(obj) +#define BIN_FROMSTR(str) PyBytes_FromString(str) +#define BIN_FORMAT PyBytes_FromFormat +#define BIN_SETLEN(obj, len) do Py_SIZE(obj) = (len); while (0) + +#define TEXT_CHECK(obj) PyUnicode_Check(obj) +#if PY_VERSION_HEX >= 0x03030000 +# define TEXT_PTR(obj) PyUnicode_AsUTF8(obj) +# define TEXT_STR(obj) PyUnicode_AsUTF8(obj) +# define TEXT_PTRLEN(obj, ptr, len) do { \ + Py_ssize_t len_; \ + *(ptr) = PyUnicode_AsUTF8AndSize((obj), &len_); \ + *(len) = len_; \ +} while (0) +# define TEXT_PREPAREWRITE(obj, ptr, sz) do { \ + (obj) = PyUnicode_New((sz), 127); \ + (ptr) = PyUnicode_DATA(obj); \ +} while (0) +# define TEXT_DONEWRITE(obj, len) do { \ + size_t len_ = (len); \ + assert(PyUnicode_IS_COMPACT_ASCII(obj)); \ + ((char *)PyUnicode_DATA(obj))[len_] = 0; \ + ((PyASCIIObject *)(obj))->length = len_; \ +} while (0) +#else +# define TEXT_PTR(obj) _PyUnicode_AsString(obj) +# define TEXT_STR(obj) _PyUnicode_AsString(obj) +# define TEXT_PTRLEN(obj, ptr, len) do { \ + Py_ssize_t len_; \ + *(ptr) = _PyUnicode_AsStringAndSize((obj), &len_); \ + *(len) = len_; \ +} while (0) +# define TEXT_PREPAREWRITE(obj, ptr, sz) do { \ + (obj) = PyBytes_FromStringAndSize(0, (sz)); \ + (ptr) = PyBytes_AS_STRING(obj); \ +} while (0) +# define TEXT_DONEWRITE(obj, len) do { \ + PyObject *new_ = PyUnicode_FromEncodedObject(obj, 0, 0); \ + assert(new_); Py_DECREF(obj); (obj) = new_; \ +} while (0) +#endif +#define TEXT_FORMAT PyUnicode_FromFormat +#define TEXT_FROMSTR(str) PyUnicode_FromString(str) +#define TEXT_FROMSTRLEN(str, len) PyUnicode_FromStringAndSize(str, len) + +#define Py_TPFLAGS_CHECKTYPES 0 + +#else + +#define BIN_CHECK(obj) PyString_Check(obj) +#define BIN_PTR(obj) PyString_AS_STRING(obj) +#define BIN_LEN(obj) PyString_GET_SIZE(obj) +#define BIN_FROMSTR(str) PyString_FromString(str) +#define BIN_FORMAT PyString_FromFormat +#define BIN_SETLEN(obj, len) do Py_SIZE(obj) = (len); while (0) + +#define TEXT_TYPE PyString_Type +#define TEXT_CHECK(obj) PyString_Check(obj) +#define TEXT_PTR(obj) PyString_AS_STRING(obj) +#define TEXT_STR(obj) PyString_AsString(obj) +#define TEXT_PTRLEN(obj, ptr, len) do { \ + *(ptr) = PyString_AS_STRING(obj); \ + *(len) = PyString_GET_SIZE(obj); \ +} while (0) +#define TEXT_FORMAT PyString_FromFormat +#define TEXT_PREPAREWRITE(obj, ptr, sz) do { \ + (obj) = PyString_FromStringAndSize(0, (sz)); \ + (ptr) = PyString_AS_STRING(obj); \ +} while (0) +#define TEXT_DONEWRITE(obj, len) \ + do _PyString_Resize(&(obj), (len)); while (0) +#define TEXT_FROMSTR(str) PyString_FromString(str) +#define TEXT_FROMSTRLEN(str, len) PyString_FromStringAndSize(str, len) + +#endif + +#ifndef PyVarObject_HEAD_INIT +# define PyVarObject_HEAD_INIT(super, sz) PyObject_HEAD_INIT(super) sz, +#endif + +#if PY_VERSION_HEX < 0x03020000 + typedef long Py_hash_t; +#endif + /*----- That's all, folks -------------------------------------------------*/ #ifdef __cplusplus diff --git a/mLib.pyx b/mLib.pyx index daa6cd9..bae8e30 100644 --- a/mLib.pyx +++ b/mLib.pyx @@ -29,6 +29,12 @@ include 'defs.pxi' ###-------------------------------------------------------------------------- +### Testing stuff. + +def refcount(object obj): + return Py_REFCNT(obj) + +###-------------------------------------------------------------------------- ### Various facilities. ## Internal utilities. @@ -45,40 +51,40 @@ include 'atom.pyx' include 'assoc.pyx' ## String utilities. -include 'str.pyx' +#include 'str.pyx' ## Encodings. -include 'codec.pyx' -include 'base64.pyx' -include 'base32.pyx' -include 'hex.pyx' -include 'url.pyx' +#include 'codec.pyx' +#include 'base64.pyx' +#include 'base32.pyx' +#include 'hex.pyx' +#include 'url.pyx' ## Error reporting. -include 'report.pyx' +#include 'report.pyx' ## File utilities. -include 'fwatch.pyx' -include 'fdutils.pyx' -include 'mdup.pyx' +#include 'fwatch.pyx' +#include 'fdutils.pyx' +#include 'mdup.pyx' ## Other useful stuff. -include 'stuff.pyx' +#include 'stuff.pyx' ## Buffering. -include 'lbuf.pyx' -include 'pkbuf.pyx' +#include 'lbuf.pyx' +#include 'pkbuf.pyx' ## Select stuff. -include 'sel-base.pyx' -include 'sel-file.pyx' -include 'sel-timer.pyx' -include 'conn.pyx' -include 'bres.pyx' -include 'sig.pyx' -include 'selbuf.pyx' -include 'selpk.pyx' -include 'ident.pyx' +#include 'sel-base.pyx' +#include 'sel-file.pyx' +#include 'sel-timer.pyx' +#include 'conn.pyx' +#include 'bres.pyx' +#include 'sig.pyx' +#include 'selbuf.pyx' +#include 'selpk.pyx' +#include 'ident.pyx' ###-------------------------------------------------------------------------- ### Set-up stuff. @@ -89,9 +95,9 @@ cdef object _tyobj(PyTypeObject *ty): Py_INCREF(obj) return obj -da_pysetup() -Array = _tyobj(&da_pytype) -ArrayIter = _tyobj(&daiter_pytype) +#da_pysetup() +#Array = _tyobj(&da_pytype) +#ArrayIter = _tyobj(&daiter_pytype) atom_pysetup() Atom = _tyobj(&atom_pytype) diff --git a/mapping.pyx b/mapping.pyx index b044ed6..d007a17 100644 --- a/mapping.pyx +++ b/mapping.pyx @@ -23,19 +23,17 @@ ### along with mLib/Python; if not, write to the Free Software Foundation, ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -cdef class Mapping - cdef class _MapIterator: cdef void *_next(me): return NULL -cdef class Mapping: +cdef class _Mapping: - ## Subclasses must implement these + ## Subclasses must implement these. cdef int _init(me) except -1: - raise TypeError, 'abstract class' + raise TypeError('abstract class') cdef void *_find(me, object key, unsigned *f) except NULL: - raise SystemError, 'unimplemented _find' + raise SystemError('unimplemented _find') cdef object _key(me, void *e): return None cdef object _value(me, void *e): @@ -45,15 +43,14 @@ cdef class Mapping: cdef void _del(me, void *e): pass cdef _MapIterator _iter(me): - raise SystemError, 'unimplemented _iter' + raise SystemError('unimplemented _iter') - ## Initialization - def __cinit__(me, *hunoz, **hukairz): - me._init() + ## Initialization. def __init__(me, stuff = None, **kw): - me.update(stuff, kw) + me._init() + me.update(stuff, **kw) - ## Bulk update + ## Bulk update. def update(me, stuff = None, **kw): """D.update([MAP], **KW): insert mappings from MAP and KW""" cdef unsigned f @@ -72,27 +69,22 @@ cdef class Mapping: me._setval(me._find(k, &f), v) return me - ## Item access + ## Item access. def __getitem__(me, key): - cdef void *e - e = me._find(key, NULL) - if not e: - raise KeyError, key + cdef void *e = me._find(key, NULL) return me._value(e) def __setitem__(me, key, value): cdef unsigned f me._setval(me._find(key, &f), value) def __delitem__(me, key): - cdef void *e cdef unsigned f - e = me._find(key, &f) + cdef void *e = me._find(key, &f) if not e: - raise KeyError, key + raise KeyError(key) me._del(e) def get(me, key, default = None): """D.get(KEY, [default = None]) -> VALUE: value at KEY, or DEFAULT""" - cdef void *e - e = me._find(key, NULL) + cdef void *e = me._find(key, NULL) if not e: return default return me._value(e) @@ -100,9 +92,8 @@ cdef class Mapping: """ D.setdefault(KEY, [default = None]) -> VALUE: return value at key, or store DEFAULT at key and return that""" - cdef void *e cdef unsigned f - e = me._find(key, &f) + cdef void *e = me._find(key, &f) if f: return me._value(e) else: @@ -112,8 +103,7 @@ cdef class Mapping: """ D.pop(KEY, [default = None]) -> VALUE: return value at key or DEFAULT, and remove KEY""" - cdef void *e - e = me._find(key, NULL) + cdef void *e = me._find(key, NULL) if not e: return default rc = me._value(e) @@ -121,42 +111,39 @@ cdef class Mapping: return rc def popitem(me): """D.popitem() -> KEY, VALUE: return and remove an association pair""" - cdef _MapIterator i - cdef void *e - i = me._iter() - e = i._next() + cdef _MapIterator i = me._iter() + cdef void *e = i._next() if not e: - raise ValueError, 'popitem(): table is empty' + raise ValueError('popitem(): table is empty') return me._key(e), me._value(e) - ## Lists of items - cdef object _list(me, object (*func)(Mapping m, void *e)): - cdef _MapIterator i - cdef void *e - i = me._iter() - l = [] - while 1: - e = i._next() - if not e: - break - l.append(func(me, e)) - return l + ## Lists of items. + IF PYVERSION < (3,): + cdef object _list(me, object (*func)(_Mapping m, void *e)): + cdef _MapIterator i = me._iter() + cdef void *e + l = [] + while 1: + e = i._next() + if not e: + break + l.append(func(me, e)) + return l - def keys(me): - """D.keys() -> LIST: return a list of known keys""" - return me._list(_map_key) - def values(me): - """D.values() -> LIST: return a list of known values""" - return me._list(_map_value) - def items(me): - """D.values() -> LIST: return a list of known (KEY, VALUE) pairs""" - return me._list(_map_item) + def keys(me): + """D.keys() -> LIST: return a list of known keys""" + return me._list(_map_key) + def values(me): + """D.values() -> LIST: return a list of known values""" + return me._list(_map_value) + def items(me): + """D.items() -> LIST: return a list of known (KEY, VALUE) pairs""" + return me._list(_map_item) def clear(me): """D.clear(): remove all mappings""" - cdef _MapIterator i + cdef _MapIterator i = me._iter() cdef void *e - i = me._iter() while 1: e = i._next() if not e: @@ -164,26 +151,38 @@ cdef class Mapping: me._del(e) return me - ## Iteration + ## Iteration. def __iter__(me): return MapKeyIter(me) - def iterkeys(me): - """D.iterkeys() -> ITER: return iterator over keys""" - return MapKeyIter(me) - def itervalues(me): - """D.itervalues() -> ITER: return iterator over values""" - return MapValueIter(me) - def iteritems(me): - """D.iteritems() -> ITER: return iterator over (KEY, VALUE) pairs""" - return MapItemIter(me) + IF PYVERSION >= (3,): + def keys(me): + """D.keys() -> ITER: return iterator over keys""" + return MapKeyIter(me) + def values(me): + """D.values() -> ITER: return iterator over values""" + return MapValueIter(me) + def items(me): + """D.items() -> ITER: return iterator over (KEY, VALUE) pairs""" + return MapItemIter(me) + ELSE: + def iterkeys(me): + """D.iterkeys() -> ITER: return iterator over keys""" + return MapKeyIter(me) + def itervalues(me): + """D.itervalues() -> ITER: return iterator over values""" + return MapValueIter(me) + def iteritems(me): + """D.iteritems() -> ITER: return iterator over (KEY, VALUE) pairs""" + return MapItemIter(me) -cdef class MapIterBase: - cdef Mapping m - cdef object (*func)(Mapping m, void *e) +cdef class _MapIterBase: + cdef _Mapping m + cdef object (*func)(_Mapping m, void *e) cdef _MapIterator i cdef int _init(me) except -1: - raise TypeError, 'abstract class' - def __cinit__(me): + raise TypeError('abstract class') + def __init__(me, _Mapping m): + me.m = m me.i = m._iter() me._init() def __iter__(me): @@ -194,24 +193,24 @@ cdef class MapIterBase: if not e: raise StopIteration return me.func(me.m, e) -cdef class MapKeyIter (MapIterBase): +cdef class MapKeyIter (_MapIterBase): cdef int _init(me) except -1: me.func = _map_key return 0 -cdef class MapValueIter (MapIterBase): +cdef class MapValueIter (_MapIterBase): cdef int _init(me) except -1: me.func = _map_value return 0 -cdef class MapItemIter (MapIterBase): +cdef class MapItemIter (_MapIterBase): cdef int _init(me) except -1: me.func = _map_item return 0 -cdef object _map_key(Mapping m, void *e): +cdef object _map_key(_Mapping m, void *e): return m._key(e) -cdef object _map_value(Mapping m, void *e): +cdef object _map_value(_Mapping m, void *e): return m._value(e) -cdef object _map_item(Mapping m, void *e): +cdef object _map_item(_Mapping m, void *e): return m._key(e), m._value(e) ###----- That's all, folks -------------------------------------------------- diff --git a/setup.py b/setup.py index 66aafc5..1ec65dc 100755 --- a/setup.py +++ b/setup.py @@ -1,28 +1,46 @@ #! /usr/bin/python +import os as OS +import sys as SYS import distutils.core as DC -import Pyrex.Distutils as PXD +import Cython.Build as CB import mdwsetup as MS -MS.pkg_config('mLib', '2.1.0') +MS.pkg_config('mLib', '2.4.99~') -mLib = DC.Extension('mLib', ['mLib.pyx', 'atom-base.c', 'array.c'], +PYVERSION = SYS.version_info[0:3] +pyxc = 'mLib-py%d.%d.%d.c' % PYVERSION +mLib = DC.Extension('mLib', [pyxc, + 'atom-base.c', + ##'array.c' + ], ##extra_compile_args = ['-O0'], include_dirs = MS.uniquify(MS.INCLUDEDIRS), library_dirs = MS.uniquify(MS.LIBDIRS), libraries = MS.uniquify(MS.LIBS)) +## The `cythonize' function generates the C sources immediately, so we have +## to generate its inputs even earlier. +genfiles = [ +# MS.Derive('base64.pyx', 'codec.pyx.in', +# {'CLASS': 'Base64', 'PREFIX': 'base64'}), +# MS.Derive('base32.pyx', 'codec.pyx.in', +# {'CLASS': 'Base32', 'PREFIX': 'base32'}), +# MS.Derive('hex.pyx', 'codec.pyx.in', +# {'CLASS': 'Hex', 'PREFIX': 'hex'}) +] +for g in genfiles: g.gen() + +## Generate the main C code. +if OS.path.exists(pyxc): OS.rename(pyxc, "mLib.c") +CB.cythonize("mLib.pyx", compile_time_env = dict(PYVERSION = PYVERSION)) +OS.rename("mLib.c", pyxc) + MS.setup(name = 'mLib-python', description = 'Python interface to mLib utilities library', author = 'Straylight/Edgeware', author_email = 'mdw@distorted.org.uk', license = 'GNU General Public License', ext_modules = [mLib], - genfiles = [MS.Derive('base64.pyx', 'codec.pyx.in', - {'CLASS': 'Base64', 'PREFIX': 'base64'}), - MS.Derive('base32.pyx', 'codec.pyx.in', - {'CLASS': 'Base32', 'PREFIX': 'base32'}), - MS.Derive('hex.pyx', 'codec.pyx.in', - {'CLASS': 'Hex', 'PREFIX': 'hex'})], - cleanfiles = ['mLib.c'], - cmdclass = { 'build_ext': PXD.build_ext }) + genfiles = genfiles, + cleanfiles = [pyxc]) diff --git a/sym.pyx b/sym.pyx index f456db8..a41fad8 100644 --- a/sym.pyx +++ b/sym.pyx @@ -27,46 +27,45 @@ cdef struct _sym_entry: sym_base _b PyObject *v -cdef class SymTable (Mapping): +cdef class SymTable (_Mapping): """ SymTable([DICT], **KW) A mapping keyed by strings. """ cdef sym_table _t - cdef int _init(me) except -1: + def __cinit__(me): sym_create(&me._t) + cdef int _init(me) except -1: return 0 cdef void *_find(me, object key, unsigned *f) except NULL: - cdef void *p + cdef const char *p cdef Py_ssize_t n - cdef _sym_entry *e - PyObject_AsReadBuffer(key, &p, &n) - if f: - f[0] = 0 + cdef _sym_entry *e = NULL + TEXT_PTRLEN(key, &p, &n) + if not f: + e = <_sym_entry *>sym_find(&me._t, p, n, 0, NULL) + if not e: + raise KeyError(key) + else: e = <_sym_entry *>sym_find(&me._t, p, n, PSIZEOF(e), f) if not f[0]: e.v = NULL - else: - e = <_sym_entry *>sym_find(&me._t, p, n, 0, NULL) return e cdef object _key(me, void *e): - return PyString_FromStringAndSize(SYM_NAME(e), SYM_LEN(e)) + return TEXT_FROMSTRLEN(SYM_NAME(e), SYM_LEN(e)) cdef object _value(me, void *e): - cdef _sym_entry *ee - ee = <_sym_entry *>e + cdef _sym_entry *ee = <_sym_entry *>e Py_INCREF(ee.v) return ee.v cdef void _setval(me, void *e, object val): - cdef _sym_entry *ee - ee = <_sym_entry *>e + cdef _sym_entry *ee = <_sym_entry *>e if ee.v: Py_DECREF(ee.v) - ee.v = v + ee.v = val Py_INCREF(ee.v) cdef void _del(me, void *e): - cdef _sym_entry *ee - ee = <_sym_entry *>e + cdef _sym_entry *ee = <_sym_entry *>e if ee.v: Py_DECREF(ee.v) sym_remove(&me._t, ee) diff --git a/test.py b/test.py new file mode 100644 index 0000000..33f781e --- /dev/null +++ b/test.py @@ -0,0 +1,70 @@ +#! /usr/bin/python +### -*- coding: utf-8 -*- + +import sys as SYS +import mLib as M + +if SYS.version_info >= (3,): + def _bin(text): return text.encode() + def _text(bin): return bin.decode() +else: + def _bin(text): return text + def _text(bin): return bin + +def must_equal(x, y): + if x == y: pass + else: raise AssertionError("%r != %r" % (x, y)) + +## Done! +print(";; test begins (Python %s)" % SYS.version) + +## crc32 +must_equal(M.crc32(_bin("abc")), 0x352441c2) +must_equal(M.CRC32().chunk(_bin("a")).chunk(_bin("bc")).done(), 0x352441c2) + +## unihash +assert M.Unihash().key is None +must_equal(M.Unihash.hash(_bin("abc")), 0xbf71f6a2) +must_equal(M.Unihash().chunk(_bin("a")).chunk(_bin("bc")).done(), 0xbf71f6a2) +key = M.UnihashKey(0x8498a262) +assert M.Unihash(key).key is key +must_equal(M.Unihash.hash(_bin("abc"), key), 0xecd1e2a2) +must_equal(key.hash(_bin("abc")), 0xecd1e2a2) +must_equal(M.Unihash(key).chunk(_bin("a")).chunk(_bin("bc")).done(), 0xecd1e2a2) +M.setglobalkey(0x8498a262) +must_equal(M.Unihash.hash(_bin("abc")), 0xecd1e2a2) + +## atom +foo = M.Atom("foo") +bar = M.Atom("bär") +assert foo != bar +assert foo is M.Atom("foo") +assert bar is M.Atom("bär") +assert foo.internedp +must_equal(foo.name, "foo") +must_equal(bar.name, "bär") +gen = M.Atom() +assert gen is not M.Atom() +assert not gen.internedp +all = set(M.atoms()) +assert foo in all +assert bar in all + +## assoc, sym +def test_mapping(mapcls, keya, keyz): + tab = mapcls() + tab[keya] = 69; must_equal(tab[keya], 69) + tab[keya] = 42; must_equal(tab[keya], 42) + tab[keyz] = 'zing'; must_equal(tab[keyz], 'zing') + del tab[keyz] + try: tab[keyz] + except KeyError: pass + else: assert False + must_equal(list(tab.keys()), [keya]) + must_equal(list(tab.values()), [42]) + must_equal(list(tab.items()), [(keya, 42)]) +test_mapping(M.AssocTable, foo, bar) +test_mapping(M.SymTable, 'foo', 'bar') + +## Done! +print(";; test completed OK") diff --git a/unihash.pyx b/unihash.pyx index 8ededb0..ba38090 100644 --- a/unihash.pyx +++ b/unihash.pyx @@ -27,40 +27,54 @@ def setglobalkey(uint32 k): """setglobalkey(K): set global hash key""" unihash_setkey(&unihash_global, k) -cdef class Key: +cdef class UnihashKey: """Key(K): universal hashing key""" cdef unihash_info _i - cdef uint32 _k + cdef readonly uint32 k def __cinit__(me, uint32 k): unihash_setkey(&me._i, k) - me._k = k - property k: - """K.k -> INT: the key value""" - def __get__(me): - return _u32(me._k) + me.k = k + def hash(me, object data): + return _unihash_hash(me, data) + +## DEPRECATED: compatibility hack +Key = UnihashKey + +cdef const unihash_info *_unihash_keydata(UnihashKey key): + if key is None: + return &unihash_global + else: + return &key._i + +cdef uint32 _unihash_hash(UnihashKey key, object data): + cdef const void *p + cdef Py_ssize_t n + cdef const unihash_info *i = _unihash_keydata(key) + PyObject_AsReadBuffer(data, &p, &n) + return unihash_hash(i, UNIHASH_INIT(i), p, n) cdef class Unihash: """Unihash([key = None]): universal hashing context""" cdef uint32 _a - cdef readonly Key key - cdef unihash_info *_i - def __cinit__(me, key = None): - cdef Key k + cdef readonly UnihashKey key + cdef const unihash_info *_i + def __init__(me, UnihashKey key = None): me.key = key - if key is None: - me._i = &unihash_global - else: - k = key - me._i = &k._i + me._i = _unihash_keydata(key) me._a = UNIHASH_INIT(me._i) def chunk(me, data): - """U.chunk(STR): hash the STR""" - cdef void *p + """U.chunk(BYTES): hash the STR""" + cdef const void *p cdef Py_ssize_t n - PyObject_AsReadBuffer(data, &p, &n) + PyObject_AsReadBuffer(data, &p, &n) me._a = unihash_hash(me._i, me._a, p, n) + return me def done(me): """U.done() -> INT: the hash of the data""" - return _u32(me._a) + return me._a + @classmethod + def hash(cls, data, UnihashKey key = None): + """Unihash.hash(BYTES, [key = None]) -> INT: the hash of the data""" + return _unihash_hash(key, data) ###----- That's all, folks -------------------------------------------------- diff --git a/utils.pyx b/utils.pyx index 8d4ffad..17fa416 100644 --- a/utils.pyx +++ b/utils.pyx @@ -23,37 +23,25 @@ ### along with mLib/Python; if not, write to the Free Software Foundation, ### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -cdef object _u32(uint32 x): - if x <= LONG_MAX: - return PyInt_FromLong(x) - else: - return PyLong_FromUnsignedLong(x) - -cdef object _oserror(): - raise OSError, (errno, strerror(errno)) - -cdef object _tobool(int i): - if i: - return True - else: - return False - -cdef int _getfd(object fdobj): - try: - fd = int(fdobj) - except TypeError: - PyErr_Clear() - fd = fdobj.fileno() - return fd - -cdef object _checkcallable(f, what): - if f is not None and not callable(f): - raise TypeError, '%s must be callable' % what - return f - -cdef object _maybecall(f, args): - if f is None: - return None - return f(*args) +#cdef object _oserror(): +# raise OSError, (errno, strerror(errno)) + +#cdef int _getfd(object fdobj): +# try: +# fd = int(fdobj) +# except TypeError: +# ##PyErr_Clear() +# fd = fdobj.fileno() +# return fd + +#cdef object _checkcallable(object f, object what): +# if f is not None and not callable(f): +# raise TypeError('%s must be callable' % what) +# return f + +#cdef object _maybecall(object f, object args): +# if f is None: +# return None +# return f(*args) ###----- That's all, folks -------------------------------------------------- -- 2.11.0