3 ### Miscellaneous utilities
5 ### (c) 2013 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This file is part of Chopwood: a password-changing service.
12 ### Chopwood is free software; you can redistribute it and/or modify
13 ### it under the terms of the GNU Affero General Public License as
14 ### published by the Free Software Foundation; either version 3 of the
15 ### License, or (at your option) any later version.
17 ### Chopwood is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ### GNU Affero General Public License for more details.
22 ### You should have received a copy of the GNU Affero General Public
23 ### License along with Chopwood; if not, see
24 ### <http://www.gnu.org/licenses/>.
26 from __future__
import with_statement
29 import contextlib
as CTX
37 try: import threading
as TH
38 except ImportError: import dummy_threading
as TH
40 ###--------------------------------------------------------------------------
44 """The identity function: returns its argument."""
48 """The function which always returns X."""
51 class struct (object):
52 """A simple object for storing data in attributes."""
54 def __init__(me
, *args
, **kw
):
56 for k
, v
in kw
.iteritems(): setattr(me
, k
, v
)
59 except AttributeError:
60 if args
: raise ValueError, 'no slots defined'
62 if len(args
) > len(slots
): raise ValueError, 'too many arguments'
63 for k
, v
in zip(slots
, args
): setattr(me
, k
, v
)
65 if hasattr(me
, k
): continue
66 try: setattr(me
, k
, cls
.DEFAULTS
[k
])
67 except KeyError: raise ValueError, "no value for `%s'" % k
70 """An object whose only purpose is to be distinct from other objects."""
71 def __init__(me
, name
):
74 return '#<%s %r>' %
(type(me
).__name__
, me
._name
)
76 class DictExpanderClass (type):
78 Metaclass for classes with autogenerated members.
80 If the class body defines a dictionary `__extra__' then the key/value pairs
81 in this dictionary are promoted into attributes of the class. This is much
82 easier -- and safer -- than fiddling about with `locals'.
84 def __new__(cls
, name
, supers
, dict):
86 ex
= dict['__extra__']
90 for k
, v
in ex
.iteritems():
93 return super(DictExpanderClass
, cls
).__new__(cls
, name
, supers
, dict)
95 class ExpectedError (Exception):
97 A (concrete) base class for various errors we expect to encounter.
99 The `msg' attribute carries a human-readable message explaining what the
100 problem actually is. The really important bit, though, is the `code'
101 attribute, which carries an HTTP status code to be reported to the user
102 agent, if we're running through CGI.
104 def __init__(me
, code
, msg
):
108 return '%s (%d)' %
(me
.msg
, me
.code
)
110 def register(dict, name
):
111 """A decorator: add the decorated function to DICT, under the key NAME."""
117 class StringSubst (object):
119 A string substitution. Initialize with a dictionary mapping source strings
120 to target strings. The object is callable, and maps strings in the obvious
123 def __init__(me
, map):
125 me
._rx
= RX
.compile('|'.join(RX
.escape(s
) for s
in map))
127 return me
._rx
.sub(lambda m
: me
._map
[m
.group(0)], s
)
129 def readline(what
, file = SYS
.stdin
):
130 """Read a single line from FILE (default stdin) and return it."""
131 try: line
= SYS
.stdin
.readline()
132 except IOError, e
: raise ExpectedError
, (500, str(e
))
133 if not line
.endswith('\n'):
134 raise ExpectedError
, (500, "Failed to read %s" % what
)
137 class EscapeHatch (BaseException
):
138 """Exception used by the `Escape' context manager"""
139 def __init__(me
): pass
141 class Escape (object):
143 A context manager. Executes its body until completion or the `Escape'
144 object itself is invoked as a function. Other exceptions propagate
148 me
.exc
= EscapeHatch()
153 def __exit__(me
, exty
, exval
, extb
):
154 return exval
is me
.exc
156 class Fluid (object):
158 Stores `fluid' variables which can be temporarily bound to new values, and
161 A caller may use the object's attributes for storing arbitrary values
162 (though storing a `bind' value would be silly). The `bind' method provides
163 a context manager which binds attributes to other values during its
164 execution. This works even with multiple threads.
167 ## We maintain two stores for variables. One is a global store, `_g'; the
168 ## other is a thread-local store `_t'. We look for a variable first in the
169 ## thread-local store, and then if necessary in the global store. Binding
170 ## works by remembering the old state of the variable on entry, setting it
171 ## in the thread-local store (always), and then restoring the old state on
174 ## A special marker for unbound variables. If a variable is bound to a
175 ## value, rebound temporarily with `bind', and then deleted, we must
176 ## pretend that it's not there, and then restore it again afterwards. We
177 ## use this tag to mark variables which have been deleted while they're
179 UNBOUND
= Tag('unbound-variable')
181 def __init__(me
, **kw
):
182 """Create a new set of fluid variables, initialized from the keywords."""
183 me
.__dict__
.update(_g
= struct(),
185 for k
, v
in kw
.iteritems():
188 def __getattr__(me
, k
):
189 """Return the current value stored with K, or raise AttributeError."""
190 try: v
= getattr(me
._t
, k
)
191 except AttributeError: v
= getattr(me
._g
, k
)
192 if v
is Fluid
.UNBOUND
: raise AttributeError, k
195 def __setattr__(me
, k
, v
):
196 """Associate the value V with the variable K."""
197 if hasattr(me
._t
, k
): setattr(me
._t
, k
, v
)
198 else: setattr(me
._g
, k
, v
)
200 def __delattr__(me
, k
):
202 Forget about the variable K, so that attempts to read it result in an
205 if hasattr(me
._t
, k
): setattr(me
._t
, k
, Fluid
.UNBOUND
)
206 else: delattr(me
._g
, k
)
209 """Return a list of the currently known variables."""
212 for s
in [me
._t
, me
._g
]:
214 if k
in seen
: continue
216 if getattr(s
, k
) is not Fluid
.UNBOUND
: keys
.append(k
)
222 A context manager: bind values to variables according to the keywords KW,
223 and execute the body; when the body exits, restore the rebound variables
224 to their previous values.
227 ## A list of things to do when we finish.
231 ## Remove K from the thread-local store. Only it might already have
232 ## been deleted, so be careful.
233 try: delattr(me
._t
, k
)
234 except AttributeError: pass
237 ## Stash a function for restoring the old state of K. We do this here
238 ## rather than inline only because Python's scoping rules are crazy and
239 ## we need to ensure that all of the necessary variables are
241 try: ov
= getattr(me
._t
, k
)
242 except AttributeError: unwind
.append(lambda: _delattr(k
))
243 else: unwind
.append(lambda: setattr(me
._t
, k
, ov
))
245 ## Rebind the variables.
246 for k
, v
in kw
.iteritems():
250 ## Run the body, and restore.
255 class Cleanup (object):
257 A context manager for stacking other context managers.
259 By itself, it does nothing. Attach other context managers with `enter' or
260 loose cleanup functions with `add'. On exit, contexts are left and
261 cleanups performed in reverse order.
267 def __exit__(me
, exty
, exval
, extb
):
269 for c
in reversed(me
._cleanups
):
270 if c(exty
, exval
, extb
): trap
= True
274 me
._cleanups
.append(ctx
.__exit__
)
277 me
._cleanups
.append(lambda exty
, exval
, extb
: func())
279 ###--------------------------------------------------------------------------
282 class Encoding (object):
284 A pairing of injective encoding on binary strings, with its appropriate
287 The two functions are available in the `encode' and `decode' attributes.
288 See also the `ENCODINGS' dictionary.
290 def __init__(me
, encode
, decode
):
295 'base64': Encoding(lambda s
: BN
.b64encode(s
),
296 lambda s
: BN
.b64decode(s
)),
297 'base32': Encoding(lambda s
: BN
.b32encode(s
).lower(),
298 lambda s
: BN
.b32decode(s
, casefold
= True)),
299 'hex': Encoding(lambda s
: BN
.b16encode(s
).lower(),
300 lambda s
: BN
.b16decode(s
, casefold
= True)),
301 None: Encoding(identity
, identity
)
304 ###--------------------------------------------------------------------------
305 ### Time and timeouts.
309 Reset our idea of the current time, as kept in the global variable `NOW'.
315 class Alarm (Exception):
317 Exception used internally by the `timeout' context manager.
319 If you're very unlucky, you might get one of these at top level.
323 class Timeout (ExpectedError
):
325 Report a timeout, from the `timeout' context manager.
327 def __init__(me
, what
):
328 ExpectedError
.__init__(me
, 500, "Timeout %s" % what
)
330 ## Set `DEADLINE' to be the absolute time of the next alarm. We'll keep this
331 ## up to date in `timeout'.
332 delta
, _
= SIG
.getitimer(SIG
.ITIMER_REAL
)
333 if delta
== 0: DEADLINE
= None
334 else: DEADLINE
= NOW
+ delta
337 """If we receive `SIGALRM', raise the alarm."""
339 SIG
.signal(SIG
.SIGALRM
, _alarm
)
342 def timeout(delta
, what
):
344 A context manager which interrupts execution of its body after DELTA
345 seconds, if it doesn't finish before then.
347 If execution is interrupted, a `Timeout' exception is raised, carrying WHY
348 (a gerund phrase) as part of its message.
353 if DEADLINE
is not None and when
>= DEADLINE
:
360 SIG
.setitimer(SIG
.ITIMER_REAL
, delta
)
367 if od
is None: SIG
.setitimer(SIG
.ITIMER_REAL
, 0)
368 else: SIG
.setitimer(SIG
.ITIMER_REAL
, DEADLINE
- NOW
)
370 ###--------------------------------------------------------------------------
374 def lockfile(lock
, t
= None):
376 Acquire an exclusive lock on a named file LOCK while executing the body.
378 If T is zero, fail immediately if the lock can't be acquired; if T is none,
379 then wait forever if necessary; otherwise give up after T seconds.
383 fd
= OS
.open(lock
, OS
.O_WRONLY | OS
.O_CREAT
, 0600)
385 F
.lockf(fd
, F
.LOCK_EX
)
387 F
.lockf(fd
, F
.LOCK_EX | F
.LOCK_NB
)
389 with
timeout(t
, "waiting for lock file `%s'" % lock
):
390 F
.lockf(fd
, F
.LOCK_EX
)
393 if fd
!= -1: OS
.close(fd
)
395 ###--------------------------------------------------------------------------
396 ### Database utilities.
398 ### Python's database API is dreadful: it exposes far too many
399 ### implementation-specific details to the programmer, who may well want to
400 ### write code which works against many different databases.
402 ### One particularly frustrating problem is the variability of placeholder
403 ### syntax in SQL statements: there's no universal convention, just a number
404 ### of possible syntaxes, at least one of which will be implemented (and some
405 ### of which are mutually incompatible). Because not doing this invites all
406 ### sorts of misery such as SQL injection vulnerabilties, we introduce a
407 ### simple abstraction. A database parameter-type object keeps track of one
408 ### particular convention, providing the correct placeholders to be inserted
409 ### into the SQL command string, and the corresponding arguments, in whatever
410 ### way is necessary.
412 ### The protocol is fairly simple. An object of the appropriate class is
413 ### instantiated for each SQL statement, providing it with a dictionary
414 ### mapping placeholder names to their values. The object's `sub' method is
415 ### called for each placeholder found in the statement, with a match object
416 ### as an argument; the match object picks out the name of the placeholder in
417 ### question in group 1, and the method returns a piece of syntax appropriate
418 ### to the database backend. Finally, the collected arguments are made
419 ### available, in whatever format is required, in the object's `args'
422 ## Turn simple Unix not-quite-glob patterns into SQL `LIKE' patterns.
423 ## Match using: x LIKE y ESCAPE '\\'
424 globtolike
= StringSubst({
425 '\\*': '*', '%': '\\%', '*': '%',
426 '\\?': '?', '_': '\\_', '?': '_'
429 class LinearParam (object):
431 Abstract parent class for `linear' parameter conventions.
433 A linear convention is one where the arguments are supplied as a list, and
434 placeholders are either all identical (with semantics `insert the next
435 argument'), or identify their argument by its position within the list.
437 def __init__(me
, kw
):
442 name
= match
.group(1)
443 me
.args
.append(me
._kw
[name
])
444 marker
= me
._format()
447 class QmarkParam (LinearParam
):
448 def _format(me
): return '?'
449 class NumericParam (LinearParam
):
450 def _format(me
): return ':%d' % me
._i
451 class FormatParam (LinearParam
):
452 def _format(me
): return '%s'
454 class DictParam (object):
456 Abstract parent class for `dictionary' parameter conventions.
458 A dictionary convention is one where the arguments are provided as a
459 dictionary, and placeholders contain a key name identifying the
460 corresponding value in that dictionary.
462 def __init__(me
, kw
):
465 name
= match
.group(1)
466 return me
._format(name
)
467 def NamedParam (object):
468 def _format(me
, name
): return ':%s' % name
469 def PyFormatParam (object):
470 def _format(me
, name
): return '%%(%s)s' % name
472 ### Since we're doing a bunch of work to paper over idiosyncratic placeholder
473 ### syntax, we might as well also sort out other problems. The `DB_FIXUPS'
474 ### dictionary maps database module names to functions which might need to do
475 ### clever stuff at connection setup time.
479 @register(DB_FIXUPS
, 'sqlite3')
480 def fixup_sqlite3(db
):
482 Unfortunately, SQLite learnt about FOREIGN KEY constraints late, and so
483 doesn't enforce them unless explicitly told to.
486 c
.execute("PRAGMA foreign_keys = ON")
488 class SimpleDBConnection (object):
490 Represents a database connection, while trying to hide the differences
491 between various kinds of database backends.
494 __metaclass__
= DictExpanderClass
496 ## A map from placeholder convention names to classes implementing them.
499 'numeric': NumericParam
,
501 'format': FormatParam
,
502 'pyformat': PyFormatParam
505 ## A pattern for our own placeholder syntax.
506 R_PLACE
= RX
.compile(r
'\$(\w+)')
508 def __init__(me
, modname
, modargs
):
510 Make a new database connection, using the module MODNAME, and passing its
511 `connect' function the MODARGS -- which may be either a list or a
515 ## Get the module, and create a connection.
516 mod
= __import__(modname
)
517 if isinstance(modargs
, dict): me
._db
= mod
.connect(**modargs
)
518 else: me
._db
= mod
.connect(*modargs
)
520 ## Apply any necessary fixups.
521 try: fixup
= DB_FIXUPS
[modname
]
522 except KeyError: pass
525 ## Grab hold of other interesting things.
527 me
.Warning = mod
.Warning
528 me
._placecls
= me
.PLACECLS
[mod
.paramstyle
]
530 def execute(me
, command
, **kw
):
532 Execute the SQL COMMAND. The keyword arguments are used to provide
533 values corresponding to `$NAME' placeholders in the COMMAND.
535 Return the receiver, so that iterator protocol is convenient.
537 me
._cur
= me
._db
.cursor()
538 plc
= me
._placecls(kw
)
539 subst
= me
.R_PLACE
.sub(plc
.sub
, command
)
540 ##PRINT('*** %s : %r' % (subst, plc.args))
541 me
._cur
.execute(subst
, plc
.args
)
545 """Iterator protocol: simply return the receiver."""
548 """Iterator protocol: return the next row from the current query."""
550 if row
is None: raise StopIteration
554 """Context protocol: begin a transaction."""
557 def __exit__(me
, exty
, exval
, tb
):
558 """Context protocol: commit or roll back a transaction."""
560 ##PRINT('>*> ROLLBACK')
563 ##PRINT('>>> COMMIT')
566 ## Import a number of methods from the underlying connection.
568 for _name
in ['fetchone', 'fetchmany', 'fetchall']:
570 extra
[name
] = lambda me
, *args
, **kw
: \
571 getattr(me
._cur
, name
)(*args
, **kw
)
573 for _name
in ['commit', 'rollback']:
575 extra
[name
] = lambda me
, *args
, **kw
: \
576 getattr(me
._db
, name
)(*args
, **kw
)
580 ###----- That's all, folks --------------------------------------------------