Source code for pysteg.sql.queue

#! /usr/bin/env python
## -*- coding: utf-8 -*-
## (C) 2012: Hans Georg Schaathun <georg@schaathun.net> 

"""
This module defines Queue class and associated SQL table to maintain
the job queue.  All the necessary functionality is provided by methods.
"""

print "[pysteg.sql.queue]"

# TODO: Consider threading to parallellise I/O

from .tables import *
from datetime import datetime
from sqlobject import *

[docs]class Queue(SQLObject): """Table to record pending jobs. Each entry concerns one image and one or more feature sets. A worker node should use a transaction to select one item where assigned is null, and then set this field with the current date and time before the transaction is released. Three modes: 1. image set/svmodel=None for normal feature calculation 2. image=None/svmodel set/testset=None for SVM training 3. image=None/svmodel and testset set for SVM testing """ entered = DateTimeCol( ) assigned = DateTimeCol( default=None ) assignee = StringCol( default=None ) features = SQLRelatedJoin( "FeatureSet" ) image = ForeignKey( "Image" ) svmodel = ForeignKey( "SVModel", default=None ) testset = ForeignKey( "TestSet", default=None ) @classmethod
[docs] def addToImage( cls, img, fset ): """Add a new image with one or more feature sets to the queue. """ stdconn = sqlhub.getConnection() trans = stdconn.transaction() sqlhub.threadConnection = trans try: S = cls.selectBy( image=img(), assigned=None ) N = S.count() if N == 1: item = S[0] print "Added to existing Queue item:", item else: item = Queue( image=img(), entered=datetime.now() ) print "New Queue item:", item if hasattr( fset, "__iter__" ): for f in fset: item.addTo( f ) else: item.addTo( fset ) except: trans.rollback() raise else: trans.commit( close=True ) return item finally: sqlhub.threadConnection = stdconn
[docs] def addTo( self, fset ): """Add feature sets to the queue item.""" if not isinstance( fset, SQLObject ): fset = FeatureSet.selectBy( key=fset ).getOne() return self.addFeatureSet( fset )
@classmethod
[docs] def getJob( cls, worker=None, SVM=True, verbosity=0 ): """Get a job from the queue. Transactions are used to make this safe to concurrency. If SVM is false, only feature extraction tasks will be accepted. This is useful if some compute nodes are used without access to the filesystem holding SVM model files. """ conn = cls._connection trans = conn.transaction() if SVM: sqlcond = cls.q.assigned == None else: sqlcond = AND(cls.q.assigned == None, cls.q.svmodel == None) taskselect = cls.select( sqlcond, forUpdate=True, connection=trans ).orderBy( "entered" ) while True: task = list( taskselect.limit(1) ) if task == []: N = cls.select( sqlcond ).count() # N = taskselect.count() if N > 0: print "[Queue] Lock collision; reattempting" continue else: return None else: task = task[0] break id = task.id task.assigned = datetime.now() task.assignee = worker trans.commit( close=True ) return cls.get(id)
[docs] def destroy( self, force=False ): """Delete the job. Unless force is True, an assigned job will not be deleted. Normally, :meth:`releaseJob` is used to release and delete a processed job. This is not safe; a transaction should be used to lock the record while deleting.""" if not force and self.assigned != None: return False return self.destroySelf()
[docs] def releaseJob( self, worker=None, success=True ): "Notify the Queue that the job has been completed." self.destroySelf()