Source code for pysteg.sql._queue

#! /usr/bin/env python
## -*- coding: utf-8 -*-
## (C) 2012: Hans Georg Schaathun <georg@schaathun.net> 

"""
This module defines Queue class and associated SQL table to maintain
the job queue.  All the necessary functionality is provided by methods.

All the necessary objects are exposed by the main package, so this 
module should not normally be explicitly imported.
"""

print "[pysteg.sql._queue]"

from .tables import *
import datetime as dt
from sqlobject import *
import sqlobject.sqlbuilder as sqlb

__all__ =[ "Queue" ]

[docs]class Queue(SQLObject): """Table to record pending jobs. Each entry concerns one image and one or more feature sets. A worker node should use a transaction to select one item where assigned is null, and then set this field with the current date and time before the transaction is released. Three modes: 1. image set/svmodel=None for normal feature calculation 2. image=None/svmodel set/testset=None for SVM training 3. image=None/svmodel and testset set for SVM testing """ entered = DateTimeCol( ) assigned = DateTimeCol( default=None ) assignee = StringCol( default=None ) features = SQLRelatedJoin( "FeatureSet", createRelatedTable=True ) image = ForeignKey( "Image" ) svmodel = ForeignKey( "SVModel", default=None ) testset = ForeignKey( "TestSet", default=None ) @classmethod
[docs] def addToImage( cls, img, fset, verbosity=2 ): """Add a new image with one or more feature sets to the queue. :Parameters: img : an :class:`Image` or :class:`TestImage` object fset : an iterator of :class:`FeatureSet` objects """ stdconn = sqlhub.getConnection() trans = stdconn.transaction() sqlhub.threadConnection = trans image = img() if image == None: print img raise ValueError, ( "Needs image for feature extraction. Received None.") try: S = cls.selectBy( image=image, assigned=None ) N = S.count() if N == 1: item = S[0] print "Added to existing Queue item:", item else: item = Queue( image=image, entered=dt.datetime.now() ) print "New Queue item:", item if hasattr( fset, "__iter__" ): item.addListTo( fset ) else: item.addTo( fset ) except: trans.rollback() raise else: trans.commit( close=True ) return item finally: sqlhub.threadConnection = stdconn
[docs] def addListTo( self, L ): """Add a list (or any other iterator) of feature sets to the queue item. The elements must be FeatureSet objects (neither id-s nor keys are acceptable).""" R = [ { "queue_id" : self.id, "feature_set_id" : fs.id } for fs in L ] ins = sqlb.Insert( "feature_set_queue", valueList=R ) query = self._connection.sqlrepr(ins) self._connection.query( query )
[docs] def addTo( self, fset ): """Add feature sets to the queue item.""" if not isinstance( fset, SQLObject ): try: fset = FeatureSet.selectBy( key=fset ).getOne() except: print "Exception", fset raise return self.addFeatureSet( fset )
@classmethod
[docs] def getJob( cls, worker=None, SVM=True, verbosity=0, **kw ): """Get a job from the queue. Transactions are used to make this safe to concurrency. If SVM is false, only feature extraction tasks will be accepted. This is useful if some compute nodes are used without access to the filesystem holding SVM model files. """ conn = cls._connection trans = conn.transaction() if SVM: sqlcond = cls.q.assigned == None else: sqlcond = AND(cls.q.assigned == None, cls.q.svmodel == None) taskselect = cls.select( sqlcond, forUpdate=True, connection=trans ).orderBy( "entered" ) while True: task = list( taskselect.limit(1) ) if task == []: N = cls.select( sqlcond ).count() # N = taskselect.count() if N > 0: print "[Queue] Lock collision; reattempting" continue else: return None else: task = task[0] break task.assigned = dt.datetime.now() task.assignee = worker trans.commit( close=True ) return cls.get(task.id)
[docs] def destroy( self, force=False ): """Delete the job. Unless force is True, an assigned job will not be deleted. Normally, :meth:`releaseJob` is used to release and delete a processed job. This is not safe; a transaction should be used to lock the record while deleting.""" if not force and self.assigned != None: return False return self.destroySelf()
[docs] def releaseJob( self, worker=None, success=True ): "Notify the Queue that the job has been completed." self.destroySelf()