Source code for pysteg.sql._queue
#! /usr/bin/env python
## -*- coding: utf-8 -*-
## (C) 2012: Hans Georg Schaathun <georg@schaathun.net>
"""
This module defines Queue class and associated SQL table to maintain
the job queue. All the necessary functionality is provided by methods.
All the necessary objects are exposed by the main package, so this
module should not normally be explicitly imported.
"""
print "[pysteg.sql._queue]"
from .tables import *
import datetime as dt
from sqlobject import *
import sqlobject.sqlbuilder as sqlb
__all__ =[ "Queue" ]
[docs]class Queue(SQLObject):
"""Table to record pending jobs. Each entry concerns one image
and one or more feature sets. A worker node should use a transaction
to select one item where assigned is null, and then set this field
with the current date and time before the transaction is released.
Three modes:
1. image set/svmodel=None for normal feature calculation
2. image=None/svmodel set/testset=None for SVM training
3. image=None/svmodel and testset set for SVM testing
"""
entered = DateTimeCol( )
assigned = DateTimeCol( default=None )
assignee = StringCol( default=None )
features = SQLRelatedJoin( "FeatureSet", createRelatedTable=True )
image = ForeignKey( "Image" )
svmodel = ForeignKey( "SVModel", default=None )
testset = ForeignKey( "TestSet", default=None )
@classmethod
[docs] def addToImage( cls, img, fset, verbosity=2 ):
"""Add a new image with one or more feature sets to the queue.
:Parameters:
img : an :class:`Image` or :class:`TestImage` object
fset : an iterator of :class:`FeatureSet` objects
"""
stdconn = sqlhub.getConnection()
trans = stdconn.transaction()
sqlhub.threadConnection = trans
image = img()
if image == None:
print img
raise ValueError, (
"Needs image for feature extraction. Received None.")
try:
S = cls.selectBy( image=image, assigned=None )
N = S.count()
if N == 1:
item = S[0]
print "Added to existing Queue item:", item
else:
item = Queue( image=image, entered=dt.datetime.now() )
print "New Queue item:", item
if hasattr( fset, "__iter__" ):
item.addListTo( fset )
else: item.addTo( fset )
except:
trans.rollback()
raise
else:
trans.commit( close=True )
return item
finally:
sqlhub.threadConnection = stdconn
[docs] def addListTo( self, L ):
"""Add a list (or any other iterator) of feature sets to the
queue item. The elements must be FeatureSet objects
(neither id-s nor keys are acceptable)."""
R = [ { "queue_id" : self.id, "feature_set_id" : fs.id } for fs in L ]
ins = sqlb.Insert( "feature_set_queue", valueList=R )
query = self._connection.sqlrepr(ins)
self._connection.query( query )
[docs] def addTo( self, fset ):
"""Add feature sets to the queue item."""
if not isinstance( fset, SQLObject ):
try:
fset = FeatureSet.selectBy( key=fset ).getOne()
except:
print "Exception", fset
raise
return self.addFeatureSet( fset )
@classmethod
[docs] def getJob( cls, worker=None, SVM=True, verbosity=0, **kw ):
"""Get a job from the queue. Transactions are used to make
this safe to concurrency.
If SVM is false, only feature extraction tasks will be accepted.
This is useful if some compute nodes are used without access to
the filesystem holding SVM model files.
"""
conn = cls._connection
trans = conn.transaction()
if SVM:
sqlcond = cls.q.assigned == None
else:
sqlcond = AND(cls.q.assigned == None, cls.q.svmodel == None)
taskselect = cls.select( sqlcond,
forUpdate=True, connection=trans ).orderBy( "entered" )
while True:
task = list( taskselect.limit(1) )
if task == []:
N = cls.select( sqlcond ).count()
# N = taskselect.count()
if N > 0:
print "[Queue] Lock collision; reattempting"
continue
else:
return None
else:
task = task[0]
break
task.assigned = dt.datetime.now()
task.assignee = worker
trans.commit( close=True )
return cls.get(task.id)
[docs] def destroy( self, force=False ):
"""Delete the job. Unless force is True, an assigned job will
not be deleted. Normally, :meth:`releaseJob` is used to release
and delete a processed job. This is not safe; a transaction should
be used to lock the record while deleting."""
if not force and self.assigned != None:
return False
return self.destroySelf()
[docs] def releaseJob( self, worker=None, success=True ):
"Notify the Queue that the job has been completed."
self.destroySelf()