Source code for pysteg.sql.queue
#! /usr/bin/env python
## -*- coding: utf-8 -*-
## (C) 2012: Hans Georg Schaathun <georg@schaathun.net>
"""
This module defines Queue class and associated SQL table to maintain
the job queue. All the necessary functionality is provided by methods.
"""
print "[pysteg.sql.queue]"
# TODO: Consider threading to parallellise I/O
from .tables import *
from datetime import datetime
from sqlobject import *
[docs]class Queue(SQLObject):
"""Table to record pending jobs. Each entry concerns one image
and one or more feature sets. A worker node should use a transaction
to select one item where assigned is null, and then set this field
with the current date and time before the transaction is released.
Three modes:
1. image set/svmodel=None for normal feature calculation
2. image=None/svmodel set/testset=None for SVM training
3. image=None/svmodel and testset set for SVM testing
"""
entered = DateTimeCol( )
assigned = DateTimeCol( default=None )
assignee = StringCol( default=None )
features = SQLRelatedJoin( "FeatureSet" )
image = ForeignKey( "Image" )
svmodel = ForeignKey( "SVModel", default=None )
testset = ForeignKey( "TestSet", default=None )
@classmethod
[docs] def addToImage( cls, img, fset ):
"""Add a new image with one or more feature sets to the queue.
"""
stdconn = sqlhub.getConnection()
trans = stdconn.transaction()
sqlhub.threadConnection = trans
try:
S = cls.selectBy( image=img(), assigned=None )
N = S.count()
if N == 1:
item = S[0]
print "Added to existing Queue item:", item
else:
item = Queue( image=img(), entered=datetime.now() )
print "New Queue item:", item
if hasattr( fset, "__iter__" ):
for f in fset: item.addTo( f )
else: item.addTo( fset )
except:
trans.rollback()
raise
else:
trans.commit( close=True )
return item
finally:
sqlhub.threadConnection = stdconn
[docs] def addTo( self, fset ):
"""Add feature sets to the queue item."""
if not isinstance( fset, SQLObject ):
fset = FeatureSet.selectBy( key=fset ).getOne()
return self.addFeatureSet( fset )
@classmethod
[docs] def getJob( cls, worker=None, SVM=True, verbosity=0 ):
"""Get a job from the queue. Transactions are used to make
this safe to concurrency.
If SVM is false, only feature extraction tasks will be accepted.
This is useful if some compute nodes are used without access to
the filesystem holding SVM model files.
"""
conn = cls._connection
trans = conn.transaction()
if SVM:
sqlcond = cls.q.assigned == None
else:
sqlcond = AND(cls.q.assigned == None, cls.q.svmodel == None)
taskselect = cls.select( sqlcond,
forUpdate=True, connection=trans ).orderBy( "entered" )
while True:
task = list( taskselect.limit(1) )
if task == []:
N = cls.select( sqlcond ).count()
# N = taskselect.count()
if N > 0:
print "[Queue] Lock collision; reattempting"
continue
else:
return None
else:
task = task[0]
break
id = task.id
task.assigned = datetime.now()
task.assignee = worker
trans.commit( close=True )
return cls.get(id)
[docs] def destroy( self, force=False ):
"""Delete the job. Unless force is True, an assigned job will
not be deleted. Normally, :meth:`releaseJob` is used to release
and delete a processed job. This is not safe; a transaction should
be used to lock the record while deleting."""
if not force and self.assigned != None:
return False
return self.destroySelf()
[docs] def releaseJob( self, worker=None, success=True ):
"Notify the Queue that the job has been completed."
self.destroySelf()