@@ -171,52 +171,11 @@ def runSomething(message):
171171 subp = subprocess .Popen (cmd .split (), stdout = subprocess .PIPE , stderr = subprocess .STDOUT )
172172 monitorAndLog (subp ,logger )
173173
174- localOut = f"/workspace/backend/{ message ['batch' ]} /{ message ['group' ]['plate' ]} "
175-
176174 # Figure out a done condition - a number of files being created, a particular file being created, an exit code, etc.
177175 # If done, get the outputs and move them to S3
178176
179- done = True
180-
181- if done :
182- time .sleep (30 )
183- mvtries = 0
184- while mvtries < 3 :
185- try :
186- printandlog ('Move attempt #' + str (mvtries + 1 ),logger )
187- cmd = 'aws s3 mv ' + localOut + ' s3://' + AWS_BUCKET + '/' + remoteOut + ' --recursive'
188- subp = subprocess .Popen (cmd .split (), stdout = subprocess .PIPE , stderr = subprocess .PIPE )
189- out ,err = subp .communicate ()
190- out = out .decode ()
191- err = err .decode ()
192- printandlog ('== OUT \n ' + out , logger )
193- if err == '' :
194- break
195- else :
196- printandlog ('== ERR \n ' + err ,logger )
197- mvtries += 1
198- except :
199- printandlog ('Move failed' ,logger )
200- printandlog ('== ERR \n ' + err ,logger )
201- time .sleep (30 )
202- mvtries += 1
203- if mvtries < 3 :
204- printandlog ('SUCCESS' ,logger )
205- logger .removeHandler (watchtowerlogger )
206- return 'SUCCESS'
207- else :
208- printandlog ('SYNC PROBLEM. Giving up on trying to sync ' + metadataID ,logger )
209- import shutil
210- shutil .rmtree (localOut , ignore_errors = True )
211- logger .removeHandler (watchtowerlogger )
212- return 'PROBLEM'
213- else :
214- printandlog ('PROBLEM: Failed exit condition for ' + metadataID ,logger )
215- logger .removeHandler (watchtowerlogger )
216- import shutil
217- shutil .rmtree (localOut , ignore_errors = True )
218- return 'PROBLEM'
219-
177+ printandlog ('DONE' ,logger )
178+ return 'SUCCESS'
220179
221180#################################
222181# MAIN WORKER LOOP
@@ -225,19 +184,17 @@ def runSomething(message):
225184def main ():
226185 queue = JobQueue (QUEUE_URL )
227186 # Main loop. Keep reading messages while they are available in SQS
228- while True :
229- msg , handle = queue .readMessage ()
230- if msg is not None :
231- result = runSomething (msg )
232- if result == 'SUCCESS' :
233- print ('Batch completed successfully.' )
234- queue .deleteMessage (handle )
235- else :
236- print ('Returning message to the queue.' )
237- queue .returnMessage (handle )
187+ msg , handle = queue .readMessage ()
188+ if msg is not None :
189+ result = runSomething (msg )
190+ if result == 'SUCCESS' :
191+ print ('Batch completed successfully.' )
192+ queue .deleteMessage (handle )
238193 else :
239- print ('No messages in the queue' )
240- break
194+ print ('Returning message to the queue.' )
195+ queue .returnMessage (handle )
196+ else :
197+ print ('No messages in the queue' )
241198
242199#################################
243200# MODULE ENTRY POINT
0 commit comments