details: http://www.bx.psu.edu/hg/galaxy/rev/941266327bc3 changeset: 1704:941266327bc3 user: Nate Coraor <nate@bx.psu.edu> date: Wed Jan 14 10:49:14 2009 -0500 description: Improvements to job recovery and clarification of the false path's job id 2 file(s) affected in this change: lib/galaxy/jobs/__init__.py lib/galaxy/tools/__init__.py diffs (79 lines): diff -r c259ce9cb50d -r 941266327bc3 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Wed Jan 14 08:50:31 2009 -0500 +++ b/lib/galaxy/jobs/__init__.py Wed Jan 14 10:49:14 2009 -0500 @@ -111,19 +111,19 @@ def __check_jobs_at_startup( self ): """ - Checks all jobs that are in the 'running' or 'queued' state in the - database and requeues or cleans up as necessary. Only run as the + Checks all jobs that are in the 'new', 'queued' or 'running' state in + the database and requeues or cleans up as necessary. Only run as the job manager starts. """ model = self.app.model - # Jobs in the NEW state won't be requeued unless we're tracking in the database - if not self.track_jobs_in_database: - for job in model.Job.filter( model.Job.c.state==model.Job.states.NEW ).all(): - log.debug( "no runner: %s is still in new state, adding to the jobs queue" %job.id ) + for job in model.Job.filter( model.Job.c.state==model.Job.states.NEW ).all(): + log.debug( "no runner: %s is still in new state, adding to the jobs queue" %job.id ) + self.queue.put( ( job.id, job.tool_id ) ) + for job in model.Job.filter( (model.Job.c.state == model.Job.states.RUNNING) | (model.Job.c.state == model.Job.states.QUEUED) ).all(): + if job.job_runner_name is None: + log.debug( "no runner: %s is still in queued state, adding to the jobs queue" %job.id ) self.queue.put( ( job.id, job.tool_id ) ) - for job in model.Job.filter( (model.Job.c.state == model.Job.states.RUNNING) | (model.Job.c.state == model.Job.states.QUEUED) ).all(): - if job.job_runner_name is not None: - # why are we passing the queue to the wrapper? + else: job_wrapper = JobWrapper( job, self.app.toolbox.tools_by_id[ job.tool_id ], self ) self.dispatcher.recover( job, job_wrapper ) @@ -383,12 +383,12 @@ if not job.state == model.Job.states.DELETED: for dataset_assoc in job.output_datasets: if self.app.config.outputs_to_working_directory: - false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.id ) ) - if os.path.exists( false_path ): + false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.dataset.id ) ) + try: shutil.move( false_path, dataset_assoc.dataset.file_name ) log.debug( "fail(): Moved %s to %s" % ( false_path, dataset_assoc.dataset.file_name ) ) - else: - log.warning( "fail(): Missing output file in working directory: %s" % false_path ) + except ( IOError, OSError ), e: + log.error( "fail(): Missing output file in working directory: %s" % e ) dataset = dataset_assoc.dataset dataset.refresh() dataset.state = dataset.states.ERROR @@ -453,12 +453,13 @@ job.state = 'ok' for dataset_assoc in job.output_datasets: if self.app.config.outputs_to_working_directory: - false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.id ) ) - if os.path.exists( false_path ): + false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % dataset_assoc.dataset.dataset.id ) ) + try: shutil.move( false_path, dataset_assoc.dataset.file_name ) log.debug( "finish(): Moved %s to %s" % ( false_path, dataset_assoc.dataset.file_name ) ) - else: - log.warning( "finish(): Missing output file in working directory: %s" % false_path ) + except ( IOError, OSError ): + self.fail( "The job's output dataset(s) could not be read" ) + return for dataset in dataset_assoc.dataset.dataset.history_associations: #need to update all associated output hdas, i.e. history was shared with job running dataset.blurb = 'done' dataset.peek = 'no peek' diff -r c259ce9cb50d -r 941266327bc3 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Wed Jan 14 08:50:31 2009 -0500 +++ b/lib/galaxy/tools/__init__.py Wed Jan 14 10:49:14 2009 -0500 @@ -1098,7 +1098,7 @@ for name, data in output_datasets.items(): # Write outputs to the working directory (for security purposes) if desired. if self.app.config.outputs_to_working_directory and working_directory is not None: - false_path = os.path.abspath( os.path.join( working_directory, "galaxy_dataset_%d.dat" % data.id ) ) + false_path = os.path.abspath( os.path.join( working_directory, "galaxy_dataset_%d.dat" % data.dataset.id ) ) param_dict[name] = DatasetFilenameWrapper( data, false_path = false_path ) open( false_path, 'w' ).close() else: