- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: dan: Fix for checking dataset access permissions when reruning a tool by job_id.
by commits-noreply＠bitbucket.org 16 May '14

16 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9fbe27547931/ Changeset: 9fbe27547931 User: dan Date: 2014-05-16 21:13:29 Summary: Fix for checking dataset access permissions when reruning a tool by job_id. Affected #: 1 file diff -r 1b06ee57191c56cd50c0248239f876f4c5311831 -r 9fbe2754793109a1680239131b5507614d431870 lib/galaxy/webapps/galaxy/controllers/tool_runner.py --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -118,6 +118,11 @@ job = trans.sa_session.query( trans.app.model.Job ).get( job_id ) except: error( "Invalid value for 'job_id' parameter" ) + if not trans.user_is_admin(): + for data_assoc in job.output_datasets: + #only allow rerunning if user is allowed access to the dataset. + if not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data_assoc.dataset.dataset ): + error( "You are not allowed to rerun this job" ) param_error_text = "Failed to get parameters for job id %d " % job_id else: if not id: @@ -130,7 +135,6 @@ id = trans.security.decode_id( id ) except: error( "Invalid value for 'id' parameter" ) - # Get the dataset object data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( id ) #only allow rerunning if user is allowed access to the dataset. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: A bit more LWR client logging.
by commits-noreply＠bitbucket.org 16 May '14

16 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1b06ee57191c/ Changeset: 1b06ee57191c User: jmchilton Date: 2014-05-16 17:24:54 Summary: A bit more LWR client logging. Affected #: 1 file diff -r 2f592b5f1883520d62fc6ceab6a1657edd188294 -r 1b06ee57191c56cd50c0248239f876f4c5311831 lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -313,7 +313,9 @@ # before queueing. setup_params = _setup_params_from_job_config(job_config) launch_params["setup_params"] = setup_params - return self.client_manager.exchange.publish("setup", launch_params) + response = self.client_manager.exchange.publish("setup", launch_params) + log.info("Job published to setup message queue.") + return response def clean(self): del self.client_manager.status_cache[self.job_id] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Update LWR client and behavior through LWR changeset 8ea7265.
by commits-noreply＠bitbucket.org 16 May '14

16 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2f592b5f1883/ Changeset: 2f592b5f1883 User: jmchilton Date: 2014-05-16 17:14:52 Summary: Update LWR client and behavior through LWR changeset 8ea7265. LWR now informed of job submission failures in MQ mode so jobs shouldn't sit in queued state forever if job fails to queue remotely. Also small fixes and more logging. Affected #: 4 files diff -r 26b9fe985ee718ddf25ee2e9e9d5096e8b315fad -r 2f592b5f1883520d62fc6ceab6a1657edd188294 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -28,6 +28,7 @@ NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "LWR misconfiguration - LWR client configured to set metadata remotely, but remote LWR isn't properly configured with a galaxy_home directory." NO_REMOTE_DATATYPES_CONFIG = "LWR client is configured to use remote datatypes configuration when setting metadata externally, but LWR is not configured with this information. Defaulting to datatypes_conf.xml." +GENERIC_REMOTE_ERROR = "Failed to communicate with remote job server." # Is there a good way to infer some default for this? Can only use # url_for from web threads. https://gist.github.com/jmchilton/9098762 @@ -266,7 +267,7 @@ if failed: job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True) except Exception: - message = "Failed to communicate with remote job server." + message = GENERIC_REMOTE_ERROR job_wrapper.fail( message, exception=True ) log.exception("failure finishing job %d" % job_wrapper.job_id) return @@ -284,7 +285,7 @@ Seperated out so we can use the worker threads for it. """ self.stop_job( self.sa_session.query( self.app.model.Job ).get( job_state.job_wrapper.job_id ) ) - job_state.job_wrapper.fail( job_state.fail_message ) + job_state.job_wrapper.fail( getattr( job_state, "fail_message", GENERIC_REMOTE_ERROR ) ) def check_pid( self, pid ): try: diff -r 26b9fe985ee718ddf25ee2e9e9d5096e8b315fad -r 2f592b5f1883520d62fc6ceab6a1657edd188294 lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -300,7 +300,7 @@ launch_params = dict(command_line=command_line, job_id=self.job_id) submit_params_dict = submit_params(self.destination_params) if submit_params_dict: - launch_params['params'] = submit_params_dict + launch_params['submit_params'] = submit_params_dict if dependencies_description: launch_params['dependencies_description'] = dependencies_description.to_dict() if env: diff -r 26b9fe985ee718ddf25ee2e9e9d5096e8b315fad -r 2f592b5f1883520d62fc6ceab6a1657edd188294 lib/galaxy/jobs/runners/lwr_client/manager.py --- a/lib/galaxy/jobs/runners/lwr_client/manager.py +++ b/lib/galaxy/jobs/runners/lwr_client/manager.py @@ -97,6 +97,7 @@ try: if "job_id" in body: self.status_cache[body["job_id"]] = body + log.debug("Handling asynchronous status update from remote LWR.") callback(body) except Exception: log.exception("Failure processing job status update message.") @@ -104,6 +105,7 @@ def run(): self.exchange.consume("status_update", callback_wrapper, check=self) + log.debug("Leaving LWR client status update thread, no additional LWR updates will be processed.") thread = threading.Thread( name="lwr_client_%s_status_update_callback" % self.manager_name, diff -r 26b9fe985ee718ddf25ee2e9e9d5096e8b315fad -r 2f592b5f1883520d62fc6ceab6a1657edd188294 lib/galaxy/jobs/runners/lwr_client/util.py --- a/lib/galaxy/jobs/runners/lwr_client/util.py +++ b/lib/galaxy/jobs/runners/lwr_client/util.py @@ -62,6 +62,7 @@ contents.append(name) return contents + def parse_amqp_connect_ssl_params(params): ssl = None rval = None Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix query filters for various Tool Shed "latest revision" grids by making sure revisions have actually been tested by the install and test framework, and not sure prepared the framework's preparation scripts.
by commits-noreply＠bitbucket.org 16 May '14

16 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/26b9fe985ee7/ Changeset: 26b9fe985ee7 User: greg Date: 2014-05-16 15:48:50 Summary: Fix query filters for various Tool Shed "latest revision" grids by making sure revisions have actually been tested by the install and test framework, and not sure prepared the framework's preparation scripts. Affected #: 1 file diff -r 7e95105c466051432e5c47ff977826a19758b867 -r 26b9fe985ee718ddf25ee2e9e9d5096e8b315fad lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -1940,8 +1940,8 @@ of type repository_suite_definition and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata \ - and repository_metadata.tool_test_results is not None \ + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ and not repository_metadata.missing_test_components \ and not repository_metadata.tools_functionally_correct \ and not repository_metadata.test_install_error: @@ -1957,7 +1957,9 @@ tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata and repository_metadata.missing_test_components: + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and repository_metadata.missing_test_components: return repository_metadata.changeset_revision return None @@ -1968,9 +1970,10 @@ and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata_if_it_includes_tools( trans, repository ) - if repository_metadata is not None and \ - not repository_metadata.missing_test_components and \ - repository_metadata.tools_functionally_correct: + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and not repository_metadata.missing_test_components \ + and repository_metadata.tools_functionally_correct: return repository_metadata.changeset_revision return None @@ -1993,7 +1996,9 @@ """ repository_metadata = get_latest_downloadable_repository_metadata_if_it_has_test_install_errors( trans, repository ) # Filter further by eliminating repositories that are missing test components. - if repository_metadata is not None and not repository_metadata.missing_test_components: + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and not repository_metadata.missing_test_components: return repository_metadata.changeset_revision return None @@ -2067,7 +2072,9 @@ well as types repository_suite_definition and tool_dependency_definition. """ repository_metadata = get_latest_downloadable_repository_metadata( trans, repository ) - if repository_metadata is not None and repository_metadata.test_install_error: + if repository_metadata is not None \ + and has_been_tested( repository_metadata ) \ + and repository_metadata.test_install_error: return repository_metadata return None @@ -2104,3 +2111,25 @@ if metadata is not None and 'invalid_tools' in metadata: return repository_metadata return None + +def has_been_tested( repository_metadata ): + """ + Return True if the received repository_metadata record'd tool_test_results column was populated by + the Tool Shed's install and test framework. + """ + tool_test_results = repository_metadata.tool_test_results + if tool_test_results is None: + return False + # The install and test framework's preparation scripts will populate the tool_test_results column + # with something like this: + # [{"test_environment": + # {"time_tested": "2014-05-15 16:15:18", + # "tool_shed_database_version": 22, + # "tool_shed_mercurial_version": "2.2.3", + # "tool_shed_revision": "13459:9a1415f8108f"} + # }] + tool_test_results = listify( tool_test_results ) + for test_results_dict in tool_test_results: + if len( test_results_dict ) > 1: + return True + return False Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 16 May '14

16 May '14

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0d81156abc40/ Changeset: 0d81156abc40 User: jmchilton Date: 2014-05-16 07:57:31 Summary: Helper method for creating JobWrappers in JobHandler. Affected #: 1 file diff -r b94bb858d29b39f5d8127414bd703ee8e4905281 -r 0d81156abc40773643f5f8d72af521fe5d1b5d9d lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -84,6 +84,13 @@ self.monitor_thread.start() log.info( "job handler queue started" ) + def job_wrapper( self, job ): + return JobWrapper( job, self ) + + def job_wrapper_for_id( self, id ): + job = self.sa_session.query( model.Job ) + return self.job_wrapper( job ) + def __check_jobs_at_startup( self ): """ Checks all jobs that are in the 'new', 'queued' or 'running' state in @@ -110,7 +117,7 @@ for job in jobs_at_startup: if job.tool_id not in self.app.toolbox.tools_by_id: log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) - JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) + self.job_wrapper( job ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) elif job.job_runner_name is not None and job.job_runner_external_id is None: # This could happen during certain revisions of Galaxy where a runner URL was persisted before the job was dispatched to a runner. log.debug( "(%s) Job runner assigned but no external ID recorded, adding to the job handler queue" % job.id ) @@ -122,7 +129,7 @@ elif job.job_runner_name is not None and job.job_runner_external_id is not None and job.destination_id is None: # This is the first start after upgrading from URLs to destinations, convert the URL to a destination and persist # TODO: test me extensively - job_wrapper = JobWrapper( job, self ) + job_wrapper = self.job_wrapper( job ) job_destination = self.dispatcher.url_to_destination(job.job_runner_name) if job_destination.id is None: job_destination.id = 'legacy_url' @@ -138,7 +145,7 @@ self.queue.put( ( job.id, job.tool_id ) ) else: # Already dispatched and running - job_wrapper = JobWrapper( job, self ) + job_wrapper = self.job_wrapper( job ) job_wrapper.job_runner_mapper.cached_job_destination = JobDestination(id=job.destination_id, runner=job.job_runner_name, params=job.destination_params) self.dispatcher.recover( job, job_wrapper ) if self.sa_session.dirty: @@ -296,21 +303,21 @@ continue # don't run jobs for which the input dataset was deleted if idata.deleted: - self.job_wrappers.pop(job.id, JobWrapper( job, self )).fail( "input data %s (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) ) + self.job_wrappers.pop(job.id, self.job_wrapper( job )).fail( "input data %s (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) ) return JOB_INPUT_DELETED # an error in the input data causes us to bail immediately elif idata.state == idata.states.ERROR: - self.job_wrappers.pop(job.id, JobWrapper( job, self )).fail( "input data %s is in error state" % ( idata.hid ) ) + self.job_wrappers.pop(job.id, self.job_wrapper( job )).fail( "input data %s is in error state" % ( idata.hid ) ) return JOB_INPUT_ERROR elif idata.state == idata.states.FAILED_METADATA: - self.job_wrappers.pop(job.id, JobWrapper( job, self )).fail( "input data %s failed to properly set metadata" % ( idata.hid ) ) + self.job_wrappers.pop(job.id, self.job_wrapper( job )).fail( "input data %s failed to properly set metadata" % ( idata.hid ) ) return JOB_INPUT_ERROR elif idata.state != idata.states.OK and not ( idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id ): # need to requeue return JOB_WAIT # Create the job wrapper so that the destination can be set if job.id not in self.job_wrappers: - self.job_wrappers[job.id] = JobWrapper(job, self) + self.job_wrappers[job.id] = self.job_wrapper( job ) # Cause the job_destination to be set and cached by the mapper try: self.job_wrappers[job.id].job_destination https://bitbucket.org/galaxy/galaxy-central/commits/6b565d3fc2a7/ Changeset: 6b565d3fc2a7 User: jmchilton Date: 2014-05-16 07:57:31 Summary: PEP-8 fixes for JobManager. Affected #: 1 file diff -r 0d81156abc40773643f5f8d72af521fe5d1b5d9d -r 6b565d3fc2a7f4f0db9f7a911e762e139c7fbb67 lib/galaxy/jobs/manager.py --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -3,20 +3,12 @@ """ import logging -import os -import random -import threading -import time -from Queue import Empty, Queue - -from galaxy import model -from galaxy.util.sleeper import Sleeper -from galaxy.jobs import handler, JobWrapper, NoopQueue -from galaxy.util.json import from_json_string +from galaxy.jobs import handler, NoopQueue log = logging.getLogger( __name__ ) + class JobManager( object ): """ Highest level interface to job management. @@ -35,14 +27,18 @@ self.job_handler = NoopHandler() self.job_queue = self.job_stop_queue = NoopQueue() self.job_handler.start() + def shutdown( self ): self.job_handler.shutdown() + class NoopHandler( object ): def __init__( self, *args, **kwargs ): self.job_queue = NoopQueue() self.job_stop_queue = NoopQueue() + def start( self ): pass + def shutdown( self, *args ): pass https://bitbucket.org/galaxy/galaxy-central/commits/7c668716a412/ Changeset: 7c668716a412 User: jmchilton Date: 2014-05-16 07:57:31 Summary: Handle LWR failed state in asynchronous MQ updates. Affected #: 1 file diff -r 6b565d3fc2a7f4f0db9f7a911e762e139c7fbb67 -r 7c668716a4125c583c0000d11a4aae9347dc8606 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -98,6 +98,9 @@ if lwr_status == "complete": self.mark_as_finished(job_state) return None + if lwr_status == "failed": + self.fail_job(job_state) + return None if lwr_status == "running" and not job_state.running: job_state.running = True job_state.job_wrapper.change_state( model.Job.states.RUNNING ) https://bitbucket.org/galaxy/galaxy-central/commits/21a568dac530/ Changeset: 21a568dac530 User: jmchilton Date: 2014-05-16 07:57:31 Summary: Correct 'abstract' method definition in AsynchronousJobRunner. Affected #: 1 file diff -r 7c668716a4125c583c0000d11a4aae9347dc8606 -r 21a568dac530aefb54e49238da2fac0934cb9308 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -400,7 +400,7 @@ self.watched = new_watched # Subclasses should implement this unless they override check_watched_items all together. - def check_watched_item(self): + def check_watched_item(self, job_state): raise NotImplementedError() def finish_job( self, job_state ): https://bitbucket.org/galaxy/galaxy-central/commits/7e95105c4660/ Changeset: 7e95105c4660 User: jmchilton Date: 2014-05-16 07:57:31 Summary: Simplified, truely asynchronous LWR MQ status updates. In addition to eliminating some egregious hacks, this reworked functionality should allow any LWR handler process to consume update messages for any job (so LWR jobs don't have to be bound to a particular handler). Affected #: 2 files diff -r 21a568dac530aefb54e49238da2fac0934cb9308 -r 7e95105c466051432e5c47ff977826a19758b867 lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -87,9 +87,9 @@ def job_wrapper( self, job ): return JobWrapper( job, self ) - def job_wrapper_for_id( self, id ): - job = self.sa_session.query( model.Job ) - return self.job_wrapper( job ) + def job_pair_for_id( self, id ): + job = self.sa_session.query( model.Job ).get( id ) + return job, self.job_wrapper( job ) def __check_jobs_at_startup( self ): """ diff -r 21a568dac530aefb54e49238da2fac0934cb9308 -r 7e95105c466051432e5c47ff977826a19758b867 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -43,8 +43,6 @@ def __init__( self, app, nworkers, transport=None, cache=None, url=None, galaxy_url=DEFAULT_GALAXY_URL, **kwds ): """Start the job runner """ super( LwrJobRunner, self ).__init__( app, nworkers ) - self.async_status_updates = dict() - self._init_monitor_thread() self._init_worker_threads() amqp_connect_ssl_args = {} amqp_consumer_timeout = False @@ -65,6 +63,10 @@ client_manager_kwargs['amqp_consumer_timeout'] = float(kwds['amqp_consumer_timeout']) self.galaxy_url = galaxy_url self.client_manager = build_client_manager(**client_manager_kwargs) + if url: + self.client_manager.ensure_has_status_update_callback(self.__async_update) + else: + self._init_monitor_thread() def url_to_destination( self, url ): """Convert a legacy URL to a job destination""" @@ -73,18 +75,6 @@ def check_watched_item(self, job_state): try: client = self.get_client_from_state(job_state) - - if hasattr(self.client_manager, 'ensure_has_status_update_callback'): - # Message queue implementation. - - # TODO: Very hacky now, refactor after Dannon merges in his - # message queue work, runners need the ability to disable - # check_watched_item like this and instead a callback needs to - # be issued post job recovery allowing a message queue - # consumer to be setup. - self.client_manager.ensure_has_status_update_callback(self.__async_update) - return job_state - status = client.get_status() except Exception: # An orphaned job was put into the queue at app startup, so remote server went down @@ -108,26 +98,9 @@ def __async_update( self, full_status ): job_id = full_status[ "job_id" ] - job_state = self.__find_watched_job( job_id ) - if not job_state: - # Probably finished too quickly, sleep and try again. - # Kind of a hack, why does monitor queue need to no wait - # get and sleep instead of doing a busy wait that would - # respond immediately. - sleep( 2 ) - job_state = self.__find_watched_job( job_id ) - if not job_state: - log.warn( "Failed to find job corresponding to status %s in %s" % ( full_status, self.watched ) ) - else: - self.__update_job_state_for_lwr_status(job_state, full_status["status"]) - - def __find_watched_job( self, job_id ): - found_job = None - for async_job_state in self.watched: - if str( async_job_state.job_id ) == job_id: - found_job = async_job_state - break - return found_job + job, job_wrapper = self.app.job_manager.job_handler.job_queue.job_pair_for_id( job_id ) + job_state = self.__job_state( job, job_wrapper ) + self.__update_job_state_for_lwr_status(job_state, full_status["status"]) def queue_job(self, job_wrapper): job_destination = job_wrapper.job_destination @@ -358,12 +331,8 @@ def recover( self, job, job_wrapper ): """Recovers jobs stuck in the queued/running state when Galaxy started""" - job_state = AsynchronousJobState() - job_state.job_id = str( job.get_job_runner_external_id() ) - job_state.runner_url = job_wrapper.get_job_runner_url() - job_state.job_destination = job_wrapper.job_destination + job_state = self.__job_state( job, job_wrapper ) job_wrapper.command_line = job.get_command_line() - job_state.job_wrapper = job_wrapper state = job.get_state() if state in [model.Job.states.RUNNING, model.Job.states.QUEUED]: log.debug( "(LWR/%s) is still in running state, adding to the LWR queue" % ( job.get_id()) ) @@ -375,6 +344,14 @@ super( LwrJobRunner, self ).shutdown() self.client_manager.shutdown() + def __job_state( self, job, job_wrapper ): + job_state = AsynchronousJobState() + job_state.job_id = str( job.get_job_runner_external_id() ) + job_state.runner_url = job_wrapper.get_job_runner_url() + job_state.job_destination = job_wrapper.job_destination + job_state.job_wrapper = job_wrapper + return job_state + def __client_outputs( self, client, job_wrapper ): work_dir_outputs = self.get_work_dir_outputs( job_wrapper ) output_files = self.get_output_files( job_wrapper ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix for previous commit.
by commits-noreply＠bitbucket.org 16 May '14

16 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b94bb858d29b/ Changeset: b94bb858d29b User: jmchilton Date: 2014-05-16 05:05:04 Summary: Fix for previous commit. Affected #: 1 file diff -r 933d0f2847332024f1fdf40234e74930014794e6 -r b94bb858d29b39f5d8127414bd703ee8e4905281 lib/galaxy/jobs/runners/lwr_client/manager.py --- a/lib/galaxy/jobs/runners/lwr_client/manager.py +++ b/lib/galaxy/jobs/runners/lwr_client/manager.py @@ -76,7 +76,7 @@ def __init__(self, **kwds): self.url = kwds.get('url') - self.manager_name = kwds.get("manager", "_default_") + self.manager_name = kwds.get("manager", None) or "_default_" self.connect_ssl = parse_amqp_connect_ssl_params(kwds.get('amqp_connect_ssl_args', None)) timeout = kwds.get('amqp_consumer_timeout', False) if timeout is False: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix: LWR - Fix manager name not passed into MQClientManager.
by commits-noreply＠bitbucket.org 16 May '14

16 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/933d0f284733/ Changeset: 933d0f284733 User: jmchilton Date: 2014-05-16 04:57:54 Summary: Bugfix: LWR - Fix manager name not passed into MQClientManager. Affected #: 1 file diff -r 559c5f4d1bdb694f90826896e5ebc837afa9b4f2 -r 933d0f2847332024f1fdf40234e74930014794e6 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -51,7 +51,13 @@ for kwd in kwds.keys(): if kwd.startswith('amqp_connect_ssl_'): amqp_connect_ssl_args[kwd] = kwds[kwd] - client_manager_kwargs = {'transport_type': transport, 'cache': string_as_bool_or_none(cache), "url": url, 'amqp_connect_ssl_args': amqp_connect_ssl_args or None} + client_manager_kwargs = { + 'transport_type': transport, + 'cache': string_as_bool_or_none(cache), + "url": url, + 'amqp_connect_ssl_args': amqp_connect_ssl_args or None, + 'manager': kwds.get("manager", None), + } if 'amqp_consumer_timeout' in kwds: if kwds['amqp_consumer_timeout'] == 'None': client_manager_kwargs['amqp_consumer_timeout'] = None Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 15 May '14

15 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fcd83883072c/ Changeset: fcd83883072c User: jmchilton Date: 2014-05-15 22:58:12 Summary: Fix GALAXY_SLOTS calculation for LSF. Thanks to Ilya Sytchev for the patch. Affected #: 1 file diff -r 47786821cd22eed10d1db9227b2ac18eff760374 -r fcd83883072c64b96ea0f12582fde3d05060b0f4 lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh --- a/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh +++ b/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh @@ -8,6 +8,8 @@ GALAXY_SLOTS="$NSLOTS" elif [ -f "$PBS_NODEFILE" ]; then GALAXY_SLOTS=`wc -l < $PBS_NODEFILE` +elif [ -n "$LSB_DJOB_NUMPROC" ]; then + GALAXY_SLOTS="$LSB_DJOB_NUMPROC" else GALAXY_SLOTS="1" unset GALAXY_SLOTS_CONFIGURED https://bitbucket.org/galaxy/galaxy-central/commits/559c5f4d1bdb/ Changeset: 559c5f4d1bdb User: jmchilton Date: 2014-05-15 22:58:58 Summary: Merge. Affected #: 1 file diff -r fcd83883072c64b96ea0f12582fde3d05060b0f4 -r 559c5f4d1bdb694f90826896e5ebc837afa9b4f2 lib/galaxy/webapps/galaxy/controllers/library_common.py --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -473,6 +473,20 @@ dbkey = dbkey[0] file_formats = [ dtype_name for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems() if dtype_value.allow_datatype_change ] file_formats.sort() + + def __ok_to_edit_metadata( ldda_id ): + #prevent modifying metadata when dataset is queued or running as input/output + #This code could be more efficient, i.e. by using mappers, but to prevent slowing down loading a History panel, we'll leave the code here for now + for job_to_dataset_association in trans.sa_session.query( self.app.model.JobToInputLibraryDatasetAssociation ) \ + .filter_by( ldda_id=ldda_id ) \ + .all() \ + + trans.sa_session.query( self.app.model.JobToOutputLibraryDatasetAssociation ) \ + .filter_by( ldda_id=ldda_id ) \ + .all(): + if job_to_dataset_association.job.state not in [ job_to_dataset_association.job.states.OK, job_to_dataset_association.job.states.ERROR, job_to_dataset_association.job.states.DELETED ]: + return False + return True + # See if we have any associated templates widgets = [] info_association, inherited = ldda.get_info_association() @@ -480,14 +494,18 @@ widgets = ldda.get_template_widgets( trans ) if params.get( 'change', False ): # The user clicked the Save button on the 'Change data type' form - if ldda.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change: - trans.app.datatypes_registry.change_datatype( ldda, params.datatype ) - trans.sa_session.flush() - message = "Data type changed for library dataset '%s'." % ldda.name - status = 'done' + if __ok_to_edit_metadata( ldda.id ): + if ldda.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change: + trans.app.datatypes_registry.change_datatype( ldda, params.datatype ) + trans.sa_session.flush() + message = "Data type changed for library dataset '%s'." % ldda.name + status = 'done' + else: + message = "You are unable to change datatypes in this manner. Changing %s to %s is not allowed." % ( ldda.extension, params.datatype ) + status = 'error' else: - message = "You are unable to change datatypes in this manner. Changing %s to %s is not allowed." % ( ldda.extension, params.datatype ) - status = 'error' + message = "This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them." + status = "error" elif params.get( 'save', False ): # The user clicked the Save button on the 'Edit Attributes' form old_name = ldda.name @@ -501,33 +519,40 @@ ldda.name = new_name ldda.info = new_info ldda.message = new_message - # The following for loop will save all metadata_spec items - for name, spec in ldda.datatype.metadata_spec.items(): - if spec.get("readonly"): - continue - optional = params.get( "is_" + name, None ) - if optional and optional == 'true': - # optional element... == 'true' actually means it is NOT checked (and therefore ommitted) - setattr( ldda.metadata, name, None ) - else: - setattr( ldda.metadata, name, spec.unwrap( params.get ( name, None ) ) ) - ldda.metadata.dbkey = dbkey - ldda.datatype.after_setting_metadata( ldda ) + if __ok_to_edit_metadata( ldda.id ): + # The following for loop will save all metadata_spec items + for name, spec in ldda.datatype.metadata_spec.items(): + if spec.get("readonly"): + continue + optional = params.get( "is_" + name, None ) + if optional and optional == 'true': + # optional element... == 'true' actually means it is NOT checked (and therefore ommitted) + setattr( ldda.metadata, name, None ) + else: + setattr( ldda.metadata, name, spec.unwrap( params.get ( name, None ) ) ) + ldda.metadata.dbkey = dbkey + ldda.datatype.after_setting_metadata( ldda ) + message = "Attributes updated for library dataset '%s'." % ldda.name + status = 'done' + else: + message = "Attributes updated, but metadata could not be changed because this dataset is currently being used as input or output. You must cancel or wait for these jobs to complete before changing metadata." + status = 'warning' trans.sa_session.flush() - message = "Attributes updated for library dataset '%s'." % ldda.name - status = 'done' elif params.get( 'detect', False ): # The user clicked the Auto-detect button on the 'Edit Attributes' form - for name, spec in ldda.datatype.metadata_spec.items(): - # We need to be careful about the attributes we are resetting - if name not in [ 'name', 'info', 'dbkey' ]: - if spec.get( 'default' ): - setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) - ldda.datatype.set_meta( ldda ) - ldda.datatype.after_setting_metadata( ldda ) + if __ok_to_edit_metadata( ldda.id ): + for name, spec in ldda.datatype.metadata_spec.items(): + # We need to be careful about the attributes we are resetting + if name not in [ 'name', 'info', 'dbkey' ]: + if spec.get( 'default' ): + setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) + message = "Attributes have been queued to be updated for library dataset '%s'." % ldda.name + status = 'done' + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':ldda } ) + else: + message = "This dataset is currently being used as input or output. You cannot change metadata until the jobs have completed or you have canceled them." + status = 'error' trans.sa_session.flush() - message = "Information updated for library dataset '%s'." % ldda.name - status = 'done' elif params.get( 'change_extended_metadata', False): em_string = util.restore_text( params.get("extended_metadata", "") ) if len(em_string): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Have auto-detect metadata option in library use external set metadata process.
by commits-noreply＠bitbucket.org 15 May '14

15 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/331232edbe8f/ Changeset: 331232edbe8f User: dan Date: 2014-05-15 22:25:44 Summary: Have auto-detect metadata option in library use external set metadata process. Affected #: 1 file diff -r 47786821cd22eed10d1db9227b2ac18eff760374 -r 331232edbe8f30eef4814cc01ac1cb3db4bd09e7 lib/galaxy/webapps/galaxy/controllers/library_common.py --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -473,6 +473,20 @@ dbkey = dbkey[0] file_formats = [ dtype_name for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems() if dtype_value.allow_datatype_change ] file_formats.sort() + + def __ok_to_edit_metadata( ldda_id ): + #prevent modifying metadata when dataset is queued or running as input/output + #This code could be more efficient, i.e. by using mappers, but to prevent slowing down loading a History panel, we'll leave the code here for now + for job_to_dataset_association in trans.sa_session.query( self.app.model.JobToInputLibraryDatasetAssociation ) \ + .filter_by( ldda_id=ldda_id ) \ + .all() \ + + trans.sa_session.query( self.app.model.JobToOutputLibraryDatasetAssociation ) \ + .filter_by( ldda_id=ldda_id ) \ + .all(): + if job_to_dataset_association.job.state not in [ job_to_dataset_association.job.states.OK, job_to_dataset_association.job.states.ERROR, job_to_dataset_association.job.states.DELETED ]: + return False + return True + # See if we have any associated templates widgets = [] info_association, inherited = ldda.get_info_association() @@ -480,14 +494,18 @@ widgets = ldda.get_template_widgets( trans ) if params.get( 'change', False ): # The user clicked the Save button on the 'Change data type' form - if ldda.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change: - trans.app.datatypes_registry.change_datatype( ldda, params.datatype ) - trans.sa_session.flush() - message = "Data type changed for library dataset '%s'." % ldda.name - status = 'done' + if __ok_to_edit_metadata( ldda.id ): + if ldda.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change: + trans.app.datatypes_registry.change_datatype( ldda, params.datatype ) + trans.sa_session.flush() + message = "Data type changed for library dataset '%s'." % ldda.name + status = 'done' + else: + message = "You are unable to change datatypes in this manner. Changing %s to %s is not allowed." % ( ldda.extension, params.datatype ) + status = 'error' else: - message = "You are unable to change datatypes in this manner. Changing %s to %s is not allowed." % ( ldda.extension, params.datatype ) - status = 'error' + message = "This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them." + status = "error" elif params.get( 'save', False ): # The user clicked the Save button on the 'Edit Attributes' form old_name = ldda.name @@ -501,33 +519,40 @@ ldda.name = new_name ldda.info = new_info ldda.message = new_message - # The following for loop will save all metadata_spec items - for name, spec in ldda.datatype.metadata_spec.items(): - if spec.get("readonly"): - continue - optional = params.get( "is_" + name, None ) - if optional and optional == 'true': - # optional element... == 'true' actually means it is NOT checked (and therefore ommitted) - setattr( ldda.metadata, name, None ) - else: - setattr( ldda.metadata, name, spec.unwrap( params.get ( name, None ) ) ) - ldda.metadata.dbkey = dbkey - ldda.datatype.after_setting_metadata( ldda ) + if __ok_to_edit_metadata( ldda.id ): + # The following for loop will save all metadata_spec items + for name, spec in ldda.datatype.metadata_spec.items(): + if spec.get("readonly"): + continue + optional = params.get( "is_" + name, None ) + if optional and optional == 'true': + # optional element... == 'true' actually means it is NOT checked (and therefore ommitted) + setattr( ldda.metadata, name, None ) + else: + setattr( ldda.metadata, name, spec.unwrap( params.get ( name, None ) ) ) + ldda.metadata.dbkey = dbkey + ldda.datatype.after_setting_metadata( ldda ) + message = "Attributes updated for library dataset '%s'." % ldda.name + status = 'done' + else: + message = "Attributes updated, but metadata could not be changed because this dataset is currently being used as input or output. You must cancel or wait for these jobs to complete before changing metadata." + status = 'warning' trans.sa_session.flush() - message = "Attributes updated for library dataset '%s'." % ldda.name - status = 'done' elif params.get( 'detect', False ): # The user clicked the Auto-detect button on the 'Edit Attributes' form - for name, spec in ldda.datatype.metadata_spec.items(): - # We need to be careful about the attributes we are resetting - if name not in [ 'name', 'info', 'dbkey' ]: - if spec.get( 'default' ): - setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) - ldda.datatype.set_meta( ldda ) - ldda.datatype.after_setting_metadata( ldda ) + if __ok_to_edit_metadata( ldda.id ): + for name, spec in ldda.datatype.metadata_spec.items(): + # We need to be careful about the attributes we are resetting + if name not in [ 'name', 'info', 'dbkey' ]: + if spec.get( 'default' ): + setattr( ldda.metadata, name, spec.unwrap( spec.get( 'default' ) ) ) + message = "Attributes have been queued to be updated for library dataset '%s'." % ldda.name + status = 'done' + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':ldda } ) + else: + message = "This dataset is currently being used as input or output. You cannot change metadata until the jobs have completed or you have canceled them." + status = 'error' trans.sa_session.flush() - message = "Information updated for library dataset '%s'." % ldda.name - status = 'done' elif params.get( 'change_extended_metadata', False): em_string = util.restore_text( params.get("extended_metadata", "") ) if len(em_string): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix incorrect LWR comment.
by commits-noreply＠bitbucket.org 15 May '14

15 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/47786821cd22/ Changeset: 47786821cd22 User: jmchilton Date: 2014-05-15 21:59:57 Summary: Fix incorrect LWR comment. Affected #: 1 file diff -r a50b64b9d70704542eba7dd69410c2a25c54e2a3 -r 47786821cd22eed10d1db9227b2ac18eff760374 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -108,7 +108,7 @@ sleep( 2 ) job_state = self.__find_watched_job( job_id ) if not job_state: - log.warn( "Failed to find job corresponding to final status %s in %s" % ( full_status, self.watched ) ) + log.warn( "Failed to find job corresponding to status %s in %s" % ( full_status, self.watched ) ) else: self.__update_job_state_for_lwr_status(job_state, full_status["status"]) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0