May 2014 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: jmchilton: Improvements to LWR documentation.
by commits-noreply＠bitbucket.org 13 May '14

13 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f5f6cd1e9383/ Changeset: f5f6cd1e9383 User: jmchilton Date: 2014-05-13 07:46:41 Summary: Improvements to LWR documentation. Add separate examples for MQ driven stuff - this should clear up things and make it clear which new parameters are required when using message queue and which are no longer required. Add more documentation for remote dependency resolution. Document rewrite_parameters. Affected #: 1 file diff -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 -r f5f6cd1e938304094c9491da10824f93cbfb2d27 job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -23,11 +23,6 @@  -  -  - </plugin> + <plugin id="amqp_lwr" type="runner" load="galaxy.jobs.runners.lwr:LwrJobRunner"> + <param id="url">amqp://guest:guest@localhost:5672//</param> +  + <param id="galaxy_url">http://localhost:8080/</param> +  +  + </plugin><plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" /><plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" /><plugin id="slurm" type="runner" load="galaxy.jobs.runners.slurm:SlurmJobRunner" /> @@ -116,7 +121,11 @@  + http transfers with file system copies, 'remote_transfer' to cause + the lwr to initiate HTTP transfers instead of Galaxy, or + 'remote_copy' to cause lwr to initiate file system copies. + If setting this to 'remote_transfer' be sure to specify a + 'galaxy_url' attribute on the runner plugin above. --> +  +  -  - </destination> + <destination id="amqp_lwr_dest" runner="amqp_lwr" > +  +  + <param id="jobs_directory">/path/to/remote/lwr/lwr_staging/</param> +  + <param id="default_file_action">remote_transfer</param> + </destination><destination id="ssh_torque" runner="cli"><param id="shell_plugin">SecureShell</param><param id="job_plugin">Torque</param> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 13 May '14

13 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/48ee04030f89/ Changeset: 48ee04030f89 User: jmchilton Date: 2014-05-13 07:13:02 Summary: Implement serializable class that can describe full dependency context for remote dependency resolution. Previously LWR jut serialized tool requirements, by transitioning to this class LWR should be able to resolve tool shed installed packages as well - requiring the additional repository and tool dependency context. Affected #: 2 files diff -r 3b29b6f83d17ed07d2c3ea49d839b29d66cad580 -r 48ee04030f893a9a9657b639bb38819fb4404388 lib/galaxy/tools/deps/dependencies.py --- /dev/null +++ b/lib/galaxy/tools/deps/dependencies.py @@ -0,0 +1,68 @@ +from galaxy.tools.deps.requirements import ToolRequirement +from galaxy.util import bunch + + +class DependenciesDescription(object): + """ Capture (in a readily serializable way) context related a tool + dependencies - both the tool's listed requirements and the tool shed + related context required to resolve dependencies via the + ToolShedPackageDependencyResolver. + + This is meant to enable remote resolution of dependencies, by the LWR or + other potential remote execution mechanisms. + """ + + def __init__(self, requirements=[], installed_tool_dependencies=[]): + self.requirements = requirements + # tool shed installed tool dependencies... + self.installed_tool_dependencies = installed_tool_dependencies + + def to_dict(self): + return dict( + requirements=[r.to_dict() for r in self.requirements], + installed_tool_dependencies=[DependenciesDescription._toolshed_install_dependency_to_dict(d) for d in self.installed_tool_dependencies] + ) + + @staticmethod + def from_dict(as_dict): + if as_dict is None: + return None + + requirements_dicts = as_dict.get('requirements', []) + requirements = [ToolRequirement.from_dict(r) for r in requirements_dicts] + installed_tool_dependencies_dicts = as_dict.get('installed_tool_dependencies', []) + installed_tool_dependencies = map(DependenciesDescription._toolshed_install_dependency_from_dict, installed_tool_dependencies_dicts) + return DependenciesDescription( + requirements=requirements, + installed_tool_dependencies=installed_tool_dependencies + ) + + @staticmethod + def _toolshed_install_dependency_from_dict(as_dict): + # Rather than requiring full models in LWR, just use simple objects + # containing only properties and associations used to resolve + # dependencies for tool execution. + repository_object = bunch.Bunch( + name=as_dict['repository_name'], + owner=as_dict['repository_owner'], + installed_changeset_revision=as_dict['repository_installed_changeset'], + ) + dependency_object = bunch.Bunch( + name=as_dict['dependency_name'], + version=as_dict['dependency_version'], + type=as_dict['dependency_type'], + tool_shed_repository=repository_object, + ) + return dependency_object + + @staticmethod + def _toolshed_install_dependency_to_dict(tool_dependency): + tool_shed_repository = tool_dependency.tool_shed_repository + return dict( + dependency_name=tool_dependency.name, + dependency_version=tool_dependency.version, + dependency_type=tool_dependency.type, + repository_name=tool_shed_repository.name, + repository_owner=tool_shed_repository.owner, + repository_installed_changeset=tool_shed_repository.installed_changeset_revision, + ) diff -r 3b29b6f83d17ed07d2c3ea49d839b29d66cad580 -r 48ee04030f893a9a9657b639bb38819fb4404388 test/unit/tools/test_tool_dependency_description.py --- /dev/null +++ b/test/unit/tools/test_tool_dependency_description.py @@ -0,0 +1,44 @@ +from galaxy.model import tool_shed_install +from galaxy.tools.deps import requirements +from galaxy.tools.deps import dependencies + + +def test_serialization(): + repository = tool_shed_install.ToolShedRepository( + owner="devteam", + name="tophat", + installed_changeset_revision="abcdefghijk", + ) + dependency = tool_shed_install.ToolDependency( + name="tophat", + version="2.0", + type="package", + status=tool_shed_install.ToolDependency.installation_status.INSTALLED, + ) + dependency.tool_shed_repository = repository + tool_requirement = requirements.ToolRequirement( + name="tophat", + version="2.0", + type="package", + ) + descript = dependencies.DependenciesDescription( + requirements=[tool_requirement], + installed_tool_dependencies=[dependency], + ) + result_descript = dependencies.DependenciesDescription.from_dict( + descript.to_dict() + ) + result_requirement = result_descript.requirements[0] + assert result_requirement.name == "tophat" + assert result_requirement.version == "2.0" + assert result_requirement.type == "package" + + result_tool_shed_dependency = result_descript.installed_tool_dependencies[0] + result_tool_shed_dependency.name = "tophat" + result_tool_shed_dependency.version = "2.0" + result_tool_shed_dependency.type = "package" + result_tool_shed_repository = result_tool_shed_dependency.tool_shed_repository + result_tool_shed_repository.name = "tophat" + result_tool_shed_repository.owner = "devteam" + result_tool_shed_repository.installed_changeset_revision = "abcdefghijk" + https://bitbucket.org/galaxy/galaxy-central/commits/24b5759f33f8/ Changeset: 24b5759f33f8 User: jmchilton Date: 2014-05-13 07:13:02 Summary: Update LWR client through LWR changeset 9fb7fcb. Among other smaller fixes, this update causes the LWR client/runner to pass more tool dependency context through to the remote LWR server when `dependency_resolution` is set to `remote`. This additional context allows tool shed packages to be resolved remotely (i.e. allows proper use of ToolShedPackageDependencyResolver on the remote LWR server). It remains the responsibility of the deployer to ensure identical versions of tool shed packages are installed on the remote LWR server and the local Galaxy instance. Affected #: 11 files diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr.py --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -5,6 +5,7 @@ from galaxy.jobs import ComputeEnvironment from galaxy.jobs import JobDestination from galaxy.jobs.command_factory import build_command +from galaxy.tools.deps import dependencies from galaxy.util import string_as_bool_or_none from galaxy.util.bunch import Bunch @@ -118,9 +119,7 @@ return try: - dependency_resolution = LwrJobRunner.__dependency_resolution( client ) - remote_dependency_resolution = dependency_resolution == "remote" - requirements = job_wrapper.tool.requirements if remote_dependency_resolution else [] + dependencies_description = LwrJobRunner.__dependencies_description( client, job_wrapper ) rewrite_paths = not LwrJobRunner.__rewrite_parameters( client ) unstructured_path_rewrites = {} if compute_environment: @@ -133,7 +132,7 @@ working_directory=job_wrapper.working_directory, tool=job_wrapper.tool, config_files=job_wrapper.extra_filenames, - requirements=requirements, + dependencies_description=dependencies_description, env=client.env, rewrite_paths=rewrite_paths, arbitrary_files=unstructured_path_rewrites, @@ -376,6 +375,19 @@ return client_outputs @staticmethod + def __dependencies_description( lwr_client, job_wrapper ): + dependency_resolution = LwrJobRunner.__dependency_resolution( lwr_client ) + remote_dependency_resolution = dependency_resolution == "remote" + if not remote_dependency_resolution: + return None + requirements = job_wrapper.tool.requirements or [] + installed_tool_dependencies = job_wrapper.tool.installed_tool_dependencies or [] + return dependencies.DependenciesDescription( + requirements=requirements, + installed_tool_dependencies=installed_tool_dependencies, + ) + + @staticmethod def __dependency_resolution( lwr_client ): dependency_resolution = lwr_client.destination_params.get( "dependency_resolution", "local" ) if dependency_resolution not in ["none", "local", "remote"]: diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/__init__.py --- a/lib/galaxy/jobs/runners/lwr_client/__init__.py +++ b/lib/galaxy/jobs/runners/lwr_client/__init__.py @@ -4,6 +4,39 @@ This module contains logic for interfacing with an external LWR server. +------------------ +Configuring Galaxy +------------------ + +Galaxy job runners are configured in Galaxy's ``job_conf.xml`` file. See ``job_conf.xml.sample_advanced`` +in your Galaxy code base or on +`Bitbucket <https://bitbucket.org/galaxy/galaxy-dist/src/tip/job_conf.xml.sample_advanc…>`_ +for information on how to configure Galaxy to interact with the LWR. + +Galaxy also supports an older, less rich configuration of job runners directly +in its main ``universe_wsgi.ini`` file. The following section describes how to +configure Galaxy to communicate with the LWR in this legacy mode. + +Legacy +------ + +A Galaxy tool can be configured to be executed remotely via LWR by +adding a line to the ``universe_wsgi.ini`` file under the +``galaxy:tool_runners`` section with the format:: + + <tool_id> = lwr://http://<lwr_host>:<lwr_port> + +As an example, if a host named remotehost is running the LWR server +application on port ``8913``, then the tool with id ``test_tool`` can +be configured to run remotely on remotehost by adding the following +line to ``universe.ini``:: + + test_tool = lwr://http://remotehost:8913 + +Remember this must be added after the ``[galaxy:tool_runners]`` header +in the ``universe.ini`` file. + + """ from .staging.down import finish_job diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/action_mapper.py --- a/lib/galaxy/jobs/runners/lwr_client/action_mapper.py +++ b/lib/galaxy/jobs/runners/lwr_client/action_mapper.py @@ -166,8 +166,8 @@ action_type = mapper.action_type file_lister = mapper.file_lister if type in ["workdir", "output_workdir"] and action_type == "none": - ## We are changing the working_directory relative to what - ## Galaxy would use, these need to be copied over. + # We are changing the working_directory relative to what + # Galaxy would use, these need to be copied over. action_type = "copy" action_class = actions.get(action_type, None) if action_class is None: diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py --- a/lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py +++ b/lib/galaxy/jobs/runners/lwr_client/amqp_exchange.py @@ -13,8 +13,8 @@ DEFAULT_EXCHANGE_NAME = "lwr" DEFAULT_EXCHANGE_TYPE = "direct" -DEFAULT_TIMEOUT = 0.2 # Set timeout to periodically give up looking and check - # if polling should end. +# Set timeout to periodically give up looking and check if polling should end. +DEFAULT_TIMEOUT = 0.2 class LwrExchange(object): @@ -48,7 +48,6 @@ queue = self.__queue(queue_name) with self.connection(self.__url, **connection_kwargs) as connection: with kombu.Consumer(connection, queues=[queue], callbacks=[callback], accept=['json']): - log.debug("Consuming queue %s" % queue) while check: try: connection.drain_events(timeout=self.__timeout) @@ -59,7 +58,6 @@ with self.connection(self.__url) as connection: with pools.producers[connection].acquire() as producer: key = self.__queue_name(name) - log.debug("Publishing with key %s and payload %s" % (key, payload)) producer.publish( payload, serializer='json', diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/client.py --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -39,9 +39,8 @@ ) else: job_directory = None - self.env = destination_params.get( "env", [] ) + self.env = destination_params.get("env", []) self.files_endpoint = destination_params.get("files_endpoint", None) - self.env = destination_params.get("env", []) self.job_directory = job_directory self.default_file_action = self.destination_params.get("default_file_action", "transfer") @@ -83,7 +82,7 @@ super(JobClient, self).__init__(destination_params, job_id) self.job_manager_interface = job_manager_interface - def launch(self, command_line, requirements=[], env=[], remote_staging=[], job_config=None): + def launch(self, command_line, dependencies_description=None, env=[], remote_staging=[], job_config=None): """ Queue up the execution of the supplied `command_line` on the remote server. Called launch for historical reasons, should be renamed to @@ -98,8 +97,8 @@ submit_params_dict = submit_params(self.destination_params) if submit_params_dict: launch_params['params'] = dumps(submit_params_dict) - if requirements: - launch_params['requirements'] = dumps([requirement.to_dict() for requirement in requirements]) + if dependencies_description: + launch_params['dependencies_description'] = dumps(dependencies_description.to_dict()) if env: launch_params['env'] = dumps(env) if remote_staging: @@ -248,8 +247,8 @@ return self._raw_execute(self._upload_file_action(args), args, contents, input_path) def _upload_file_action(self, args): - ## Hack for backward compatibility, instead of using new upload_file - ## path. Use old paths. + # Hack for backward compatibility, instead of using new upload_file + # path. Use old paths. input_type = args['input_type'] action = { # For backward compatibility just target upload_input_extra for all @@ -295,15 +294,15 @@ raise Exception(error_message) self.client_manager = client_manager - def launch(self, command_line, requirements=[], env=[], remote_staging=[], job_config=None): + def launch(self, command_line, dependencies_description=None, env=[], remote_staging=[], job_config=None): """ """ launch_params = dict(command_line=command_line, job_id=self.job_id) submit_params_dict = submit_params(self.destination_params) if submit_params_dict: launch_params['params'] = submit_params_dict - if requirements: - launch_params['requirements'] = [requirement.to_dict() for requirement in requirements] + if dependencies_description: + launch_params['dependencies_description'] = dependencies_description.to_dict() if env: launch_params['env'] = env if remote_staging: diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/destination.py --- a/lib/galaxy/jobs/runners/lwr_client/destination.py +++ b/lib/galaxy/jobs/runners/lwr_client/destination.py @@ -31,9 +31,9 @@ if not url.endswith("/"): url += "/" - ## Check for private token embedded in the URL. A URL of the form - ## https://moo@cow:8913 will try to contact https://cow:8913 - ## with a private key of moo + # Check for private token embedded in the URL. A URL of the form + # https://moo@cow:8913 will try to contact https://cow:8913 + # with a private key of moo private_token_format = "https?://(.*)@.*/?" private_token_match = match(private_token_format, url) private_token = None diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/interface.py --- a/lib/galaxy/jobs/runners/lwr_client/interface.py +++ b/lib/galaxy/jobs/runners/lwr_client/interface.py @@ -67,8 +67,8 @@ self.object_store = object_store def __app_args(self): - ## Arguments that would be specified from LwrApp if running - ## in web server. + # Arguments that would be specified from LwrApp if running + # in web server. return { 'manager': self.job_manager, 'file_cache': self.file_cache, diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/staging/__init__.py --- a/lib/galaxy/jobs/runners/lwr_client/staging/__init__.py +++ b/lib/galaxy/jobs/runners/lwr_client/staging/__init__.py @@ -30,8 +30,9 @@ be transferred to remote server). working_directory : str Local path created by Galaxy for running this job. - requirements : list - List of requirements for tool execution. + dependencies_description : list + galaxy.tools.deps.dependencies.DependencyDescription object describing + tool dependency context for remote depenency resolution. env: list List of dict object describing environment variables to populate. version_file : str @@ -55,7 +56,7 @@ input_files, client_outputs, working_directory, - requirements, + dependencies_description=None, env=[], arbitrary_files=None, rewrite_paths=True, @@ -66,7 +67,7 @@ self.input_files = input_files self.client_outputs = client_outputs self.working_directory = working_directory - self.requirements = requirements + self.dependencies_description = dependencies_description self.env = env self.rewrite_paths = rewrite_paths self.arbitrary_files = arbitrary_files or {} @@ -79,6 +80,15 @@ def version_file(self): return self.client_outputs.version_file + @property + def tool_dependencies(self): + if not self.remote_dependency_resolution: + return None + return dict( + requirements=(self.tool.requirements or []), + installed_tool_dependencies=(self.tool.installed_tool_dependencies or []) + ) + class ClientOutputs(object): """ Abstraction describing the output datasets EXPECTED by the Galaxy job diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/staging/up.py --- a/lib/galaxy/jobs/runners/lwr_client/staging/up.py +++ b/lib/galaxy/jobs/runners/lwr_client/staging/up.py @@ -24,7 +24,7 @@ job_id = file_stager.job_id launch_kwds = dict( command_line=rebuilt_command_line, - requirements=client_job_description.requirements, + dependencies_description=client_job_description.dependencies_description, env=client_job_description.env, ) if file_stager.job_config: diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/jobs/runners/lwr_client/transport/__init__.py --- a/lib/galaxy/jobs/runners/lwr_client/transport/__init__.py +++ b/lib/galaxy/jobs/runners/lwr_client/transport/__init__.py @@ -15,8 +15,8 @@ def __get_transport_type(transport_type, os_module): if not transport_type: use_curl = os_module.getenv('LWR_CURL_TRANSPORT', "0") - ## If LWR_CURL_TRANSPORT is unset or set to 0, use default, - ## else use curl. + # If LWR_CURL_TRANSPORT is unset or set to 0, use default, + # else use curl. if use_curl.isdigit() and not int(use_curl): transport_type = 'urllib' else: diff -r 48ee04030f893a9a9657b639bb38819fb4404388 -r 24b5759f33f87879e945a2f14ef5e8abfce488c1 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2673,12 +2673,18 @@ def build_dependency_shell_commands( self ): """Return a list of commands to be run to populate the current environment to include this tools requirements.""" + return self.app.toolbox.dependency_manager.dependency_shell_commands( + self.requirements, + installed_tool_dependencies=self.installed_tool_dependencies + ) + + @property + def installed_tool_dependencies(self): if self.tool_shed_repository: installed_tool_dependencies = self.tool_shed_repository.tool_dependencies_installed_or_in_error else: installed_tool_dependencies = None - return self.app.toolbox.dependency_manager.dependency_shell_commands( self.requirements, - installed_tool_dependencies=installed_tool_dependencies ) + return installed_tool_dependencies def build_redirect_url_params( self, param_dict ): """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: davebgx: Fix for retrieving job streams with the twill interactor.
by commits-noreply＠bitbucket.org 12 May '14

12 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3b29b6f83d17/ Changeset: 3b29b6f83d17 User: davebgx Date: 2014-05-12 22:27:49 Summary: Fix for retrieving job streams with the twill interactor. Affected #: 1 file diff -r 246b419fc92a91f31eb7cdd5e2de4a4c15a4b631 -r 3b29b6f83d17ed07d2c3ea49d839b29d66cad580 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -386,7 +386,8 @@ self.twill_test_case.verify_dataset_correctness( outfile, hid=hid, attributes=attributes, shed_tool_id=shed_tool_id, maxseconds=maxseconds ) def get_job_stream( self, history_id, output_data, stream ): - return self.twill_test_case._get_job_stream_output( output_data.get( 'id' ), stream=stream, format=False ) + encoded_id = self.twill_test_case.security.encode_id( output_data.get( 'id' ) ) + return self.twill_test_case._get_job_stream_output( encoded_id, stream=stream, format=False ) def stage_data_async( self, test_data, history, shed_tool_id, async=True ): name = test_data.get( 'name', None ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: When working with <conversions> in DataToolParameters, have the parsed conv_types be a list, as is now expected in subsequent usage.
by commits-noreply＠bitbucket.org 12 May '14

12 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/246b419fc92a/ Changeset: 246b419fc92a User: dan Date: 2014-05-12 21:27:36 Summary: When working with <conversions> in DataToolParameters, have the parsed conv_types be a list, as is now expected in subsequent usage. Affected #: 1 file diff -r 2b7e97a34afa6b33e9950df9213b3cb9be9026a3 -r 246b419fc92a91f31eb7cdd5e2de4a4c15a4b631 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1659,7 +1659,7 @@ conv_extensions = conv_elem.get( "type" ) # target datatype extension # FIXME: conv_extensions should be able to be an ordered list assert None not in [ name, type ], 'A name (%s) and type (%s) are required for explicit conversion' % ( name, type ) - conv_types = tool.app.datatypes_registry.get_datatype_by_extension( conv_extensions.lower() ) + conv_types = [ tool.app.datatypes_registry.get_datatype_by_extension( conv_extensions.lower() ) ] self.conversions.append( ( name, conv_extensions, conv_types ) ) def get_html_field( self, trans=None, value=None, other_values={} ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: martenson: important docs for select2 backbone view
by commits-noreply＠bitbucket.org 12 May '14

12 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2b7e97a34afa/ Changeset: 2b7e97a34afa User: martenson Date: 2014-05-12 20:56:06 Summary: important docs for select2 backbone view Affected #: 2 files diff -r 8e2135ee278aa44eb5a8da04a392bfa44efc0279 -r 2b7e97a34afa6b33e9950df9213b3cb9be9026a3 static/scripts/mvc/ui/ui-select.js --- a/static/scripts/mvc/ui/ui-select.js +++ b/static/scripts/mvc/ui/ui-select.js @@ -1,7 +1,12 @@ // dependencies define(['utils/utils'], function(Utils) { -// plugin +/** + * A plugin for initializing select2 input items. + * Make sure the select2 library itself is loaded beforehand. + * Also the element to which select2 will be appended has to + * be created before select2 initialization (and passed as option). + */ var View = Backbone.View.extend( { // options @@ -181,13 +186,13 @@ }, // element - _template: function(options) { + _template: function() { return '<input type="hidden"/>'; } }); return { View : View -} +}; }); diff -r 8e2135ee278aa44eb5a8da04a392bfa44efc0279 -r 2b7e97a34afa6b33e9950df9213b3cb9be9026a3 static/scripts/packed/mvc/ui/ui-select.js --- a/static/scripts/packed/mvc/ui/ui-select.js +++ b/static/scripts/packed/mvc/ui/ui-select.js @@ -1,1 +1,1 @@ -define(["utils/utils"],function(a){var b=Backbone.View.extend({optionsDefault:{css:"",placeholder:"No data available",data:[],value:null},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));if(!this.options.container){console.log("ui-select::initialize() : container not specified.");return}this.options.container.append(this.$el);this.select_data=this.options.data;this._refresh();if(this.options.value){this._setValue(this.options.value)}var c=this;if(this.options.onchange){this.$el.on("change",function(){c.options.onchange(c.value())})}},value:function(c){var d=this._getValue();if(c!==undefined){this._setValue(c)}var e=this._getValue();if((e!=d&&this.options.onchange)){this.options.onchange(e)}return e},text:function(){return this.$el.select2("data").text},disabled:function(){return !this.$el.select2("enable")},enable:function(){this.$el.select2("enable",true)},disable:function(){this.$el.select2("enable",false)},add:function(c){this.select_data.push({id:c.id,text:c.text});this._refresh()},del:function(d){var c=this._getIndex(d);if(c!=-1){this.select_data.splice(c,1);this._refresh()}},remove:function(){this.$el.select2("destroy")},update:function(c){this.select_data=[];for(var d in c.data){this.select_data.push(c.data[d])}this._refresh()},_refresh:function(){var c=this._getValue();this.$el.select2({data:this.select_data,containerCssClass:this.options.css,placeholder:this.options.placeholder});this._setValue(c)},_getIndex:function(d){for(var c in this.select_data){if(this.select_data[c].id==d){return c}}return -1},_getValue:function(){return this.$el.select2("val")},_setValue:function(d){var c=this._getIndex(d);if(c==-1){if(this.select_data.length>0){d=this.select_data[0].id}}this.$el.select2("val",d)},_template:function(c){return'<input type="hidden"/>'}});return{View:b}}); \ No newline at end of file +define(["utils/utils"],function(a){var b=Backbone.View.extend({optionsDefault:{css:"",placeholder:"No data available",data:[],value:null},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));if(!this.options.container){console.log("ui-select::initialize() : container not specified.");return}this.options.container.append(this.$el);this.select_data=this.options.data;this._refresh();if(this.options.value){this._setValue(this.options.value)}var c=this;if(this.options.onchange){this.$el.on("change",function(){c.options.onchange(c.value())})}},value:function(c){var d=this._getValue();if(c!==undefined){this._setValue(c)}var e=this._getValue();if((e!=d&&this.options.onchange)){this.options.onchange(e)}return e},text:function(){return this.$el.select2("data").text},disabled:function(){return !this.$el.select2("enable")},enable:function(){this.$el.select2("enable",true)},disable:function(){this.$el.select2("enable",false)},add:function(c){this.select_data.push({id:c.id,text:c.text});this._refresh()},del:function(d){var c=this._getIndex(d);if(c!=-1){this.select_data.splice(c,1);this._refresh()}},remove:function(){this.$el.select2("destroy")},update:function(c){this.select_data=[];for(var d in c.data){this.select_data.push(c.data[d])}this._refresh()},_refresh:function(){var c=this._getValue();this.$el.select2({data:this.select_data,containerCssClass:this.options.css,placeholder:this.options.placeholder});this._setValue(c)},_getIndex:function(d){for(var c in this.select_data){if(this.select_data[c].id==d){return c}}return -1},_getValue:function(){return this.$el.select2("val")},_setValue:function(d){var c=this._getIndex(d);if(c==-1){if(this.select_data.length>0){d=this.select_data[0].id}}this.$el.select2("val",d)},_template:function(){return'<input type="hidden"/>'}});return{View:b}}); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix typo in the Tool Shed's package recipe install step handler - thanks Bjoren Gruning.
by commits-noreply＠bitbucket.org 12 May '14

12 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8e2135ee278a/ Changeset: 8e2135ee278a User: greg Date: 2014-05-12 18:47:57 Summary: Fix typo in the Tool Shed's package recipe install step handler - thanks Bjoren Gruning. Affected #: 1 file diff -r 7950ab309ad1cb06cf3af973adcca1e3209c9241 -r 8e2135ee278aa44eb5a8da04a392bfa44efc0279 lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -1337,7 +1337,7 @@ """ env_vars = dict() env_vars = install_environment.environment_dict() - tool_shed_repository = tool_depenedncy.tool_shed_repository + tool_shed_repository = tool_dependency.tool_shed_repository env_vars.update( td_common_util.get_env_var_values( install_environment ) ) language = action_dict[ 'language' ] with settings( warn_only=True, **env_vars ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Small improvements to collection creation API.
by commits-noreply＠bitbucket.org 12 May '14

12 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7950ab309ad1/ Changeset: 7950ab309ad1 User: jmchilton Date: 2014-05-12 16:54:02 Summary: Small improvements to collection creation API. Return full collection when creating collection and improved error handling during creation. Affected #: 2 files diff -r 4af4aad0ce3a47407298e05bab4729091361ac47 -r 7950ab309ad1cb06cf3af973adcca1e3209c9241 lib/galaxy/dataset_collections/__init__.py --- a/lib/galaxy/dataset_collections/__init__.py +++ b/lib/galaxy/dataset_collections/__init__.py @@ -202,6 +202,10 @@ def __load_elements( self, trans, element_identifiers ): elements = odict.odict() for element_identifier in element_identifiers: + if "name" not in element_identifier: + raise RequestParameterInvalidException( + "Cannot load invalid dataset identifier - missing name - %s" % element_identifier + ) elements[ element_identifier[ "name" ] ] = self.__load_element( trans, element_identifier ) return elements diff -r 4af4aad0ce3a47407298e05bab4729091361ac47 -r 7950ab309ad1cb06cf3af973adcca1e3209c9241 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -295,7 +295,7 @@ create_params = api_payload_to_create_params( payload ) service = trans.app.dataset_collections_service dataset_collection_instance = service.create( trans, parent=history, **create_params ) - return self.__collection_dict( trans, dataset_collection_instance ) + return self.__collection_dict( trans, dataset_collection_instance, view="element" ) @expose_api_anonymous def update( self, trans, history_id, id, payload, **kwd ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 11 May '14

11 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2d08e99be908/ Changeset: 2d08e99be908 User: jmchilton Date: 2014-05-11 20:56:49 Summary: Allow more typical expanded HDCA value with src when mapping over collection via API. Flat ID still used by GUI - but when coming through API this is probably the better choice. Affected #: 2 files diff -r a74fd6395843cb384928ad367fd6f5f6d5eff65c -r 2d08e99be908216de8897b1c545896fedaa84379 lib/galaxy/tools/parameters/meta.py --- a/lib/galaxy/tools/parameters/meta.py +++ b/lib/galaxy/tools/parameters/meta.py @@ -40,7 +40,13 @@ if "|" in incoming_val: encoded_hdc_id, subcollection_type = incoming_val.split( "|", 1 ) else: - encoded_hdc_id = incoming_val + try: + src = incoming_val[ "src" ] + if src != "hdca": + raise exceptions.ToolMetaParameterException( "Invalid dataset collection source type %s" % src ) + encoded_hdc_id = incoming_val[ "id" ] + except TypeError: + encoded_hdc_id = incoming_val subcollection_type = None hdc_id = trans.app.security.decode_id( encoded_hdc_id ) hdc = trans.sa_session.query( model.HistoryDatasetCollectionAssociation ).get( hdc_id ) diff -r a74fd6395843cb384928ad367fd6f5f6d5eff65c -r 2d08e99be908216de8897b1c545896fedaa84379 test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -159,6 +159,9 @@ history_id = self.dataset_populator.new_history() hdca_id = self.__build_pair( history_id, [ "123", "456" ] ) inputs = { + # Such inputs can be simple hdca ids (for GUI) or + # {src: "hdca", id: <hdca_id>} for API. This tests the + # first, next test method tests other. "input1|__collection_multirun__": hdca_id, } create = self._run_cat1( history_id, inputs=inputs, assert_ok=True ) @@ -181,7 +184,7 @@ history_id = self.dataset_populator.new_history() hdca_id = self.__build_nested_list( history_id ) inputs = { - "input1|__collection_multirun__": hdca_id, + "input1|__collection_multirun__": dict( src="hdca", id=hdca_id ), } create = self._run_cat1( history_id, inputs=inputs, assert_ok=True ) outputs = create[ 'outputs' ] https://bitbucket.org/galaxy/galaxy-central/commits/4af4aad0ce3a/ Changeset: 4af4aad0ce3a User: jmchilton Date: 2014-05-11 20:56:49 Summary: Add API functional test for workflow extraction of collection reductions. Affected #: 1 file diff -r 2d08e99be908216de8897b1c545896fedaa84379 -r 4af4aad0ce3a47407298e05bab4729091361ac47 test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -142,38 +142,22 @@ "input|__collection_multirun__": hdca_id, "num_lines": 2 } - run_output1 = self.dataset_populator.run_tool( - tool_id="random_lines1", - inputs=inputs1, - history_id=history_id, - ) - implicit_hdca1 = run_output1[ "implicit_collections" ][ 0 ] - job_id1 = run_output1[ "jobs" ][ 0 ][ "id" ] - self.dataset_populator.wait_for_history( history_id, assert_ok=True, timeout=20 ) + implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id( history_id, "random_lines1", inputs1 ) inputs2 = { "input|__collection_multirun__": implicit_hdca1[ "id" ], "num_lines": 1 } - run_output2 = self.dataset_populator.run_tool( - tool_id="random_lines1", - inputs=inputs2, - history_id=history_id, - ) - # implicit_hdca2 = run_output2[ "implicit_collections" ][ 0 ] - job_id2 = run_output2[ "jobs" ][ 0 ][ "id" ] - self.dataset_populator.wait_for_history( history_id, assert_ok=True, timeout=20 ) + _, job_id2 = self._run_tool_get_collection_and_job_id( history_id, "random_lines1", inputs2 ) downloaded_workflow = self._extract_and_download_workflow( from_history_id=history_id, dataset_collection_ids=dumps( [ hdca[ "hid" ] ] ), job_ids=dumps( [ job_id1, job_id2 ] ), workflow_name="test import from mapping history", ) + # Assert workflow is input connected to a tool step with one output + # connected to another tool step. assert len( downloaded_workflow[ "steps" ] ) == 3 - collection_steps = self._get_steps_of_type( downloaded_workflow, "data_collection_input", expected_len=1 ) - collection_step = collection_steps[ 0 ] - collection_step_state = loads( collection_step[ "tool_state" ] ) - self.assertEquals( collection_step_state[ "collection_type" ], u"paired" ) - collect_step_idx = collection_step[ "id" ] + collect_step_idx = self._assert_first_step_is_paired_input( downloaded_workflow ) tool_steps = self._get_steps_of_type( downloaded_workflow, "tool", expected_len=2 ) tool_step_idxs = [] tool_input_step_idxs = [] @@ -187,6 +171,63 @@ assert tool_input_step_idxs[ 0 ] == collect_step_idx assert tool_input_step_idxs[ 1 ] == tool_step_idxs[ 0 ] + @skip_without_tool( "random_lines1" ) + @skip_without_tool( "multi_data_param" ) + def test_extract_reduction_from_history( self ): + history_id = self.dataset_populator.new_history() + hdca = self.dataset_collection_populator.create_pair_in_history( history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"] ).json() + hdca_id = hdca[ "id" ] + inputs1 = { + "input|__collection_multirun__": hdca_id, + "num_lines": 2 + } + implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id( history_id, "random_lines1", inputs1 ) + inputs2 = { + "f1": "__collection_reduce__|%s" % ( implicit_hdca1[ "id" ] ), + "f2": "__collection_reduce__|%s" % ( implicit_hdca1[ "id" ] ) + } + reduction_run_output = self.dataset_populator.run_tool( + tool_id="multi_data_param", + inputs=inputs2, + history_id=history_id, + ) + job_id2 = reduction_run_output[ "jobs" ][ 0 ][ "id" ] + self.dataset_populator.wait_for_history( history_id, assert_ok=True, timeout=20 ) + downloaded_workflow = self._extract_and_download_workflow( + from_history_id=history_id, + dataset_collection_ids=dumps( [ hdca[ "hid" ] ] ), + job_ids=dumps( [ job_id1, job_id2 ] ), + workflow_name="test import reduction", + ) + assert len( downloaded_workflow[ "steps" ] ) == 3 + collect_step_idx = self._assert_first_step_is_paired_input( downloaded_workflow ) + tool_steps = self._get_steps_of_type( downloaded_workflow, "tool", expected_len=2 ) + random_lines_map_step = tool_steps[ 0 ] + reduction_step = tool_steps[ 1 ] + random_lines_input = random_lines_map_step[ "input_connections" ][ "input" ] + assert random_lines_input[ "id" ] == collect_step_idx + reduction_step_input = reduction_step[ "input_connections" ][ "f1" ] + assert reduction_step_input[ "id"] == random_lines_map_step[ "id" ] + + def _run_tool_get_collection_and_job_id( self, history_id, tool_id, inputs ): + run_output1 = self.dataset_populator.run_tool( + tool_id=tool_id, + inputs=inputs, + history_id=history_id, + ) + implicit_hdca = run_output1[ "implicit_collections" ][ 0 ] + job_id = run_output1[ "jobs" ][ 0 ][ "id" ] + self.dataset_populator.wait_for_history( history_id, assert_ok=True, timeout=20 ) + return implicit_hdca, job_id + + def _assert_first_step_is_paired_input( self, downloaded_workflow ): + collection_steps = self._get_steps_of_type( downloaded_workflow, "data_collection_input", expected_len=1 ) + collection_step = collection_steps[ 0 ] + collection_step_state = loads( collection_step[ "tool_state" ] ) + self.assertEquals( collection_step_state[ "collection_type" ], u"paired" ) + collect_step_idx = collection_step[ "id" ] + return collect_step_idx + def _extract_and_download_workflow( self, **extract_payload ): create_workflow_response = self._post( "workflows", data=extract_payload ) self._assert_status_code_is( create_workflow_response, 200 ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Jeremy Goecks: Add example configuration for CLI Slurm runner.
by commits-noreply＠bitbucket.org 09 May '14

09 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a74fd6395843/ Changeset: a74fd6395843 User: Jeremy Goecks Date: 2014-05-09 22:20:33 Summary: Add example configuration for CLI Slurm runner. Affected #: 1 file diff -r ae7da2ebe7737d6e6ba142d1ac24a24b467285ad -r a74fd6395843cb384928ad367fd6f5f6d5eff65c job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -164,6 +164,18 @@ <param id="shell_hostname">foo.example.org</param><param id="job_Resource_List">walltime=24:00:00,ncpus=4</param></destination> + +  + <destination id="ssh_slurm" runner="cli"> + <param id="shell_plugin">SecureShell</param> + <param id="job_plugin">Slurm</param> + <param id="shell_username">foo</param> + <param id="shell_hostname">my_host</param> + <param id="job_time">2:00:00</param> + <param id="job_ncpus">4</param> + <param id="job_partition">my_partition</param> + </destination> + <destination id="condor" runner="condor"><!-- With no params, jobs are submitted to the 'vanilla' universe with: notification = NEVER Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Jeremy Goecks: Add a CLI-driven Slurm runner so that Galaxy can be run on non-submit hosts with a Slurm cluster. Add some flexibility for obtaining the external job ID in the CLI runner as well.
by commits-noreply＠bitbucket.org 09 May '14

09 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ae7da2ebe773/ Changeset: ae7da2ebe773 User: Jeremy Goecks Date: 2014-05-09 22:15:27 Summary: Add a CLI-driven Slurm runner so that Galaxy can be run on non-submit hosts with a Slurm cluster. Add some flexibility for obtaining the external job ID in the CLI runner as well. Affected #: 2 files diff -r 3210eaa1db6cb6af4e5c4846c1fec39585405cd9 -r ae7da2ebe7737d6e6ba142d1ac24a24b467285ad lib/galaxy/jobs/runners/cli.py --- a/lib/galaxy/jobs/runners/cli.py +++ b/lib/galaxy/jobs/runners/cli.py @@ -99,7 +99,9 @@ log.error('(%s) submission failed (stderr): %s' % (galaxy_id_tag, cmd_out.stderr)) job_wrapper.fail("failure submitting job") return - external_job_id = cmd_out.stdout.strip() + # Some job runners return something like 'Submitted batch job XXXX' + # Strip and split to get job ID. + external_job_id = cmd_out.stdout.strip().split()[-1] if not external_job_id: log.error('(%s) submission did not return a job identifier, failing job' % galaxy_id_tag) job_wrapper.fail("failure submitting job") diff -r 3210eaa1db6cb6af4e5c4846c1fec39585405cd9 -r ae7da2ebe7737d6e6ba142d1ac24a24b467285ad lib/galaxy/jobs/runners/util/cli/job/slurm.py --- /dev/null +++ b/lib/galaxy/jobs/runners/util/cli/job/slurm.py @@ -0,0 +1,94 @@ +# A simple CLI runner for slurm that can be used when running Galaxy from a +# non-submit host and using a Slurm cluster. + +try: + from galaxy.model import Job + job_states = Job.states +except ImportError: + # Not in Galaxy, map Galaxy job states to LWR ones. + from galaxy.util import enum + job_states = enum(RUNNING='running', OK='complete', QUEUED='queued') + +from ..job import BaseJobExec + +__all__ = ('Slurm',) + +from logging import getLogger +log = getLogger(__name__) + +argmap = { + 'time': '-t', + 'ncpus': '-c', + 'partition': '-p' +} + +class Slurm(BaseJobExec): + + def __init__(self, **params): + self.params = {} + for k, v in params.items(): + self.params[k] = v + + def job_script_kwargs(self, ofile, efile, job_name): + scriptargs = {'-o': ofile, + '-e': efile, + '-J': job_name} + + # Map arguments using argmap. + for k, v in self.params.items(): + if k == 'plugin': + continue + try: + if not k.startswith('-'): + k = argmap[k] + scriptargs[k] = v + except: + log.warning('Unrecognized long argument passed to Slurm CLI plugin: %s' % k) + + # Generated template. + template_scriptargs = '' + for k, v in scriptargs.items(): + template_scriptargs += '#SBATCH %s %s\n' % (k, v) + return dict(headers=template_scriptargs) + + def submit(self, script_file): + return 'sbatch %s' % script_file + + def delete(self, job_id): + return 'scancel %s' % job_id + + def get_status(self, job_ids=None): + return 'squeue -a -o \\"%A %t\\"' + + def get_single_status(self, job_id): + return 'squeue -a -o \\"%A %t\\" -j ' + job_id + + def parse_status(self, status, job_ids): + # Get status for each job, skipping header. + rval = {} + for line in status.splitlines()[ 1: ]: + id, state = line.split() + if id in job_ids: + # map job states to Galaxy job states. + rval[id] = self._get_job_state(state) + return rval + + def parse_single_status(self, status, job_id): + status = status.splitlines() + if len( status ) > 1: + # Job still on cluster and has state. + id, state = status[1].split() + return self._get_job_state(state) + return job_states.OK + + def _get_job_state(self, state): + try: + return { + 'F': job_states.ERROR, + 'R': job_states.RUNNING, + 'CG': job_states.RUNNING, + 'PD': job_states.QUEUED, + 'CD': job_states.OK + }.get(state) + except KeyError: + raise KeyError("Failed to map slurm status code [%s] to job state." % state) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0