May 2014 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: jmchilton: Simpler, more powerful subcollection mapping.
by commits-noreply＠bitbucket.org 07 May '14

07 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9e502242af28/ Changeset: 9e502242af28 User: jmchilton Date: 2014-05-07 18:29:51 Summary: Simpler, more powerful subcollection mapping. Collections can be mapped over 'data' parameters and sufficiently nested collections can be mapped over 'data_collection' parameters (for instance a list of 5 pairs can be supplied to a tool taking in a pair and 5 jobs will be executed). I (perhaps poorly) term these concepts collection mapping and subcollection mapping. Prior to this changeset - the API for doing collection 'mapping' and 'subcollection mapping' was somewhat more divergent and the tool execution code explicitly forbid doing both kinds of mappings in the same tool execution even if the effective collections could be matched (e.g. it could not map a 'data' parameter over a list of 5 datasets and a pair parameter a list of 5 pairs in the same execution). This changeset should remedy this - as long as the effective collection mappings can match up such jobs should be possible. The workflow editor (and I think runner) already thought this was possible, so this changeset reduces the tool-workflow impedance mismatch - an existing problem exacerbated by recent dataset collections introduction. This all needs much more testing - test workflows execute this way, functional test of a tool execution that combines collection mapping and subcollection mapping, etc.... Affected #: 4 files diff -r c5b736d3e4e5027391a04fd187e2f9d2aa4ebe46 -r 9e502242af28473660ee70ee0f98345f3604e137 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1925,7 +1925,7 @@ # Fixed set of input parameters may correspond to any number of jobs. # Expand these out to individual parameters for given jobs (tool # executions). - expanded_incomings, collection_info = expand_meta_parameters( trans, incoming, self.inputs ) + expanded_incomings, collection_info = expand_meta_parameters( trans, self, incoming ) if not expanded_incomings: raise exceptions.MessageException( "Tool execution failed, trying to run a tool over an empty collection." ) diff -r c5b736d3e4e5027391a04fd187e2f9d2aa4ebe46 -r 9e502242af28473660ee70ee0f98345f3604e137 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -2021,7 +2021,7 @@ self._ensure_selection( field ) return field - def _get_select_dataset_collection_field( self, trans, history, multiple=False, suffix="|__subcollection_multirun__", value=None, other_values=None ): + def _get_select_dataset_collection_field( self, trans, history, multiple=False, suffix="|__collection_multirun__", value=None, other_values=None ): field_name = "%s%s" % ( self.name, suffix ) field = form_builder.SelectField( field_name, multiple, None, self.refresh_on_change, refresh_on_change_values=self.refresh_on_change_values ) dataset_matcher = DatasetMatcher( trans, self, value, other_values ) diff -r c5b736d3e4e5027391a04fd187e2f9d2aa4ebe46 -r 9e502242af28473660ee70ee0f98345f3604e137 lib/galaxy/tools/parameters/meta.py --- a/lib/galaxy/tools/parameters/meta.py +++ b/lib/galaxy/tools/parameters/meta.py @@ -9,7 +9,7 @@ log = logging.getLogger( __name__ ) -def expand_meta_parameters( trans, incoming, inputs ): +def expand_meta_parameters( trans, tool, incoming ): """ Take in a dictionary of raw incoming parameters and expand to a list of expanded incoming parameters (one set of parameters per tool @@ -34,26 +34,23 @@ def collection_classifier( input_key ): multirun_key = "%s|__collection_multirun__" % input_key if multirun_key in incoming: - encoded_hdc_id = incoming[ multirun_key ] - hdc_id = trans.app.security.decode_id( encoded_hdc_id ) - hdc = trans.sa_session.query( model.HistoryDatasetCollectionAssociation ).get( hdc_id ) - collections_to_match.add( input_key, hdc ) - hdas = hdc.collection.dataset_instances - return permutations.input_classification.MATCHED, hdas - else: - return permutations.input_classification.SINGLE, incoming[ input_key ] - - def subcollection_classifier( input_key ): - multirun_key = "%s|__subcollection_multirun__" % input_key - if multirun_key in incoming: incoming_val = incoming[ multirun_key ] - # value will be "hdca_id|subcollection_type" - encoded_hdc_id, subcollection_type = incoming_val.split( "|", 1 ) + # If subcollectin multirun of data_collection param - value will + # be "hdca_id|subcollection_type" else it will just be hdca_id + if "|" in incoming_val: + encoded_hdc_id, subcollection_type = incoming_val.split( "|", 1 ) + else: + encoded_hdc_id = incoming_val + subcollection_type = None hdc_id = trans.app.security.decode_id( encoded_hdc_id ) hdc = trans.sa_session.query( model.HistoryDatasetCollectionAssociation ).get( hdc_id ) collections_to_match.add( input_key, hdc, subcollection_type=subcollection_type ) - subcollection_elements = subcollections.split_dataset_collection_instance( hdc, subcollection_type ) - return permutations.input_classification.MATCHED, subcollection_elements + if subcollection_type is not None: + subcollection_elements = subcollections.split_dataset_collection_instance( hdc, subcollection_type ) + return permutations.input_classification.MATCHED, subcollection_elements + else: + hdas = hdc.collection.dataset_instances + return permutations.input_classification.MATCHED, hdas else: return permutations.input_classification.SINGLE, incoming[ input_key ] @@ -72,26 +69,17 @@ multirun_found = False collection_multirun_found = False - subcollection_multirun_found = False for key, value in incoming.iteritems(): multirun_found = try_replace_key( key, "|__multirun__" ) or multirun_found collection_multirun_found = try_replace_key( key, "|__collection_multirun__" ) or collection_multirun_found - subcollection_multirun_found = try_replace_key( key, "|__subcollection_multirun__" ) or subcollection_multirun_found - if sum( [ 1 if f else 0 for f in [ multirun_found, collection_multirun_found, subcollection_multirun_found ] ] ) > 1: + if sum( [ 1 if f else 0 for f in [ multirun_found, collection_multirun_found ] ] ) > 1: # In theory doable, but to complicated for a first pass. message = "Cannot specify parallel execution across both multiple datasets and dataset collections." raise exceptions.ToolMetaParameterException( message ) if multirun_found: return permutations.expand_multi_inputs( incoming_template, classifier ), None - elif subcollection_multirun_found: - expanded_incomings = permutations.expand_multi_inputs( incoming_template, subcollection_classifier ) - if collections_to_match.has_collections(): - collection_info = trans.app.dataset_collections_service.match_collections( collections_to_match ) - else: - collection_info = None - return expanded_incomings, collection_info else: expanded_incomings = permutations.expand_multi_inputs( incoming_template, collection_classifier ) if collections_to_match.has_collections(): diff -r c5b736d3e4e5027391a04fd187e2f9d2aa4ebe46 -r 9e502242af28473660ee70ee0f98345f3604e137 test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -264,7 +264,7 @@ history_id = self.dataset_populator.new_history() hdca_list_id = self.__build_nested_list( history_id ) inputs = { - "f1|__subcollection_multirun__": "%s|paired" % hdca_list_id + "f1|__collection_multirun__": "%s|paired" % hdca_list_id } # Following wait not really needed - just getting so many database # locked errors with sqlite. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 07 May '14

07 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fff767c478fb/ Changeset: fff767c478fb User: dannon Date: 2014-05-07 17:48:11 Summary: Merge stable. Affected #: 1 file https://bitbucket.org/galaxy/galaxy-central/commits/c5b736d3e4e5/ Changeset: c5b736d3e4e5 User: dannon Date: 2014-05-07 17:49:14 Summary: Merge. Affected #: 2 files diff -r fff767c478fbce151720d49729d5444f4f85cd28 -r c5b736d3e4e5027391a04fd187e2f9d2aa4ebe46 lib/galaxy/exceptions/__init__.py --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -104,16 +104,16 @@ class Conflict( MessageException ): status_code = 409 - err_code = error_codes.CONFLICT + err_code = error_codes.CONFLICT class InconsistentDatabase ( MessageException ): status_code = 500 - err_code = error_codes.INCONSISTENT_DATABASE + err_code = error_codes.INCONSISTENT_DATABASE class InternalServerError ( MessageException ): status_code = 500 - err_code = error_codes.INTERNAL_SERVER_ERROR + err_code = error_codes.INTERNAL_SERVER_ERROR class NotImplemented ( MessageException ): status_code = 501 - err_code = error_codes.NOT_IMPLEMENTED + err_code = error_codes.NOT_IMPLEMENTED diff -r fff767c478fbce151720d49729d5444f4f85cd28 -r c5b736d3e4e5027391a04fd187e2f9d2aa4ebe46 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -192,7 +192,6 @@ webapp.mapper.connect( 'workflow_usage', '/api/workflows/{workflow_id}/usage', controller='workflows', action='workflow_usage', conditions=dict(method=['GET'])) webapp.mapper.connect( 'workflow_usage_contents', '/api/workflows/{workflow_id}/usage/{usage_id}', controller='workflows', action='workflow_usage_contents', conditions=dict(method=['GET'])) - # ============================ # ===== AUTHENTICATE API ===== # ============================ @@ -248,8 +247,8 @@ parent_resources=dict( member_name='library', collection_name='libraries' ) ) _add_item_extended_metadata_controller( webapp, - name_prefix="library_dataset_", - path_prefix='/api/libraries/:library_id/contents/:library_content_id' ) + name_prefix="library_dataset_", + path_prefix='/api/libraries/:library_id/contents/:library_content_id' ) # ======================= # ===== FOLDERS API ===== Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: martenson: pep8 fixes
by commits-noreply＠bitbucket.org 07 May '14

07 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e8336f4c485b/ Changeset: e8336f4c485b User: martenson Date: 2014-05-07 17:47:28 Summary: pep8 fixes Affected #: 2 files diff -r a9e4f0ea892c8b7da3f1e468195dc1d29914fd38 -r e8336f4c485b0eedd75b9185f78ff5acaca7a8b3 lib/galaxy/exceptions/__init__.py --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -104,16 +104,16 @@ class Conflict( MessageException ): status_code = 409 - err_code = error_codes.CONFLICT + err_code = error_codes.CONFLICT class InconsistentDatabase ( MessageException ): status_code = 500 - err_code = error_codes.INCONSISTENT_DATABASE + err_code = error_codes.INCONSISTENT_DATABASE class InternalServerError ( MessageException ): status_code = 500 - err_code = error_codes.INTERNAL_SERVER_ERROR + err_code = error_codes.INTERNAL_SERVER_ERROR class NotImplemented ( MessageException ): status_code = 501 - err_code = error_codes.NOT_IMPLEMENTED + err_code = error_codes.NOT_IMPLEMENTED diff -r a9e4f0ea892c8b7da3f1e468195dc1d29914fd38 -r e8336f4c485b0eedd75b9185f78ff5acaca7a8b3 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -192,7 +192,6 @@ webapp.mapper.connect( 'workflow_usage', '/api/workflows/{workflow_id}/usage', controller='workflows', action='workflow_usage', conditions=dict(method=['GET'])) webapp.mapper.connect( 'workflow_usage_contents', '/api/workflows/{workflow_id}/usage/{usage_id}', controller='workflows', action='workflow_usage_contents', conditions=dict(method=['GET'])) - # ============================ # ===== AUTHENTICATE API ===== # ============================ @@ -248,8 +247,8 @@ parent_resources=dict( member_name='library', collection_name='libraries' ) ) _add_item_extended_metadata_controller( webapp, - name_prefix="library_dataset_", - path_prefix='/api/libraries/:library_id/contents/:library_content_id' ) + name_prefix="library_dataset_", + path_prefix='/api/libraries/:library_id/contents/:library_content_id' ) # ======================= # ===== FOLDERS API ===== Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Force an absolute path for the temp directory in the local runner
by commits-noreply＠bitbucket.org 07 May '14

07 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3534f67acd2a/ Changeset: 3534f67acd2a Branch: stable User: dannon Date: 2014-05-07 15:45:53 Summary: Force an absolute path for the temp directory in the local runner Affected #: 1 file diff -r 61b4a993d39c5341afea857859221a4f5b94334f -r 3534f67acd2a4f84899335326ac2cc8bcf05201d lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -39,7 +39,7 @@ #Set TEMP if a valid temp value is not already set if not ( 'TMPDIR' in self._environ or 'TEMP' in self._environ or 'TMP' in self._environ ): - self._environ[ 'TEMP' ] = tempfile.gettempdir() + self._environ[ 'TEMP' ] = os.path.abspath(tempfile.gettempdir()) super( LocalJobRunner, self ).__init__( app, nworkers ) self._init_worker_threads() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Force an absolute path for the temp directory in the local runner
by commits-noreply＠bitbucket.org 07 May '14

07 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a9e4f0ea892c/ Changeset: a9e4f0ea892c User: dannon Date: 2014-05-07 15:45:53 Summary: Force an absolute path for the temp directory in the local runner Affected #: 1 file diff -r ac95389c7eb502345d0321c54b750f134c5e6ba9 -r a9e4f0ea892c8b7da3f1e468195dc1d29914fd38 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -39,7 +39,7 @@ #Set TEMP if a valid temp value is not already set if not ( 'TMPDIR' in self._environ or 'TEMP' in self._environ or 'TMP' in self._environ ): - self._environ[ 'TEMP' ] = tempfile.gettempdir() + self._environ[ 'TEMP' ] = os.path.abspath(tempfile.gettempdir()) super( LocalJobRunner, self ).__init__( app, nworkers ) self._init_worker_threads() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix type introduced in renaming the InstallManager to the ToolMigrationManager - thanks Nicola Soranzo.
by commits-noreply＠bitbucket.org 07 May '14

07 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ac95389c7eb5/ Changeset: ac95389c7eb5 User: greg Date: 2014-05-07 12:42:28 Summary: Fix type introduced in renaming the InstallManager to the ToolMigrationManager - thanks Nicola Soranzo. Affected #: 2 files diff -r e7b579bec0e277f0c2e26cf23661ebfa7fa5ec2d -r ac95389c7eb502345d0321c54b750f134c5e6ba9 lib/tool_shed/galaxy_install/migrate/common.py --- a/lib/tool_shed/galaxy_install/migrate/common.py +++ b/lib/tool_shed/galaxy_install/migrate/common.py @@ -48,7 +48,7 @@ latest_migration_script_number = int( tools_migration_config.split( '_' )[ 0 ] ) # The value of migrated_tools_config is migrated_tools_conf.xml, and is reserved for # containing only those tools that have been eliminated from the distribution and moved - # to the tool shed. A side-effect of instantiating the ToolMigrationlManager is the automatic + # to the tool shed. A side-effect of instantiating the ToolMigrationManager is the automatic # installation of all appropriate tool shed repositories. self.tool_migration_manager = \ tool_migration_manager.ToolMigrationManager( app=self, diff -r e7b579bec0e277f0c2e26cf23661ebfa7fa5ec2d -r ac95389c7eb502345d0321c54b750f134c5e6ba9 lib/tool_shed/galaxy_install/tool_migration_manager.py --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -24,7 +24,7 @@ log = logging.getLogger( __name__ ) -class ToolMigrationlManager( object ): +class ToolMigrationManager( object ): def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ): """ @@ -446,7 +446,7 @@ from_tool_migration_manager=True ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR: - print '\nThe following error occurred from the InstallManager while installing tool dependency ', installed_tool_dependency.name, ':' + print '\nThe ToolMigrationManager returned the following error while installing tool dependency ', installed_tool_dependency.name, ':' print installed_tool_dependency.error_message, '\n\n' if 'datatypes' in metadata_dict: tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Expose use_remote_user via configuration API.
by commits-noreply＠bitbucket.org 07 May '14

07 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e7b579bec0e2/ Changeset: e7b579bec0e2 User: jmchilton Date: 2014-05-07 08:19:09 Summary: Expose use_remote_user via configuration API. Attempting to improve automated testing... marginally. Affected #: 1 file diff -r ea869e917062e0d92da847e50f210dcb1e0cbfd2 -r e7b579bec0e277f0c2e26cf23661ebfa7fa5ec2d lib/galaxy/webapps/galaxy/api/configuration.py --- a/lib/galaxy/webapps/galaxy/api/configuration.py +++ b/lib/galaxy/webapps/galaxy/api/configuration.py @@ -9,6 +9,7 @@ import logging log = logging.getLogger( __name__ ) + class ConfigurationController( BaseAPIController ): # config attributes viewable by non-admin users EXPOSED_USER_OPTIONS = [ @@ -20,6 +21,7 @@ 'logo_url', 'terms_url', 'allow_user_dataset_purge', + 'use_remote_user' ] # config attributes viewable by admin users EXPOSED_ADMIN_OPTIONS = [ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Improve API test skipping when required tools are absent.
by commits-noreply＠bitbucket.org 07 May '14

07 May '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ea869e917062/ Changeset: ea869e917062 User: jmchilton Date: 2014-05-07 07:24:00 Summary: Improve API test skipping when required tools are absent. Replace ad-hoc tools API test method for skipping tests with a more general purpose decorator. Use new decorator to specify required tools for workflow tests. Affected #: 3 files diff -r 7eb53364de36c60654ee6e89207a63b280c74af1 -r ea869e917062e0d92da847e50f210dcb1e0cbfd2 test/api/helpers.py --- a/test/api/helpers.py +++ b/test/api/helpers.py @@ -1,3 +1,5 @@ +from operator import itemgetter + import time import json import StringIO @@ -12,6 +14,35 @@ DEFAULT_HISTORY_TIMEOUT = 5 # Secs to wait on history to turn ok +def skip_without_tool( tool_id ): + """ Decorate an API test method as requiring a specific tool, + have nose skip the test case is the tool is unavailable. + """ + + def method_wrapper( method ): + + def get_tool_ids( api_test_case ): + index = api_test_case.galaxy_interactor.get( "tools", data=dict(in_panel=False) ) + tools = index.json() + # In panels by default, so flatten out sections... + tool_ids = map( itemgetter( "id" ), tools ) + return tool_ids + + def wrapped_method( api_test_case, *args, **kwargs ): + if tool_id not in get_tool_ids( api_test_case ): + from nose.plugins.skip import SkipTest + raise SkipTest( ) + + return method( api_test_case, *args, **kwargs ) + + # Must preserve method name so nose can detect and report tests by + # name. + wrapped_method.__name__ = method.__name__ + return wrapped_method + + return method_wrapper + + # Deprecated mixin, use dataset populator instead. # TODO: Rework existing tests to target DatasetPopulator in a setup method instead. class TestsDatasets: diff -r 7eb53364de36c60654ee6e89207a63b280c74af1 -r ea869e917062e0d92da847e50f210dcb1e0cbfd2 test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -3,6 +3,7 @@ from operator import itemgetter from .helpers import DatasetPopulator from .helpers import DatasetCollectionPopulator +from .helpers import skip_without_tool class ToolsTestCase( api.ApiTestCase ): @@ -51,8 +52,8 @@ result_content = self._upload_and_get_content( table ) self.assertEquals( result_content, table ) + @skip_without_tool( "cat1" ) def test_run_cat1( self ): - self.__skip_without_tool( "cat1" ) # Run simple non-upload tool with an input data parameter. history_id = self.dataset_populator.new_history() new_dataset = self.dataset_populator.new_dataset( history_id, content='Cat1Test' ) @@ -66,8 +67,8 @@ output1_content = self._get_content( history_id, dataset=output1 ) self.assertEqual( output1_content.strip(), "Cat1Test" ) + @skip_without_tool( "cat1" ) def test_run_cat1_with_two_inputs( self ): - self.__skip_without_tool( "cat1" ) # Run tool with an multiple data parameter and grouping (repeat) history_id = self.dataset_populator.new_history() new_dataset1 = self.dataset_populator.new_dataset( history_id, content='Cat1Test' ) @@ -83,8 +84,8 @@ output1_content = self._get_content( history_id, dataset=output1 ) self.assertEqual( output1_content.strip(), "Cat1Test\nCat2Test" ) + @skip_without_tool( "cat1" ) def test_multirun_cat1( self ): - self.__skip_without_tool( "cat1" ) history_id = self.dataset_populator.new_history() new_dataset1 = self.dataset_populator.new_dataset( history_id, content='123' ) new_dataset2 = self.dataset_populator.new_dataset( history_id, content='456' ) @@ -104,8 +105,8 @@ self.assertEquals( output1_content.strip(), "123" ) self.assertEquals( output2_content.strip(), "456" ) + @skip_without_tool( "cat1" ) def test_multirun_in_repeat( self ): - self.__skip_without_tool( "cat1" ) history_id = self.dataset_populator.new_history() new_dataset1 = self.dataset_populator.new_dataset( history_id, content='123' ) new_dataset2 = self.dataset_populator.new_dataset( history_id, content='456' ) @@ -127,8 +128,8 @@ self.assertEquals( output1_content.strip(), "Common\n123" ) self.assertEquals( output2_content.strip(), "Common\n456" ) + @skip_without_tool( "cat1" ) def test_multirun_on_multiple_inputs( self ): - self.__skip_without_tool( "cat1" ) history_id = self.dataset_populator.new_history() new_dataset1 = self.dataset_populator.new_dataset( history_id, content='123' ) new_dataset2 = self.dataset_populator.new_dataset( history_id, content='456' ) @@ -153,8 +154,8 @@ assert "123\n0ab" in outputs_contents assert "456\n0ab" in outputs_contents + @skip_without_tool( "cat1" ) def test_map_over_collection( self ): - self.__skip_without_tool( "cat1" ) history_id = self.dataset_populator.new_history() hdca_id = self.__build_pair( history_id, [ "123", "456" ] ) inputs = { @@ -175,8 +176,8 @@ self.assertEquals( output1_content.strip(), "123" ) self.assertEquals( output2_content.strip(), "456" ) + @skip_without_tool( "cat1" ) def test_map_over_nested_collections( self ): - self.__skip_without_tool( "cat1" ) history_id = self.dataset_populator.new_history() hdca_id = self.__build_nested_list( history_id ) inputs = { @@ -203,6 +204,7 @@ first_object_left_element = first_object[ "elements" ][ 0 ] self.assertEquals( outputs[ 0 ][ "id" ], first_object_left_element[ "object" ][ "id" ] ) + @skip_without_tool( "cat1" ) def test_map_over_two_collections( self ): history_id = self.dataset_populator.new_history() hdca1_id = self.__build_pair( history_id, [ "123", "456" ] ) @@ -221,8 +223,8 @@ self.assertEquals( output1_content.strip(), "123\n789" ) self.assertEquals( output2_content.strip(), "456\n0ab" ) + @skip_without_tool( "cat1" ) def test_cannot_map_over_incompatible_collections( self ): - self.__skip_without_tool( "cat1" ) history_id = self.dataset_populator.new_history() hdca1_id = self.__build_pair( history_id, [ "123", "456" ] ) hdca2_id = self.dataset_collection_populator.create_list_in_history( history_id ).json()[ "id" ] @@ -235,8 +237,8 @@ # on server. assert run_response.status_code >= 400 + @skip_without_tool( "multi_data_param" ) def test_reduce_collections( self ): - self.__skip_without_tool( "multi_data_param" ) history_id = self.dataset_populator.new_history() hdca1_id = self.__build_pair( history_id, [ "123", "456" ] ) hdca2_id = self.dataset_collection_populator.create_list_in_history( history_id ).json()[ "id" ] @@ -257,8 +259,8 @@ assert output1_content.strip() == "123\n456" assert len( output2_content.strip().split("\n") ) == 3, output2_content + @skip_without_tool( "collection_paired_test" ) def test_subcollection_mapping( self ): - self.__skip_without_tool( "collection_paired_test" ) history_id = self.dataset_populator.new_history() hdca_list_id = self.__build_nested_list( history_id ) inputs = { @@ -334,11 +336,6 @@ tool_ids = map( itemgetter( "id" ), tools ) return tool_ids - def __skip_without_tool( self, tool_id ): - from nose.plugins.skip import SkipTest - if tool_id not in self.__tool_ids( ): - raise SkipTest( ) - def __build_nested_list( self, history_id ): hdca1_id = self.__build_pair( history_id, [ "123", "456" ] ) hdca2_id = self.__build_pair( history_id, [ "789", "0ab" ] ) diff -r 7eb53364de36c60654ee6e89207a63b280c74af1 -r ea869e917062e0d92da847e50f210dcb1e0cbfd2 test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -5,6 +5,7 @@ from .helpers import WorkflowPopulator from .helpers import DatasetPopulator from .helpers import DatasetCollectionPopulator +from .helpers import skip_without_tool from base.interactor import delete_request # requests like delete @@ -62,6 +63,7 @@ first_input = downloaded_workflow[ "steps" ][ "0" ][ "inputs" ][ 0 ] assert first_input[ "name" ] == "WorkflowInput1" + @skip_without_tool( "cat1" ) def test_run_workflow( self ): workflow = self.workflow_populator.load_workflow( name="test_for_run" ) workflow_request, history_id = self._setup_workflow_run( workflow ) @@ -71,6 +73,7 @@ self._assert_status_code_is( run_workflow_response, 200 ) self.dataset_populator.wait_for_history( history_id, assert_ok=True ) + @skip_without_tool( "cat1" ) def test_extract_from_history( self ): workflow = self.workflow_populator.load_workflow( name="test_for_extract" ) workflow_request, history_id = self._setup_workflow_run( workflow ) @@ -104,6 +107,7 @@ self.assertEquals( downloaded_workflow[ "name" ], "test import from history" ) assert len( downloaded_workflow[ "steps" ] ) == 3 + @skip_without_tool( "collection_paired_test" ) def test_extract_workflows_with_dataset_collections( self ): history_id = self.dataset_populator.new_history() hdca = self.dataset_collection_populator.create_pair_in_history( history_id ).json() @@ -141,6 +145,7 @@ collection_step_state = loads( collection_step[ "tool_state" ] ) self.assertEquals( collection_step_state[ "collection_type" ], u"paired" ) + @skip_without_tool( "random_lines1" ) def test_run_replace_params_by_tool( self ): workflow_request, history_id = self._setup_random_x2_workflow( "test_for_replace_tool_params" ) workflow_request[ "parameters" ] = dumps( dict( random_lines1=dict( num_lines=5 ) ) ) @@ -151,6 +156,7 @@ self.__assert_lines_hid_line_count_is( history_id, 2, 5 ) self.__assert_lines_hid_line_count_is( history_id, 3, 5 ) + @skip_without_tool( "random_lines1" ) def test_run_replace_params_by_steps( self ): workflow_request, history_id = self._setup_random_x2_workflow( "test_for_replace_step_params" ) workflow_summary_response = self._get( "workflows/%s" % workflow_request[ "workflow_id" ] ) @@ -177,6 +183,7 @@ pja = pjas[ 0 ] self._assert_has_keys( pja, "action_type", "output_name", "action_arguments" ) + @skip_without_tool( "cat1" ) def test_invocation_usage( self ): workflow = self.workflow_populator.load_workflow( name="test_usage" ) workflow_request, history_id = self._setup_workflow_run( workflow ) @@ -200,6 +207,7 @@ for step in usage_details[ "steps" ].values(): self._assert_has_keys( step, "workflow_step_id", "order_index" ) + @skip_without_tool( "cat1" ) def test_post_job_action( self ): """ Tests both import and execution of post job actions. """ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 06 May '14

06 May '14

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a44bea8165f0/ Changeset: a44bea8165f0 User: jmchilton Date: 2014-05-06 23:08:30 Summary: Bugfix: galaxy.workflow.extract expects hids - API was consuming encoded ids. API now properly expects hids. That was unfortunate - need to figure out why the existing functional test did not break because of this and improve it. Affected #: 2 files diff -r c3370be77427ba8cdd575306d804b27facb9f49f -r a44bea8165f021e5f116bb99071526d4ab8dfb04 lib/galaxy/webapps/galaxy/api/workflows.py --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -171,8 +171,8 @@ from_history_id = payload.get( 'from_history_id' ) history = self.get_history( trans, from_history_id, check_ownership=False, check_accessible=True ) job_ids = map( trans.security.decode_id, payload.get( "job_ids", [] ) ) - dataset_ids = map( trans.security.decode_id, payload.get( "dataset_ids", [] ) ) - dataset_collection_ids = map( trans.security.decode_id, payload.get( "dataset_collection_ids", [] ) ) + dataset_ids = payload.get( "dataset_ids", [] ) + dataset_collection_ids = payload.get( "dataset_collection_ids", [] ) workflow_name = payload[ "workflow_name" ] stored_workflow = extract_workflow( trans=trans, diff -r c3370be77427ba8cdd575306d804b27facb9f49f -r a44bea8165f021e5f116bb99071526d4ab8dfb04 test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -73,7 +73,7 @@ workflow_request, history_id = self._setup_workflow_run( workflow ) contents_response = self._get( "histories/%s/contents" % history_id ) self._assert_status_code_is( contents_response, 200 ) - hda_ids = map( lambda c: c[ "id" ], contents_response.json() ) + hda_ids = map( lambda c: c[ "hid" ], contents_response.json() ) run_workflow_response = self._post( "workflows", data=workflow_request ) self._assert_status_code_is( run_workflow_response, 200 ) https://bitbucket.org/galaxy/galaxy-central/commits/7eb53364de36/ Changeset: 7eb53364de36 User: jmchilton Date: 2014-05-06 23:08:30 Summary: Bugfix: Track collection_type when extracting collection inputs from histories. Add functional test to verify the correctness of this tracking. TODO: refactor __skip_unless_tool from tools test so this new workflow test can be skipped if not run with -with_framework_test_tools. Affected #: 2 files diff -r a44bea8165f021e5f116bb99071526d4ab8dfb04 -r 7eb53364de36c60654ee6e89207a63b280c74af1 lib/galaxy/workflow/extract.py --- a/lib/galaxy/workflow/extract.py +++ b/lib/galaxy/workflow/extract.py @@ -87,7 +87,8 @@ for hid in dataset_collection_ids: step = model.WorkflowStep() step.type = 'data_collection_input' - step.tool_inputs = dict( name="Input Dataset Collection" ) + collection_type = summary.collection_types[ hid ] + step.tool_inputs = dict( name="Input Dataset Collection", collection_type=collection_type ) hid_to_output_pair[ hid ] = ( step, 'output' ) steps.append( step ) # Tool steps @@ -167,6 +168,8 @@ self.warnings = set() self.jobs = odict() self.implicit_map_jobs = [] + self.collection_types = {} + self.__summarize() def __summarize( self ): @@ -177,7 +180,9 @@ implicit_outputs = [] for content in self.history.active_contents: if content.history_content_type == "dataset_collection": + hid = content.hid content = self.__original_hdca( content ) + self.collection_types[ hid ] = content.collection.collection_type if not content.implicit_output_name: job = DatasetCollectionCreationJob( content ) self.jobs[ job ] = [ ( None, content ) ] diff -r a44bea8165f021e5f116bb99071526d4ab8dfb04 -r 7eb53364de36c60654ee6e89207a63b280c74af1 test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -1,8 +1,10 @@ from base import api from json import dumps +from json import loads import time from .helpers import WorkflowPopulator from .helpers import DatasetPopulator +from .helpers import DatasetCollectionPopulator from base.interactor import delete_request # requests like delete @@ -18,6 +20,7 @@ super( WorkflowsApiTestCase, self ).setUp() self.workflow_populator = WorkflowPopulator( self.galaxy_interactor ) self.dataset_populator = DatasetPopulator( self.galaxy_interactor ) + self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor ) def test_delete( self ): workflow_id = self.workflow_populator.simple_workflow( "test_delete" ) @@ -91,16 +94,53 @@ job_ids=dumps( [ cat1_job_id ] ), workflow_name="test import from history", ) - run_workflow_response = self._post( "workflows", data=create_from_data ) - self._assert_status_code_is( run_workflow_response, 200 ) + create_workflow_response = self._post( "workflows", data=create_from_data ) + self._assert_status_code_is( create_workflow_response, 200 ) - new_workflow_id = run_workflow_response.json()[ "id" ] + new_workflow_id = create_workflow_response.json()[ "id" ] download_response = self._get( "workflows/%s/download" % new_workflow_id ) self._assert_status_code_is( download_response, 200 ) downloaded_workflow = download_response.json() self.assertEquals( downloaded_workflow[ "name" ], "test import from history" ) assert len( downloaded_workflow[ "steps" ] ) == 3 + def test_extract_workflows_with_dataset_collections( self ): + history_id = self.dataset_populator.new_history() + hdca = self.dataset_collection_populator.create_pair_in_history( history_id ).json() + hdca_id = hdca[ "id" ] + inputs = { + "f1": dict( src="hdca", id=hdca_id ) + } + payload = self.dataset_populator.run_tool_payload( + tool_id="collection_paired_test", + inputs=inputs, + history_id=history_id, + ) + tool_response = self._post( "tools", data=payload ) + self._assert_status_code_is( tool_response, 200 ) + job_id = tool_response.json()[ "jobs" ][ 0 ][ "id" ] + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) + create_from_data = dict( + from_history_id=history_id, + dataset_collection_ids=dumps( [ hdca[ "hid" ] ] ), + job_ids=dumps( [ job_id ] ), + workflow_name="test import from history", + ) + create_workflow_response = self._post( "workflows", data=create_from_data ) + self._assert_status_code_is( create_workflow_response, 200 ) + create_workflow_response.json()[ "id" ] + + new_workflow_id = create_workflow_response.json()[ "id" ] + download_response = self._get( "workflows/%s/download" % new_workflow_id ) + self._assert_status_code_is( download_response, 200 ) + downloaded_workflow = download_response.json() + assert len( downloaded_workflow[ "steps" ] ) == 2 + collection_steps = [ s for s in downloaded_workflow[ "steps" ].values() if s[ "type" ] == "data_collection_input" ] + assert len( collection_steps ) == 1 + collection_step = collection_steps[ 0 ] + collection_step_state = loads( collection_step[ "tool_state" ] ) + self.assertEquals( collection_step_state[ "collection_type" ], u"paired" ) + def test_run_replace_params_by_tool( self ): workflow_request, history_id = self._setup_random_x2_workflow( "test_for_replace_tool_params" ) workflow_request[ "parameters" ] = dumps( dict( random_lines1=dict( num_lines=5 ) ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 06 May '14

06 May '14

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e3d9d40cd05a/ Changeset: e3d9d40cd05a User: jmchilton Date: 2014-05-06 20:52:49 Summary: More refactoring away from mixins and toward 'data populators' in API tests. Affected #: 3 files diff -r 6d6449af7d623b9b4a5bb0118750ad49f725b49a -r e3d9d40cd05a78d0a2202e68daab0c5c0f3b345f test/api/helpers.py --- a/test/api/helpers.py +++ b/test/api/helpers.py @@ -84,8 +84,8 @@ class WorkflowPopulator( object ): # Impulse is to make this a Mixin, but probably better as an object. - def __init__( self, api_test_case ): - self.api_test_case = api_test_case + def __init__( self, galaxy_interactor ): + self.galaxy_interactor = galaxy_interactor def load_workflow( self, name, content=workflow_str, add_pja=False ): workflow = json.loads( content ) @@ -111,7 +111,7 @@ workflow=json.dumps( workflow ), **create_kwds ) - upload_response = self.api_test_case._post( "workflows/upload", data=data ) + upload_response = self.galaxy_interactor.post( "workflows/upload", data=data ) uploaded_workflow_id = upload_response.json()[ "id" ] return uploaded_workflow_id diff -r 6d6449af7d623b9b4a5bb0118750ad49f725b49a -r e3d9d40cd05a78d0a2202e68daab0c5c0f3b345f test/api/test_search.py --- a/test/api/test_search.py +++ b/test/api/test_search.py @@ -7,7 +7,7 @@ class SearchApiTestCase( api.ApiTestCase ): def test_search_workflows( self ): - workflow_populator = WorkflowPopulator( self ) + workflow_populator = WorkflowPopulator( self.galaxy_interactor ) workflow_id = workflow_populator.simple_workflow( "test_for_search" ) search_response = self.__search( "select * from workflow" ) assert self.__has_result_with_name( search_response, "test_for_search (imported from API)" ), search_response.json() diff -r 6d6449af7d623b9b4a5bb0118750ad49f725b49a -r e3d9d40cd05a78d0a2202e68daab0c5c0f3b345f test/api/test_workflows.py --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -1,8 +1,8 @@ from base import api from json import dumps import time -from .helpers import TestsDatasets from .helpers import WorkflowPopulator +from .helpers import DatasetPopulator from base.interactor import delete_request # requests like delete @@ -12,11 +12,12 @@ # - Allow post to workflows/<workflow_id>/run in addition to posting to # /workflows with id in payload. # - Much more testing obviously, always more testing. -class WorkflowsApiTestCase( api.ApiTestCase, TestsDatasets ): +class WorkflowsApiTestCase( api.ApiTestCase ): def setUp( self ): super( WorkflowsApiTestCase, self ).setUp() - self.workflow_populator = WorkflowPopulator( self ) + self.workflow_populator = WorkflowPopulator( self.galaxy_interactor ) + self.dataset_populator = DatasetPopulator( self.galaxy_interactor ) def test_delete( self ): workflow_id = self.workflow_populator.simple_workflow( "test_delete" ) @@ -65,7 +66,7 @@ # something like that. run_workflow_response = self._post( "workflows", data=workflow_request ) self._assert_status_code_is( run_workflow_response, 200 ) - self._wait_for_history( history_id, assert_ok=True ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) def test_extract_from_history( self ): workflow = self.workflow_populator.load_workflow( name="test_for_extract" ) @@ -77,7 +78,7 @@ run_workflow_response = self._post( "workflows", data=workflow_request ) self._assert_status_code_is( run_workflow_response, 200 ) - self._wait_for_history( history_id, assert_ok=True ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) data = dict( history_id=history_id, tool_id="cat1" ) jobs_response = self._get( "jobs", data=data ) self._assert_status_code_is( jobs_response, 200 ) @@ -105,7 +106,7 @@ workflow_request[ "parameters" ] = dumps( dict( random_lines1=dict( num_lines=5 ) ) ) run_workflow_response = self._post( "workflows", data=workflow_request ) self._assert_status_code_is( run_workflow_response, 200 ) - self._wait_for_history( history_id, assert_ok=True ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) # Would be 8 and 6 without modification self.__assert_lines_hid_line_count_is( history_id, 2, 5 ) self.__assert_lines_hid_line_count_is( history_id, 3, 5 ) @@ -120,7 +121,7 @@ workflow_request[ "parameters" ] = params run_workflow_response = self._post( "workflows", data=workflow_request ) self._assert_status_code_is( run_workflow_response, 200 ) - self._wait_for_history( history_id, assert_ok=True ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) # Would be 8 and 6 without modification self.__assert_lines_hid_line_count_is( history_id, 2, 8 ) self.__assert_lines_hid_line_count_is( history_id, 3, 5 ) @@ -166,7 +167,7 @@ workflow_request, history_id = self._setup_workflow_run( workflow ) run_workflow_response = self._post( "workflows", data=workflow_request ) self._assert_status_code_is( run_workflow_response, 200 ) - self._wait_for_history( history_id, assert_ok=True ) + self.dataset_populator.wait_for_history( history_id, assert_ok=True ) time.sleep(.1) # Give another little bit of time for rename (needed?) contents = self._get( "histories/%s/contents" % history_id ).json() # loading workflow with add_pja=True causes workflow output to be @@ -183,9 +184,9 @@ step_1 = key if label == "WorkflowInput2": step_2 = key - history_id = self._new_history() - hda1 = self._new_dataset( history_id, content="1 2 3" ) - hda2 = self._new_dataset( history_id, content="4 5 6" ) + history_id = self.dataset_populator.new_history() + hda1 = self.dataset_populator.new_dataset( history_id, content="1 2 3" ) + hda2 = self.dataset_populator.new_dataset( history_id, content="4 5 6" ) workflow_request = dict( history="hist_id=%s" % history_id, workflow_id=uploaded_workflow_id, @@ -201,9 +202,9 @@ uploaded_workflow_id = self.workflow_populator.create_workflow( workflow ) workflow_inputs = self._workflow_inputs( uploaded_workflow_id ) key = workflow_inputs.keys()[ 0 ] - history_id = self._new_history() + history_id = self.dataset_populator.new_history() ten_lines = "\n".join( map( str, range( 10 ) ) ) - hda1 = self._new_dataset( history_id, content=ten_lines ) + hda1 = self.dataset_populator.new_dataset( history_id, content=ten_lines ) workflow_request = dict( history="hist_id=%s" % history_id, workflow_id=uploaded_workflow_id, https://bitbucket.org/galaxy/galaxy-central/commits/d82e6db63d3b/ Changeset: d82e6db63d3b User: jmchilton Date: 2014-05-06 20:52:49 Summary: Fix location of test_dataset_collections.py. Didn't rebase properly. Affected #: 2 files diff -r e3d9d40cd05a78d0a2202e68daab0c5c0f3b345f -r d82e6db63d3b581dadedcb4cc38e150445b9637c test/api/test_dataset_collections.py --- /dev/null +++ b/test/api/test_dataset_collections.py @@ -0,0 +1,206 @@ +from base import api +import json +from .helpers import DatasetPopulator + + +# TODO: Move into helpers with rest of populators +class DatasetCollectionPopulator( object ): + + def __init__( self, galaxy_interactor ): + self.galaxy_interactor = galaxy_interactor + self.dataset_populator = DatasetPopulator( galaxy_interactor ) + + def create_list_from_pairs( self, history_id, pairs ): + element_identifiers = [] + for i, pair in enumerate( pairs ): + element_identifiers.append( dict( + name="test%d" % i, + src="hdca", + id=pair + ) ) + + payload = dict( + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(element_identifiers), + collection_type="list:paired", + ) + return self.__create( payload ) + + def create_pair_in_history( self, history_id, **kwds ): + payload = self.create_pair_payload( + history_id, + instance_type="history", + **kwds + ) + return self.__create( payload ) + + def create_list_in_history( self, history_id, **kwds ): + payload = self.create_list_payload( + history_id, + instance_type="history", + **kwds + ) + return self.__create( payload ) + + def create_list_payload( self, history_id, **kwds ): + return self.__create_payload( history_id, identifiers_func=self.list_identifiers, collection_type="list", **kwds ) + + def create_pair_payload( self, history_id, **kwds ): + return self.__create_payload( history_id, identifiers_func=self.pair_identifiers, collection_type="paired", **kwds ) + + def __create_payload( self, history_id, identifiers_func, collection_type, **kwds ): + contents = None + if "contents" in kwds: + contents = kwds[ "contents" ] + del kwds[ "contents" ] + + if "element_identifiers" not in kwds: + kwds[ "element_identifiers" ] = json.dumps( identifiers_func( history_id, contents=contents ) ) + + payload = dict( + history_id=history_id, + collection_type=collection_type, + **kwds + ) + return payload + + def pair_identifiers( self, history_id, contents=None ): + hda1, hda2 = self.__datasets( history_id, count=2, contents=contents ) + + element_identifiers = [ + dict( name="left", src="hda", id=hda1[ "id" ] ), + dict( name="right", src="hda", id=hda2[ "id" ] ), + ] + return element_identifiers + + def list_identifiers( self, history_id, contents=None ): + hda1, hda2, hda3 = self.__datasets( history_id, count=3, contents=contents ) + element_identifiers = [ + dict( name="data1", src="hda", id=hda1[ "id" ] ), + dict( name="data2", src="hda", id=hda2[ "id" ] ), + dict( name="data3", src="hda", id=hda3[ "id" ] ), + ] + return element_identifiers + + def __create( self, payload ): + create_response = self.galaxy_interactor.post( "dataset_collections", data=payload ) + return create_response + + def __datasets( self, history_id, count, contents=None ): + datasets = [] + for i in xrange( count ): + new_kwds = {} + if contents: + new_kwds[ "content" ] = contents[ i ] + datasets.append( self.dataset_populator.new_dataset( history_id, **new_kwds ) ) + return datasets + + +class DatasetCollectionApiTestCase( api.ApiTestCase ): + + def setUp( self ): + super( DatasetCollectionApiTestCase, self ).setUp() + self.dataset_populator = DatasetPopulator( self.galaxy_interactor ) + self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor ) + self.history_id = self.dataset_populator.new_history() + + def test_create_pair_from_history( self ): + payload = self.dataset_collection_populator.create_pair_payload( + self.history_id, + instance_type="history", + ) + create_response = self._post( "dataset_collections", payload ) + dataset_collection = self._check_create_response( create_response ) + returned_datasets = dataset_collection[ "elements" ] + assert len( returned_datasets ) == 2, dataset_collection + + def test_create_list_from_history( self ): + element_identifiers = self.dataset_collection_populator.list_identifiers( self.history_id ) + + payload = dict( + instance_type="history", + history_id=self.history_id, + element_identifiers=json.dumps(element_identifiers), + collection_type="list", + ) + + create_response = self._post( "dataset_collections", payload ) + dataset_collection = self._check_create_response( create_response ) + returned_datasets = dataset_collection[ "elements" ] + assert len( returned_datasets ) == 3, dataset_collection + + def test_create_list_of_existing_pairs( self ): + pair_payload = self.dataset_collection_populator.create_pair_payload( + self.history_id, + instance_type="history", + ) + pair_create_response = self._post( "dataset_collections", pair_payload ) + dataset_collection = self._check_create_response( pair_create_response ) + hdca_id = dataset_collection[ "id" ] + + element_identifiers = [ + dict( name="test1", src="hdca", id=hdca_id ) + ] + + payload = dict( + instance_type="history", + history_id=self.history_id, + element_identifiers=json.dumps(element_identifiers), + collection_type="list", + ) + create_response = self._post( "dataset_collections", payload ) + dataset_collection = self._check_create_response( create_response ) + returned_collections = dataset_collection[ "elements" ] + assert len( returned_collections ) == 1, dataset_collection + + def test_create_list_of_new_pairs( self ): + pair_identifiers = self.dataset_collection_populator.pair_identifiers( self.history_id ) + element_identifiers = [ dict( + src="new_collection", + name="test_pair", + collection_type="paired", + element_identifiers=pair_identifiers, + ) ] + payload = dict( + collection_type="list:paired", + instance_type="history", + history_id=self.history_id, + name="nested_collecion", + element_identifiers=json.dumps( element_identifiers ), + ) + create_response = self._post( "dataset_collections", payload ) + dataset_collection = self._check_create_response( create_response ) + assert dataset_collection[ "collection_type" ] == "list:paired" + returned_collections = dataset_collection[ "elements" ] + assert len( returned_collections ) == 1, dataset_collection + pair_1_element = returned_collections[ 0 ] + self._assert_has_keys( pair_1_element, "element_index" ) + pair_1_object = pair_1_element[ "object" ] + self._assert_has_keys( pair_1_object, "collection_type", "elements" ) + self.assertEquals( pair_1_object[ "collection_type" ], "paired" ) + pair_elements = pair_1_object[ "elements" ] + assert len( pair_elements ) == 2 + pair_1_element_1 = pair_elements[ 0 ] + assert pair_1_element_1[ "element_index" ] == 0 + + def test_hda_security( self ): + element_identifiers = self.dataset_collection_populator.pair_identifiers( self.history_id ) + + with self._different_user( ): + history_id = self.dataset_populator.new_history() + payload = dict( + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(element_identifiers), + collection_type="paired", + ) + + create_response = self._post( "dataset_collections", payload ) + self._assert_status_code_is( create_response, 403 ) + + def _check_create_response( self, create_response ): + self._assert_status_code_is( create_response, 200 ) + dataset_collection = create_response.json() + self._assert_has_keys( dataset_collection, "elements", "url", "name", "collection_type" ) + return dataset_collection diff -r e3d9d40cd05a78d0a2202e68daab0c5c0f3b345f -r d82e6db63d3b581dadedcb4cc38e150445b9637c test/functional/api/test_dataset_collections.py --- a/test/functional/api/test_dataset_collections.py +++ /dev/null @@ -1,206 +0,0 @@ -from base import api -import json -from .helpers import DatasetPopulator - - -# TODO: Move into helpers with rest of populators -class DatasetCollectionPopulator( object ): - - def __init__( self, galaxy_interactor ): - self.galaxy_interactor = galaxy_interactor - self.dataset_populator = DatasetPopulator( galaxy_interactor ) - - def create_list_from_pairs( self, history_id, pairs ): - element_identifiers = [] - for i, pair in enumerate( pairs ): - element_identifiers.append( dict( - name="test%d" % i, - src="hdca", - id=pair - ) ) - - payload = dict( - instance_type="history", - history_id=history_id, - element_identifiers=json.dumps(element_identifiers), - collection_type="list:paired", - ) - return self.__create( payload ) - - def create_pair_in_history( self, history_id, **kwds ): - payload = self.create_pair_payload( - history_id, - instance_type="history", - **kwds - ) - return self.__create( payload ) - - def create_list_in_history( self, history_id, **kwds ): - payload = self.create_list_payload( - history_id, - instance_type="history", - **kwds - ) - return self.__create( payload ) - - def create_list_payload( self, history_id, **kwds ): - return self.__create_payload( history_id, identifiers_func=self.list_identifiers, collection_type="list", **kwds ) - - def create_pair_payload( self, history_id, **kwds ): - return self.__create_payload( history_id, identifiers_func=self.pair_identifiers, collection_type="paired", **kwds ) - - def __create_payload( self, history_id, identifiers_func, collection_type, **kwds ): - contents = None - if "contents" in kwds: - contents = kwds[ "contents" ] - del kwds[ "contents" ] - - if "element_identifiers" not in kwds: - kwds[ "element_identifiers" ] = json.dumps( identifiers_func( history_id, contents=contents ) ) - - payload = dict( - history_id=history_id, - collection_type=collection_type, - **kwds - ) - return payload - - def pair_identifiers( self, history_id, contents=None ): - hda1, hda2 = self.__datasets( history_id, count=2, contents=contents ) - - element_identifiers = [ - dict( name="left", src="hda", id=hda1[ "id" ] ), - dict( name="right", src="hda", id=hda2[ "id" ] ), - ] - return element_identifiers - - def list_identifiers( self, history_id, contents=None ): - hda1, hda2, hda3 = self.__datasets( history_id, count=3, contents=contents ) - element_identifiers = [ - dict( name="data1", src="hda", id=hda1[ "id" ] ), - dict( name="data2", src="hda", id=hda2[ "id" ] ), - dict( name="data3", src="hda", id=hda3[ "id" ] ), - ] - return element_identifiers - - def __create( self, payload ): - create_response = self.galaxy_interactor.post( "dataset_collections", data=payload ) - return create_response - - def __datasets( self, history_id, count, contents=None ): - datasets = [] - for i in xrange( count ): - new_kwds = {} - if contents: - new_kwds[ "content" ] = contents[ i ] - datasets.append( self.dataset_populator.new_dataset( history_id, **new_kwds ) ) - return datasets - - -class DatasetCollectionApiTestCase( api.ApiTestCase ): - - def setUp( self ): - super( DatasetCollectionApiTestCase, self ).setUp() - self.dataset_populator = DatasetPopulator( self.galaxy_interactor ) - self.dataset_collection_populator = DatasetCollectionPopulator( self.galaxy_interactor ) - self.history_id = self.dataset_populator.new_history() - - def test_create_pair_from_history( self ): - payload = self.dataset_collection_populator.create_pair_payload( - self.history_id, - instance_type="history", - ) - create_response = self._post( "dataset_collections", payload ) - dataset_collection = self._check_create_response( create_response ) - returned_datasets = dataset_collection[ "elements" ] - assert len( returned_datasets ) == 2, dataset_collection - - def test_create_list_from_history( self ): - element_identifiers = self.dataset_collection_populator.list_identifiers( self.history_id ) - - payload = dict( - instance_type="history", - history_id=self.history_id, - element_identifiers=json.dumps(element_identifiers), - collection_type="list", - ) - - create_response = self._post( "dataset_collections", payload ) - dataset_collection = self._check_create_response( create_response ) - returned_datasets = dataset_collection[ "elements" ] - assert len( returned_datasets ) == 3, dataset_collection - - def test_create_list_of_existing_pairs( self ): - pair_payload = self.dataset_collection_populator.create_pair_payload( - self.history_id, - instance_type="history", - ) - pair_create_response = self._post( "dataset_collections", pair_payload ) - dataset_collection = self._check_create_response( pair_create_response ) - hdca_id = dataset_collection[ "id" ] - - element_identifiers = [ - dict( name="test1", src="hdca", id=hdca_id ) - ] - - payload = dict( - instance_type="history", - history_id=self.history_id, - element_identifiers=json.dumps(element_identifiers), - collection_type="list", - ) - create_response = self._post( "dataset_collections", payload ) - dataset_collection = self._check_create_response( create_response ) - returned_collections = dataset_collection[ "elements" ] - assert len( returned_collections ) == 1, dataset_collection - - def test_create_list_of_new_pairs( self ): - pair_identifiers = self.dataset_collection_populator.pair_identifiers( self.history_id ) - element_identifiers = [ dict( - src="new_collection", - name="test_pair", - collection_type="paired", - element_identifiers=pair_identifiers, - ) ] - payload = dict( - collection_type="list:paired", - instance_type="history", - history_id=self.history_id, - name="nested_collecion", - element_identifiers=json.dumps( element_identifiers ), - ) - create_response = self._post( "dataset_collections", payload ) - dataset_collection = self._check_create_response( create_response ) - assert dataset_collection[ "collection_type" ] == "list:paired" - returned_collections = dataset_collection[ "elements" ] - assert len( returned_collections ) == 1, dataset_collection - pair_1_element = returned_collections[ 0 ] - self._assert_has_keys( pair_1_element, "element_index" ) - pair_1_object = pair_1_element[ "object" ] - self._assert_has_keys( pair_1_object, "collection_type", "elements" ) - self.assertEquals( pair_1_object[ "collection_type" ], "paired" ) - pair_elements = pair_1_object[ "elements" ] - assert len( pair_elements ) == 2 - pair_1_element_1 = pair_elements[ 0 ] - assert pair_1_element_1[ "element_index" ] == 0 - - def test_hda_security( self ): - element_identifiers = self.dataset_collection_populator.pair_identifiers( self.history_id ) - - with self._different_user( ): - history_id = self.dataset_populator.new_history() - payload = dict( - instance_type="history", - history_id=history_id, - element_identifiers=json.dumps(element_identifiers), - collection_type="paired", - ) - - create_response = self._post( "dataset_collections", payload ) - self._assert_status_code_is( create_response, 403 ) - - def _check_create_response( self, create_response ): - self._assert_status_code_is( create_response, 200 ) - dataset_collection = create_response.json() - self._assert_has_keys( dataset_collection, "elements", "url", "name", "collection_type" ) - return dataset_collection https://bitbucket.org/galaxy/galaxy-central/commits/4cf29a388e80/ Changeset: 4cf29a388e80 User: jmchilton Date: 2014-05-06 20:52:49 Summary: Move dataset colleciton test populator. Affected #: 4 files diff -r d82e6db63d3b581dadedcb4cc38e150445b9637c -r 4cf29a388e802d073d28deb7c365c6ec5f555122 test/api/helpers.py --- a/test/api/helpers.py +++ b/test/api/helpers.py @@ -189,6 +189,99 @@ return show().json() +class DatasetCollectionPopulator( object ): + + def __init__( self, galaxy_interactor ): + self.galaxy_interactor = galaxy_interactor + self.dataset_populator = DatasetPopulator( galaxy_interactor ) + + def create_list_from_pairs( self, history_id, pairs ): + element_identifiers = [] + for i, pair in enumerate( pairs ): + element_identifiers.append( dict( + name="test%d" % i, + src="hdca", + id=pair + ) ) + + payload = dict( + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(element_identifiers), + collection_type="list:paired", + ) + return self.__create( payload ) + + def create_pair_in_history( self, history_id, **kwds ): + payload = self.create_pair_payload( + history_id, + instance_type="history", + **kwds + ) + return self.__create( payload ) + + def create_list_in_history( self, history_id, **kwds ): + payload = self.create_list_payload( + history_id, + instance_type="history", + **kwds + ) + return self.__create( payload ) + + def create_list_payload( self, history_id, **kwds ): + return self.__create_payload( history_id, identifiers_func=self.list_identifiers, collection_type="list", **kwds ) + + def create_pair_payload( self, history_id, **kwds ): + return self.__create_payload( history_id, identifiers_func=self.pair_identifiers, collection_type="paired", **kwds ) + + def __create_payload( self, history_id, identifiers_func, collection_type, **kwds ): + contents = None + if "contents" in kwds: + contents = kwds[ "contents" ] + del kwds[ "contents" ] + + if "element_identifiers" not in kwds: + kwds[ "element_identifiers" ] = json.dumps( identifiers_func( history_id, contents=contents ) ) + + payload = dict( + history_id=history_id, + collection_type=collection_type, + **kwds + ) + return payload + + def pair_identifiers( self, history_id, contents=None ): + hda1, hda2 = self.__datasets( history_id, count=2, contents=contents ) + + element_identifiers = [ + dict( name="left", src="hda", id=hda1[ "id" ] ), + dict( name="right", src="hda", id=hda2[ "id" ] ), + ] + return element_identifiers + + def list_identifiers( self, history_id, contents=None ): + hda1, hda2, hda3 = self.__datasets( history_id, count=3, contents=contents ) + element_identifiers = [ + dict( name="data1", src="hda", id=hda1[ "id" ] ), + dict( name="data2", src="hda", id=hda2[ "id" ] ), + dict( name="data3", src="hda", id=hda3[ "id" ] ), + ] + return element_identifiers + + def __create( self, payload ): + create_response = self.galaxy_interactor.post( "dataset_collections", data=payload ) + return create_response + + def __datasets( self, history_id, count, contents=None ): + datasets = [] + for i in xrange( count ): + new_kwds = {} + if contents: + new_kwds[ "content" ] = contents[ i ] + datasets.append( self.dataset_populator.new_dataset( history_id, **new_kwds ) ) + return datasets + + def wait_on_state( state_func, assert_ok=False, timeout=5 ): delta = .1 iteration = 0 diff -r d82e6db63d3b581dadedcb4cc38e150445b9637c -r 4cf29a388e802d073d28deb7c365c6ec5f555122 test/api/test_dataset_collections.py --- a/test/api/test_dataset_collections.py +++ b/test/api/test_dataset_collections.py @@ -1,100 +1,7 @@ from base import api import json from .helpers import DatasetPopulator - - -# TODO: Move into helpers with rest of populators -class DatasetCollectionPopulator( object ): - - def __init__( self, galaxy_interactor ): - self.galaxy_interactor = galaxy_interactor - self.dataset_populator = DatasetPopulator( galaxy_interactor ) - - def create_list_from_pairs( self, history_id, pairs ): - element_identifiers = [] - for i, pair in enumerate( pairs ): - element_identifiers.append( dict( - name="test%d" % i, - src="hdca", - id=pair - ) ) - - payload = dict( - instance_type="history", - history_id=history_id, - element_identifiers=json.dumps(element_identifiers), - collection_type="list:paired", - ) - return self.__create( payload ) - - def create_pair_in_history( self, history_id, **kwds ): - payload = self.create_pair_payload( - history_id, - instance_type="history", - **kwds - ) - return self.__create( payload ) - - def create_list_in_history( self, history_id, **kwds ): - payload = self.create_list_payload( - history_id, - instance_type="history", - **kwds - ) - return self.__create( payload ) - - def create_list_payload( self, history_id, **kwds ): - return self.__create_payload( history_id, identifiers_func=self.list_identifiers, collection_type="list", **kwds ) - - def create_pair_payload( self, history_id, **kwds ): - return self.__create_payload( history_id, identifiers_func=self.pair_identifiers, collection_type="paired", **kwds ) - - def __create_payload( self, history_id, identifiers_func, collection_type, **kwds ): - contents = None - if "contents" in kwds: - contents = kwds[ "contents" ] - del kwds[ "contents" ] - - if "element_identifiers" not in kwds: - kwds[ "element_identifiers" ] = json.dumps( identifiers_func( history_id, contents=contents ) ) - - payload = dict( - history_id=history_id, - collection_type=collection_type, - **kwds - ) - return payload - - def pair_identifiers( self, history_id, contents=None ): - hda1, hda2 = self.__datasets( history_id, count=2, contents=contents ) - - element_identifiers = [ - dict( name="left", src="hda", id=hda1[ "id" ] ), - dict( name="right", src="hda", id=hda2[ "id" ] ), - ] - return element_identifiers - - def list_identifiers( self, history_id, contents=None ): - hda1, hda2, hda3 = self.__datasets( history_id, count=3, contents=contents ) - element_identifiers = [ - dict( name="data1", src="hda", id=hda1[ "id" ] ), - dict( name="data2", src="hda", id=hda2[ "id" ] ), - dict( name="data3", src="hda", id=hda3[ "id" ] ), - ] - return element_identifiers - - def __create( self, payload ): - create_response = self.galaxy_interactor.post( "dataset_collections", data=payload ) - return create_response - - def __datasets( self, history_id, count, contents=None ): - datasets = [] - for i in xrange( count ): - new_kwds = {} - if contents: - new_kwds[ "content" ] = contents[ i ] - datasets.append( self.dataset_populator.new_dataset( history_id, **new_kwds ) ) - return datasets +from .helpers import DatasetCollectionPopulator class DatasetCollectionApiTestCase( api.ApiTestCase ): diff -r d82e6db63d3b581dadedcb4cc38e150445b9637c -r 4cf29a388e802d073d28deb7c365c6ec5f555122 test/api/test_history_contents.py --- a/test/api/test_history_contents.py +++ b/test/api/test_history_contents.py @@ -3,7 +3,7 @@ from .helpers import TestsDatasets from .helpers import LibraryPopulator -from .test_dataset_collections import DatasetCollectionPopulator +from .helpers import DatasetCollectionPopulator from base.interactor import ( put_request, delete_request, diff -r d82e6db63d3b581dadedcb4cc38e150445b9637c -r 4cf29a388e802d073d28deb7c365c6ec5f555122 test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -2,7 +2,7 @@ from base import api from operator import itemgetter from .helpers import DatasetPopulator -from .test_dataset_collections import DatasetCollectionPopulator +from .helpers import DatasetCollectionPopulator class ToolsTestCase( api.ApiTestCase ): https://bitbucket.org/galaxy/galaxy-central/commits/c3370be77427/ Changeset: c3370be77427 User: jmchilton Date: 2014-05-06 20:52:49 Summary: Fix bug in test/api/test_tools.py. Affected #: 1 file diff -r 4cf29a388e802d073d28deb7c365c6ec5f555122 -r c3370be77427ba8cdd575306d804b27facb9f49f test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -167,7 +167,6 @@ self.assertEquals( len( jobs ), 2 ) self.assertEquals( len( outputs ), 2 ) self.assertEquals( len( implicit_collections ), 1 ) - self.dataset_populator.wait_for_history( history_id, assert_ok=True ) output1 = outputs[ 0 ] output2 = outputs[ 1 ] @@ -285,7 +284,8 @@ return self._run_outputs( self._run( tool_id, history_id, inputs ) ) def _run_outputs( self, create_response ): - self._assert_status_code_is( create_response, 200, assert_ok=True )[ 'outputs' ] + self._assert_status_code_is( create_response, 200 ) + return create_response.json()[ 'outputs' ] def _run_cat1( self, history_id, inputs, assert_ok=False ): return self._run( 'cat1', history_id, inputs, assert_ok=assert_ok ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0