3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/55d684093172/ Changeset: 55d684093172 User: Dave Bouvier Date: 2013-11-25 19:01:44 Summary: Fix for importing repositories. Affected #: 1 file diff -r 8ae8e19ea0762af833558f75b84fe19840c6e783 -r 55d684093172360d6329349337313da202e94108 lib/tool_shed/util/import_util.py --- a/lib/tool_shed/util/import_util.py +++ b/lib/tool_shed/util/import_util.py @@ -164,7 +164,7 @@ # No repository with the specified name and owner currently exists, so make sure the current user can create one. if trans.user_is_admin(): repository_info_dict[ 'status' ] = None - elif trans.app.security_agent.user_can_import_repository_archive( trans.user, owner ): + elif trans.app.security_agent.user_can_import_repository_archive( trans.user, repository_info_dict[ 'owner' ] ): repository_info_dict[ 'status' ] = None else: repository_info_dict[ 'status' ] = 'Not authorized to import' https://bitbucket.org/galaxy/galaxy-central/commits/398287552d91/ Changeset: 398287552d91 User: Dave Bouvier Date: 2013-11-25 19:02:09 Summary: Functional tests for importing and exporting repositories. Affected #: 3 files diff -r 55d684093172360d6329349337313da202e94108 -r 398287552d91093721ea0de635ed6b0d7c7cd147 test/tool_shed/test_data/repository_capsules/0490_filtering.tar.gz Binary file test/tool_shed/test_data/repository_capsules/0490_filtering.tar.gz has changed diff -r 55d684093172360d6329349337313da202e94108 -r 398287552d91093721ea0de635ed6b0d7c7cd147 test/tool_shed/test_data/repository_capsules/0500_emboss_5.tar.gz Binary file test/tool_shed/test_data/repository_capsules/0500_emboss_5.tar.gz has changed diff -r 55d684093172360d6329349337313da202e94108 -r 398287552d91093721ea0de635ed6b0d7c7cd147 test/tool_shed/test_data/repository_capsules/0510_trans_proteomic_pipeline.tar.gz Binary file test/tool_shed/test_data/repository_capsules/0510_trans_proteomic_pipeline.tar.gz has changed https://bitbucket.org/galaxy/galaxy-central/commits/d37d6e711d67/ Changeset: d37d6e711d67 User: Dave Bouvier Date: 2013-11-25 19:02:09 Summary: Functional tests for importing and exporting repositories. Affected #: 5 files diff -r 398287552d91093721ea0de635ed6b0d7c7cd147 -r d37d6e711d671763d725ffb4931f0acc379964dc test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -4,6 +4,7 @@ import re import test_db_util import simplejson +import shutil import logging import time import tempfile @@ -12,6 +13,7 @@ import galaxy.model as galaxy_model import galaxy.util as util from tool_shed.util import shed_util_common as suc +from tool_shed.util import xml_util from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib from tool_shed.util.encoding_util import tool_shed_encode, tool_shed_decode @@ -98,6 +100,16 @@ self.check_repository_changelog( repository ) self.check_string_count_in_page( 'Repository metadata is associated with this change set.', metadata_count ) + def check_exported_repository_dependency( self, dependency_filename, repository_name, repository_owner ): + root, error_message = xml_util.parse_xml( dependency_filename ) + for elem in root.findall( 'repository' ): + if 'changeset_revision' in elem: + raise AssertionError( 'Exported repository %s with owner %s has a dependency with a defined changeset revision.' % \ + ( repository_name, repository_owner ) ) + if 'toolshed' in elem: + raise AssertionError( 'Exported repository %s with owner %s has a dependency with a defined tool shed.' % \ + ( repository_name, repository_owner ) ) + def check_for_valid_tools( self, repository, strings_displayed=[], strings_not_displayed=[] ): strings_displayed.append( 'Valid tools' ) self.display_manage_repository_page( repository, strings_displayed, strings_not_displayed ) @@ -144,6 +156,24 @@ self.visit_galaxy_url( url ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def check_manifest( self, manifest_filepath, owner=None ): + root, error_message = xml_util.parse_xml( manifest_filepath ) + for elem in root.findall( 'repository' ): + repository_name = elem.get( 'name' ) + manifest_owner = elem.get( 'username' ) + if owner is not None: + assert manifest_owner == owner, 'Expected repository %s to be owned by %s, but found %s' % \ + ( elem.get( 'name' ), owner, manifest_owner ) + toolshed = elem.get( 'toolshed' ) + changeset_revision = elem.get( 'changeset_revision' ) + assert toolshed is None, 'Repository definition %s has a tool shed attribute %s.' % ( repository_name, toolshed ) + assert changeset_revision is None, 'Repository definition %s specifies a changeset revision %s.' % \ + ( repository_name, changeset_revision ) + repository_archive = elem.find( 'archive' ).text + filepath, filename = os.path.split( manifest_filepath ) + repository_path = os.path.join( filepath, repository_archive ) + self.verify_repository_in_capsule( repository_path, repository_name, owner ) + def check_repository_changelog( self, repository, strings_displayed=[], strings_not_displayed=[] ): url = '/repository/view_changelog?id=%s' % self.security.encode_id( repository.id ) self.visit_url( url ) @@ -1207,6 +1237,15 @@ tc.submit( "upload_button" ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def verify_capsule_contents( self, capsule_filepath, owner ): + tar_object = tarfile.open( capsule_filepath, 'r:*' ) + extraction_path = tempfile.mkdtemp() + tar_object.extractall( extraction_path ) + for root, dirs, files in os.walk( extraction_path ): + if 'manifest.xml' in files: + self.check_manifest( os.path.join( root, 'manifest.xml' ), owner=owner ) + shutil.rmtree( extraction_path ) + def verify_installed_repositories( self, installed_repositories=[], uninstalled_repositories=[] ): for repository_name, repository_owner in installed_repositories: galaxy_repository = test_db_util.get_installed_repository_by_name_owner( repository_name, repository_owner ) @@ -1279,6 +1318,17 @@ # or we know that the repository was not correctly installed! assert found, 'No entry for %s in %s.' % ( required_data_table_entry, self.shed_tool_data_table_conf ) + def verify_repository_in_capsule( self, repository_archive, repository_name, repository_owner ): + repository_extraction_dir = tempfile.mkdtemp() + repository_tar_object = tarfile.open( repository_archive, 'r:*' ) + repository_tar_object.extractall( repository_extraction_dir ) + for root, dirs, files in os.walk( repository_extraction_dir ): + for filename in files: + if filename in [ 'tool_dependencies.xml', 'repository_dependencies.xml' ]: + dependency_filepath = os.path.join( root, filename ) + self.check_exported_repository_dependency( dependency_filepath, repository_name, repository_owner ) + shutil.rmtree( repository_extraction_dir ) + def verify_repository_reviews( self, repository, reviewer=None, strings_displayed=[], strings_not_displayed=[] ): changeset_revision = self.get_repository_tip( repository ) # Verify that the currently logged in user has a repository review for the specified repository, reviewer, and changeset revision. diff -r 398287552d91093721ea0de635ed6b0d7c7cd147 -r d37d6e711d671763d725ffb4931f0acc379964dc test/tool_shed/functional/test_0490_export_import_repositories.py --- /dev/null +++ b/test/tool_shed/functional/test_0490_export_import_repositories.py @@ -0,0 +1,90 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +import logging +log = logging.getLogger( __name__ ) + +repository_name = 'filtering_0490' +repository_description = "Galaxy's filtering tool for test 0490" +repository_long_description = "Long description of Galaxy's filtering tool for test 0490" + +category_name = 'Test 0490 Repository Import Export' +category_description = 'Test script 0490 for importing and exporting single repositories.' + +''' +First test: + +1. Import a repository capsule containing a repository with no dependencies, e.g. filter1. +2. Check that the repository to be imported is not marked as preexisting. The word ' Exists' should not be displayed, but '<b>Exists</b>' will. +3. Export that repository. Check the capsule's contents, verify that changeset revision and tool shed are not set. +4. Import the capsule again. Check that the repository to be imported is marked as preexisting. The word ' Exists' should be + displayed, as will '<b>Exists</b>'. + +''' + +class TestExportImportRepository( ShedTwillTestCase ): + '''Test exporting and importing repositories.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts and login as an admin user.""" + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + + def test_0005_import_repository_capsule( self ): + """Import the filter_0490 repository capsule.""" + ''' + This is step 1 - Import a repository capsule containing a repository with no dependencies, e.g. filter1. + Check that the repository to be imported is not marked as preexisting. The string ' Exists' should not + be displayed, but '<b>Exists</b>' should. + ''' + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + self.import_capsule( self.get_filename( 'repository_capsules/0490_filtering.tar.gz' ), + strings_displayed=[ repository_name, '<b>Exists' ], + strings_not_displayed=[ ' Exists' ], + strings_displayed_after_submit=[ 'Repository <b>filtering_0490</b> has been created.' ], + strings_not_displayed_after_submit=[ 'Import not necessary' ] ) + + def test_0010_export_repository_capsule( self ): + '''Export the repository that was imported in the previous step.''' + ''' + This is step 2 - Export that repository. + Export the repository to a temporary location. + ''' + global capsule_filepath + repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + capsule_filepath = self.export_capsule( repository ) + log.debug( os.path.exists( capsule_filepath ) ) + + def test_0015_verify_exported_capsule( self ): + '''Verify the exported capsule contents.''' + ''' + This is step 3 - Check the capsule's contents, verify that changeset revision and tool shed are not set. + Extract the exported capsule tarball to a temporary path, and confirm that the manifest does not specify + a tool shed or changeset revision. + ''' + global capsule_filepath + self.verify_capsule_contents( capsule_filepath, owner=common.test_user_1_name ) + + def test_0020_import_repository_capsule( self ): + '''Import the exported repository capsule.''' + ''' + This is step 4 - Import the capsule again. Check that the repository to be imported is marked as preexisting. + The string ' Exists' should be displayed, as should '<b>Exists</b>'. + ''' + global capsule_filepath + self.import_capsule( capsule_filepath, + strings_displayed=[ repository_name, ' Exists', self.url ], + strings_not_displayed_after_submit=[ 'Repository <b>filtering_0490</b> has been created.' ], + strings_displayed_after_submit=[ 'Import not necessary', 'Exists' ] ) diff -r 398287552d91093721ea0de635ed6b0d7c7cd147 -r d37d6e711d671763d725ffb4931f0acc379964dc test/tool_shed/functional/test_0500_export_repository_simple_dependency.py --- /dev/null +++ b/test/tool_shed/functional/test_0500_export_repository_simple_dependency.py @@ -0,0 +1,89 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +import logging +log = logging.getLogger( __name__ ) + +emboss_repository_name = 'emboss_5_0500' +emboss_repository_description = "Galaxy wrappers for Emboss version 5.0.0 tools" +emboss_repository_long_description = "Galaxy wrappers for Emboss version 5.0.0 tools" +datatypes_repository_name = 'emboss_datatypes_0500' +datatypes_repository_description = 'Galaxy applicable data formats used by Emboss tools.' +datatypes_repository_long_description = 'Galaxy applicable data formats used by Emboss tools. This repository contains no tools.' + +category_name = 'Test 0500 Repository Dependency Import Export' +category_description = 'Test script 0500 for importing and exporting repositories with simple repository dependencies.' + +''' +1. Export a repository with no dependencies, e.g. filter1. +2. Temporarily extract the repository capsule. +2a. For every owner in the manifest, set to a different user. +3. Import it into the same tool shed. +4. Check that the repository to be imported has no status in the status column. +5. Click the import button. +6. Verify the resulting page is correct. +''' + +class TestExportImportRepository( ShedTwillTestCase ): + '''Test exporting and importing repositories.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts and login as an admin user.""" + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + + def test_0005_create_category_and_repository( self ): + """Create categories for this test suite""" + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + self.import_capsule( self.get_filename( 'repository_capsules/0500_emboss_5.tar.gz' ), + strings_displayed=[ emboss_repository_name, datatypes_repository_name, '<b>Exists' ], + strings_not_displayed=[ ' Exists' ], + strings_displayed_after_submit=[ 'Repository <b>emboss_5_0500</b> has been created.', + 'Repository <b>emboss_datatypes_0500</b> has been created.' ], + strings_not_displayed_after_submit=[ 'Import not necessary' ] ) + + def test_0010_export_repository_capsule( self ): + '''Export the repository that was imported in the previous step.''' + ''' + This is step 2 - Export that repository. + Export the repository to a temporary location. + ''' + global capsule_filepath + repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + capsule_filepath = self.export_capsule( repository ) + log.debug( os.path.exists( capsule_filepath ) ) + + def test_0015_verify_exported_capsule( self ): + '''Verify the exported capsule contents.''' + ''' + This is step 3 - Check the capsule's contents, verify that changeset revision and tool shed are not set. + Extract the exported capsule tarball to a temporary path, and confirm that the manifest does not specify + a tool shed or changeset revision. + ''' + global capsule_filepath + self.verify_capsule_contents( capsule_filepath, owner=common.test_user_1_name ) + + def test_0020_import_repository_capsule( self ): + '''Import the exported repository capsule.''' + ''' + This is step 4 - Import the capsule again. Check that the repository to be imported is marked as preexisting. + The string ' Exists' should be displayed, as should '<b>Exists</b>'. + ''' + global capsule_filepath + self.import_capsule( capsule_filepath, + strings_displayed=[ emboss_repository_name, datatypes_repository_name, ' Exists', self.url ], + strings_not_displayed_after_submit=[ 'Repository <b>emboss_5_0500</b> has been created.', + 'Repository <b>emboss_datatypes_0500</b> has been created.' ], + strings_displayed_after_submit=[ 'Import not necessary', ' Exists' ] ) diff -r 398287552d91093721ea0de635ed6b0d7c7cd147 -r d37d6e711d671763d725ffb4931f0acc379964dc test/tool_shed/functional/test_0510_export_import_repository_complex_dependencies.py --- /dev/null +++ b/test/tool_shed/functional/test_0510_export_import_repository_complex_dependencies.py @@ -0,0 +1,99 @@ +from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os +import tool_shed.base.test_db_util as test_db_util + +import logging +log = logging.getLogger( __name__ ) + +category_name = 'Test 0510 Import Export Complex Dependencies' +category_description = 'Test script 0510 for importing and exporting repositories with complex repository dependencies.' + +''' +Import a repository capsule with a complex repository dependency with trans_proteomic_pipeline and required repositories. +Check that the repository to be imported is not marked as preexisting. The word ' Exists' should not be displayed, but '<b>Exists</b>' will. +Click the import button. +Verify the resulting page is correct. +Verify the dependency structure that has been created. +Export the trans_proteomic_pipeline repository with dependencies. +Check the capsule's contents, verify that changeset revision and tool shed are not set. +''' + +class TestExportImportRepository( ShedTwillTestCase ): + '''Test exporting and importing repositories with complex dependencies.''' + + def test_0000_initiate_users( self ): + """Create necessary user accounts and login as an admin user.""" + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + test_user_1 = test_db_util.get_user( common.test_user_1_email ) + assert test_user_1 is not None, 'Problem retrieving user with email %s from the database' % common.test_user_1_email + test_user_1_private_role = test_db_util.get_private_role( test_user_1 ) + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + admin_user = test_db_util.get_user( common.admin_email ) + assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email + admin_user_private_role = test_db_util.get_private_role( admin_user ) + + def test_0005_create_category_and_repositories( self ): + """Create categories for this test suite""" + self.logout() + self.login( email=common.admin_email, username=common.admin_username ) + self.create_category( name=category_name, description=category_description ) + self.logout() + self.login( email=common.test_user_1_email, username=common.test_user_1_name ) + self.import_capsule( self.get_filename( 'repository_capsules/0510_trans_proteomic_pipeline.tar.gz' ), + strings_displayed=[ 'package_trans_proteomic_pipeline_4_6_3', + 'package_perl_5_18', + 'package_libpng_1_2', + 'package_libgd_2_1', + 'package_expat_2_1', + '<b>Exists' ], + strings_not_displayed=[ ' Exists' ], + strings_displayed_after_submit=[ 'Repository <b>package_trans_proteomic_pipeline_4_6_3</b> has been created.', + 'Repository <b>package_perl_5_18</b> has been created.', + 'Repository <b>package_libpng_1_2</b> has been created.', + 'Repository <b>package_libgd_2_1</b> has been created.', + 'Repository <b>package_expat_2_1</b> has been created.' ], + strings_not_displayed_after_submit=[ 'Import not necessary' ] ) + + def test_0010_export_repository_capsule( self ): + '''Export the repository that was imported in the previous step.''' + ''' + This is step 2 - Export that repository. + Export the repository to a temporary location. + ''' + global capsule_filepath + repository = test_db_util.get_repository_by_name_and_owner( 'package_trans_proteomic_pipeline_4_6_3', common.test_user_1_name ) + capsule_filepath = self.export_capsule( repository ) + log.debug( os.path.exists( capsule_filepath ) ) + + def test_0015_verify_exported_capsule( self ): + '''Verify the exported capsule contents.''' + ''' + This is step 3 - Check the capsule's contents, verify that changeset revision and tool shed are not set. + Extract the exported capsule tarball to a temporary path, and confirm that the manifest does not specify + a tool shed or changeset revision. + ''' + global capsule_filepath + self.verify_capsule_contents( capsule_filepath, owner=common.test_user_1_name ) + + def test_0020_import_repository_capsule( self ): + '''Import the exported repository capsule.''' + ''' + This is step 4 - Import the capsule again. Check that the repository to be imported is marked as preexisting. + The string ' Exists' should be displayed, as should '<b>Exists</b>'. + ''' + global capsule_filepath + self.import_capsule( capsule_filepath, + strings_displayed=[ 'package_trans_proteomic_pipeline_4_6_3', + 'package_perl_5_18', + 'package_libpng_1_2', + 'package_libgd_2_1', + 'package_expat_2_1', + ' Exists', + self.url ], + strings_not_displayed_after_submit=[ 'Repository <b>package_trans_proteomic_pipeline_4_6_3</b> has been created.', + 'Repository <b>package_perl_5_18</b> has been created.', + 'Repository <b>package_libpng_1_2</b> has been created.', + 'Repository <b>package_libgd_2_1</b> has been created.', + 'Repository <b>package_expat_2_1</b> has been created.' ], + strings_displayed_after_submit=[ 'Import not necessary', ' Exists' ] ) diff -r 398287552d91093721ea0de635ed6b0d7c7cd147 -r d37d6e711d671763d725ffb4931f0acc379964dc test/tool_shed/functional_tests.py --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -92,8 +92,9 @@ tool_sheds_conf_xml_template = '''<?xml version="1.0"?><tool_sheds> + <tool_shed name="Galaxy main tool shed" url="http://toolshed.g2.bx.psu.edu/"/> + <tool_shed name="Galaxy test tool shed" url="http://testtoolshed.g2.bx.psu.edu/"/><tool_shed name="Embedded tool shed for functional tests" url="http://${shed_url}:${shed_port}/"/> - <tool_shed name="Galaxy main tool shed" url="http://toolshed.g2.bx.psu.edu/"/></tool_sheds> ''' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.