1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/a7a0a5962648/ changeset: a7a0a5962648 user: greg date: 2012-08-14 22:24:54 summary: Fix for recently introduced bug that breaks installation of tool shed repositories into a local Galaxy instance if the repository includes tools that use the ToolDataTableManage. Also, move processing of sample files and other config files included in installed tool shed repositories to disk rather than memory. affected #: 4 files diff -r b4f52783c54ac175d306d5064ef5ab7bd5149ca9 -r a7a0a5962648eb68a27ed9e460d1b4ea012c2302 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -136,27 +136,22 @@ repository_clone_url=repository_clone_url, relative_install_dir=relative_install_dir, repository_files_dir=None, - resetting_all_metadata_on_repository=False ) + resetting_all_metadata_on_repository=False, + webapp='galaxy' ) tool_shed_repository.metadata = metadata_dict self.app.sa_session.add( tool_shed_repository ) self.app.sa_session.flush() if 'tool_dependencies' in metadata_dict: # All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed. - tool_dependencies = create_tool_dependency_objects( self.app, tool_shed_repository, tool_shed_repository.installed_changeset_revision, set_status=True ) + tool_dependencies = create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True ) else: tool_dependencies = None if 'tools' in metadata_dict: - work_dir = tempfile.mkdtemp() repository_tools_tups = get_repository_tools_tups( self.app, metadata_dict ) if repository_tools_tups: sample_files = metadata_dict.get( 'sample_files', [] ) # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_data_table_entry( self.app, - tool_shed_repository, - tool_shed_repository.installed_changeset_revision, - self.tool_path, - repository_tools_tups, - work_dir ) + repository_tools_tups = handle_missing_data_table_entry( self.app, relative_install_dir, self.tool_path, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. repository_tools_tups, sample_files_copied = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. @@ -166,12 +161,8 @@ update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) - # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = get_config_from_repository( self.app, - 'tool_dependencies.xml', - tool_shed_repository, - tool_shed_repository.installed_changeset_revision, - work_dir ) + # Get the tool_dependencies.xml file from disk. + tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', relative_install_dir ) installed_tool_dependencies = handle_tool_dependencies( app=self.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, @@ -189,10 +180,6 @@ self.migrated_tools_config, tool_panel_dict=tool_panel_dict_for_display, new_install=True ) - try: - shutil.rmtree( work_dir ) - except: - pass if 'datatypes' in metadata_dict: tool_shed_repository.status = self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: @@ -200,11 +187,7 @@ self.app.sa_session.add( tool_shed_repository ) self.app.sa_session.flush() work_dir = tempfile.mkdtemp() - datatypes_config = get_config_from_repository( self.app, - 'datatypes_conf.xml', - tool_shed_repository, - tool_shed_repository.installed_changeset_revision, - work_dir ) + datatypes_config = get_config_from_disk( 'datatypes_conf.xml', relative_install_dir ) # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started # after this installation completes. converter_path, display_path = alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, relative_install_dir, override=False ) diff -r b4f52783c54ac175d306d5064ef5ab7bd5149ca9 -r a7a0a5962648eb68a27ed9e460d1b4ea012c2302 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -248,7 +248,7 @@ except: pass return converter_path, display_path -def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): +def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files, webapp='galaxy' ): """ Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make sure the files exist. @@ -291,8 +291,9 @@ correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file ) correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) - # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - reset_tool_data_tables( app ) + if webapp == 'community': + # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. + reset_tool_data_tables( app ) return invalid_files_and_errors_tups def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): # Persist the current in-memory list of config_elems to a file named by the value of config_filename. @@ -427,16 +428,11 @@ sa_session.add( tool_shed_repository ) sa_session.flush() return tool_shed_repository -def create_tool_dependency_objects( app, tool_shed_repository, current_changeset_revision, set_status=True ): +def create_tool_dependency_objects( app, tool_shed_repository, relative_install_dir, set_status=True ): # Create or update a ToolDependency for each entry in tool_dependencies_config. This method is called when installing a new tool_shed_repository. tool_dependency_objects = [] - work_dir = tempfile.mkdtemp() # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = get_config_from_repository( app, - 'tool_dependencies.xml', - tool_shed_repository, - current_changeset_revision, - work_dir ) + tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', relative_install_dir ) tree = ElementTree.parse( tool_dependencies_config ) root = tree.getroot() ElementInclude.include( root ) @@ -454,10 +450,6 @@ status=app.model.ToolDependency.installation_status.NEVER_INSTALLED, set_status=set_status ) tool_dependency_objects.append( tool_dependency ) - try: - shutil.rmtree( work_dir ) - except: - pass return tool_dependency_objects def generate_clone_url( trans, repository ): """Generate the URL for cloning a repository.""" @@ -548,7 +540,8 @@ if not can_generate_dependency_metadata: break return can_generate_dependency_metadata -def generate_metadata_for_changeset_revision( app, repository_clone_url, relative_install_dir=None, repository_files_dir=None, resetting_all_metadata_on_repository=False ): +def generate_metadata_for_changeset_revision( app, repository_clone_url, relative_install_dir=None, repository_files_dir=None, + resetting_all_metadata_on_repository=False, webapp='galaxy' ): """ Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip, the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's @@ -629,7 +622,7 @@ invalid_tool_configs.append( name ) invalid_file_tups.append( ( name, str( e ) ) ) if tool is not None: - invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_files ) + invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_files, webapp=webapp ) can_set_metadata = True for tup in invalid_files_and_errors_tups: if name in tup: @@ -664,7 +657,7 @@ metadata_dict = generate_tool_dependency_metadata( tool_dependencies_config, metadata_dict ) if invalid_tool_configs: metadata_dict [ 'invalid_tools' ] = invalid_tool_configs - if resetting_all_metadata_on_repository: + if webapp == 'community' and resetting_all_metadata_on_repository: # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. reset_tool_data_tables( app ) # Reset the value of the app's tool_data_path to it's original value. @@ -1255,7 +1248,7 @@ return shed_url # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. return None -def handle_missing_data_table_entry( app, repository, changeset_revision, tool_path, repository_tools_tups, dir ): +def handle_missing_data_table_entry( app, relative_install_dir, tool_path, repository_tools_tups ): """ Inspect each tool to see if any have input parameters that are dynamically generated select lists that require entries in the tool_data_table_conf.xml file. This method is called only from Galaxy (not the tool shed) when a repository is being installed @@ -1269,7 +1262,7 @@ break if missing_data_table_entry: # The repository must contain a tool_data_table_conf.xml.sample file that includes all required entries for all tools in the repository. - sample_tool_data_table_conf = get_config_from_repository( app, 'tool_data_table_conf.xml.sample', repository, changeset_revision, dir ) + sample_tool_data_table_conf = get_config_from_disk( 'tool_data_table_conf.xml.sample', relative_install_dir ) # Add entries to the ToolDataTableManager's in-memory data_tables dictionary as well as the list of data_table_elems and the list of # data_table_elem_names. error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf, persist=True ) @@ -1394,14 +1387,8 @@ def load_installed_datatypes( app, repository, relative_install_dir, deactivate=False ): # Load proprietary datatypes and return information needed for loading proprietary datatypes converters and display applications later. metadata = repository.metadata - work_dir = tempfile.mkdtemp() repository_dict = None - datatypes_config = get_config_from_repository( app, - 'datatypes_conf.xml', - repository, - repository.changeset_revision, - work_dir, - install_dir=relative_install_dir ) + datatypes_config = get_config_from_disk( 'datatypes_conf.xml', relative_install_dir ) if datatypes_config: converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate ) if converter_path or display_path: @@ -1413,10 +1400,6 @@ tool_dicts=metadata.get( 'tools', [] ), converter_path=converter_path, display_path=display_path ) - try: - shutil.rmtree( work_dir ) - except: - pass return repository_dict def load_installed_display_applications( app, installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype display applications diff -r b4f52783c54ac175d306d5064ef5ab7bd5149ca9 -r a7a0a5962648eb68a27ed9e460d1b4ea012c2302 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -525,13 +525,8 @@ # Get the tool_shed_repository from one of the tool_dependencies. message = '' tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository - work_dir = tempfile.mkdtemp() # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = get_config_from_repository( trans.app, - 'tool_dependencies.xml', - tool_shed_repository, - tool_shed_repository.changeset_revision, - work_dir ) + tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', tool_shed_repository.repo_path( trans.app ) ) installed_tool_dependencies = handle_tool_dependencies( app=trans.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, @@ -539,10 +534,6 @@ for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == trans.app.model.ToolDependency.installation_status.ERROR: message += ' %s' % installed_tool_dependency.error_message - try: - shutil.rmtree( work_dir ) - except: - pass tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in tool_dependencies ] if message: status = 'error' @@ -663,11 +654,7 @@ tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = get_config_from_repository( trans.app, - 'tool_dependencies.xml', - tool_shed_repository, - tool_shed_repository.installed_changeset_revision, - work_dir ) + tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', relative_install_dir ) installed_tool_dependencies = handle_tool_dependencies( app=trans.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, @@ -691,24 +678,19 @@ repository_clone_url=repository_clone_url, relative_install_dir=relative_install_dir, repository_files_dir=None, - resetting_all_metadata_on_repository=False ) + resetting_all_metadata_on_repository=False, + webapp='galaxy' ) tool_shed_repository.metadata = metadata_dict trans.sa_session.add( tool_shed_repository ) trans.sa_session.flush() if 'tool_dependencies' in metadata_dict and not reinstalling: - tool_dependencies = create_tool_dependency_objects( trans.app, tool_shed_repository, tool_shed_repository.installed_changeset_revision, set_status=True ) + tool_dependencies = create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True ) if 'tools' in metadata_dict: tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) repository_tools_tups = get_repository_tools_tups( trans.app, metadata_dict ) if repository_tools_tups: # Handle missing data table entries for tool parameters that are dynamically generated select lists. - work_dir = tempfile.mkdtemp() - repository_tools_tups = handle_missing_data_table_entry( trans.app, - tool_shed_repository, - tool_shed_repository.changeset_revision, - tool_path, - repository_tools_tups, - work_dir ) + repository_tools_tups = handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. sample_files = metadata_dict.get( 'sample_files', [] ) repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) @@ -723,22 +705,13 @@ shed_tool_conf=shed_tool_conf, tool_panel_dict=tool_panel_dict, new_install=True ) - try: - shutil.rmtree( work_dir ) - except: - pass if 'datatypes' in metadata_dict: tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES if not tool_shed_repository.includes_datatypes: tool_shed_repository.includes_datatypes = True trans.sa_session.add( tool_shed_repository ) trans.sa_session.flush() - work_dir = tempfile.mkdtemp() - datatypes_config = get_config_from_repository( trans.app, - 'datatypes_conf.xml', - tool_shed_repository, - tool_shed_repository.changeset_revision, - work_dir ) + datatypes_config = get_config_from_disk( 'datatypes_conf.xml', relative_install_dir ) # Load data types required by tools. converter_path, display_path = alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, relative_install_dir, override=False ) if converter_path or display_path: @@ -756,10 +729,6 @@ if display_path: # Load proprietary datatype display applications trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - try: - shutil.rmtree( work_dir ) - except: - pass @web.expose @web.require_admin def manage_repository( self, trans, **kwd ): @@ -1507,7 +1476,7 @@ trans.sa_session.flush() # Create tool_dependency records if necessary. if 'tool_dependencies' in metadata_dict: - tool_dependencies = create_tool_dependency_objects( trans.app, repository, repository.changeset_revision, set_status=False ) + tool_dependencies = create_tool_dependency_objects( trans.app, repository, relative_install_dir, set_status=False ) message = "The installed repository named '%s' has been updated to change set revision '%s'. " % ( name, latest_changeset_revision ) # See if any tool dependencies can be installed. shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository ) diff -r b4f52783c54ac175d306d5064ef5ab7bd5149ca9 -r a7a0a5962648eb68a27ed9e460d1b4ea012c2302 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -795,7 +795,8 @@ repository_clone_url=repository_clone_url, relative_install_dir=repo_dir, repository_files_dir=work_dir, - resetting_all_metadata_on_repository=True ) + resetting_all_metadata_on_repository=True, + webapp='community' ) if current_metadata_dict: if not metadata_changeset_revision and not metadata_dict: # We're at the first change set in the change log. @@ -885,7 +886,8 @@ repository_clone_url=repository_clone_url, relative_install_dir=repo_dir, repository_files_dir=None, - resetting_all_metadata_on_repository=False ) + resetting_all_metadata_on_repository=False, + webapp='community' ) if metadata_dict: downloadable = is_downloadable( metadata_dict ) repository_metadata = None Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.