1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/3d9f0ce7bb61/ changeset: 3d9f0ce7bb61 user: greg date: 2012-01-13 21:29:04 summary: Add the ability to deactivate / activate installed tool shed repositories. When a repository is deactivated, it's contained tools are eliminated from the tool panel on the fly. Similarly, if a deactivated repository is activated, it's contained tools are loaded into the tool panel on the fly. This works for tools contained in sections or outside sections in the tool panel. Also fix a bug which resulted in a server error when installing a tool shed repository that includs a .loc.sample file. affected #: 10 files diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -38,13 +38,13 @@ self.install_section( elem ) def install_repository( self, elem, section_name='', section_id='' ): # Install a single repository into the tool config. If outside of any sections, the entry looks something like: - # <repository name="cut_wrapper" description="Galaxy wrapper for the Cut tool" changeset_revision="f3ed6cfe6402"> + # <repository name="cut_wrapper" description="Galaxy wrapper for the Cut tool" installed_changeset_revision="f3ed6cfe6402"> # <tool id="Cut1" version="1.0.1" /> # </repository> name = elem.get( 'name' ) description = elem.get( 'description' ) changeset_revision = elem.get( 'changeset_revision' ) - # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<changeset revision> + # Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision> clone_dir = os.path.join( self.tool_path, self.tool_shed, 'repos', self.repository_owner, name, changeset_revision ) if self.__isinstalled( elem, clone_dir ): log.debug( "Skipping automatic install of repository '%s' because it has already been installed in location '%s'" % ( name, clone_dir ) ) @@ -61,6 +61,7 @@ new_section_elem = Element( 'section' ) new_section_elem.attrib[ 'name' ] = section_name new_section_elem.attrib[ 'id' ] = section_id + new_section_elem.attrib[ 'version' ] = '' tool_section = ToolSection( new_section_elem ) self.app.toolbox.tool_panel[ section_key ] = tool_section else: @@ -74,7 +75,7 @@ returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision ) if returncode == 0: metadata_dict = load_repository_contents( app=self.app, - name=name, + repository_name=name, description=description, owner=self.repository_owner, changeset_revision=changeset_revision, @@ -124,7 +125,7 @@ def install_section( self, elem ): # Install 1 or more repositories into a section in the tool config. An entry looks something like: # <section name="EMBOSS" id="EMBOSSLite"> - # <repository name="emboss_5" description="Galaxy wrappers for EMBOSS version 5 tools" changeset_revision="bdd88ae5d0ac"> + # <repository name="emboss_5" description="Galaxy wrappers for EMBOSS version 5 tools" installed_changeset_revision="bdd88ae5d0ac"> # <tool file="emboss_5/emboss_antigenic.xml" id="EMBOSS: antigenic1" version="5.0.0" /> # ... # </repository> diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -24,6 +24,7 @@ from galaxy.tools.actions import DefaultToolAction from galaxy.tools.deps import DependencyManager from galaxy.model import directory_hash_id +from galaxy.model.orm import * from galaxy.util.none_like import NoneDataset from galaxy.datatypes import sniff from cgi import FieldStorage @@ -37,15 +38,11 @@ pass class ToolBox( object ): - """ - Container for a collection of tools - """ - + """Container for a collection of tools""" def __init__( self, config_filenames, tool_root_dir, app ): """ - Create a toolbox from the config file names by `config_filename`, - using `tool_root_directory` as the base directory for finding - individual tool config files. + Create a toolbox from the config files named by `config_filenames`, using + `tool_root_dir` as the base directory for finding individual tool config files. """ # The shed_tool_confs dictionary contains shed_conf_filename : tool_path pairs. self.shed_tool_confs = {} @@ -62,7 +59,7 @@ try: self.init_tools( config_filename ) except: - log.exception( "ToolBox error reading %s", config_filename ) + log.exception( "Error loading tools defined in config %s", config_filename ) def init_tools( self, config_filename ): """ Read the configuration file and load each tool. @@ -82,7 +79,7 @@ log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")") self.sa_session.query( self.app.model.ToolTagAssociation ).delete() self.sa_session.flush() - log.info( "parsing the tool configuration %s" % config_filename ) + log.info( "Parsing the tool configuration %s" % config_filename ) tree = util.parse_xml( config_filename ) root = tree.getroot() tool_path = root.get( 'tool_path' ) @@ -94,53 +91,107 @@ tool_path = self.tool_root_dir for elem in root: if elem.tag == 'tool': - self.load_tool_tag_set( elem, self.tool_panel, tool_path, guid=elem.get( 'guid' ) ) + self.load_tool_tag_set( elem, self.tool_panel, tool_path, guid=elem.get( 'guid' ), section=None ) elif elem.tag == 'workflow': self.load_workflow_tag_set( elem, self.tool_panel ) - elif elem.tag == 'section' : + elif elem.tag == 'section': self.load_section_tag_set( elem, self.tool_panel, tool_path ) elif elem.tag == 'label': self.load_label_tag_set( elem, self.tool_panel ) - def load_tool_tag_set( self, elem, panel_dict, tool_path, guid=None ): + def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): + return self.sa_session.query( self.app.model.ToolShedRepository ) \ + .filter( and_( self.app.model.ToolShedRepository.table.c.tool_shed == tool_shed, + self.app.model.ToolShedRepository.table.c.name == name, + self.app.model.ToolShedRepository.table.c.owner == owner, + self.app.model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ + .first() + def load_tool_tag_set( self, elem, panel_dict, tool_path, guid=None, section=None ): try: path = elem.get( "file" ) - tool = self.load_tool( os.path.join( tool_path, path ), guid=guid ) if guid is not None: - # Tool was installed from a Galaxy tool shed. - tool.tool_shed = elem.find( "tool_shed" ).text - tool.repository_name = elem.find( "repository_name" ).text - tool.repository_owner = elem.find( "repository_owner" ).text - tool.changeset_revision = elem.find( "changeset_revision" ).text - tool.old_id = elem.find( "id" ).text - tool.version = elem.find( "version" ).text - if self.app.config.get_bool( 'enable_tool_tags', False ): - tag_names = elem.get( "tags", "" ).split( "," ) - for tag_name in tag_names: - if tag_name == '': - continue - tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first() - if not tag: - tag = self.app.model.Tag( name=tag_name ) - self.sa_session.add( tag ) - self.sa_session.flush() - tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) - self.sa_session.add( tta ) - self.sa_session.flush() - else: - for tagged_tool in tag.tagged_tools: - if tagged_tool.tool_id == tool.id: - break + # The tool is contained in an installed tool shed repository, so load + # the tool only if the repository has not been marked deleted. + tool_shed = elem.find( "tool_shed" ).text + repository_name = elem.find( "repository_name" ).text + repository_owner = elem.find( "repository_owner" ).text + installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) + if installed_changeset_revision_elem is None: + # Backward compatibility issue - the tag used to be named 'changeset_revision'. + installed_changeset_revision_elem = elem.find( "changeset_revision" ) + installed_changeset_revision = installed_changeset_revision_elem.text + tool_shed_repository = self.__get_tool_shed_repository( tool_shed, repository_name, repository_owner, installed_changeset_revision ) + if tool_shed_repository: + # Only load tools if the repository is not deactivated or uninstalled. + can_load = not tool_shed_repository.deleted + if can_load: + metadata = tool_shed_repository.metadata + update_needed = False + if 'tool_panel_section' in metadata: + if section: + # If the tool_panel_section dictionary is included in the metadata, update it if necessary. + tool_panel_section = metadata[ 'tool_panel_section' ] + if tool_panel_section [ 'id' ] != section.id or \ + tool_panel_section [ 'version' ] != section.version or \ + tool_panel_section [ 'name' ] != section.name: + tool_panel_section [ 'id' ] = section.id + tool_panel_section [ 'version' ] = section.version + tool_panel_section [ 'name' ] = section.name + update_needed = True else: + # The tool_panel_section was introduced late, so set it's value if its missing in the metadata. + if section: + tool_panel_section = dict( id=section.id, version=section.version, name=section.name ) + update_needed = True + else: + tool_panel_section = dict( id='', version='', name='' ) + update_needed = True + if update_needed: + metadata[ 'tool_panel_section' ] = tool_panel_section + tool_shed_repository.metadata = metadata + self.sa_session.add( tool_shed_repository ) + self.sa_session.flush() + else: + # If there is not yet a tool_shed_repository record, we're in the process of installing + # a new repository, so any included tools can be loaded into the tool panel. + can_load = True + else: + can_load = True + if can_load: + tool = self.load_tool( os.path.join( tool_path, path ), guid=guid ) + if guid is not None: + tool.tool_shed = tool_shed + tool.repository_name = repository_name + tool.repository_owner = repository_owner + tool.installed_changeset_revision = installed_changeset_revision + tool.old_id = elem.find( "id" ).text + tool.version = elem.find( "version" ).text + if self.app.config.get_bool( 'enable_tool_tags', False ): + tag_names = elem.get( "tags", "" ).split( "," ) + for tag_name in tag_names: + if tag_name == '': + continue + tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first() + if not tag: + tag = self.app.model.Tag( name=tag_name ) + self.sa_session.add( tag ) + self.sa_session.flush() tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) self.sa_session.add( tta ) self.sa_session.flush() - if tool.id not in self.tools_by_id: - # Allow for the same tool to be loaded into multiple places in the - # tool panel. - self.tools_by_id[ tool.id ] = tool - key = 'tool_' + tool.id - panel_dict[ key ] = tool - log.debug( "Loaded tool: %s %s" % ( tool.id, tool.version ) ) + else: + for tagged_tool in tag.tagged_tools: + if tagged_tool.tool_id == tool.id: + break + else: + tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) + self.sa_session.add( tta ) + self.sa_session.flush() + if tool.id not in self.tools_by_id: + # Allow for the same tool to be loaded into multiple places in the tool panel. + self.tools_by_id[ tool.id ] = tool + key = 'tool_' + tool.id + panel_dict[ key ] = tool + log.debug( "Loaded tool: %s %s" % ( tool.id, tool.version ) ) except: log.exception( "error reading tool from path: %s" % path ) def load_workflow_tag_set( self, elem, panel_dict ): @@ -161,22 +212,23 @@ def load_section_tag_set( self, elem, panel_dict, tool_path ): key = 'section_' + elem.get( "id" ) if key in panel_dict: - # Appending a tool to an existing section in self.tool_panel - elems = panel_dict[ key ].elems - log.debug( "Appending to section: %s" % elem.get( "name" ) ) + # Loading an existing section in self.tool_panel + section = panel_dict[ key ] + elems = section.elems + log.debug( "Reloading section: %s" % elem.get( "name" ) ) else: - # Appending a new section to self.tool_panel + # Adding a new section to self.tool_panel section = ToolSection( elem ) elems = section.elems log.debug( "Loading section: %s" % section.name ) - for section_elem in elem: - if section_elem.tag == 'tool': - self.load_tool_tag_set( section_elem, elems, tool_path, guid=section_elem.get( 'guid' ) ) - elif section_elem.tag == 'workflow': - self.load_workflow_tag_set( section_elem, elems ) - elif section_elem.tag == 'label': - self.load_label_tag_set( section_elem, elems ) - if key not in panel_dict: + for sub_elem in elem: + if sub_elem.tag == 'tool': + self.load_tool_tag_set( sub_elem, elems, tool_path, guid=sub_elem.get( 'guid' ), section=section ) + elif sub_elem.tag == 'workflow': + self.load_workflow_tag_set( sub_elem, elems ) + elif sub_elem.tag == 'label': + self.load_label_tag_set( sub_elem, elems ) + if key not in panel_dict and section.elems: panel_dict[ key ] = section def load_tool( self, config_file, guid=None ): """ @@ -409,7 +461,7 @@ self.tool_shed = None self.repository_name = None self.repository_owner = None - self.changeset_revision = None + self.installed_changeset_revision = None self.old_id = None self.version = None # Parse XML element containing configuration diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -3,7 +3,6 @@ from time import strftime from galaxy import util from galaxy.util.json import * -from galaxy.tools import ToolSection from galaxy.tools.search import ToolBoxSearch from galaxy.model.orm import * @@ -78,6 +77,17 @@ os.chdir( current_working_dir ) tmp_stderr.close() return returncode, tmp_name +def copy_sample_loc_file( app, filename ): + """Copy xxx.loc.sample to ~/tool-data/xxx.loc.sample and ~/tool-data/xxx.loc""" + head, sample_loc_file = os.path.split( filename ) + loc_file = sample_loc_file.replace( '.sample', '' ) + tool_data_path = os.path.abspath( app.config.tool_data_path ) + # It's ok to overwrite the .sample version of the file. + shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, sample_loc_file ) ) + # Only create the .loc file if it does not yet exist. We don't + # overwrite it in case it contains stuff proprietary to the local instance. + if not os.path.exists( os.path.join( tool_data_path, loc_file ) ): + shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) ) def create_or_update_tool_shed_repository( app, name, description, changeset_revision, repository_clone_url, metadata_dict, owner='' ): # This method is used by the InstallManager, which does not have access to trans. sa_session = app.model.context.current @@ -149,7 +159,7 @@ if datatypes: metadata_dict[ 'datatypes' ] = datatypes return metadata_dict -def generate_metadata( toolbox, relative_install_dir, repository_clone_url ): +def generate_metadata( toolbox, relative_install_dir, repository_clone_url, tool_section_dict=None ): """ Browse the repository files on disk to generate metadata. Since we are using disk files, it is imperative that the repository is updated to the desired change set revision before metadata @@ -158,6 +168,11 @@ metadata_dict = {} sample_files = [] datatypes_config = None + # Keep track of the section in the tool panel in which this repository's tools will be contained. + if tool_section_dict: + metadata_dict[ 'tool_panel_section' ] = tool_section_dict + else: + metadata_dict[ 'tool_panel_section' ] = dict( id='', version='', name='' ) # Find datatypes_conf.xml if it exists. for root, dirs, files in os.walk( relative_install_dir ): if root.find( '.hg' ) < 0: @@ -264,6 +279,7 @@ def generate_tool_panel_elem_list( repository_name, repository_clone_url, changeset_revision, repository_tools_tups, tool_section=None, owner='' ): """Generate a list of ElementTree Element objects for each section or list of tools.""" elem_list = [] + tool_elem = None tmp_url = clean_repository_clone_url( repository_clone_url ) if not owner: owner = get_repository_owner( tmp_url ) @@ -271,6 +287,7 @@ root_elem = Element( 'section' ) root_elem.attrib[ 'name' ] = tool_section.name root_elem.attrib[ 'id' ] = tool_section.id + root_elem.attrib[ 'version' ] = tool_section.version for repository_tool_tup in repository_tools_tups: tool_file_path, guid, tool = repository_tool_tup if tool_section: @@ -285,7 +302,7 @@ repository_name_elem.text = repository_name repository_owner_elem = SubElement( tool_elem, 'repository_owner' ) repository_owner_elem.text = owner - changeset_revision_elem = SubElement( tool_elem, 'changeset_revision' ) + changeset_revision_elem = SubElement( tool_elem, 'installed_changeset_revision' ) changeset_revision_elem.text = changeset_revision id_elem = SubElement( tool_elem, 'id' ) id_elem.text = tool.id @@ -293,7 +310,7 @@ version_elem.text = tool.version if tool_section: elem_list.append( root_elem ) - else: + elif tool_elem: elem_list.append( tool_elem ) return elem_list def generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ): @@ -324,6 +341,15 @@ if repo_path.startswith( '/' ): repo_path = repo_path.replace( '/', '', 1 ) return repo_path.lstrip( '/' ).split( '/' )[ 0 ] +def get_repository_tools_tups( app, metadata_dict ): + repository_tools_tups = [] + if 'tools' in metadata_dict: + for tool_dict in metadata_dict[ 'tools' ]: + relative_path = tool_dict[ 'tool_config' ] + guid = tool_dict[ 'guid' ] + tool = app.toolbox.load_tool( os.path.abspath( relative_path ) ) + repository_tools_tups.append( ( relative_path, guid, tool ) ) + return repository_tools_tups def get_tool_id_guid_map( app, tool_id, version, tool_shed, repository_owner, repository_name ): # This method is used by the InstallManager, which does not have access to trans. sa_session = app.model.context.current @@ -545,7 +571,7 @@ # Load proprietary datatypes app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=datatypes_config, imported_modules=imported_modules ) return converter_path, display_path -def load_repository_contents( app, name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir, +def load_repository_contents( app, repository_name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir, current_working_dir, tmp_name, tool_section=None, shed_tool_conf=None, new_install=True ): # This method is used by the InstallManager, which does not have access to trans. # Generate the metadata for the installed tool shed repository. It is imperative that @@ -556,7 +582,16 @@ # and when updates have been pulled to previously installed repositories (in which case the # default value None is set for tool_section and shed_tool_conf, and the value of new_install # is passed as False). - metadata_dict = generate_metadata( app.toolbox, relative_install_dir, repository_clone_url ) + if tool_section: + section_id = tool_section.id + section_version = tool_section.version + section_name = tool_section.name + else: + section_id = '' + section_version = '' + section_name = '' + tool_section_dict = dict( id=section_id, version=section_version, name=section_name ) + metadata_dict = generate_metadata( app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_section_dict ) if 'datatypes_config' in metadata_dict: datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) # Load data types required by tools. @@ -568,12 +603,7 @@ # Load proprietary datatype display applications app.datatypes_registry.load_display_applications( display_path=display_path ) if 'tools' in metadata_dict: - repository_tools_tups = [] - for tool_dict in metadata_dict[ 'tools' ]: - relative_path = tool_dict[ 'tool_config' ] - guid = tool_dict[ 'guid' ] - tool = app.toolbox.load_tool( os.path.abspath( relative_path ) ) - repository_tools_tups.append( ( relative_path, guid, tool ) ) + repository_tools_tups = get_repository_tools_tups( app, metadata_dict ) if repository_tools_tups: sample_files = metadata_dict.get( 'sample_files', [] ) # Handle missing data table entries for tool parameters that are dynamically generated select lists. @@ -583,28 +613,20 @@ # Handle tools that use fabric scripts to install dependencies. handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) if new_install: - # Generate a new entry for the tool config. - elem_list = generate_tool_panel_elem_list( name, - repository_clone_url, - changeset_revision, - repository_tools_tups, - tool_section=tool_section, - owner=owner ) - if tool_section: - for section_elem in elem_list: - # Load the section into the tool panel. - app.toolbox.load_section_tag_set( section_elem, app.toolbox.tool_panel, tool_path ) - else: - # Load the tools into the tool panel outside of any sections. - for tool_elem in elem_list: - guid = tool_elem.get( 'guid' ) - app.toolbox.load_tool_tag_set( tool_elem, app.toolbox.tool_panel, tool_path=tool_path, guid=guid ) - for elem_entry in elem_list: - # Append the new entry (either section or list of tools) to the shed_tool_config file. - add_shed_tool_conf_entry( app, shed_tool_conf, elem_entry ) - if app.toolbox_search.enabled: - # If search support for tools is enabled, index the new installed tools. - app.toolbox_search = ToolBoxSearch( app.toolbox ) + load_altered_part_of_tool_panel( app=app, + repository_name=repository_name, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + repository_tools_tups=repository_tools_tups, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + tool_path=tool_path, + owner=owner, + add_to_config=True ) + else: + if app.toolbox_search.enabled: + # If search support for tools is enabled, index the new installed tools. + app.toolbox_search = ToolBoxSearch( app.toolbox ) # Remove the temporary file try: os.unlink( tmp_name ) @@ -612,9 +634,34 @@ pass # Add a new record to the tool_shed_repository table if one doesn't # already exist. If one exists but is marked deleted, undelete it. - log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % name ) - create_or_update_tool_shed_repository( app, name, description, changeset_revision, repository_clone_url, metadata_dict ) + log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % repository_name ) + create_or_update_tool_shed_repository( app, repository_name, description, changeset_revision, repository_clone_url, metadata_dict ) return metadata_dict +def load_altered_part_of_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, + tool_section, shed_tool_conf, tool_path, owner, add_to_config=True ): + # Generate a new entry for the tool config. + elem_list = generate_tool_panel_elem_list( repository_name, + repository_clone_url, + changeset_revision, + repository_tools_tups, + tool_section=tool_section, + owner=owner ) + if tool_section: + for section_elem in elem_list: + # Load the section into the tool panel. + app.toolbox.load_section_tag_set( section_elem, app.toolbox.tool_panel, tool_path ) + else: + # Load the tools into the tool panel outside of any sections. + for tool_elem in elem_list: + guid = tool_elem.get( 'guid' ) + app.toolbox.load_tool_tag_set( tool_elem, app.toolbox.tool_panel, tool_path=tool_path, guid=guid ) + if add_to_config: + # Append the new entry (either section or list of tools) to the shed_tool_config file. + for elem_entry in elem_list: + add_shed_tool_conf_entry( app, shed_tool_conf, elem_entry ) + if app.toolbox_search.enabled: + # If search support for tools is enabled, index the new installed tools. + app.toolbox_search = ToolBoxSearch( app.toolbox ) def pull_repository( current_working_dir, repo_files_dir, name ): # Pull the latest possible contents to the repository. log.debug( "Pulling latest updates to the repository named '%s'" % name ) diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -1323,7 +1323,6 @@ status = kwd.get( 'status', 'done' ) if webapp == 'galaxy': cloned_repositories = trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( trans.model.ToolShedRepository.deleted == False ) \ .first() return trans.fill_template( '/webapps/galaxy/admin/index.mako', webapp=webapp, @@ -2422,17 +2421,6 @@ ## ---- Utility methods ------------------------------------------------------- -def copy_sample_loc_file( app, filename ): - """Copy xxx.loc.sample to ~/tool-data/xxx.loc.sample and ~/tool-data/xxx.loc""" - head, sample_loc_file = os.path.split( filename ) - loc_file = sample_loc_file.replace( '.sample', '' ) - tool_data_path = os.path.abspath( app.config.tool_data_path ) - # It's ok to overwrite the .sample version of the file. - shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, sample_loc_file ) ) - # Only create the .loc file if it does not yet exist. We don't - # overwrite it in case it contains stuff proprietary to the local instance. - if not os.path.exists( os.path.join( tool_data_path, loc_file ) ): - shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) ) def get_user( trans, user_id ): """Get a User from the database by id.""" user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) ) diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -1,5 +1,6 @@ from galaxy.web.controllers.admin import * from galaxy.util.shed_util import * +from galaxy import tools log = logging.getLogger( __name__ ) @@ -101,6 +102,14 @@ operations = [ grids.GridOperation( "Get updates", allow_multiple=False, condition=( lambda item: not item.deleted ), + async_compatible=False ), + grids.GridOperation( "Deactivate or uninstall", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ), + grids.GridOperation( "Activate or reinstall", + allow_multiple=False, + condition=( lambda item: item.deleted ), async_compatible=False ) ] standard_filters = [] default_filter = dict( deleted="False" ) @@ -108,8 +117,7 @@ preserve_state = False use_paging = True def build_initial_query( self, trans, **kwd ): - return trans.sa_session.query( self.model_class ) \ - .filter( self.model_class.table.c.deleted == False ) + return trans.sa_session.query( self.model_class ) class AdminToolshed( AdminGalaxy ): @@ -140,7 +148,12 @@ return self.manage_repository( trans, **kwd ) if operation == "get updates": return self.check_for_updates( trans, **kwd ) - kwd[ 'message' ] = 'Names of repositories for which updates are available are highlighted in red.' + if operation == "activate or reinstall": + return self.activate_or_reinstall_repository( trans, **kwd ) + if operation == "deactivate or uninstall": + return self.deactivate_or_uninstall_repository( trans, **kwd ) + if 'message' not in kwd or not kwd[ 'message' ]: + kwd[ 'message' ] = 'Names of repositories for which updates are available are highlighted in red.' return self.repository_list_grid( trans, **kwd ) @web.expose @web.require_admin @@ -218,7 +231,8 @@ elem = Element( 'section' ) elem.attrib[ 'name' ] = new_tool_panel_section elem.attrib[ 'id' ] = section_id - tool_section = ToolSection( elem ) + elem.attrib[ 'version' ] = '' + tool_section = tools.ToolSection( elem ) trans.app.toolbox.tool_panel[ new_section_key ] = tool_section else: section_key = 'section_%s' % tool_panel_section @@ -245,7 +259,7 @@ if returncode == 0: owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) metadata_dict = load_repository_contents( app=trans.app, - name=name, + repository_name=name, description=description, owner=owner, changeset_revision=changeset_revision, @@ -305,13 +319,180 @@ status=status ) @web.expose @web.require_admin + def deactivate_or_uninstall_repository( self, trans, **kwd ): + # TODO: handle elimination of workflows loaded into the tool panel that are in the repository being deactivated. + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + remove_from_disk = params.get( 'remove_from_disk', '' ) + remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) + repository = get_repository( trans, kwd[ 'id' ] ) + if params.get( 'deactivate_or_uninstall_repository_button', False ): + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) + metadata = repository.metadata + repository_tools_tups = get_repository_tools_tups( trans.app, metadata ) + # Generate the list of tool panel keys derived from the tools included in the repository. + repository_tool_panel_keys = [] + if repository_tools_tups: + repository_tool_panel_keys = [ 'tool_%s' % repository_tools_tup[ 1 ] for repository_tools_tup in repository_tools_tups ] + tool_panel_section = metadata[ 'tool_panel_section' ] + section_id = tool_panel_section[ 'id' ] + if section_id in [ '' ]: + # If the repository includes tools, they were loaded into the tool panel outside of any sections. + tool_section = None + self.__deactivate( trans, repository_tool_panel_keys ) + else: + # If the repository includes tools, they were loaded into the tool panel inside a section. + section_key = 'section_%s' % str( section_id ) + if section_key in trans.app.toolbox.tool_panel: + tool_section = trans.app.toolbox.tool_panel[ section_key ] + self.__deactivate( trans, repository_tool_panel_keys, tool_section=tool_section, section_key=section_key ) + else: + # The tool panel section could not be found, so handle deactivating tools + # as if they were loaded into the tool panel outside of any sections. + tool_section = None + self.__deactivate( trans, repository_tool_panel_keys ) + repository.deleted = True + trans.sa_session.add( repository ) + trans.sa_session.flush() + repository_clone_url = self.__generate_clone_url( trans, repository ) + load_altered_part_of_tool_panel( app=trans.app, + repository_name=repository.name, + repository_clone_url=repository_clone_url, + changeset_revision=repository.installed_changeset_revision, + repository_tools_tups=repository_tools_tups, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + tool_path=tool_path, + owner=repository.owner, + add_to_config=False ) + if remove_from_disk_checked: + # TODO: Remove repository from disk and alter the shed tool config. + message = 'The repository named <b>%s</b> has been uninstalled.' % repository.name + else: + message = 'The repository named <b>%s</b> has been deactivated.' % repository.name + status = 'done' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) + remove_from_disk_check_box = CheckboxField( 'remove_from_disk', checked=remove_from_disk_checked ) + return trans.fill_template( '/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako', + repository=repository, + remove_from_disk_check_box=remove_from_disk_check_box, + message=message, + status=status ) + def __deactivate( self, trans, repository_tool_panel_keys, tool_section=None, section_key=None ): + # Delete tools loaded into the tool panel. If the tool_panel is received, + # the tools were loaded into the tool panel outside of any sections. If + # the tool_section is received, all tools were loaded into that section. + if tool_section: + # The tool_section.elems dictionary looks something like: + # {'tool_gvk.bx.psu.edu:9009/repos/test/filter/Filter1/1.0.1': <galaxy.tools.Tool instance at 0x10769ae60>} + for item_id, item in tool_section.elems.items(): + if item_id in repository_tool_panel_keys: + del tool_section.elems[ item_id ] + if not tool_section.elems: + del trans.app.toolbox.tool_panel[ section_key ] + else: + # The tool panel looks something like: + # {'tool_gvk.bx.psu.edu:9009/repos/test/blast2go/blast2go/0.0.2': <galaxy.tools.Tool instance at 0x1076a5f80>} + for item_id, item in trans.app.toolbox.tool_panel.items(): + if item_id in repository_tool_panel_keys: + del trans.app.toolbox.tool_panel[ item_id ] + @web.expose + @web.require_admin + def activate_or_reinstall_repository( self, trans, **kwd ): + # TODO: handle re-introduction of workflows loaded into the tool panel that are in the repository being activated. + repository = get_repository( trans, kwd[ 'id' ] ) + repository.deleted = False + trans.sa_session.add( repository ) + trans.sa_session.flush() + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) + metadata = repository.metadata + repository_tools_tups = get_repository_tools_tups( trans.app, metadata ) + tool_panel_section = metadata[ 'tool_panel_section' ] + section_id = tool_panel_section[ 'id' ] + # Generate the list of tool panel keys derived from the tools included in the repository. + #repository_tool_panel_keys = [] + #if repository_tools_tups: + # repository_tool_panel_keys = [ 'tool_%s' % repository_tools_tup[ 1 ] for repository_tools_tup in repository_tools_tups ] + if section_id in [ '' ]: + # If the repository includes tools, reload them into the tool panel outside of any sections. + self.__activate( trans, repository, repository_tools_tups, tool_section=None, section_key=None ) + else: + # If the repository includes tools, reload them into the appropriate tool panel section. + section_key = 'section_%s' % str( section_id ) + if section_key in trans.app.toolbox.tool_panel: + # Load the repository tools into a section that still exists in the tool panel. + tool_section = trans.app.toolbox.tool_panel[ section_key ] + self.__activate( trans, repository, repository_tools_tups, tool_section=tool_section, section_key=section_key ) + else: + # Load the repository tools into a section that no longer exists in the tool panel. + # We first see if the section exists in the tool config, which will be the case if + # the repository was only deactivated and not uninstalled. + section_loaded = False + for tcf in trans.app.config.tool_configs: + # Only inspect tool configs that contain installed tool shed repositories. + if tcf not in [ 'tool_conf.xml' ]: + log.info( "Parsing the tool configuration %s" % tcf ) + tree = util.parse_xml( tcf ) + root = tree.getroot() + tool_path = root.get( 'tool_path' ) + if tool_path is not None: + # Tool configs that contain tools installed from tool shed repositories + # must have a tool_path attribute. + for elem in root: + if elem.tag == 'section': + id_attr = elem.get( 'id' ) + if id_attr == section_id: + tool_section = elem + trans.app.toolbox.load_section_tag_set( tool_section, trans.app.toolbox.tool_panel, tool_path ) + section_loaded = True + break + if section_loaded: + break + message = 'The repository named <b>%s</b> has been activated.' % repository.name + status = 'done' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) + def __activate( self, trans, repository, repository_tools_tups, tool_section=None, section_key=None ): + if tool_section: + elems = tool_section.elems + for repository_tools_tup in repository_tools_tups: + relative_path, guid, tool = repository_tools_tup + tool.tool_shed = repository.tool_shed + tool.repository_name = repository.name + tool.repository_owner = repository.owner + tool.installed_changeset_revision = repository.installed_changeset_revision + # Set the tool's old_id to the id used before the tool shed existed. + tool.old_id = tool.id + # Set the tool's id to the tool shed guid. + tool.id = guid + if tool_section: + if tool.id not in elems: + elems[ 'tool_%s' % tool.id ] = tool + log.debug( "Reactivated tool: %s %s" % ( tool.id, tool.version ) ) + else: + if tool.id not in trans.app.toolbox.tools_by_id: + # Allow for the same tool to be loaded into multiple places in the tool panel. + trans.app.toolbox.tools_by_id[ tool.id ] = tool + trans.app.toolbox.tool_panel[ 'tool_%s' % tool.id ] = tool + log.debug( "Reactivated tool: %s %s" % ( tool.id, tool.version ) ) + if tool_section: + trans.app.toolbox.tool_panel[ section_key ] = tool_section + log.debug( "Appended reactivated tool to section: %s" % tool_section.name ) + @web.expose + @web.require_admin def manage_repository( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository = get_repository( trans, kwd[ 'id' ] ) description = util.restore_text( params.get( 'description', repository.description ) ) - tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) if params.get( 'edit_repository_button', False ): if description != repository.description: @@ -321,7 +502,12 @@ message = "The repository information has been updated." elif params.get( 'set_metadata_button', False ): repository_clone_url = self.__generate_clone_url( trans, repository ) - metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url ) + # In case metadata was previously generated for this repository, we'll + # check to see if it has information needed for the tool_section_dict. + metadata = repository.metadata + if 'tool_panel_section' in metadata: + tool_section_dict = metadata[ 'tool_panel_section' ] + metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_section_dict ) if metadata_dict: repository.metadata = metadata_dict trans.sa_session.add( repository ) @@ -360,7 +546,7 @@ message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name else: current_working_dir = os.getcwd() - tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) if relative_install_dir: repo_files_dir = os.path.join( relative_install_dir, name ) returncode, tmp_name = pull_repository( current_working_dir, repo_files_dir, name ) @@ -440,17 +626,17 @@ # Get the relative tool installation paths from each of the shed tool configs. shed_tool_confs = trans.app.toolbox.shed_tool_confs relative_install_dir = None - # The shed_tool_confs dictionary contains { shed_conf_filename : tool_path } pairs. - for shed_conf_filename, tool_path in shed_tool_confs.items(): + # The shed_tool_confs dictionary contains { shed_tool_conf : tool_path } pairs. + for shed_tool_conf, tool_path in shed_tool_confs.items(): relative_install_dir = os.path.join( tool_path, partial_install_dir ) if os.path.isdir( relative_install_dir ): break - return tool_path, relative_install_dir + return shed_tool_conf, tool_path, relative_install_dir def __generate_tool_path( self, repository_clone_url, changeset_revision ): """ Generate a tool path that guarantees repositories with the same name will always be installed in different directories. The tool path will be of the form: - <tool shed url>/repos/<repository owner>/<repository name>/<changeset revision> + <tool shed url>/repos/<repository owner>/<repository name>/<installed changeset revision> http://test@bx.psu.edu:9009/repos/test/filter """ tmp_url = clean_repository_clone_url( repository_clone_url ) diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -4,7 +4,7 @@ from galaxy.tools import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.hash_util import * -from galaxy.util.shed_util import generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata +from galaxy.util.shed_util import copy_sample_loc_file, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata from galaxy.web.base.controller import * from galaxy.webapps.community import model from galaxy.model.orm import * diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 templates/admin/tool_shed_repository/browse_repository.mako --- a/templates/admin/tool_shed_repository/browse_repository.mako +++ b/templates/admin/tool_shed_repository/browse_repository.mako @@ -10,6 +10,7 @@ <div popupmenu="repository-${repository.id}-popup"><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='deactivate_or_uninstall_repository', id=trans.security.encode_id( repository.id ) )}">Deactivate or Uninstall</a></div></ul> diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako @@ -0,0 +1,77 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> +<%namespace file="/admin/tool_shed_repository/common.mako" import="*" /> + +<br/><br/> +<ul class="manage-table-actions"> + <li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li> + <div popupmenu="repository-${repository.id}-popup"> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + </div> +</ul> + +%if message: + ${render_msg( message, status )} +%endif + +<div class="toolForm"> + <div class="toolFormTitle">${repository.name}</div> + <div class="toolFormBody"> + <form name="uninstall_repository" id="uninstall_repository" action="${h.url_for( controller='admin_toolshed', action='deactivate_or_uninstall_repository', id=trans.security.encode_id( repository.id ) )}" method="post" > + <div class="form-row"> + <label>Description:</label> + ${repository.description} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Revision:</label> + ${repository.changeset_revision}</a> + </div> + <div class="form-row"> + <label>Tool shed:</label> + ${repository.tool_shed} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Owner:</label> + ${repository.owner} + </div> + <div class="form-row"> + <label>Deleted:</label> + ${repository.deleted} + </div> + <div class="form-row"> + ${remove_from_disk_check_box.get_html( disabled=True )} + <label for="repository" style="display: inline;font-weight:normal;">Check to uninstall (not yet implemented) or leave blank to deactivate</label> + <br/><br/> + <label>Deactivating this repository will result in the following:</label> + <div class="toolParamHelp" style="clear: both;"> + 1. This repository record's deleted column in the tool_shed_repository database table will be set to True. + </div> + <div class="toolParamHelp" style="clear: both;"> + 2. The repository and all of it's contents will remain on disk. + </div> + <div class="toolParamHelp" style="clear: both;"> + 3. If this repository includes tools, they will not be loaded into the tool panel, but the tool config file in which they are defined will not be altered. + </div> + <br/> + <label>Uninstalling (not yet implemented) this repository will result in the following:</label> + <div class="toolParamHelp" style="clear: both;"> + 1. This repository record's deleted column in the tool_shed_repository database table will be set to True. + </div> + <div class="toolParamHelp" style="clear: both;"> + 2. The repository and all of it's contents will be removed from disk. + </div> + <div class="toolParamHelp" style="clear: both;"> + 3. If this repository includes tools, they will be removed from the tool config file in which they are defined and they will not be loaded into the tool panel. + </div> + <div style="clear: both"></div> + </div> + <div class="form-row"> + <input type="submit" name="deactivate_or_uninstall_repository_button" value="Deactivate or Uninstall"/> + </div> + </form> + </div> +</div> diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 templates/admin/tool_shed_repository/manage_repository.mako --- a/templates/admin/tool_shed_repository/manage_repository.mako +++ b/templates/admin/tool_shed_repository/manage_repository.mako @@ -8,6 +8,7 @@ <div popupmenu="repository-${repository.id}-popup"><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='deactivate_or_uninstall_repository', id=trans.security.encode_id( repository.id ) )}">Deactivate or Uninstall</a></div></ul> diff -r 4a130280e6142f2c241e33a2bacc6d52e566047c -r 3d9f0ce7bb6138b20c200203014e60a4e9342160 templates/admin/tool_shed_repository/view_tool_metadata.mako --- a/templates/admin/tool_shed_repository/view_tool_metadata.mako +++ b/templates/admin/tool_shed_repository/view_tool_metadata.mako @@ -8,6 +8,7 @@ <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='deactivate_or_uninstall_repository', id=trans.security.encode_id( repository.id ) )}">Deactivate or Uninstall</a></div></ul> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.