1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/5340450cdab2/ changeset: 5340450cdab2 user: greg date: 2011-12-02 20:49:43 summary: 1. Add 3 columns to the tool_shed_repository table in the Galaxy model / database; metadata (json), includes_datatypes (boolean) and update_available (boolean) 2. Add the ability to set metadata for tool shed repositories installed into local Galaxy instances (at time of install and on the Manage repository page) 3. Add the ability to view tool shed repository metadata (including tool guids) when managing installed tool shed repositories within Galaxy 4. Create a new admin_toolshed controller, and move all tool shed related code from the admin controller to this new controller affected #: 16 files diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2646,7 +2646,8 @@ pass class ToolShedRepository( object ): - def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, changeset_revision=None, deleted=False ): + def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, + changeset_revision=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False ): self.id = id self.create_time = create_time self.tool_shed = tool_shed @@ -2654,6 +2655,9 @@ self.description = description self.owner = owner self.changeset_revision = changeset_revision + self.metadata = metadata + self.includes_datatypes = includes_datatypes + self.update_available = update_available self.deleted = deleted ## ---- Utility methods ------------------------------------------------------- diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -373,6 +373,9 @@ Column( "description" , TEXT ), Column( "owner", TrimmedString( 255 ), index=True ), Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "metadata", JSONType, nullable=True ), + Column( "includes_datatypes", Boolean, index=True, default=False ), + Column( "update_available", Boolean, default=False ), Column( "deleted", Boolean, index=True, default=False ) ) Job.table = Table( "job", metadata, diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py @@ -0,0 +1,80 @@ +""" +Migration script to add the metadata, update_available and includes_datatypes columns to the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + c = Column( "metadata", JSONType(), nullable=True ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.metadata + except Exception, e: + print "Adding metadata column to the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Adding metadata column to the tool_shed_repository table failed: %s" % str( e ) ) + c = Column( "includes_datatypes", Boolean, index=True, default=False ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.includes_datatypes + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + db_session.execute( "UPDATE tool_shed_repository SET includes_datatypes=%s" % default_false ) + except Exception, e: + print "Adding includes_datatypes column to the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Adding includes_datatypes column to the tool_shed_repository table failed: %s" % str( e ) ) + c = Column( "update_available", Boolean, default=False ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.update_available + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + db_session.execute( "UPDATE tool_shed_repository SET update_available=%s" % default_false ) + except Exception, e: + print "Adding update_available column to the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Adding update_available column to the tool_shed_repository table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.metadata.drop() + except Exception, e: + print "Dropping column metadata from the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Dropping column metadata from the tool_shed_repository table failed: %s" % str( e ) ) + try: + ToolShedRepository_table.c.includes_datatypes.drop() + except Exception, e: + print "Dropping column includes_datatypes from the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Dropping column includes_datatypes from the tool_shed_repository table failed: %s" % str( e ) ) + try: + ToolShedRepository_table.c.update_available.drop() + except Exception, e: + print "Dropping column update_available from the tool_shed_repository table failed: %s" % str( e ) + log.debug( "Dropping column update_available from the tool_shed_repository table failed: %s" % str( e ) ) diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 lib/galaxy/web/controllers/admin.py --- a/lib/galaxy/web/controllers/admin.py +++ b/lib/galaxy/web/controllers/admin.py @@ -5,13 +5,13 @@ from galaxy.tools.search import ToolBoxSearch from galaxy.tools import ToolSection, json_fix from galaxy.util import parse_xml, inflector -import logging -log = logging.getLogger( __name__ ) - from galaxy.actions.admin import AdminActions from galaxy.web.params import QuotaParamParser from galaxy.exceptions import * import galaxy.datatypes.registry +import logging + +log = logging.getLogger( __name__ ) class UserListGrid( grids.Grid ): class EmailColumn( grids.TextColumn ): @@ -383,63 +383,12 @@ preserve_state = False use_paging = True -class RepositoryListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_shed_repository ): - return tool_shed_repository.name - class DescriptionColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_shed_repository ): - return tool_shed_repository.description - class OwnerColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_shed_repository ): - return tool_shed_repository.owner - class RevisionColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_shed_repository ): - return tool_shed_repository.changeset_revision - class ToolShedColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_shed_repository ): - return tool_shed_repository.tool_shed - # Grid definition - title = "Installed tool shed repositories" - model_class = model.ToolShedRepository - template='/admin/tool_shed_repository/grid.mako' - default_sort_key = "name" - columns = [ - NameColumn( "Name", - key="name", - link=( lambda item: dict( operation="manage_repository", id=item.id, webapp="galaxy" ) ), - attach_popup=False ), - DescriptionColumn( "Description" ), - OwnerColumn( "Owner" ), - RevisionColumn( "Revision" ), - ToolShedColumn( "Tool shed" ), - # Columns that are valid for filtering but are not visible. - grids.DeletedColumn( "Deleted", - key="deleted", - visible=False, - filterable="advanced" ) - ] - columns.append( grids.MulticolFilterColumn( "Search repository name", - cols_to_filter=[ columns[0] ], - key="free-text-search", - visible=False, - filterable="standard" ) ) - standard_filters = [] - default_filter = dict( deleted="False" ) - num_rows_per_page = 50 - preserve_state = False - use_paging = True - def build_initial_query( self, trans, **kwd ): - return trans.sa_session.query( self.model_class ) \ - .filter( self.model_class.table.c.deleted == False ) - class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamParser ): user_list_grid = UserListGrid() role_list_grid = RoleListGrid() group_list_grid = GroupListGrid() quota_list_grid = QuotaListGrid() - repository_list_grid = RepositoryListGrid() delete_operation = grids.GridOperation( "Delete", condition=( lambda item: not item.deleted ), allow_multiple=True ) undelete_operation = grids.GridOperation( "Undelete", condition=( lambda item: item.deleted and not item.purged ), allow_multiple=True ) purge_operation = grids.GridOperation( "Purge", condition=( lambda item: item.deleted and not item.purged ), allow_multiple=True ) @@ -676,391 +625,6 @@ return quota, params @web.expose @web.require_admin - def browse_tool_shed_repository( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - repository = get_repository( trans, kwd[ 'id' ] ) - relative_install_dir = self.__get_relative_install_dir( trans, repository ) - repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) - tool_dicts = [] - workflow_dicts = [] - for root, dirs, files in os.walk( repo_files_dir ): - if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: - if '.hg' in dirs: - # Don't visit .hg directories. - dirs.remove( '.hg' ) - if 'hgrc' in files: - # Don't include hgrc files. - files.remove( 'hgrc' ) - for name in files: - # Find all tool configs. - if name.endswith( '.xml' ): - try: - full_path = os.path.abspath( os.path.join( root, name ) ) - tool = trans.app.toolbox.load_tool( full_path ) - if tool is not None: - tool_config = os.path.join( root, name ) - # Handle tool.requirements. - tool_requirements = [] - for tr in tool.requirements: - name=tr.name - type=tr.type - if type == 'fabfile': - version = None - fabfile = tr.fabfile - method = tr.method - else: - version = tr.version - fabfile = None - method = None - requirement_dict = dict( name=name, - type=type, - version=version, - fabfile=fabfile, - method=method ) - tool_requirements.append( requirement_dict ) - tool_dict = dict( id=tool.id, - old_id=tool.old_id, - name=tool.name, - version=tool.version, - description=tool.description, - requirements=tool_requirements, - tool_config=tool_config ) - tool_dicts.append( tool_dict ) - except Exception, e: - # The file is not a Galaxy tool config. - pass - # Find all exported workflows - elif name.endswith( '.ga' ): - try: - full_path = os.path.abspath( os.path.join( root, name ) ) - # Convert workflow data from json - fp = open( full_path, 'rb' ) - workflow_text = fp.read() - fp.close() - workflow_dict = from_json_string( workflow_text ) - if workflow_dict[ 'a_galaxy_workflow' ] == 'true': - workflow_dicts.append( dict( full_path=full_path, workflow_dict=workflow_dict ) ) - except Exception, e: - # The file is not a Galaxy workflow. - pass - return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako', - repository=repository, - tool_dicts=tool_dicts, - workflow_dicts=workflow_dicts, - message=message, - status=status ) - @web.expose - @web.require_admin - def browse_tool_shed_repositories( self, trans, **kwd ): - if 'operation' in kwd: - operation = kwd.pop( 'operation' ).lower() - if operation == "manage_repository": - return self.manage_tool_shed_repository( trans, **kwd ) - # Render the list view - return self.repository_list_grid( trans, **kwd ) - @web.expose - @web.require_admin - def browse_tool_sheds( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - return trans.fill_template( '/webapps/galaxy/admin/tool_sheds.mako', - webapp='galaxy', - message=message, - status='error' ) - @web.expose - @web.require_admin - def find_tools_in_tool_shed( self, trans, **kwd ): - tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = url_for( '', qualified=True ) - url = '%s/repository/find_tools?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) - return trans.response.send_redirect( url ) - @web.expose - @web.require_admin - def find_workflows_in_tool_shed( self, trans, **kwd ): - tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = url_for( '', qualified=True ) - url = '%s/repository/find_workflows?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) - return trans.response.send_redirect( url ) - @web.expose - @web.require_admin - def browse_tool_shed( self, trans, **kwd ): - tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = url_for( '', qualified=True ) - url = '%s/repository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) - return trans.response.send_redirect( url ) - @web.expose - @web.require_admin - def install_tool_shed_repository( self, trans, **kwd ): - if not trans.app.toolbox.shed_tool_confs: - message = 'The <b>tool_config_file</b> setting in <b>universe_wsgi.ini</b> must include at least one shed tool configuration file name with a ' - message += '<b><toolbox></b> tag that includes a <b>tool_path</b> attribute value which is a directory relative to the Galaxy installation ' - message += 'directory in order to automatically install tools from a Galaxy tool shed (e.g., the file name <b>shed_tool_conf.xml</b> whose ' - message += '<b><toolbox></b> tag is <b><toolbox tool_path="../shed_tools"></b>).<p/>See the ' - message += '<a href="http://wiki.g2.bx.psu.edu/Tool%20Shed#Automatic_installation_of_Galaxy_tool_..." ' - message += 'target="_blank">Automatic installation of Galaxy tool shed repository tools into a local Galaxy instance</a> section of the ' - message += '<a href="http://wiki.g2.bx.psu.edu/Tool%20Shed" target="_blank">Galaxy tool shed wiki</a> for all of the details.' - return trans.show_error_message( message ) - message = kwd.get( 'message', '' ) - status = kwd.get( 'status', 'done' ) - tool_shed_url = kwd[ 'tool_shed_url' ] - repo_info_dict = kwd[ 'repo_info_dict' ] - new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) - tool_panel_section = kwd.get( 'tool_panel_section', '' ) - if kwd.get( 'select_tool_panel_section_button', False ): - shed_tool_conf = kwd[ 'shed_tool_conf' ] - # Get the tool path. - for k, tool_path in trans.app.toolbox.shed_tool_confs.items(): - if k == shed_tool_conf: - break - if new_tool_panel_section or tool_panel_section: - if new_tool_panel_section: - section_id = new_tool_panel_section.lower().replace( ' ', '_' ) - new_section_key = 'section_%s' % str( section_id ) - if new_section_key in trans.app.toolbox.tool_panel: - # Appending a tool to an existing section in trans.app.toolbox.tool_panel - log.debug( "Appending to tool panel section: %s" % new_tool_panel_section ) - tool_section = trans.app.toolbox.tool_panel[ new_section_key ] - else: - # Appending a new section to trans.app.toolbox.tool_panel - log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) - elem = Element( 'section' ) - elem.attrib[ 'name' ] = new_tool_panel_section - elem.attrib[ 'id' ] = section_id - tool_section = ToolSection( elem ) - trans.app.toolbox.tool_panel[ new_section_key ] = tool_section - else: - section_key = 'section_%s' % tool_panel_section - tool_section = trans.app.toolbox.tool_panel[ section_key ] - # Decode the encoded repo_info_dict param value. - repo_info_dict = tool_shed_decode( repo_info_dict ) - # Clone the repository to the configured location. - current_working_dir = os.getcwd() - installed_repository_names = [] - for name, repo_info_tuple in repo_info_dict.items(): - description, repository_clone_url, changeset_revision = repo_info_tuple - clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) - if os.path.exists( clone_dir ): - # Repository and revision has already been cloned. - # TODO: implement the ability to re-install or revert an existing repository. - message += 'Revision <b>%s</b> of repository <b>%s</b> was previously installed.<br/>' % ( changeset_revision, name ) - else: - os.makedirs( clone_dir ) - log.debug( 'Cloning %s...' % repository_clone_url ) - cmd = 'hg clone %s' % repository_clone_url - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( clone_dir ) - proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode == 0: - # Add a new record to the tool_shed_repository table if one doesn't - # already exist. If one exists but is marked deleted, undelete it. - self.__create_or_undelete_tool_shed_repository( trans, name, description, changeset_revision, repository_clone_url ) - # Update the cloned repository to changeset_revision. - repo_files_dir = os.path.join( clone_dir, name ) - log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision ) - cmd = 'hg update -r %s' % changeset_revision - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode == 0: - # Load data types required by tools. - self.__load_datatypes( trans, repo_files_dir ) - # Load tools and tool data files required by them. - sample_files, repository_tools_tups = self.__get_repository_tools_and_sample_files( trans, tool_path, repo_files_dir ) - if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = self.__handle_missing_data_table_entry( trans, tool_path, sample_files, repository_tools_tups ) - # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = self.__handle_missing_index_file( trans, tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. - self.__handle_tool_dependencies( current_working_dir, repo_files_dir, repository_tools_tups ) - # Generate an in-memory tool conf section that includes the new tools. - new_tool_section = self.__generate_tool_panel_section( name, - repository_clone_url, - changeset_revision, - tool_section, - repository_tools_tups ) - # Create a temporary file to persist the in-memory tool section - # TODO: Figure out how to do this in-memory using xml.etree. - tmp_name = tempfile.NamedTemporaryFile().name - persisted_new_tool_section = open( tmp_name, 'wb' ) - persisted_new_tool_section.write( new_tool_section ) - persisted_new_tool_section.close() - # Parse the persisted tool panel section - tree = parse_xml( tmp_name ) - root = tree.getroot() - # Load the tools in the section into the tool panel. - trans.app.toolbox.load_section_tag_set( root, trans.app.toolbox.tool_panel, tool_path ) - # Remove the temporary file - try: - os.unlink( tmp_name ) - except: - pass - # Append the new section to the shed_tool_config file. - self.__add_shed_tool_conf_entry( trans, shed_tool_conf, new_tool_section ) - if trans.app.toolbox_search.enabled: - # If search support for tools is enabled, index the new installed tools. - trans.app.toolbox_search = ToolBoxSearch( trans.app.toolbox ) - installed_repository_names.append( name ) - else: - tmp_stderr = open( tmp_name, 'rb' ) - message += '%s<br/>' % tmp_stderr.read() - tmp_stderr.close() - status = 'error' - else: - tmp_stderr = open( tmp_name, 'rb' ) - message += '%s<br/>' % tmp_stderr.read() - tmp_stderr.close() - status = 'error' - if installed_repository_names: - installed_repository_names.sort() - num_repositories_installed = len( installed_repository_names ) - message += 'Installed %d %s and all tools were loaded into tool panel section <b>%s</b>:<br/>Installed repositories: ' % \ - ( num_repositories_installed, inflector.cond_plural( num_repositories_installed, 'repository' ), tool_section.name ) - for i, repo_name in enumerate( installed_repository_names ): - if i == len( installed_repository_names ) -1: - message += '%s.<br/>' % repo_name - else: - message += '%s, ' % repo_name - return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tool_shed_repositories', - message=message, - status=status ) ) - else: - message = 'Choose the section in your tool panel to contain the installed tools.' - status = 'error' - if len( trans.app.toolbox.shed_tool_confs.keys() ) > 1: - shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) - shed_tool_conf = None - else: - shed_tool_conf = trans.app.toolbox.shed_tool_confs.keys()[0].lstrip( './' ) - shed_tool_conf_select_field = None - tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) - return trans.fill_template( '/admin/select_tool_panel_section.mako', - tool_shed_url=tool_shed_url, - repo_info_dict=repo_info_dict, - shed_tool_conf=shed_tool_conf, - shed_tool_conf_select_field=shed_tool_conf_select_field, - tool_panel_section_select_field=tool_panel_section_select_field, - new_tool_panel_section=new_tool_panel_section, - message=message, - status=status ) - @web.expose - @web.require_admin - def manage_tool_shed_repository( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - repository_id = params.get( 'id', None ) - repository = get_repository( trans, repository_id ) - description = util.restore_text( params.get( 'description', repository.description ) ) - if params.get( 'edit_repository_button', False ): - if description != repository.description: - repository.description = description - trans.sa_session.add( repository ) - trans.sa_session.flush() - message = "The repository information has been updated." - relative_install_dir = self.__get_relative_install_dir( trans, repository ) - if relative_install_dir: - repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) - else: - repo_files_dir = 'unknown' - return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', - repository=repository, - description=description, - repo_files_dir=repo_files_dir, - message=message, - status=status ) - @web.expose - @web.require_admin - def check_for_updates( self, trans, **kwd ): - # Send a request to the relevant tool shed to see if there are any updates. - repository = get_repository( trans, kwd[ 'id' ] ) - tool_shed_url = get_url_from_repository_tool_shed( trans, repository ) - url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, url_for( '', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) - return trans.response.send_redirect( url ) - @web.expose - @web.require_admin - def update_to_changeset_revision( self, trans, **kwd ): - """Update a cloned repository to the latest revision possible.""" - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - tool_shed_url = kwd[ 'tool_shed_url' ] - name = params.get( 'name', None ) - owner = params.get( 'owner', None ) - changeset_revision = params.get( 'changeset_revision', None ) - latest_changeset_revision = params.get( 'latest_changeset_revision', None ) - repository = get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed_url, name, owner, changeset_revision ) - if changeset_revision and latest_changeset_revision: - if changeset_revision == latest_changeset_revision: - message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name - else: - current_working_dir = os.getcwd() - relative_install_dir = self.__get_relative_install_dir( trans, repository ) - if relative_install_dir: - # Update the cloned repository to changeset_revision. - repo_files_dir = os.path.join( relative_install_dir, name ) - log.debug( "Updating cloned repository named '%s' from revision '%s' to revision '%s'..." % \ - ( name, changeset_revision, latest_changeset_revision ) ) - cmd = 'hg pull' - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode == 0: - cmd = 'hg update -r %s' % latest_changeset_revision - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode == 0: - # Update the repository changeset_revision in the database. - repository.changeset_revision = latest_changeset_revision - trans.sa_session.add( repository ) - trans.sa_session.flush() - message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \ - ( name, latest_changeset_revision ) - else: - tmp_stderr = open( tmp_name, 'rb' ) - message = tmp_stderr.read() - tmp_stderr.close() - status = 'error' - else: - tmp_stderr = open( tmp_name, 'rb' ) - message = tmp_stderr.read() - tmp_stderr.close() - status = 'error' - else: - message = "The directory containing the cloned repository named '%s' cannot be found." % name - status = 'error' - else: - message = "The latest changeset revision could not be retrieved for the repository named '%s'." % name - status = 'error' - return trans.response.send_redirect( web.url_for( controller='admin', - action='manage_tool_shed_repository', - id=trans.security.encode_id( repository.id ), - message=message, - status=status ) ) - @web.expose - @web.require_admin def impersonate( self, trans, email=None, **kwd ): if not trans.app.config.allow_user_impersonation: return trans.show_error_message( "User impersonation is not enabled in this instance of Galaxy." ) @@ -1079,354 +643,4 @@ if emails is None: emails = [ u.email for u in trans.sa_session.query( trans.app.model.User ).enable_eagerloads( False ).all() ] return trans.fill_template( 'admin/impersonate.mako', emails=emails, message=message, status=status ) - - def __get_relative_install_dir( self, trans, repository ): - # Get the directory where the repository is install. - tool_shed = self.__clean_tool_shed_url( repository.tool_shed ) - partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.changeset_revision ) - # Get the relative tool installation paths from each of the shed tool configs. - shed_tool_confs = trans.app.toolbox.shed_tool_confs - relative_install_dir = None - # The shed_tool_confs dictionary contains { shed_conf_filename : tool_path } pairs. - for shed_conf_filename, tool_path in shed_tool_confs.items(): - relative_install_dir = os.path.join( tool_path, partial_install_dir ) - if os.path.isdir( relative_install_dir ): - break - return relative_install_dir - def __handle_missing_data_table_entry( self, trans, tool_path, sample_files, repository_tools_tups ): - # Inspect each tool to see if any have input parameters that are dynamically - # generated select lists that require entries in the tool_data_table_conf.xml file. - missing_data_table_entry = False - for index, repository_tools_tup in enumerate( repository_tools_tups ): - tup_path, repository_tool = repository_tools_tup - if repository_tool.params_with_missing_data_table_entry: - missing_data_table_entry = True - break - if missing_data_table_entry: - # The repository must contain a tool_data_table_conf.xml.sample file that includes - # all required entries for all tools in the repository. - for sample_file in sample_files: - head, tail = os.path.split( sample_file ) - if tail == 'tool_data_table_conf.xml.sample': - break - error, correction_msg = handle_sample_tool_data_table_conf_file( trans, sample_file ) - if error: - # TODO: Do more here than logging an exception. - log.debug( exception_msg ) - # Reload the tool into the local list of repository_tools_tups. - repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) - repository_tools_tups[ index ] = ( tup_path, repository_tool ) - return repository_tools_tups - def __handle_missing_index_file( self, trans, tool_path, sample_files, repository_tools_tups ): - # Inspect each tool to see if it has any input parameters that - # are dynamically generated select lists that depend on a .loc file. - missing_files_handled = [] - for index, repository_tools_tup in enumerate( repository_tools_tups ): - tup_path, repository_tool = repository_tools_tup - params_with_missing_index_file = repository_tool.params_with_missing_index_file - for param in params_with_missing_index_file: - options = param.options - missing_head, missing_tail = os.path.split( options.missing_index_file ) - if missing_tail not in missing_files_handled: - # The repository must contain the required xxx.loc.sample file. - for sample_file in sample_files: - sample_head, sample_tail = os.path.split( sample_file ) - if sample_tail == '%s.sample' % missing_tail: - copy_sample_loc_file( trans, sample_file ) - if options.tool_data_table and options.tool_data_table.missing_index_file: - options.tool_data_table.handle_found_index_file( options.missing_index_file ) - missing_files_handled.append( missing_tail ) - break - # Reload the tool into the local list of repository_tools_tups. - repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) - repository_tools_tups[ index ] = ( tup_path, repository_tool ) - return repository_tools_tups - def __handle_tool_dependencies( self, current_working_dir, repo_files_dir, repository_tools_tups ): - # Inspect each tool to see if it includes a "requirement" that refers to a fabric - # script. For those that do, execute the fabric script to install tool dependencies. - for index, repository_tools_tup in enumerate( repository_tools_tups ): - tup_path, repository_tool = repository_tools_tup - for requirement in repository_tool.requirements: - if requirement.type == 'fabfile': - log.debug( 'Executing fabric script to install dependencies for tool "%s"...' % repository_tool.name ) - fabfile = requirement.fabfile - method = requirement.method - # Find the relative path to the fabfile. - relative_fabfile_path = None - for root, dirs, files in os.walk( repo_files_dir ): - for name in files: - if name == fabfile: - relative_fabfile_path = os.path.join( root, name ) - break - if relative_fabfile_path: - # cmd will look something like: fab -f fabfile.py install_bowtie - cmd = 'fab -f %s %s' % ( relative_fabfile_path, method ) - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode != 0: - # TODO: do something more here than logging the problem. - tmp_stderr = open( tmp_name, 'rb' ) - error = tmp_stderr.read() - tmp_stderr.close() - log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) - def __load_datatypes( self, trans, repo_files_dir ): - # Find datatypes_conf.xml if it exists. - datatypes_config = None - for root, dirs, files in os.walk( repo_files_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == 'datatypes_conf.xml': - datatypes_config = os.path.abspath( os.path.join( root, name ) ) - break - if datatypes_config: - imported_module = None - # Parse datatypes_config. - tree = parse_xml( datatypes_config ) - datatypes_config_root = tree.getroot() - relative_path_to_datatype_file_name = None - datatype_files = datatypes_config_root.find( 'datatype_files' ) - # Currently only a single datatype_file is supported. For example: - # <datatype_files> - # <datatype_file name="gmap.py"/> - # </datatype_files> - for elem in datatype_files.findall( 'datatype_file' ): - datatype_file_name = elem.get( 'name', None ) - if datatype_file_name: - # Find the file in the installed repository. - for root, dirs, files in os.walk( repo_files_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == datatype_file_name: - relative_path_to_datatype_file_name = os.path.join( root, name ) - break - break - if relative_path_to_datatype_file_name: - relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) - registration = datatypes_config_root.find( 'registration' ) - # Get the module by parsing the <datatype> tag. - for elem in registration.findall( 'datatype' ): - # A 'type' attribute is currently required. The attribute - # should be something like: type="gmap:GmapDB". - dtype = elem.get( 'type', None ) - if dtype: - fields = dtype.split( ':' ) - datatype_module = fields[0] - datatype_class_name = fields[1] - # Since we currently support only a single datatype_file, - # we have what we need. - break - try: - sys.path.insert( 0, relative_head ) - imported_module = __import__( datatype_module ) - sys.path.pop( 0 ) - except Exception, e: - log.debug( "Exception importing datatypes code file included in installed repository: %s" % str( e ) ) - trans.app.datatypes_registry.load_datatypes( root_dir=trans.app.config.root, config=datatypes_config, imported_module=imported_module ) - def __get_repository_tools_and_sample_files( self, trans, tool_path, repo_files_dir ): - # The sample_files list contains all files whose name ends in .sample - sample_files = [] - # Find all special .sample files first. - for root, dirs, files in os.walk( repo_files_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name.endswith( '.sample' ): - sample_files.append( os.path.abspath( os.path.join( root, name ) ) ) - # The repository_tools_tups list contains tuples of ( relative_path_to_tool_config, tool ) pairs - repository_tools_tups = [] - for root, dirs, files in os.walk( repo_files_dir ): - if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: - if '.hg' in dirs: - dirs.remove( '.hg' ) - if 'hgrc' in files: - files.remove( 'hgrc' ) - for name in files: - # Find all tool configs. - if name.endswith( '.xml' ): - relative_path = os.path.join( root, name ) - full_path = os.path.abspath( os.path.join( root, name ) ) - try: - repository_tool = trans.app.toolbox.load_tool( full_path ) - if repository_tool: - # At this point, we need to lstrip tool_path from relative_path. - tup_path = relative_path.replace( tool_path, '' ).lstrip( '/' ) - repository_tools_tups.append( ( tup_path, repository_tool ) ) - except Exception, e: - # We have an invalid .xml file, so not a tool config. - log.debug( "Ignoring invalid tool config (%s). Error: %s" % ( str( relative_path ), str( e ) ) ) - return sample_files, repository_tools_tups - def __create_or_undelete_tool_shed_repository( self, trans, name, description, changeset_revision, repository_clone_url ): - tmp_url = self.__clean_repository_clone_url( repository_clone_url ) - tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) - owner = self.__get_repository_owner( tmp_url ) - flush_needed = False - tool_shed_repository = get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed, name, owner, changeset_revision ) - if tool_shed_repository: - if tool_shed_repository.deleted: - tool_shed_repository.deleted = False - flush_needed = True - else: - tool_shed_repository = trans.model.ToolShedRepository( tool_shed=tool_shed, - name=name, - description=description, - owner=owner, - changeset_revision=changeset_revision ) - flush_needed = True - if flush_needed: - trans.sa_session.add( tool_shed_repository ) - trans.sa_session.flush() - def __add_shed_tool_conf_entry( self, trans, shed_tool_conf, new_tool_section ): - # Add an entry in the shed_tool_conf file. An entry looks something like: - # <section name="Filter and Sort" id="filter"> - # <tool file="filter/filtering.xml" guid="toolshed.g2.bx.psu.edu/repos/test/filter/1.0.2"/> - # </section> - # Make a backup of the hgweb.config file since we're going to be changing it. - if not os.path.exists( shed_tool_conf ): - output = open( shed_tool_conf, 'w' ) - output.write( '<?xml version="1.0"?>\n' ) - output.write( '<toolbox tool_path="%s">\n' % tool_path ) - output.write( '</toolbox>\n' ) - output.close() - self.__make_shed_tool_conf_copy( trans, shed_tool_conf ) - tmp_fd, tmp_fname = tempfile.mkstemp() - new_shed_tool_conf = open( tmp_fname, 'wb' ) - for i, line in enumerate( open( shed_tool_conf ) ): - if line.startswith( '</toolbox>' ): - # We're at the end of the original config file, so add our entry. - new_shed_tool_conf.write( new_tool_section ) - new_shed_tool_conf.write( line ) - else: - new_shed_tool_conf.write( line ) - new_shed_tool_conf.close() - shutil.move( tmp_fname, os.path.abspath( shed_tool_conf ) ) - def __make_shed_tool_conf_copy( self, trans, shed_tool_conf ): - # Make a backup of the shed_tool_conf file. - today = date.today() - backup_date = today.strftime( "%Y_%m_%d" ) - shed_tool_conf_copy = '%s/%s_%s_backup' % ( trans.app.config.root, shed_tool_conf, backup_date ) - shutil.copy( os.path.abspath( shed_tool_conf ), os.path.abspath( shed_tool_conf_copy ) ) - def __clean_tool_shed_url( self, tool_shed_url ): - if tool_shed_url.find( ':' ) > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - return tool_shed_url.split( ':' )[ 0 ] - return tool_shed_url.rstrip( '/' ) - def __clean_repository_clone_url( self, repository_clone_url ): - if repository_clone_url.find( '@' ) > 0: - # We have an url that includes an authenticated user, something like: - # http://test@bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split( '@' ) - tmp_url = items[ 1 ] - elif repository_clone_url.find( '//' ) > 0: - # We have an url that includes only a protocol, something like: - # http://bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split( '//' ) - tmp_url = items[ 1 ] - else: - tmp_url = repository_clone_url - return tmp_url - def __get_repository_owner( self, cleaned_repository_url ): - items = cleaned_repository_url.split( 'repos' ) - repo_path = items[ 1 ] - return repo_path.lstrip( '/' ).split( '/' )[ 0 ] - def __generate_tool_path( self, repository_clone_url, changeset_revision ): - """ - Generate a tool path that guarantees repositories with the same name will always be installed - in different directories. The tool path will be of the form: - <tool shed url>/repos/<repository owner>/<repository name>/<changeset revision> - http://test@bx.psu.edu:9009/repos/test/filter - """ - tmp_url = self.__clean_repository_clone_url( repository_clone_url ) - # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column - items = tmp_url.split( 'repos' ) - tool_shed_url = items[ 0 ] - repo_path = items[ 1 ] - tool_shed_url = self.__clean_tool_shed_url( tool_shed_url ) - return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) - def __generate_tool_guid( self, repository_clone_url, tool ): - """ - Generate a guid for the installed tool. It is critical that this guid matches the guid for - the tool in the Galaxy tool shed from which it is being installed. The form of the guid is - <tool shed host>/repos/<repository owner>/<repository name>/<tool id>/<tool version> - """ - tmp_url = self.__clean_repository_clone_url( repository_clone_url ) - return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) - def __generate_tool_panel_section( self, repository_name, repository_clone_url, changeset_revision, tool_section, repository_tools_tups ): - """ - Write an in-memory tool panel section so we can load it into the tool panel and then - append it to the appropriate shed tool config. - TODO: re-write using ElementTree. - """ - tmp_url = self.__clean_repository_clone_url( repository_clone_url ) - section_str = '' - section_str += ' <section name="%s" id="%s">\n' % ( tool_section.name, tool_section.id ) - for repository_tool_tup in repository_tools_tups: - tool_file_path, tool = repository_tool_tup - guid = self.__generate_tool_guid( repository_clone_url, tool ) - section_str += ' <tool file="%s" guid="%s">\n' % ( tool_file_path, guid ) - section_str += ' <tool_shed>%s</tool_shed>\n' % tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) - section_str += ' <repository_name>%s</repository_name>\n' % repository_name - section_str += ' <repository_owner>%s</repository_owner>\n' % self.__get_repository_owner( tmp_url ) - section_str += ' <changeset_revision>%s</changeset_revision>\n' % changeset_revision - section_str += ' <id>%s</id>\n' % tool.id - section_str += ' <version>%s</version>\n' % tool.version - section_str += ' </tool>\n' - section_str += ' </section>\n' - return section_str - -## ---- Utility methods ------------------------------------------------------- - -def build_shed_tool_conf_select_field( trans ): - """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs.""" - options = [] - for shed_tool_conf_filename, tool_path in trans.app.toolbox.shed_tool_confs.items(): - options.append( ( shed_tool_conf_filename.lstrip( './' ), shed_tool_conf_filename ) ) - select_field = SelectField( name='shed_tool_conf' ) - for option_tup in options: - select_field.add_option( option_tup[0], option_tup[1] ) - return select_field -def build_tool_panel_section_select_field( trans ): - """Build a SelectField whose options are the sections of the current in-memory toolbox.""" - options = [] - for k, tool_section in trans.app.toolbox.tool_panel.items(): - options.append( ( tool_section.name, tool_section.id ) ) - select_field = SelectField( name='tool_panel_section', display='radio' ) - for option_tup in options: - select_field.add_option( option_tup[0], option_tup[1] ) - return select_field -def get_repository( trans, id ): - """Get a tool_shed_repository from the database via id""" - return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) -def get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision ): - """Get a repository from the database via name owner and changeset_revision""" - return trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( and_( trans.model.ToolShedRepository.table.c.name == name, - trans.model.ToolShedRepository.table.c.owner == owner, - trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ - .first() -def get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed, name, owner, changeset_revision ): - if tool_shed.find( '//' ) > 0: - tool_shed = tool_shed.split( '//' )[1] - return trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( and_( trans.model.ToolShedRepository.table.c.tool_shed == tool_shed, - trans.model.ToolShedRepository.table.c.name == name, - trans.model.ToolShedRepository.table.c.owner == owner, - trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ - .first() -def get_url_from_repository_tool_shed( trans, repository ): - # The stored value of repository.tool_shed is something like: - # toolshed.g2.bx.psu.edu - # We need the URL to this tool shed, which is something like: - # http://toolshed.g2.bx.psu.edu/ - for shed_name, shed_url in trans.app.tool_shed_registry.tool_sheds.items(): - if shed_url.find( repository.tool_shed ) >= 0: - if shed_url.endswith( '/' ): - shed_url = shed_url.rstrip( '/' ) - return shed_url - # The tool shed from which the repository was originally - # installed must no longer be configured in tool_sheds_conf.xml. - return None - \ No newline at end of file + \ No newline at end of file diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 lib/galaxy/web/controllers/admin_toolshed.py --- /dev/null +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -0,0 +1,889 @@ +from galaxy.web.controllers.admin import * +import logging + +log = logging.getLogger( __name__ ) + +class RepositoryListGrid( grids.Grid ): + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + if tool_shed_repository.update_available: + return '<div class="count-box state-color-error">%s</div>' % tool_shed_repository.name + return tool_shed_repository.name + class DescriptionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.description + class OwnerColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.owner + class RevisionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.changeset_revision + class ToolShedColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.tool_shed + # Grid definition + title = "Installed tool shed repositories" + model_class = model.ToolShedRepository + template='/admin/tool_shed_repository/grid.mako' + default_sort_key = "name" + columns = [ + NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="manage_repository", id=item.id, webapp="galaxy" ) ), + attach_popup=True ), + DescriptionColumn( "Description" ), + OwnerColumn( "Owner" ), + RevisionColumn( "Revision" ), + ToolShedColumn( "Tool shed" ), + # Columns that are valid for filtering but are not visible. + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, + filterable="advanced" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ grids.GridOperation( "Get updates", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) ] + standard_filters = [] + default_filter = dict( deleted="False" ) + num_rows_per_page = 50 + preserve_state = False + use_paging = True + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( self.model_class ) \ + .filter( self.model_class.table.c.deleted == False ) + +class AdminToolshed( AdminGalaxy ): + + repository_list_grid = RepositoryListGrid() + + @web.expose + @web.require_admin + def browse_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + repository = get_repository( trans, kwd[ 'id' ] ) + return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako', + repository=repository, + message=message, + status=status ) + @web.expose + @web.require_admin + def browse_repositories( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd.pop( 'operation' ).lower() + if operation == "manage_repository": + return self.manage_repository( trans, **kwd ) + if operation == "get updates": + return self.check_for_updates( trans, **kwd ) + kwd[ 'message' ] = 'Names of repositories for which updates are available are highlighted in red.' + return self.repository_list_grid( trans, **kwd ) + @web.expose + @web.require_admin + def browse_tool_sheds( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + return trans.fill_template( '/webapps/galaxy/admin/tool_sheds.mako', + webapp='galaxy', + message=message, + status='error' ) + @web.expose + @web.require_admin + def find_tools_in_tool_shed( self, trans, **kwd ): + tool_shed_url = kwd[ 'tool_shed_url' ] + galaxy_url = url_for( '', qualified=True ) + url = '%s/repository/find_tools?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def find_workflows_in_tool_shed( self, trans, **kwd ): + tool_shed_url = kwd[ 'tool_shed_url' ] + galaxy_url = url_for( '', qualified=True ) + url = '%s/repository/find_workflows?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def browse_tool_shed( self, trans, **kwd ): + tool_shed_url = kwd[ 'tool_shed_url' ] + galaxy_url = url_for( '', qualified=True ) + url = '%s/repository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def install_repository( self, trans, **kwd ): + if not trans.app.toolbox.shed_tool_confs: + message = 'The <b>tool_config_file</b> setting in <b>universe_wsgi.ini</b> must include at least one shed tool configuration file name with a ' + message += '<b><toolbox></b> tag that includes a <b>tool_path</b> attribute value which is a directory relative to the Galaxy installation ' + message += 'directory in order to automatically install tools from a Galaxy tool shed (e.g., the file name <b>shed_tool_conf.xml</b> whose ' + message += '<b><toolbox></b> tag is <b><toolbox tool_path="../shed_tools"></b>).<p/>See the ' + message += '<a href="http://wiki.g2.bx.psu.edu/Tool%20Shed#Automatic_installation_of_Galaxy_tool_..." ' + message += 'target="_blank">Automatic installation of Galaxy tool shed repository tools into a local Galaxy instance</a> section of the ' + message += '<a href="http://wiki.g2.bx.psu.edu/Tool%20Shed" target="_blank">Galaxy tool shed wiki</a> for all of the details.' + return trans.show_error_message( message ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + tool_shed_url = kwd[ 'tool_shed_url' ] + repo_info_dict = kwd[ 'repo_info_dict' ] + new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) + tool_panel_section = kwd.get( 'tool_panel_section', '' ) + if kwd.get( 'select_tool_panel_section_button', False ): + shed_tool_conf = kwd[ 'shed_tool_conf' ] + # Get the tool path. + for k, tool_path in trans.app.toolbox.shed_tool_confs.items(): + if k == shed_tool_conf: + break + if new_tool_panel_section or tool_panel_section: + if new_tool_panel_section: + section_id = new_tool_panel_section.lower().replace( ' ', '_' ) + new_section_key = 'section_%s' % str( section_id ) + if new_section_key in trans.app.toolbox.tool_panel: + # Appending a tool to an existing section in trans.app.toolbox.tool_panel + log.debug( "Appending to tool panel section: %s" % new_tool_panel_section ) + tool_section = trans.app.toolbox.tool_panel[ new_section_key ] + else: + # Appending a new section to trans.app.toolbox.tool_panel + log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) + elem = Element( 'section' ) + elem.attrib[ 'name' ] = new_tool_panel_section + elem.attrib[ 'id' ] = section_id + tool_section = ToolSection( elem ) + trans.app.toolbox.tool_panel[ new_section_key ] = tool_section + else: + section_key = 'section_%s' % tool_panel_section + tool_section = trans.app.toolbox.tool_panel[ section_key ] + # Decode the encoded repo_info_dict param value. + repo_info_dict = tool_shed_decode( repo_info_dict ) + # Clone the repository to the configured location. + current_working_dir = os.getcwd() + installed_repository_names = [] + for name, repo_info_tuple in repo_info_dict.items(): + metadata_dict = None + description, repository_clone_url, changeset_revision = repo_info_tuple + clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) + if os.path.exists( clone_dir ): + # Repository and revision has already been cloned. + # TODO: implement the ability to re-install or revert an existing repository. + message += 'Revision <b>%s</b> of repository <b>%s</b> was previously installed.<br/>' % ( changeset_revision, name ) + else: + os.makedirs( clone_dir ) + log.debug( 'Cloning %s...' % repository_clone_url ) + cmd = 'hg clone %s' % repository_clone_url + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( clone_dir ) + proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + # Update the cloned repository to changeset_revision. It is imperative that the + # installed repository is updated to the desired changeset_revision before metadata + # is set because the process for setting metadata uses the repository files on disk. + relative_install_dir = os.path.join( clone_dir, name ) + log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision ) + cmd = 'hg update -r %s' % changeset_revision + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( relative_install_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + # Generate the metadata for the installed tool shed repository. It is imperative that + # the installed repository is updated to the desired changeset_revision before metadata + # is set because the process for setting metadata uses the repository files on disk. + metadata_dict = self.__generate_metadata( trans, relative_install_dir, repository_clone_url ) + if 'datatypes_config' in metadata_dict: + datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) + # Load data types required by tools. + self.__load_datatypes( trans, datatypes_config, relative_install_dir ) + if 'tools' in metadata_dict: + repository_tools_tups = [] + for tool_dict in metadata_dict[ 'tools' ]: + relative_path = tool_dict[ 'tool_config' ] + tool = trans.app.toolbox.load_tool( os.path.abspath( relative_path ) ) + repository_tools_tups.append( ( relative_path, tool ) ) + if repository_tools_tups: + sample_files = metadata_dict.get( 'sample_files', [] ) + # Handle missing data table entries for tool parameters that are dynamically generated select lists. + repository_tools_tups = self.__handle_missing_data_table_entry( trans, tool_path, sample_files, repository_tools_tups ) + # Handle missing index files for tool parameters that are dynamically generated select lists. + repository_tools_tups = self.__handle_missing_index_file( trans, tool_path, sample_files, repository_tools_tups ) + # Handle tools that use fabric scripts to install dependencies. + self.__handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) + # Generate an in-memory tool conf section that includes the new tools. + new_tool_section = self.__generate_tool_panel_section( name, + repository_clone_url, + changeset_revision, + tool_section, + repository_tools_tups ) + # Create a temporary file to persist the in-memory tool section + # TODO: Figure out how to do this in-memory using xml.etree. + tmp_name = tempfile.NamedTemporaryFile().name + persisted_new_tool_section = open( tmp_name, 'wb' ) + persisted_new_tool_section.write( new_tool_section ) + persisted_new_tool_section.close() + # Parse the persisted tool panel section + tree = parse_xml( tmp_name ) + root = tree.getroot() + # Load the tools in the section into the tool panel. + trans.app.toolbox.load_section_tag_set( root, trans.app.toolbox.tool_panel, tool_path ) + # Remove the temporary file + try: + os.unlink( tmp_name ) + except: + pass + # Append the new section to the shed_tool_config file. + self.__add_shed_tool_conf_entry( trans, shed_tool_conf, new_tool_section ) + if trans.app.toolbox_search.enabled: + # If search support for tools is enabled, index the new installed tools. + trans.app.toolbox_search = ToolBoxSearch( trans.app.toolbox ) + # Add a new record to the tool_shed_repository table if one doesn't + # already exist. If one exists but is marked deleted, undelete it. + self.__create_or_undelete_tool_shed_repository( trans, + name, + description, + changeset_revision, + repository_clone_url, + metadata_dict ) + installed_repository_names.append( name ) + else: + tmp_stderr = open( tmp_name, 'rb' ) + message += '%s<br/>' % tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + tmp_stderr = open( tmp_name, 'rb' ) + message += '%s<br/>' % tmp_stderr.read() + tmp_stderr.close() + status = 'error' + if installed_repository_names: + installed_repository_names.sort() + num_repositories_installed = len( installed_repository_names ) + message += 'Installed %d %s and all tools were loaded into tool panel section <b>%s</b>:<br/>Installed repositories: ' % \ + ( num_repositories_installed, inflector.cond_plural( num_repositories_installed, 'repository' ), tool_section.name ) + for i, repo_name in enumerate( installed_repository_names ): + if i == len( installed_repository_names ) -1: + message += '%s.<br/>' % repo_name + else: + message += '%s, ' % repo_name + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) + else: + message = 'Choose the section in your tool panel to contain the installed tools.' + status = 'error' + if len( trans.app.toolbox.shed_tool_confs.keys() ) > 1: + shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) + shed_tool_conf = None + else: + shed_tool_conf = trans.app.toolbox.shed_tool_confs.keys()[0].lstrip( './' ) + shed_tool_conf_select_field = None + tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) + return trans.fill_template( '/admin/tool_shed_repository/select_tool_panel_section.mako', + tool_shed_url=tool_shed_url, + repo_info_dict=repo_info_dict, + shed_tool_conf=shed_tool_conf, + shed_tool_conf_select_field=shed_tool_conf_select_field, + tool_panel_section_select_field=tool_panel_section_select_field, + new_tool_panel_section=new_tool_panel_section, + message=message, + status=status ) + @web.expose + @web.require_admin + def manage_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + repository = get_repository( trans, kwd[ 'id' ] ) + description = util.restore_text( params.get( 'description', repository.description ) ) + relative_install_dir = self.__get_relative_install_dir( trans, repository ) + repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) + if params.get( 'edit_repository_button', False ): + if description != repository.description: + repository.description = description + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "The repository information has been updated." + elif params.get( 'set_metadata_button', False ): + repository_clone_url = self.__generate_clone_url( trans, repository ) + metadata_dict = self.__generate_metadata( trans, relative_install_dir, repository_clone_url ) + if metadata_dict: + repository.metadata = metadata_dict + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "Repository metadata has been reset." + return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', + repository=repository, + description=description, + repo_files_dir=repo_files_dir, + message=message, + status=status ) + @web.expose + @web.require_admin + def check_for_updates( self, trans, **kwd ): + # Send a request to the relevant tool shed to see if there are any updates. + repository = get_repository( trans, kwd[ 'id' ] ) + tool_shed_url = get_url_from_repository_tool_shed( trans, repository ) + url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( tool_shed_url, url_for( '', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def update_to_changeset_revision( self, trans, **kwd ): + """Update a cloned repository to the latest revision possible.""" + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + tool_shed_url = kwd[ 'tool_shed_url' ] + name = params.get( 'name', None ) + owner = params.get( 'owner', None ) + changeset_revision = params.get( 'changeset_revision', None ) + latest_changeset_revision = params.get( 'latest_changeset_revision', None ) + repository = get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed_url, name, owner, changeset_revision ) + if changeset_revision and latest_changeset_revision: + if changeset_revision == latest_changeset_revision: + message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name + else: + current_working_dir = os.getcwd() + relative_install_dir = self.__get_relative_install_dir( trans, repository ) + if relative_install_dir: + # Update the cloned repository to changeset_revision. + repo_files_dir = os.path.join( relative_install_dir, name ) + log.debug( "Updating cloned repository named '%s' from revision '%s' to revision '%s'..." % \ + ( name, changeset_revision, latest_changeset_revision ) ) + cmd = 'hg pull' + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + cmd = 'hg update -r %s' % latest_changeset_revision + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + # Update the repository changeset_revision in the database. + repository.changeset_revision = latest_changeset_revision + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \ + ( name, latest_changeset_revision ) + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + message = "The directory containing the cloned repository named '%s' cannot be found." % name + status = 'error' + else: + message = "The latest changeset revision could not be retrieved for the repository named '%s'." % name + status = 'error' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='manage_repository', + id=trans.security.encode_id( repository.id ), + message=message, + status=status ) ) + @web.expose + @web.require_admin + def view_tool_metadata( self, trans, repository_id, tool_id, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + webapp = params.get( 'webapp', 'community' ) + repository = get_repository( trans, repository_id ) + metadata = {} + tool = None + if 'tools' in repository.metadata: + for tool_metadata_dict in repository.metadata[ 'tools' ]: + if tool_metadata_dict[ 'id' ] == tool_id: + metadata = tool_metadata_dict + tool = trans.app.toolbox.load_tool( os.path.abspath( metadata[ 'tool_config' ] ) ) + break + return trans.fill_template( "/admin/tool_shed_repository/view_tool_metadata.mako", + repository=repository, + tool=tool, + metadata=metadata, + message=message, + status=status ) + def __generate_metadata( self, trans, relative_install_dir, repository_clone_url ): + """ + Browse the repository files on disk to generate metadata. Since we are using disk files, it + is imperative that the repository is updated to the desired change set revision before metadata + is generated. + """ + metadata_dict = {} + sample_files = [] + datatypes_config = None + # Find datatypes_conf.xml if it exists. + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == 'datatypes_conf.xml': + relative_path = os.path.join( root, name ) + datatypes_config = os.path.abspath( relative_path ) + break + if datatypes_config: + metadata_dict[ 'datatypes_config' ] = relative_path + metadata_dict = self.__generate_datatypes_metadata( trans, datatypes_config, metadata_dict ) + # Find all special .sample files. + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name.endswith( '.sample' ): + sample_files.append( os.path.join( root, name ) ) + if sample_files: + metadata_dict[ 'sample_files' ] = sample_files + # Find all tool configs and exported workflows. + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: + if '.hg' in dirs: + dirs.remove( '.hg' ) + for name in files: + # Find all tool configs. + if name != 'datatypes_conf.xml' and name.endswith( '.xml' ): + full_path = os.path.abspath( os.path.join( root, name ) ) + try: + tool = trans.app.toolbox.load_tool( full_path ) + except Exception, e: + tool = None + if tool is not None: + tool_config = os.path.join( root, name ) + metadata_dict = self.__generate_tool_metadata( trans, tool_config, tool, repository_clone_url, metadata_dict ) + # Find all exported workflows + elif name.endswith( '.ga' ): + relative_path = os.path.join( root, name ) + fp = open( relative_path, 'rb' ) + workflow_text = fp.read() + fp.close() + exported_workflow_dict = from_json_string( workflow_text ) + if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': + metadata_dict = self.__generate_workflow_metadata( trans, relative_path, exported_workflow_dict, metadata_dict ) + return metadata_dict + def __generate_datatypes_metadata( self, trans, datatypes_config, metadata_dict ): + """ + Update the received metadata_dict with changes that have been applied + to the received datatypes_config. + """ + # Parse datatypes_config. + tree = ElementTree.parse( datatypes_config ) + root = tree.getroot() + ElementInclude.include( root ) + repository_datatype_code_files = [] + datatype_files = root.find( 'datatype_files' ) + if datatype_files: + for elem in datatype_files.findall( 'datatype_file' ): + name = elem.get( 'name', None ) + repository_datatype_code_files.append( name ) + metadata_dict[ 'datatype_files' ] = repository_datatype_code_files + datatypes = [] + registration = root.find( 'registration' ) + if registration: + for elem in registration.findall( 'datatype' ): + extension = elem.get( 'extension', None ) + dtype = elem.get( 'type', None ) + mimetype = elem.get( 'mimetype', None ) + datatypes.append( dict( extension=extension, + dtype=dtype, + mimetype=mimetype ) ) + metadata_dict[ 'datatypes' ] = datatypes + return metadata_dict + def __generate_tool_metadata( self, trans, tool_config, tool, repository_clone_url, metadata_dict ): + """ + Update the received metadata_dict with changes that have been + applied to the received tool. + """ + # Generate the guid + guid = self.__generate_tool_guid( repository_clone_url, tool ) + # Handle tool.requirements. + tool_requirements = [] + for tr in tool.requirements: + name=tr.name + type=tr.type + if type == 'fabfile': + version = None + fabfile = tr.fabfile + method = tr.method + else: + version = tr.version + fabfile = None + method = None + requirement_dict = dict( name=name, + type=type, + version=version, + fabfile=fabfile, + method=method ) + tool_requirements.append( requirement_dict ) + # Handle tool.tests. + tool_tests = [] + if tool.tests: + for ttb in tool.tests: + test_dict = dict( name=ttb.name, + required_files=ttb.required_files, + inputs=ttb.inputs, + outputs=ttb.outputs ) + tool_tests.append( test_dict ) + tool_dict = dict( id=tool.id, + guid=guid, + name=tool.name, + version=tool.version, + description=tool.description, + version_string_cmd = tool.version_string_cmd, + tool_config=tool_config, + requirements=tool_requirements, + tests=tool_tests ) + if 'tools' in metadata_dict: + metadata_dict[ 'tools' ].append( tool_dict ) + else: + metadata_dict[ 'tools' ] = [ tool_dict ] + return metadata_dict + def __generate_workflow_metadata( self, trans, relative_path, exported_workflow_dict, metadata_dict ): + """ + Update the received metadata_dict with changes that have been applied + to the received exported_workflow_dict. Store everything in the database. + """ + if 'workflows' in metadata_dict: + metadata_dict[ 'workflows' ].append( ( relative_path, exported_workflow_dict ) ) + else: + metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ] + return metadata_dict + def __get_relative_install_dir( self, trans, repository ): + # Get the directory where the repository is install. + tool_shed = self.__clean_tool_shed_url( repository.tool_shed ) + partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.changeset_revision ) + # Get the relative tool installation paths from each of the shed tool configs. + shed_tool_confs = trans.app.toolbox.shed_tool_confs + relative_install_dir = None + # The shed_tool_confs dictionary contains { shed_conf_filename : tool_path } pairs. + for shed_conf_filename, tool_path in shed_tool_confs.items(): + relative_install_dir = os.path.join( tool_path, partial_install_dir ) + if os.path.isdir( relative_install_dir ): + break + return relative_install_dir + def __handle_missing_data_table_entry( self, trans, tool_path, sample_files, repository_tools_tups ): + # Inspect each tool to see if any have input parameters that are dynamically + # generated select lists that require entries in the tool_data_table_conf.xml file. + missing_data_table_entry = False + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + if repository_tool.params_with_missing_data_table_entry: + missing_data_table_entry = True + break + if missing_data_table_entry: + # The repository must contain a tool_data_table_conf.xml.sample file that includes + # all required entries for all tools in the repository. + for sample_file in sample_files: + head, tail = os.path.split( sample_file ) + if tail == 'tool_data_table_conf.xml.sample': + break + error, correction_msg = handle_sample_tool_data_table_conf_file( trans, sample_file ) + if error: + # TODO: Do more here than logging an exception. + log.debug( exception_msg ) + # Reload the tool into the local list of repository_tools_tups. + repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) + repository_tools_tups[ index ] = ( tup_path, repository_tool ) + return repository_tools_tups + def __handle_missing_index_file( self, trans, tool_path, sample_files, repository_tools_tups ): + # Inspect each tool to see if it has any input parameters that + # are dynamically generated select lists that depend on a .loc file. + missing_files_handled = [] + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + params_with_missing_index_file = repository_tool.params_with_missing_index_file + for param in params_with_missing_index_file: + options = param.options + missing_head, missing_tail = os.path.split( options.missing_index_file ) + if missing_tail not in missing_files_handled: + # The repository must contain the required xxx.loc.sample file. + for sample_file in sample_files: + sample_head, sample_tail = os.path.split( sample_file ) + if sample_tail == '%s.sample' % missing_tail: + copy_sample_loc_file( trans, sample_file ) + if options.tool_data_table and options.tool_data_table.missing_index_file: + options.tool_data_table.handle_found_index_file( options.missing_index_file ) + missing_files_handled.append( missing_tail ) + break + # Reload the tool into the local list of repository_tools_tups. + repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) + repository_tools_tups[ index ] = ( tup_path, repository_tool ) + return repository_tools_tups + def __handle_tool_dependencies( self, current_working_dir, repo_files_dir, repository_tools_tups ): + # Inspect each tool to see if it includes a "requirement" that refers to a fabric + # script. For those that do, execute the fabric script to install tool dependencies. + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + for requirement in repository_tool.requirements: + if requirement.type == 'fabfile': + log.debug( 'Executing fabric script to install dependencies for tool "%s"...' % repository_tool.name ) + fabfile = requirement.fabfile + method = requirement.method + # Find the relative path to the fabfile. + relative_fabfile_path = None + for root, dirs, files in os.walk( repo_files_dir ): + for name in files: + if name == fabfile: + relative_fabfile_path = os.path.join( root, name ) + break + if relative_fabfile_path: + # cmd will look something like: fab -f fabfile.py install_bowtie + cmd = 'fab -f %s %s' % ( relative_fabfile_path, method ) + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode != 0: + # TODO: do something more here than logging the problem. + tmp_stderr = open( tmp_name, 'rb' ) + error = tmp_stderr.read() + tmp_stderr.close() + log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) + def __load_datatypes( self, trans, datatypes_config, relative_intall_dir ): + imported_module = None + # Parse datatypes_config. + tree = parse_xml( datatypes_config ) + datatypes_config_root = tree.getroot() + relative_path_to_datatype_file_name = None + datatype_files = datatypes_config_root.find( 'datatype_files' ) + # Currently only a single datatype_file is supported. For example: + # <datatype_files> + # <datatype_file name="gmap.py"/> + # </datatype_files> + for elem in datatype_files.findall( 'datatype_file' ): + datatype_file_name = elem.get( 'name', None ) + if datatype_file_name: + # Find the file in the installed repository. + for root, dirs, files in os.walk( relative_intall_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == datatype_file_name: + relative_path_to_datatype_file_name = os.path.join( root, name ) + break + break + if relative_path_to_datatype_file_name: + relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) + registration = datatypes_config_root.find( 'registration' ) + # Get the module by parsing the <datatype> tag. + for elem in registration.findall( 'datatype' ): + # A 'type' attribute is currently required. The attribute + # should be something like: type="gmap:GmapDB". + dtype = elem.get( 'type', None ) + if dtype: + fields = dtype.split( ':' ) + datatype_module = fields[0] + datatype_class_name = fields[1] + # Since we currently support only a single datatype_file, + # we have what we need. + break + try: + sys.path.insert( 0, relative_head ) + imported_module = __import__( datatype_module ) + sys.path.pop( 0 ) + except Exception, e: + log.debug( "Exception importing datatypes code file included in installed repository: %s" % str( e ) ) + trans.app.datatypes_registry.load_datatypes( root_dir=trans.app.config.root, config=datatypes_config, imported_module=imported_module ) + def __create_or_undelete_tool_shed_repository( self, trans, name, description, changeset_revision, repository_clone_url, metadata_dict ): + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) + owner = self.__get_repository_owner( tmp_url ) + includes_datatypes = 'datatypes_config' in metadata_dict + flush_needed = False + tool_shed_repository = get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed, name, owner, changeset_revision ) + if tool_shed_repository: + if tool_shed_repository.deleted: + tool_shed_repository.deleted = False + # Reset includes_datatypes in case metadata changed since last installed. + tool_shed_repository.includes_datatypes = includes_datatypes + flush_needed = True + else: + tool_shed_repository = trans.model.ToolShedRepository( tool_shed=tool_shed, + name=name, + description=description, + owner=owner, + changeset_revision=changeset_revision, + metadata=metadata_dict, + includes_datatypes=includes_datatypes ) + flush_needed = True + if flush_needed: + trans.sa_session.add( tool_shed_repository ) + trans.sa_session.flush() + def __add_shed_tool_conf_entry( self, trans, shed_tool_conf, new_tool_section ): + # Add an entry in the shed_tool_conf file. An entry looks something like: + # <section name="Filter and Sort" id="filter"> + # <tool file="filter/filtering.xml" guid="toolshed.g2.bx.psu.edu/repos/test/filter/1.0.2"/> + # </section> + # Make a backup of the hgweb.config file since we're going to be changing it. + if not os.path.exists( shed_tool_conf ): + output = open( shed_tool_conf, 'w' ) + output.write( '<?xml version="1.0"?>\n' ) + output.write( '<toolbox tool_path="%s">\n' % tool_path ) + output.write( '</toolbox>\n' ) + output.close() + self.__make_shed_tool_conf_copy( trans, shed_tool_conf ) + tmp_fd, tmp_fname = tempfile.mkstemp() + new_shed_tool_conf = open( tmp_fname, 'wb' ) + for i, line in enumerate( open( shed_tool_conf ) ): + if line.startswith( '</toolbox>' ): + # We're at the end of the original config file, so add our entry. + new_shed_tool_conf.write( new_tool_section ) + new_shed_tool_conf.write( line ) + else: + new_shed_tool_conf.write( line ) + new_shed_tool_conf.close() + shutil.move( tmp_fname, os.path.abspath( shed_tool_conf ) ) + def __make_shed_tool_conf_copy( self, trans, shed_tool_conf ): + # Make a backup of the shed_tool_conf file. + today = date.today() + backup_date = today.strftime( "%Y_%m_%d" ) + shed_tool_conf_copy = '%s/%s_%s_backup' % ( trans.app.config.root, shed_tool_conf, backup_date ) + shutil.copy( os.path.abspath( shed_tool_conf ), os.path.abspath( shed_tool_conf_copy ) ) + def __clean_tool_shed_url( self, tool_shed_url ): + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + return tool_shed_url.split( ':' )[ 0 ] + return tool_shed_url.rstrip( '/' ) + def __clean_repository_clone_url( self, repository_clone_url ): + if repository_clone_url.find( '@' ) > 0: + # We have an url that includes an authenticated user, something like: + # http://test@bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split( '@' ) + tmp_url = items[ 1 ] + elif repository_clone_url.find( '//' ) > 0: + # We have an url that includes only a protocol, something like: + # http://bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split( '//' ) + tmp_url = items[ 1 ] + else: + tmp_url = repository_clone_url + return tmp_url + def __get_repository_owner( self, cleaned_repository_url ): + items = cleaned_repository_url.split( 'repos' ) + repo_path = items[ 1 ] + return repo_path.lstrip( '/' ).split( '/' )[ 0 ] + def __generate_tool_path( self, repository_clone_url, changeset_revision ): + """ + Generate a tool path that guarantees repositories with the same name will always be installed + in different directories. The tool path will be of the form: + <tool shed url>/repos/<repository owner>/<repository name>/<changeset revision> + http://test@bx.psu.edu:9009/repos/test/filter + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column + items = tmp_url.split( 'repos' ) + tool_shed_url = items[ 0 ] + repo_path = items[ 1 ] + tool_shed_url = self.__clean_tool_shed_url( tool_shed_url ) + return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) + def __generate_clone_url( self, trans, repository ): + """Generate the URL for cloning a repository.""" + tool_shed_url = get_url_from_repository_tool_shed( trans, repository ) + return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) + def __generate_tool_guid( self, repository_clone_url, tool ): + """ + Generate a guid for the installed tool. It is critical that this guid matches the guid for + the tool in the Galaxy tool shed from which it is being installed. The form of the guid is + <tool shed host>/repos/<repository owner>/<repository name>/<tool id>/<tool version> + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) + def __generate_tool_panel_section( self, repository_name, repository_clone_url, changeset_revision, tool_section, repository_tools_tups ): + """ + Write an in-memory tool panel section so we can load it into the tool panel and then + append it to the appropriate shed tool config. + TODO: re-write using ElementTree. + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + section_str = '' + section_str += ' <section name="%s" id="%s">\n' % ( tool_section.name, tool_section.id ) + for repository_tool_tup in repository_tools_tups: + tool_file_path, tool = repository_tool_tup + guid = self.__generate_tool_guid( repository_clone_url, tool ) + section_str += ' <tool file="%s" guid="%s">\n' % ( tool_file_path, guid ) + section_str += ' <tool_shed>%s</tool_shed>\n' % tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) + section_str += ' <repository_name>%s</repository_name>\n' % repository_name + section_str += ' <repository_owner>%s</repository_owner>\n' % self.__get_repository_owner( tmp_url ) + section_str += ' <changeset_revision>%s</changeset_revision>\n' % changeset_revision + section_str += ' <id>%s</id>\n' % tool.id + section_str += ' <version>%s</version>\n' % tool.version + section_str += ' </tool>\n' + section_str += ' </section>\n' + return section_str + +## ---- Utility methods ------------------------------------------------------- + +def build_shed_tool_conf_select_field( trans ): + """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs.""" + options = [] + for shed_tool_conf_filename, tool_path in trans.app.toolbox.shed_tool_confs.items(): + options.append( ( shed_tool_conf_filename.lstrip( './' ), shed_tool_conf_filename ) ) + select_field = SelectField( name='shed_tool_conf' ) + for option_tup in options: + select_field.add_option( option_tup[0], option_tup[1] ) + return select_field +def build_tool_panel_section_select_field( trans ): + """Build a SelectField whose options are the sections of the current in-memory toolbox.""" + options = [] + for k, tool_section in trans.app.toolbox.tool_panel.items(): + options.append( ( tool_section.name, tool_section.id ) ) + select_field = SelectField( name='tool_panel_section', display='radio' ) + for option_tup in options: + select_field.add_option( option_tup[0], option_tup[1] ) + return select_field +def get_repository( trans, id ): + """Get a tool_shed_repository from the database via id""" + return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) +def get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision ): + """Get a repository from the database via name owner and changeset_revision""" + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( and_( trans.model.ToolShedRepository.table.c.name == name, + trans.model.ToolShedRepository.table.c.owner == owner, + trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + .first() +def get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed, name, owner, changeset_revision ): + if tool_shed.find( '//' ) > 0: + tool_shed = tool_shed.split( '//' )[1] + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( and_( trans.model.ToolShedRepository.table.c.tool_shed == tool_shed, + trans.model.ToolShedRepository.table.c.name == name, + trans.model.ToolShedRepository.table.c.owner == owner, + trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + .first() +def get_url_from_repository_tool_shed( trans, repository ): + # The stored value of repository.tool_shed is something like: + # toolshed.g2.bx.psu.edu + # We need the URL to this tool shed, which is something like: + # http://toolshed.g2.bx.psu.edu/ + for shed_name, shed_url in trans.app.tool_shed_registry.tool_sheds.items(): + if shed_url.find( repository.tool_shed ) >= 0: + if shed_url.endswith( '/' ): + shed_url = shed_url.rstrip( '/' ) + return shed_url + # The tool shed from which the repository was originally + # installed must no longer be configured in tool_sheds_conf.xml. + return None diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -1230,8 +1230,8 @@ elif installed_repository_file: # The workflow was read from a file included with an installed tool shed repository. message = "Workflow <b>%s</b> imported successfully." % workflow.name - return trans.response.send_redirect( web.url_for( controller='admin', - action='browse_tool_shed_repository', + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repository', id=repository_id, message=message, status=status ) ) diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -437,7 +437,7 @@ if operation == "install": galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) encoded_repo_info_dict = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) - url = '%s/admin/install_tool_shed_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s' % \ + url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s' % \ ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict ) return trans.response.send_redirect( url ) else: @@ -513,7 +513,7 @@ if operation == "install": galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) encoded_repo_info_dict = self.__encode_repo_info_dict( trans, webapp, util.listify( item_id ) ) - url = '%s/admin/install_tool_shed_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s' % \ + url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&webapp=%s&repo_info_dict=%s' % \ ( galaxy_url, url_for( '', qualified=True ), webapp, encoded_repo_info_dict ) return trans.response.send_redirect( url ) else: @@ -759,7 +759,7 @@ repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision ) encoded_repo_info_dict = encode( repo_info_dict ) # Redirect back to local Galaxy to perform install. - url = '%s/admin/install_tool_shed_repository?tool_shed_url=%s&repo_info_dict=%s' % \ + url = '%s/admin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s' % \ ( galaxy_url, url_for( '', qualified=True ), encoded_repo_info_dict ) return trans.response.send_redirect( url ) @web.expose @@ -773,7 +773,7 @@ changeset_revision = params.get( 'changeset_revision', None ) webapp = params.get( 'webapp', 'community' ) # Start building up the url to redirect back to the calling Galaxy instance. - url = '%s/admin/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '', qualified=True ) ) + url = '%s/admin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '', qualified=True ) ) repository = get_repository_by_name_and_owner( trans, name, owner ) url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ ( repository.name, repository.user.username, changeset_revision ) @@ -1035,7 +1035,7 @@ hgweb_config = "%s/hgweb.config" % trans.app.config.root if repository_path.startswith( './' ): repository_path = repository_path.replace( './', '', 1 ) - entry = "repos/%s/%s = %s" % ( repository.user.username, repository.name, repository_path.lstrip( './' ) ) + entry = "repos/%s/%s = %s" % ( repository.user.username, repository.name, repository_path ) tmp_fd, tmp_fname = tempfile.mkstemp() if os.path.exists( hgweb_config ): # Make a backup of the hgweb.config file since we're going to be changing it. diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/admin/select_tool_panel_section.mako --- a/templates/admin/select_tool_panel_section.mako +++ /dev/null @@ -1,61 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> - -%if message: - ${render_msg( message, status )} -%endif - -<div class="warningmessage"> - The core Galaxy development team does not maintain the contents of many Galaxy tool shed repositories. Some repository tools - may include code that produces malicious behavior, so be aware of what you are installing. - <p/> - If you discover a repository that causes problems after installation, contact <a href="http://wiki.g2.bx.psu.edu/Support" target="_blank">Galaxy support</a>, - sending all necessary information, and appropriate action will be taken. - <p/> - <a href="http://wiki.g2.bx.psu.edu/Tool%20Shed#Contacting_the_owner_of_a_repository" target="_blank">Contact the repository owner</a> for general questions - or concerns. -</div> -<br/> -<div class="warningmessage"> - Installation may take a while, depending upon the size of the repository contents. Wait until a message is displayed in your - browser after clicking the <b>Install</b> button below. -</div> -<br/> - -<div class="toolForm"> - <div class="toolFormTitle">Choose section to load tools into tool panel</div> - <div class="toolFormBody"> - <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin', action='install_tool_shed_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict )}" method="post" > - %if shed_tool_conf_select_field: - <div class="form-row"> - <label>Shed tool configuration file:</label> - ${shed_tool_conf_select_field.get_html()} - <div class="toolParamHelp" style="clear: both;"> - Your Galaxy instance is configured with ${len( shed_tool_conf_select_field.options )} shed tool configuration files, - so choose one in which to configure the installed tools. - </div> - </div> - <div style="clear: both"></div> - %else: - <input type="hidden" name="shed_tool_conf" value="${shed_tool_conf}"/> - %endif - <div class="form-row"> - <label>Add new tool panel section:</label> - <input name="new_tool_panel_section" type="textfield" value="${new_tool_panel_section}" size="40"/> - <div class="toolParamHelp" style="clear: both;"> - Add a new tool panel section or choose an existing section in your tool panel below to contain the installed tools. - </div> - </div> - <div class="form-row"> - <label>Select existing tool panel section:</label> - ${tool_panel_section_select_field.get_html()} - <div class="toolParamHelp" style="clear: both;"> - Choose an existing section in your tool panel to contain the installed tools. - </div> - </div> - <div class="form-row"> - <input type="submit" name="select_tool_panel_section_button" value="Install"/> - </div> - </form> - </div> -</div> diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/admin/tool_shed_repository/browse_repository.mako --- a/templates/admin/tool_shed_repository/browse_repository.mako +++ b/templates/admin/tool_shed_repository/browse_repository.mako @@ -1,5 +1,6 @@ <%inherit file="/base.mako"/><%namespace file="/message.mako" import="render_msg" /> +<%namespace file="/admin/tool_shed_repository/common.mako" import="*" /><% from galaxy.web.base.controller import tool_shed_encode, tool_shed_decode %> @@ -7,8 +8,8 @@ <ul class="manage-table-actions"><li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li><div popupmenu="repository-${repository.id}-popup"> - <a class="action-button" href="${h.url_for( controller='admin', action='manage_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a> - <a class="action-button" href="${h.url_for( controller='admin', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a></div></ul> @@ -16,112 +17,4 @@ ${render_msg( message, status )} %endif -<div class="toolForm"> - <div class="toolFormTitle">Installed tool shed repository '${repository.name}'</div> - <div class="toolFormBody"> - - %if tool_dicts: - <div class="form-row"> - <table width="100%"> - <tr bgcolor="#D8D8D8" width="100%"> - <td><b>Tools</b></td> - </tr> - </table> - </div> - <div class="form-row"> - <table class="grid"> - <tr> - <td><b>name</b></td> - <td><b>description</b></td> - <td><b>version</b></td> - <td><b>requirements</b></td> - </tr> - %for tool_dict in tool_dicts: - <tr> - <td>${tool_dict[ 'name' ]}</div> - </td> - <td>${tool_dict[ 'description' ]}</td> - <td>${tool_dict[ 'version' ]}</td> - <td> - <% - if 'requirements' in tool_dict: - requirements = tool_dict[ 'requirements' ] - else: - requirements = None - %> - %if requirements: - <% - requirements_str = '' - for requirement_dict in tool_dict[ 'requirements' ]: - requirements_str += '%s (%s), ' % ( requirement_dict[ 'name' ], requirement_dict[ 'type' ] ) - requirements_str = requirements_str.rstrip( ', ' ) - %> - ${requirements_str} - %else: - none - %endif - </td> - </tr> - %endfor - </table> - </div> - <div style="clear: both"></div> - %endif - %if workflow_dicts: - <div class="form-row"> - <table width="100%"> - <tr bgcolor="#D8D8D8" width="100%"> - <td><b>Workflows</b></td> - </tr> - </table> - </div> - <div style="clear: both"></div> - <div class="form-row"> - <table class="grid"> - <tr> - <td><b>name</b></td> - <td><b>steps</b></td> - <td><b>format-version</b></td> - <td><b>annotation</b></td> - </tr> - <% index = 0 %> - %for wf_dict in workflow_dicts: - <% - full_path = wf_dict[ 'full_path' ] - workflow_dict = wf_dict[ 'workflow_dict' ] - workflow_name = workflow_dict[ 'name' ] - if 'steps' in workflow_dict: - ## Initially steps were not stored in the metadata record. - steps = workflow_dict[ 'steps' ] - else: - steps = [] - format_version = workflow_dict[ 'format-version' ] - annotation = workflow_dict[ 'annotation' ] - %> - <tr> - <td> - <div class="menubutton" style="float: left;" id="workflow-${index}-popup"> - ${workflow_name} - <div popupmenu="workflow-${index}-popup"> - <a class="action-button" href="${h.url_for( controller='workflow', action='import_workflow', installed_repository_file=full_path, repository_id=trans.security.encode_id( repository.id ) )}">Import to Galaxy</a> - </div> - </div> - </td> - <td> - %if 'steps' in workflow_dict: - ${len( steps )} - %else: - unknown - %endif - </td> - <td>${format_version}</td> - <td>${annotation}</td> - </tr> - <% index += 1 %> - %endfor - </table> - </div> - <div style="clear: both"></div> - %endif - </div> -</div> +${render_metadata( repository, can_reset_metadata=False )} diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/admin/tool_shed_repository/common.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/common.mako @@ -0,0 +1,171 @@ +<%def name="render_metadata( repository, can_reset_metadata=False )"> + <div class="toolForm"> + <div class="toolFormTitle">Repository contents</div> + <div class="toolFormBody"> + <% metadata = repository.metadata %> + %if metadata: + %if 'tools' in metadata: + <div class="form-row"> + <table width="100%"> + <tr bgcolor="#D8D8D8" width="100%"> + <td><b>Tools</b><i> - click the name to view tool related information</i></td> + </tr> + </table> + </div> + <div class="form-row"> + <% tool_dicts = metadata[ 'tools' ] %> + <table class="grid"> + <tr> + <td><b>name</b></td> + <td><b>description</b></td> + <td><b>version</b></td> + <td><b>requirements</b></td> + </tr> + %for tool_dict in tool_dicts: + <tr> + <td> + <a class="view-info" href="${h.url_for( controller='admin_toolshed', action='view_tool_metadata', repository_id=trans.security.encode_id( repository.id ), tool_id=tool_dict[ 'id' ] )}"> + ${tool_dict[ 'name' ]} + </a> + </td> + <td>${tool_dict[ 'description' ]}</td> + <td>${tool_dict[ 'version' ]}</td> + <td> + <% + if 'requirements' in tool_dict: + requirements = tool_dict[ 'requirements' ] + else: + requirements = None + %> + %if requirements: + <% + requirements_str = '' + for requirement_dict in tool_dict[ 'requirements' ]: + requirements_str += '%s (%s), ' % ( requirement_dict[ 'name' ], requirement_dict[ 'type' ] ) + requirements_str = requirements_str.rstrip( ', ' ) + %> + ${requirements_str} + %else: + none + %endif + </td> + </tr> + %endfor + </table> + </div> + <div style="clear: both"></div> + %endif + %if 'workflows' in metadata: + <div class="form-row"> + <table width="100%"> + <tr bgcolor="#D8D8D8" width="100%"> + <td><b>Workflows</b><i> - click the name to import</i></td> + </tr> + </table> + </div> + <div style="clear: both"></div> + <div class="form-row"> + <% workflow_tups = metadata[ 'workflows' ] %> + <table class="grid"> + <tr> + <td><b>name</b></td> + <td><b>steps</b></td> + <td><b>format-version</b></td> + <td><b>annotation</b></td> + </tr> + <% index = 0 %> + %for workflow_tup in workflow_tups: + <% + import os.path + relative_path = workflow_tup[ 0 ] + full_path = os.path.abspath( relative_path ) + workflow_dict = workflow_tup[ 1 ] + workflow_name = workflow_dict[ 'name' ] + if 'steps' in workflow_dict: + ## Initially steps were not stored in the metadata record. + steps = workflow_dict[ 'steps' ] + else: + steps = [] + format_version = workflow_dict[ 'format-version' ] + annotation = workflow_dict[ 'annotation' ] + %> + <tr> + <td> + <div class="menubutton" style="float: left;" id="workflow-${index}-popup"> + ${workflow_name} + <div popupmenu="workflow-${index}-popup"> + <a class="action-button" href="${h.url_for( controller='workflow', action='import_workflow', installed_repository_file=full_path, repository_id=trans.security.encode_id( repository.id ) )}">Import to Galaxy</a> + </div> + </div> + </td> + <td> + %if steps: + ${len( steps )} + %else: + unknown + %endif + </td> + <td>${format_version}</td> + <td>${annotation}</td> + </tr> + <% index += 1 %> + %endfor + </table> + </div> + <div style="clear: both"></div> + %endif + %if 'datatypes' in metadata: + <div class="form-row"> + <table width="100%"> + <tr bgcolor="#D8D8D8" width="100%"> + <td><b>Data types</b></td> + </tr> + </table> + </div> + <div style="clear: both"></div> + <div class="form-row"> + <% datatypes_dicts = metadata[ 'datatypes' ] %> + <table class="grid"> + <tr> + <td><b>extension</b></td> + <td><b>dtype</b></td> + <td><b>mimetype</b></td> + </tr> + %for datatypes_dict in datatypes_dicts: + <% + extension = datatypes_dict[ 'extension' ] + dtype = datatypes_dict[ 'dtype' ] + mimetype = datatypes_dict[ 'mimetype' ] + %> + <tr> + <td>${extension}</td> + <td>${dtype}</td> + <td> + %if mimetype: + ${mimetype} + %else: +   + %endif + </td> + </tr> + %endfor + </table> + </div> + <div style="clear: both"></div> + %endif + %endif + %if can_reset_metadata: + <form name="set_metadata" action="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}" method="post"> + <div class="form-row"> + <div style="float: left; width: 250px; margin-right: 10px;"> + <input type="submit" name="set_metadata_button" value="Reset metadata"/> + </div> + <div class="toolParamHelp" style="clear: both;"> + Inspect the repository and reset the above attributes. + </div> + </div> + </form> + %endif + </div> + </div> +</%def> diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/admin/tool_shed_repository/manage_repository.mako --- a/templates/admin/tool_shed_repository/manage_repository.mako +++ b/templates/admin/tool_shed_repository/manage_repository.mako @@ -1,12 +1,13 @@ <%inherit file="/base.mako"/><%namespace file="/message.mako" import="render_msg" /> +<%namespace file="/admin/tool_shed_repository/common.mako" import="*" /><br/><br/><ul class="manage-table-actions"><li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li><div popupmenu="repository-${repository.id}-popup"> - <a class="action-button" href="${h.url_for( controller='admin', action='browse_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a> - <a class="action-button" href="${h.url_for( controller='admin', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a></div></ul> @@ -15,9 +16,9 @@ %endif <div class="toolForm"> - <div class="toolFormTitle">${repository.name}</div> + <div class="toolFormTitle">Installed tool shed repository '${repository.name}'</div><div class="toolFormBody"> - <form name="edit_repository" id="edit_repository" action="${h.url_for( controller='admin', action='manage_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}" method="post" > + <form name="edit_repository" id="edit_repository" action="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}" method="post" ><div class="form-row"><label>Tool shed:</label> ${repository.tool_shed} @@ -55,3 +56,6 @@ </form></div></div> +<p/> +${render_metadata( repository, can_reset_metadata=True )} +<p/> diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/admin/tool_shed_repository/select_tool_panel_section.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/select_tool_panel_section.mako @@ -0,0 +1,61 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, status )} +%endif + +<div class="warningmessage"> + The core Galaxy development team does not maintain the contents of many Galaxy tool shed repositories. Some repository tools + may include code that produces malicious behavior, so be aware of what you are installing. + <p/> + If you discover a repository that causes problems after installation, contact <a href="http://wiki.g2.bx.psu.edu/Support" target="_blank">Galaxy support</a>, + sending all necessary information, and appropriate action will be taken. + <p/> + <a href="http://wiki.g2.bx.psu.edu/Tool%20Shed#Contacting_the_owner_of_a_repository" target="_blank">Contact the repository owner</a> for general questions + or concerns. +</div> +<br/> +<div class="warningmessage"> + Installation may take a while, depending upon the size of the repository contents. Wait until a message is displayed in your + browser after clicking the <b>Install</b> button below. +</div> +<br/> + +<div class="toolForm"> + <div class="toolFormTitle">Choose section to load tools into tool panel</div> + <div class="toolFormBody"> + <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict )}" method="post" > + %if shed_tool_conf_select_field: + <div class="form-row"> + <label>Shed tool configuration file:</label> + ${shed_tool_conf_select_field.get_html()} + <div class="toolParamHelp" style="clear: both;"> + Your Galaxy instance is configured with ${len( shed_tool_conf_select_field.options )} shed tool configuration files, + so choose one in which to configure the installed tools. + </div> + </div> + <div style="clear: both"></div> + %else: + <input type="hidden" name="shed_tool_conf" value="${shed_tool_conf}"/> + %endif + <div class="form-row"> + <label>Add new tool panel section:</label> + <input name="new_tool_panel_section" type="textfield" value="${new_tool_panel_section}" size="40"/> + <div class="toolParamHelp" style="clear: both;"> + Add a new tool panel section or choose an existing section in your tool panel below to contain the installed tools. + </div> + </div> + <div class="form-row"> + <label>Select existing tool panel section:</label> + ${tool_panel_section_select_field.get_html()} + <div class="toolParamHelp" style="clear: both;"> + Choose an existing section in your tool panel to contain the installed tools. + </div> + </div> + <div class="form-row"> + <input type="submit" name="select_tool_panel_section_button" value="Install"/> + </div> + </form> + </div> +</div> diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/admin/tool_shed_repository/view_tool_metadata.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/view_tool_metadata.mako @@ -0,0 +1,155 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +<br/><br/> +<ul class="manage-table-actions"> + <li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li> + <div popupmenu="repository-${repository.id}-popup"> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> + </div> +</ul> + +%if message: + ${render_msg( message, status )} +%endif + +%if metadata: + <p/> + <div class="toolForm"> + <div class="toolFormTitle">${metadata[ 'name' ]} tool metadata</div> + <div class="toolFormBody"> + <div class="form-row"> + <label>Name:</label> + ${metadata[ 'name' ]} + <div style="clear: both"></div> + </div> + %if 'description' in metadata: + <div class="form-row"> + <label>Description:</label> + ${metadata[ 'description' ]} + <div style="clear: both"></div> + </div> + %endif + %if 'id' in metadata: + <div class="form-row"> + <label>Id:</label> + ${metadata[ 'id' ]} + <div style="clear: both"></div> + </div> + %endif + %if 'guid' in metadata: + <div class="form-row"> + <label>Guid:</label> + ${metadata[ 'guid' ]} + <div style="clear: both"></div> + </div> + %endif + %if 'version' in metadata: + <div class="form-row"> + <label>Version:</label> + ${metadata[ 'version' ]} + <div style="clear: both"></div> + </div> + %endif + %if 'version_string_cmd' in metadata: + <div class="form-row"> + <label>Version command string:</label> + ${metadata[ 'version_string_cmd' ]} + <div style="clear: both"></div> + </div> + %endif + %if tool: + <div class="form-row"> + <label>Command:</label> + <pre>${tool.command}</pre> + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Interpreter:</label> + ${tool.interpreter} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Is multi-byte:</label> + ${tool.is_multi_byte} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Forces a history refresh:</label> + ${tool.force_history_refresh} + <div style="clear: both"></div> + </div> + <div class="form-row"> + <label>Parallelism:</label> + ${tool.parallelism} + <div style="clear: both"></div> + </div> + %endif + <% + if 'requirements' in metadata: + requirements = metadata[ 'requirements' ] + else: + requirements = None + %> + %if requirements: + <% + requirements_str = '' + for requirement_dict in metadata[ 'requirements' ]: + requirements_str += '%s (%s), ' % ( requirement_dict[ 'name' ], requirement_dict[ 'type' ] ) + requirements_str = requirements_str.rstrip( ', ' ) + %> + <div class="form-row"> + <label>Requirements:</label> + ${requirements_str} + <div style="clear: both"></div> + </div> + %endif + <% + if 'tests' in metadata: + tests = metadata[ 'tests' ] + else: + tests = None + %> + %if tests: + <div class="form-row"> + <label>Functional tests:</label></td> + <table class="grid"> + <tr> + <td><b>name</b></td> + <td><b>inputs</b></td> + <td><b>outputs</b></td> + <td><b>required files</b></td> + </tr> + %for test_dict in tests: + <% + inputs = test_dict[ 'inputs' ] + outputs = test_dict[ 'outputs' ] + required_files = test_dict[ 'required_files' ] + %> + <tr> + <td>${test_dict[ 'name' ]}</td> + <td> + %for input in inputs: + <b>${input[0]}:</b> ${input[1]}<br/> + %endfor + </td> + <td> + %for output in outputs: + <b>${output[0]}:</b> ${output[1]}<br/> + %endfor + </td> + <td> + %for required_file in required_files: + ${required_file[0]}<br/> + %endfor + </td> + </tr> + %endfor + </table> + </div> + %endif + </div> + </div> +%endif diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/webapps/community/base_panels.mako --- a/templates/webapps/community/base_panels.mako +++ b/templates/webapps/community/base_panels.mako @@ -34,7 +34,8 @@ <div class="submenu"><ul><li><a target="_blank" href="${app.config.get( "support_url", "http://wiki.g2.bx.psu.edu/Support" )}">Support</a></li> - <li><a target="_blank" href="${app.config.get( "wiki_url", "http://wiki.g2.bx.psu.edu/" )}">Galaxy Wiki</a></li> + <li><a target="_blank" href="${app.config.get( "wiki_url", "http://wiki.g2.bx.psu.edu/Tool%20Shed" )}">Tool shed wiki</a></li> + <li><a target="_blank" href="${app.config.get( "wiki_url", "http://wiki.g2.bx.psu.edu/" )}">Galaxy wiki</a></li><li><a target="_blank" href="${app.config.get( "screencasts_url", "http://galaxycast.org" )}">Video tutorials (screencasts)</a></li><li><a target="_blank" href="${app.config.get( "citation_url", "http://wiki.g2.bx.psu.edu/Citing%20Galaxy" )}">How to Cite Galaxy</a></li></ul> diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/webapps/galaxy/admin/index.mako --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -67,7 +67,7 @@ <div class="toolTitle"><a href="${h.url_for( controller='admin', action='memdump' )}" target="galaxy_main">Profile memory usage</a></div><div class="toolTitle"><a href="${h.url_for( controller='admin', action='jobs' )}" target="galaxy_main">Manage jobs</a></div> %if cloned_repositories: - <div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_tool_shed_repositories' )}" target="galaxy_main">Manage installed tool shed repositories</a></div> + <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='browse_repositories' )}" target="galaxy_main">Manage installed tool shed repositories</a></div> %endif </div></div> @@ -76,7 +76,7 @@ <div class="toolSectionTitle">Tool sheds</div><div class="toolSectionBody"><div class="toolSectionBg"> - <div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_tool_sheds' )}" target="galaxy_main">Search and browse tool sheds</a></div> + <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='browse_tool_sheds' )}" target="galaxy_main">Search and browse tool sheds</a></div></div></div> %endif diff -r 87dc2336c1892878b26a110bb1857722fd3f4903 -r 5340450cdab2a7887d8d9e9186ae438d7b2aee93 templates/webapps/galaxy/admin/tool_sheds.mako --- a/templates/webapps/galaxy/admin/tool_sheds.mako +++ b/templates/webapps/galaxy/admin/tool_sheds.mako @@ -22,12 +22,12 @@ <tr class="libraryTitle"><td><div style="float: left; margin-left: 1px;" class="menubutton split popup" id="dataset-${shed_id}-popup"> - <a class="view-info" href="${h.url_for( controller='admin', action='browse_tool_shed', tool_shed_url=url )}">${name}</a> + <a class="view-info" href="${h.url_for( controller='admin_toolshed', action='browse_tool_shed', tool_shed_url=url )}">${name}</a></div><div popupmenu="dataset-${shed_id}-popup"> - <a class="action-button" href="${h.url_for( controller='admin', action='browse_tool_shed', tool_shed_url=url )}">Browse valid repositories</a> - <a class="action-button" href="${h.url_for( controller='admin', action='find_tools_in_tool_shed', tool_shed_url=url )}">Search for valid tools</a> - <a class="action-button" href="${h.url_for( controller='admin', action='find_workflows_in_tool_shed', tool_shed_url=url )}">Search for workflows</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='browse_tool_shed', tool_shed_url=url )}">Browse valid repositories</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='find_tools_in_tool_shed', tool_shed_url=url )}">Search for valid tools</a> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='find_workflows_in_tool_shed', tool_shed_url=url )}">Search for workflows</a></div></td></tr> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.