1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/8dce400655d6/ changeset: 8dce400655d6 user: greg date: 2011-09-30 15:48:56 summary: Add the ability for a Galaxy admin to get updates from the tool shed for cloned tool shed repositories installed on their local Galaxy instance. affected #: 10 files (-1 bytes) --- a/lib/galaxy/model/__init__.py Fri Sep 30 09:26:22 2011 -0400 +++ b/lib/galaxy/model/__init__.py Fri Sep 30 09:48:56 2011 -0400 @@ -2630,6 +2630,17 @@ class APIKeys( object ): pass +class ToolShedRepository( object ): + def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, changeset_revision=None, deleted=False ): + self.id = id + self.create_time = create_time + self.tool_shed = tool_shed + self.name = name + self.description = description + self.owner = owner + self.changeset_revision = changeset_revision + self.deleted = deleted + ## ---- Utility methods ------------------------------------------------------- def directory_hash_id( id ): --- a/lib/galaxy/model/mapping.py Fri Sep 30 09:26:22 2011 -0400 +++ b/lib/galaxy/model/mapping.py Fri Sep 30 09:48:56 2011 -0400 @@ -363,6 +363,17 @@ Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), Column( "deleted", Boolean, index=True, default=False ) ) +ToolShedRepository.table = Table( "tool_shed_repository", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "tool_shed", TrimmedString( 255 ), index=True ), + Column( "name", TrimmedString( 255 ), index=True ), + Column( "description" , TEXT ), + Column( "owner", TrimmedString( 255 ), index=True ), + Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "deleted", Boolean, index=True, default=False ) ) + Job.table = Table( "job", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1583,7 +1594,9 @@ annotations=relation( PageAnnotationAssociation, order_by=PageAnnotationAssociation.table.c.id, backref="pages" ), ratings=relation( PageRatingAssociation, order_by=PageRatingAssociation.table.c.id, backref="pages" ) ) ) - + +assign_mapper( context, ToolShedRepository, ToolShedRepository.table ) + # Set up proxy so that # Page.users_shared_with # returns a list of users that page is shared with. --- a/lib/galaxy/web/base/controller.py Fri Sep 30 09:26:22 2011 -0400 +++ b/lib/galaxy/web/base/controller.py Fri Sep 30 09:48:56 2011 -0400 @@ -1293,6 +1293,7 @@ role_list_grid = None group_list_grid = None quota_list_grid = None + repository_list_grid = None @web.expose @web.require_admin @@ -1302,8 +1303,11 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) if webapp == 'galaxy': + cloned_repositories = trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.deleted == False ) return trans.fill_template( '/webapps/galaxy/admin/index.mako', webapp=webapp, + cloned_repositories=cloned_repositories, message=message, status=status ) else: @@ -2382,362 +2386,9 @@ msg = msg, status = status, job_lock = trans.app.job_manager.job_queue.job_lock ) - @web.expose - @web.require_admin - def browse_tool_shed( self, trans, **kwd ): - tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = trans.request.host - url = '%s/repository/browse_downloadable_repositories?galaxy_url=%s&webapp=community' % ( tool_shed_url, galaxy_url ) - return trans.response.send_redirect( url ) - @web.expose - @web.require_admin - def install_tool_shed_repository( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - tool_shed_url = kwd[ 'tool_shed_url' ] - repository_name = kwd[ 'repository_name' ] - changeset_revision = kwd[ 'changeset_revision' ] - repository_clone_url = kwd[ 'repository_clone_url' ] - if kwd.get( 'select_tool_panel_section_button', False ): - shed_tool_conf = kwd[ 'shed_tool_conf' ] - # Get the tool path. - for k, tool_path in trans.app.toolbox.shed_tool_confs.items(): - if k == shed_tool_conf: - break - if 'tool_panel_section' in kwd: - section_key = 'section_%s' % kwd[ 'tool_panel_section' ] - tool_section = trans.app.toolbox.tool_panel[ section_key ] - # Clone the repository to the configured location. - current_working_dir = os.getcwd() - clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) - if os.path.exists( clone_dir ): - # Repository and revision has already been cloned. - # TODO: implement the ability to re-install or revert an existing repository. - message = 'Revision <b>%s</b> of repository <b>%s</b> has already been installed. Updating an existing repository is not yet supported.' % \ - ( changeset_revision, repository_name ) - status = 'error' - else: - os.makedirs( clone_dir ) - log.debug( 'Cloning %s...' % repository_clone_url ) - cmd = 'hg clone %s' % repository_clone_url - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( clone_dir ) - proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode == 0: - repo_files_dir = os.path.join( clone_dir, repository_name ) - log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision ) - cmd = 'hg update -r %s' % changeset_revision - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode == 0: - sample_files, repository_tools_tups = self.__get_repository_tools_and_sample_files( trans, tool_path, repo_files_dir ) - if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = self.__handle_missing_data_table_entry( trans, tool_path, sample_files, repository_tools_tups ) - # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = self.__handle_missing_index_file( trans, tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. - self.__handle_tool_dependencies( current_working_dir, repo_files_dir, repository_tools_tups ) - # Generate an in-memory tool conf section that includes the new tools. - new_tool_section = self.__generate_tool_panel_section( repository_name, - repository_clone_url, - changeset_revision, - tool_section, - repository_tools_tups ) - # Create a temporary file to persist the in-memory tool section - # TODO: Figure out how to do this in-memory using xml.etree. - tmp_name = tempfile.NamedTemporaryFile().name - persisted_new_tool_section = open( tmp_name, 'wb' ) - persisted_new_tool_section.write( new_tool_section ) - persisted_new_tool_section.close() - # Parse the persisted tool panel section - tree = ElementTree.parse( tmp_name ) - root = tree.getroot() - ElementInclude.include( root ) - # Load the tools in the section into the tool panel. - trans.app.toolbox.load_section_tag_set( root, trans.app.toolbox.tool_panel, tool_path ) - # Remove the temporary file - try: - os.unlink( tmp_name ) - except: - pass - # Append the new section to the shed_tool_config file. - self.__add_shed_tool_conf_entry( trans, shed_tool_conf, new_tool_section ) - message = 'Revision <b>%s</b> of repository <b>%s</b> has been installed in tool panel section <b>%s</b>.' % \ - ( changeset_revision, repository_name, tool_section.name ) - return trans.show_ok_message( message ) - else: - tmp_stderr = open( tmp_name, 'rb' ) - message = tmp_stderr.read() - tmp_stderr.close() - status = 'error' - else: - tmp_stderr = open( tmp_name, 'rb' ) - message = tmp_stderr.read() - tmp_stderr.close() - status = 'error' - else: - message = 'Choose the section in your tool panel to contain the installed tools.' - status = 'error' - if len( trans.app.toolbox.shed_tool_confs.keys() ) > 1: - shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) - shed_tool_conf = None - else: - shed_tool_conf = trans.app.toolbox.shed_tool_confs.keys()[0].lstrip( './' ) - shed_tool_conf_select_field = None - tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) - return trans.fill_template( '/admin/select_tool_panel_section.mako', - tool_shed_url=tool_shed_url, - repository_name=repository_name, - changeset_revision=changeset_revision, - repository_clone_url=repository_clone_url, - shed_tool_conf=shed_tool_conf, - shed_tool_conf_select_field=shed_tool_conf_select_field, - tool_panel_section_select_field=tool_panel_section_select_field, - message=message, - status=status ) - def __handle_missing_data_table_entry( self, trans, tool_path, sample_files, repository_tools_tups ): - # Inspect each tool to see if any have input parameters that are dynamically - # generated select lists that require entries in the tool_data_table_conf.xml file. - missing_data_table_entry = False - for index, repository_tools_tup in enumerate( repository_tools_tups ): - tup_path, repository_tool = repository_tools_tup - if repository_tool.params_with_missing_data_table_entry: - missing_data_table_entry = True - break - if missing_data_table_entry: - # The repository must contain a tool_data_table_conf.xml.sample file that includes - # all required entries for all tools in the repository. - for sample_file in sample_files: - head, tail = os.path.split( sample_file ) - if tail == 'tool_data_table_conf.xml.sample': - break - error, correction_msg = handle_sample_tool_data_table_conf_file( trans, sample_file ) - if error: - # TODO: Do more here than logging an exception. - log.debug( exception_msg ) - # Reload the tool into the local list of repository_tools_tups. - repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) - repository_tools_tups[ index ] = ( tup_path, repository_tool ) - return repository_tools_tups - def __handle_missing_index_file( self, trans, tool_path, sample_files, repository_tools_tups ): - # Inspect each tool to see if it has any input parameters that - # are dynamically generated select lists that depend on a .loc file. - missing_files_handled = [] - for index, repository_tools_tup in enumerate( repository_tools_tups ): - tup_path, repository_tool = repository_tools_tup - params_with_missing_index_file = repository_tool.params_with_missing_index_file - for param in params_with_missing_index_file: - options = param.options - missing_head, missing_tail = os.path.split( options.missing_index_file ) - if missing_tail not in missing_files_handled: - # The repository must contain the required xxx.loc.sample file. - for sample_file in sample_files: - sample_head, sample_tail = os.path.split( sample_file ) - if sample_tail == '%s.sample' % missing_tail: - copy_sample_loc_file( trans, sample_file ) - if options.tool_data_table and options.tool_data_table.missing_index_file: - options.tool_data_table.handle_found_index_file( options.missing_index_file ) - missing_files_handled.append( missing_tail ) - break - # Reload the tool into the local list of repository_tools_tups. - repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) - repository_tools_tups[ index ] = ( tup_path, repository_tool ) - return repository_tools_tups - def __handle_tool_dependencies( self, current_working_dir, repo_files_dir, repository_tools_tups ): - # Inspect each tool to see if it includes a "requirement" that refers to a fabric - # script. For those that do, execute the fabric script to install tool dependencies. - for index, repository_tools_tup in enumerate( repository_tools_tups ): - tup_path, repository_tool = repository_tools_tup - for requirement in repository_tool.requirements: - if requirement.type == 'fabfile': - log.debug( 'Executing fabric script to install dependencies for tool "%s"...' % repository_tool.name ) - fabfile = requirement.fabfile - method = requirement.method - # Find the relative path to the fabfile. - relative_fabfile_path = None - for root, dirs, files in os.walk( repo_files_dir ): - for name in files: - if name == fabfile: - relative_fabfile_path = os.path.join( root, name ) - break - if relative_fabfile_path: - # cmd will look something like: fab -f fabfile.py install_bowtie - cmd = 'fab -f %s %s' % ( relative_fabfile_path, method ) - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - if returncode != 0: - # TODO: do something more here than logging the problem. - tmp_stderr = open( tmp_name, 'rb' ) - error = tmp_stderr.read() - tmp_stderr.close() - log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) - def __get_repository_tools_and_sample_files( self, trans, tool_path, repo_files_dir ): - # The sample_files list contains all files whose name ends in .sample - sample_files = [] - # The repository_tools_tups list contains tuples of ( relative_path_to_tool_config, tool ) pairs - repository_tools_tups = [] - for root, dirs, files in os.walk( repo_files_dir ): - if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: - if '.hg' in dirs: - # Don't visit .hg directories - should be impossible since we don't - # allow uploaded archives that contain .hg dirs, but just in case... - dirs.remove( '.hg' ) - if 'hgrc' in files: - # Don't include hgrc files in commit. - files.remove( 'hgrc' ) - # Find all special .sample files first. - for name in files: - if name.endswith( '.sample' ): - sample_files.append( os.path.abspath( os.path.join( root, name ) ) ) - for name in files: - # Find all tool configs. - if name.endswith( '.xml' ): - relative_path = os.path.join( root, name ) - full_path = os.path.abspath( os.path.join( root, name ) ) - try: - repository_tool = trans.app.toolbox.load_tool( full_path ) - if repository_tool: - # At this point, we need to lstrip tool_path from relative_path. - tup_path = relative_path.replace( tool_path, '' ).lstrip( '/' ) - repository_tools_tups.append( ( tup_path, repository_tool ) ) - except Exception, e: - # We have an invalid .xml file, so not a tool config. - log.debug( "Ignoring invalid tool config (%s). Error: %s" % ( str( relative_path ), str( e ) ) ) - return sample_files, repository_tools_tups - def __add_shed_tool_conf_entry( self, trans, shed_tool_conf, new_tool_section ): - # Add an entry in the shed_tool_conf file. An entry looks something like: - # <section name="Filter and Sort" id="filter"> - # <tool file="filter/filtering.xml" guid="toolshed.g2.bx.psu.edu/repos/test/filter/1.0.2"/> - # </section> - # Make a backup of the hgweb.config file since we're going to be changing it. - if not os.path.exists( shed_tool_conf ): - output = open( shed_tool_conf, 'w' ) - output.write( '<?xml version="1.0"?>\n' ) - output.write( '<toolbox tool_path="%s">\n' % tool_path ) - output.write( '</toolbox>\n' ) - output.close() - self.__make_shed_tool_conf_copy( trans, shed_tool_conf ) - tmp_fd, tmp_fname = tempfile.mkstemp() - new_shed_tool_conf = open( tmp_fname, 'wb' ) - for i, line in enumerate( open( shed_tool_conf ) ): - if line.startswith( '</toolbox>' ): - # We're at the end of the original config file, so add our entry. - new_shed_tool_conf.write( new_tool_section ) - new_shed_tool_conf.write( line ) - else: - new_shed_tool_conf.write( line ) - new_shed_tool_conf.close() - shutil.move( tmp_fname, os.path.abspath( shed_tool_conf ) ) - def __make_shed_tool_conf_copy( self, trans, shed_tool_conf ): - # Make a backup of the shed_tool_conf file. - today = date.today() - backup_date = today.strftime( "%Y_%m_%d" ) - shed_tool_conf_copy = '%s/%s_%s_backup' % ( trans.app.config.root, shed_tool_conf, backup_date ) - shutil.copy( os.path.abspath( shed_tool_conf ), os.path.abspath( shed_tool_conf_copy ) ) - def __clean_tool_shed_url( self, tool_shed_url ): - if tool_shed_url.find( ':' ) > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - return tool_shed_url.split( ':' )[ 0 ] - return tool_shed_url.rstrip( '/' ) - def __clean_repository_clone_url( self, repository_clone_url ): - if repository_clone_url.find( '@' ) > 0: - # We have an url that includes an authenticated user, something like: - # http://test@bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split( '@' ) - tmp_url = items[ 1 ] - elif repository_clone_url.find( '\/\/' ) > 0: - # We have an url that includes only a protocol, something like: - # http://bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split( '\/\/' ) - tmp_url = items[ 1 ] - else: - tmp_url = repository_clone_url - return tmp_url - def __get_repository_owner( self, cleaned_repository_url ): - items = cleaned_repository_url.split( 'repos' ) - repo_path = items[ 1 ] - return repo_path.lstrip( '/' ).split( '/' )[ 0 ] - def __generate_tool_path( self, repository_clone_url, changeset_revision ): - """ - Generate a tool path that guarantees repositories with the same name will always be installed - in different directories. The tool path will be of the form: - <tool shed url>/repos/<repository owner>/<repository name>/<changeset revision> - http://test@bx.psu.edu:9009/repos/test/filter - """ - tmp_url = self.__clean_repository_clone_url( repository_clone_url ) - # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column - items = tmp_url.split( 'repos' ) - tool_shed_url = items[ 0 ] - repo_path = items[ 1 ] - tool_shed_url = self.__clean_tool_shed_url( tool_shed_url ) - return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) - def __generate_tool_guid( self, repository_clone_url, tool ): - """ - Generate a guid for the installed tool. It is critical that this guid matches the guid for - the tool in the Galaxy tool shed from which it is being installed. The form of the guid is - <tool shed host>/repos/<repository owner>/<repository name>/<tool id>/<tool version> - """ - tmp_url = self.__clean_repository_clone_url( repository_clone_url ) - return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) - def __generate_tool_panel_section( self, repository_name, repository_clone_url, changeset_revision, tool_section, repository_tools_tups ): - """ - Write an in-memory tool panel section so we can load it into the tool panel and then - append it to the appropriate shed tool config. - TODO: re-write using ElementTree. - """ - tmp_url = self.__clean_repository_clone_url( repository_clone_url ) - section_str = '' - section_str += ' <section name="%s" id="%s">\n' % ( tool_section.name, tool_section.id ) - for repository_tool_tup in repository_tools_tups: - tool_file_path, tool = repository_tool_tup - guid = self.__generate_tool_guid( repository_clone_url, tool ) - section_str += ' <tool file="%s" guid="%s">\n' % ( tool_file_path, guid ) - section_str += ' <tool_shed>%s</tool_shed>\n' % tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) - section_str += ' <repository_name>%s</repository_name>\n' % repository_name - section_str += ' <repository_owner>%s</repository_owner>\n' % self.__get_repository_owner( tmp_url ) - section_str += ' <changeset_revision>%s</changeset_revision>\n' % changeset_revision - section_str += ' <id>%s</id>\n' % tool.id - section_str += ' <version>%s</version>\n' % tool.version - section_str += ' </tool>\n' - section_str += ' </section>\n' - return section_str ## ---- Utility methods ------------------------------------------------------- -def build_shed_tool_conf_select_field( trans ): - """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs.""" - options = [] - for shed_tool_conf_filename, tool_path in trans.app.toolbox.shed_tool_confs.items(): - options.append( ( shed_tool_conf_filename.lstrip( './' ), shed_tool_conf_filename ) ) - select_field = SelectField( name='shed_tool_conf' ) - for option_tup in options: - select_field.add_option( option_tup[0], option_tup[1] ) - return select_field -def build_tool_panel_section_select_field( trans ): - """Build a SelectField whose options are the sections of the current in-memory toolbox.""" - options = [] - for k, tool_section in trans.app.toolbox.tool_panel.items(): - options.append( ( tool_section.name, tool_section.id ) ) - select_field = SelectField( name='tool_panel_section', display='radio' ) - for option_tup in options: - select_field.add_option( option_tup[0], option_tup[1] ) - return select_field def copy_sample_loc_file( trans, filename ): """Copy xxx.loc.sample to ~/tool-data/xxx.loc.sample and ~/tool-data/xxx.loc""" head, sample_loc_file = os.path.split( filename ) @@ -2757,6 +2408,12 @@ if not user: return trans.show_error_message( "User not found for id (%s)" % str( id ) ) return user +def get_user_by_username( trans, username ): + """Get a user from the database by username""" + # TODO: Add exception handling here. + return trans.sa_session.query( trans.model.User ) \ + .filter( trans.model.User.table.c.username == username ) \ + .one() def get_role( trans, id ): """Get a Role from the database by id.""" # Load user from database --- a/lib/galaxy/web/controllers/admin.py Fri Sep 30 09:26:22 2011 -0400 +++ b/lib/galaxy/web/controllers/admin.py Fri Sep 30 09:48:56 2011 -0400 @@ -384,14 +384,66 @@ preserve_state = False use_paging = True +class RepositoryListGrid( grids.Grid ): + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.name + class DescriptionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.description + class OwnerColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.owner + class RevisionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.changeset_revision + class ToolShedColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.tool_shed + # Grid definition + title = "Tool shed repositories" + model_class = model.ToolShedRepository + template='/admin/tool_shed_repository/grid.mako' + default_sort_key = "name" + columns = [ + NameColumn( "Name", + key="name", + attach_popup=True ), + DescriptionColumn( "Description" ), + OwnerColumn( "Owner" ), + RevisionColumn( "Revision" ), + ToolShedColumn( "Tool shed" ), + # Columns that are valid for filtering but are not visible. + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, + filterable="advanced" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ grids.GridOperation( "Get updates", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) ] + standard_filters = [] + default_filter = dict( deleted="False" ) + num_rows_per_page = 50 + preserve_state = False + use_paging = True + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( self.model_class ) + class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamParser ): user_list_grid = UserListGrid() role_list_grid = RoleListGrid() group_list_grid = GroupListGrid() quota_list_grid = QuotaListGrid() + repository_list_grid = RepositoryListGrid() - # Galaxy Quota Stuff @web.expose @web.require_admin def quotas( self, trans, **kwargs ): @@ -417,7 +469,6 @@ return self.edit_quota( trans, **kwargs ) # Render the list view return self.quota_list_grid( trans, **kwargs ) - @web.expose @web.require_admin def create_quota( self, trans, **kwd ): @@ -464,7 +515,6 @@ out_groups=params.out_groups, message=params.message, status=params.status ) - @web.expose @web.require_admin def rename_quota( self, trans, **kwd ): @@ -478,7 +528,6 @@ webapp=params.webapp, message=params.message, status=params.status ) - @web.expose @web.require_admin def manage_users_and_groups_for_quota( self, trans, **kwd ): @@ -513,7 +562,6 @@ webapp=params.webapp, message=params.message, status=params.status ) - @web.expose @web.require_admin def edit_quota( self, trans, **kwd ): @@ -527,7 +575,6 @@ webapp=params.webapp, message=params.message, status=params.status ) - @web.expose @web.require_admin def set_quota_default( self, trans, **kwd ): @@ -546,7 +593,6 @@ webapp=params.webapp, message=params.message, status=params.status ) - @web.expose @web.require_admin def unset_quota_default( self, trans, **kwd ): @@ -558,8 +604,6 @@ webapp=params.webapp, message=util.sanitize_text( params.message ), status='error' ) ) - - @web.expose @web.require_admin def mark_quota_deleted( self, trans, **kwd ): @@ -571,7 +615,6 @@ webapp=params.webapp, message=util.sanitize_text( params.message ), status='error' ) ) - @web.expose @web.require_admin def undelete_quota( self, trans, **kwd ): @@ -583,7 +626,6 @@ webapp=params.webapp, message=util.sanitize_text( params.message ), status='error' ) ) - @web.expose @web.require_admin def purge_quota( self, trans, **kwd ): @@ -595,7 +637,6 @@ webapp=params.webapp, message=util.sanitize_text( params.message ), status='error' ) ) - def _quota_op( self, trans, do_op, op_method, kwd, listify=False ): params = self.get_quota_params( kwd ) if listify: @@ -633,3 +674,488 @@ params.message = e.err_msg params.status = e.type return quota, params + @web.expose + @web.require_admin + def browse_repositories( self, trans, **kwd ): + if 'operation' in kwd: + operation = kwd.pop('operation').lower() + if operation == "get updates": + return self.check_for_updates( trans, **kwd ) + # Render the list view + return self.repository_list_grid( trans, **kwd ) + @web.expose + @web.require_admin + def browse_tool_shed( self, trans, **kwd ): + tool_shed_url = kwd[ 'tool_shed_url' ] + galaxy_url = trans.request.host + url = '%s/repository/browse_downloadable_repositories?galaxy_url=%s&webapp=community' % ( tool_shed_url, galaxy_url ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def install_tool_shed_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + tool_shed_url = kwd[ 'tool_shed_url' ] + name = kwd[ 'name' ] + description = kwd[ 'description' ] + changeset_revision = kwd[ 'changeset_revision' ] + repository_clone_url = kwd[ 'repository_clone_url' ] + if kwd.get( 'select_tool_panel_section_button', False ): + shed_tool_conf = kwd[ 'shed_tool_conf' ] + # Get the tool path. + for k, tool_path in trans.app.toolbox.shed_tool_confs.items(): + if k == shed_tool_conf: + break + if 'tool_panel_section' in kwd: + section_key = 'section_%s' % kwd[ 'tool_panel_section' ] + tool_section = trans.app.toolbox.tool_panel[ section_key ] + # Clone the repository to the configured location. + current_working_dir = os.getcwd() + clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) + if os.path.exists( clone_dir ): + # Repository and revision has already been cloned. + # TODO: implement the ability to re-install or revert an existing repository. + message = 'Revision <b>%s</b> of repository <b>%s</b> has already been installed. Updating an existing repository is not yet supported.' % \ + ( changeset_revision, name ) + status = 'error' + else: + os.makedirs( clone_dir ) + log.debug( 'Cloning %s...' % repository_clone_url ) + cmd = 'hg clone %s' % repository_clone_url + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( clone_dir ) + proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + # Add a new record to the tool_shed_repository table. + tool_shed_repository = self.__create_tool_shed_repository( trans, + name, + description, + changeset_revision, + repository_clone_url ) + # Update the cloned repository to changeset_revision. + repo_files_dir = os.path.join( clone_dir, name ) + log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision ) + cmd = 'hg update -r %s' % changeset_revision + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + sample_files, repository_tools_tups = self.__get_repository_tools_and_sample_files( trans, tool_path, repo_files_dir ) + if repository_tools_tups: + # Handle missing data table entries for tool parameters that are dynamically generated select lists. + repository_tools_tups = self.__handle_missing_data_table_entry( trans, tool_path, sample_files, repository_tools_tups ) + # Handle missing index files for tool parameters that are dynamically generated select lists. + repository_tools_tups = self.__handle_missing_index_file( trans, tool_path, sample_files, repository_tools_tups ) + # Handle tools that use fabric scripts to install dependencies. + self.__handle_tool_dependencies( current_working_dir, repo_files_dir, repository_tools_tups ) + # Generate an in-memory tool conf section that includes the new tools. + new_tool_section = self.__generate_tool_panel_section( name, + repository_clone_url, + changeset_revision, + tool_section, + repository_tools_tups ) + # Create a temporary file to persist the in-memory tool section + # TODO: Figure out how to do this in-memory using xml.etree. + tmp_name = tempfile.NamedTemporaryFile().name + persisted_new_tool_section = open( tmp_name, 'wb' ) + persisted_new_tool_section.write( new_tool_section ) + persisted_new_tool_section.close() + # Parse the persisted tool panel section + tree = ElementTree.parse( tmp_name ) + root = tree.getroot() + ElementInclude.include( root ) + # Load the tools in the section into the tool panel. + trans.app.toolbox.load_section_tag_set( root, trans.app.toolbox.tool_panel, tool_path ) + # Remove the temporary file + try: + os.unlink( tmp_name ) + except: + pass + # Append the new section to the shed_tool_config file. + self.__add_shed_tool_conf_entry( trans, shed_tool_conf, new_tool_section ) + message = 'Revision <b>%s</b> of repository <b>%s</b> has been installed in tool panel section <b>%s</b>.' % \ + ( changeset_revision, name, tool_section.name ) + return trans.show_ok_message( message ) + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + message = 'Choose the section in your tool panel to contain the installed tools.' + status = 'error' + if len( trans.app.toolbox.shed_tool_confs.keys() ) > 1: + shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) + shed_tool_conf = None + else: + shed_tool_conf = trans.app.toolbox.shed_tool_confs.keys()[0].lstrip( './' ) + shed_tool_conf_select_field = None + tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) + return trans.fill_template( '/admin/select_tool_panel_section.mako', + tool_shed_url=tool_shed_url, + name=name, + description=description, + changeset_revision=changeset_revision, + repository_clone_url=repository_clone_url, + shed_tool_conf=shed_tool_conf, + shed_tool_conf_select_field=shed_tool_conf_select_field, + tool_panel_section_select_field=tool_panel_section_select_field, + message=message, + status=status ) + @web.expose + @web.require_admin + def check_for_updates( self, trans, **kwd ): + params = util.Params( kwd ) + repository_id = params.get( 'id', None ) + repository = get_repository( trans, repository_id ) + galaxy_url = trans.request.host + # Send a request to the relevant tool shed to see if there are any updates. + # TODO: support https in the following url. + url = 'http://%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=community' % \ + ( repository.tool_shed, galaxy_url, repository.name, repository.owner, repository.changeset_revision ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def update_to_changeset_revision( self, trans, **kwd ): + """Update a cloned repository to the latest revision possible.""" + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + tool_shed_url = kwd[ 'tool_shed_url' ] + name = params.get( 'name', None ) + owner = params.get( 'owner', None ) + changeset_revision = params.get( 'changeset_revision', None ) + latest_changeset_revision = params.get( 'latest_changeset_revision', None ) + if changeset_revision and latest_changeset_revision: + if changeset_revision == latest_changeset_revision: + message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name + else: + repository = get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision ) + current_working_dir = os.getcwd() + # Get the directory where the repository is cloned. + cleaned_tool_shed_url = self.__clean_tool_shed_url( tool_shed_url ) + partial_cloned_dir = '%s/repos/%s/%s/%s' % ( cleaned_tool_shed_url, owner, name, changeset_revision ) + # Get the relative tool installation paths from each of the shed tool configs. + shed_tool_confs = trans.app.toolbox.shed_tool_confs + relative_cloned_dir = None + # The shed_tool_confs dictionary contains shed_conf_filename : tool_path pairs. + for shed_conf_filename, tool_path in shed_tool_confs.items(): + relative_cloned_dir = os.path.join( tool_path, partial_cloned_dir ) + if os.path.isdir( relative_cloned_dir ): + break + if relative_cloned_dir: + # Update the cloned repository to changeset_revision. + repo_files_dir = os.path.join( relative_cloned_dir, name ) + log.debug( "Updating cloned repository named '%s' from revision '%s' to revision '%s'..." % \ + ( name, changeset_revision, latest_changeset_revision ) ) + cmd = 'hg pull' + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + cmd = 'hg update -r %s' % latest_changeset_revision + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode == 0: + # Update the repository changeset_revision in the database. + repository.changeset_revision = latest_changeset_revision + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \ + ( name, latest_changeset_revision ) + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + tmp_stderr = open( tmp_name, 'rb' ) + message = tmp_stderr.read() + tmp_stderr.close() + status = 'error' + else: + message = "The directory containing the cloned repository named '%s' cannot be found." % name + status = 'error' + else: + message = "The latest changeset revision could not be retrieved for the repository named '%s'." % name + status = 'error' + return trans.response.send_redirect( web.url_for( controller='admin', + action='browse_repositories', + message=message, + status=status ) ) + def __handle_missing_data_table_entry( self, trans, tool_path, sample_files, repository_tools_tups ): + # Inspect each tool to see if any have input parameters that are dynamically + # generated select lists that require entries in the tool_data_table_conf.xml file. + missing_data_table_entry = False + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + if repository_tool.params_with_missing_data_table_entry: + missing_data_table_entry = True + break + if missing_data_table_entry: + # The repository must contain a tool_data_table_conf.xml.sample file that includes + # all required entries for all tools in the repository. + for sample_file in sample_files: + head, tail = os.path.split( sample_file ) + if tail == 'tool_data_table_conf.xml.sample': + break + error, correction_msg = handle_sample_tool_data_table_conf_file( trans, sample_file ) + if error: + # TODO: Do more here than logging an exception. + log.debug( exception_msg ) + # Reload the tool into the local list of repository_tools_tups. + repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) + repository_tools_tups[ index ] = ( tup_path, repository_tool ) + return repository_tools_tups + def __handle_missing_index_file( self, trans, tool_path, sample_files, repository_tools_tups ): + # Inspect each tool to see if it has any input parameters that + # are dynamically generated select lists that depend on a .loc file. + missing_files_handled = [] + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + params_with_missing_index_file = repository_tool.params_with_missing_index_file + for param in params_with_missing_index_file: + options = param.options + missing_head, missing_tail = os.path.split( options.missing_index_file ) + if missing_tail not in missing_files_handled: + # The repository must contain the required xxx.loc.sample file. + for sample_file in sample_files: + sample_head, sample_tail = os.path.split( sample_file ) + if sample_tail == '%s.sample' % missing_tail: + copy_sample_loc_file( trans, sample_file ) + if options.tool_data_table and options.tool_data_table.missing_index_file: + options.tool_data_table.handle_found_index_file( options.missing_index_file ) + missing_files_handled.append( missing_tail ) + break + # Reload the tool into the local list of repository_tools_tups. + repository_tool = trans.app.toolbox.load_tool( os.path.join( tool_path, tup_path ) ) + repository_tools_tups[ index ] = ( tup_path, repository_tool ) + return repository_tools_tups + def __handle_tool_dependencies( self, current_working_dir, repo_files_dir, repository_tools_tups ): + # Inspect each tool to see if it includes a "requirement" that refers to a fabric + # script. For those that do, execute the fabric script to install tool dependencies. + for index, repository_tools_tup in enumerate( repository_tools_tups ): + tup_path, repository_tool = repository_tools_tup + for requirement in repository_tool.requirements: + if requirement.type == 'fabfile': + log.debug( 'Executing fabric script to install dependencies for tool "%s"...' % repository_tool.name ) + fabfile = requirement.fabfile + method = requirement.method + # Find the relative path to the fabfile. + relative_fabfile_path = None + for root, dirs, files in os.walk( repo_files_dir ): + for name in files: + if name == fabfile: + relative_fabfile_path = os.path.join( root, name ) + break + if relative_fabfile_path: + # cmd will look something like: fab -f fabfile.py install_bowtie + cmd = 'fab -f %s %s' % ( relative_fabfile_path, method ) + tmp_name = tempfile.NamedTemporaryFile().name + tmp_stderr = open( tmp_name, 'wb' ) + os.chdir( repo_files_dir ) + proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) + returncode = proc.wait() + os.chdir( current_working_dir ) + tmp_stderr.close() + if returncode != 0: + # TODO: do something more here than logging the problem. + tmp_stderr = open( tmp_name, 'rb' ) + error = tmp_stderr.read() + tmp_stderr.close() + log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) + def __get_repository_tools_and_sample_files( self, trans, tool_path, repo_files_dir ): + # The sample_files list contains all files whose name ends in .sample + sample_files = [] + # The repository_tools_tups list contains tuples of ( relative_path_to_tool_config, tool ) pairs + repository_tools_tups = [] + for root, dirs, files in os.walk( repo_files_dir ): + if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0: + if '.hg' in dirs: + # Don't visit .hg directories - should be impossible since we don't + # allow uploaded archives that contain .hg dirs, but just in case... + dirs.remove( '.hg' ) + if 'hgrc' in files: + # Don't include hgrc files in commit. + files.remove( 'hgrc' ) + # Find all special .sample files first. + for name in files: + if name.endswith( '.sample' ): + sample_files.append( os.path.abspath( os.path.join( root, name ) ) ) + for name in files: + # Find all tool configs. + if name.endswith( '.xml' ): + relative_path = os.path.join( root, name ) + full_path = os.path.abspath( os.path.join( root, name ) ) + try: + repository_tool = trans.app.toolbox.load_tool( full_path ) + if repository_tool: + # At this point, we need to lstrip tool_path from relative_path. + tup_path = relative_path.replace( tool_path, '' ).lstrip( '/' ) + repository_tools_tups.append( ( tup_path, repository_tool ) ) + except Exception, e: + # We have an invalid .xml file, so not a tool config. + log.debug( "Ignoring invalid tool config (%s). Error: %s" % ( str( relative_path ), str( e ) ) ) + return sample_files, repository_tools_tups + def __create_tool_shed_repository( self, trans, name, description, changeset_revision, repository_clone_url ): + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) + owner = self.__get_repository_owner( tmp_url ) + tool_shed_repository = trans.model.ToolShedRepository( tool_shed=tool_shed, + name=name, + description=description, + owner=owner, + changeset_revision=changeset_revision ) + trans.sa_session.add( tool_shed_repository ) + trans.sa_session.flush() + def __add_shed_tool_conf_entry( self, trans, shed_tool_conf, new_tool_section ): + # Add an entry in the shed_tool_conf file. An entry looks something like: + # <section name="Filter and Sort" id="filter"> + # <tool file="filter/filtering.xml" guid="toolshed.g2.bx.psu.edu/repos/test/filter/1.0.2"/> + # </section> + # Make a backup of the hgweb.config file since we're going to be changing it. + if not os.path.exists( shed_tool_conf ): + output = open( shed_tool_conf, 'w' ) + output.write( '<?xml version="1.0"?>\n' ) + output.write( '<toolbox tool_path="%s">\n' % tool_path ) + output.write( '</toolbox>\n' ) + output.close() + self.__make_shed_tool_conf_copy( trans, shed_tool_conf ) + tmp_fd, tmp_fname = tempfile.mkstemp() + new_shed_tool_conf = open( tmp_fname, 'wb' ) + for i, line in enumerate( open( shed_tool_conf ) ): + if line.startswith( '</toolbox>' ): + # We're at the end of the original config file, so add our entry. + new_shed_tool_conf.write( new_tool_section ) + new_shed_tool_conf.write( line ) + else: + new_shed_tool_conf.write( line ) + new_shed_tool_conf.close() + shutil.move( tmp_fname, os.path.abspath( shed_tool_conf ) ) + def __make_shed_tool_conf_copy( self, trans, shed_tool_conf ): + # Make a backup of the shed_tool_conf file. + today = date.today() + backup_date = today.strftime( "%Y_%m_%d" ) + shed_tool_conf_copy = '%s/%s_%s_backup' % ( trans.app.config.root, shed_tool_conf, backup_date ) + shutil.copy( os.path.abspath( shed_tool_conf ), os.path.abspath( shed_tool_conf_copy ) ) + def __clean_tool_shed_url( self, tool_shed_url ): + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + return tool_shed_url.split( ':' )[ 0 ] + return tool_shed_url.rstrip( '/' ) + def __clean_repository_clone_url( self, repository_clone_url ): + if repository_clone_url.find( '@' ) > 0: + # We have an url that includes an authenticated user, something like: + # http://test@bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split( '@' ) + tmp_url = items[ 1 ] + elif repository_clone_url.find( '\/\/' ) > 0: + # We have an url that includes only a protocol, something like: + # http://bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split( '\/\/' ) + tmp_url = items[ 1 ] + else: + tmp_url = repository_clone_url + return tmp_url + def __get_repository_owner( self, cleaned_repository_url ): + items = cleaned_repository_url.split( 'repos' ) + repo_path = items[ 1 ] + return repo_path.lstrip( '/' ).split( '/' )[ 0 ] + def __generate_tool_path( self, repository_clone_url, changeset_revision ): + """ + Generate a tool path that guarantees repositories with the same name will always be installed + in different directories. The tool path will be of the form: + <tool shed url>/repos/<repository owner>/<repository name>/<changeset revision> + http://test@bx.psu.edu:9009/repos/test/filter + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column + items = tmp_url.split( 'repos' ) + tool_shed_url = items[ 0 ] + repo_path = items[ 1 ] + tool_shed_url = self.__clean_tool_shed_url( tool_shed_url ) + return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) + def __generate_tool_guid( self, repository_clone_url, tool ): + """ + Generate a guid for the installed tool. It is critical that this guid matches the guid for + the tool in the Galaxy tool shed from which it is being installed. The form of the guid is + <tool shed host>/repos/<repository owner>/<repository name>/<tool id>/<tool version> + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) + def __generate_tool_panel_section( self, repository_name, repository_clone_url, changeset_revision, tool_section, repository_tools_tups ): + """ + Write an in-memory tool panel section so we can load it into the tool panel and then + append it to the appropriate shed tool config. + TODO: re-write using ElementTree. + """ + tmp_url = self.__clean_repository_clone_url( repository_clone_url ) + section_str = '' + section_str += ' <section name="%s" id="%s">\n' % ( tool_section.name, tool_section.id ) + for repository_tool_tup in repository_tools_tups: + tool_file_path, tool = repository_tool_tup + guid = self.__generate_tool_guid( repository_clone_url, tool ) + section_str += ' <tool file="%s" guid="%s">\n' % ( tool_file_path, guid ) + section_str += ' <tool_shed>%s</tool_shed>\n' % tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) + section_str += ' <repository_name>%s</repository_name>\n' % repository_name + section_str += ' <repository_owner>%s</repository_owner>\n' % self.__get_repository_owner( tmp_url ) + section_str += ' <changeset_revision>%s</changeset_revision>\n' % changeset_revision + section_str += ' <id>%s</id>\n' % tool.id + section_str += ' <version>%s</version>\n' % tool.version + section_str += ' </tool>\n' + section_str += ' </section>\n' + return section_str + +## ---- Utility methods ------------------------------------------------------- + +def build_shed_tool_conf_select_field( trans ): + """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs.""" + options = [] + for shed_tool_conf_filename, tool_path in trans.app.toolbox.shed_tool_confs.items(): + options.append( ( shed_tool_conf_filename.lstrip( './' ), shed_tool_conf_filename ) ) + select_field = SelectField( name='shed_tool_conf' ) + for option_tup in options: + select_field.add_option( option_tup[0], option_tup[1] ) + return select_field +def build_tool_panel_section_select_field( trans ): + """Build a SelectField whose options are the sections of the current in-memory toolbox.""" + options = [] + for k, tool_section in trans.app.toolbox.tool_panel.items(): + options.append( ( tool_section.name, tool_section.id ) ) + select_field = SelectField( name='tool_panel_section', display='radio' ) + for option_tup in options: + select_field.add_option( option_tup[0], option_tup[1] ) + return select_field +def get_repository( trans, id ): + """Get a tool_shed_repository from the database via id""" + return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) +def get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision ): + """Get a repository from the database via name owner and changeset_revision""" + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( and_( trans.model.ToolShedRepository.table.c.name == name, + trans.model.ToolShedRepository.table.c.owner == owner, + trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \ + .first() --- a/lib/galaxy/webapps/community/controllers/common.py Fri Sep 30 09:26:22 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/common.py Fri Sep 30 09:48:56 2011 -0400 @@ -86,6 +86,13 @@ def get_repository( trans, id ): """Get a repository from the database via id""" return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) +def get_repository_by_name_and_owner( trans, name, owner ): + """Get a repository from the database via name and owner""" + user = get_user_by_username( trans, owner ) + return trans.sa_session.query( trans.model.Repository ) \ + .filter( and_( trans.model.Repository.table.c.name == name, + trans.model.Repository.table.c.user_id == user.id ) ) \ + .first() def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): """Get metadata for a specified repository change set from the database""" return trans.sa_session.query( trans.model.RepositoryMetadata ) \ @@ -95,6 +102,10 @@ def get_repository_metadata_by_id( trans, id ): """Get repository metadata from the database""" return trans.sa_session.query( trans.model.RepositoryMetadata ).get( trans.security.decode_id( id ) ) +def get_repository_metadata_by_repository_id( trans, id ): + """Get all metadata records for a specified repository.""" + return trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) def get_revision_label( trans, repository, changeset_revision ): """ Return a string consisting of the human read-able @@ -469,7 +480,7 @@ _ui.setconfig( 'ui', 'quiet', True ) return _ui def get_user( trans, id ): - """Get a user from the database""" + """Get a user from the database by id""" return trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( id ) ) def handle_email_alerts( trans, repository ): repo_dir = repository.repo_path --- a/lib/galaxy/webapps/community/controllers/repository.py Fri Sep 30 09:26:22 2011 -0400 +++ b/lib/galaxy/webapps/community/controllers/repository.py Fri Sep 30 09:48:56 2011 -0400 @@ -341,8 +341,100 @@ tool_shed_url = trans.request.host repository_clone_url = generate_clone_url( trans, repository_id ) # TODO: support https in the following url. - url = 'http://%s/admin/install_tool_shed_repository?tool_shed_url=%s&repository_name=%s&repository_clone_url=%s&changeset_revision=%s' % \ - ( galaxy_url, tool_shed_url, repository.name, repository_clone_url, changeset_revision ) + url = 'http://%s/admin/install_tool_shed_repository?tool_shed_url=%s&name=%s&description=%s&repository_clone_url=%s&changeset_revision=%s' % \ + ( galaxy_url, tool_shed_url, repository.name, repository.description, repository_clone_url, changeset_revision ) + return trans.response.send_redirect( url ) + @web.expose + def check_for_updates( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + # The sender didn't store galaxy_url in a cookie since + # this method immediately redirects back to the caller. + galaxy_url = kwd[ 'galaxy_url' ] + name = params.get( 'name', None ) + owner = params.get( 'owner', None ) + changeset_revision = params.get( 'changeset_revision', None ) + webapp = params.get( 'webapp', None ) + tool_shed_url = trans.request.host + # Start building up the url to redirect back to the calling Galaxy instance. + # TODO: support https in the following url. + url = 'http://%s/admin/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, tool_shed_url ) + repository = get_repository_by_name_and_owner( trans, name, owner ) + #if error: + # url += '&message=%s&status=error' % message + #else: + url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ + ( repository.name, repository.user.username, changeset_revision ) + if changeset_revision == repository.tip: + # If changeset_revision is the repository tip, then + # we know there are no additional updates for the tools. + url += repository.tip + else: + repository_metadata = get_repository_metadata_by_changeset_revision( trans, + trans.security.encode_id( repository.id ), + changeset_revision ) + if repository_metadata: + # If changeset_revision is in the repository_metadata table for this + # repository, then we know there are no additional updates for the tools. + url += changeset_revision + else: + # The changeset_revision column in the repository_metadata table has been + # updated with a new changeset_revision value since the repository was cloned. + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + # Load each tool in the repository's changeset_revision to generate a list of + # tool guids, since guids differentiate tools by id and version. + ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + if ctx is not None: + tool_guids = [] + for filename in ctx: + # Find all tool configs in this repository changeset_revision. + if filename.endswith( '.xml' ): + fctx = ctx[ filename ] + # Write the contents of the old tool config to a temporary file. + fh = tempfile.NamedTemporaryFile( 'w' ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'w' ) + fh.write( fctx.data() ) + fh.close() + try: + tool = load_tool( trans, tmp_filename ) + if tool is not None: + tool_guids.append( generate_tool_guid( trans, repository, tool ) ) + except: + # File must not be a valid tool config even though it has a .xml extension. + pass + try: + os.unlink( tmp_filename ) + except: + pass + tool_guids.sort() + if tool_guids: + # Compare our list of tool guids against those in each repository_metadata record + # for the repository to find the repository_metadata record with the changeset_revision + # value we want to pass back to the caller. + found = False + for repository_metadata in get_repository_metadata_by_repository_id( trans, trans.security.encode_id( repository.id ) ): + metadata = repository_metadata.metadata + metadata_tool_guids = [] + for tool_dict in metadata[ 'tools' ]: + metadata_tool_guids.append( tool_dict[ 'guid' ] ) + metadata_tool_guids.sort() + if tool_guids == metadata_tool_guids: + # We've found the repository_metadata record whose changeset_revision + # value has been updated. + url += repository_metadata.changeset_revision + found = True + break + if not found: + # There must be a problem in the data, so we'll just send back the received changeset_revision. + log.debug( "Possible data corruption - updated repository_metadata cannot be found for repository id %d." % repository.id ) + url += changeset_revision + else: + # There are not tools in the changeset_revision, so no tool updates are possible. + url += changeset_revision return trans.response.send_redirect( url ) @web.expose def browse_repositories( self, trans, **kwd ): --- a/templates/admin/select_tool_panel_section.mako Fri Sep 30 09:26:22 2011 -0400 +++ b/templates/admin/select_tool_panel_section.mako Fri Sep 30 09:48:56 2011 -0400 @@ -23,9 +23,9 @@ <br/><div class="toolForm"> - <div class="toolFormTitle">Install tools from repository '${repository_name}'</div> + <div class="toolFormTitle">Install tools from repository '${name}'</div><div class="toolFormBody"> - <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin', action='install_tool_shed_repository', tool_shed_url=tool_shed_url, repository_name=repository_name, changeset_revision=changeset_revision, repository_clone_url=repository_clone_url )}" method="post" > + <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin', action='install_tool_shed_repository', tool_shed_url=tool_shed_url, name=name, description=description, changeset_revision=changeset_revision, repository_clone_url=repository_clone_url )}" method="post" > %if shed_tool_conf_select_field: <div class="form-row"><label>Shed tool configuration file:</label> --- a/templates/webapps/galaxy/admin/index.mako Fri Sep 30 09:26:22 2011 -0400 +++ b/templates/webapps/galaxy/admin/index.mako Fri Sep 30 09:48:56 2011 -0400 @@ -62,6 +62,9 @@ <div class="toolTitle"><a href="${h.url_for( controller='admin', action='reload_tool' )}" target="galaxy_main">Reload a tool's configuration</a></div><div class="toolTitle"><a href="${h.url_for( controller='admin', action='memdump' )}" target="galaxy_main">Profile memory usage</a></div><div class="toolTitle"><a href="${h.url_for( controller='admin', action='jobs' )}" target="galaxy_main">Manage jobs</a></div> + %if cloned_repositories: + <div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_repositories' )}" target="galaxy_main">Manage cloned tool shed repositories</a></div> + %endif </div></div> %if trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.