[galaxy-commits] commit/galaxy-central: greg: Refinements for installing tool shed repositories and tool dependencies.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/cc5fea2e382e/ changeset: cc5fea2e382e user: greg date: 2012-06-29 21:44:49 summary: Refinements for installing tool shed repositories and tool dependencies. affected #: 18 files diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2647,15 +2647,23 @@ pass class ToolShedRepository( object ): - installation_status = Bunch( CLONED='cloned', - SETTING_TOOL_VERSIONS='setting tool versions', - INSTALLING_TOOL_DEPENDENCIES='installing tool dependencies', - INSTALLED='installed', - ERROR='error', - UNINSTALLED='uninstalled' ) + installation_status = Bunch( NEW='New', + CLONING='Cloning', + SETTING_TOOL_VERSIONS='Setting tool versions', + INSTALLING_TOOL_DEPENDENCIES='Installing tool dependencies', + LOADING_PROPRIETARY_DATATYPES='Loading proprietary datatypes', + INSTALLED='Installed', + DEACTIVATED='Deactivated', + ERROR='Error', + UNINSTALLED='Uninstalled' ) + states = Bunch( INSTALLING = 'running', + OK = 'ok', + WARNING = 'queued', + ERROR = 'error', + UNINSTALLED = 'deleted_new' ) def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, installed_changeset_revision=None, changeset_revision=None, ctx_rev=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False, - uninstalled=False, dist_to_shed=False ): + uninstalled=False, dist_to_shed=False, status=None, error_message=None ): self.id = id self.create_time = create_time self.tool_shed = tool_shed @@ -2671,6 +2679,8 @@ self.deleted = deleted self.uninstalled = uninstalled self.dist_to_shed = dist_to_shed + self.status = status + self.error_message = error_message def repo_files_directory( self, app ): repo_path = self.repo_path( app ) if repo_path: @@ -2723,6 +2733,16 @@ dependencies_being_installed.append( tool_dependency ) return dependencies_being_installed @property + def tool_dependencies_missing_or_being_installed( self ): + dependencies_missing_or_being_installed = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status in [ ToolDependency.installation_status.ERROR, + ToolDependency.installation_status.INSTALLING, + ToolDependency.installation_status.NEVER_INSTALLED, + ToolDependency.installation_status.UNINSTALLED ]: + dependencies_missing_or_being_installed.append( tool_dependency ) + return dependencies_missing_or_being_installed + @property def tool_dependencies_with_installation_errors( self ): dependencies_with_installation_errors = [] for tool_dependency in self.tool_dependencies: @@ -2746,6 +2766,7 @@ UNINSTALLED='Uninstalled' ) states = Bunch( INSTALLING = 'running', OK = 'ok', + WARNING = 'queued', ERROR = 'error', UNINSTALLED = 'deleted_new' ) def __init__( self, tool_shed_repository_id=None, name=None, version=None, type=None, status=None, error_message=None ): diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -388,7 +388,9 @@ Column( "update_available", Boolean, default=False ), Column( "deleted", Boolean, index=True, default=False ), Column( "uninstalled", Boolean, default=False ), - Column( "dist_to_shed", Boolean, default=False ) ) + Column( "dist_to_shed", Boolean, default=False ), + Column( "status", TrimmedString( 255 ) ), + Column( "error_message", TEXT ) ) ToolDependency.table = Table( "tool_dependency", metadata, Column( "id", Integer, primary_key=True ), diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 lib/galaxy/model/migrate/versions/0103_add_tool_shed_repository_status_columns.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0103_add_tool_shed_repository_status_columns.py @@ -0,0 +1,68 @@ +"""Migration script to add status and error_message columns to the tool_shed_repository table.""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + # Add the status column to the tool_shed_repository table. + col = Column( "status", TrimmedString( 255 ) ) + try: + col.create( ToolShedRepository_table ) + assert col is ToolShedRepository_table.c.status + except Exception, e: + print "Adding status column to the tool_shed_repository table failed: %s" % str( e ) + # Add the error_message column to the tool_shed_repository table. + col = Column( "error_message", TEXT ) + try: + col.create( ToolShedRepository_table ) + assert col is ToolShedRepository_table.c.error_message + except Exception, e: + print "Adding error_message column to the tool_shed_repository table failed: %s" % str( e ) + # Update the status column value for tool_shed_repositories to the default value 'Installed'. + cmd = "UPDATE tool_shed_repository SET status = 'Installed';" + try: + db_session.execute( cmd ) + except Exception, e: + print "Exception executing sql command: " + print cmd + print str( e ) + # Update the status column for tool_shed_repositories that have been uninstalled. + cmd = "UPDATE tool_shed_repository SET status = 'Uninstalled' WHERE uninstalled;" + try: + db_session.execute( cmd ) + except Exception, e: + print "Exception executing sql command: " + print cmd + print str( e ) + # Update the status column for tool_shed_repositories that have been deactivated. + cmd = "UPDATE tool_shed_repository SET status = 'Deactivated' where deleted and not uninstalled;" + try: + db_session.execute( cmd ) + except Exception, e: + print "Exception executing sql command: " + print cmd + print str( e ) +def downgrade(): + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.status.drop() + except Exception, e: + print "Dropping column status from the tool_shed_repository table failed: %s" % str( e ) + try: + ToolShedRepository_table.c.error_message.drop() + except Exception, e: + print "Dropping column error_message from the tool_shed_repository table failed: %s" % str( e ) diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -119,12 +119,8 @@ if not is_displayed: is_displayed = True return is_displayed, tool_sections - def handle_repository_contents( self, repository_clone_url, relative_install_dir, repository_elem, repository_name, description, - installed_changeset_revision, ctx_rev, install_dependencies ): - # Generate the metadata for the installed tool shed repository, among other things. It is critical that the installed repository is - # updated to the desired changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. - # The values for the keys in each of the following dictionaries will be a list to allow for the same tool to be displayed in multiple places - # in the tool panel. + def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies ): + """Generate the metadata for the installed tool shed repository, among other things.""" tool_panel_dict_for_display = odict() for tool_elem in repository_elem: # The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" /> @@ -137,21 +133,12 @@ for k, v in tool_panel_dict_for_tool_config.items(): tool_panel_dict_for_display[ k ] = v metadata_dict = generate_metadata_using_disk_files( self.toolbox, relative_install_dir, repository_clone_url ) - # Add a new record to the tool_shed_repository table if one doesn't already exist. If one exists but is marked - # deleted, undelete it. It is critical that this happens before the call to add_to_tool_panel() below because - # tools will not be properly loaded if the repository is marked deleted. - print "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % repository_name - tool_shed_repository = create_or_update_tool_shed_repository( self.app, - repository_name, - description, - installed_changeset_revision, - ctx_rev, - repository_clone_url, - metadata_dict, - dist_to_shed=True ) + tool_shed_repository.metadata = metadata_dict + self.app.sa_session.add( tool_shed_repository ) + self.app.sa_session.flush() if 'tool_dependencies' in metadata_dict: - # All tool_dependency objects must be created before the tools are processed no matter whether tool dependencies are going to be installed. - tool_dependencies = create_tool_dependency_objects( self.app, tool_shed_repository, installed_changeset_revision ) + # All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed. + tool_dependencies = create_tool_dependency_objects( self.app, tool_shed_repository, tool_shed_repository.installed_changeset_revision ) else: tool_dependencies = None if 'tools' in metadata_dict: @@ -162,7 +149,7 @@ # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = handle_missing_data_table_entry( self.app, tool_shed_repository, - installed_changeset_revision, + tool_shed_repository.installed_changeset_revision, self.tool_path, repository_tools_tups, work_dir ) @@ -171,13 +158,16 @@ # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. copy_sample_files( self.app, sample_files, sample_files_copied=sample_files_copied ) if install_dependencies and tool_dependencies and 'tool_dependencies' in metadata_dict: + # Install tool dependencies. + update_tool_shed_repository_status( self.app, + tool_shed_repository, + self.app.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) # Get the tool_dependencies.xml file from the repository. tool_dependencies_config = get_config_from_repository( self.app, 'tool_dependencies.xml', tool_shed_repository, - installed_changeset_revision, + tool_shed_repository.installed_changeset_revision, work_dir ) - # Install tool dependencies. installed_tool_dependencies = handle_tool_dependencies( app=self.app, tool_shed_repository=tool_shed_repository, tool_dependencies_config=tool_dependencies_config, @@ -187,9 +177,9 @@ print '\nThe following error occurred from the InstallManager while installing tool dependency ', installed_tool_dependency.name, ':' print installed_tool_dependency.error_message, '\n\n' add_to_tool_panel( self.app, - repository_name, + tool_shed_repository.name, repository_clone_url, - installed_changeset_revision, + tool_shed_repository.installed_changeset_revision, repository_tools_tups, self.repository_owner, self.migrated_tools_config, @@ -200,11 +190,14 @@ except: pass if 'datatypes' in metadata_dict: + update_tool_shed_repository_status( self.app, + tool_shed_repository, + self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ) work_dir = make_tmp_directory() datatypes_config = get_config_from_repository( self.app, 'datatypes_conf.xml', tool_shed_repository, - installed_changeset_revision, + tool_shed_repository.installed_changeset_revision, work_dir ) # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started # after this installation completes. @@ -212,9 +205,9 @@ if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, - name=repository_name, + name=tool_shed_repository.name, owner=self.repository_owner, - installed_changeset_revision=installed_changeset_revision, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, tool_dicts=metadata_dict.get( 'tools', [] ), converter_path=converter_path, display_path=display_path ) @@ -228,7 +221,6 @@ shutil.rmtree( work_dir ) except: pass - return tool_shed_repository, metadata_dict def install_repository( self, repository_elem, install_dependencies ): # Install a single repository, loading contained tools into the tool panel. name = repository_elem.get( 'name' ) @@ -243,16 +235,31 @@ repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) relative_install_dir = os.path.join( clone_dir, name ) ctx_rev = get_ctx_rev( tool_shed_url, name, self.repository_owner, installed_changeset_revision ) + print "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % name + tool_shed_repository = create_or_update_tool_shed_repository( app=self.app, + name=name, + description=description, + installed_changeset_revision=installed_changeset_revision, + ctx_rev=ctx_rev, + repository_clone_url=repository_clone_url, + metadata_dict={}, + status=self.app.model.ToolShedRepository.installation_status.NEW, + current_changeset_revision=None, + owner=self.repository_owner, + dist_to_shed=True ) + update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.CLONING ) clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev ) - tool_shed_repository, metadata_dict = self.handle_repository_contents( repository_clone_url, - relative_install_dir, - repository_elem, - name, - description, - installed_changeset_revision, - ctx_rev, - install_dependencies ) + self.handle_repository_contents( tool_shed_repository=tool_shed_repository, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + repository_elem=repository_elem, + install_dependencies=install_dependencies ) + self.app.sa_session.refresh( tool_shed_repository ) + metadata_dict = tool_shed_repository.metadata if 'tools' in metadata_dict: + update_tool_shed_repository_status( self.app, + tool_shed_repository, + self.app.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) # Get the tool_versions from the tool shed for each tool in the installed change set. url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ ( tool_shed_url, tool_shed_repository.name, self.repository_owner, installed_changeset_revision ) @@ -291,6 +298,7 @@ parent_id=tool_version_using_old_id.id ) self.app.sa_session.add( tool_version_association ) self.app.sa_session.flush() + update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLED ) @property def non_shed_tool_panel_configs( self ): # Get the non-shed related tool panel config file names from the Galaxy config - the default is tool_conf.xml. diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -279,11 +279,11 @@ def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" commands.clone( get_configured_ui(), - repository_clone_url, - dest=repository_file_dir, + str( repository_clone_url ), + dest=str( repository_file_dir ), pull=True, noupdate=False, - rev=[ ctx_rev ] ) + rev=util.listify( str( ctx_rev ) ) ) def copy_sample_file( app, filename, dest_path=None ): """Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc. The default value for dest_path is ~/tool-data.""" if dest_path is None: @@ -313,7 +313,7 @@ converter_path=converter_path, display_path=display_path ) def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, - current_changeset_revision=None, owner='', dist_to_shed=False ): + status, current_changeset_revision=None, owner='', dist_to_shed=False ): # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used # to be in the Galaxy distribution, but have been moved to the main Galaxy tool shed. if current_changeset_revision is None: @@ -337,8 +337,17 @@ tool_shed_repository.ctx_rev = ctx_rev tool_shed_repository.metadata = metadata_dict tool_shed_repository.includes_datatypes = includes_datatypes - tool_shed_repository.deleted = False - tool_shed_repository.uninstalled = False + tool_shed_repository.status = status + if status in [ app.model.ToolShedRepository.installation_status.NEW, + app.model.ToolShedRepository.installation_status.CLONING, + app.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, + app.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, + app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES, + app.model.ToolShedRepository.installation_status.INSTALLED, + app.model.ToolShedRepository.installation_status.ERROR ]: + tool_shed_repository.deleted = False + if status not in [ app.model.ToolShedRepository.installation_status.UNINSTALLED ]: + tool_shed_repository.uninstalled = False else: tool_shed_repository = app.model.ToolShedRepository( tool_shed=tool_shed, name=name, @@ -349,7 +358,8 @@ ctx_rev=ctx_rev, metadata=metadata_dict, includes_datatypes=includes_datatypes, - dist_to_shed=dist_to_shed ) + dist_to_shed=dist_to_shed, + status=status ) sa_session.add( tool_shed_repository ) sa_session.flush() return tool_shed_repository @@ -1255,87 +1265,6 @@ def load_installed_display_applications( installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate ) -def load_repository_contents( trans, repository_name, description, owner, installed_changeset_revision, current_changeset_revision, ctx_rev, - tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None ): - """ - Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy (never the tool shed) - when an admin is installing a new repository or reinstalling an uninstalled repository. - """ - metadata_dict = generate_metadata_using_disk_files( trans.app.toolbox, relative_install_dir, repository_clone_url ) - # Add a new record to the tool_shed_repository table if one doesn't already exist. If one exists but is marked deleted, undelete it. This - # must happen before the call to add_to_tool_panel() below because tools will not be properly loaded if the repository is marked deleted. - log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % repository_name ) - tool_shed_repository = create_or_update_tool_shed_repository( trans.app, - repository_name, - description, - installed_changeset_revision, - ctx_rev, - repository_clone_url, - metadata_dict, - current_changeset_revision=current_changeset_revision, - owner='', - dist_to_shed=False ) - if 'tools' in metadata_dict: - tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) - repository_tools_tups = get_repository_tools_tups( trans.app, metadata_dict ) - if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - work_dir = make_tmp_directory() - repository_tools_tups = handle_missing_data_table_entry( trans.app, - tool_shed_repository, - current_changeset_revision, - tool_path, - repository_tools_tups, - work_dir ) - # Handle missing index files for tool parameters that are dynamically generated select lists. - sample_files = metadata_dict.get( 'sample_files', [] ) - repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) - # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. - copy_sample_files( trans.app, sample_files, sample_files_copied=sample_files_copied ) - add_to_tool_panel( app=trans.app, - repository_name=repository_name, - repository_clone_url=repository_clone_url, - changeset_revision=current_changeset_revision, - repository_tools_tups=repository_tools_tups, - owner=owner, - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=True ) - try: - shutil.rmtree( work_dir ) - except: - pass - if 'datatypes' in metadata_dict: - work_dir = make_tmp_directory() - datatypes_config = get_config_from_repository( trans.app, - 'datatypes_conf.xml', - tool_shed_repository, - current_changeset_revision, - work_dir ) - # Load data types required by tools. - converter_path, display_path = alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, relative_install_dir, override=False ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=repository_name, - owner=owner, - installed_changeset_revision=installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - if converter_path: - # Load proprietary datatype converters - trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) - if display_path: - # Load proprietary datatype display applications - trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - try: - shutil.rmtree( work_dir ) - except: - pass - if 'tool_dependencies' in metadata_dict: - tool_dependencies = create_tool_dependency_objects( trans.app, tool_shed_repository, current_changeset_revision ) - return tool_shed_repository, metadata_dict def make_tmp_directory(): tmp_dir = os.getenv( 'TMPDIR', '' ) if tmp_dir: @@ -1582,3 +1511,8 @@ commands.update( get_configured_ui(), repo, rev=ctx_rev ) +def update_tool_shed_repository_status( app, tool_shed_repository, status ): + sa_session = app.model.context.current + tool_shed_repository.status = status + sa_session.add( tool_shed_repository ) + sa_session.flush() diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 lib/galaxy/web/controllers/admin_toolshed.py --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -7,17 +7,17 @@ from galaxy import eggs, tools eggs.require( 'mercurial' ) -from mercurial import hg +from mercurial import hg, ui, commands log = logging.getLogger( __name__ ) MAX_CONTENT_SIZE = 32768 -class RepositoryListGrid( grids.Grid ): +class InstalledRepositoryGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): if tool_shed_repository.update_available: - return '<div class="count-box state-color-error">%s</div>' % tool_shed_repository.name + return '<div class="count-box state-color-running">%s</div>' % tool_shed_repository.name return tool_shed_repository.name class DescriptionColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): @@ -28,6 +28,30 @@ class RevisionColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): return tool_shed_repository.changeset_revision + class StatusColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + if tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.CLONING, + trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, + trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]: + bgcolor = trans.model.ToolShedRepository.states.INSTALLING + elif tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, + trans.model.ToolShedRepository.installation_status.UNINSTALLED ]: + bgcolor = trans.model.ToolShedRepository.states.UNINSTALLED + elif tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.ERROR ]: + bgcolor = trans.model.ToolShedRepository.states.ERROR + elif tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.DEACTIVATED ]: + bgcolor = trans.model.ToolShedRepository.states.WARNING + elif tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.INSTALLED ]: + if tool_shed_repository.missing_tool_dependencies: + bgcolor = trans.model.ToolShedRepository.states.WARNING + else: + bgcolor = trans.model.ToolShedRepository.states.OK + else: + bgcolor = trans.model.ToolShedRepository.states.ERROR + rval = '<div class="count-box state-color-%s">' % bgcolor + rval += '%s</div>' % tool_shed_repository.status + return rval class ToolShedColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): return tool_shed_repository.tool_shed @@ -44,6 +68,8 @@ DescriptionColumn( "Description" ), OwnerColumn( "Owner" ), RevisionColumn( "Revision" ), + StatusColumn( "Installation Status", + filterable="advanced" ), ToolShedColumn( "Tool shed" ), # Columns that are valid for filtering but are not visible. grids.DeletedColumn( "Deleted", @@ -77,6 +103,101 @@ def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( self.model_class ) +class RepositoryInstallationGrid( grids.Grid ): + class NameColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.name + class DescriptionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.description + class OwnerColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.owner + class RevisionColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + return tool_shed_repository.changeset_revision + class StatusColumn( grids.TextColumn ): + def get_value( self, trans, grid, tool_shed_repository ): + if tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.CLONING, + trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, + trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]: + bgcolor = trans.model.ToolShedRepository.states.INSTALLING + elif tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, + trans.model.ToolShedRepository.installation_status.UNINSTALLED ]: + bgcolor = trans.model.ToolShedRepository.states.UNINSTALLED + elif tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.ERROR ]: + bgcolor = trans.model.ToolShedRepository.states.ERROR + elif tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.DEACTIVATED ]: + bgcolor = trans.model.ToolShedRepository.states.WARNING + elif tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.INSTALLED ]: + if tool_shed_repository.missing_tool_dependencies: + bgcolor = trans.model.ToolShedRepository.states.WARNING + else: + bgcolor = trans.model.ToolShedRepository.states.OK + else: + bgcolor = trans.model.ToolShedRepository.states.ERROR + rval = '<div class="count-box state-color-%s" id="ToolShedRepositoryStatus-%s">' % ( bgcolor, trans.security.encode_id( tool_shed_repository.id ) ) + rval += '%s</div>' % tool_shed_repository.status + return rval + + webapp = "galaxy" + title = "Monitor installing tool shed repositories" + template = "admin/tool_shed_repository/repository_installation_grid.mako" + model_class = model.ToolShedRepository + default_sort_key = "-create_time" + num_rows_per_page = 50 + preserve_state = True + use_paging = False + columns = [ + NameColumn( "Name", + link=( lambda item: iff( item.status in \ + [ model.ToolShedRepository.installation_status.NEW, + model.ToolShedRepository.installation_status.CLONING, + model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, + model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, + model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES, + model.ToolShedRepository.installation_status.UNINSTALLED ], \ + None, dict( action="manage_repository", id=item.id ) ) ), + filterable="advanced" ), + DescriptionColumn( "Description", + filterable="advanced" ), + OwnerColumn( "Owner", + filterable="advanced" ), + RevisionColumn( "Revision", + filterable="advanced" ), + StatusColumn( "Installation Status", + filterable="advanced" ), + ] + operations = [] + def build_initial_query( self, trans, **kwd ): + clause_list = [] + tool_shed_repository_ids = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) + if tool_shed_repository_ids: + for tool_shed_repository_id in tool_shed_repository_ids: + clause_list.append( self.model_class.table.c.id == trans.security.decode_id( tool_shed_repository_id ) ) + if clause_list: + return trans.sa_session.query( self.model_class ) \ + .filter( or_( *clause_list ) ) + for tool_shed_repository in trans.sa_session.query( self.model_class ) \ + .filter( self.model_class.table.c.deleted == False ): + if tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, + trans.model.ToolShedRepository.installation_status.CLONING, + trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, + trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]: + clause_list.append( self.model_class.table.c.id == trans.security.decode_id( tool_shed_repository.id ) ) + if clause_list: + return trans.sa_session.query( self.model_class ) \ + .filter( or_( *clause_list ) ) + return trans.sa_session.query( self.model_class ) \ + .filter( self.model_class.table.c.status == trans.model.ToolShedRepository.installation_status.NEW ) + def apply_query_filter( self, trans, query, **kwd ): + tool_shed_repository_id = kwd.get( 'tool_shed_repository_id', None ) + if tool_shed_repository_id: + return query.filter_by( tool_shed_repository_id=trans.security.decode_id( tool_shed_repository_id ) ) + return query + class ToolDependencyGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_dependency ): @@ -90,18 +211,17 @@ class StatusColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_dependency ): if tool_dependency.status in [ trans.model.ToolDependency.installation_status.INSTALLING ]: - return tool_dependency.status - else: - if tool_dependency.status in [ trans.model.ToolDependency.installation_status.NEVER_INSTALLED, - trans.model.ToolDependency.installation_status.UNINSTALLED ]: - bgcolor = trans.model.ToolDependency.states.UNINSTALLED - elif tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR ]: - bgcolor = trans.model.ToolDependency.states.ERROR - elif tool_dependency.status in [ trans.model.ToolDependency.installation_status.INSTALLED ]: - bgcolor = trans.model.ToolDependency.states.OK - rval = '<div class="count-box state-color-%s" id="ToolDependencyStatus-%s">' % ( bgcolor, trans.security.encode_id( tool_dependency.id ) ) - rval += '%s</div>' % tool_dependency.status - return rval + bgcolor = trans.model.ToolDependency.states.INSTALLING + elif tool_dependency.status in [ trans.model.ToolDependency.installation_status.NEVER_INSTALLED, + trans.model.ToolDependency.installation_status.UNINSTALLED ]: + bgcolor = trans.model.ToolDependency.states.UNINSTALLED + elif tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR ]: + bgcolor = trans.model.ToolDependency.states.ERROR + elif tool_dependency.status in [ trans.model.ToolDependency.installation_status.INSTALLED ]: + bgcolor = trans.model.ToolDependency.states.OK + rval = '<div class="count-box state-color-%s" id="ToolDependencyStatus-%s">' % ( bgcolor, trans.security.encode_id( tool_dependency.id ) ) + rval += '%s</div>' % tool_dependency.status + return rval webapp = "galaxy" title = "Tool Dependencies" @@ -138,7 +258,7 @@ model.ToolDependency.installation_status.ERROR ] ) ) ] def build_initial_query( self, trans, **kwd ): - tool_dependency_ids = kwd.get( 'tool_dependency_ids', None ) + tool_dependency_ids = util.listify( kwd.get( 'tool_dependency_ids', None ) ) if tool_dependency_ids: clause_list = [] for tool_dependency_id in tool_dependency_ids: @@ -148,13 +268,14 @@ return trans.sa_session.query( self.model_class ) def apply_query_filter( self, trans, query, **kwd ): tool_dependency_id = kwd.get( 'tool_dependency_id', None ) - if not tool_dependency_id: - return query - return query.filter_by( tool_dependency_id=trans.security.decode_id( tool_dependency_id ) ) + if tool_dependency_id: + return query.filter_by( tool_dependency_id=trans.security.decode_id( tool_dependency_id ) ) + return query class AdminToolshed( AdminGalaxy ): - repository_list_grid = RepositoryListGrid() + installed_repository_grid = InstalledRepositoryGrid() + repository_installation_grid = RepositoryInstallationGrid() tool_dependency_grid = ToolDependencyGrid() @web.expose @@ -165,6 +286,7 @@ shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository ) repository_clone_url = self.__generate_clone_url( trans, repository ) repository.deleted = False + repository.status = trans.model.ToolShedRepository.installation_status.INSTALLED trans.sa_session.add( repository ) trans.sa_session.flush() if repository.includes_tools: @@ -220,8 +342,8 @@ if operation == "deactivate or uninstall": return self.deactivate_or_uninstall_repository( trans, **kwd ) if 'message' not in kwd or not kwd[ 'message' ]: - kwd[ 'message' ] = 'Names of repositories for which updates are available are highlighted in red.' - return self.repository_list_grid( trans, **kwd ) + kwd[ 'message' ] = 'Names of repositories for which updates are available are highlighted in yellow.' + return self.installed_repository_grid( trans, **kwd ) @web.expose @web.require_admin def browse_tool_dependency( self, trans, **kwd ): @@ -262,52 +384,23 @@ return trans.response.send_redirect( url ) @web.expose @web.require_admin - def confirm_tool_dependency_install( self, trans, **kwd ): - """Display a page enabling the Galaxy administrator to choose to install tool dependencies for a tool shed repository they are installing.""" - # This method is called from the tool shed (never Galaxy) when a tool shed repository that includes a file named tool_dependencies.xml - # is being installed into a local Galaxy instance. - message = kwd.get( 'message', '' ) - status = kwd.get( 'status', 'done' ) - tool_shed_url = kwd[ 'tool_shed_url' ] - repo_info_dict = kwd[ 'repo_info_dict' ] - includes_tools = util.string_as_bool( kwd.get( 'includes_tools', False ) ) - # Decode the encoded repo_info_dict param value. - dict_with_tool_dependencies = tool_shed_decode( repo_info_dict ) - # The repo_info_dict includes tool dependencies which we need to display so the user knows what will be installed. - new_repo_info_dict = {} - for name, repo_info_tuple in dict_with_tool_dependencies.items(): - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple - # Create a new repo_info_dict by eliminating tool-dependencies from the repo_info_tuple. - new_repo_info_dict[ name ] = ( description, repository_clone_url, changeset_revision, ctx_rev ) - repo_info_dict = tool_shed_encode( new_repo_info_dict ) - install_tool_dependencies_check_box = CheckboxField( 'install_tool_dependencies', checked=True ) - return trans.fill_template( '/admin/tool_shed_repository/confirm_tool_dependency_install.mako', - tool_shed_url=tool_shed_url, - repo_info_dict=repo_info_dict, - dict_with_tool_dependencies=dict_with_tool_dependencies, - includes_tools=includes_tools, - install_tool_dependencies_check_box=install_tool_dependencies_check_box, - message=message, - status=status ) - @web.expose - @web.require_admin def deactivate_or_uninstall_repository( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) remove_from_disk = params.get( 'remove_from_disk', '' ) remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) - repository = get_repository( trans, kwd[ 'id' ] ) - shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + tool_shed_repository = get_repository( trans, kwd[ 'id' ] ) + shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) repository_install_dir = os.path.abspath ( relative_install_dir ) errors = '' if params.get( 'deactivate_or_uninstall_repository_button', False ): - if repository.includes_tools: + if tool_shed_repository.includes_tools: # Handle tool panel alterations. - remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall=remove_from_disk_checked ) - if repository.includes_datatypes: + remove_from_tool_panel( trans, tool_shed_repository, shed_tool_conf, uninstall=remove_from_disk_checked ) + if tool_shed_repository.includes_datatypes: # Deactivate proprietary datatypes. - installed_repository_dict = load_installed_datatypes( trans.app, repository, repository_install_dir, deactivate=True ) + installed_repository_dict = load_installed_datatypes( trans.app, tool_shed_repository, repository_install_dir, deactivate=True ) if installed_repository_dict[ 'converter_path' ]: load_installed_datatype_converters( trans.app, installed_repository_dict, deactivate=True ) if installed_repository_dict[ 'display_path' ]: @@ -322,24 +415,28 @@ log.debug( "Error removing repository installation directory %s: %s" % ( str( repository_install_dir ), str( e ) ) ) removed = False if removed: - repository.uninstalled = True + tool_shed_repository.uninstalled = True # Remove all installed tool dependencies. - for tool_dependency in repository.installed_tool_dependencies: + for tool_dependency in tool_shed_repository.installed_tool_dependencies: uninstalled, error_message = remove_tool_dependency( trans, tool_dependency ) if error_message: errors = '%s %s' % ( errors, error_message ) - repository.deleted = True - trans.sa_session.add( repository ) + tool_shed_repository.deleted = True + if remove_from_disk_checked: + tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.UNINSTALLED + else: + tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.DEACTIVATED + trans.sa_session.add( tool_shed_repository ) trans.sa_session.flush() if remove_from_disk_checked: - message = 'The repository named <b>%s</b> has been uninstalled. ' % repository.name + message = 'The repository named <b>%s</b> has been uninstalled. ' % tool_shed_repository.name if errors: message += 'Attempting to uninstall tool dependencies resulted in errors: %s' % errors status = 'error' else: status = 'done' else: - message = 'The repository named <b>%s</b> has been deactivated. ' % repository.name + message = 'The repository named <b>%s</b> has been deactivated. ' % tool_shed_repository.name status = 'done' return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='browse_repositories', @@ -347,7 +444,7 @@ status=status ) ) remove_from_disk_check_box = CheckboxField( 'remove_from_disk', checked=remove_from_disk_checked ) return trans.fill_template( '/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako', - repository=repository, + repository=tool_shed_repository, remove_from_disk_check_box=remove_from_disk_check_box, message=message, status=status ) @@ -365,6 +462,20 @@ galaxy_url = url_for( '/', qualified=True ) url = '%srepository/find_workflows?galaxy_url=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, galaxy_url ) return trans.response.send_redirect( url ) + def generate_tool_path( self, repository_clone_url, changeset_revision ): + """ + Generate a tool path that guarantees repositories with the same name will always be installed + in different directories. The tool path will be of the form: + <tool shed url>/repos/<repository owner>/<repository name>/<installed changeset revision> + http://test@bx.psu.edu:9009/repos/test/filter + """ + tmp_url = clean_repository_clone_url( repository_clone_url ) + # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column + items = tmp_url.split( 'repos' ) + tool_shed_url = items[ 0 ] + repo_path = items[ 1 ] + tool_shed_url = clean_tool_shed_url( tool_shed_url ) + return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) @web.json @web.require_admin def get_file_contents( self, trans, file_path ): @@ -374,6 +485,23 @@ return get_repository_file_contents( file_path ) @web.expose @web.require_admin + def initiate_repository_installation( self, trans, shed_repository_ids, encoded_kwd ): + tsr_ids = util.listify( shed_repository_ids ) + tool_shed_repositories = [] + for tsr_id in tsr_ids: + tsr = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) + tool_shed_repositories.append( tsr ) + clause_list = [] + for tsr_id in tsr_ids: + clause_list.append( trans.model.ToolShedRepository.table.c.id == trans.security.decode_id( tsr_id ) ) + query = trans.sa_session.query( trans.model.ToolShedRepository ).filter( or_( *clause_list ) ) + return trans.fill_template( 'admin/tool_shed_repository/initiate_repository_installation.mako', + encoded_kwd=encoded_kwd, + query=query, + tool_shed_repositories=tool_shed_repositories, + initiate_repository_installation_ids=shed_repository_ids ) + @web.expose + @web.require_admin def initiate_tool_dependency_installation( self, trans, tool_dependencies ): """Install specified dependencies for repository tools.""" # Get the tool_shed_repository from one of the tool_dependencies. @@ -403,9 +531,10 @@ else: message = "Installed tool dependencies: %s" % ','.join( td.name for td in installed_tool_dependencies ) status = 'done' + td_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies ] return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='manage_tool_dependencies', - tool_dependency_ids=tool_dependency_ids, + tool_dependency_ids=td_ids, message=message, status=status ) ) @web.expose @@ -423,22 +552,33 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) tool_shed_url = kwd[ 'tool_shed_url' ] - repo_info_dict = kwd[ 'repo_info_dict' ] new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) tool_panel_section = kwd.get( 'tool_panel_section', '' ) includes_tools = util.string_as_bool( kwd.get( 'includes_tools', False ) ) + includes_tool_dependencies = util.string_as_bool( kwd.get( 'includes_tool_dependencies', False ) ) + if includes_tool_dependencies: + dict_with_tool_dependencies = tool_shed_decode( kwd[ 'repo_info_dict' ] ) + # The repo_info_dict includes tool dependencies which we need to display so the user knows what will be installed. + repo_info_dict = {} + for name, repo_info_tuple in dict_with_tool_dependencies.items(): + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple + # Create a new repo_info_dict by eliminating tool_dependencies from the repo_info_tuple. + repo_info_dict[ name ] = ( description, repository_clone_url, changeset_revision, ctx_rev ) + else: + dict_with_tool_dependencies = {} + repo_info_dict = tool_shed_decode( kwd[ 'repo_info_dict' ] ) install_tool_dependencies = kwd.get( 'install_tool_dependencies', '' ) + tool_section = None if not includes_tools or ( includes_tools and kwd.get( 'select_tool_panel_section_button', False ) ): if includes_tools: install_tool_dependencies = CheckboxField.is_checked( install_tool_dependencies ) shed_tool_conf = kwd[ 'shed_tool_conf' ] else: install_tool_dependencies = False - # If installing a repository that includes no tools, get the relative tool_path from the file - # to which the migrated_tools_config setting points. + # If installing a repository that includes no tools, get the relative tool_path from the file to which the + # migrated_tools_config setting points. shed_tool_conf = trans.app.config.migrated_tools_config - # Get the tool path by searching the list of shed_tool_confs for the dictionary - # that contains the information about shed_tool_conf. + # Get the tool path by searching the list of shed_tool_confs for the dictionary that contains the information about shed_tool_conf. for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: config_filename = shed_tool_conf_dict[ 'config_filename' ] if config_filename == shed_tool_conf: @@ -471,79 +611,62 @@ tool_section = trans.app.toolbox.tool_panel[ section_key ] else: tool_section = None - # Decode the encoded repo_info_dict param value. - repo_info_dict = tool_shed_decode( repo_info_dict ) - # Clone the repository to the configured location. - installed_repository_names = [] + # Make sure all tool_shed_repository records exist. + new_tool_shed_repositories = [] for name, repo_info_tuple in repo_info_dict.items(): description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple - clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) + clone_dir = os.path.join( tool_path, self.generate_tool_path( repository_clone_url, changeset_revision ) ) relative_install_dir = os.path.join( clone_dir, name ) - if os.path.exists( clone_dir ): - # Repository and revision has already been cloned. - message += 'Revision <b>%s</b> of repository <b>%s</b> was previously installed.<br/>' % ( changeset_revision, name ) + owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) + # Make sure the repository was not already installed. + installed_changeset_revision = self.repository_was_previously_installed( trans, tool_shed_url, name, repo_info_tuple, clone_dir ) + if installed_changeset_revision: + message = "Tool shed repository <b>%s</b> with owner <b>%s</b> and changeset revision <b>%s</b> " % ( name, owner, changeset_revision ) + message += "was previously installed using changeset revision <b>%s</b>. You can get " % installed_changeset_revision + message += "the latest updates for the installed repository using the <b>Get updates</b> option from the repository's " + message += "<b>Repository Actions</b> pop-up menu." + status = 'error' else: - clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev ) - owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) - tool_shed = clean_tool_shed_url( tool_shed_url ) - tool_shed_repository, metadata_dict, = load_repository_contents( trans, - repository_name=name, - description=description, - owner=owner, - installed_changeset_revision=changeset_revision, - current_changeset_revision=changeset_revision, - ctx_rev=ctx_rev, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - tool_shed=tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf ) - if 'tools' in metadata_dict: - # Get the tool_versions from the tool shed for each tool in the installed change set. - url = '%srepository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ - ( tool_shed_url, name, owner, changeset_revision ) - response = urllib2.urlopen( url ) - text = response.read() - response.close() - if text: - tool_version_dicts = from_json_string( text ) - handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository ) - else: - message += "Version information for the tools included in the <b>%s</b> repository is missing. " % name - message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions " - message += "from the installed repository's <b>Repository Actions</b> menu. " - status = 'error' - installed_repository_names.append( name ) - if installed_repository_names: - installed_repository_names.sort() - num_repositories_installed = len( installed_repository_names ) - if tool_section: - message += 'Installed %d %s and all tools were loaded into tool panel section <b>%s</b>:<br/>Installed repositories: ' % \ - ( num_repositories_installed, - inflector.cond_plural( num_repositories_installed, 'repository' ), - tool_section.name ) - else: - message += 'Installed %d %s and all tools were loaded into the tool panel outside of any sections.<br/>Installed repositories: ' % \ - ( num_repositories_installed, - inflector.cond_plural( num_repositories_installed, 'repository' ) ) - for i, repo_name in enumerate( installed_repository_names ): - if i == len( installed_repository_names ) -1: - message += '%s.<br/>' % repo_name - else: - message += '%s, ' % repo_name - if install_tool_dependencies: - tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.missing_tool_dependencies ] + print "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % name + tool_shed_repository = create_or_update_tool_shed_repository( app=trans.app, + name=name, + description=description, + installed_changeset_revision=changeset_revision, + ctx_rev=ctx_rev, + repository_clone_url=repository_clone_url, + metadata_dict={}, + status=trans.model.ToolShedRepository.installation_status.NEW, + current_changeset_revision=changeset_revision, + owner=owner, + dist_to_shed=False ) + new_tool_shed_repositories.append( tool_shed_repository ) + if new_tool_shed_repositories: + tsrids_list = [ trans.security.encode_id( tsr.id ) for tsr in new_tool_shed_repositories ] + new_kwd = dict( includes_tool_dependencies=kwd.get( 'includes_tool_dependencies', False ), + includes_tools=kwd.get( 'includes_tools', False ), + install_tool_dependencies=install_tool_dependencies, + repo_info_dict=kwd[ 'repo_info_dict' ], + message=message, + new_tool_panel_section=kwd.get( 'new_tool_panel_section', '' ), + shed_tool_conf = kwd[ 'shed_tool_conf' ], + status=status, + tool_panel_section=kwd.get( 'tool_panel_section', '' ), + tool_path=tool_path, + tool_section=tool_section, + tool_shed_repository_ids=tsrids_list, + tool_shed_url=kwd[ 'tool_shed_url' ] ) + encoded_kwd = tool_shed_encode( new_kwd ) + tsrids_str = ','.join( tsrids_list ) return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='manage_tool_dependencies', - operation='install', - tool_dependency_ids=tool_dependency_ids, - status=status, - message=message ) ) - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='browse_repositories', - message=message, - status=status ) ) + action='initiate_repository_installation', + shed_repository_ids=tsrids_str, + encoded_kwd=encoded_kwd ) ) + else: + kwd[ 'message' ] = message + kwd[ 'status' ] = status + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='manage_repositories', + **kwd ) ) if len( trans.app.toolbox.shed_tool_confs ) > 1: shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) shed_tool_conf = None @@ -553,35 +676,35 @@ shed_tool_conf = shed_tool_conf.replace( './', '', 1 ) shed_tool_conf_select_field = None tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) - if includes_tools: + if includes_tools and len( repo_info_dict ) == 1: # If we're installing a single repository that contains a readme file, get it's contents to display. - decoded_repo_info_dict = tool_shed_decode( repo_info_dict ) - if len( decoded_repo_info_dict ) == 1: - name = decoded_repo_info_dict.keys()[ 0 ] - repo_info_tuple = decoded_repo_info_dict[ name ] - description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple - owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) - url = '%srepository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, name, owner, changeset_revision ) - response = urllib2.urlopen( url ) - raw_text = response.read() - response.close() - readme_text = '' - for i, line in enumerate( raw_text ): - readme_text = '%s%s' % ( readme_text, to_html_str( line ) ) - if len( readme_text ) > MAX_CONTENT_SIZE: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) - readme_text = '%s%s' % ( readme_text, to_html_str( large_str ) ) - break - else: - readme_text = '' + name = repo_info_dict.keys()[ 0 ] + repo_info_tuple = repo_info_dict[ name ] + description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple + repository_owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) + url = '%srepository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ + ( tool_shed_url, name, repository_owner, changeset_revision ) + response = urllib2.urlopen( url ) + raw_text = response.read() + response.close() + readme_text = '' + for i, line in enumerate( raw_text ): + readme_text = '%s%s' % ( readme_text, to_html_str( line ) ) + if len( readme_text ) > MAX_CONTENT_SIZE: + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) + readme_text = '%s%s' % ( readme_text, to_html_str( large_str ) ) + break else: - readme_text = '' + readme_text = '' + install_tool_dependencies_check_box = CheckboxField( 'install_tool_dependencies', checked=True ) return trans.fill_template( '/admin/tool_shed_repository/select_tool_panel_section.mako', tool_shed_url=tool_shed_url, - repo_info_dict=repo_info_dict, + repo_info_dict=kwd[ 'repo_info_dict' ], + dict_with_tool_dependencies=dict_with_tool_dependencies, shed_tool_conf=shed_tool_conf, includes_tools=includes_tools, - install_tool_dependencies=install_tool_dependencies, + includes_tool_dependencies=includes_tool_dependencies, + install_tool_dependencies_check_box=install_tool_dependencies_check_box, shed_tool_conf_select_field=shed_tool_conf_select_field, tool_panel_section_select_field=tool_panel_section_select_field, new_tool_panel_section=new_tool_panel_section, @@ -610,10 +733,10 @@ tool_dependencies_for_installation.append( tool_dependency ) if tool_dependencies_for_installation: # Redirect back to the ToolDependencyGrid before initiating installation. - tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in tool_dependencies_for_installation ] + encoded_tool_dependency_for_installation_ids = [ trans.security.encode_id( td.id ) for td in tool_dependencies_for_installation ] new_kwd = dict( action='manage_tool_dependencies', operation='initiate_tool_dependency_installation', - tool_dependency_ids=tool_dependency_ids, + tool_dependency_ids=encoded_tool_dependency_for_installation_ids, message=message, status=status ) return self.tool_dependency_grid( trans, **new_kwd ) @@ -631,12 +754,196 @@ status=status ) @web.expose @web.require_admin + def install_tool_shed_repositories( self, trans, tool_shed_repositories, **kwd ): + """Install specified tool shed repositories.""" + repo_info_dict = tool_shed_decode( kwd[ 'repo_info_dict' ] ) + tool_path = kwd[ 'tool_path' ] + includes_tool_dependencies = util.string_as_bool( kwd[ 'includes_tool_dependencies' ] ) + install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) + for tool_shed_repository in tool_shed_repositories: + # Clone each repository to the configured location. + update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING ) + repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] + if includes_tool_dependencies: + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple + else: + description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple + clone_dir = os.path.join( tool_path, self.generate_tool_path( repository_clone_url, tool_shed_repository.installed_changeset_revision ) ) + relative_install_dir = os.path.join( clone_dir, tool_shed_repository.name ) + clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev ) + self.handle_repository_contents( trans, + tool_shed_repository=tool_shed_repository, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + tool_shed=tool_shed_repository.tool_shed, + tool_section=kwd.get( 'tool_section', '' ), + shed_tool_conf=kwd.get( 'shed_tool_conf', '' ) ) + trans.sa_session.refresh( tool_shed_repository ) + metadata = tool_shed_repository.metadata + if 'tools' in metadata: + # Get the tool_versions from the tool shed for each tool in the installed change set. + update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) + url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ + ( tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + tool_version_dicts = from_json_string( text ) + handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository ) + else: + message += "Version information for the tools included in the <b>%s</b> repository is missing. " % name + message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions " + message += "from the installed repository's <b>Repository Actions</b> menu. " + status = 'error' + if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: + work_dir = make_tmp_directory() + # Install tool dependencies. + update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + # Get the tool_dependencies.xml file from the repository. + tool_dependencies_config = get_config_from_repository( trans.app, + 'tool_dependencies.xml', + tool_shed_repository, + tool_shed_repository.installed_changeset_revision, + work_dir ) + installed_tool_dependencies = handle_tool_dependencies( app=trans.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_shed_repository.tool_dependencies ) + try: + shutil.rmtree( work_dir ) + except: + pass + if 'datatypes' in metadata: + update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ) + work_dir = make_tmp_directory() + datatypes_config = get_config_from_repository( trans.app, + 'datatypes_conf.xml', + tool_shed_repository, + tool_shed_repository.installed_changeset_revision, + work_dir ) + # Load proprietary data types required by tools. + converter_path, display_path = alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, relative_install_dir, override=False ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + if converter_path: + # Load proprietary datatype converters + trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) + if display_path: + # Load proprietary datatype display applications + trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + try: + shutil.rmtree( work_dir ) + except: + pass + update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) + tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + tool_shed_repository_ids=tsr_ids_for_monitoring ) ) + def handle_repository_contents( self, trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, + tool_section=None, shed_tool_conf=None ): + """ + Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy (never the tool shed) + when an admin is installing a new repository or reinstalling an uninstalled repository. + """ + metadata_dict = generate_metadata_using_disk_files( trans.app.toolbox, relative_install_dir, repository_clone_url ) + tool_shed_repository.metadata = metadata_dict + trans.sa_session.add( tool_shed_repository ) + trans.sa_session.flush() + if 'tool_dependencies' in metadata_dict: + tool_dependencies = create_tool_dependency_objects( trans.app, tool_shed_repository, tool_shed_repository.installed_changeset_revision ) + if 'tools' in metadata_dict: + tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) + repository_tools_tups = get_repository_tools_tups( trans.app, metadata_dict ) + if repository_tools_tups: + # Handle missing data table entries for tool parameters that are dynamically generated select lists. + work_dir = make_tmp_directory() + repository_tools_tups = handle_missing_data_table_entry( trans.app, + tool_shed_repository, + tool_shed_repository.changeset_revision, + tool_path, + repository_tools_tups, + work_dir ) + # Handle missing index files for tool parameters that are dynamically generated select lists. + sample_files = metadata_dict.get( 'sample_files', [] ) + repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + copy_sample_files( trans.app, sample_files, sample_files_copied=sample_files_copied ) + add_to_tool_panel( app=trans.app, + repository_name=tool_shed_repository.name, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.changeset_revision, + repository_tools_tups=repository_tools_tups, + owner=tool_shed_repository.owner, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=True ) + try: + shutil.rmtree( work_dir ) + except: + pass + if 'datatypes' in metadata_dict: + update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ) + work_dir = make_tmp_directory() + datatypes_config = get_config_from_repository( trans.app, + 'datatypes_conf.xml', + tool_shed_repository, + tool_shed_repository.changeset_revision, + work_dir ) + # Load data types required by tools. + converter_path, display_path = alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, relative_install_dir, override=False ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + if converter_path: + # Load proprietary datatype converters + trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) + if display_path: + # Load proprietary datatype display applications + trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + try: + shutil.rmtree( work_dir ) + except: + pass + @web.expose + @web.require_admin def manage_repository( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) repository_id = kwd[ 'id' ] repository = get_repository( trans, repository_id ) + if repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, + trans.model.ToolShedRepository.installation_status.CLONING ]: + message = "The repository '%s' is not yet cloned, please try again..." + status = 'warning' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + **kwd ) ) description = util.restore_text( params.get( 'description', repository.description ) ) shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository ) repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) @@ -662,6 +969,62 @@ status=status ) @web.expose @web.require_admin + def manage_repositories( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + tsrid = params.get( 'tool_shed_repository_id', None ) + tsridslist = util.listify( params.get( 'tool_shed_repository_ids', None ) ) + if not tsridslist: + tsridslist = util.listify( params.get( 'id', None ) ) + if tsrid and tsrid not in tsridslist: + tsridslist.append( tsrid ) + if 'operation' in kwd: + operation = kwd[ 'operation' ].lower() + if not tsridslist: + message = 'Select at least 1 tool shed repository to %s.' % operation + kwd[ 'message' ] = message + kwd[ 'status' ] = 'error' + del kwd[ 'operation' ] + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='manage_repositories', + **kwd ) ) + if operation == 'browse': + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repository', + **kwd ) ) + elif operation == 'uninstall': + repositories_for_uninstallation = [] + for repository_id in tool_shed_repository_id: + repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( repository_id ) ) + if repository.status in [ trans.model.ToolShedRepository.installation_status.INSTALLED, + trans.model.ToolShedRepository.installation_status.ERROR ]: + repositories_for_uninstallation.append( repository ) + if repositories_for_uninstallation: + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='uninstall_repositories', + **kwd ) ) + else: + kwd[ 'message' ] = 'All selected tool shed repositories are already uninstalled.' + kwd[ 'status' ] = 'error' + elif operation == "install": + encoded_kwd = kwd[ 'encoded_kwd' ] + decoded_kwd = tool_shed_decode( encoded_kwd ) + tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] + repositories_for_installation = [] + for tsr_id in tsr_ids: + repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) + if repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, + trans.model.ToolShedRepository.installation_status.UNINSTALLED ]: + repositories_for_installation.append( repository ) + if repositories_for_installation: + self.install_tool_shed_repositories( trans, repositories_for_installation, **decoded_kwd ) + else: + kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' + kwd[ 'status' ] = 'error' + return self.repository_installation_grid( trans, **kwd ) + @web.expose + @web.require_admin def manage_tool_dependencies( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -672,7 +1035,6 @@ tool_dependency_ids = util.listify( params.get( 'id', None ) ) if tool_dependency_id and tool_dependency_id not in tool_dependency_ids: tool_dependency_ids.append( tool_dependency_id ) - tool_dependencies = [] # We need a tool_shed_repository, so get it from one of the tool_dependencies. tool_dependency = get_tool_dependency( trans, tool_dependency_ids[ 0 ] ) tool_shed_repository = tool_dependency.tool_shed_repository @@ -739,6 +1101,18 @@ kwd[ 'message' ] = 'All selected tool dependencies are already installed.' kwd[ 'status' ] = 'error' return self.tool_dependency_grid( trans, **kwd ) + @web.expose + @web.require_admin + def monitor_repository_installation( self, trans, **kwd ): + params = util.Params( kwd ) + tsrid = params.get( 'tool_shed_repository_id', None ) + tsridslist = util.listify( params.get( 'tool_shed_repository_ids', None ) ) + if not tsridslist: + tsridslist = util.listify( params.get( 'id', None ) ) + if tsrid and tsrid not in tsridslist: + tsridslist.append( tsrid ) + kwd[ 'tool_shed_repository_ids' ] = tsridslist + return self.repository_installation_grid( trans, **kwd ) @web.json @web.require_admin def open_folder( self, trans, folder_path ): @@ -752,31 +1126,32 @@ message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd[ 'id' ] - repository = get_repository( trans, repository_id ) + tool_shed_repository = get_repository( trans, repository_id ) no_changes = kwd.get( 'no_changes', '' ) no_changes_checked = CheckboxField.is_checked( no_changes ) install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) - shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository ) - repository_clone_url = generate_clone_url( trans, repository ) - clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, repository.installed_changeset_revision ) ) - relative_install_dir = os.path.join( clone_dir, repository.name ) - tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - if not repository.ctx_rev: + shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository ) + repository_clone_url = generate_clone_url( trans, tool_shed_repository ) + clone_dir = os.path.join( tool_path, self.generate_tool_path( repository_clone_url, tool_shed_repository.installed_changeset_revision ) ) + relative_install_dir = os.path.join( clone_dir, tool_shed_repository.name ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, tool_shed_repository ) + if not tool_shed_repository.ctx_rev: # The ctx_rev column was introduced late, so may be null for some installed ToolShedRepositories. - ctx_rev = get_ctx_rev( tool_shed_url, repository.name, repository.owner, repository.installed_changeset_revision ) + ctx_rev = get_ctx_rev( tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.installed_changeset_revision ) else: - ctx_rev = repository.ctx_rev + ctx_rev = tool_shed_repository.ctx_rev + update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING ) clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev ) # Since we're reinstalling the repository we need to find the latest changeset revision to which is can be updated. - current_changeset_revision, current_ctx_rev = get_update_to_changeset_revision_and_ctx_rev( trans, repository ) + current_changeset_revision, current_ctx_rev = get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository ) if current_ctx_rev != ctx_rev: repo = hg.repository( get_configured_ui(), path=os.path.abspath( relative_install_dir ) ) pull_repository( repo, repository_clone_url, current_changeset_revision ) update_repository( repo, ctx_rev=current_ctx_rev ) tool_section = None - if repository.includes_tools: + metadata = tool_shed_repository.metadata + if tool_shed_repository.includes_tools: # Get the location in the tool panel in which each tool was originally loaded. - metadata = repository.metadata if 'tool_panel_section' in metadata: tool_panel_dict = metadata[ 'tool_panel_section' ] if not tool_panel_dict: @@ -829,52 +1204,83 @@ tool_section = trans.app.toolbox.tool_panel[ section_key ] else: tool_section = None - tool_shed_repository, metadata_dict, load_repository_contents( trans, - repository_name=repository.name, - description=repository.description, - owner=repository.owner, - installed_changeset_revision=repository.installed_changeset_revision, - current_changeset_revision=current_changeset_revision, - ctx_rev=ctx_rev, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - tool_shed=repository.tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf ) - repository.uninstalled = False - repository.deleted = False - trans.sa_session.add( repository ) + self.handle_repository_contents( trans, + tool_shed_repository=tool_shed_repository, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + tool_shed=tool_shed_repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf ) + + if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: + # Install tool dependencies. + update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + work_dir = make_tmp_directory() + # Get the tool_dependencies.xml file from the repository. + tool_dependencies_config = get_config_from_repository( trans.app, + 'tool_dependencies.xml', + tool_shed_repository, + tool_shed_repository.installed_changeset_revision, + work_dir ) + installed_tool_dependencies = handle_tool_dependencies( app=trans.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_dependencies ) + tool_shed_repository.uninstalled = False + tool_shed_repository.deleted = False + tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.INSTALLED + trans.sa_session.add( tool_shed_repository ) trans.sa_session.flush() - message += 'The <b>%s</b> repository has been reinstalled. ' % repository.name - if install_tool_dependencies: - message += 'The following tool dependencies are now being installed, please wait...' - tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.missing_tool_dependencies ] - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='manage_tool_dependencies', - operation='install', - tool_dependency_ids=tool_dependency_ids, - status=status, - message=message ) ) return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='browse_repositories', message=message, status=status ) ) @web.json - def repository_installation_status_updates( self, trans, id=None, status=None ): + def repository_installation_status_updates( self, trans, ids=None, status_list=None ): # Avoid caching trans.response.headers[ 'Pragma' ] = 'no-cache' trans.response.headers[ 'Expires' ] = '0' - # Create new HTML for any that have changed - rval = {} - if id is not None and status is not None: - repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) - if repository.status != status: - repository.status = status - rval[ id ] = { "status": repository.status, - "html_status": unicode( trans.fill_template( "admin/tool_shed_repository/repository_installation_status.mako", - repository=repository ), - 'utf-8' ) } + # Create new HTML for any ToolShedRepository records whose status that has changed. + rval = [] + if ids is not None and status_list is not None: + ids = util.listify( ids ) + status_list = util.listify( status_list ) + for id, status in zip( ids, status_list ): + repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) + if repository.status != status: + rval.append( dict( id=id, + status=repository.status, + html_status=unicode( trans.fill_template( "admin/tool_shed_repository/repository_installation_status.mako", + repository=repository ), + 'utf-8' ) ) ) + return rval + @web.expose + @web.require_admin + def repository_was_previously_installed( self, trans, tool_shed_url, repository_name, repo_info_tuple, clone_dir ): + # Handle case where the repository was previously installed using an older changeset_revsion, but later the repository was updated + # in the tool shed and now we're trying to install the latest changeset revision of the same repository instead of updating the one + # that was previously installed. + description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple + if os.path.exists( clone_dir ): + # The installed changeset revision already exists on disk. + return changeset_revision + repository_owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) + url = '%s/repository/previous_changeset_revisions?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ + ( tool_shed_url, url_for( '/', qualified=True ), repository_name, repository_owner, changeset_revision ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + clone_path, clone_directory = os.path.split( clone_dir ) + changeset_revisions = util.listify( text ) + for previous_changeset_revision in changeset_revisions: + new_clone_dir = os.path.join( clone_path, previous_changeset_revision ) + if os.path.exists( new_clone_dir ): + return previous_changeset_revision + return None @web.expose @web.require_admin def reselect_tool_panel_section( self, trans, **kwd ): @@ -948,18 +1354,19 @@ # Avoid caching trans.response.headers[ 'Pragma' ] = 'no-cache' trans.response.headers[ 'Expires' ] = '0' - # Create new HTML for any that have changed - rval = {} + # Create new HTML for any ToolDependency records whose status that has changed. + rval = [] if ids is not None and status_list is not None: - ids = ids.split( "," ) - status_list = status_list.split( "," ) + ids = util.listify( ids ) + status_list = util.listify( status_list ) for id, status in zip( ids, status_list ): tool_dependency = trans.sa_session.query( trans.model.ToolDependency ).get( trans.security.decode_id( id ) ) if tool_dependency.status != status: - rval[ id ] = { "status": tool_dependency.status, - "html_status": unicode( trans.fill_template( "admin/tool_shed_repository/tool_dependency_installation_status.mako", - tool_dependency=tool_dependency ), - 'utf-8' ) } + rval.append( dict( id=id, + status=tool_dependency.status, + html_status=unicode( trans.fill_template( "admin/tool_shed_repository/tool_dependency_installation_status.mako", + tool_dependency=tool_dependency ), + 'utf-8' ) ) ) return rval @web.expose @web.require_admin @@ -992,9 +1399,11 @@ status = 'error' else: message = "These tool dependencies have been uninstalled: %s" % ','.join( td.name for td in tool_dependencies_for_uninstallation ) + tool_shed_repository = tool_dependencies[ 0 ].tool_shed_repository + td_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies ] return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='manage_tool_dependencies', - tool_dependency_ids=tool_dependency_ids, + tool_dependency_ids=td_ids, status=status, message=message ) ) return trans.fill_template( '/admin/tool_shed_repository/uninstall_tool_dependencies.mako', @@ -1081,20 +1490,6 @@ """Generate the URL for cloning a repository.""" tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) - def __generate_tool_path( self, repository_clone_url, changeset_revision ): - """ - Generate a tool path that guarantees repositories with the same name will always be installed - in different directories. The tool path will be of the form: - <tool shed url>/repos/<repository owner>/<repository name>/<installed changeset revision> - http://test@bx.psu.edu:9009/repos/test/filter - """ - tmp_url = clean_repository_clone_url( repository_clone_url ) - # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column - items = tmp_url.split( 'repos' ) - tool_shed_url = items[ 0 ] - repo_path = items[ 1 ] - tool_shed_url = clean_tool_shed_url( tool_shed_url ) - return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) ## ---- Utility methods ------------------------------------------------------- diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -10,7 +10,7 @@ from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * from galaxy.util.shed_util import get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, make_tmp_directory, NOT_TOOL_CONFIGS -from galaxy.util.shed_util import open_repository_files_folder, strip_path +from galaxy.util.shed_util import open_repository_files_folder, reversed_lower_upper_bounded_changelog, strip_path from galaxy.tool_shed.encoding_util import * from common import * @@ -1096,8 +1096,9 @@ repo = hg.repository( get_configured_ui(), repo_dir ) # Default to the received changeset revision and ctx_rev. update_to_ctx = get_changectx_for_changeset( repo, changeset_revision ) + ctx_rev = str( update_to_ctx.rev() ) latest_changeset_revision = changeset_revision - update_dict = dict( changeset_revision=update_to_ctx, ctx_rev=str( update_to_ctx.rev() ) ) + update_dict = dict( changeset_revision=changeset_revision, ctx_rev=ctx_rev ) if changeset_revision == repository.tip: # If changeset_revision is the repository tip, there are no additional updates. return tool_shed_encode( update_dict ) @@ -1227,14 +1228,9 @@ changeset_revision, str( ctx.rev() ) ) encoded_repo_info_dict = encode( repo_info_dict ) - if includes_tool_dependencies: - # Redirect back to local Galaxy to present the option to install tool dependencies. - url = '%sadmin_toolshed/confirm_tool_dependency_install?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \ - ( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ) ) - else: - # Redirect back to local Galaxy to perform install. - url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \ - ( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ) ) + # Redirect back to local Galaxy to perform install. + url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s&includes_tool_dependencies=%s' % \ + ( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ), str( includes_tool_dependencies ) ) return trans.response.send_redirect( url ) @web.expose def load_invalid_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ): @@ -1563,6 +1559,34 @@ message=message, status=status ) @web.expose + def previous_changeset_revisions( self, trans, **kwd ): + """ + Handle a request from a local Galaxy instance. This method will handle the case where the repository was previously installed using an + older changeset_revsion, but later the repository was updated in the tool shed and the Galaxy admin is trying to install the latest + changeset revision of the same repository instead of updating the one that was previously installed. + """ + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + # If the request originated with the UpdateManager, it will not include a galaxy_url. + galaxy_url = kwd.get( 'galaxy_url', '' ) + name = params.get( 'name', None ) + owner = params.get( 'owner', None ) + changeset_revision = params.get( 'changeset_revision', None ) + repository = get_repository_by_name_and_owner( trans, name, owner ) + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + # Get the lower bound changeset revision + lower_bound_changeset_revision = get_previous_valid_changset_revision( repository, repo, changeset_revision ) + # Build the list of changeset revision hashes. + changeset_hashes = [] + for changeset in reversed_lower_upper_bounded_changelog( repo, lower_bound_changeset_revision, changeset_revision ): + changeset_hashes.append( str( repo.changectx( changeset ) ) ) + if changeset_hashes: + changeset_hashes_str = ','.join( changeset_hashes ) + return changeset_hashes_str + return '' + @web.expose @web.require_login( "rate repositories" ) def rate_repository( self, trans, **kwd ): """ Rate a repository and return updated rating data. """ diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/common.mako --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -54,7 +54,7 @@ url: "${h.url_for( controller='admin_toolshed', action='get_file_contents' )}", dataType: "json", data: { file_path: selected_value }, - success : function ( data ) { + success : function( data ) { cell.html( '<label>'+data+'</label>' ) } }); @@ -69,27 +69,31 @@ <%def name="dependency_status_updater()"><script type="text/javascript"> - - // Tool dependency status updater - used to update the installation status on the Tool Dependencies grid. + // Tool dependency status updater - used to update the installation status on the Tool Dependencies Grid. // Looks for changes in tool dependency installation status using an async request. Keeps calling itself // (via setTimeout) until dependency installation status is neither 'Installing' nor 'Building'. - var tool_dependency_status_updater = function ( dependency_status_list ) { + var tool_dependency_status_updater = function( dependency_status_list ) { // See if there are any items left to track var empty = true; - for ( i in dependency_status_list ) { - empty = false; - break; + for ( var item in dependency_status_list ) { + //alert( "item" + item.toSource() ); + //alert( "dependency_status_list[item] " + dependency_status_list[item].toSource() ); + //alert( "dependency_status_list[item]['status']" + dependency_status_list[item]['status'] ); + if ( dependency_status_list[item]['status'] != 'Installed' ) { + empty = false; + break; + } } if ( ! empty ) { setTimeout( function() { tool_dependency_status_updater_callback( dependency_status_list ) }, 3000 ); } }; - var tool_dependency_status_updater_callback = function ( dependency_status_list ) { - var ids = [] - var status_list = [] - $.each( dependency_status_list, function ( id, dependency_status ) { - ids.push( id ); - status_list.push( dependency_status ); + var tool_dependency_status_updater_callback = function( dependency_status_list ) { + var ids = []; + var status_list = []; + $.each( dependency_status_list, function( index, dependency_status ) { + ids.push( dependency_status[ 'id' ] ); + status_list.push( dependency_status[ 'status' ] ); }); // Make ajax call $.ajax( { @@ -97,17 +101,75 @@ url: "${h.url_for( controller='admin_toolshed', action='tool_dependency_status_updates' )}", dataType: "json", data: { ids: ids.join( "," ), status_list: status_list.join( "," ) }, - success : function ( data ) { - $.each( data, function( id, val ) { + success : function( data ) { + $.each( data, function( index, val ) { // Replace HTML - var cell1 = $("#ToolDependencyStatus-" + id); - cell1.html( val.html_status ); - dependency_status_list[ id ] = val.status; + var cell1 = $( "#ToolDependencyStatus-" + val[ 'id' ] ); + cell1.html( val[ 'html_status' ] ); + dependency_status_list[ index ] = val; }); tool_dependency_status_updater( dependency_status_list ); }, error: function() { - tool_dependency_status_updater( dependency_status_list ); + alert( "tool_dependency_status_updater_callback failed..." ); + } + }); + }; + </script> +</%def> + +<%def name="repository_installation_status_updater()"> + <script type="text/javascript"> + // Tool shed repository status updater - used to update the installation status on the Repository Installation Grid. + // Looks for changes in repository installation status using an async request. Keeps calling itself (via setTimeout) until + // repository installation status is not one of: 'New', 'Cloning', 'Setting tool versions', 'Installing tool dependencies', + // 'Loading proprietary datatypes'. + var tool_shed_repository_status_updater = function( repository_status_list ) { + // See if there are any items left to track + //alert( "repository_status_list start " + repository_status_list.toSource() ); + var empty = true; + for ( var item in repository_status_list ) { + //alert( "item" + item.toSource() ); + //alert( "repository_status_list[item] " + repository_status_list[item].toSource() ); + //alert( "repository_status_list[item]['status']" + repository_status_list[item]['status'] ); + if (repository_status_list[item]['status'] != 'Installed'){ + empty = false; + break; + } + } + if ( ! empty ) { + setTimeout( function() { tool_shed_repository_status_updater_callback( repository_status_list ) }, 3000 ); + } + }; + var tool_shed_repository_status_updater_callback = function( repository_status_list ) { + //alert( repository_status_list ); + //alert( repository_status_list.toSource() ); + var ids = []; + var status_list = []; + $.each( repository_status_list, function( index, repository_status ) { + //alert('repository_status '+ repository_status.toSource() ); + //alert('id '+ repository_status['id'] ); + //alert( 'status'+ repository_status['status'] ); + ids.push( repository_status[ 'id' ] ); + status_list.push( repository_status[ 'status' ] ); + }); + // Make ajax call + $.ajax( { + type: "POST", + url: "${h.url_for( controller='admin_toolshed', action='repository_installation_status_updates' )}", + dataType: "json", + data: { ids: ids.join( "," ), status_list: status_list.join( "," ) }, + success : function( data ) { + $.each( data, function( index, val ) { + // Replace HTML + var cell1 = $( "#RepositoryStatus-" + val[ 'id' ] ); + cell1.html( val[ 'html_status' ] ); + repository_status_list[ index ] = val; + }); + tool_shed_repository_status_updater( repository_status_list ); + }, + error: function() { + alert( "tool_shed_repository_status_updater_callback failed..." ); } }); }; @@ -126,71 +188,27 @@ %if can_update: <script type="text/javascript"> // Tool dependency installation status updater - tool_dependency_status_updater( {${ ",".join( [ '"%s" : "%s"' % ( trans.security.encode_id( td.id ), td.status ) for td in query ] ) }}); + tool_dependency_status_updater( [${ ",".join( [ '{"id" : "%s", "status" : "%s"}' % ( trans.security.encode_id( td.id ), td.status ) for td in query ] ) } ] ); </script> %endif </%def> -<%def name="repository_installation_status_updater()"> - <script type="text/javascript"> - - // Tool shed repository status updater - used to update the installation status on the repository_installation.mako template. - // Looks for changes in repository installation status using an async request. Keeps calling itself (via setTimeout) until - // repository installation status is neither 'cloning', 'cloned' nor 'installing tool dependencies'. - var tool_shed_repository_status_updater = function ( repository_status_list ) { - // See if there are any items left to track - var empty = true; - for ( i in repository_status_list ) { - empty = false; - break; - } - if ( ! empty ) { - setTimeout( function() { tool_shed_repository_status_updater_callback( repository_status_list ) }, 3000 ); - } - }; - var tool_shed_repository_status_updater_callback = function ( repository_status_list ) { - var ids = [] - var status_list = [] - $.each( repository_status_list, function ( id, repository_status ) { - ids.push( id ); - status_list.push( repository_status ); - }); - // Make ajax call - $.ajax( { - type: "POST", - url: "${h.url_for( controller='admin_toolshed', action='repository_installation_status_updates' )}", - dataType: "json", - data: { id: ids[0], status_list: status_list.join( "," ) }, - success : function ( data ) { - $.each( data, function( id, val ) { - // Replace HTML - var cell1 = $("#RepositoryStatus-" + id); - cell1.html( val.html_status ); - repository_status_list[ id ] = val.status; - }); - tool_shed_repository_status_updater( repository_status_list ); - }, - error: function() { - tool_shed_repository_status_updater( repository_status_list ); - } - }); - }; - </script> -</%def> - <%def name="repository_installation_updater()"><% - can_update = True - if tool_shed_repository: - can_update = tool_shed_repository.status not in [ trans.model.ToolShedRepository.installation_status.INSTALLED, - trans.model.ToolShedRepository.installation_status.ERROR, - trans.model.ToolShedRepository.installation_status.UNINSTALLED ] + can_update = False + if query.count(): + for tool_shed_repository in query: + if tool_shed_repository.status not in [ trans.model.ToolShedRepository.installation_status.INSTALLED, + trans.model.ToolShedRepository.installation_status.ERROR, + trans.model.ToolShedRepository.installation_status.DEACTIVATED, + trans.model.ToolShedRepository.installation_status.UNINSTALLED ]: + can_update = True + break %> %if can_update: <script type="text/javascript"> // Tool shed repository installation status updater - repository_installation_status_updater( {${ ",".join( [ '"%s" : "%s"' % ( trans.security.encode_id( repository.id ), repository.status ) for repository in query ] ) }}); + tool_shed_repository_status_updater( [${ ",".join( [ '{"id" : "%s", "status" : "%s"}' % ( trans.security.encode_id( tsr.id ), tsr.status ) for tsr in query ] ) } ] ); </script> %endif </%def> - diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/confirm_tool_dependency_install.mako --- a/templates/admin/tool_shed_repository/confirm_tool_dependency_install.mako +++ /dev/null @@ -1,86 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> - -<% import os %> - -%if message: - ${render_msg( message, status )} -%endif - -<div class="warningmessage"> - <p> - The tool dependencies listed below can be automatically installed with the repository. Installing them provides significant - benefits and Galaxy includes various features to manage them. - </p> - <p> - Each of these dependencies may require their own build requirements (e.g., CMake, g++, etc). Galaxy will not attempt to install - these build requirements, so if any are missing from your environment tool dependency installation may partially fail. The - repository and all of it's contents will be installed in any case. - </p> - <p> - If tool dependency installation fails in any way, you can install the missing build requirements and have Galaxy attempt to install - the tool dependencies again using the <b>Install tool dependencies</b> pop-up menu option on the <b>Manage repository</b> page. - </p> -</div> - -<div class="toolForm"> - <div class="toolFormBody"> - <form name="confirm_tool_dependency_install" id="confirm_tool_dependency_install" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, includes_tools=includes_tools )}" method="post" > - <div style="clear: both"></div> - <div class="form-row"> - <label>Install tool dependencies?</label> - ${install_tool_dependencies_check_box.get_html()} - <div class="toolParamHelp" style="clear: both;"> - Un-check to skip automatic installation of these tool dependencies. - </div> - </div> - <div style="clear: both"></div> - <div class="form-row"> - <table class="grid"> - <tr><td colspan="4" bgcolor="#D8D8D8"><b>Tool dependencies</b></td></tr> - <tr> - <th>Name</th> - <th>Version</th> - <th>Type</th> - <th>Install directory</th> - </tr> - %for repository_name, repo_info_tuple in dict_with_tool_dependencies.items(): - <% - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple - %> - %for dependency_key, requirements_dict in tool_dependencies.items(): - <% - name = requirements_dict[ 'name' ] - version = requirements_dict[ 'version' ] - type = requirements_dict[ 'type' ] - install_dir = os.path.join( trans.app.config.tool_dependency_dir, - name, - version, - repository_owner, - repository_name, - changeset_revision ) - readme_text = requirements_dict.get( 'readme', None ) - %> - %if not os.path.exists( install_dir ): - <tr> - <td>${name}</td> - <td>${version}</td> - <td>${type}</td> - <td>${install_dir}</td> - </tr> - %if readme_text: - <tr><td colspan="4" bgcolor="#FFFFCC">${name} ${version} requirements and installation information</td></tr> - <tr><td colspan="4"><pre>${readme_text}</pre></td></tr> - %endif - %endif - %endfor - %endfor - </table> - <div style="clear: both"></div> - </div> - <div class="form-row"> - <input type="submit" name="confirm_tool_dependency_install_button" value="Continue"/> - </div> - </form> - </div> -</div> diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako --- a/templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako +++ b/templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako @@ -10,7 +10,8 @@ <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> %if repository.tool_dependencies: - <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_tool_dependencies', id=trans.security.encode_id( repository.id ) )}">Manage tool dependencies</a> + <% tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in repository.tool_dependencies ] %> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_tool_dependencies', tool_dependency_ids=tool_dependency_ids )}">Manage tool dependencies</a> %endif </div></ul> diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/initiate_repository_installation.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/initiate_repository_installation.mako @@ -0,0 +1,75 @@ +<%inherit file="/base.mako"/> +<%namespace file="/admin/tool_shed_repository/common.mako" import="*" /> + +<%def name="javascripts()"> + ${parent.javascripts()} + ${repository_installation_status_updater()} + ${repository_installation_updater()} + ${self.repository_installation_javascripts()} +</%def> + +<%def name="repository_installation_javascripts()"> + <script type="text/javascript"> + $(document).ready(function( ){ + initiate_repository_installation( "${initiate_repository_installation_ids}", "${encoded_kwd}" ); + }); + var initiate_repository_installation = function ( iri_ids, encoded_kwd ) { + // Make ajax call + $.ajax( { + type: "POST", + url: "${h.url_for( controller='admin_toolshed', action='manage_repositories' )}", + dataType: "html", + data: { operation: "install", tool_shed_repository_ids: iri_ids, encoded_kwd: encoded_kwd }, + success : function ( data ) { + //alert( "Initializing repository installation succeeded" ); + }, + error: function() { + alert( "Initializing repository installation failed" ); + }, + }); + }; + </script> +</%def> + +%if tool_shed_repositories: + <div class="toolForm"> + <div class="toolFormTitle">Monitor installing tool shed repositories</div> + <div class="toolFormBody"> + <table class="grid"> + <tr> + <td>Name</td> + <td>Description</td> + <td>Owner</td> + <td>Revision</td> + <td>Status</td> + </tr> + %for tool_shed_repository in tool_shed_repositories: + <% + encoded_repository_id = trans.security.encode_id( tool_shed_repository.id ) + ids_of_tool_dependencies_missing_or_being_installed = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies_missing_or_being_installed ] + link_to_manage_tool_dependencies = tool_shed_repository.status in [ trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ] + %> + <tr> + <td> + %if link_to_manage_tool_dependencies: + <a class="view-info" href="${h.url_for( controller='admin_toolshed', action='manage_tool_dependencies', tool_dependency_ids=ids_of_tool_dependencies_missing_or_being_installed )}"> + ${tool_shed_repository.name} + </a> + %else: + <a class="view-info" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=encoded_repository_id )}"> + ${tool_shed_repository.name} + </a> + %endif + </td> + <td>${tool_shed_repository.description}</td> + <td>${tool_shed_repository.owner}</td> + <td>${tool_shed_repository.changeset_revision}</td> + <td><div id="RepositoryStatus-${encoded_repository_id}">${tool_shed_repository.status}</div></td> + </tr> + %endfor + </table> + <br clear="left"/> + </div> + </div> +%endif + diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/repository_installation_grid.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/repository_installation_grid.mako @@ -0,0 +1,8 @@ +<%inherit file="/grid_base.mako"/> +<%namespace file="/admin/tool_shed_repository/common.mako" import="*" /> + +<%def name="javascripts()"> + ${parent.javascripts()} + ${repository_installation_status_updater()} + ${repository_installation_updater()} +</%def> diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/repository_installation_status.mako --- /dev/null +++ b/templates/admin/tool_shed_repository/repository_installation_status.mako @@ -0,0 +1,29 @@ +<%def name="render_repository_status( repository )"> + <% + if repository.status in [ trans.model.ToolShedRepository.installation_status.CLONING, + trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, + trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]: + bgcolor = trans.model.ToolShedRepository.states.INSTALLING + elif repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, + trans.model.ToolShedRepository.installation_status.UNINSTALLED ]: + bgcolor = trans.model.ToolShedRepository.states.UNINSTALLED + elif repository.status in [ trans.model.ToolShedRepository.installation_status.ERROR ]: + bgcolor = trans.model.ToolShedRepository.states.ERROR + elif repository.status in [ trans.model.ToolShedRepository.installation_status.DEACTIVATED ]: + bgcolor = trans.model.ToolShedRepository.states.WARNING + elif repository.status in [ trans.model.ToolShedRepository.installation_status.INSTALLED ]: + if repository.missing_tool_dependencies: + bgcolor = trans.model.ToolShedRepository.states.WARNING + else: + bgcolor = trans.model.ToolShedRepository.states.OK + else: + bgcolor = trans.model.ToolShedRepository.states.ERROR + rval = '<div class="count-box state-color-%s" id="ToolShedRepositoryStatus-%s">' % ( bgcolor, trans.security.encode_id( repository.id ) ) + rval += '%s</div>' % repository.status + return rval + %> + ${rval} +</%def> + +${render_repository_status( repository )} diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/reselect_tool_panel_section.mako --- a/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako +++ b/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako @@ -8,12 +8,7 @@ <div class="toolForm"><div class="toolFormTitle">Choose the tool panel section to contain the installed tools (optional)</div><div class="toolFormBody"> - %if repository.tool_dependencies: - <% tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in repository.tool_dependencies ] %> - <form name="reselect_tool_panel_section" id="reselect_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='manage_tool_dependencies', operation='install', tool_dependency_ids=tool_dependency_ids )}" method="post" > - %else: - <form name="reselect_tool_panel_section" id="reselect_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='reinstall_repository', id=trans.security.encode_id( repository.id ) )}" method="post" > - %endif + <form name="reselect_tool_panel_section" id="reselect_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='reinstall_repository', id=trans.security.encode_id( repository.id ) )}" method="post" ><div class="form-row"> ${no_changes_check_box.get_html()} <label style="display: inline;">No changes</label> diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/select_tool_panel_section.mako --- a/templates/admin/tool_shed_repository/select_tool_panel_section.mako +++ b/templates/admin/tool_shed_repository/select_tool_panel_section.mako @@ -1,6 +1,8 @@ <%inherit file="/base.mako"/><%namespace file="/message.mako" import="render_msg" /> +<% import os %> + %if message: ${render_msg( message, status )} %endif @@ -20,24 +22,83 @@ </p></div><br/> -<div class="warningmessage"> - <p> - Installation may take a while with large repositories or if you elect to install tool dependencies. <b>Always wait until a message is - displayed in your browser after clicking the <b>Install</b> button below.</b> If you get bored, watching your Galaxy server's paster log - will help pass the time. - </p> - <p> - If installing tool dependencies, information about installation process will be saved in various files named with a ".log" - extension in the directory: - ${trans.app.config.tool_dependency_dir}/<i>package name</i>/<i>package version</i>/<i>repository owner</i>/<i>repository name</i>/<i>repository changeset revision</i> - </p> -</div> -<br/><div class="toolForm"> - <div class="toolFormTitle">Choose the tool panel section to contain the installed tools (optional)</div> + <div class="toolFormTitle">Confirm tool dependency installation</div><div class="toolFormBody"> - <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, includes_tools=includes_tools, install_tool_dependencies=install_tool_dependencies )}" method="post" > + <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, includes_tools=includes_tools, includes_tool_dependencies=includes_tool_dependencies )}" method="post" > + <div style="clear: both"></div> + %if includes_tool_dependencies: + <div class="form-row"> + <div class="toolParamHelp" style="clear: both;"> + <p> + These tool dependencies can be automatically installed with the repository. Installing them provides significant benefits and + Galaxy includes various features to manage them. + </p> + <p> + Each of these dependencies may require their own build requirements (e.g., CMake, g++, etc). Galaxy will not attempt to install + these build requirements, so tool dependency installation may partially fail if any are missing from your environment, but the + repository and all of it's contents will be installed. You can install the missing build requirements and have Galaxy attempt + to install the tool dependencies again if tool dependency installation fails in any way. + </p> + </div> + </div> + <div class="form-row"> + <label>Install tool dependencies?</label> + ${install_tool_dependencies_check_box.get_html()} + <div class="toolParamHelp" style="clear: both;"> + Un-check to skip automatic installation of these tool dependencies. + </div> + </div> + <div style="clear: both"></div> + <div class="form-row"> + <table class="grid"> + <tr><td colspan="4" bgcolor="#D8D8D8"><b>Tool dependencies</b></td></tr> + <tr> + <th>Name</th> + <th>Version</th> + <th>Type</th> + <th>Install directory</th> + </tr> + %for repository_name, repo_info_tuple in dict_with_tool_dependencies.items(): + <% description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple %> + %for dependency_key, requirements_dict in tool_dependencies.items(): + <% + name = requirements_dict[ 'name' ] + version = requirements_dict[ 'version' ] + type = requirements_dict[ 'type' ] + install_dir = os.path.join( trans.app.config.tool_dependency_dir, + name, + version, + repository_owner, + repository_name, + changeset_revision ) + tool_dependency_readme_text = requirements_dict.get( 'readme', None ) + %> + %if not os.path.exists( install_dir ): + <tr> + <td>${name}</td> + <td>${version}</td> + <td>${type}</td> + <td>${install_dir}</td> + </tr> + %if tool_dependency_readme_text: + <tr><td colspan="4" bgcolor="#FFFFCC">${name} ${version} requirements and installation information</td></tr> + <tr><td colspan="4"><pre>${tool_dependency_readme_text}</pre></td></tr> + %endif + %endif + %endfor + %endfor + </table> + <div style="clear: both"></div> + </div> + %endif + <div style="clear: both"></div> + <div class="form-row"> + <table class="colored" width="100%"> + <th bgcolor="#EBD9B2">Choose the tool panel section to contain the installed tools (optional)</th> + </table> + </div> %if shed_tool_conf_select_field: <div class="form-row"><label>Shed tool configuration file:</label> diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/tool_dependency_installation_status.mako --- a/templates/admin/tool_shed_repository/tool_dependency_installation_status.mako +++ b/templates/admin/tool_shed_repository/tool_dependency_installation_status.mako @@ -1,11 +1,17 @@ <%def name="render_tool_dependency_status( tool_dependency )"><% - if tool_dependency.status == trans.model.ToolDependency.installation_status.INSTALLING: + if tool_dependency.status in [ trans.model.ToolDependency.installation_status.INSTALLING ]: bgcolor = trans.model.ToolDependency.states.INSTALLING - rval = '<div class="count-box state-color-%s" id="ToolDependencyStatus-%s">' % ( bgcolor, trans.security.encode_id( tool_dependency.id ) ) - rval += '%s</div>' % tool_dependency.status - else: - rval = tool_dependency.status + elif tool_dependency.status in [ trans.model.ToolDependency.installation_status.NEVER_INSTALLED, + trans.model.ToolDependency.installation_status.UNINSTALLED ]: + bgcolor = trans.model.ToolDependency.states.UNINSTALLED + elif tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR ]: + bgcolor = trans.model.ToolDependency.states.ERROR + elif tool_dependency.status in [ trans.model.ToolDependency.installation_status.INSTALLED ]: + bgcolor = trans.model.ToolDependency.states.OK + rval = '<div class="count-box state-color-%s" id="ToolDependencyStatus-%s">' % ( bgcolor, trans.security.encode_id( tool_dependency.id ) ) + rval += '%s</div>' % tool_dependency.status + return rval %> ${rval} </%def> diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/admin/tool_shed_repository/view_tool_metadata.mako --- a/templates/admin/tool_shed_repository/view_tool_metadata.mako +++ b/templates/admin/tool_shed_repository/view_tool_metadata.mako @@ -9,7 +9,8 @@ <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a><a class="action-button" href="${h.url_for( controller='admin_toolshed', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a> %if repository.tool_dependencies: - <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_tool_dependencies', id=trans.security.encode_id( repository.id ) )}">Manage tool dependencies</a> + <% tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in repository.tool_dependencies ] %> + <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='manage_tool_dependencies', tool_dependency_ids=tool_dependency_ids )}">Manage tool dependencies</a> %endif <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='deactivate_or_uninstall_repository', id=trans.security.encode_id( repository.id ) )}">Deactivate or uninstall repository</a></div> diff -r 1a4d6f11f28453aa98b32439a8df479a2830bc50 -r cc5fea2e382ed8edadf0d694f559b46e86a66884 templates/webapps/galaxy/admin/index.mako --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -73,6 +73,7 @@ %if cloned_repositories: <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='browse_repositories' )}" target="galaxy_main">Manage installed tool shed repositories</a></div> %endif + <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='monitor_repository_installation' )}" target="galaxy_main">Monitor installing tool shed repositories</a></div></div></div> %if trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.