November 2012 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: dannon: Add pause icon from fontawesome to paused dataset display.
by Bitbucket 16 Nov '12

16 Nov '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/9e1d821b5c12/ changeset: 9e1d821b5c12 user: dannon date: 2012-11-16 17:35:07 summary: Add pause icon from fontawesome to paused dataset display. affected #: 2 files diff -r 951ae6588d1630858ed55afcff4d426761a25268 -r 9e1d821b5c12f56bbfe1923ea1092608054c9d08 static/june_2007_style/base.less --- a/static/june_2007_style/base.less +++ b/static/june_2007_style/base.less @@ -1640,6 +1640,12 @@ div.historyItem-paused { // border-color: @history_paused_border; background: @history_paused_bg; + .state-icon { + .ficon(); + &:before { + content: "\f04c"; + } + } } // Special case for showing the spinner but not changing the background diff -r 951ae6588d1630858ed55afcff4d426761a25268 -r 9e1d821b5c12f56bbfe1923ea1092608054c9d08 static/june_2007_style/blue/base.css --- a/static/june_2007_style/blue/base.css +++ b/static/june_2007_style/blue/base.css @@ -851,7 +851,7 @@ div.historyItem-upload{background:#ccccff;}div.historyItem-upload .state-icon{background-image:url(data_upload.gif);} div.historyItem-queued{background:#eeeeee;} div.historyItem-noPermission{filter:alpha(opacity=60);-moz-opacity:.60;opacity:.60;} -div.historyItem-paused{background:#d9edf7;} +div.historyItem-paused{background:#d9edf7;}div.historyItem-paused .state-icon{font-family:FontAwesome;font-weight:normal;font-style:normal;display:inline-block;}div.historyItem-paused .state-icon:before{content:"\f04c";} div.historyItemTitleBar.spinner .state-icon{background:url(data_running.gif) 0 1px no-repeat !important;} div.historyItemButtons{float:right;} div.historyItemBody div{padding-top:2px;} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Add paused dataset state for new history panel. Still needs icon and complete info text.
by Bitbucket 16 Nov '12

16 Nov '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/c3ac1fae1a29/ changeset: c3ac1fae1a29 user: dannon date: 2012-11-16 15:40:20 summary: Add paused dataset state for new history panel. Still needs icon and complete info text. affected #: 2 files diff -r 4212a41ba5ee1f2a45f4d5f84c110808b167c87a -r c3ac1fae1a290c4012f2e7f3d7a87126622b0fc3 static/scripts/mvc/dataset/hda-base.js --- a/static/scripts/mvc/dataset/hda-base.js +++ b/static/scripts/mvc/dataset/hda-base.js @@ -328,34 +328,37 @@ switch( this.model.get( 'state' ) ){ case HistoryDatasetAssociation.STATES.NOT_VIEWABLE : this._render_body_not_viewable( body ); - break; + break; case HistoryDatasetAssociation.STATES.UPLOAD : - this._render_body_uploading( body ); - break; + this._render_body_uploading( body ); + break; + case HistoryDatasetAssociation.STATES.PAUSED: + this._render_body_paused( body ); + break; case HistoryDatasetAssociation.STATES.QUEUED : - this._render_body_queued( body ); - break; + this._render_body_queued( body ); + break; case HistoryDatasetAssociation.STATES.RUNNING : - this._render_body_running( body ); - break; + this._render_body_running( body ); + break; case HistoryDatasetAssociation.STATES.ERROR : - this._render_body_error( body ); - break; + this._render_body_error( body ); + break; case HistoryDatasetAssociation.STATES.DISCARDED : - this._render_body_discarded( body ); - break; + this._render_body_discarded( body ); + break; case HistoryDatasetAssociation.STATES.SETTING_METADATA : - this._render_body_setting_metadata( body ); - break; + this._render_body_setting_metadata( body ); + break; case HistoryDatasetAssociation.STATES.EMPTY : - this._render_body_empty( body ); - break; + this._render_body_empty( body ); + break; case HistoryDatasetAssociation.STATES.FAILED_METADATA : - this._render_body_failed_metadata( body ); - break; + this._render_body_failed_metadata( body ); + break; case HistoryDatasetAssociation.STATES.OK : - this._render_body_ok( body ); - break; + this._render_body_ok( body ); + break; default: //??: no body? body.append( $( '<div>Error: unknown dataset state "' + state + '".</div>' ) ); @@ -392,6 +395,14 @@ parent.append( $( '<div>' + _l( 'Job is waiting to run' ) + '.</div>' ) ); parent.append( this._render_primaryActionButtons( this.defaultPrimaryActionButtonRenderers )); }, + + /** Render an HDA whose job is paused. + * @param {jQuery} parent DOM to which to append this body + */ + _render_body_paused: function( parent ){ + parent.append( $( '<div>' + _l( 'Job is paused. Use the history menu to unpause' ) + '.</div>' ) ); + parent.append( this._render_primaryActionButtons( this.defaultPrimaryActionButtonRenderers )); + }, /** Render an HDA whose job is running. * @param {jQuery} parent DOM to which to append this body diff -r 4212a41ba5ee1f2a45f4d5f84c110808b167c87a -r c3ac1fae1a290c4012f2e7f3d7a87126622b0fc3 static/scripts/mvc/dataset/hda-model.js --- a/static/scripts/mvc/dataset/hda-model.js +++ b/static/scripts/mvc/dataset/hda-model.js @@ -164,6 +164,8 @@ UPLOAD : 'upload', /** the job that will produce the dataset queued in the runner */ QUEUED : 'queued', + /** the job that will produce the dataset paused */ + PAUSED : 'paused', /** the job that will produce the dataset is running */ RUNNING : 'running', /** metadata for the dataset is being discovered/set */ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: More shed_util refactoring.
by Bitbucket 16 Nov '12

16 Nov '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/4212a41ba5ee/ changeset: 4212a41ba5ee user: greg date: 2012-11-16 15:19:11 summary: More shed_util refactoring. affected #: 2 files diff -r 4f68935907f4f86313f63cad33a310e65b2ee02c -r 4212a41ba5ee1f2a45f4d5f84c110808b167c87a lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -596,19 +596,6 @@ fh.write( fctx.data() ) fh.close() return sample_files, deleted_sample_files -def get_repository_files( trans, folder_path ): - contents = [] - for item in os.listdir( folder_path ): - # Skip .hg directories - if str( item ).startswith( '.hg' ): - continue - if os.path.isdir( os.path.join( folder_path, item ) ): - # Append a '/' character so that our jquery dynatree will function properly. - item = '%s/' % item - contents.append( item ) - if contents: - contents.sort() - return contents def get_repository_owner( cleaned_repository_url ): items = cleaned_repository_url.split( 'repos' ) repo_path = items[ 1 ] diff -r 4f68935907f4f86313f63cad33a310e65b2ee02c -r 4212a41ba5ee1f2a45f4d5f84c110808b167c87a lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -768,6 +768,19 @@ to_html = '%s%s' % ( to_html, to_html_str( large_str ) ) break return to_html +def get_repository_files( trans, folder_path ): + contents = [] + for item in os.listdir( folder_path ): + # Skip .hg directories + if str( item ).startswith( '.hg' ): + continue + if os.path.isdir( os.path.join( folder_path, item ) ): + # Append a '/' character so that our jquery dynatree will function properly. + item = '%s/' % item + contents.append( item ) + if contents: + contents.sort() + return contents def get_repository_in_tool_shed( trans, id ): """Get a repository on the tool shed side from the database via id""" return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix imports in shed_util_common.py
by Bitbucket 15 Nov '12

15 Nov '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/4f68935907f4/ changeset: 4f68935907f4 user: greg date: 2012-11-15 23:01:56 summary: Fix imports in shed_util_common.py affected #: 1 file diff -r a43b16b64b1a4b0b7fe2d5be0d87a9047c3a54a3 -r 4f68935907f4f86313f63cad33a310e65b2ee02c lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -13,6 +13,10 @@ pkg_resources.require( 'mercurial' ) from mercurial import hg, ui, commands +pkg_resources.require( 'elementtree' ) +from elementtree import ElementTree, ElementInclude +from elementtree.ElementTree import Element, SubElement + log = logging.getLogger( __name__ ) INITIAL_CHANGELOG_HASH = '000000000000' @@ -592,8 +596,8 @@ try: # Make sure we're looking at a valid repository_dependencies.xml file. tree = util.parse_xml( repository_dependencies_config ) - root = element_tree.getroot() - is_valid = element_tree_root.tag == 'repositories' + root = tree.getroot() + is_valid = root.tag == 'repositories' except Exception, e: log.debug( "Error parsing %s, exception: %s" % ( repository_dependencies_config, str( e ) ) ) is_valid = False Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: More shed_util refactoring.
by Bitbucket 15 Nov '12

15 Nov '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/a43b16b64b1a/ changeset: a43b16b64b1a user: greg date: 2012-11-15 22:54:10 summary: More shed_util refactoring. affected #: 6 files diff -r 7a51b701af8825baaf4aeb68f422304434c01a10 -r a43b16b64b1a4b0b7fe2d5be0d87a9047c3a54a3 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -157,154 +157,6 @@ except: pass return converter_path, display_path -def can_generate_tool_dependency_metadata( root, metadata_dict ): - """ - Make sure the combination of name, version and type (the type will be the value of elem.tag) of each root element tag in the tool_dependencies.xml - file is defined in the <requirement> tag for at least one tool in the repository. - """ - can_generate_dependency_metadata = False - for elem in root: - tool_dependency_type = elem.tag - tool_dependency_version = elem.get( 'version', None ) - if tool_dependency_type == 'package': - can_generate_dependency_metadata = False - tool_dependency_name = elem.get( 'name', None ) - if tool_dependency_name and tool_dependency_version: - for tool_dict in metadata_dict.get( 'tools', [] ): - requirements = tool_dict.get( 'requirements', [] ) - for requirement_dict in requirements: - req_name = requirement_dict.get( 'name', None ) - req_version = requirement_dict.get( 'version', None ) - req_type = requirement_dict.get( 'type', None ) - if req_name==tool_dependency_name and req_version==tool_dependency_version and req_type==tool_dependency_type: - can_generate_dependency_metadata = True - break - if requirements and not can_generate_dependency_metadata: - # We've discovered at least 1 combination of name, version and type that is not defined in the <requirement> - # tag for any tool in the repository. - break - if not can_generate_dependency_metadata: - break - elif tool_dependency_type == 'set_environment': - # Here elem is something like: <set_environment version="1.0"> - for env_var_elem in elem: - can_generate_dependency_metadata = False - # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> - env_var_name = env_var_elem.get( 'name', None ) - if env_var_name: - for tool_dict in metadata_dict.get( 'tools', [] ): - requirements = tool_dict.get( 'requirements', [] ) - for requirement_dict in requirements: - # {"name": "R_SCRIPT_PATH", "type": "set_environment", "version": null} - req_name = requirement_dict.get( 'name', None ) - req_type = requirement_dict.get( 'type', None ) - if req_name==env_var_name and req_type==tool_dependency_type: - can_generate_dependency_metadata = True - break - if requirements and not can_generate_dependency_metadata: - # We've discovered at least 1 combination of name, version and type that is not defined in the <requirement> - # tag for any tool in the repository. - break - return can_generate_dependency_metadata -def clean_repository_metadata( trans, id, changeset_revisions ): - # Delete all repository_metadata records associated with the repository that have a changeset_revision that is not in changeset_revisions. - # We sometimes see multiple records with the same changeset revision value - no idea how this happens. We'll assume we can delete the older - # records, so we'll order by update_time descending and delete records that have the same changeset_revision we come across later.. - changeset_revisions_checked = [] - for repository_metadata in trans.sa_session.query( trans.model.RepositoryMetadata ) \ - .filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) \ - .order_by( trans.model.RepositoryMetadata.table.c.changeset_revision, - trans.model.RepositoryMetadata.table.c.update_time.desc() ): - changeset_revision = repository_metadata.changeset_revision - can_delete = changeset_revision in changeset_revisions_checked or changeset_revision not in changeset_revisions - if can_delete: - trans.sa_session.delete( repository_metadata ) - trans.sa_session.flush() -def compare_changeset_revisions( ancestor_changeset_revision, ancestor_metadata_dict, current_changeset_revision, current_metadata_dict ): - # The metadata associated with ancestor_changeset_revision is ancestor_metadata_dict. This changeset_revision is an ancestor of - # current_changeset_revision which is associated with current_metadata_dict. A new repository_metadata record will be created only - # when this method returns the string 'not equal and not subset'. - ancestor_datatypes = ancestor_metadata_dict.get( 'datatypes', [] ) - ancestor_tools = ancestor_metadata_dict.get( 'tools', [] ) - ancestor_guids = [ tool_dict[ 'guid' ] for tool_dict in ancestor_tools ] - ancestor_guids.sort() - ancestor_tool_dependencies = ancestor_metadata_dict.get( 'tool_dependencies', [] ) - ancestor_workflows = ancestor_metadata_dict.get( 'workflows', [] ) - current_datatypes = current_metadata_dict.get( 'datatypes', [] ) - current_tools = current_metadata_dict.get( 'tools', [] ) - current_guids = [ tool_dict[ 'guid' ] for tool_dict in current_tools ] - current_guids.sort() - current_tool_dependencies = current_metadata_dict.get( 'tool_dependencies', [] ) - current_workflows = current_metadata_dict.get( 'workflows', [] ) - # Handle case where no metadata exists for either changeset. - if not ancestor_guids and not current_guids and not ancestor_workflows and not current_workflows and not ancestor_datatypes and not current_datatypes: - return 'no metadata' - workflow_comparison = compare_workflows( ancestor_workflows, current_workflows ) - datatype_comparison = compare_datatypes( ancestor_datatypes, current_datatypes ) - # Handle case where all metadata is the same. - if ancestor_guids == current_guids and workflow_comparison == 'equal' and datatype_comparison == 'equal': - return 'equal' - if workflow_comparison in [ 'equal', 'subset' ] and datatype_comparison in [ 'equal', 'subset' ]: - is_subset = True - for guid in ancestor_guids: - if guid not in current_guids: - is_subset = False - break - if is_subset: - return 'subset' - return 'not equal and not subset' -def compare_datatypes( ancestor_datatypes, current_datatypes ): - # Determine if ancestor_datatypes is the same as current_datatypes - # or if ancestor_datatypes is a subset of current_datatypes. Each - # datatype dict looks something like: - # {"dtype": "galaxy.datatypes.images:Image", "extension": "pdf", "mimetype": "application/pdf"} - if len( ancestor_datatypes ) <= len( current_datatypes ): - for ancestor_datatype in ancestor_datatypes: - # Currently the only way to differentiate datatypes is by name. - ancestor_datatype_dtype = ancestor_datatype[ 'dtype' ] - ancestor_datatype_extension = ancestor_datatype[ 'extension' ] - ancestor_datatype_mimetype = ancestor_datatype.get( 'mimetype', None ) - found_in_current = False - for current_datatype in current_datatypes: - if current_datatype[ 'dtype' ] == ancestor_datatype_dtype and \ - current_datatype[ 'extension' ] == ancestor_datatype_extension and \ - current_datatype.get( 'mimetype', None ) == ancestor_datatype_mimetype: - found_in_current = True - break - if not found_in_current: - return 'not equal and not subset' - if len( ancestor_datatypes ) == len( current_datatypes ): - return 'equal' - else: - return 'subset' - return 'not equal and not subset' -def compare_workflows( ancestor_workflows, current_workflows ): - # Determine if ancestor_workflows is the same as current_workflows - # or if ancestor_workflows is a subset of current_workflows. - if len( ancestor_workflows ) <= len( current_workflows ): - for ancestor_workflow_tup in ancestor_workflows: - # ancestor_workflows is a list of tuples where each contained tuple is - # [ <relative path to the .ga file in the repository>, <exported workflow dict> ] - ancestor_workflow_dict = ancestor_workflow_tup[1] - # Currently the only way to differentiate workflows is by name. - ancestor_workflow_name = ancestor_workflow_dict[ 'name' ] - num_ancestor_workflow_steps = len( ancestor_workflow_dict[ 'steps' ] ) - found_in_current = False - for current_workflow_tup in current_workflows: - current_workflow_dict = current_workflow_tup[1] - # Assume that if the name and number of steps are euqal, - # then the workflows are the same. Of course, this may - # not be true... - if current_workflow_dict[ 'name' ] == ancestor_workflow_name and len( current_workflow_dict[ 'steps' ] ) == num_ancestor_workflow_steps: - found_in_current = True - break - if not found_in_current: - return 'not equal and not subset' - if len( ancestor_workflows ) == len( current_workflows ): - return 'equal' - else: - return 'subset' - return 'not equal and not subset' def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): # Persist the current in-memory list of config_elems to a file named by the value of config_filename. fd, filename = tempfile.mkstemp() @@ -326,20 +178,6 @@ copy_sample_file( trans.app, relative_path, dest_path=dest_path ) sample_files.append( name ) return sample_files -def clean_repository_clone_url( repository_clone_url ): - if repository_clone_url.find( '@' ) > 0: - # We have an url that includes an authenticated user, something like: - # http://test@bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split( '@' ) - tmp_url = items[ 1 ] - elif repository_clone_url.find( '//' ) > 0: - # We have an url that includes only a protocol, something like: - # http://bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split( '//' ) - tmp_url = items[ 1 ] - else: - tmp_url = repository_clone_url - return tmp_url def clean_tool_shed_url( tool_shed_url ): if tool_shed_url.find( ':' ) > 0: # Eliminate the port, if any, since it will result in an invalid directory name. @@ -472,166 +310,6 @@ """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) return url_join( tool_shed_url, 'repos', repository.owner, repository.name ) -def generate_datatypes_metadata( datatypes_config, metadata_dict ): - """Update the received metadata_dict with information from the parsed datatypes_config.""" - tree = ElementTree.parse( datatypes_config ) - root = tree.getroot() - ElementInclude.include( root ) - repository_datatype_code_files = [] - datatype_files = root.find( 'datatype_files' ) - if datatype_files: - for elem in datatype_files.findall( 'datatype_file' ): - name = elem.get( 'name', None ) - repository_datatype_code_files.append( name ) - metadata_dict[ 'datatype_files' ] = repository_datatype_code_files - datatypes = [] - registration = root.find( 'registration' ) - if registration: - for elem in registration.findall( 'datatype' ): - datatypes_dict = {} - display_in_upload = elem.get( 'display_in_upload', None ) - if display_in_upload: - datatypes_dict[ 'display_in_upload' ] = display_in_upload - dtype = elem.get( 'type', None ) - if dtype: - datatypes_dict[ 'dtype' ] = dtype - extension = elem.get( 'extension', None ) - if extension: - datatypes_dict[ 'extension' ] = extension - max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) - if max_optional_metadata_filesize: - datatypes_dict[ 'max_optional_metadata_filesize' ] = max_optional_metadata_filesize - mimetype = elem.get( 'mimetype', None ) - if mimetype: - datatypes_dict[ 'mimetype' ] = mimetype - subclass = elem.get( 'subclass', None ) - if subclass: - datatypes_dict[ 'subclass' ] = subclass - if datatypes_dict: - datatypes.append( datatypes_dict ) - if datatypes: - metadata_dict[ 'datatypes' ] = datatypes - return metadata_dict -def generate_environment_dependency_metadata( elem, tool_dependencies_dict ): - """The value of env_var_name must match the value of the "set_environment" type in the tool config's <requirements> tag set.""" - requirements_dict = {} - for env_elem in elem: - env_name = env_elem.get( 'name', None ) - if env_name: - requirements_dict [ 'name' ] = env_name - requirements_dict [ 'type' ] = 'environment variable' - if requirements_dict: - if 'set_environment' in tool_dependencies_dict: - tool_dependencies_dict[ 'set_environment' ].append( requirements_dict ) - else: - tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] - return tool_dependencies_dict -def generate_package_dependency_metadata( elem, tool_dependencies_dict ): - """The value of package_name must match the value of the "package" type in the tool config's <requirements> tag set.""" - requirements_dict = {} - package_name = elem.get( 'name', None ) - package_version = elem.get( 'version', None ) - if package_name and package_version: - dependency_key = '%s/%s' % ( package_name, package_version ) - requirements_dict [ 'name' ] = package_name - requirements_dict [ 'version' ] = package_version - requirements_dict [ 'type' ] = 'package' - for sub_elem in elem: - if sub_elem.tag == 'readme': - requirements_dict[ 'readme' ] = sub_elem.text - if requirements_dict: - tool_dependencies_dict[ dependency_key ] = requirements_dict - return tool_dependencies_dict -def generate_tool_dependency_metadata( app, repository, tool_dependencies_config, metadata_dict, original_repository_metadata=None ): - """ - If the combination of name, version and type of each element is defined in the <requirement> tag for at least one tool in the repository, - then update the received metadata_dict with information from the parsed tool_dependencies_config. - """ - if original_repository_metadata: - # Keep a copy of the original tool dependencies dictionary in the metadata. - original_tool_dependencies_dict = original_repository_metadata.get( 'tool_dependencies', None ) - else: - original_tool_dependencies_dict = None - try: - tree = ElementTree.parse( tool_dependencies_config ) - except Exception, e: - log.debug( "Exception attempting to parse tool_dependencies.xml: %s" %str( e ) ) - return metadata_dict - root = tree.getroot() - ElementInclude.include( root ) - tool_dependencies_dict = {} - if can_generate_tool_dependency_metadata( root, metadata_dict ): - for elem in root: - if elem.tag == 'package': - tool_dependencies_dict = generate_package_dependency_metadata( elem, tool_dependencies_dict ) - elif elem.tag == 'set_environment': - tool_dependencies_dict = generate_environment_dependency_metadata( elem, tool_dependencies_dict ) - # Handle tool dependency installation via other means here (future). - if tool_dependencies_dict: - metadata_dict[ 'tool_dependencies' ] = tool_dependencies_dict - else: - log.debug( "Name, version and type from the <requirement> tag does not match the information in the tool_dependencies.xml file. Tool dependencies will be ignored." ) - if tool_dependencies_dict: - if original_tool_dependencies_dict: - # We're generating metadata on an update pulled to a tool shed repository installed into a Galaxy instance, so handle changes to - # tool dependencies appropriately. - handle_existing_tool_dependencies_that_changed_in_update( app, repository, original_tool_dependencies_dict, tool_dependencies_dict ) - metadata_dict[ 'tool_dependencies' ] = tool_dependencies_dict - return metadata_dict -def generate_tool_guid( repository_clone_url, tool ): - """ - Generate a guid for the installed tool. It is critical that this guid matches the guid for - the tool in the Galaxy tool shed from which it is being installed. The form of the guid is - <tool shed host>/repos/<repository owner>/<repository name>/<tool id>/<tool version> - """ - tmp_url = clean_repository_clone_url( repository_clone_url ) - return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) -def generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ): - """Update the received metadata_dict with changes that have been applied to the received tool.""" - # Generate the guid - guid = generate_tool_guid( repository_clone_url, tool ) - # Handle tool.requirements. - tool_requirements = [] - for tr in tool.requirements: - requirement_dict = dict( name=tr.name, - type=tr.type, - version=tr.version ) - tool_requirements.append( requirement_dict ) - # Handle tool.tests. - tool_tests = [] - if tool.tests: - for ttb in tool.tests: - required_files = [] - for required_file in ttb.required_files: - value, extra = required_file - required_files.append( ( value ) ) - inputs = [] - for input in ttb.inputs: - name, value, extra = input - inputs.append( ( name, value ) ) - outputs = [] - for output in ttb.outputs: - name, file_name, extra = output - outputs.append( ( name, strip_path( file_name ) if file_name else None ) ) - test_dict = dict( name=ttb.name, - required_files=required_files, - inputs=inputs, - outputs=outputs ) - tool_tests.append( test_dict ) - tool_dict = dict( id=tool.id, - guid=guid, - name=tool.name, - version=tool.version, - description=tool.description, - version_string_cmd = tool.version_string_cmd, - tool_config=tool_config, - requirements=tool_requirements, - tests=tool_tests ) - if 'tools' in metadata_dict: - metadata_dict[ 'tools' ].append( tool_dict ) - else: - metadata_dict[ 'tools' ] = [ tool_dict ] - return metadata_dict def generate_tool_elem( tool_shed, repository_name, changeset_revision, owner, tool_file_path, tool, tool_section ): if tool_section is not None: tool_elem = SubElement( tool_section, 'tool' ) @@ -652,7 +330,6 @@ version_elem = SubElement( tool_elem, 'version' ) version_elem.text = tool.version return tool_elem - def generate_tool_panel_elem_list( repository_name, repository_clone_url, changeset_revision, tool_panel_dict, repository_tools_tups, owner='' ): """Generate a list of ElementTree Element objects for each section or tool.""" elem_list = [] @@ -958,34 +635,6 @@ if tool: repository_tools_tups.append( ( relative_path, guid, tool ) ) return repository_tools_tups -def get_sample_files_from_disk( repository_files_dir, tool_path = None, relative_install_dir=None, resetting_all_metadata_on_repository=False ): - if resetting_all_metadata_on_repository: - # Keep track of the location where the repository is temporarily cloned so that we can strip it when setting metadata. - work_dir = repository_files_dir - sample_file_metadata_paths = [] - sample_file_copy_paths = [] - for root, dirs, files in os.walk( repository_files_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name.endswith( '.sample' ): - if resetting_all_metadata_on_repository: - full_path_to_sample_file = os.path.join( root, name ) - stripped_path_to_sample_file = full_path_to_sample_file.replace( work_dir, '' ) - if stripped_path_to_sample_file.startswith( '/' ): - stripped_path_to_sample_file = stripped_path_to_sample_file[ 1: ] - relative_path_to_sample_file = os.path.join( relative_install_dir, stripped_path_to_sample_file ) - if os.path.exists( relative_path_to_sample_file ): - sample_file_copy_paths.append( relative_path_to_sample_file ) - else: - sample_file_copy_paths.append( full_path_to_sample_file ) - else: - relative_path_to_sample_file = os.path.join( root, name ) - sample_file_copy_paths.append( relative_path_to_sample_file ) - if tool_path and relative_install_dir: - if relative_path_to_sample_file.startswith( os.path.join( tool_path, relative_install_dir ) ): - relative_path_to_sample_file = relative_path_to_sample_file[ len( tool_path ) + 1 :] - sample_file_metadata_paths.append( relative_path_to_sample_file ) - return sample_file_metadata_paths, sample_file_copy_paths def get_shed_tool_conf_dict( app, shed_tool_conf ): """ Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry @@ -1120,13 +769,6 @@ return shed_url # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. return None -def get_readme_file_names( repository_name ): - readme_files = [ 'readme', 'read_me', 'install' ] - valid_filenames = [ r for r in readme_files ] - for r in readme_files: - valid_filenames.append( '%s.txt' % r ) - valid_filenames.append( '%s.txt' % repository_name ) - return valid_filenames def handle_missing_data_table_entry( app, relative_install_dir, tool_path, repository_tools_tups ): """ Inspect each tool to see if any have input parameters that are dynamically generated select lists that require entries in the @@ -1522,34 +1164,6 @@ removed = True error_message = '' return removed, error_message -def reset_all_metadata_on_installed_repository( trans, id ): - """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" - repository = get_installed_tool_shed_repository( trans, id ) - tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - repository_clone_url = generate_clone_url_for_installed_repository( trans, repository ) - tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) - if relative_install_dir: - original_metadata_dict = repository.metadata - metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - repository_clone_url=repository_clone_url, - shed_config_dict = repository.get_shed_config_dict( trans.app ), - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=False ) - repository.metadata = metadata_dict - if metadata_dict != original_metadata_dict: - update_in_shed_tool_config( trans.app, repository ) - trans.sa_session.add( repository ) - trans.sa_session.flush() - log.debug( 'Metadata has been reset on repository %s.' % repository.name ) - else: - log.debug( 'Metadata did not need to be reset on repository %s.' % repository.name ) - else: - log.debug( 'Error locating installation directory for repository %s.' % repository.name ) - return invalid_file_tups, metadata_dict def to_html_str( text ): """Translates the characters in text to an html string""" translated = [] diff -r 7a51b701af8825baaf4aeb68f422304434c01a10 -r a43b16b64b1a4b0b7fe2d5be0d87a9047c3a54a3 lib/galaxy/util/shed_util_common.py --- a/lib/galaxy/util/shed_util_common.py +++ b/lib/galaxy/util/shed_util_common.py @@ -1,4 +1,4 @@ -import os, shutil, tempfile, logging +import os, shutil, tempfile, logging, string from galaxy import util from galaxy.tools import parameters from galaxy.util import inflector @@ -23,7 +23,7 @@ '&' : '&', '\'' : ''' } MAX_CONTENT_SIZE = 32768 -NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'tool_dependencies.xml' ] +NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'repository_dependencies.xml', 'tool_dependencies.xml' ] GALAXY_ADMIN_TOOL_SHED_CONTROLLER = 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER' TOOL_SHED_ADMIN_CONTROLLER = 'TOOL_SHED_ADMIN_CONTROLLER' VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) @@ -49,6 +49,55 @@ option_value = trans.security.encode_id( repository.id ) repositories_select_field.add_option( option_label, option_value ) return repositories_select_field +def can_generate_tool_dependency_metadata( root, metadata_dict ): + """ + Make sure the combination of name, version and type (the type will be the value of elem.tag) of each root element tag in the tool_dependencies.xml + file is defined in the <requirement> tag for at least one tool in the repository. + """ + can_generate_dependency_metadata = False + for elem in root: + tool_dependency_type = elem.tag + tool_dependency_version = elem.get( 'version', None ) + if tool_dependency_type == 'package': + can_generate_dependency_metadata = False + tool_dependency_name = elem.get( 'name', None ) + if tool_dependency_name and tool_dependency_version: + for tool_dict in metadata_dict.get( 'tools', [] ): + requirements = tool_dict.get( 'requirements', [] ) + for requirement_dict in requirements: + req_name = requirement_dict.get( 'name', None ) + req_version = requirement_dict.get( 'version', None ) + req_type = requirement_dict.get( 'type', None ) + if req_name==tool_dependency_name and req_version==tool_dependency_version and req_type==tool_dependency_type: + can_generate_dependency_metadata = True + break + if requirements and not can_generate_dependency_metadata: + # We've discovered at least 1 combination of name, version and type that is not defined in the <requirement> + # tag for any tool in the repository. + break + if not can_generate_dependency_metadata: + break + elif tool_dependency_type == 'set_environment': + # Here elem is something like: <set_environment version="1.0"> + for env_var_elem in elem: + can_generate_dependency_metadata = False + # <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable> + env_var_name = env_var_elem.get( 'name', None ) + if env_var_name: + for tool_dict in metadata_dict.get( 'tools', [] ): + requirements = tool_dict.get( 'requirements', [] ) + for requirement_dict in requirements: + # {"name": "R_SCRIPT_PATH", "type": "set_environment", "version": null} + req_name = requirement_dict.get( 'name', None ) + req_type = requirement_dict.get( 'type', None ) + if req_name==env_var_name and req_type==tool_dependency_type: + can_generate_dependency_metadata = True + break + if requirements and not can_generate_dependency_metadata: + # We've discovered at least 1 combination of name, version and type that is not defined in the <requirement> + # tag for any tool in the repository. + break + return can_generate_dependency_metadata def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): """ Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make @@ -93,6 +142,34 @@ correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) return invalid_files_and_errors_tups +def clean_repository_clone_url( repository_clone_url ): + if repository_clone_url.find( '@' ) > 0: + # We have an url that includes an authenticated user, something like: + # http://test@bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split( '@' ) + tmp_url = items[ 1 ] + elif repository_clone_url.find( '//' ) > 0: + # We have an url that includes only a protocol, something like: + # http://bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split( '//' ) + tmp_url = items[ 1 ] + else: + tmp_url = repository_clone_url + return tmp_url +def clean_repository_metadata( trans, id, changeset_revisions ): + # Delete all repository_metadata records associated with the repository that have a changeset_revision that is not in changeset_revisions. + # We sometimes see multiple records with the same changeset revision value - no idea how this happens. We'll assume we can delete the older + # records, so we'll order by update_time descending and delete records that have the same changeset_revision we come across later.. + changeset_revisions_checked = [] + for repository_metadata in trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) \ + .order_by( trans.model.RepositoryMetadata.table.c.changeset_revision, + trans.model.RepositoryMetadata.table.c.update_time.desc() ): + changeset_revision = repository_metadata.changeset_revision + can_delete = changeset_revision in changeset_revisions_checked or changeset_revision not in changeset_revisions + if can_delete: + trans.sa_session.delete( repository_metadata ) + trans.sa_session.flush() def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" try: @@ -107,6 +184,91 @@ error_message = 'Error cloning repository: %s' % str( e ) log.debug( error_message ) return False, error_message +def compare_changeset_revisions( ancestor_changeset_revision, ancestor_metadata_dict, current_changeset_revision, current_metadata_dict ): + # The metadata associated with ancestor_changeset_revision is ancestor_metadata_dict. This changeset_revision is an ancestor of + # current_changeset_revision which is associated with current_metadata_dict. A new repository_metadata record will be created only + # when this method returns the string 'not equal and not subset'. + ancestor_datatypes = ancestor_metadata_dict.get( 'datatypes', [] ) + ancestor_tools = ancestor_metadata_dict.get( 'tools', [] ) + ancestor_guids = [ tool_dict[ 'guid' ] for tool_dict in ancestor_tools ] + ancestor_guids.sort() + ancestor_tool_dependencies = ancestor_metadata_dict.get( 'tool_dependencies', [] ) + ancestor_workflows = ancestor_metadata_dict.get( 'workflows', [] ) + current_datatypes = current_metadata_dict.get( 'datatypes', [] ) + current_tools = current_metadata_dict.get( 'tools', [] ) + current_guids = [ tool_dict[ 'guid' ] for tool_dict in current_tools ] + current_guids.sort() + current_tool_dependencies = current_metadata_dict.get( 'tool_dependencies', [] ) + current_workflows = current_metadata_dict.get( 'workflows', [] ) + # Handle case where no metadata exists for either changeset. + if not ancestor_guids and not current_guids and not ancestor_workflows and not current_workflows and not ancestor_datatypes and not current_datatypes: + return 'no metadata' + workflow_comparison = compare_workflows( ancestor_workflows, current_workflows ) + datatype_comparison = compare_datatypes( ancestor_datatypes, current_datatypes ) + # Handle case where all metadata is the same. + if ancestor_guids == current_guids and workflow_comparison == 'equal' and datatype_comparison == 'equal': + return 'equal' + if workflow_comparison in [ 'equal', 'subset' ] and datatype_comparison in [ 'equal', 'subset' ]: + is_subset = True + for guid in ancestor_guids: + if guid not in current_guids: + is_subset = False + break + if is_subset: + return 'subset' + return 'not equal and not subset' +def compare_datatypes( ancestor_datatypes, current_datatypes ): + # Determine if ancestor_datatypes is the same as current_datatypes + # or if ancestor_datatypes is a subset of current_datatypes. Each + # datatype dict looks something like: + # {"dtype": "galaxy.datatypes.images:Image", "extension": "pdf", "mimetype": "application/pdf"} + if len( ancestor_datatypes ) <= len( current_datatypes ): + for ancestor_datatype in ancestor_datatypes: + # Currently the only way to differentiate datatypes is by name. + ancestor_datatype_dtype = ancestor_datatype[ 'dtype' ] + ancestor_datatype_extension = ancestor_datatype[ 'extension' ] + ancestor_datatype_mimetype = ancestor_datatype.get( 'mimetype', None ) + found_in_current = False + for current_datatype in current_datatypes: + if current_datatype[ 'dtype' ] == ancestor_datatype_dtype and \ + current_datatype[ 'extension' ] == ancestor_datatype_extension and \ + current_datatype.get( 'mimetype', None ) == ancestor_datatype_mimetype: + found_in_current = True + break + if not found_in_current: + return 'not equal and not subset' + if len( ancestor_datatypes ) == len( current_datatypes ): + return 'equal' + else: + return 'subset' + return 'not equal and not subset' +def compare_workflows( ancestor_workflows, current_workflows ): + # Determine if ancestor_workflows is the same as current_workflows + # or if ancestor_workflows is a subset of current_workflows. + if len( ancestor_workflows ) <= len( current_workflows ): + for ancestor_workflow_tup in ancestor_workflows: + # ancestor_workflows is a list of tuples where each contained tuple is + # [ <relative path to the .ga file in the repository>, <exported workflow dict> ] + ancestor_workflow_dict = ancestor_workflow_tup[1] + # Currently the only way to differentiate workflows is by name. + ancestor_workflow_name = ancestor_workflow_dict[ 'name' ] + num_ancestor_workflow_steps = len( ancestor_workflow_dict[ 'steps' ] ) + found_in_current = False + for current_workflow_tup in current_workflows: + current_workflow_dict = current_workflow_tup[1] + # Assume that if the name and number of steps are euqal, + # then the workflows are the same. Of course, this may + # not be true... + if current_workflow_dict[ 'name' ] == ancestor_workflow_name and len( current_workflow_dict[ 'steps' ] ) == num_ancestor_workflow_steps: + found_in_current = True + break + if not found_in_current: + return 'not equal and not subset' + if len( ancestor_workflows ) == len( current_workflows ): + return 'equal' + else: + return 'subset' + return 'not equal and not subset' def concat_messages( msg1, msg2 ): if msg1: if msg2: @@ -165,6 +327,60 @@ return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) else: return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name ) +def generate_datatypes_metadata( datatypes_config, metadata_dict ): + """Update the received metadata_dict with information from the parsed datatypes_config.""" + tree = ElementTree.parse( datatypes_config ) + root = tree.getroot() + ElementInclude.include( root ) + repository_datatype_code_files = [] + datatype_files = root.find( 'datatype_files' ) + if datatype_files: + for elem in datatype_files.findall( 'datatype_file' ): + name = elem.get( 'name', None ) + repository_datatype_code_files.append( name ) + metadata_dict[ 'datatype_files' ] = repository_datatype_code_files + datatypes = [] + registration = root.find( 'registration' ) + if registration: + for elem in registration.findall( 'datatype' ): + datatypes_dict = {} + display_in_upload = elem.get( 'display_in_upload', None ) + if display_in_upload: + datatypes_dict[ 'display_in_upload' ] = display_in_upload + dtype = elem.get( 'type', None ) + if dtype: + datatypes_dict[ 'dtype' ] = dtype + extension = elem.get( 'extension', None ) + if extension: + datatypes_dict[ 'extension' ] = extension + max_optional_metadata_filesize = elem.get( 'max_optional_metadata_filesize', None ) + if max_optional_metadata_filesize: + datatypes_dict[ 'max_optional_metadata_filesize' ] = max_optional_metadata_filesize + mimetype = elem.get( 'mimetype', None ) + if mimetype: + datatypes_dict[ 'mimetype' ] = mimetype + subclass = elem.get( 'subclass', None ) + if subclass: + datatypes_dict[ 'subclass' ] = subclass + if datatypes_dict: + datatypes.append( datatypes_dict ) + if datatypes: + metadata_dict[ 'datatypes' ] = datatypes + return metadata_dict +def generate_environment_dependency_metadata( elem, tool_dependencies_dict ): + """The value of env_var_name must match the value of the "set_environment" type in the tool config's <requirements> tag set.""" + requirements_dict = {} + for env_elem in elem: + env_name = env_elem.get( 'name', None ) + if env_name: + requirements_dict [ 'name' ] = env_name + requirements_dict [ 'type' ] = 'environment variable' + if requirements_dict: + if 'set_environment' in tool_dependencies_dict: + tool_dependencies_dict[ 'set_environment' ].append( requirements_dict ) + else: + tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] + return tool_dependencies_dict def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): if as_html: new_line = '<br/>' @@ -201,7 +417,7 @@ correction_msg = exception_msg.replace( '<br/>', new_line ).replace( '<b>', bold_start ).replace( '</b>', bold_end ) message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) return message -def generate_metadata_for_changeset_revision( app, repository, repository_clone_url, shed_config_dict={}, relative_install_dir=None, repository_files_dir=None, +def generate_metadata_for_changeset_revision( app, repository, repository_clone_url, shed_config_dict=None, relative_install_dir=None, repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ): """ Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip, @@ -212,6 +428,8 @@ The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. """ + if shed_config_dict is None: + shed_config_dict = {} if updating_installed_repository: # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which # we have pulled updates. @@ -270,18 +488,23 @@ if '.hg' in dirs: dirs.remove( '.hg' ) for name in files: + # See if we have a repository dependencies defined. + if name == 'repository_dependencies.xml': + relative_path_to_repository_dependencies = get_relative_path_to_repository_file( root, + name, + relative_install_dir, + work_dir, + shed_config_dict, + resetting_all_metadata_on_repository ) + metadata_dict = generate_repository_dependency_metadata( relative_path_to_repository_dependencies, metadata_dict ) # See if we have a READ_ME file. - if name.lower() in readme_file_names: - if resetting_all_metadata_on_repository: - full_path_to_readme = os.path.join( root, name ) - stripped_path_to_readme = full_path_to_readme.replace( work_dir, '' ) - if stripped_path_to_readme.startswith( '/' ): - stripped_path_to_readme = stripped_path_to_readme[ 1: ] - relative_path_to_readme = os.path.join( relative_install_dir, stripped_path_to_readme ) - else: - relative_path_to_readme = os.path.join( root, name ) - if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_readme.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): - relative_path_to_readme = relative_path_to_readme[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] + elif name.lower() in readme_file_names: + relative_path_to_readme = get_relative_path_to_repository_file( root, + name, + relative_install_dir, + work_dir, + shed_config_dict, + resetting_all_metadata_on_repository ) metadata_dict[ 'readme' ] = relative_path_to_readme # See if we have a tool config. elif name not in NOT_TOOL_CONFIGS and name.endswith( '.xml' ): @@ -311,16 +534,14 @@ invalid_tool_configs.append( name ) break if can_set_metadata: - if resetting_all_metadata_on_repository: - full_path_to_tool_config = os.path.join( root, name ) - stripped_path_to_tool_config = full_path_to_tool_config.replace( work_dir, '' ) - if stripped_path_to_tool_config.startswith( '/' ): - stripped_path_to_tool_config = stripped_path_to_tool_config[ 1: ] - relative_path_to_tool_config = os.path.join( relative_install_dir, stripped_path_to_tool_config ) - else: - relative_path_to_tool_config = os.path.join( root, name ) - if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_tool_config.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): - relative_path_to_tool_config = relative_path_to_tool_config[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] + relative_path_to_tool_config = get_relative_path_to_repository_file( root, + name, + relative_install_dir, + work_dir, + shed_config_dict, + resetting_all_metadata_on_repository ) + + metadata_dict = generate_tool_metadata( relative_path_to_tool_config, tool, repository_clone_url, metadata_dict ) else: for tup in invalid_files_and_errors_tups: @@ -350,6 +571,131 @@ app.config.tool_data_path = original_tool_data_path app.config.tool_data_table_config_path = original_tool_data_table_config_path return metadata_dict, invalid_file_tups +def generate_package_dependency_metadata( elem, tool_dependencies_dict ): + """The value of package_name must match the value of the "package" type in the tool config's <requirements> tag set.""" + requirements_dict = {} + package_name = elem.get( 'name', None ) + package_version = elem.get( 'version', None ) + if package_name and package_version: + dependency_key = '%s/%s' % ( package_name, package_version ) + requirements_dict [ 'name' ] = package_name + requirements_dict [ 'version' ] = package_version + requirements_dict [ 'type' ] = 'package' + for sub_elem in elem: + if sub_elem.tag == 'readme': + requirements_dict[ 'readme' ] = sub_elem.text + if requirements_dict: + tool_dependencies_dict[ dependency_key ] = requirements_dict + return tool_dependencies_dict +def generate_repository_dependency_metadata( repository_dependencies_config, metadata_dict ): + repository_dependencies_tups = [] + try: + # Make sure we're looking at a valid repository_dependencies.xml file. + tree = util.parse_xml( repository_dependencies_config ) + root = element_tree.getroot() + is_valid = element_tree_root.tag == 'repositories' + except Exception, e: + log.debug( "Error parsing %s, exception: %s" % ( repository_dependencies_config, str( e ) ) ) + is_valid = False + if is_valid: + for repository_elem in root.findall( 'repository' ): + repository_dependencies_tups.append( ( repository_elem.attrib[ 'toolshed' ], + repository_elem.attrib[ 'name' ], + repository_elem.attrib[ 'owner'], + repository_elem.attrib[ 'changeset_revision' ] ) ) + if repository_dependencies_tups: + metadata_dict[ 'repository_dependencies' ] = repository_dependencies_tups + return metadata_dict +def generate_tool_dependency_metadata( app, repository, tool_dependencies_config, metadata_dict, original_repository_metadata=None ): + """ + If the combination of name, version and type of each element is defined in the <requirement> tag for at least one tool in the repository, + then update the received metadata_dict with information from the parsed tool_dependencies_config. + """ + if original_repository_metadata: + # Keep a copy of the original tool dependencies dictionary in the metadata. + original_tool_dependencies_dict = original_repository_metadata.get( 'tool_dependencies', None ) + else: + original_tool_dependencies_dict = None + try: + tree = ElementTree.parse( tool_dependencies_config ) + except Exception, e: + log.debug( "Exception attempting to parse tool_dependencies.xml: %s" %str( e ) ) + return metadata_dict + root = tree.getroot() + ElementInclude.include( root ) + tool_dependencies_dict = {} + if can_generate_tool_dependency_metadata( root, metadata_dict ): + for elem in root: + if elem.tag == 'package': + tool_dependencies_dict = generate_package_dependency_metadata( elem, tool_dependencies_dict ) + elif elem.tag == 'set_environment': + tool_dependencies_dict = generate_environment_dependency_metadata( elem, tool_dependencies_dict ) + # Handle tool dependency installation via other means here (future). + if tool_dependencies_dict: + metadata_dict[ 'tool_dependencies' ] = tool_dependencies_dict + else: + log.debug( "Name, version and type from the <requirement> tag does not match the information in the tool_dependencies.xml file. Tool dependencies will be ignored." ) + if tool_dependencies_dict: + if original_tool_dependencies_dict: + # We're generating metadata on an update pulled to a tool shed repository installed into a Galaxy instance, so handle changes to + # tool dependencies appropriately. + handle_existing_tool_dependencies_that_changed_in_update( app, repository, original_tool_dependencies_dict, tool_dependencies_dict ) + metadata_dict[ 'tool_dependencies' ] = tool_dependencies_dict + return metadata_dict +def generate_tool_guid( repository_clone_url, tool ): + """ + Generate a guid for the installed tool. It is critical that this guid matches the guid for + the tool in the Galaxy tool shed from which it is being installed. The form of the guid is + <tool shed host>/repos/<repository owner>/<repository name>/<tool id>/<tool version> + """ + tmp_url = clean_repository_clone_url( repository_clone_url ) + return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) +def generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ): + """Update the received metadata_dict with changes that have been applied to the received tool.""" + # Generate the guid + guid = generate_tool_guid( repository_clone_url, tool ) + # Handle tool.requirements. + tool_requirements = [] + for tr in tool.requirements: + requirement_dict = dict( name=tr.name, + type=tr.type, + version=tr.version ) + tool_requirements.append( requirement_dict ) + # Handle tool.tests. + tool_tests = [] + if tool.tests: + for ttb in tool.tests: + required_files = [] + for required_file in ttb.required_files: + value, extra = required_file + required_files.append( ( value ) ) + inputs = [] + for input in ttb.inputs: + name, value, extra = input + inputs.append( ( name, value ) ) + outputs = [] + for output in ttb.outputs: + name, file_name, extra = output + outputs.append( ( name, strip_path( file_name ) if file_name else None ) ) + test_dict = dict( name=ttb.name, + required_files=required_files, + inputs=inputs, + outputs=outputs ) + tool_tests.append( test_dict ) + tool_dict = dict( id=tool.id, + guid=guid, + name=tool.name, + version=tool.version, + description=tool.description, + version_string_cmd = tool.version_string_cmd, + tool_config=tool_config, + requirements=tool_requirements, + tests=tool_tests ) + if 'tools' in metadata_dict: + metadata_dict[ 'tools' ].append( tool_dict ) + else: + metadata_dict[ 'tools' ] = [ tool_dict ] + return metadata_dict def get_changectx_for_changeset( repo, changeset_revision, **kwd ): """Retrieve a specified changectx from a repository""" for changeset in repo.changelog: @@ -393,6 +739,13 @@ if deleted: return 'DELETED' return None +def get_readme_file_names( repository_name ): + readme_files = [ 'readme', 'read_me', 'install' ] + valid_filenames = [ r for r in readme_files ] + for r in readme_files: + valid_filenames.append( '%s.txt' % r ) + valid_filenames.append( '%s.txt' % repository_name ) + return valid_filenames def get_repository_file_contents( file_path ): if is_gzip( file_path ): to_html = to_html_str( '\ngzip compressed file\n' ) @@ -454,6 +807,34 @@ fh.close() return tmp_filename return None +def get_sample_files_from_disk( repository_files_dir, tool_path = None, relative_install_dir=None, resetting_all_metadata_on_repository=False ): + if resetting_all_metadata_on_repository: + # Keep track of the location where the repository is temporarily cloned so that we can strip it when setting metadata. + work_dir = repository_files_dir + sample_file_metadata_paths = [] + sample_file_copy_paths = [] + for root, dirs, files in os.walk( repository_files_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name.endswith( '.sample' ): + if resetting_all_metadata_on_repository: + full_path_to_sample_file = os.path.join( root, name ) + stripped_path_to_sample_file = full_path_to_sample_file.replace( work_dir, '' ) + if stripped_path_to_sample_file.startswith( '/' ): + stripped_path_to_sample_file = stripped_path_to_sample_file[ 1: ] + relative_path_to_sample_file = os.path.join( relative_install_dir, stripped_path_to_sample_file ) + if os.path.exists( relative_path_to_sample_file ): + sample_file_copy_paths.append( relative_path_to_sample_file ) + else: + sample_file_copy_paths.append( full_path_to_sample_file ) + else: + relative_path_to_sample_file = os.path.join( root, name ) + sample_file_copy_paths.append( relative_path_to_sample_file ) + if tool_path and relative_install_dir: + if relative_path_to_sample_file.startswith( os.path.join( tool_path, relative_install_dir ) ): + relative_path_to_sample_file = relative_path_to_sample_file[ len( tool_path ) + 1 :] + sample_file_metadata_paths.append( relative_path_to_sample_file ) + return sample_file_metadata_paths, sample_file_copy_paths def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): parent_id = None # Compare from most recent to oldest. @@ -472,6 +853,20 @@ if parent_id is None: # The tool did not change through all of the changeset revisions. return old_id +def get_relative_path_to_repository_file( root, name, relative_install_dir, work_dir, shed_config_dict, resetting_all_metadata_on_repository ): + if resetting_all_metadata_on_repository: + full_path_to_file = os.path.join( root, name ) + stripped_path_to_file = full_path_to_file.replace( work_dir, '' ) + if stripped_path_to_file.startswith( '/' ): + stripped_path_to_file = stripped_path_to_file[ 1: ] + relative_path_to_file = os.path.join( relative_install_dir, stripped_path_to_file ) + else: + relative_path_to_file = os.path.join( root, name ) + if relative_install_dir and \ + shed_config_dict.get( 'tool_path' ) and \ + relative_path_to_file.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): + relative_path_to_file = relative_path_to_file[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] + return relative_path_to_file def handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ): # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. message = '' @@ -566,6 +961,34 @@ shutil.rmtree( dir ) except: pass +def reset_all_metadata_on_installed_repository( trans, id ): + """Reset all metadata on a single tool shed repository installed into a Galaxy instance.""" + repository = get_installed_tool_shed_repository( trans, id ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) + repository_clone_url = generate_clone_url_for_installed_repository( trans, repository ) + tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app ) + if relative_install_dir: + original_metadata_dict = repository.metadata + metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app, + repository=repository, + repository_clone_url=repository_clone_url, + shed_config_dict = repository.get_shed_config_dict( trans.app ), + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=False ) + repository.metadata = metadata_dict + if metadata_dict != original_metadata_dict: + update_in_shed_tool_config( trans.app, repository ) + trans.sa_session.add( repository ) + trans.sa_session.flush() + log.debug( 'Metadata has been reset on repository %s.' % repository.name ) + else: + log.debug( 'Metadata did not need to be reset on repository %s.' % repository.name ) + else: + log.debug( 'Error locating installation directory for repository %s.' % repository.name ) + return invalid_file_tups, metadata_dict def reset_all_metadata_on_repository_in_tool_shed( trans, id ): """Reset all metadata on a single repository in a tool shed.""" def reset_all_tool_versions( trans, id, repo ): diff -r 7a51b701af8825baaf4aeb68f422304434c01a10 -r a43b16b64b1a4b0b7fe2d5be0d87a9047c3a54a3 templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -83,7 +83,7 @@ hg clone <a href="${clone_str}">${clone_str}</a></%def> -<%def name="render_repository_items( repository_metadata_id, metadata, can_set_metadata=False )"> +<%def name="render_repository_items( repository_metadata_id, changeset_revision, metadata, can_set_metadata=False )"><% from galaxy.tool_shed.encoding_util import tool_shed_encode %> %if metadata or can_set_metadata: <p/> @@ -91,6 +91,28 @@ <div class="toolFormTitle">Preview tools and inspect metadata by tool version</div><div class="toolFormBody"> %if metadata: + %if 'repository_dependencies' in metadata: + <div class="form-row"> + <table class="grid"> + <tr> + <td><b>tool shed</b></td> + <td><b>name</b></td> + <td><b>version</b></td> + <td><b>type</b></td> + </tr> + %for repository_dependency_tup in metadata[ 'repository_dependencies' ]: + <% toolshed, name, owner, changeset_revision = repository_dependency_tup %> + <tr> + <td>${toolshed | h}</td> + <td>${name | h}</td> + <td>${owner | h}</td> + <td>${changeset_revision | h}</td> + </tr> + %endfor + </table> + </div> + <div style="clear: both"></div> + %endif %if 'tool_dependencies' in metadata: <% # See if tool dependencies are packages, environment settings or both. diff -r 7a51b701af8825baaf4aeb68f422304434c01a10 -r a43b16b64b1a4b0b7fe2d5be0d87a9047c3a54a3 templates/webapps/community/repository/manage_repository.mako --- a/templates/webapps/community/repository/manage_repository.mako +++ b/templates/webapps/community/repository/manage_repository.mako @@ -204,7 +204,7 @@ </form></div></div> -${render_repository_items( repository_metadata_id, metadata, can_set_metadata=True )} +${render_repository_items( repository_metadata_id, changeset_revision, metadata, can_set_metadata=True )} <p/><div class="toolForm"><div class="toolFormTitle">Manage categories</div> diff -r 7a51b701af8825baaf4aeb68f422304434c01a10 -r a43b16b64b1a4b0b7fe2d5be0d87a9047c3a54a3 templates/webapps/community/repository/preview_tools_in_changeset.mako --- a/templates/webapps/community/repository/preview_tools_in_changeset.mako +++ b/templates/webapps/community/repository/preview_tools_in_changeset.mako @@ -79,4 +79,4 @@ </div></div><p/> -${render_repository_items( repository_metadata_id, metadata )} +${render_repository_items( repository_metadata_id, changeset_revision, metadata, can_set_metadata=False )} diff -r 7a51b701af8825baaf4aeb68f422304434c01a10 -r a43b16b64b1a4b0b7fe2d5be0d87a9047c3a54a3 templates/webapps/community/repository/view_repository.mako --- a/templates/webapps/community/repository/view_repository.mako +++ b/templates/webapps/community/repository/view_repository.mako @@ -181,7 +181,7 @@ %endif </div></div> -${render_repository_items( repository_metadata_id, metadata )} +${render_repository_items( repository_metadata_id, changeset_revision, metadata, can_set_metadata=False )} %if repository.categories: <p/><div class="toolForm"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 15 Nov '12

15 Nov '12

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/e1895e14176e/ changeset: e1895e14176e user: greg date: 2012-11-15 20:57:34 summary: Refactor ~/galaxy/util/shed_util.py affected #: 13 files diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -6,6 +6,7 @@ from galaxy.tools import ToolSection from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * +from galaxy.util.shed_util_common import * from galaxy.util.odict import odict from galaxy.tool_shed.common_util import * diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/tool_shed/update_manager.py --- a/lib/galaxy/tool_shed/update_manager.py +++ b/lib/galaxy/tool_shed/update_manager.py @@ -4,6 +4,7 @@ import threading, urllib2, logging from galaxy.util import string_as_bool from galaxy.util.shed_util import * +from galaxy.util.shed_util_common import * log = logging.getLogger( __name__ ) diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -33,6 +33,7 @@ from galaxy.util.hash_util import * from galaxy.util import listify from galaxy.util.shed_util import * +from galaxy.util.shed_util_common import * from galaxy.web import url_for from galaxy.visualization.genome.visual_analytics import TracksterConfig diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1,14 +1,9 @@ -import sys, os, tempfile, shutil, logging, string, urllib2 -import galaxy.tools.data -from datetime import date, datetime, timedelta +import os, tempfile, shutil, logging, urllib2 from galaxy import util -from galaxy.web import url_for -from galaxy.web.form_builder import SelectField -from galaxy.tools import parameters from galaxy.datatypes.checkers import * from galaxy.datatypes.sniff import is_column_based from galaxy.util.json import * -from galaxy.util import inflector +from galaxy.util.shed_util_common import * from galaxy.tools.search import ToolBoxSearch from galaxy.tool_shed.tool_dependencies.install_util import create_or_update_tool_dependency, install_package, set_environment from galaxy.tool_shed.encoding_util import * @@ -26,19 +21,6 @@ log = logging.getLogger( __name__ ) -GALAXY_ADMIN_TOOL_SHED_CONTROLLER = 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER' -INITIAL_CHANGELOG_HASH = '000000000000' -# Characters that must be html escaped -MAPPED_CHARS = { '>' :'>', - '<' :'<', - '"' : '"', - '&' : '&', - '\'' : ''' } -MAX_CONTENT_SIZE = 32768 -NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'tool_dependencies.xml' ] -VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) -TOOL_SHED_ADMIN_CONTROLLER = 'TOOL_SHED_ADMIN_CONTROLLER' - def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ): # A tool shed repository is being installed so change the shed_tool_conf file. Parse the config file to generate the entire list # of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have @@ -175,27 +157,6 @@ except: pass return converter_path, display_path -def build_repository_ids_select_field( trans, cntrller, name='repository_ids', multiple=True, display='checkboxes' ): - """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" - repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - if cntrller == TOOL_SHED_ADMIN_CONTROLLER: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - elif cntrller == GALAXY_ADMIN_TOOL_SHED_CONTROLLER: - for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ - .order_by( trans.model.ToolShedRepository.table.c.name, - trans.model.ToolShedRepository.table.c.owner ): - option_label = '%s (%s)' % ( repository.name, repository.owner ) - option_value = trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - return repositories_select_field def can_generate_tool_dependency_metadata( root, metadata_dict ): """ Make sure the combination of name, version and type (the type will be the value of elem.tag) of each root element tag in the tool_dependencies.xml @@ -245,50 +206,6 @@ # tag for any tool in the repository. break return can_generate_dependency_metadata -def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): - """ - Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make - sure the files exist. - """ - invalid_files_and_errors_tups = [] - correction_msg = '' - for input_param in tool.input_params: - if isinstance( input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: - # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, make sure all requirements exist. - options = input_param.dynamic_options or input_param.options - if options: - if options.tool_data_table or options.missing_tool_data_table_name: - # Make sure the repository contains a tool_data_table_conf.xml.sample file. - sample_tool_data_table_conf = get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) - if sample_tool_data_table_conf: - error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf ) - if error: - invalid_files_and_errors_tups.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) - else: - options.missing_tool_data_table_name = None - else: - correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. Upload a file named tool_data_table_conf.xml.sample " - correction_msg += "to the repository that includes the required entry to correct this error.<br/>" - invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) - if options.index_file or options.missing_index_file: - # Make sure the repository contains the required xxx.loc.sample file. - index_file = options.index_file or options.missing_index_file - index_file_name = strip_path( index_file ) - sample_found = False - for sample_file in sample_files: - sample_file_name = strip_path( sample_file ) - if sample_file_name == '%s.sample' % index_file_name: - options.index_file = index_file_name - options.missing_index_file = None - if options.tool_data_table: - options.tool_data_table.missing_index_file = None - sample_found = True - break - if not sample_found: - correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file ) - correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) - invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) - return invalid_files_and_errors_tups def clean_repository_metadata( trans, id, changeset_revisions ): # Delete all repository_metadata records associated with the repository that have a changeset_revision that is not in changeset_revisions. # We sometimes see multiple records with the same changeset revision value - no idea how this happens. We'll assume we can delete the older @@ -388,17 +305,6 @@ else: return 'subset' return 'not equal and not subset' -def concat_messages( msg1, msg2 ): - if msg1: - if msg2: - message = '%s %s' % ( msg1, msg2 ) - else: - message = msg1 - elif msg2: - message = msg2 - else: - message = '' - return message def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): # Persist the current in-memory list of config_elems to a file named by the value of config_filename. fd, filename = tempfile.mkstemp() @@ -439,35 +345,6 @@ # Eliminate the port, if any, since it will result in an invalid directory name. return tool_shed_url.split( ':' )[ 0 ] return tool_shed_url.rstrip( '/' ) -def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): - """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" - try: - commands.clone( get_configured_ui(), - str( repository_clone_url ), - dest=str( repository_file_dir ), - pull=True, - noupdate=False, - rev=util.listify( str( ctx_rev ) ) ) - return True, None - except Exception, e: - error_message = 'Error cloning repository: %s' % str( e ) - log.debug( error_message ) - return False, error_message -def copy_sample_file( app, filename, dest_path=None ): - """Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path is ~/tool-data.""" - if dest_path is None: - dest_path = os.path.abspath( app.config.tool_data_path ) - sample_file_name = strip_path( filename ) - copied_file = sample_file_name.replace( '.sample', '' ) - full_source_path = os.path.abspath( filename ) - full_destination_path = os.path.join( dest_path, sample_file_name ) - # Don't copy a file to itself - not sure how this happens, but sometimes it does... - if full_source_path != full_destination_path: - # It's ok to overwrite the .sample version of the file. - shutil.copy( full_source_path, full_destination_path ) - # Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance. - if not os.path.exists( os.path.join( dest_path, copied_file ) ): - shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) def copy_sample_files( app, sample_files, tool_path=None, sample_files_copied=None, dest_path=None ): """ Copy all appropriate files to dest_path in the local Galaxy environment that have not already been copied. Those that have been copied @@ -484,15 +361,6 @@ # Attempt to ensure we're copying an appropriate file. if is_data_index_sample_file( filename ): copy_sample_file( app, filename, dest_path=dest_path ) -def create_repo_info_dict( repository, owner, repository_clone_url, changeset_revision, ctx_rev, metadata ): - repo_info_dict = {} - repo_info_dict[ repository.name ] = ( repository.description, - repository_clone_url, - changeset_revision, - ctx_rev, - owner, - metadata.get( 'tool_dependencies', None ) ) - return repo_info_dict def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): return dict( tool_shed=tool_shed, repository_name=name, @@ -501,20 +369,6 @@ tool_dicts=tool_dicts, converter_path=converter_path, display_path=display_path ) -def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ): - downloadable = is_downloadable( metadata_dict ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) - if repository_metadata: - repository_metadata.metadata = metadata_dict - repository_metadata.downloadable = downloadable - else: - repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, - changeset_revision=changeset_revision, - metadata=metadata_dict, - downloadable=downloadable ) - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() - return repository_metadata def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used @@ -618,15 +472,6 @@ """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) return url_join( tool_shed_url, 'repos', repository.owner, repository.name ) -def generate_clone_url_for_repository_in_tool_shed( trans, repository ): - """Generate the URL for cloning a repository that is in the tool shed.""" - base_url = url_for( '/', qualified=True ).rstrip( '/' ) - if trans.user: - protocol, base = base_url.split( '://' ) - username = '%s@' % trans.user.username - return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) - else: - return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name ) def generate_datatypes_metadata( datatypes_config, metadata_dict ): """Update the received metadata_dict with information from the parsed datatypes_config.""" tree = ElementTree.parse( datatypes_config ) @@ -681,191 +526,6 @@ else: tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] return tool_dependencies_dict -def generate_metadata_for_changeset_revision( app, repository, repository_clone_url, shed_config_dict={}, relative_install_dir=None, repository_files_dir=None, - resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ): - """ - Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip, - the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's - disk files, so the value of repository_files_dir will not always be repository.repo_path( app ) (it could be an absolute path to a temporary - directory containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path( app ). - - The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries - should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. - """ - if updating_installed_repository: - # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which - # we have pulled updates. - original_repository_metadata = repository.metadata - else: - original_repository_metadata = None - readme_file_names = get_readme_file_names( repository.name ) - metadata_dict = { 'shed_config_filename': shed_config_dict.get( 'config_filename' ) } - invalid_file_tups = [] - invalid_tool_configs = [] - tool_dependencies_config = None - original_tool_data_path = app.config.tool_data_path - original_tool_data_table_config_path = app.config.tool_data_table_config_path - if resetting_all_metadata_on_repository: - if not relative_install_dir: - raise Exception( "The value of repository.repo_path( app ) must be sent when resetting all metadata on a repository." ) - # Keep track of the location where the repository is temporarily cloned so that we can strip the path when setting metadata. The value of - # repository_files_dir is the full path to the temporary directory to which the repository was cloned. - work_dir = repository_files_dir - files_dir = repository_files_dir - # Since we're working from a temporary directory, we can safely copy sample files included in the repository to the repository root. - app.config.tool_data_path = repository_files_dir - app.config.tool_data_table_config_path = repository_files_dir - else: - # Use a temporary working directory to copy all sample files. - work_dir = tempfile.mkdtemp() - # All other files are on disk in the repository's repo_path, which is the value of relative_install_dir. - files_dir = relative_install_dir - if shed_config_dict.get( 'tool_path' ): - files_dir = os.path.join( shed_config_dict['tool_path'], files_dir ) - app.config.tool_data_path = work_dir - app.config.tool_data_table_config_path = work_dir - # Handle proprietary datatypes, if any. - datatypes_config = get_config_from_disk( 'datatypes_conf.xml', files_dir ) - if datatypes_config: - metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict ) - # Get the relative path to all sample files included in the repository for storage in the repository's metadata. - sample_file_metadata_paths, sample_file_copy_paths = get_sample_files_from_disk( repository_files_dir=files_dir, - tool_path=shed_config_dict.get( 'tool_path' ), - relative_install_dir=relative_install_dir, - resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) - if sample_file_metadata_paths: - metadata_dict[ 'sample_files' ] = sample_file_metadata_paths - # Copy all sample files included in the repository to a single directory location so we can load tools that depend on them. - for sample_file in sample_file_copy_paths: - copy_sample_file( app, sample_file, dest_path=work_dir ) - # If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory. - relative_path, filename = os.path.split( sample_file ) - if filename == 'tool_data_table_conf.xml.sample': - new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file, - tool_data_path=original_tool_data_path, - shed_tool_data_table_config=app.config.shed_tool_data_table_config, - persist=persist ) - for root, dirs, files in os.walk( files_dir ): - if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: - if '.hg' in dirs: - dirs.remove( '.hg' ) - for name in files: - # See if we have a READ_ME file. - if name.lower() in readme_file_names: - if resetting_all_metadata_on_repository: - full_path_to_readme = os.path.join( root, name ) - stripped_path_to_readme = full_path_to_readme.replace( work_dir, '' ) - if stripped_path_to_readme.startswith( '/' ): - stripped_path_to_readme = stripped_path_to_readme[ 1: ] - relative_path_to_readme = os.path.join( relative_install_dir, stripped_path_to_readme ) - else: - relative_path_to_readme = os.path.join( root, name ) - if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_readme.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): - relative_path_to_readme = relative_path_to_readme[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] - metadata_dict[ 'readme' ] = relative_path_to_readme - # See if we have a tool config. - elif name not in NOT_TOOL_CONFIGS and name.endswith( '.xml' ): - full_path = str( os.path.abspath( os.path.join( root, name ) ) ) - if os.path.getsize( full_path ) > 0: - if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ] - or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ): - try: - # Make sure we're looking at a tool config and not a display application config or something else. - element_tree = util.parse_xml( full_path ) - element_tree_root = element_tree.getroot() - is_tool = element_tree_root.tag == 'tool' - except Exception, e: - log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) ) - is_tool = False - if is_tool: - tool, valid, error_message = load_tool_from_config( app, full_path ) - if tool is None: - if not valid: - invalid_file_tups.append( ( name, error_message ) ) - else: - invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_file_metadata_paths ) - can_set_metadata = True - for tup in invalid_files_and_errors_tups: - if name in tup: - can_set_metadata = False - invalid_tool_configs.append( name ) - break - if can_set_metadata: - if resetting_all_metadata_on_repository: - full_path_to_tool_config = os.path.join( root, name ) - stripped_path_to_tool_config = full_path_to_tool_config.replace( work_dir, '' ) - if stripped_path_to_tool_config.startswith( '/' ): - stripped_path_to_tool_config = stripped_path_to_tool_config[ 1: ] - relative_path_to_tool_config = os.path.join( relative_install_dir, stripped_path_to_tool_config ) - else: - relative_path_to_tool_config = os.path.join( root, name ) - if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_tool_config.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): - relative_path_to_tool_config = relative_path_to_tool_config[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] - metadata_dict = generate_tool_metadata( relative_path_to_tool_config, tool, repository_clone_url, metadata_dict ) - else: - for tup in invalid_files_and_errors_tups: - invalid_file_tups.append( tup ) - # Find all exported workflows. - elif name.endswith( '.ga' ): - relative_path = os.path.join( root, name ) - if os.path.getsize( os.path.abspath( relative_path ) ) > 0: - fp = open( relative_path, 'rb' ) - workflow_text = fp.read() - fp.close() - exported_workflow_dict = from_json_string( workflow_text ) - if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': - metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ) - if 'tools' in metadata_dict: - # This step must be done after metadata for tools has been defined. - tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', files_dir ) - if tool_dependencies_config: - metadata_dict = generate_tool_dependency_metadata( app, - repository, - tool_dependencies_config, - metadata_dict, - original_repository_metadata=original_repository_metadata ) - if invalid_tool_configs: - metadata_dict [ 'invalid_tools' ] = invalid_tool_configs - # Reset the value of the app's tool_data_path and tool_data_table_config_path to their respective original values. - app.config.tool_data_path = original_tool_data_path - app.config.tool_data_table_config_path = original_tool_data_table_config_path - return metadata_dict, invalid_file_tups -def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): - if as_html: - new_line = '<br/>' - bold_start = '<b>' - bold_end = '</b>' - else: - new_line = '\n' - bold_start = '' - bold_end = '' - message = '' - if not displaying_invalid_tool: - if metadata_dict: - message += "Metadata was defined for some items in revision '%s'. " % str( repository.tip( trans.app ) ) - message += "Correct the following problems if necessary and reset metadata.%s" % new_line - else: - message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip( trans.app ) ) - message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.%s" % new_line - for itc_tup in invalid_file_tups: - tool_file, exception_msg = itc_tup - if exception_msg.find( 'No such file or directory' ) >= 0: - exception_items = exception_msg.split() - missing_file_items = exception_items[ 7 ].split( '/' ) - missing_file = missing_file_items[ -1 ].rstrip( '\'' ) - if missing_file.endswith( '.loc' ): - sample_ext = '%s.sample' % missing_file - else: - sample_ext = missing_file - correction_msg = "This file refers to a missing file %s%s%s. " % ( bold_start, str( missing_file ), bold_end ) - correction_msg += "Upload a file named %s%s%s to the repository to correct this error." % ( bold_start, sample_ext, bold_end ) - else: - if as_html: - correction_msg = exception_msg - else: - correction_msg = exception_msg.replace( '<br/>', new_line ).replace( '<b>', bold_start ).replace( '</b>', bold_end ) - message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) - return message def generate_package_dependency_metadata( elem, tool_dependencies_dict ): """The value of package_name must match the value of the "package" type in the tool config's <requirements> tag set.""" requirements_dict = {} @@ -1155,13 +815,6 @@ else: metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ] return metadata_dict -def get_changectx_for_changeset( repo, changeset_revision, **kwd ): - """Retrieve a specified changectx from a repository""" - for changeset in repo.changelog: - ctx = repo.changectx( changeset ) - if str( ctx ) == changeset_revision: - return ctx - return None def get_config( config_file, repo, ctx, dir ): """Return the latest version of config_filename from the repository manifest.""" config_file = strip_path( config_file ) @@ -1172,22 +825,6 @@ if ctx_file_name == config_file: return get_named_tmpfile_from_ctx( changeset_ctx, ctx_file, dir ) return None -def get_config_from_disk( config_file, relative_install_dir ): - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == config_file: - return os.path.abspath( os.path.join( root, name ) ) - return None -def get_configured_ui(): - # Configure any desired ui settings. - _ui = ui.ui() - # The following will suppress all messages. This is - # the same as adding the following setting to the repo - # hgrc file' [ui] section: - # quiet = True - _ui.setconfig( 'ui', 'quiet', True ) - return _ui def get_converter_and_display_paths( registration_elem, relative_install_dir ): """Find the relative path to data type converters and display applications included in installed tool shed repositories.""" converter_path = None @@ -1247,33 +884,6 @@ ctx_rev = response.read() response.close() return ctx_rev -def get_file_context_from_ctx( ctx, filename ): - # We have to be careful in determining if we found the correct file because multiple files with the same name may be in different directories - # within ctx if the files were moved within the change set. For example, in the following ctx.files() list, the former may have been moved to - # the latter: ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample']. Another scenario - # is that the file has been deleted. - deleted = False - filename = strip_path( filename ) - for ctx_file in ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if filename == ctx_file_name: - try: - # If the file was moved, its destination will be returned here. - fctx = ctx[ ctx_file ] - return fctx - except LookupError, e: - # Set deleted for now, and continue looking in case the file was moved instead of deleted. - deleted = True - if deleted: - return 'DELETED' - return None -def get_file_from_changeset_revision( app, repository, repo_files_dir, changeset_revision, file_name, dir ): - """Return file_name from the received changeset_revision of the repository manifest.""" - stripped_file_name = strip_path( file_name ) - repo = hg.repository( get_configured_ui(), repo_files_dir ) - ctx = get_changectx_for_changeset( repo, changeset_revision ) - named_tmp_file = get_named_tmpfile_from_ctx( ctx, file_name, dir ) - return named_tmp_file def get_installed_tool_shed_repository( trans, id ): """Get a repository on the Galaxy side from the database via id""" return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) @@ -1309,63 +919,6 @@ fh.write( fctx.data() ) fh.close() return sample_files, deleted_sample_files -def get_named_tmpfile_from_ctx( ctx, filename, dir ): - filename = strip_path( filename ) - for ctx_file in ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if filename == ctx_file_name: - try: - # If the file was moved, its destination file contents will be returned here. - fctx = ctx[ ctx_file ] - except LookupError, e: - # Continue looking in case the file was moved. - fctx = None - continue - if fctx: - fh = tempfile.NamedTemporaryFile( 'wb', dir=dir ) - tmp_filename = fh.name - fh.close() - fh = open( tmp_filename, 'wb' ) - fh.write( fctx.data() ) - fh.close() - return tmp_filename - return None -def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): - parent_id = None - # Compare from most recent to oldest. - changeset_revisions.reverse() - for changeset_revision in changeset_revisions: - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) - metadata = repository_metadata.metadata - tools_dicts = metadata.get( 'tools', [] ) - for tool_dict in tools_dicts: - if tool_dict[ 'guid' ] == guid: - # The tool has not changed between the compared changeset revisions. - continue - if tool_dict[ 'id' ] == old_id and tool_dict[ 'version' ] != version: - # The tool version is different, so we've found the parent. - return tool_dict[ 'guid' ] - if parent_id is None: - # The tool did not change through all of the changeset revisions. - return old_id -def get_repository_file_contents( file_path ): - if is_gzip( file_path ): - to_html = to_html_str( '\ngzip compressed file\n' ) - elif is_bz2( file_path ): - to_html = to_html_str( '\nbz2 compressed file\n' ) - elif check_zip( file_path ): - to_html = to_html_str( '\nzip compressed file\n' ) - elif check_binary( file_path ): - to_html = to_html_str( '\nBinary file\n' ) - else: - to_html = '' - for i, line in enumerate( open( file_path ) ): - to_html = '%s%s' % ( to_html, to_html_str( line ) ) - if len( to_html ) > MAX_CONTENT_SIZE: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) - to_html = '%s%s' % ( to_html, to_html_str( large_str ) ) - break - return to_html def get_repository_files( trans, folder_path ): contents = [] for item in os.listdir( folder_path ): @@ -1379,28 +932,6 @@ if contents: contents.sort() return contents -def get_repository_in_tool_shed( trans, id ): - """Get a repository on the tool shed side from the database via id""" - return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) -def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): - """Get metadata for a specified repository change set from the database""" - # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. Duplicate records were somehow - # created in the past. The cause of this issue has been resolved, but we'll leave this method as is for a while longer to ensure all duplicate - # records are removed. - all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \ - .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), - trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ - .order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \ - .all() - if len( all_metadata_records ) > 1: - # Delete all recrds older than the last one updated. - for repository_metadata in all_metadata_records[ 1: ]: - trans.sa_session.delete( repository_metadata ) - trans.sa_session.flush() - return all_metadata_records[ 0 ] - elif all_metadata_records: - return all_metadata_records[ 0 ] - return None def get_repository_owner( cleaned_repository_url ): items = cleaned_repository_url.split( 'repos' ) repo_path = items[ 1 ] @@ -1667,55 +1198,6 @@ repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied -def handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ): - # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. - message = '' - sample_files = copy_disk_sample_files_to_dir( trans, repo_files_dir, work_dir ) - if sample_files: - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) - tool, valid, message2 = load_tool_from_config( trans.app, tool_config_filepath ) - message = concat_messages( message, message2 ) - return tool, valid, message, sample_files -def handle_sample_files_and_load_tool_from_tmp_config( trans, repo, changeset_revision, tool_config_filename, work_dir ): - tool = None - message = '' - ctx = get_changectx_for_changeset( repo, changeset_revision ) - # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in - # the manifest, but have been deleted from disk. - sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) - if sample_files: - trans.app.config.tool_data_path = work_dir - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - if tool_data_table_config: - error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) - if error: - log.debug( message ) - manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) - if manifest_ctx and ctx_file: - tool, message2 = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir ) - message = concat_messages( message, message2 ) - return tool, message, sample_files -def handle_sample_tool_data_table_conf_file( app, filename, persist=False ): - """ - Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur - if call is from the Galaxy side, not the tool shed), the new entries will be appended to Galaxy's shed_tool_data_table_conf.xml file on disk. - """ - error = False - message = '' - try: - new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=filename, - tool_data_path=app.config.tool_data_path, - shed_tool_data_table_config=app.config.shed_tool_data_table_config, - persist=persist ) - except Exception, e: - message = str( e ) - error = True - return error, message def handle_tool_dependencies( app, tool_shed_repository, tool_dependencies_config, tool_dependencies ): """ Install and build tool dependencies defined in the tool_dependencies_config. This config's tag sets can currently refer to installation @@ -1800,8 +1282,6 @@ return False # Default to copying the file if none of the above are true. return True -def is_downloadable( metadata_dict ): - return 'datatypes' in metadata_dict or 'tools' in metadata_dict or 'workflows' in metadata_dict def load_installed_datatype_converters( app, installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype converters app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=installed_repository_dict, deactivate=deactivate ) @@ -1825,22 +1305,6 @@ def load_installed_display_applications( app, installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate ) -def load_tool_from_config( app, full_path ): - try: - tool = app.toolbox.load_tool( full_path ) - valid = True - error_message = None - except KeyError, e: - tool = None - valid = False - error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) - error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' - error_message += 'this error. ' - except Exception, e: - tool = None - valid = False - error_message = str( e ) - return tool, valid, error_message def load_tool_from_tmp_config( trans, repo, ctx, ctx_file, work_dir ): tool = None message = '' @@ -1866,27 +1330,6 @@ except: pass return tool, message -def open_repository_files_folder( trans, folder_path ): - try: - files_list = get_repository_files( trans, folder_path ) - except OSError, e: - if str( e ).find( 'No such file or directory' ) >= 0: - # We have a repository with no contents. - return [] - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents def panel_entry_per_tool( tool_section_dict ): # Return True if tool_section_dict looks like this. # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]} @@ -1906,12 +1349,6 @@ repo, source=repository_clone_url, rev=[ ctx_rev ] ) -def remove_dir( dir ): - if os.path.exists( dir ): - try: - shutil.rmtree( dir ) - except: - pass def remove_from_shed_tool_config( trans, shed_tool_conf_dict, guids_to_remove ): # A tool shed repository is being uninstalled so change the shed_tool_conf file. Parse the config file to generate the entire list # of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have @@ -2113,209 +1550,6 @@ else: log.debug( 'Error locating installation directory for repository %s.' % repository.name ) return invalid_file_tups, metadata_dict -def reset_all_metadata_on_repository_in_tool_shed( trans, id ): - """Reset all metadata on a single repository in a tool shed.""" - def reset_all_tool_versions( trans, id, repo ): - changeset_revisions = [] - for changeset in repo.changelog: - changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) - if repository_metadata: - metadata = repository_metadata.metadata - if metadata: - if metadata.get( 'tools', None ): - changeset_revisions.append( changeset_revision ) - # The list of changeset_revisions is now filtered to contain only those that are downloadable and contain tools. - # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. - for index, changeset_revision in enumerate( changeset_revisions ): - tool_versions_dict = {} - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) - metadata = repository_metadata.metadata - tool_dicts = metadata[ 'tools' ] - if index == 0: - # The first changset_revision is a special case because it will have no ancestor changeset_revisions in which to match tools. - # The parent tool id for tools in the first changeset_revision will be the "old_id" in the tool config. - for tool_dict in tool_dicts: - tool_versions_dict[ tool_dict[ 'guid' ] ] = tool_dict[ 'id' ] - else: - for tool_dict in tool_dicts: - parent_id = get_parent_id( trans, - id, - tool_dict[ 'id' ], - tool_dict[ 'version' ], - tool_dict[ 'guid' ], - changeset_revisions[ 0:index ] ) - tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id - if tool_versions_dict: - repository_metadata.tool_versions = tool_versions_dict - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() - repository = get_repository_in_tool_shed( trans, id ) - log.debug( "Resetting all metadata on repository: %s" % repository.name ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( get_configured_ui(), repo_dir ) - repository_clone_url = generate_clone_url_for_repository_in_tool_shed( trans, repository ) - # The list of changeset_revisions refers to repository_metadata records that have been created or updated. When the following loop - # completes, we'll delete all repository_metadata records for this repository that do not have a changeset_revision value in this list. - changeset_revisions = [] - # When a new repository_metadata record is created, it always uses the values of metadata_changeset_revision and metadata_dict. - metadata_changeset_revision = None - metadata_dict = None - ancestor_changeset_revision = None - ancestor_metadata_dict = None - invalid_file_tups = [] - home_dir = os.getcwd() - for changeset in repo.changelog: - work_dir = tempfile.mkdtemp() - current_changeset_revision = str( repo.changectx( changeset ) ) - ctx = repo.changectx( changeset ) - log.debug( "Cloning repository revision: %s", str( ctx.rev() ) ) - cloned_ok, error_message = clone_repository( repository_clone_url, work_dir, str( ctx.rev() ) ) - if cloned_ok: - log.debug( "Generating metadata for changset revision: %s", str( ctx.rev() ) ) - current_metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - repository_clone_url=repository_clone_url, - relative_install_dir=repo_dir, - repository_files_dir=work_dir, - resetting_all_metadata_on_repository=True, - updating_installed_repository=False, - persist=False ) - if current_metadata_dict: - if not metadata_changeset_revision and not metadata_dict: - # We're at the first change set in the change log. - metadata_changeset_revision = current_changeset_revision - metadata_dict = current_metadata_dict - if ancestor_changeset_revision: - # Compare metadata from ancestor and current. The value of comparison will be one of: - # 'no metadata' - no metadata for either ancestor or current, so continue from current - # 'equal' - ancestor metadata is equivalent to current metadata, so continue from current - # 'subset' - ancestor metadata is a subset of current metadata, so continue from current - # 'not equal and not subset' - ancestor metadata is neither equal to nor a subset of current metadata, so persist ancestor metadata. - comparison = compare_changeset_revisions( ancestor_changeset_revision, - ancestor_metadata_dict, - current_changeset_revision, - current_metadata_dict ) - if comparison in [ 'no metadata', 'equal', 'subset' ]: - ancestor_changeset_revision = current_changeset_revision - ancestor_metadata_dict = current_metadata_dict - elif comparison == 'not equal and not subset': - metadata_changeset_revision = ancestor_changeset_revision - metadata_dict = ancestor_metadata_dict - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) - changeset_revisions.append( metadata_changeset_revision ) - ancestor_changeset_revision = current_changeset_revision - ancestor_metadata_dict = current_metadata_dict - else: - # We're at the beginning of the change log. - ancestor_changeset_revision = current_changeset_revision - ancestor_metadata_dict = current_metadata_dict - if not ctx.children(): - metadata_changeset_revision = current_changeset_revision - metadata_dict = current_metadata_dict - # We're at the end of the change log. - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) - changeset_revisions.append( metadata_changeset_revision ) - ancestor_changeset_revision = None - ancestor_metadata_dict = None - elif ancestor_metadata_dict: - # We reach here only if current_metadata_dict is empty and ancestor_metadata_dict is not. - if not ctx.children(): - # We're at the end of the change log. - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) - changeset_revisions.append( metadata_changeset_revision ) - ancestor_changeset_revision = None - ancestor_metadata_dict = None - remove_dir( work_dir ) - # Delete all repository_metadata records for this repository that do not have a changeset_revision value in changeset_revisions. - clean_repository_metadata( trans, id, changeset_revisions ) - # Set tool version information for all downloadable changeset revisions. Get the list of changeset revisions from the changelog. - reset_all_tool_versions( trans, id, repo ) - # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - reset_tool_data_tables( trans.app ) - return invalid_file_tups, metadata_dict -def reset_metadata_on_selected_repositories( trans, **kwd ): - # This method is called from both Galaxy and the Tool Shed, so the cntrller param is required. - repository_ids = util.listify( kwd.get( 'repository_ids', None ) ) - CONTROLLER = kwd[ 'CONTROLLER' ] - message = '' - status = 'done' - if repository_ids: - successful_count = 0 - unsuccessful_count = 0 - for repository_id in repository_ids: - try: - if CONTROLLER == 'TOOL_SHED_ADMIN_CONTROLLER': - repository = get_repository_in_tool_shed( trans, repository_id ) - invalid_file_tups, metadata_dict = reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) - elif CONTROLLER == 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER': - repository = get_installed_tool_shed_repository( trans, repository_id ) - invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans, repository_id ) - if invalid_file_tups: - message = generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) - log.debug( message ) - unsuccessful_count += 1 - else: - log.debug( "Successfully reset metadata on repository %s" % repository.name ) - successful_count += 1 - except Exception, e: - log.debug( "Error attempting to reset metadata on repository '%s': %s" % ( repository.name, str( e ) ) ) - unsuccessful_count += 1 - message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) - if unsuccessful_count: - message += "Error setting metadata on %d %s - see the paster log for details. " % ( unsuccessful_count, - inflector.cond_plural( unsuccessful_count, "repository" ) ) - else: - message = 'Select at least one repository to on which to reset all metadata.' - status = 'error' - return message, status -def reset_tool_data_tables( app ): - # Reset the tool_data_tables to an empty dictionary. - app.tool_data_tables.data_tables = {} -def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): - """ - Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, but up to and - including the included_upper_bounds_changeset_revision. The value of excluded_lower_bounds_changeset_revision will be the value of - INITIAL_CHANGELOG_HASH if no valid changesets exist before included_upper_bounds_changeset_revision. - """ - # To set excluded_lower_bounds_changeset_revision, calling methods should do the following, where the value of changeset_revision - # is a downloadable changeset_revision. - # excluded_lower_bounds_changeset_revision = get_previous_downloadable_changset_revision( repository, repo, changeset_revision ) - if excluded_lower_bounds_changeset_revision == INITIAL_CHANGELOG_HASH: - appending_started = True - else: - appending_started = False - reversed_changelog = [] - for changeset in repo.changelog: - changeset_hash = str( repo.changectx( changeset ) ) - if appending_started: - reversed_changelog.insert( 0, changeset ) - if changeset_hash == excluded_lower_bounds_changeset_revision and not appending_started: - appending_started = True - if changeset_hash == included_upper_bounds_changeset_revision: - break - return reversed_changelog -def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): - return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) -def strip_path( fpath ): - if not fpath: - return fpath - try: - file_path, file_name = os.path.split( fpath ) - except: - file_name = fpath - return file_name -def to_html_escaped( text ): - """Translates the characters in text to html values""" - translated = [] - for c in text: - if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS: - translated.append( c ) - elif c in MAPPED_CHARS: - translated.append( MAPPED_CHARS[ c ] ) - else: - translated.append( '' ) - return ''.join( translated ) def to_html_str( text ): """Translates the characters in text to an html string""" translated = [] @@ -2443,32 +1677,8 @@ elem = guid_to_tool_elem_dict[ guid ] config_elems.append( elem ) config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) -def update_repository( repo, ctx_rev=None ): - """ - Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired - changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. - """ - # TODO: We may have files on disk in the repo directory that aren't being tracked, so they must be removed. - # The codes used to show the status of files are as follows. - # M = modified - # A = added - # R = removed - # C = clean - # ! = deleted, but still tracked - # ? = not tracked - # I = ignored - # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that - # purging is not supported by the mercurial API. See the deprecated update_for_browsing() method in common.py. - commands.update( get_configured_ui(), - repo, - rev=ctx_rev ) def update_tool_shed_repository_status( app, tool_shed_repository, status ): sa_session = app.model.context.current tool_shed_repository.status = status sa_session.add( tool_shed_repository ) sa_session.flush() -def url_join( *args ): - parts = [] - for arg in args: - parts.append( arg.strip( '/' ) ) - return '/'.join( parts ) diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/util/shed_util_common.py --- /dev/null +++ b/lib/galaxy/util/shed_util_common.py @@ -0,0 +1,784 @@ +import os, shutil, tempfile, logging +from galaxy import util +from galaxy.tools import parameters +from galaxy.util import inflector +from galaxy.web import url_for +from galaxy.web.form_builder import SelectField +from galaxy.datatypes.checkers import * +from galaxy.model.orm import * + +from galaxy import eggs +import pkg_resources + +pkg_resources.require( 'mercurial' ) +from mercurial import hg, ui, commands + +log = logging.getLogger( __name__ ) + +INITIAL_CHANGELOG_HASH = '000000000000' +# Characters that must be html escaped +MAPPED_CHARS = { '>' :'>', + '<' :'<', + '"' : '"', + '&' : '&', + '\'' : ''' } +MAX_CONTENT_SIZE = 32768 +NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'tool_dependencies.xml' ] +GALAXY_ADMIN_TOOL_SHED_CONTROLLER = 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER' +TOOL_SHED_ADMIN_CONTROLLER = 'TOOL_SHED_ADMIN_CONTROLLER' +VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) + +def build_repository_ids_select_field( trans, cntrller, name='repository_ids', multiple=True, display='checkboxes' ): + """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" + repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) + if cntrller == TOOL_SHED_ADMIN_CONTROLLER: + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ): + owner = repository.user.username + option_label = '%s (%s)' % ( repository.name, owner ) + option_value = '%s' % trans.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) + elif cntrller == GALAXY_ADMIN_TOOL_SHED_CONTROLLER: + for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ + .order_by( trans.model.ToolShedRepository.table.c.name, + trans.model.ToolShedRepository.table.c.owner ): + option_label = '%s (%s)' % ( repository.name, repository.owner ) + option_value = trans.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) + return repositories_select_field +def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): + """ + Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make + sure the files exist. + """ + invalid_files_and_errors_tups = [] + correction_msg = '' + for input_param in tool.input_params: + if isinstance( input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: + # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, make sure all requirements exist. + options = input_param.dynamic_options or input_param.options + if options: + if options.tool_data_table or options.missing_tool_data_table_name: + # Make sure the repository contains a tool_data_table_conf.xml.sample file. + sample_tool_data_table_conf = get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) + if sample_tool_data_table_conf: + error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf ) + if error: + invalid_files_and_errors_tups.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) + else: + options.missing_tool_data_table_name = None + else: + correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. Upload a file named tool_data_table_conf.xml.sample " + correction_msg += "to the repository that includes the required entry to correct this error.<br/>" + invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) + if options.index_file or options.missing_index_file: + # Make sure the repository contains the required xxx.loc.sample file. + index_file = options.index_file or options.missing_index_file + index_file_name = strip_path( index_file ) + sample_found = False + for sample_file in sample_files: + sample_file_name = strip_path( sample_file ) + if sample_file_name == '%s.sample' % index_file_name: + options.index_file = index_file_name + options.missing_index_file = None + if options.tool_data_table: + options.tool_data_table.missing_index_file = None + sample_found = True + break + if not sample_found: + correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file ) + correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) + invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) + return invalid_files_and_errors_tups +def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): + """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" + try: + commands.clone( get_configured_ui(), + str( repository_clone_url ), + dest=str( repository_file_dir ), + pull=True, + noupdate=False, + rev=util.listify( str( ctx_rev ) ) ) + return True, None + except Exception, e: + error_message = 'Error cloning repository: %s' % str( e ) + log.debug( error_message ) + return False, error_message +def concat_messages( msg1, msg2 ): + if msg1: + if msg2: + message = '%s %s' % ( msg1, msg2 ) + else: + message = msg1 + elif msg2: + message = msg2 + else: + message = '' + return message +def copy_sample_file( app, filename, dest_path=None ): + """Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path is ~/tool-data.""" + if dest_path is None: + dest_path = os.path.abspath( app.config.tool_data_path ) + sample_file_name = strip_path( filename ) + copied_file = sample_file_name.replace( '.sample', '' ) + full_source_path = os.path.abspath( filename ) + full_destination_path = os.path.join( dest_path, sample_file_name ) + # Don't copy a file to itself - not sure how this happens, but sometimes it does... + if full_source_path != full_destination_path: + # It's ok to overwrite the .sample version of the file. + shutil.copy( full_source_path, full_destination_path ) + # Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance. + if not os.path.exists( os.path.join( dest_path, copied_file ) ): + shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) +def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ): + downloadable = is_downloadable( metadata_dict ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + if repository_metadata: + repository_metadata.metadata = metadata_dict + repository_metadata.downloadable = downloadable + else: + repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, + changeset_revision=changeset_revision, + metadata=metadata_dict, + downloadable=downloadable ) + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + return repository_metadata +def create_repo_info_dict( repository, owner, repository_clone_url, changeset_revision, ctx_rev, metadata ): + repo_info_dict = {} + repo_info_dict[ repository.name ] = ( repository.description, + repository_clone_url, + changeset_revision, + ctx_rev, + owner, + metadata.get( 'tool_dependencies', None ) ) + return repo_info_dict +def generate_clone_url_for_repository_in_tool_shed( trans, repository ): + """Generate the URL for cloning a repository that is in the tool shed.""" + base_url = url_for( '/', qualified=True ).rstrip( '/' ) + if trans.user: + protocol, base = base_url.split( '://' ) + username = '%s@' % trans.user.username + return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) + else: + return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name ) +def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): + if as_html: + new_line = '<br/>' + bold_start = '<b>' + bold_end = '</b>' + else: + new_line = '\n' + bold_start = '' + bold_end = '' + message = '' + if not displaying_invalid_tool: + if metadata_dict: + message += "Metadata was defined for some items in revision '%s'. " % str( repository.tip( trans.app ) ) + message += "Correct the following problems if necessary and reset metadata.%s" % new_line + else: + message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip( trans.app ) ) + message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.%s" % new_line + for itc_tup in invalid_file_tups: + tool_file, exception_msg = itc_tup + if exception_msg.find( 'No such file or directory' ) >= 0: + exception_items = exception_msg.split() + missing_file_items = exception_items[ 7 ].split( '/' ) + missing_file = missing_file_items[ -1 ].rstrip( '\'' ) + if missing_file.endswith( '.loc' ): + sample_ext = '%s.sample' % missing_file + else: + sample_ext = missing_file + correction_msg = "This file refers to a missing file %s%s%s. " % ( bold_start, str( missing_file ), bold_end ) + correction_msg += "Upload a file named %s%s%s to the repository to correct this error." % ( bold_start, sample_ext, bold_end ) + else: + if as_html: + correction_msg = exception_msg + else: + correction_msg = exception_msg.replace( '<br/>', new_line ).replace( '<b>', bold_start ).replace( '</b>', bold_end ) + message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) + return message +def generate_metadata_for_changeset_revision( app, repository, repository_clone_url, shed_config_dict={}, relative_install_dir=None, repository_files_dir=None, + resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ): + """ + Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip, + the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's + disk files, so the value of repository_files_dir will not always be repository.repo_path( app ) (it could be an absolute path to a temporary + directory containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path( app ). + + The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries + should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. + """ + if updating_installed_repository: + # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which + # we have pulled updates. + original_repository_metadata = repository.metadata + else: + original_repository_metadata = None + readme_file_names = get_readme_file_names( repository.name ) + metadata_dict = { 'shed_config_filename': shed_config_dict.get( 'config_filename' ) } + invalid_file_tups = [] + invalid_tool_configs = [] + tool_dependencies_config = None + original_tool_data_path = app.config.tool_data_path + original_tool_data_table_config_path = app.config.tool_data_table_config_path + if resetting_all_metadata_on_repository: + if not relative_install_dir: + raise Exception( "The value of repository.repo_path( app ) must be sent when resetting all metadata on a repository." ) + # Keep track of the location where the repository is temporarily cloned so that we can strip the path when setting metadata. The value of + # repository_files_dir is the full path to the temporary directory to which the repository was cloned. + work_dir = repository_files_dir + files_dir = repository_files_dir + # Since we're working from a temporary directory, we can safely copy sample files included in the repository to the repository root. + app.config.tool_data_path = repository_files_dir + app.config.tool_data_table_config_path = repository_files_dir + else: + # Use a temporary working directory to copy all sample files. + work_dir = tempfile.mkdtemp() + # All other files are on disk in the repository's repo_path, which is the value of relative_install_dir. + files_dir = relative_install_dir + if shed_config_dict.get( 'tool_path' ): + files_dir = os.path.join( shed_config_dict['tool_path'], files_dir ) + app.config.tool_data_path = work_dir + app.config.tool_data_table_config_path = work_dir + # Handle proprietary datatypes, if any. + datatypes_config = get_config_from_disk( 'datatypes_conf.xml', files_dir ) + if datatypes_config: + metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict ) + # Get the relative path to all sample files included in the repository for storage in the repository's metadata. + sample_file_metadata_paths, sample_file_copy_paths = get_sample_files_from_disk( repository_files_dir=files_dir, + tool_path=shed_config_dict.get( 'tool_path' ), + relative_install_dir=relative_install_dir, + resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) + if sample_file_metadata_paths: + metadata_dict[ 'sample_files' ] = sample_file_metadata_paths + # Copy all sample files included in the repository to a single directory location so we can load tools that depend on them. + for sample_file in sample_file_copy_paths: + copy_sample_file( app, sample_file, dest_path=work_dir ) + # If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory. + relative_path, filename = os.path.split( sample_file ) + if filename == 'tool_data_table_conf.xml.sample': + new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file, + tool_data_path=original_tool_data_path, + shed_tool_data_table_config=app.config.shed_tool_data_table_config, + persist=persist ) + for root, dirs, files in os.walk( files_dir ): + if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: + if '.hg' in dirs: + dirs.remove( '.hg' ) + for name in files: + # See if we have a READ_ME file. + if name.lower() in readme_file_names: + if resetting_all_metadata_on_repository: + full_path_to_readme = os.path.join( root, name ) + stripped_path_to_readme = full_path_to_readme.replace( work_dir, '' ) + if stripped_path_to_readme.startswith( '/' ): + stripped_path_to_readme = stripped_path_to_readme[ 1: ] + relative_path_to_readme = os.path.join( relative_install_dir, stripped_path_to_readme ) + else: + relative_path_to_readme = os.path.join( root, name ) + if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_readme.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): + relative_path_to_readme = relative_path_to_readme[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] + metadata_dict[ 'readme' ] = relative_path_to_readme + # See if we have a tool config. + elif name not in NOT_TOOL_CONFIGS and name.endswith( '.xml' ): + full_path = str( os.path.abspath( os.path.join( root, name ) ) ) + if os.path.getsize( full_path ) > 0: + if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ] + or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ): + try: + # Make sure we're looking at a tool config and not a display application config or something else. + element_tree = util.parse_xml( full_path ) + element_tree_root = element_tree.getroot() + is_tool = element_tree_root.tag == 'tool' + except Exception, e: + log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) ) + is_tool = False + if is_tool: + tool, valid, error_message = load_tool_from_config( app, full_path ) + if tool is None: + if not valid: + invalid_file_tups.append( ( name, error_message ) ) + else: + invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_file_metadata_paths ) + can_set_metadata = True + for tup in invalid_files_and_errors_tups: + if name in tup: + can_set_metadata = False + invalid_tool_configs.append( name ) + break + if can_set_metadata: + if resetting_all_metadata_on_repository: + full_path_to_tool_config = os.path.join( root, name ) + stripped_path_to_tool_config = full_path_to_tool_config.replace( work_dir, '' ) + if stripped_path_to_tool_config.startswith( '/' ): + stripped_path_to_tool_config = stripped_path_to_tool_config[ 1: ] + relative_path_to_tool_config = os.path.join( relative_install_dir, stripped_path_to_tool_config ) + else: + relative_path_to_tool_config = os.path.join( root, name ) + if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_tool_config.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): + relative_path_to_tool_config = relative_path_to_tool_config[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] + metadata_dict = generate_tool_metadata( relative_path_to_tool_config, tool, repository_clone_url, metadata_dict ) + else: + for tup in invalid_files_and_errors_tups: + invalid_file_tups.append( tup ) + # Find all exported workflows. + elif name.endswith( '.ga' ): + relative_path = os.path.join( root, name ) + if os.path.getsize( os.path.abspath( relative_path ) ) > 0: + fp = open( relative_path, 'rb' ) + workflow_text = fp.read() + fp.close() + exported_workflow_dict = from_json_string( workflow_text ) + if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': + metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ) + if 'tools' in metadata_dict: + # This step must be done after metadata for tools has been defined. + tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', files_dir ) + if tool_dependencies_config: + metadata_dict = generate_tool_dependency_metadata( app, + repository, + tool_dependencies_config, + metadata_dict, + original_repository_metadata=original_repository_metadata ) + if invalid_tool_configs: + metadata_dict [ 'invalid_tools' ] = invalid_tool_configs + # Reset the value of the app's tool_data_path and tool_data_table_config_path to their respective original values. + app.config.tool_data_path = original_tool_data_path + app.config.tool_data_table_config_path = original_tool_data_table_config_path + return metadata_dict, invalid_file_tups +def get_changectx_for_changeset( repo, changeset_revision, **kwd ): + """Retrieve a specified changectx from a repository""" + for changeset in repo.changelog: + ctx = repo.changectx( changeset ) + if str( ctx ) == changeset_revision: + return ctx + return None +def get_config_from_disk( config_file, relative_install_dir ): + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == config_file: + return os.path.abspath( os.path.join( root, name ) ) + return None +def get_configured_ui(): + # Configure any desired ui settings. + _ui = ui.ui() + # The following will suppress all messages. This is + # the same as adding the following setting to the repo + # hgrc file' [ui] section: + # quiet = True + _ui.setconfig( 'ui', 'quiet', True ) + return _ui +def get_file_context_from_ctx( ctx, filename ): + # We have to be careful in determining if we found the correct file because multiple files with the same name may be in different directories + # within ctx if the files were moved within the change set. For example, in the following ctx.files() list, the former may have been moved to + # the latter: ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample']. Another scenario + # is that the file has been deleted. + deleted = False + filename = strip_path( filename ) + for ctx_file in ctx.files(): + ctx_file_name = strip_path( ctx_file ) + if filename == ctx_file_name: + try: + # If the file was moved, its destination will be returned here. + fctx = ctx[ ctx_file ] + return fctx + except LookupError, e: + # Set deleted for now, and continue looking in case the file was moved instead of deleted. + deleted = True + if deleted: + return 'DELETED' + return None +def get_repository_file_contents( file_path ): + if is_gzip( file_path ): + to_html = to_html_str( '\ngzip compressed file\n' ) + elif is_bz2( file_path ): + to_html = to_html_str( '\nbz2 compressed file\n' ) + elif check_zip( file_path ): + to_html = to_html_str( '\nzip compressed file\n' ) + elif check_binary( file_path ): + to_html = to_html_str( '\nBinary file\n' ) + else: + to_html = '' + for i, line in enumerate( open( file_path ) ): + to_html = '%s%s' % ( to_html, to_html_str( line ) ) + if len( to_html ) > MAX_CONTENT_SIZE: + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) + to_html = '%s%s' % ( to_html, to_html_str( large_str ) ) + break + return to_html +def get_repository_in_tool_shed( trans, id ): + """Get a repository on the tool shed side from the database via id""" + return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) +def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): + """Get metadata for a specified repository change set from the database""" + # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. Duplicate records were somehow + # created in the past. The cause of this issue has been resolved, but we'll leave this method as is for a while longer to ensure all duplicate + # records are removed. + all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), + trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ + .order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \ + .all() + if len( all_metadata_records ) > 1: + # Delete all recrds older than the last one updated. + for repository_metadata in all_metadata_records[ 1: ]: + trans.sa_session.delete( repository_metadata ) + trans.sa_session.flush() + return all_metadata_records[ 0 ] + elif all_metadata_records: + return all_metadata_records[ 0 ] + return None +def get_named_tmpfile_from_ctx( ctx, filename, dir ): + filename = strip_path( filename ) + for ctx_file in ctx.files(): + ctx_file_name = strip_path( ctx_file ) + if filename == ctx_file_name: + try: + # If the file was moved, its destination file contents will be returned here. + fctx = ctx[ ctx_file ] + except LookupError, e: + # Continue looking in case the file was moved. + fctx = None + continue + if fctx: + fh = tempfile.NamedTemporaryFile( 'wb', dir=dir ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'wb' ) + fh.write( fctx.data() ) + fh.close() + return tmp_filename + return None +def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): + parent_id = None + # Compare from most recent to oldest. + changeset_revisions.reverse() + for changeset_revision in changeset_revisions: + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + metadata = repository_metadata.metadata + tools_dicts = metadata.get( 'tools', [] ) + for tool_dict in tools_dicts: + if tool_dict[ 'guid' ] == guid: + # The tool has not changed between the compared changeset revisions. + continue + if tool_dict[ 'id' ] == old_id and tool_dict[ 'version' ] != version: + # The tool version is different, so we've found the parent. + return tool_dict[ 'guid' ] + if parent_id is None: + # The tool did not change through all of the changeset revisions. + return old_id +def handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ): + # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. + message = '' + sample_files = copy_disk_sample_files_to_dir( trans, repo_files_dir, work_dir ) + if sample_files: + if 'tool_data_table_conf.xml.sample' in sample_files: + # Load entries into the tool_data_tables if the tool requires them. + tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) + error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) + tool, valid, message2 = load_tool_from_config( trans.app, tool_config_filepath ) + message = concat_messages( message, message2 ) + return tool, valid, message, sample_files +def handle_sample_files_and_load_tool_from_tmp_config( trans, repo, changeset_revision, tool_config_filename, work_dir ): + tool = None + message = '' + ctx = get_changectx_for_changeset( repo, changeset_revision ) + # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in + # the manifest, but have been deleted from disk. + sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) + if sample_files: + trans.app.config.tool_data_path = work_dir + if 'tool_data_table_conf.xml.sample' in sample_files: + # Load entries into the tool_data_tables if the tool requires them. + tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) + if tool_data_table_config: + error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) + if error: + log.debug( message ) + manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) + if manifest_ctx and ctx_file: + tool, message2 = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir ) + message = concat_messages( message, message2 ) + return tool, message, sample_files +def handle_sample_tool_data_table_conf_file( app, filename, persist=False ): + """ + Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur + if call is from the Galaxy side, not the tool shed), the new entries will be appended to Galaxy's shed_tool_data_table_conf.xml file on disk. + """ + error = False + message = '' + try: + new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=filename, + tool_data_path=app.config.tool_data_path, + shed_tool_data_table_config=app.config.shed_tool_data_table_config, + persist=persist ) + except Exception, e: + message = str( e ) + error = True + return error, message +def is_downloadable( metadata_dict ): + return 'datatypes' in metadata_dict or 'tools' in metadata_dict or 'workflows' in metadata_dict +def load_tool_from_config( app, full_path ): + try: + tool = app.toolbox.load_tool( full_path ) + valid = True + error_message = None + except KeyError, e: + tool = None + valid = False + error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) + error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' + error_message += 'this error. ' + except Exception, e: + tool = None + valid = False + error_message = str( e ) + return tool, valid, error_message +def open_repository_files_folder( trans, folder_path ): + try: + files_list = get_repository_files( trans, folder_path ) + except OSError, e: + if str( e ).find( 'No such file or directory' ) >= 0: + # We have a repository with no contents. + return [] + folder_contents = [] + for filename in files_list: + is_folder = False + if filename and filename[-1] == os.sep: + is_folder = True + if filename: + full_path = os.path.join( folder_path, filename ) + node = { "title": filename, + "isFolder": is_folder, + "isLazy": is_folder, + "tooltip": full_path, + "key": full_path } + folder_contents.append( node ) + return folder_contents +def remove_dir( dir ): + if os.path.exists( dir ): + try: + shutil.rmtree( dir ) + except: + pass +def reset_all_metadata_on_repository_in_tool_shed( trans, id ): + """Reset all metadata on a single repository in a tool shed.""" + def reset_all_tool_versions( trans, id, repo ): + changeset_revisions = [] + for changeset in repo.changelog: + changeset_revision = str( repo.changectx( changeset ) ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + if metadata.get( 'tools', None ): + changeset_revisions.append( changeset_revision ) + # The list of changeset_revisions is now filtered to contain only those that are downloadable and contain tools. + # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. + for index, changeset_revision in enumerate( changeset_revisions ): + tool_versions_dict = {} + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + metadata = repository_metadata.metadata + tool_dicts = metadata[ 'tools' ] + if index == 0: + # The first changset_revision is a special case because it will have no ancestor changeset_revisions in which to match tools. + # The parent tool id for tools in the first changeset_revision will be the "old_id" in the tool config. + for tool_dict in tool_dicts: + tool_versions_dict[ tool_dict[ 'guid' ] ] = tool_dict[ 'id' ] + else: + for tool_dict in tool_dicts: + parent_id = get_parent_id( trans, + id, + tool_dict[ 'id' ], + tool_dict[ 'version' ], + tool_dict[ 'guid' ], + changeset_revisions[ 0:index ] ) + tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id + if tool_versions_dict: + repository_metadata.tool_versions = tool_versions_dict + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + repository = get_repository_in_tool_shed( trans, id ) + log.debug( "Resetting all metadata on repository: %s" % repository.name ) + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( get_configured_ui(), repo_dir ) + repository_clone_url = generate_clone_url_for_repository_in_tool_shed( trans, repository ) + # The list of changeset_revisions refers to repository_metadata records that have been created or updated. When the following loop + # completes, we'll delete all repository_metadata records for this repository that do not have a changeset_revision value in this list. + changeset_revisions = [] + # When a new repository_metadata record is created, it always uses the values of metadata_changeset_revision and metadata_dict. + metadata_changeset_revision = None + metadata_dict = None + ancestor_changeset_revision = None + ancestor_metadata_dict = None + invalid_file_tups = [] + home_dir = os.getcwd() + for changeset in repo.changelog: + work_dir = tempfile.mkdtemp() + current_changeset_revision = str( repo.changectx( changeset ) ) + ctx = repo.changectx( changeset ) + log.debug( "Cloning repository revision: %s", str( ctx.rev() ) ) + cloned_ok, error_message = clone_repository( repository_clone_url, work_dir, str( ctx.rev() ) ) + if cloned_ok: + log.debug( "Generating metadata for changset revision: %s", str( ctx.rev() ) ) + current_metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app, + repository=repository, + repository_clone_url=repository_clone_url, + relative_install_dir=repo_dir, + repository_files_dir=work_dir, + resetting_all_metadata_on_repository=True, + updating_installed_repository=False, + persist=False ) + if current_metadata_dict: + if not metadata_changeset_revision and not metadata_dict: + # We're at the first change set in the change log. + metadata_changeset_revision = current_changeset_revision + metadata_dict = current_metadata_dict + if ancestor_changeset_revision: + # Compare metadata from ancestor and current. The value of comparison will be one of: + # 'no metadata' - no metadata for either ancestor or current, so continue from current + # 'equal' - ancestor metadata is equivalent to current metadata, so continue from current + # 'subset' - ancestor metadata is a subset of current metadata, so continue from current + # 'not equal and not subset' - ancestor metadata is neither equal to nor a subset of current metadata, so persist ancestor metadata. + comparison = compare_changeset_revisions( ancestor_changeset_revision, + ancestor_metadata_dict, + current_changeset_revision, + current_metadata_dict ) + if comparison in [ 'no metadata', 'equal', 'subset' ]: + ancestor_changeset_revision = current_changeset_revision + ancestor_metadata_dict = current_metadata_dict + elif comparison == 'not equal and not subset': + metadata_changeset_revision = ancestor_changeset_revision + metadata_dict = ancestor_metadata_dict + repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + changeset_revisions.append( metadata_changeset_revision ) + ancestor_changeset_revision = current_changeset_revision + ancestor_metadata_dict = current_metadata_dict + else: + # We're at the beginning of the change log. + ancestor_changeset_revision = current_changeset_revision + ancestor_metadata_dict = current_metadata_dict + if not ctx.children(): + metadata_changeset_revision = current_changeset_revision + metadata_dict = current_metadata_dict + # We're at the end of the change log. + repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + changeset_revisions.append( metadata_changeset_revision ) + ancestor_changeset_revision = None + ancestor_metadata_dict = None + elif ancestor_metadata_dict: + # We reach here only if current_metadata_dict is empty and ancestor_metadata_dict is not. + if not ctx.children(): + # We're at the end of the change log. + repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + changeset_revisions.append( metadata_changeset_revision ) + ancestor_changeset_revision = None + ancestor_metadata_dict = None + remove_dir( work_dir ) + # Delete all repository_metadata records for this repository that do not have a changeset_revision value in changeset_revisions. + clean_repository_metadata( trans, id, changeset_revisions ) + # Set tool version information for all downloadable changeset revisions. Get the list of changeset revisions from the changelog. + reset_all_tool_versions( trans, id, repo ) + # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. + reset_tool_data_tables( trans.app ) + return invalid_file_tups, metadata_dict +def reset_metadata_on_selected_repositories( trans, **kwd ): + # This method is called from both Galaxy and the Tool Shed, so the cntrller param is required. + repository_ids = util.listify( kwd.get( 'repository_ids', None ) ) + CONTROLLER = kwd[ 'CONTROLLER' ] + message = '' + status = 'done' + if repository_ids: + successful_count = 0 + unsuccessful_count = 0 + for repository_id in repository_ids: + try: + if CONTROLLER == 'TOOL_SHED_ADMIN_CONTROLLER': + repository = get_repository_in_tool_shed( trans, repository_id ) + invalid_file_tups, metadata_dict = reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) + elif CONTROLLER == 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER': + repository = get_installed_tool_shed_repository( trans, repository_id ) + invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + log.debug( message ) + unsuccessful_count += 1 + else: + log.debug( "Successfully reset metadata on repository %s" % repository.name ) + successful_count += 1 + except Exception, e: + log.debug( "Error attempting to reset metadata on repository '%s': %s" % ( repository.name, str( e ) ) ) + unsuccessful_count += 1 + message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) + if unsuccessful_count: + message += "Error setting metadata on %d %s - see the paster log for details. " % ( unsuccessful_count, + inflector.cond_plural( unsuccessful_count, "repository" ) ) + else: + message = 'Select at least one repository to on which to reset all metadata.' + status = 'error' + return message, status +def reset_tool_data_tables( app ): + # Reset the tool_data_tables to an empty dictionary. + app.tool_data_tables.data_tables = {} +def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): + """ + Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, but up to and + including the included_upper_bounds_changeset_revision. The value of excluded_lower_bounds_changeset_revision will be the value of + INITIAL_CHANGELOG_HASH if no valid changesets exist before included_upper_bounds_changeset_revision. + """ + # To set excluded_lower_bounds_changeset_revision, calling methods should do the following, where the value of changeset_revision + # is a downloadable changeset_revision. + # excluded_lower_bounds_changeset_revision = get_previous_downloadable_changset_revision( repository, repo, changeset_revision ) + if excluded_lower_bounds_changeset_revision == INITIAL_CHANGELOG_HASH: + appending_started = True + else: + appending_started = False + reversed_changelog = [] + for changeset in repo.changelog: + changeset_hash = str( repo.changectx( changeset ) ) + if appending_started: + reversed_changelog.insert( 0, changeset ) + if changeset_hash == excluded_lower_bounds_changeset_revision and not appending_started: + appending_started = True + if changeset_hash == included_upper_bounds_changeset_revision: + break + return reversed_changelog +def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): + return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) +def strip_path( fpath ): + if not fpath: + return fpath + try: + file_path, file_name = os.path.split( fpath ) + except: + file_name = fpath + return file_name +def update_repository( repo, ctx_rev=None ): + """ + Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired + changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. + """ + # TODO: We may have files on disk in the repo directory that aren't being tracked, so they must be removed. + # The codes used to show the status of files are as follows. + # M = modified + # A = added + # R = removed + # C = clean + # ! = deleted, but still tracked + # ? = not tracked + # I = ignored + # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that + # purging is not supported by the mercurial API. See the deprecated update_for_browsing() method in common.py. + commands.update( get_configured_ui(), + repo, + rev=ctx_rev ) +def url_join( *args ): + parts = [] + for arg in args: + parts.append( arg.strip( '/' ) ) + return '/'.join( parts ) diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/webapps/community/controllers/admin.py --- a/lib/galaxy/webapps/community/controllers/admin.py +++ b/lib/galaxy/webapps/community/controllers/admin.py @@ -5,9 +5,7 @@ from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.web.form_builder import SelectField from galaxy.util import inflector -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import build_repository_ids_select_field, get_changectx_for_changeset, get_configured_ui, get_repository_in_tool_shed -from galaxy.util.shed_util import reset_metadata_on_selected_repositories, TOOL_SHED_ADMIN_CONTROLLER +from galaxy.util.shed_util_common import * from common import * from repository import RepositoryGrid, CategoryGrid diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -5,13 +5,7 @@ from galaxy.tools import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.hash_util import * -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import check_tool_input_params, clone_repository, concat_messages, copy_sample_file, create_or_update_repository_metadata -from galaxy.util.shed_util import generate_clone_url_for_repository_in_tool_shed, generate_message_for_invalid_tools, generate_metadata_for_changeset_revision -from galaxy.util.shed_util import get_changectx_for_changeset, get_config_from_disk, get_configured_ui, get_file_context_from_ctx, get_named_tmpfile_from_ctx -from galaxy.util.shed_util import get_parent_id, get_repository_in_tool_shed, get_repository_metadata_by_changeset_revision -from galaxy.util.shed_util import handle_sample_files_and_load_tool_from_disk, handle_sample_files_and_load_tool_from_tmp_config, INITIAL_CHANGELOG_HASH -from galaxy.util.shed_util import is_downloadable, load_tool_from_config, remove_dir, reset_tool_data_tables, reversed_upper_bounded_changelog, strip_path +from galaxy.util.shed_util_common import * from galaxy.web.base.controller import * from galaxy.web.base.controllers.admin import * from galaxy.webapps.community import model diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -9,14 +9,7 @@ from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import create_repo_info_dict, generate_clone_url_for_repository_in_tool_shed, generate_message_for_invalid_tools -from galaxy.util.shed_util import get_changectx_for_changeset, get_configured_ui, get_file_from_changeset_revision -from galaxy.util.shed_util import get_repository_file_contents, get_repository_in_tool_shed, get_repository_metadata_by_changeset_revision -from galaxy.util.shed_util import handle_sample_files_and_load_tool_from_disk, handle_sample_files_and_load_tool_from_tmp_config -from galaxy.util.shed_util import INITIAL_CHANGELOG_HASH, load_tool_from_config, NOT_TOOL_CONFIGS, open_repository_files_folder, remove_dir -from galaxy.util.shed_util import reset_all_metadata_on_repository_in_tool_shed, reversed_lower_upper_bounded_changelog -from galaxy.util.shed_util import reversed_upper_bounded_changelog, strip_path, to_html_escaped, update_repository, url_join +from galaxy.util.shed_util_common import * from galaxy.tool_shed.encoding_util import * from common import * @@ -1265,6 +1258,13 @@ trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' return get_repository_file_contents( file_path ) + def get_file_from_changeset_revision( self, repo_files_dir, changeset_revision, file_name, dir ): + """Return file_name from the received changeset_revision of the repository manifest.""" + stripped_file_name = strip_path( file_name ) + repo = hg.repository( get_configured_ui(), repo_files_dir ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) + named_tmp_file = get_named_tmpfile_from_ctx( ctx, file_name, dir ) + return named_tmp_file def get_metadata( self, trans, repository_id, changeset_revision ): repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) if repository_metadata and repository_metadata.metadata: @@ -2231,6 +2231,17 @@ if list: return ','.join( list ) return '' + def to_html_escaped( self, text ): + """Translates the characters in text to html values""" + translated = [] + for c in text: + if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS: + translated.append( c ) + elif c in MAPPED_CHARS: + translated.append( MAPPED_CHARS[ c ] ) + else: + translated.append( '' ) + return ''.join( translated ) def __validate_repository_name( self, name, user ): # Repository names must be unique for each user, must be at least four characters # in length and must contain only lower-case letters, numbers, and the '_' character. @@ -2304,7 +2315,7 @@ anchors = modified + added + removed + deleted + unknown + ignored + clean diffs = [] for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node() ): - diffs.append( to_html_escaped( diff ) ) + diffs.append( self.to_html_escaped( diff ) ) is_malicious = changeset_is_malicious( trans, id, repository.tip( trans.app ) ) metadata = self.get_metadata( trans, id, ctx_str ) return trans.fill_template( '/webapps/community/repository/view_changeset.mako', @@ -2356,12 +2367,7 @@ except IOError: work_dir = tempfile.mkdtemp() try: - manifest_readme_file = get_file_from_changeset_revision( trans.app, - repository, - repo_files_dir, - changeset_revision, - readme_file, - work_dir ) + manifest_readme_file = self.get_file_from_changeset_revision( repo_files_dir, changeset_revision, readme_file, work_dir ) f = open( manifest_readme_file, 'r' ) raw_text = f.read() f.close() diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/webapps/community/controllers/repository_review.py --- a/lib/galaxy/webapps/community/controllers/repository_review.py +++ b/lib/galaxy/webapps/community/controllers/repository_review.py @@ -8,8 +8,7 @@ from sqlalchemy.sql.expression import func from common import * from repository import RepositoryGrid -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import get_configured_ui, get_repository_in_tool_shed +from galaxy.util.shed_util_common import * from galaxy.util.odict import odict from galaxy import eggs diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/webapps/community/controllers/upload.py --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -3,9 +3,7 @@ from galaxy.model.orm import * from galaxy.datatypes.checkers import * from common import * -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import get_configured_ui, get_repository_in_tool_shed, reset_tool_data_tables, handle_sample_tool_data_table_conf_file -from galaxy.util.shed_util import update_repository +from galaxy.util.shed_util_common import * from galaxy import eggs eggs.require('mercurial') diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/webapps/community/controllers/workflow.py --- a/lib/galaxy/webapps/community/controllers/workflow.py +++ b/lib/galaxy/webapps/community/controllers/workflow.py @@ -10,8 +10,7 @@ from galaxy.webapps.galaxy.controllers.workflow import attach_ordered_steps from galaxy.model.orm import * from common import * -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import get_repository_in_tool_shed +from galaxy.util.shed_util_common import * from galaxy.tool_shed.encoding_util import * class RepoInputDataModule( InputDataModule ): diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -2,6 +2,7 @@ from admin import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * +from galaxy.util.shed_util_common import * from galaxy.tool_shed.encoding_util import * from galaxy import eggs, tools diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r e1895e14176e80bbeccb918dba03723b07cc8112 templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -77,7 +77,7 @@ <%def name="render_clone_str( repository )"><% - from galaxy.util.shed_util import generate_clone_url_for_repository_in_tool_shed + from galaxy.util.shed_util_common import generate_clone_url_for_repository_in_tool_shed clone_str = generate_clone_url_for_repository_in_tool_shed( trans, repository ) %> hg clone <a href="${clone_str}">${clone_str}</a> https://bitbucket.org/galaxy/galaxy-central/changeset/7a51b701af88/ changeset: 7a51b701af88 user: natefoo date: 2012-11-15 21:06:43 summary: Merge affected #: 13 files diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/tool_shed/install_manager.py --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -6,6 +6,7 @@ from galaxy.tools import ToolSection from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * +from galaxy.util.shed_util_common import * from galaxy.util.odict import odict from galaxy.tool_shed.common_util import * diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/tool_shed/update_manager.py --- a/lib/galaxy/tool_shed/update_manager.py +++ b/lib/galaxy/tool_shed/update_manager.py @@ -4,6 +4,7 @@ import threading, urllib2, logging from galaxy.util import string_as_bool from galaxy.util.shed_util import * +from galaxy.util.shed_util_common import * log = logging.getLogger( __name__ ) diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -33,6 +33,7 @@ from galaxy.util.hash_util import * from galaxy.util import listify from galaxy.util.shed_util import * +from galaxy.util.shed_util_common import * from galaxy.web import url_for from galaxy.visualization.genome.visual_analytics import TracksterConfig diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1,14 +1,9 @@ -import sys, os, tempfile, shutil, logging, string, urllib2 -import galaxy.tools.data -from datetime import date, datetime, timedelta +import os, tempfile, shutil, logging, urllib2 from galaxy import util -from galaxy.web import url_for -from galaxy.web.form_builder import SelectField -from galaxy.tools import parameters from galaxy.datatypes.checkers import * from galaxy.datatypes.sniff import is_column_based from galaxy.util.json import * -from galaxy.util import inflector +from galaxy.util.shed_util_common import * from galaxy.tools.search import ToolBoxSearch from galaxy.tool_shed.tool_dependencies.install_util import create_or_update_tool_dependency, install_package, set_environment from galaxy.tool_shed.encoding_util import * @@ -26,19 +21,6 @@ log = logging.getLogger( __name__ ) -GALAXY_ADMIN_TOOL_SHED_CONTROLLER = 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER' -INITIAL_CHANGELOG_HASH = '000000000000' -# Characters that must be html escaped -MAPPED_CHARS = { '>' :'>', - '<' :'<', - '"' : '"', - '&' : '&', - '\'' : ''' } -MAX_CONTENT_SIZE = 32768 -NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'tool_dependencies.xml' ] -VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) -TOOL_SHED_ADMIN_CONTROLLER = 'TOOL_SHED_ADMIN_CONTROLLER' - def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ): # A tool shed repository is being installed so change the shed_tool_conf file. Parse the config file to generate the entire list # of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have @@ -175,27 +157,6 @@ except: pass return converter_path, display_path -def build_repository_ids_select_field( trans, cntrller, name='repository_ids', multiple=True, display='checkboxes' ): - """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" - repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - if cntrller == TOOL_SHED_ADMIN_CONTROLLER: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - elif cntrller == GALAXY_ADMIN_TOOL_SHED_CONTROLLER: - for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ - .order_by( trans.model.ToolShedRepository.table.c.name, - trans.model.ToolShedRepository.table.c.owner ): - option_label = '%s (%s)' % ( repository.name, repository.owner ) - option_value = trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - return repositories_select_field def can_generate_tool_dependency_metadata( root, metadata_dict ): """ Make sure the combination of name, version and type (the type will be the value of elem.tag) of each root element tag in the tool_dependencies.xml @@ -245,50 +206,6 @@ # tag for any tool in the repository. break return can_generate_dependency_metadata -def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): - """ - Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make - sure the files exist. - """ - invalid_files_and_errors_tups = [] - correction_msg = '' - for input_param in tool.input_params: - if isinstance( input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: - # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, make sure all requirements exist. - options = input_param.dynamic_options or input_param.options - if options: - if options.tool_data_table or options.missing_tool_data_table_name: - # Make sure the repository contains a tool_data_table_conf.xml.sample file. - sample_tool_data_table_conf = get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) - if sample_tool_data_table_conf: - error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf ) - if error: - invalid_files_and_errors_tups.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) - else: - options.missing_tool_data_table_name = None - else: - correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. Upload a file named tool_data_table_conf.xml.sample " - correction_msg += "to the repository that includes the required entry to correct this error.<br/>" - invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) - if options.index_file or options.missing_index_file: - # Make sure the repository contains the required xxx.loc.sample file. - index_file = options.index_file or options.missing_index_file - index_file_name = strip_path( index_file ) - sample_found = False - for sample_file in sample_files: - sample_file_name = strip_path( sample_file ) - if sample_file_name == '%s.sample' % index_file_name: - options.index_file = index_file_name - options.missing_index_file = None - if options.tool_data_table: - options.tool_data_table.missing_index_file = None - sample_found = True - break - if not sample_found: - correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file ) - correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) - invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) - return invalid_files_and_errors_tups def clean_repository_metadata( trans, id, changeset_revisions ): # Delete all repository_metadata records associated with the repository that have a changeset_revision that is not in changeset_revisions. # We sometimes see multiple records with the same changeset revision value - no idea how this happens. We'll assume we can delete the older @@ -388,17 +305,6 @@ else: return 'subset' return 'not equal and not subset' -def concat_messages( msg1, msg2 ): - if msg1: - if msg2: - message = '%s %s' % ( msg1, msg2 ) - else: - message = msg1 - elif msg2: - message = msg2 - else: - message = '' - return message def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ): # Persist the current in-memory list of config_elems to a file named by the value of config_filename. fd, filename = tempfile.mkstemp() @@ -439,35 +345,6 @@ # Eliminate the port, if any, since it will result in an invalid directory name. return tool_shed_url.split( ':' )[ 0 ] return tool_shed_url.rstrip( '/' ) -def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): - """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" - try: - commands.clone( get_configured_ui(), - str( repository_clone_url ), - dest=str( repository_file_dir ), - pull=True, - noupdate=False, - rev=util.listify( str( ctx_rev ) ) ) - return True, None - except Exception, e: - error_message = 'Error cloning repository: %s' % str( e ) - log.debug( error_message ) - return False, error_message -def copy_sample_file( app, filename, dest_path=None ): - """Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path is ~/tool-data.""" - if dest_path is None: - dest_path = os.path.abspath( app.config.tool_data_path ) - sample_file_name = strip_path( filename ) - copied_file = sample_file_name.replace( '.sample', '' ) - full_source_path = os.path.abspath( filename ) - full_destination_path = os.path.join( dest_path, sample_file_name ) - # Don't copy a file to itself - not sure how this happens, but sometimes it does... - if full_source_path != full_destination_path: - # It's ok to overwrite the .sample version of the file. - shutil.copy( full_source_path, full_destination_path ) - # Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance. - if not os.path.exists( os.path.join( dest_path, copied_file ) ): - shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) def copy_sample_files( app, sample_files, tool_path=None, sample_files_copied=None, dest_path=None ): """ Copy all appropriate files to dest_path in the local Galaxy environment that have not already been copied. Those that have been copied @@ -484,15 +361,6 @@ # Attempt to ensure we're copying an appropriate file. if is_data_index_sample_file( filename ): copy_sample_file( app, filename, dest_path=dest_path ) -def create_repo_info_dict( repository, owner, repository_clone_url, changeset_revision, ctx_rev, metadata ): - repo_info_dict = {} - repo_info_dict[ repository.name ] = ( repository.description, - repository_clone_url, - changeset_revision, - ctx_rev, - owner, - metadata.get( 'tool_dependencies', None ) ) - return repo_info_dict def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): return dict( tool_shed=tool_shed, repository_name=name, @@ -501,20 +369,6 @@ tool_dicts=tool_dicts, converter_path=converter_path, display_path=display_path ) -def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ): - downloadable = is_downloadable( metadata_dict ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) - if repository_metadata: - repository_metadata.metadata = metadata_dict - repository_metadata.downloadable = downloadable - else: - repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, - changeset_revision=changeset_revision, - metadata=metadata_dict, - downloadable=downloadable ) - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() - return repository_metadata def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used @@ -618,15 +472,6 @@ """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) return url_join( tool_shed_url, 'repos', repository.owner, repository.name ) -def generate_clone_url_for_repository_in_tool_shed( trans, repository ): - """Generate the URL for cloning a repository that is in the tool shed.""" - base_url = url_for( '/', qualified=True ).rstrip( '/' ) - if trans.user: - protocol, base = base_url.split( '://' ) - username = '%s@' % trans.user.username - return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) - else: - return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name ) def generate_datatypes_metadata( datatypes_config, metadata_dict ): """Update the received metadata_dict with information from the parsed datatypes_config.""" tree = ElementTree.parse( datatypes_config ) @@ -681,191 +526,6 @@ else: tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ] return tool_dependencies_dict -def generate_metadata_for_changeset_revision( app, repository, repository_clone_url, shed_config_dict={}, relative_install_dir=None, repository_files_dir=None, - resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ): - """ - Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip, - the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's - disk files, so the value of repository_files_dir will not always be repository.repo_path( app ) (it could be an absolute path to a temporary - directory containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path( app ). - - The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries - should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. - """ - if updating_installed_repository: - # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which - # we have pulled updates. - original_repository_metadata = repository.metadata - else: - original_repository_metadata = None - readme_file_names = get_readme_file_names( repository.name ) - metadata_dict = { 'shed_config_filename': shed_config_dict.get( 'config_filename' ) } - invalid_file_tups = [] - invalid_tool_configs = [] - tool_dependencies_config = None - original_tool_data_path = app.config.tool_data_path - original_tool_data_table_config_path = app.config.tool_data_table_config_path - if resetting_all_metadata_on_repository: - if not relative_install_dir: - raise Exception( "The value of repository.repo_path( app ) must be sent when resetting all metadata on a repository." ) - # Keep track of the location where the repository is temporarily cloned so that we can strip the path when setting metadata. The value of - # repository_files_dir is the full path to the temporary directory to which the repository was cloned. - work_dir = repository_files_dir - files_dir = repository_files_dir - # Since we're working from a temporary directory, we can safely copy sample files included in the repository to the repository root. - app.config.tool_data_path = repository_files_dir - app.config.tool_data_table_config_path = repository_files_dir - else: - # Use a temporary working directory to copy all sample files. - work_dir = tempfile.mkdtemp() - # All other files are on disk in the repository's repo_path, which is the value of relative_install_dir. - files_dir = relative_install_dir - if shed_config_dict.get( 'tool_path' ): - files_dir = os.path.join( shed_config_dict['tool_path'], files_dir ) - app.config.tool_data_path = work_dir - app.config.tool_data_table_config_path = work_dir - # Handle proprietary datatypes, if any. - datatypes_config = get_config_from_disk( 'datatypes_conf.xml', files_dir ) - if datatypes_config: - metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict ) - # Get the relative path to all sample files included in the repository for storage in the repository's metadata. - sample_file_metadata_paths, sample_file_copy_paths = get_sample_files_from_disk( repository_files_dir=files_dir, - tool_path=shed_config_dict.get( 'tool_path' ), - relative_install_dir=relative_install_dir, - resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) - if sample_file_metadata_paths: - metadata_dict[ 'sample_files' ] = sample_file_metadata_paths - # Copy all sample files included in the repository to a single directory location so we can load tools that depend on them. - for sample_file in sample_file_copy_paths: - copy_sample_file( app, sample_file, dest_path=work_dir ) - # If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory. - relative_path, filename = os.path.split( sample_file ) - if filename == 'tool_data_table_conf.xml.sample': - new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file, - tool_data_path=original_tool_data_path, - shed_tool_data_table_config=app.config.shed_tool_data_table_config, - persist=persist ) - for root, dirs, files in os.walk( files_dir ): - if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: - if '.hg' in dirs: - dirs.remove( '.hg' ) - for name in files: - # See if we have a READ_ME file. - if name.lower() in readme_file_names: - if resetting_all_metadata_on_repository: - full_path_to_readme = os.path.join( root, name ) - stripped_path_to_readme = full_path_to_readme.replace( work_dir, '' ) - if stripped_path_to_readme.startswith( '/' ): - stripped_path_to_readme = stripped_path_to_readme[ 1: ] - relative_path_to_readme = os.path.join( relative_install_dir, stripped_path_to_readme ) - else: - relative_path_to_readme = os.path.join( root, name ) - if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_readme.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): - relative_path_to_readme = relative_path_to_readme[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] - metadata_dict[ 'readme' ] = relative_path_to_readme - # See if we have a tool config. - elif name not in NOT_TOOL_CONFIGS and name.endswith( '.xml' ): - full_path = str( os.path.abspath( os.path.join( root, name ) ) ) - if os.path.getsize( full_path ) > 0: - if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ] - or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ): - try: - # Make sure we're looking at a tool config and not a display application config or something else. - element_tree = util.parse_xml( full_path ) - element_tree_root = element_tree.getroot() - is_tool = element_tree_root.tag == 'tool' - except Exception, e: - log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) ) - is_tool = False - if is_tool: - tool, valid, error_message = load_tool_from_config( app, full_path ) - if tool is None: - if not valid: - invalid_file_tups.append( ( name, error_message ) ) - else: - invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_file_metadata_paths ) - can_set_metadata = True - for tup in invalid_files_and_errors_tups: - if name in tup: - can_set_metadata = False - invalid_tool_configs.append( name ) - break - if can_set_metadata: - if resetting_all_metadata_on_repository: - full_path_to_tool_config = os.path.join( root, name ) - stripped_path_to_tool_config = full_path_to_tool_config.replace( work_dir, '' ) - if stripped_path_to_tool_config.startswith( '/' ): - stripped_path_to_tool_config = stripped_path_to_tool_config[ 1: ] - relative_path_to_tool_config = os.path.join( relative_install_dir, stripped_path_to_tool_config ) - else: - relative_path_to_tool_config = os.path.join( root, name ) - if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_tool_config.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): - relative_path_to_tool_config = relative_path_to_tool_config[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] - metadata_dict = generate_tool_metadata( relative_path_to_tool_config, tool, repository_clone_url, metadata_dict ) - else: - for tup in invalid_files_and_errors_tups: - invalid_file_tups.append( tup ) - # Find all exported workflows. - elif name.endswith( '.ga' ): - relative_path = os.path.join( root, name ) - if os.path.getsize( os.path.abspath( relative_path ) ) > 0: - fp = open( relative_path, 'rb' ) - workflow_text = fp.read() - fp.close() - exported_workflow_dict = from_json_string( workflow_text ) - if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': - metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ) - if 'tools' in metadata_dict: - # This step must be done after metadata for tools has been defined. - tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', files_dir ) - if tool_dependencies_config: - metadata_dict = generate_tool_dependency_metadata( app, - repository, - tool_dependencies_config, - metadata_dict, - original_repository_metadata=original_repository_metadata ) - if invalid_tool_configs: - metadata_dict [ 'invalid_tools' ] = invalid_tool_configs - # Reset the value of the app's tool_data_path and tool_data_table_config_path to their respective original values. - app.config.tool_data_path = original_tool_data_path - app.config.tool_data_table_config_path = original_tool_data_table_config_path - return metadata_dict, invalid_file_tups -def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): - if as_html: - new_line = '<br/>' - bold_start = '<b>' - bold_end = '</b>' - else: - new_line = '\n' - bold_start = '' - bold_end = '' - message = '' - if not displaying_invalid_tool: - if metadata_dict: - message += "Metadata was defined for some items in revision '%s'. " % str( repository.tip( trans.app ) ) - message += "Correct the following problems if necessary and reset metadata.%s" % new_line - else: - message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip( trans.app ) ) - message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.%s" % new_line - for itc_tup in invalid_file_tups: - tool_file, exception_msg = itc_tup - if exception_msg.find( 'No such file or directory' ) >= 0: - exception_items = exception_msg.split() - missing_file_items = exception_items[ 7 ].split( '/' ) - missing_file = missing_file_items[ -1 ].rstrip( '\'' ) - if missing_file.endswith( '.loc' ): - sample_ext = '%s.sample' % missing_file - else: - sample_ext = missing_file - correction_msg = "This file refers to a missing file %s%s%s. " % ( bold_start, str( missing_file ), bold_end ) - correction_msg += "Upload a file named %s%s%s to the repository to correct this error." % ( bold_start, sample_ext, bold_end ) - else: - if as_html: - correction_msg = exception_msg - else: - correction_msg = exception_msg.replace( '<br/>', new_line ).replace( '<b>', bold_start ).replace( '</b>', bold_end ) - message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) - return message def generate_package_dependency_metadata( elem, tool_dependencies_dict ): """The value of package_name must match the value of the "package" type in the tool config's <requirements> tag set.""" requirements_dict = {} @@ -1155,13 +815,6 @@ else: metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ] return metadata_dict -def get_changectx_for_changeset( repo, changeset_revision, **kwd ): - """Retrieve a specified changectx from a repository""" - for changeset in repo.changelog: - ctx = repo.changectx( changeset ) - if str( ctx ) == changeset_revision: - return ctx - return None def get_config( config_file, repo, ctx, dir ): """Return the latest version of config_filename from the repository manifest.""" config_file = strip_path( config_file ) @@ -1172,22 +825,6 @@ if ctx_file_name == config_file: return get_named_tmpfile_from_ctx( changeset_ctx, ctx_file, dir ) return None -def get_config_from_disk( config_file, relative_install_dir ): - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == config_file: - return os.path.abspath( os.path.join( root, name ) ) - return None -def get_configured_ui(): - # Configure any desired ui settings. - _ui = ui.ui() - # The following will suppress all messages. This is - # the same as adding the following setting to the repo - # hgrc file' [ui] section: - # quiet = True - _ui.setconfig( 'ui', 'quiet', True ) - return _ui def get_converter_and_display_paths( registration_elem, relative_install_dir ): """Find the relative path to data type converters and display applications included in installed tool shed repositories.""" converter_path = None @@ -1247,33 +884,6 @@ ctx_rev = response.read() response.close() return ctx_rev -def get_file_context_from_ctx( ctx, filename ): - # We have to be careful in determining if we found the correct file because multiple files with the same name may be in different directories - # within ctx if the files were moved within the change set. For example, in the following ctx.files() list, the former may have been moved to - # the latter: ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample']. Another scenario - # is that the file has been deleted. - deleted = False - filename = strip_path( filename ) - for ctx_file in ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if filename == ctx_file_name: - try: - # If the file was moved, its destination will be returned here. - fctx = ctx[ ctx_file ] - return fctx - except LookupError, e: - # Set deleted for now, and continue looking in case the file was moved instead of deleted. - deleted = True - if deleted: - return 'DELETED' - return None -def get_file_from_changeset_revision( app, repository, repo_files_dir, changeset_revision, file_name, dir ): - """Return file_name from the received changeset_revision of the repository manifest.""" - stripped_file_name = strip_path( file_name ) - repo = hg.repository( get_configured_ui(), repo_files_dir ) - ctx = get_changectx_for_changeset( repo, changeset_revision ) - named_tmp_file = get_named_tmpfile_from_ctx( ctx, file_name, dir ) - return named_tmp_file def get_installed_tool_shed_repository( trans, id ): """Get a repository on the Galaxy side from the database via id""" return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) @@ -1309,63 +919,6 @@ fh.write( fctx.data() ) fh.close() return sample_files, deleted_sample_files -def get_named_tmpfile_from_ctx( ctx, filename, dir ): - filename = strip_path( filename ) - for ctx_file in ctx.files(): - ctx_file_name = strip_path( ctx_file ) - if filename == ctx_file_name: - try: - # If the file was moved, its destination file contents will be returned here. - fctx = ctx[ ctx_file ] - except LookupError, e: - # Continue looking in case the file was moved. - fctx = None - continue - if fctx: - fh = tempfile.NamedTemporaryFile( 'wb', dir=dir ) - tmp_filename = fh.name - fh.close() - fh = open( tmp_filename, 'wb' ) - fh.write( fctx.data() ) - fh.close() - return tmp_filename - return None -def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): - parent_id = None - # Compare from most recent to oldest. - changeset_revisions.reverse() - for changeset_revision in changeset_revisions: - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) - metadata = repository_metadata.metadata - tools_dicts = metadata.get( 'tools', [] ) - for tool_dict in tools_dicts: - if tool_dict[ 'guid' ] == guid: - # The tool has not changed between the compared changeset revisions. - continue - if tool_dict[ 'id' ] == old_id and tool_dict[ 'version' ] != version: - # The tool version is different, so we've found the parent. - return tool_dict[ 'guid' ] - if parent_id is None: - # The tool did not change through all of the changeset revisions. - return old_id -def get_repository_file_contents( file_path ): - if is_gzip( file_path ): - to_html = to_html_str( '\ngzip compressed file\n' ) - elif is_bz2( file_path ): - to_html = to_html_str( '\nbz2 compressed file\n' ) - elif check_zip( file_path ): - to_html = to_html_str( '\nzip compressed file\n' ) - elif check_binary( file_path ): - to_html = to_html_str( '\nBinary file\n' ) - else: - to_html = '' - for i, line in enumerate( open( file_path ) ): - to_html = '%s%s' % ( to_html, to_html_str( line ) ) - if len( to_html ) > MAX_CONTENT_SIZE: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) - to_html = '%s%s' % ( to_html, to_html_str( large_str ) ) - break - return to_html def get_repository_files( trans, folder_path ): contents = [] for item in os.listdir( folder_path ): @@ -1379,28 +932,6 @@ if contents: contents.sort() return contents -def get_repository_in_tool_shed( trans, id ): - """Get a repository on the tool shed side from the database via id""" - return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) -def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): - """Get metadata for a specified repository change set from the database""" - # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. Duplicate records were somehow - # created in the past. The cause of this issue has been resolved, but we'll leave this method as is for a while longer to ensure all duplicate - # records are removed. - all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \ - .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), - trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ - .order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \ - .all() - if len( all_metadata_records ) > 1: - # Delete all recrds older than the last one updated. - for repository_metadata in all_metadata_records[ 1: ]: - trans.sa_session.delete( repository_metadata ) - trans.sa_session.flush() - return all_metadata_records[ 0 ] - elif all_metadata_records: - return all_metadata_records[ 0 ] - return None def get_repository_owner( cleaned_repository_url ): items = cleaned_repository_url.split( 'repos' ) repo_path = items[ 1 ] @@ -1667,55 +1198,6 @@ repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied -def handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ): - # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. - message = '' - sample_files = copy_disk_sample_files_to_dir( trans, repo_files_dir, work_dir ) - if sample_files: - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) - tool, valid, message2 = load_tool_from_config( trans.app, tool_config_filepath ) - message = concat_messages( message, message2 ) - return tool, valid, message, sample_files -def handle_sample_files_and_load_tool_from_tmp_config( trans, repo, changeset_revision, tool_config_filename, work_dir ): - tool = None - message = '' - ctx = get_changectx_for_changeset( repo, changeset_revision ) - # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in - # the manifest, but have been deleted from disk. - sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) - if sample_files: - trans.app.config.tool_data_path = work_dir - if 'tool_data_table_conf.xml.sample' in sample_files: - # Load entries into the tool_data_tables if the tool requires them. - tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) - if tool_data_table_config: - error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) - if error: - log.debug( message ) - manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) - if manifest_ctx and ctx_file: - tool, message2 = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir ) - message = concat_messages( message, message2 ) - return tool, message, sample_files -def handle_sample_tool_data_table_conf_file( app, filename, persist=False ): - """ - Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur - if call is from the Galaxy side, not the tool shed), the new entries will be appended to Galaxy's shed_tool_data_table_conf.xml file on disk. - """ - error = False - message = '' - try: - new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=filename, - tool_data_path=app.config.tool_data_path, - shed_tool_data_table_config=app.config.shed_tool_data_table_config, - persist=persist ) - except Exception, e: - message = str( e ) - error = True - return error, message def handle_tool_dependencies( app, tool_shed_repository, tool_dependencies_config, tool_dependencies ): """ Install and build tool dependencies defined in the tool_dependencies_config. This config's tag sets can currently refer to installation @@ -1800,8 +1282,6 @@ return False # Default to copying the file if none of the above are true. return True -def is_downloadable( metadata_dict ): - return 'datatypes' in metadata_dict or 'tools' in metadata_dict or 'workflows' in metadata_dict def load_installed_datatype_converters( app, installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype converters app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=installed_repository_dict, deactivate=deactivate ) @@ -1825,22 +1305,6 @@ def load_installed_display_applications( app, installed_repository_dict, deactivate=False ): # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate ) -def load_tool_from_config( app, full_path ): - try: - tool = app.toolbox.load_tool( full_path ) - valid = True - error_message = None - except KeyError, e: - tool = None - valid = False - error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) - error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' - error_message += 'this error. ' - except Exception, e: - tool = None - valid = False - error_message = str( e ) - return tool, valid, error_message def load_tool_from_tmp_config( trans, repo, ctx, ctx_file, work_dir ): tool = None message = '' @@ -1866,27 +1330,6 @@ except: pass return tool, message -def open_repository_files_folder( trans, folder_path ): - try: - files_list = get_repository_files( trans, folder_path ) - except OSError, e: - if str( e ).find( 'No such file or directory' ) >= 0: - # We have a repository with no contents. - return [] - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents def panel_entry_per_tool( tool_section_dict ): # Return True if tool_section_dict looks like this. # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]} @@ -1906,12 +1349,6 @@ repo, source=repository_clone_url, rev=[ ctx_rev ] ) -def remove_dir( dir ): - if os.path.exists( dir ): - try: - shutil.rmtree( dir ) - except: - pass def remove_from_shed_tool_config( trans, shed_tool_conf_dict, guids_to_remove ): # A tool shed repository is being uninstalled so change the shed_tool_conf file. Parse the config file to generate the entire list # of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have @@ -2113,209 +1550,6 @@ else: log.debug( 'Error locating installation directory for repository %s.' % repository.name ) return invalid_file_tups, metadata_dict -def reset_all_metadata_on_repository_in_tool_shed( trans, id ): - """Reset all metadata on a single repository in a tool shed.""" - def reset_all_tool_versions( trans, id, repo ): - changeset_revisions = [] - for changeset in repo.changelog: - changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) - if repository_metadata: - metadata = repository_metadata.metadata - if metadata: - if metadata.get( 'tools', None ): - changeset_revisions.append( changeset_revision ) - # The list of changeset_revisions is now filtered to contain only those that are downloadable and contain tools. - # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. - for index, changeset_revision in enumerate( changeset_revisions ): - tool_versions_dict = {} - repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) - metadata = repository_metadata.metadata - tool_dicts = metadata[ 'tools' ] - if index == 0: - # The first changset_revision is a special case because it will have no ancestor changeset_revisions in which to match tools. - # The parent tool id for tools in the first changeset_revision will be the "old_id" in the tool config. - for tool_dict in tool_dicts: - tool_versions_dict[ tool_dict[ 'guid' ] ] = tool_dict[ 'id' ] - else: - for tool_dict in tool_dicts: - parent_id = get_parent_id( trans, - id, - tool_dict[ 'id' ], - tool_dict[ 'version' ], - tool_dict[ 'guid' ], - changeset_revisions[ 0:index ] ) - tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id - if tool_versions_dict: - repository_metadata.tool_versions = tool_versions_dict - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() - repository = get_repository_in_tool_shed( trans, id ) - log.debug( "Resetting all metadata on repository: %s" % repository.name ) - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( get_configured_ui(), repo_dir ) - repository_clone_url = generate_clone_url_for_repository_in_tool_shed( trans, repository ) - # The list of changeset_revisions refers to repository_metadata records that have been created or updated. When the following loop - # completes, we'll delete all repository_metadata records for this repository that do not have a changeset_revision value in this list. - changeset_revisions = [] - # When a new repository_metadata record is created, it always uses the values of metadata_changeset_revision and metadata_dict. - metadata_changeset_revision = None - metadata_dict = None - ancestor_changeset_revision = None - ancestor_metadata_dict = None - invalid_file_tups = [] - home_dir = os.getcwd() - for changeset in repo.changelog: - work_dir = tempfile.mkdtemp() - current_changeset_revision = str( repo.changectx( changeset ) ) - ctx = repo.changectx( changeset ) - log.debug( "Cloning repository revision: %s", str( ctx.rev() ) ) - cloned_ok, error_message = clone_repository( repository_clone_url, work_dir, str( ctx.rev() ) ) - if cloned_ok: - log.debug( "Generating metadata for changset revision: %s", str( ctx.rev() ) ) - current_metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app, - repository=repository, - repository_clone_url=repository_clone_url, - relative_install_dir=repo_dir, - repository_files_dir=work_dir, - resetting_all_metadata_on_repository=True, - updating_installed_repository=False, - persist=False ) - if current_metadata_dict: - if not metadata_changeset_revision and not metadata_dict: - # We're at the first change set in the change log. - metadata_changeset_revision = current_changeset_revision - metadata_dict = current_metadata_dict - if ancestor_changeset_revision: - # Compare metadata from ancestor and current. The value of comparison will be one of: - # 'no metadata' - no metadata for either ancestor or current, so continue from current - # 'equal' - ancestor metadata is equivalent to current metadata, so continue from current - # 'subset' - ancestor metadata is a subset of current metadata, so continue from current - # 'not equal and not subset' - ancestor metadata is neither equal to nor a subset of current metadata, so persist ancestor metadata. - comparison = compare_changeset_revisions( ancestor_changeset_revision, - ancestor_metadata_dict, - current_changeset_revision, - current_metadata_dict ) - if comparison in [ 'no metadata', 'equal', 'subset' ]: - ancestor_changeset_revision = current_changeset_revision - ancestor_metadata_dict = current_metadata_dict - elif comparison == 'not equal and not subset': - metadata_changeset_revision = ancestor_changeset_revision - metadata_dict = ancestor_metadata_dict - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) - changeset_revisions.append( metadata_changeset_revision ) - ancestor_changeset_revision = current_changeset_revision - ancestor_metadata_dict = current_metadata_dict - else: - # We're at the beginning of the change log. - ancestor_changeset_revision = current_changeset_revision - ancestor_metadata_dict = current_metadata_dict - if not ctx.children(): - metadata_changeset_revision = current_changeset_revision - metadata_dict = current_metadata_dict - # We're at the end of the change log. - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) - changeset_revisions.append( metadata_changeset_revision ) - ancestor_changeset_revision = None - ancestor_metadata_dict = None - elif ancestor_metadata_dict: - # We reach here only if current_metadata_dict is empty and ancestor_metadata_dict is not. - if not ctx.children(): - # We're at the end of the change log. - repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) - changeset_revisions.append( metadata_changeset_revision ) - ancestor_changeset_revision = None - ancestor_metadata_dict = None - remove_dir( work_dir ) - # Delete all repository_metadata records for this repository that do not have a changeset_revision value in changeset_revisions. - clean_repository_metadata( trans, id, changeset_revisions ) - # Set tool version information for all downloadable changeset revisions. Get the list of changeset revisions from the changelog. - reset_all_tool_versions( trans, id, repo ) - # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. - reset_tool_data_tables( trans.app ) - return invalid_file_tups, metadata_dict -def reset_metadata_on_selected_repositories( trans, **kwd ): - # This method is called from both Galaxy and the Tool Shed, so the cntrller param is required. - repository_ids = util.listify( kwd.get( 'repository_ids', None ) ) - CONTROLLER = kwd[ 'CONTROLLER' ] - message = '' - status = 'done' - if repository_ids: - successful_count = 0 - unsuccessful_count = 0 - for repository_id in repository_ids: - try: - if CONTROLLER == 'TOOL_SHED_ADMIN_CONTROLLER': - repository = get_repository_in_tool_shed( trans, repository_id ) - invalid_file_tups, metadata_dict = reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) - elif CONTROLLER == 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER': - repository = get_installed_tool_shed_repository( trans, repository_id ) - invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans, repository_id ) - if invalid_file_tups: - message = generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) - log.debug( message ) - unsuccessful_count += 1 - else: - log.debug( "Successfully reset metadata on repository %s" % repository.name ) - successful_count += 1 - except Exception, e: - log.debug( "Error attempting to reset metadata on repository '%s': %s" % ( repository.name, str( e ) ) ) - unsuccessful_count += 1 - message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) - if unsuccessful_count: - message += "Error setting metadata on %d %s - see the paster log for details. " % ( unsuccessful_count, - inflector.cond_plural( unsuccessful_count, "repository" ) ) - else: - message = 'Select at least one repository to on which to reset all metadata.' - status = 'error' - return message, status -def reset_tool_data_tables( app ): - # Reset the tool_data_tables to an empty dictionary. - app.tool_data_tables.data_tables = {} -def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): - """ - Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, but up to and - including the included_upper_bounds_changeset_revision. The value of excluded_lower_bounds_changeset_revision will be the value of - INITIAL_CHANGELOG_HASH if no valid changesets exist before included_upper_bounds_changeset_revision. - """ - # To set excluded_lower_bounds_changeset_revision, calling methods should do the following, where the value of changeset_revision - # is a downloadable changeset_revision. - # excluded_lower_bounds_changeset_revision = get_previous_downloadable_changset_revision( repository, repo, changeset_revision ) - if excluded_lower_bounds_changeset_revision == INITIAL_CHANGELOG_HASH: - appending_started = True - else: - appending_started = False - reversed_changelog = [] - for changeset in repo.changelog: - changeset_hash = str( repo.changectx( changeset ) ) - if appending_started: - reversed_changelog.insert( 0, changeset ) - if changeset_hash == excluded_lower_bounds_changeset_revision and not appending_started: - appending_started = True - if changeset_hash == included_upper_bounds_changeset_revision: - break - return reversed_changelog -def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): - return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) -def strip_path( fpath ): - if not fpath: - return fpath - try: - file_path, file_name = os.path.split( fpath ) - except: - file_name = fpath - return file_name -def to_html_escaped( text ): - """Translates the characters in text to html values""" - translated = [] - for c in text: - if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS: - translated.append( c ) - elif c in MAPPED_CHARS: - translated.append( MAPPED_CHARS[ c ] ) - else: - translated.append( '' ) - return ''.join( translated ) def to_html_str( text ): """Translates the characters in text to an html string""" translated = [] @@ -2443,32 +1677,8 @@ elem = guid_to_tool_elem_dict[ guid ] config_elems.append( elem ) config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path ) -def update_repository( repo, ctx_rev=None ): - """ - Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired - changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. - """ - # TODO: We may have files on disk in the repo directory that aren't being tracked, so they must be removed. - # The codes used to show the status of files are as follows. - # M = modified - # A = added - # R = removed - # C = clean - # ! = deleted, but still tracked - # ? = not tracked - # I = ignored - # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that - # purging is not supported by the mercurial API. See the deprecated update_for_browsing() method in common.py. - commands.update( get_configured_ui(), - repo, - rev=ctx_rev ) def update_tool_shed_repository_status( app, tool_shed_repository, status ): sa_session = app.model.context.current tool_shed_repository.status = status sa_session.add( tool_shed_repository ) sa_session.flush() -def url_join( *args ): - parts = [] - for arg in args: - parts.append( arg.strip( '/' ) ) - return '/'.join( parts ) diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/util/shed_util_common.py --- /dev/null +++ b/lib/galaxy/util/shed_util_common.py @@ -0,0 +1,784 @@ +import os, shutil, tempfile, logging +from galaxy import util +from galaxy.tools import parameters +from galaxy.util import inflector +from galaxy.web import url_for +from galaxy.web.form_builder import SelectField +from galaxy.datatypes.checkers import * +from galaxy.model.orm import * + +from galaxy import eggs +import pkg_resources + +pkg_resources.require( 'mercurial' ) +from mercurial import hg, ui, commands + +log = logging.getLogger( __name__ ) + +INITIAL_CHANGELOG_HASH = '000000000000' +# Characters that must be html escaped +MAPPED_CHARS = { '>' :'>', + '<' :'<', + '"' : '"', + '&' : '&', + '\'' : ''' } +MAX_CONTENT_SIZE = 32768 +NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'tool_dependencies.xml' ] +GALAXY_ADMIN_TOOL_SHED_CONTROLLER = 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER' +TOOL_SHED_ADMIN_CONTROLLER = 'TOOL_SHED_ADMIN_CONTROLLER' +VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) + +def build_repository_ids_select_field( trans, cntrller, name='repository_ids', multiple=True, display='checkboxes' ): + """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" + repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) + if cntrller == TOOL_SHED_ADMIN_CONTROLLER: + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ): + owner = repository.user.username + option_label = '%s (%s)' % ( repository.name, owner ) + option_value = '%s' % trans.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) + elif cntrller == GALAXY_ADMIN_TOOL_SHED_CONTROLLER: + for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ + .order_by( trans.model.ToolShedRepository.table.c.name, + trans.model.ToolShedRepository.table.c.owner ): + option_label = '%s (%s)' % ( repository.name, repository.owner ) + option_value = trans.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) + return repositories_select_field +def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): + """ + Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make + sure the files exist. + """ + invalid_files_and_errors_tups = [] + correction_msg = '' + for input_param in tool.input_params: + if isinstance( input_param, parameters.basic.SelectToolParameter ) and input_param.is_dynamic: + # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml, make sure all requirements exist. + options = input_param.dynamic_options or input_param.options + if options: + if options.tool_data_table or options.missing_tool_data_table_name: + # Make sure the repository contains a tool_data_table_conf.xml.sample file. + sample_tool_data_table_conf = get_config_from_disk( 'tool_data_table_conf.xml.sample', repo_dir ) + if sample_tool_data_table_conf: + error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_tool_data_table_conf ) + if error: + invalid_files_and_errors_tups.append( ( 'tool_data_table_conf.xml.sample', correction_msg ) ) + else: + options.missing_tool_data_table_name = None + else: + correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. Upload a file named tool_data_table_conf.xml.sample " + correction_msg += "to the repository that includes the required entry to correct this error.<br/>" + invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) + if options.index_file or options.missing_index_file: + # Make sure the repository contains the required xxx.loc.sample file. + index_file = options.index_file or options.missing_index_file + index_file_name = strip_path( index_file ) + sample_found = False + for sample_file in sample_files: + sample_file_name = strip_path( sample_file ) + if sample_file_name == '%s.sample' % index_file_name: + options.index_file = index_file_name + options.missing_index_file = None + if options.tool_data_table: + options.tool_data_table.missing_index_file = None + sample_found = True + break + if not sample_found: + correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file ) + correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name ) + invalid_files_and_errors_tups.append( ( tool_config_name, correction_msg ) ) + return invalid_files_and_errors_tups +def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): + """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" + try: + commands.clone( get_configured_ui(), + str( repository_clone_url ), + dest=str( repository_file_dir ), + pull=True, + noupdate=False, + rev=util.listify( str( ctx_rev ) ) ) + return True, None + except Exception, e: + error_message = 'Error cloning repository: %s' % str( e ) + log.debug( error_message ) + return False, error_message +def concat_messages( msg1, msg2 ): + if msg1: + if msg2: + message = '%s %s' % ( msg1, msg2 ) + else: + message = msg1 + elif msg2: + message = msg2 + else: + message = '' + return message +def copy_sample_file( app, filename, dest_path=None ): + """Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path is ~/tool-data.""" + if dest_path is None: + dest_path = os.path.abspath( app.config.tool_data_path ) + sample_file_name = strip_path( filename ) + copied_file = sample_file_name.replace( '.sample', '' ) + full_source_path = os.path.abspath( filename ) + full_destination_path = os.path.join( dest_path, sample_file_name ) + # Don't copy a file to itself - not sure how this happens, but sometimes it does... + if full_source_path != full_destination_path: + # It's ok to overwrite the .sample version of the file. + shutil.copy( full_source_path, full_destination_path ) + # Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance. + if not os.path.exists( os.path.join( dest_path, copied_file ) ): + shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) ) +def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ): + downloadable = is_downloadable( metadata_dict ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + if repository_metadata: + repository_metadata.metadata = metadata_dict + repository_metadata.downloadable = downloadable + else: + repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id, + changeset_revision=changeset_revision, + metadata=metadata_dict, + downloadable=downloadable ) + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + return repository_metadata +def create_repo_info_dict( repository, owner, repository_clone_url, changeset_revision, ctx_rev, metadata ): + repo_info_dict = {} + repo_info_dict[ repository.name ] = ( repository.description, + repository_clone_url, + changeset_revision, + ctx_rev, + owner, + metadata.get( 'tool_dependencies', None ) ) + return repo_info_dict +def generate_clone_url_for_repository_in_tool_shed( trans, repository ): + """Generate the URL for cloning a repository that is in the tool shed.""" + base_url = url_for( '/', qualified=True ).rstrip( '/' ) + if trans.user: + protocol, base = base_url.split( '://' ) + username = '%s@' % trans.user.username + return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name ) + else: + return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name ) +def generate_message_for_invalid_tools( trans, invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ): + if as_html: + new_line = '<br/>' + bold_start = '<b>' + bold_end = '</b>' + else: + new_line = '\n' + bold_start = '' + bold_end = '' + message = '' + if not displaying_invalid_tool: + if metadata_dict: + message += "Metadata was defined for some items in revision '%s'. " % str( repository.tip( trans.app ) ) + message += "Correct the following problems if necessary and reset metadata.%s" % new_line + else: + message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip( trans.app ) ) + message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.%s" % new_line + for itc_tup in invalid_file_tups: + tool_file, exception_msg = itc_tup + if exception_msg.find( 'No such file or directory' ) >= 0: + exception_items = exception_msg.split() + missing_file_items = exception_items[ 7 ].split( '/' ) + missing_file = missing_file_items[ -1 ].rstrip( '\'' ) + if missing_file.endswith( '.loc' ): + sample_ext = '%s.sample' % missing_file + else: + sample_ext = missing_file + correction_msg = "This file refers to a missing file %s%s%s. " % ( bold_start, str( missing_file ), bold_end ) + correction_msg += "Upload a file named %s%s%s to the repository to correct this error." % ( bold_start, sample_ext, bold_end ) + else: + if as_html: + correction_msg = exception_msg + else: + correction_msg = exception_msg.replace( '<br/>', new_line ).replace( '<b>', bold_start ).replace( '</b>', bold_end ) + message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line ) + return message +def generate_metadata_for_changeset_revision( app, repository, repository_clone_url, shed_config_dict={}, relative_install_dir=None, repository_files_dir=None, + resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ): + """ + Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip, + the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's + disk files, so the value of repository_files_dir will not always be repository.repo_path( app ) (it could be an absolute path to a temporary + directory containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path( app ). + + The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries + should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. + """ + if updating_installed_repository: + # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which + # we have pulled updates. + original_repository_metadata = repository.metadata + else: + original_repository_metadata = None + readme_file_names = get_readme_file_names( repository.name ) + metadata_dict = { 'shed_config_filename': shed_config_dict.get( 'config_filename' ) } + invalid_file_tups = [] + invalid_tool_configs = [] + tool_dependencies_config = None + original_tool_data_path = app.config.tool_data_path + original_tool_data_table_config_path = app.config.tool_data_table_config_path + if resetting_all_metadata_on_repository: + if not relative_install_dir: + raise Exception( "The value of repository.repo_path( app ) must be sent when resetting all metadata on a repository." ) + # Keep track of the location where the repository is temporarily cloned so that we can strip the path when setting metadata. The value of + # repository_files_dir is the full path to the temporary directory to which the repository was cloned. + work_dir = repository_files_dir + files_dir = repository_files_dir + # Since we're working from a temporary directory, we can safely copy sample files included in the repository to the repository root. + app.config.tool_data_path = repository_files_dir + app.config.tool_data_table_config_path = repository_files_dir + else: + # Use a temporary working directory to copy all sample files. + work_dir = tempfile.mkdtemp() + # All other files are on disk in the repository's repo_path, which is the value of relative_install_dir. + files_dir = relative_install_dir + if shed_config_dict.get( 'tool_path' ): + files_dir = os.path.join( shed_config_dict['tool_path'], files_dir ) + app.config.tool_data_path = work_dir + app.config.tool_data_table_config_path = work_dir + # Handle proprietary datatypes, if any. + datatypes_config = get_config_from_disk( 'datatypes_conf.xml', files_dir ) + if datatypes_config: + metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict ) + # Get the relative path to all sample files included in the repository for storage in the repository's metadata. + sample_file_metadata_paths, sample_file_copy_paths = get_sample_files_from_disk( repository_files_dir=files_dir, + tool_path=shed_config_dict.get( 'tool_path' ), + relative_install_dir=relative_install_dir, + resetting_all_metadata_on_repository=resetting_all_metadata_on_repository ) + if sample_file_metadata_paths: + metadata_dict[ 'sample_files' ] = sample_file_metadata_paths + # Copy all sample files included in the repository to a single directory location so we can load tools that depend on them. + for sample_file in sample_file_copy_paths: + copy_sample_file( app, sample_file, dest_path=work_dir ) + # If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory. + relative_path, filename = os.path.split( sample_file ) + if filename == 'tool_data_table_conf.xml.sample': + new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file, + tool_data_path=original_tool_data_path, + shed_tool_data_table_config=app.config.shed_tool_data_table_config, + persist=persist ) + for root, dirs, files in os.walk( files_dir ): + if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: + if '.hg' in dirs: + dirs.remove( '.hg' ) + for name in files: + # See if we have a READ_ME file. + if name.lower() in readme_file_names: + if resetting_all_metadata_on_repository: + full_path_to_readme = os.path.join( root, name ) + stripped_path_to_readme = full_path_to_readme.replace( work_dir, '' ) + if stripped_path_to_readme.startswith( '/' ): + stripped_path_to_readme = stripped_path_to_readme[ 1: ] + relative_path_to_readme = os.path.join( relative_install_dir, stripped_path_to_readme ) + else: + relative_path_to_readme = os.path.join( root, name ) + if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_readme.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): + relative_path_to_readme = relative_path_to_readme[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] + metadata_dict[ 'readme' ] = relative_path_to_readme + # See if we have a tool config. + elif name not in NOT_TOOL_CONFIGS and name.endswith( '.xml' ): + full_path = str( os.path.abspath( os.path.join( root, name ) ) ) + if os.path.getsize( full_path ) > 0: + if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ] + or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ): + try: + # Make sure we're looking at a tool config and not a display application config or something else. + element_tree = util.parse_xml( full_path ) + element_tree_root = element_tree.getroot() + is_tool = element_tree_root.tag == 'tool' + except Exception, e: + log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) ) + is_tool = False + if is_tool: + tool, valid, error_message = load_tool_from_config( app, full_path ) + if tool is None: + if not valid: + invalid_file_tups.append( ( name, error_message ) ) + else: + invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_file_metadata_paths ) + can_set_metadata = True + for tup in invalid_files_and_errors_tups: + if name in tup: + can_set_metadata = False + invalid_tool_configs.append( name ) + break + if can_set_metadata: + if resetting_all_metadata_on_repository: + full_path_to_tool_config = os.path.join( root, name ) + stripped_path_to_tool_config = full_path_to_tool_config.replace( work_dir, '' ) + if stripped_path_to_tool_config.startswith( '/' ): + stripped_path_to_tool_config = stripped_path_to_tool_config[ 1: ] + relative_path_to_tool_config = os.path.join( relative_install_dir, stripped_path_to_tool_config ) + else: + relative_path_to_tool_config = os.path.join( root, name ) + if relative_install_dir and shed_config_dict.get( 'tool_path' ) and relative_path_to_tool_config.startswith( os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir ) ): + relative_path_to_tool_config = relative_path_to_tool_config[ len( shed_config_dict.get( 'tool_path' ) ) + 1: ] + metadata_dict = generate_tool_metadata( relative_path_to_tool_config, tool, repository_clone_url, metadata_dict ) + else: + for tup in invalid_files_and_errors_tups: + invalid_file_tups.append( tup ) + # Find all exported workflows. + elif name.endswith( '.ga' ): + relative_path = os.path.join( root, name ) + if os.path.getsize( os.path.abspath( relative_path ) ) > 0: + fp = open( relative_path, 'rb' ) + workflow_text = fp.read() + fp.close() + exported_workflow_dict = from_json_string( workflow_text ) + if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': + metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ) + if 'tools' in metadata_dict: + # This step must be done after metadata for tools has been defined. + tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', files_dir ) + if tool_dependencies_config: + metadata_dict = generate_tool_dependency_metadata( app, + repository, + tool_dependencies_config, + metadata_dict, + original_repository_metadata=original_repository_metadata ) + if invalid_tool_configs: + metadata_dict [ 'invalid_tools' ] = invalid_tool_configs + # Reset the value of the app's tool_data_path and tool_data_table_config_path to their respective original values. + app.config.tool_data_path = original_tool_data_path + app.config.tool_data_table_config_path = original_tool_data_table_config_path + return metadata_dict, invalid_file_tups +def get_changectx_for_changeset( repo, changeset_revision, **kwd ): + """Retrieve a specified changectx from a repository""" + for changeset in repo.changelog: + ctx = repo.changectx( changeset ) + if str( ctx ) == changeset_revision: + return ctx + return None +def get_config_from_disk( config_file, relative_install_dir ): + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == config_file: + return os.path.abspath( os.path.join( root, name ) ) + return None +def get_configured_ui(): + # Configure any desired ui settings. + _ui = ui.ui() + # The following will suppress all messages. This is + # the same as adding the following setting to the repo + # hgrc file' [ui] section: + # quiet = True + _ui.setconfig( 'ui', 'quiet', True ) + return _ui +def get_file_context_from_ctx( ctx, filename ): + # We have to be careful in determining if we found the correct file because multiple files with the same name may be in different directories + # within ctx if the files were moved within the change set. For example, in the following ctx.files() list, the former may have been moved to + # the latter: ['tmap_wrapper_0.0.19/tool_data_table_conf.xml.sample', 'tmap_wrapper_0.3.3/tool_data_table_conf.xml.sample']. Another scenario + # is that the file has been deleted. + deleted = False + filename = strip_path( filename ) + for ctx_file in ctx.files(): + ctx_file_name = strip_path( ctx_file ) + if filename == ctx_file_name: + try: + # If the file was moved, its destination will be returned here. + fctx = ctx[ ctx_file ] + return fctx + except LookupError, e: + # Set deleted for now, and continue looking in case the file was moved instead of deleted. + deleted = True + if deleted: + return 'DELETED' + return None +def get_repository_file_contents( file_path ): + if is_gzip( file_path ): + to_html = to_html_str( '\ngzip compressed file\n' ) + elif is_bz2( file_path ): + to_html = to_html_str( '\nbz2 compressed file\n' ) + elif check_zip( file_path ): + to_html = to_html_str( '\nzip compressed file\n' ) + elif check_binary( file_path ): + to_html = to_html_str( '\nBinary file\n' ) + else: + to_html = '' + for i, line in enumerate( open( file_path ) ): + to_html = '%s%s' % ( to_html, to_html_str( line ) ) + if len( to_html ) > MAX_CONTENT_SIZE: + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) + to_html = '%s%s' % ( to_html, to_html_str( large_str ) ) + break + return to_html +def get_repository_in_tool_shed( trans, id ): + """Get a repository on the tool shed side from the database via id""" + return trans.sa_session.query( trans.model.Repository ).get( trans.security.decode_id( id ) ) +def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ): + """Get metadata for a specified repository change set from the database""" + # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. Duplicate records were somehow + # created in the past. The cause of this issue has been resolved, but we'll leave this method as is for a while longer to ensure all duplicate + # records are removed. + all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \ + .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ), + trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ + .order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \ + .all() + if len( all_metadata_records ) > 1: + # Delete all recrds older than the last one updated. + for repository_metadata in all_metadata_records[ 1: ]: + trans.sa_session.delete( repository_metadata ) + trans.sa_session.flush() + return all_metadata_records[ 0 ] + elif all_metadata_records: + return all_metadata_records[ 0 ] + return None +def get_named_tmpfile_from_ctx( ctx, filename, dir ): + filename = strip_path( filename ) + for ctx_file in ctx.files(): + ctx_file_name = strip_path( ctx_file ) + if filename == ctx_file_name: + try: + # If the file was moved, its destination file contents will be returned here. + fctx = ctx[ ctx_file ] + except LookupError, e: + # Continue looking in case the file was moved. + fctx = None + continue + if fctx: + fh = tempfile.NamedTemporaryFile( 'wb', dir=dir ) + tmp_filename = fh.name + fh.close() + fh = open( tmp_filename, 'wb' ) + fh.write( fctx.data() ) + fh.close() + return tmp_filename + return None +def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ): + parent_id = None + # Compare from most recent to oldest. + changeset_revisions.reverse() + for changeset_revision in changeset_revisions: + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + metadata = repository_metadata.metadata + tools_dicts = metadata.get( 'tools', [] ) + for tool_dict in tools_dicts: + if tool_dict[ 'guid' ] == guid: + # The tool has not changed between the compared changeset revisions. + continue + if tool_dict[ 'id' ] == old_id and tool_dict[ 'version' ] != version: + # The tool version is different, so we've found the parent. + return tool_dict[ 'guid' ] + if parent_id is None: + # The tool did not change through all of the changeset revisions. + return old_id +def handle_sample_files_and_load_tool_from_disk( trans, repo_files_dir, tool_config_filepath, work_dir ): + # Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories. + message = '' + sample_files = copy_disk_sample_files_to_dir( trans, repo_files_dir, work_dir ) + if sample_files: + if 'tool_data_table_conf.xml.sample' in sample_files: + # Load entries into the tool_data_tables if the tool requires them. + tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) + error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) + tool, valid, message2 = load_tool_from_config( trans.app, tool_config_filepath ) + message = concat_messages( message, message2 ) + return tool, valid, message, sample_files +def handle_sample_files_and_load_tool_from_tmp_config( trans, repo, changeset_revision, tool_config_filename, work_dir ): + tool = None + message = '' + ctx = get_changectx_for_changeset( repo, changeset_revision ) + # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in + # the manifest, but have been deleted from disk. + sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) + if sample_files: + trans.app.config.tool_data_path = work_dir + if 'tool_data_table_conf.xml.sample' in sample_files: + # Load entries into the tool_data_tables if the tool requires them. + tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' ) + if tool_data_table_config: + error, message = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config ) + if error: + log.debug( message ) + manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision ) + if manifest_ctx and ctx_file: + tool, message2 = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir ) + message = concat_messages( message, message2 ) + return tool, message, sample_files +def handle_sample_tool_data_table_conf_file( app, filename, persist=False ): + """ + Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur + if call is from the Galaxy side, not the tool shed), the new entries will be appended to Galaxy's shed_tool_data_table_conf.xml file on disk. + """ + error = False + message = '' + try: + new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=filename, + tool_data_path=app.config.tool_data_path, + shed_tool_data_table_config=app.config.shed_tool_data_table_config, + persist=persist ) + except Exception, e: + message = str( e ) + error = True + return error, message +def is_downloadable( metadata_dict ): + return 'datatypes' in metadata_dict or 'tools' in metadata_dict or 'workflows' in metadata_dict +def load_tool_from_config( app, full_path ): + try: + tool = app.toolbox.load_tool( full_path ) + valid = True + error_message = None + except KeyError, e: + tool = None + valid = False + error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e ) + error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct ' + error_message += 'this error. ' + except Exception, e: + tool = None + valid = False + error_message = str( e ) + return tool, valid, error_message +def open_repository_files_folder( trans, folder_path ): + try: + files_list = get_repository_files( trans, folder_path ) + except OSError, e: + if str( e ).find( 'No such file or directory' ) >= 0: + # We have a repository with no contents. + return [] + folder_contents = [] + for filename in files_list: + is_folder = False + if filename and filename[-1] == os.sep: + is_folder = True + if filename: + full_path = os.path.join( folder_path, filename ) + node = { "title": filename, + "isFolder": is_folder, + "isLazy": is_folder, + "tooltip": full_path, + "key": full_path } + folder_contents.append( node ) + return folder_contents +def remove_dir( dir ): + if os.path.exists( dir ): + try: + shutil.rmtree( dir ) + except: + pass +def reset_all_metadata_on_repository_in_tool_shed( trans, id ): + """Reset all metadata on a single repository in a tool shed.""" + def reset_all_tool_versions( trans, id, repo ): + changeset_revisions = [] + for changeset in repo.changelog: + changeset_revision = str( repo.changectx( changeset ) ) + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + if repository_metadata: + metadata = repository_metadata.metadata + if metadata: + if metadata.get( 'tools', None ): + changeset_revisions.append( changeset_revision ) + # The list of changeset_revisions is now filtered to contain only those that are downloadable and contain tools. + # If a repository includes tools, build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. + for index, changeset_revision in enumerate( changeset_revisions ): + tool_versions_dict = {} + repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) + metadata = repository_metadata.metadata + tool_dicts = metadata[ 'tools' ] + if index == 0: + # The first changset_revision is a special case because it will have no ancestor changeset_revisions in which to match tools. + # The parent tool id for tools in the first changeset_revision will be the "old_id" in the tool config. + for tool_dict in tool_dicts: + tool_versions_dict[ tool_dict[ 'guid' ] ] = tool_dict[ 'id' ] + else: + for tool_dict in tool_dicts: + parent_id = get_parent_id( trans, + id, + tool_dict[ 'id' ], + tool_dict[ 'version' ], + tool_dict[ 'guid' ], + changeset_revisions[ 0:index ] ) + tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id + if tool_versions_dict: + repository_metadata.tool_versions = tool_versions_dict + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() + repository = get_repository_in_tool_shed( trans, id ) + log.debug( "Resetting all metadata on repository: %s" % repository.name ) + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( get_configured_ui(), repo_dir ) + repository_clone_url = generate_clone_url_for_repository_in_tool_shed( trans, repository ) + # The list of changeset_revisions refers to repository_metadata records that have been created or updated. When the following loop + # completes, we'll delete all repository_metadata records for this repository that do not have a changeset_revision value in this list. + changeset_revisions = [] + # When a new repository_metadata record is created, it always uses the values of metadata_changeset_revision and metadata_dict. + metadata_changeset_revision = None + metadata_dict = None + ancestor_changeset_revision = None + ancestor_metadata_dict = None + invalid_file_tups = [] + home_dir = os.getcwd() + for changeset in repo.changelog: + work_dir = tempfile.mkdtemp() + current_changeset_revision = str( repo.changectx( changeset ) ) + ctx = repo.changectx( changeset ) + log.debug( "Cloning repository revision: %s", str( ctx.rev() ) ) + cloned_ok, error_message = clone_repository( repository_clone_url, work_dir, str( ctx.rev() ) ) + if cloned_ok: + log.debug( "Generating metadata for changset revision: %s", str( ctx.rev() ) ) + current_metadata_dict, invalid_file_tups = generate_metadata_for_changeset_revision( app=trans.app, + repository=repository, + repository_clone_url=repository_clone_url, + relative_install_dir=repo_dir, + repository_files_dir=work_dir, + resetting_all_metadata_on_repository=True, + updating_installed_repository=False, + persist=False ) + if current_metadata_dict: + if not metadata_changeset_revision and not metadata_dict: + # We're at the first change set in the change log. + metadata_changeset_revision = current_changeset_revision + metadata_dict = current_metadata_dict + if ancestor_changeset_revision: + # Compare metadata from ancestor and current. The value of comparison will be one of: + # 'no metadata' - no metadata for either ancestor or current, so continue from current + # 'equal' - ancestor metadata is equivalent to current metadata, so continue from current + # 'subset' - ancestor metadata is a subset of current metadata, so continue from current + # 'not equal and not subset' - ancestor metadata is neither equal to nor a subset of current metadata, so persist ancestor metadata. + comparison = compare_changeset_revisions( ancestor_changeset_revision, + ancestor_metadata_dict, + current_changeset_revision, + current_metadata_dict ) + if comparison in [ 'no metadata', 'equal', 'subset' ]: + ancestor_changeset_revision = current_changeset_revision + ancestor_metadata_dict = current_metadata_dict + elif comparison == 'not equal and not subset': + metadata_changeset_revision = ancestor_changeset_revision + metadata_dict = ancestor_metadata_dict + repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + changeset_revisions.append( metadata_changeset_revision ) + ancestor_changeset_revision = current_changeset_revision + ancestor_metadata_dict = current_metadata_dict + else: + # We're at the beginning of the change log. + ancestor_changeset_revision = current_changeset_revision + ancestor_metadata_dict = current_metadata_dict + if not ctx.children(): + metadata_changeset_revision = current_changeset_revision + metadata_dict = current_metadata_dict + # We're at the end of the change log. + repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + changeset_revisions.append( metadata_changeset_revision ) + ancestor_changeset_revision = None + ancestor_metadata_dict = None + elif ancestor_metadata_dict: + # We reach here only if current_metadata_dict is empty and ancestor_metadata_dict is not. + if not ctx.children(): + # We're at the end of the change log. + repository_metadata = create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict ) + changeset_revisions.append( metadata_changeset_revision ) + ancestor_changeset_revision = None + ancestor_metadata_dict = None + remove_dir( work_dir ) + # Delete all repository_metadata records for this repository that do not have a changeset_revision value in changeset_revisions. + clean_repository_metadata( trans, id, changeset_revisions ) + # Set tool version information for all downloadable changeset revisions. Get the list of changeset revisions from the changelog. + reset_all_tool_versions( trans, id, repo ) + # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. + reset_tool_data_tables( trans.app ) + return invalid_file_tups, metadata_dict +def reset_metadata_on_selected_repositories( trans, **kwd ): + # This method is called from both Galaxy and the Tool Shed, so the cntrller param is required. + repository_ids = util.listify( kwd.get( 'repository_ids', None ) ) + CONTROLLER = kwd[ 'CONTROLLER' ] + message = '' + status = 'done' + if repository_ids: + successful_count = 0 + unsuccessful_count = 0 + for repository_id in repository_ids: + try: + if CONTROLLER == 'TOOL_SHED_ADMIN_CONTROLLER': + repository = get_repository_in_tool_shed( trans, repository_id ) + invalid_file_tups, metadata_dict = reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) + elif CONTROLLER == 'GALAXY_ADMIN_TOOL_SHED_CONTROLLER': + repository = get_installed_tool_shed_repository( trans, repository_id ) + invalid_file_tups, metadata_dict = reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + log.debug( message ) + unsuccessful_count += 1 + else: + log.debug( "Successfully reset metadata on repository %s" % repository.name ) + successful_count += 1 + except Exception, e: + log.debug( "Error attempting to reset metadata on repository '%s': %s" % ( repository.name, str( e ) ) ) + unsuccessful_count += 1 + message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) + if unsuccessful_count: + message += "Error setting metadata on %d %s - see the paster log for details. " % ( unsuccessful_count, + inflector.cond_plural( unsuccessful_count, "repository" ) ) + else: + message = 'Select at least one repository to on which to reset all metadata.' + status = 'error' + return message, status +def reset_tool_data_tables( app ): + # Reset the tool_data_tables to an empty dictionary. + app.tool_data_tables.data_tables = {} +def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): + """ + Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, but up to and + including the included_upper_bounds_changeset_revision. The value of excluded_lower_bounds_changeset_revision will be the value of + INITIAL_CHANGELOG_HASH if no valid changesets exist before included_upper_bounds_changeset_revision. + """ + # To set excluded_lower_bounds_changeset_revision, calling methods should do the following, where the value of changeset_revision + # is a downloadable changeset_revision. + # excluded_lower_bounds_changeset_revision = get_previous_downloadable_changset_revision( repository, repo, changeset_revision ) + if excluded_lower_bounds_changeset_revision == INITIAL_CHANGELOG_HASH: + appending_started = True + else: + appending_started = False + reversed_changelog = [] + for changeset in repo.changelog: + changeset_hash = str( repo.changectx( changeset ) ) + if appending_started: + reversed_changelog.insert( 0, changeset ) + if changeset_hash == excluded_lower_bounds_changeset_revision and not appending_started: + appending_started = True + if changeset_hash == included_upper_bounds_changeset_revision: + break + return reversed_changelog +def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revision ): + return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) +def strip_path( fpath ): + if not fpath: + return fpath + try: + file_path, file_name = os.path.split( fpath ) + except: + file_name = fpath + return file_name +def update_repository( repo, ctx_rev=None ): + """ + Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired + changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. + """ + # TODO: We may have files on disk in the repo directory that aren't being tracked, so they must be removed. + # The codes used to show the status of files are as follows. + # M = modified + # A = added + # R = removed + # C = clean + # ! = deleted, but still tracked + # ? = not tracked + # I = ignored + # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that + # purging is not supported by the mercurial API. See the deprecated update_for_browsing() method in common.py. + commands.update( get_configured_ui(), + repo, + rev=ctx_rev ) +def url_join( *args ): + parts = [] + for arg in args: + parts.append( arg.strip( '/' ) ) + return '/'.join( parts ) diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/webapps/community/controllers/admin.py --- a/lib/galaxy/webapps/community/controllers/admin.py +++ b/lib/galaxy/webapps/community/controllers/admin.py @@ -5,9 +5,7 @@ from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.web.form_builder import SelectField from galaxy.util import inflector -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import build_repository_ids_select_field, get_changectx_for_changeset, get_configured_ui, get_repository_in_tool_shed -from galaxy.util.shed_util import reset_metadata_on_selected_repositories, TOOL_SHED_ADMIN_CONTROLLER +from galaxy.util.shed_util_common import * from common import * from repository import RepositoryGrid, CategoryGrid diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -5,13 +5,7 @@ from galaxy.tools import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.hash_util import * -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import check_tool_input_params, clone_repository, concat_messages, copy_sample_file, create_or_update_repository_metadata -from galaxy.util.shed_util import generate_clone_url_for_repository_in_tool_shed, generate_message_for_invalid_tools, generate_metadata_for_changeset_revision -from galaxy.util.shed_util import get_changectx_for_changeset, get_config_from_disk, get_configured_ui, get_file_context_from_ctx, get_named_tmpfile_from_ctx -from galaxy.util.shed_util import get_parent_id, get_repository_in_tool_shed, get_repository_metadata_by_changeset_revision -from galaxy.util.shed_util import handle_sample_files_and_load_tool_from_disk, handle_sample_files_and_load_tool_from_tmp_config, INITIAL_CHANGELOG_HASH -from galaxy.util.shed_util import is_downloadable, load_tool_from_config, remove_dir, reset_tool_data_tables, reversed_upper_bounded_changelog, strip_path +from galaxy.util.shed_util_common import * from galaxy.web.base.controller import * from galaxy.web.base.controllers.admin import * from galaxy.webapps.community import model diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -9,14 +9,7 @@ from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import create_repo_info_dict, generate_clone_url_for_repository_in_tool_shed, generate_message_for_invalid_tools -from galaxy.util.shed_util import get_changectx_for_changeset, get_configured_ui, get_file_from_changeset_revision -from galaxy.util.shed_util import get_repository_file_contents, get_repository_in_tool_shed, get_repository_metadata_by_changeset_revision -from galaxy.util.shed_util import handle_sample_files_and_load_tool_from_disk, handle_sample_files_and_load_tool_from_tmp_config -from galaxy.util.shed_util import INITIAL_CHANGELOG_HASH, load_tool_from_config, NOT_TOOL_CONFIGS, open_repository_files_folder, remove_dir -from galaxy.util.shed_util import reset_all_metadata_on_repository_in_tool_shed, reversed_lower_upper_bounded_changelog -from galaxy.util.shed_util import reversed_upper_bounded_changelog, strip_path, to_html_escaped, update_repository, url_join +from galaxy.util.shed_util_common import * from galaxy.tool_shed.encoding_util import * from common import * @@ -1265,6 +1258,13 @@ trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' return get_repository_file_contents( file_path ) + def get_file_from_changeset_revision( self, repo_files_dir, changeset_revision, file_name, dir ): + """Return file_name from the received changeset_revision of the repository manifest.""" + stripped_file_name = strip_path( file_name ) + repo = hg.repository( get_configured_ui(), repo_files_dir ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) + named_tmp_file = get_named_tmpfile_from_ctx( ctx, file_name, dir ) + return named_tmp_file def get_metadata( self, trans, repository_id, changeset_revision ): repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) if repository_metadata and repository_metadata.metadata: @@ -2231,6 +2231,17 @@ if list: return ','.join( list ) return '' + def to_html_escaped( self, text ): + """Translates the characters in text to html values""" + translated = [] + for c in text: + if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS: + translated.append( c ) + elif c in MAPPED_CHARS: + translated.append( MAPPED_CHARS[ c ] ) + else: + translated.append( '' ) + return ''.join( translated ) def __validate_repository_name( self, name, user ): # Repository names must be unique for each user, must be at least four characters # in length and must contain only lower-case letters, numbers, and the '_' character. @@ -2304,7 +2315,7 @@ anchors = modified + added + removed + deleted + unknown + ignored + clean diffs = [] for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node() ): - diffs.append( to_html_escaped( diff ) ) + diffs.append( self.to_html_escaped( diff ) ) is_malicious = changeset_is_malicious( trans, id, repository.tip( trans.app ) ) metadata = self.get_metadata( trans, id, ctx_str ) return trans.fill_template( '/webapps/community/repository/view_changeset.mako', @@ -2356,12 +2367,7 @@ except IOError: work_dir = tempfile.mkdtemp() try: - manifest_readme_file = get_file_from_changeset_revision( trans.app, - repository, - repo_files_dir, - changeset_revision, - readme_file, - work_dir ) + manifest_readme_file = self.get_file_from_changeset_revision( repo_files_dir, changeset_revision, readme_file, work_dir ) f = open( manifest_readme_file, 'r' ) raw_text = f.read() f.close() diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/webapps/community/controllers/repository_review.py --- a/lib/galaxy/webapps/community/controllers/repository_review.py +++ b/lib/galaxy/webapps/community/controllers/repository_review.py @@ -8,8 +8,7 @@ from sqlalchemy.sql.expression import func from common import * from repository import RepositoryGrid -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import get_configured_ui, get_repository_in_tool_shed +from galaxy.util.shed_util_common import * from galaxy.util.odict import odict from galaxy import eggs diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/webapps/community/controllers/upload.py --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -3,9 +3,7 @@ from galaxy.model.orm import * from galaxy.datatypes.checkers import * from common import * -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import get_configured_ui, get_repository_in_tool_shed, reset_tool_data_tables, handle_sample_tool_data_table_conf_file -from galaxy.util.shed_util import update_repository +from galaxy.util.shed_util_common import * from galaxy import eggs eggs.require('mercurial') diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/webapps/community/controllers/workflow.py --- a/lib/galaxy/webapps/community/controllers/workflow.py +++ b/lib/galaxy/webapps/community/controllers/workflow.py @@ -10,8 +10,7 @@ from galaxy.webapps.galaxy.controllers.workflow import attach_ordered_steps from galaxy.model.orm import * from common import * -# TODO: re-factor shed_util to eliminate the following restricted imports -from galaxy.util.shed_util import get_repository_in_tool_shed +from galaxy.util.shed_util_common import * from galaxy.tool_shed.encoding_util import * class RepoInputDataModule( InputDataModule ): diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -2,6 +2,7 @@ from admin import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * +from galaxy.util.shed_util_common import * from galaxy.tool_shed.encoding_util import * from galaxy import eggs, tools diff -r ad6c2f4b3433eab6ba577aee8fcd88266121798f -r 7a51b701af8825baaf4aeb68f422304434c01a10 templates/webapps/community/repository/common.mako --- a/templates/webapps/community/repository/common.mako +++ b/templates/webapps/community/repository/common.mako @@ -77,7 +77,7 @@ <%def name="render_clone_str( repository )"><% - from galaxy.util.shed_util import generate_clone_url_for_repository_in_tool_shed + from galaxy.util.shed_util_common import generate_clone_url_for_repository_in_tool_shed clone_str = generate_clone_url_for_repository_in_tool_shed( trans, repository ) %> hg clone <a href="${clone_str}">${clone_str}</a> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Extend tool panel to handle new parameters that have been added since job was run. Fix whitespace as well. Thanks to Jim Johnson for the inspiration.
by Bitbucket 15 Nov '12

15 Nov '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/ad6c2f4b3433/ changeset: ad6c2f4b3433 user: jgoecks date: 2012-11-15 20:37:20 summary: Extend tool panel to handle new parameters that have been added since job was run. Fix whitespace as well. Thanks to Jim Johnson for the inspiration. affected #: 1 file diff -r 4a581a24f07d20e17ce5fbe4c5ddb6715abbe335 -r ad6c2f4b3433eab6ba577aee8fcd88266121798f templates/show_params.mako --- a/templates/show_params.mako +++ b/templates/show_params.mako @@ -11,26 +11,42 @@ </style><%def name="inputs_recursive( input_params, param_values, depth=1 )"> - %for input_index, input in enumerate( input_params.itervalues() ): - %if input.type == "repeat": - %for i in range( len(param_values[input.name]) ): - ${ inputs_recursive(input.inputs, param_values[input.name][i], depth=depth+1) } - %endfor - %elif input.type == "conditional": - <% current_case = param_values[input.name]['__current_case__'] %> - <tr> - ${ inputs_recursive_indent( text=input.test_param.label,depth=depth )} -  - <td>${input.cases[current_case].value}</td> - </tr> - ${ inputs_recursive(input.cases[current_case].inputs, param_values[input.name], depth=depth+1) } - %elif getattr(input, "label", None): - <tr> - ${inputs_recursive_indent( text=input.label,depth=depth )} - <td>${input.value_to_display_text(param_values[input.name], trans.app)}</td> - </tr> - %endif - %endfor + %for input_index, input in enumerate( input_params.itervalues() ): + %if input.name in param_values: + %if input.type == "repeat": + %for i in range( len(param_values[input.name]) ): + ${ inputs_recursive(input.inputs, param_values[input.name][i], depth=depth+1) } + %endfor + %elif input.type == "conditional": + <% current_case = param_values[input.name]['__current_case__'] %> + <tr> + ${ inputs_recursive_indent( text=input.test_param.label, depth=depth )} +  + <td>${input.cases[current_case].value}</td> + </tr> + ${ inputs_recursive(input.cases[current_case].inputs, param_values[input.name], depth=depth+1) } + %elif getattr(input, "label", None): + <tr> + ${inputs_recursive_indent( text=input.label, depth=depth )} + <td>${input.value_to_display_text(param_values[input.name], trans.app)}</td> + </tr> + %endif + %else: + ## Parameter does not have a stored value. + <tr> + <% + # Get parameter label. + if input.type == "conditional": + label = input.test_param.label + else: + label = input.label + %> + ${inputs_recursive_indent( text=label, depth=depth )} + <td><em>not used (parameter was added after this job was run)</em></td> + </tr> + %endif + + %endfor </%def> ## function to add a indentation depending on the depth in a <tr> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: history panel: fix to packed template name
by Bitbucket 15 Nov '12

15 Nov '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/4a581a24f07d/ changeset: 4a581a24f07d user: carlfeberhard date: 2012-11-15 19:43:54 summary: history panel: fix to packed template name affected #: 2 files diff -r b0afeaf880fc9913abc1ddffab25b953f2359fea -r 4a581a24f07d20e17ce5fbe4c5ddb6715abbe335 static/scripts/packed/templates/compiled/template-hda-failedMetaData.js --- a/static/scripts/packed/templates/compiled/template-hda-failedMetaData.js +++ /dev/null @@ -1,1 +0,0 @@ -(function(){var b=Handlebars.template,a=Handlebars.templates=Handlebars.templates||{};a["template-hda-failedMetadata"]=b(function(g,m,f,l,k){f=f||g.helpers;var c,i,o=this,h="function",n=f.blockHelperMissing,j=this.escapeExpression;function e(t,s){var q="",r,p;q+="\n";p=f.local;if(p){r=p.call(t,{hash:{},inverse:o.noop,fn:o.program(2,d,s)})}else{r=t.local;r=typeof r===h?r():r}if(!f.local){r=n.call(t,r,{hash:{},inverse:o.noop,fn:o.program(2,d,s)})}if(r||r===0){q+=r}q+='\nYou may be able to <a href="';r=t.urls;r=r==null||r===false?r:r.edit;r=typeof r===h?r():r;q+=j(r)+'" target="galaxy_main">set it manually or retry auto-detection</a>.\n';return q}function d(q,p){return"An error occurred setting the metadata for this dataset."}i=f.warningmessagesmall;if(i){c=i.call(m,{hash:{},inverse:o.noop,fn:o.program(1,e,k)})}else{c=m.warningmessagesmall;c=typeof c===h?c():c}if(!f.warningmessagesmall){c=n.call(m,c,{hash:{},inverse:o.noop,fn:o.program(1,e,k)})}if(c||c===0){return c}else{return""}})})(); \ No newline at end of file diff -r b0afeaf880fc9913abc1ddffab25b953f2359fea -r 4a581a24f07d20e17ce5fbe4c5ddb6715abbe335 static/scripts/packed/templates/compiled/template-hda-failedMetadata.js --- /dev/null +++ b/static/scripts/packed/templates/compiled/template-hda-failedMetadata.js @@ -0,0 +1,1 @@ +(function(){var b=Handlebars.template,a=Handlebars.templates=Handlebars.templates||{};a["template-hda-failedMetadata"]=b(function(g,m,f,l,k){f=f||g.helpers;var c,i,o=this,h="function",n=f.blockHelperMissing,j=this.escapeExpression;function e(t,s){var q="",r,p;q+="\n";p=f.local;if(p){r=p.call(t,{hash:{},inverse:o.noop,fn:o.program(2,d,s)})}else{r=t.local;r=typeof r===h?r():r}if(!f.local){r=n.call(t,r,{hash:{},inverse:o.noop,fn:o.program(2,d,s)})}if(r||r===0){q+=r}q+='\nYou may be able to <a href="';r=t.urls;r=r==null||r===false?r:r.edit;r=typeof r===h?r():r;q+=j(r)+'" target="galaxy_main">set it manually or retry auto-detection</a>.\n';return q}function d(q,p){return"An error occurred setting the metadata for this dataset."}i=f.warningmessagesmall;if(i){c=i.call(m,{hash:{},inverse:o.noop,fn:o.program(1,e,k)})}else{c=m.warningmessagesmall;c=typeof c===h?c():c}if(!f.warningmessagesmall){c=n.call(m,c,{hash:{},inverse:o.noop,fn:o.program(1,e,k)})}if(c||c===0){return c}else{return""}})})(); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 15 Nov '12

15 Nov '12

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/dae89d6c6e82/ changeset: dae89d6c6e82 user: carlfeberhard date: 2012-11-15 19:11:21 summary: history panel: fix to template name affected #: 1 file diff -r 8e52e426487bf732d051e538ac4cd7d352aa5065 -r dae89d6c6e82a095cd07d0ffa124f76d659863bc static/scripts/templates/compiled/template-hda-failedMetadata.js --- /dev/null +++ b/static/scripts/templates/compiled/template-hda-failedMetadata.js @@ -0,0 +1,33 @@ +(function() { + var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; +templates['template-hda-failedMetadata'] = template(function (Handlebars,depth0,helpers,partials,data) { + helpers = helpers || Handlebars.helpers; + var stack1, foundHelper, self=this, functionType="function", blockHelperMissing=helpers.blockHelperMissing, escapeExpression=this.escapeExpression; + +function program1(depth0,data) { + + var buffer = "", stack1, foundHelper; + buffer += "\n"; + foundHelper = helpers.local; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(2, program2, data)}); } + else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(2, program2, data)}); } + if(stack1 || stack1 === 0) { buffer += stack1; } + buffer += "\nYou may be able to <a href=\""; + stack1 = depth0.urls; + stack1 = stack1 == null || stack1 === false ? stack1 : stack1.edit; + stack1 = typeof stack1 === functionType ? stack1() : stack1; + buffer += escapeExpression(stack1) + "\" target=\"galaxy_main\">set it manually or retry auto-detection</a>.\n"; + return buffer;} +function program2(depth0,data) { + + + return "An error occurred setting the metadata for this dataset.";} + + foundHelper = helpers.warningmessagesmall; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(1, program1, data)}); } + else { stack1 = depth0.warningmessagesmall; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + if (!helpers.warningmessagesmall) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(1, program1, data)}); } + if(stack1 || stack1 === 0) { return stack1; } + else { return ''; }}); +})(); \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/changeset/b0afeaf880fc/ changeset: b0afeaf880fc user: carlfeberhard date: 2012-11-15 19:11:56 summary: pack scripts affected #: 4 files diff -r dae89d6c6e82a095cd07d0ffa124f76d659863bc -r b0afeaf880fc9913abc1ddffab25b953f2359fea static/scripts/packed/mvc/dataset/hda-base.js --- a/static/scripts/packed/mvc/dataset/hda-base.js +++ b/static/scripts/packed/mvc/dataset/hda-base.js @@ -1,1 +1,1 @@ -var HDABaseView=BaseView.extend(LoggableMixin).extend({tagName:"div",className:"historyItemContainer",initialize:function(a){this.log(this+".initialize:",a);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];if(!a.urlTemplates){throw ("HDAView needs urlTemplates on initialize")}this.urls=this._renderUrls(a.urlTemplates,this.model.toJSON());this.expanded=a.expanded||false;this.model.bind("change",this.render,this)},_renderUrls:function(d,a){var b=this,c={};_.each(d,function(e,f){if(_.isObject(e)){c[f]=b._renderUrls(e,a)}else{if(f==="meta_download"){c[f]=b._renderMetaDownloadUrls(e,a)}else{c[f]=_.template(e,a)}}});return c},_renderMetaDownloadUrls:function(b,a){return _.map(a.meta_files,function(c){return{url:_.template(b,{id:a.id,file_type:c.file_type}),file_type:c.file_type}})},render:function(){var b=this,e=this.model.get("id"),c=this.model.get("state"),a=$("<div/>").attr("id","historyItem-"+e),d=(this.$el.children().size()===0);this.$el.attr("id","historyItemContainer-"+e);a.addClass("historyItemWrapper").addClass("historyItem").addClass("historyItem-"+c);a.append(this._render_warnings());a.append(this._render_titleBar());this.body=$(this._render_body());a.append(this.body);make_popup_menus(a);a.find(".tooltip").tooltip({placement:"bottom"});this.$el.fadeOut("fast",function(){b.$el.children().remove();b.$el.append(a).fadeIn("fast",function(){b.log(b+" rendered:",b.$el);var f="rendered";if(d){f+=":initial"}else{if(b.model.inReadyState()){f+=":ready"}}b.trigger(f)})});return this},_render_warnings:function(){return $(jQuery.trim(HDABaseView.templates.messages(this.model.toJSON())))},_render_titleBar:function(){var a=$('<div class="historyItemTitleBar" style="overflow: hidden"></div>');a.append(this._render_titleButtons());a.append('<span class="state-icon"></span>');a.append(this._render_titleLink());return a},_render_titleButtons:function(){var a=$('<div class="historyItemButtons"></div>');a.append(this._render_displayButton());return a},_render_displayButton:function(){if((!this.model.inReadyState())||(this.model.get("state")===HistoryDatasetAssociation.STATES.ERROR)||(this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){this.displayButton=null;return null}var a={icon_class:"display",target:"galaxy_main"};if(this.model.get("purged")){a.enabled=false;a.title=_l("Cannot display datasets removed from disk")}else{a.title=_l("Display data in browser");a.href=this.urls.display}this.displayButton=new IconButtonView({model:new IconButton(a)});return this.displayButton.render().$el},_render_titleLink:function(){return $(jQuery.trim(HDABaseView.templates.titleLink(_.extend(this.model.toJSON(),{urls:this.urls}))))},_render_hdaSummary:function(){var a=_.extend(this.model.toJSON(),{urls:this.urls});return HDABaseView.templates.hdaSummary(a)},_render_primaryActionButtons:function(c){var a=this,b=$("<div/>").attr("id","primary-actions-"+this.model.get("id"));_.each(c,function(d){b.append(d.call(a))});return b},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var a=HDABaseView.templates.downloadLinks(_.extend(this.model.toJSON(),{urls:this.urls}));return $(a)},_render_showParamsButton:function(){this.showParamsButton=new IconButtonView({model:new IconButton({title:_l("View details"),href:this.urls.show_params,target:"galaxy_main",icon_class:"information"})});return this.showParamsButton.render().$el},_render_displayApps:function(){if(!this.model.hasData()){return null}var a=$("<div/>").addClass("display-apps");if(!_.isEmpty(this.model.get("display_types"))){a.append(HDABaseView.templates.displayApps({displayApps:this.model.get("display_types")}))}if(!_.isEmpty(this.model.get("display_apps"))){a.append(HDABaseView.templates.displayApps({displayApps:this.model.get("display_apps")}))}return a},_render_peek:function(){if(!this.model.get("peek")){return null}return $("<div/>").append($("<pre/>").attr("id","peek"+this.model.get("id")).addClass("peek").append(this.model.get("peek")))},_render_body:function(){var a=$("<div/>").attr("id","info-"+this.model.get("id")).addClass("historyItemBody").attr("style","display: block");switch(this.model.get("state")){case HistoryDatasetAssociation.STATES.NOT_VIEWABLE:this._render_body_not_viewable(a);break;case HistoryDatasetAssociation.STATES.UPLOAD:this._render_body_uploading(a);break;case HistoryDatasetAssociation.STATES.QUEUED:this._render_body_queued(a);break;case HistoryDatasetAssociation.STATES.RUNNING:this._render_body_running(a);break;case HistoryDatasetAssociation.STATES.ERROR:this._render_body_error(a);break;case HistoryDatasetAssociation.STATES.DISCARDED:this._render_body_discarded(a);break;case HistoryDatasetAssociation.STATES.SETTING_METADATA:this._render_body_setting_metadata(a);break;case HistoryDatasetAssociation.STATES.EMPTY:this._render_body_empty(a);break;case HistoryDatasetAssociation.STATES.FAILED_METADATA:this._render_body_failed_metadata(a);break;case HistoryDatasetAssociation.STATES.OK:this._render_body_ok(a);break;default:a.append($('<div>Error: unknown dataset state "'+state+'".</div>'))}a.append('<div style="clear: both"></div>');if(this.expanded){a.show()}else{a.hide()}return a},_render_body_not_viewable:function(a){a.append($("<div>"+_l("You do not have permission to view dataset")+".</div>"))},_render_body_uploading:function(a){a.append($("<div>"+_l("Dataset is uploading")+"</div>"))},_render_body_queued:function(a){a.append($("<div>"+_l("Job is waiting to run")+".</div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_running:function(a){a.append("<div>"+_l("Job is currently running")+".</div>");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_error:function(a){if(!this.model.get("purged")){a.append($("<div>"+this.model.get("misc_blurb")+"</div>"))}a.append((_l("An error occurred running this job")+": <i>"+$.trim(this.model.get("misc_info"))+"</i>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers.concat([this._render_downloadButton])))},_render_body_discarded:function(a){a.append("<div>"+_l("The job creating this dataset was cancelled before completion")+".</div>");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_setting_metadata:function(a){a.append($("<div>"+_l("Metadata is being auto-detected")+".</div>"))},_render_body_empty:function(a){a.append($("<div>"+_l("No data")+": <i>"+this.model.get("misc_blurb")+"</i></div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_failed_metadata:function(a){a.append($(HDABaseView.templates.failedMetadata(this.model.toJSON())));this._render_body_ok(a)},_render_body_ok:function(a){a.append(this._render_hdaSummary());if(this.model.isDeletedOrPurged()){a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));return}a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));a.append('<div class="clear"/>');a.append(this._render_displayApps());a.append(this._render_peek())},events:{"click .historyItemTitle":"toggleBodyVisibility"},toggleBodyVisibility:function(c,a){var b=this,d=this.$el.find(".historyItemBody");a=(a===undefined)?(!d.is(":visible")):(a);if(a){d.slideDown("fast",function(){b.trigger("body-expanded",b.model.get("id"))})}else{d.slideUp("fast",function(){b.trigger("body-collapsed",b.model.get("id"))})}},toString:function(){var a=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+a+")"}});HDABaseView.templates={warningMsg:Handlebars.templates["template-warningmessagesmall"],messages:Handlebars.templates["template-hda-warning-messages"],titleLink:Handlebars.templates["template-hda-titleLink"],hdaSummary:Handlebars.templates["template-hda-hdaSummary"],downloadLinks:Handlebars.templates["template-hda-downloadLinks"],failedMetadata:Handlebars.templates["template-hda-failedMetaData"],displayApps:Handlebars.templates["template-hda-displayApps"]}; \ No newline at end of file +var HDABaseView=BaseView.extend(LoggableMixin).extend({tagName:"div",className:"historyItemContainer",initialize:function(a){this.log(this+".initialize:",a);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];if(!a.urlTemplates){throw ("HDAView needs urlTemplates on initialize")}this.urls=this._renderUrls(a.urlTemplates,this.model.toJSON());this.expanded=a.expanded||false;this.model.bind("change",this.render,this)},_renderUrls:function(d,a){var b=this,c={};_.each(d,function(e,f){if(_.isObject(e)){c[f]=b._renderUrls(e,a)}else{if(f==="meta_download"){c[f]=b._renderMetaDownloadUrls(e,a)}else{c[f]=_.template(e,a)}}});return c},_renderMetaDownloadUrls:function(b,a){return _.map(a.meta_files,function(c){return{url:_.template(b,{id:a.id,file_type:c.file_type}),file_type:c.file_type}})},render:function(){var b=this,e=this.model.get("id"),c=this.model.get("state"),a=$("<div/>").attr("id","historyItem-"+e),d=(this.$el.children().size()===0);this.$el.attr("id","historyItemContainer-"+e);a.addClass("historyItemWrapper").addClass("historyItem").addClass("historyItem-"+c);a.append(this._render_warnings());a.append(this._render_titleBar());this.body=$(this._render_body());a.append(this.body);make_popup_menus(a);a.find(".tooltip").tooltip({placement:"bottom"});this.$el.fadeOut("fast",function(){b.$el.children().remove();b.$el.append(a).fadeIn("fast",function(){b.log(b+" rendered:",b.$el);var f="rendered";if(d){f+=":initial"}else{if(b.model.inReadyState()){f+=":ready"}}b.trigger(f)})});return this},_render_warnings:function(){return $(jQuery.trim(HDABaseView.templates.messages(this.model.toJSON())))},_render_titleBar:function(){var a=$('<div class="historyItemTitleBar" style="overflow: hidden"></div>');a.append(this._render_titleButtons());a.append('<span class="state-icon"></span>');a.append(this._render_titleLink());return a},_render_titleButtons:function(){var a=$('<div class="historyItemButtons"></div>');a.append(this._render_displayButton());return a},_render_displayButton:function(){if((!this.model.inReadyState())||(this.model.get("state")===HistoryDatasetAssociation.STATES.ERROR)||(this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){this.displayButton=null;return null}var a={icon_class:"display",target:"galaxy_main"};if(this.model.get("purged")){a.enabled=false;a.title=_l("Cannot display datasets removed from disk")}else{a.title=_l("Display data in browser");a.href=this.urls.display}this.displayButton=new IconButtonView({model:new IconButton(a)});return this.displayButton.render().$el},_render_titleLink:function(){return $(jQuery.trim(HDABaseView.templates.titleLink(_.extend(this.model.toJSON(),{urls:this.urls}))))},_render_hdaSummary:function(){var a=_.extend(this.model.toJSON(),{urls:this.urls});return HDABaseView.templates.hdaSummary(a)},_render_primaryActionButtons:function(c){var a=this,b=$("<div/>").attr("id","primary-actions-"+this.model.get("id"));_.each(c,function(d){b.append(d.call(a))});return b},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var a=HDABaseView.templates.downloadLinks(_.extend(this.model.toJSON(),{urls:this.urls}));return $(a)},_render_showParamsButton:function(){this.showParamsButton=new IconButtonView({model:new IconButton({title:_l("View details"),href:this.urls.show_params,target:"galaxy_main",icon_class:"information"})});return this.showParamsButton.render().$el},_render_displayApps:function(){if(!this.model.hasData()){return null}var a=$("<div/>").addClass("display-apps");if(!_.isEmpty(this.model.get("display_types"))){a.append(HDABaseView.templates.displayApps({displayApps:this.model.get("display_types")}))}if(!_.isEmpty(this.model.get("display_apps"))){a.append(HDABaseView.templates.displayApps({displayApps:this.model.get("display_apps")}))}return a},_render_peek:function(){if(!this.model.get("peek")){return null}return $("<div/>").append($("<pre/>").attr("id","peek"+this.model.get("id")).addClass("peek").append(this.model.get("peek")))},_render_body:function(){var a=$("<div/>").attr("id","info-"+this.model.get("id")).addClass("historyItemBody").attr("style","display: block");switch(this.model.get("state")){case HistoryDatasetAssociation.STATES.NOT_VIEWABLE:this._render_body_not_viewable(a);break;case HistoryDatasetAssociation.STATES.UPLOAD:this._render_body_uploading(a);break;case HistoryDatasetAssociation.STATES.QUEUED:this._render_body_queued(a);break;case HistoryDatasetAssociation.STATES.RUNNING:this._render_body_running(a);break;case HistoryDatasetAssociation.STATES.ERROR:this._render_body_error(a);break;case HistoryDatasetAssociation.STATES.DISCARDED:this._render_body_discarded(a);break;case HistoryDatasetAssociation.STATES.SETTING_METADATA:this._render_body_setting_metadata(a);break;case HistoryDatasetAssociation.STATES.EMPTY:this._render_body_empty(a);break;case HistoryDatasetAssociation.STATES.FAILED_METADATA:this._render_body_failed_metadata(a);break;case HistoryDatasetAssociation.STATES.OK:this._render_body_ok(a);break;default:a.append($('<div>Error: unknown dataset state "'+state+'".</div>'))}a.append('<div style="clear: both"></div>');if(this.expanded){a.show()}else{a.hide()}return a},_render_body_not_viewable:function(a){a.append($("<div>"+_l("You do not have permission to view dataset")+".</div>"))},_render_body_uploading:function(a){a.append($("<div>"+_l("Dataset is uploading")+"</div>"))},_render_body_queued:function(a){a.append($("<div>"+_l("Job is waiting to run")+".</div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_running:function(a){a.append("<div>"+_l("Job is currently running")+".</div>");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_error:function(a){if(!this.model.get("purged")){a.append($("<div>"+this.model.get("misc_blurb")+"</div>"))}a.append((_l("An error occurred running this job")+": <i>"+$.trim(this.model.get("misc_info"))+"</i>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers.concat([this._render_downloadButton])))},_render_body_discarded:function(a){a.append("<div>"+_l("The job creating this dataset was cancelled before completion")+".</div>");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_setting_metadata:function(a){a.append($("<div>"+_l("Metadata is being auto-detected")+".</div>"))},_render_body_empty:function(a){a.append($("<div>"+_l("No data")+": <i>"+this.model.get("misc_blurb")+"</i></div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_failed_metadata:function(a){a.append($(HDABaseView.templates.failedMetadata(this.model.toJSON())));this._render_body_ok(a)},_render_body_ok:function(a){a.append(this._render_hdaSummary());if(this.model.isDeletedOrPurged()){a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));return}a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));a.append('<div class="clear"/>');a.append(this._render_displayApps());a.append(this._render_peek())},events:{"click .historyItemTitle":"toggleBodyVisibility"},toggleBodyVisibility:function(c,a){var b=this,d=this.$el.find(".historyItemBody");a=(a===undefined)?(!d.is(":visible")):(a);if(a){d.slideDown("fast",function(){b.trigger("body-expanded",b.model.get("id"))})}else{d.slideUp("fast",function(){b.trigger("body-collapsed",b.model.get("id"))})}},toString:function(){var a=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+a+")"}});HDABaseView.templates={warningMsg:Handlebars.templates["template-warningmessagesmall"],messages:Handlebars.templates["template-hda-warning-messages"],titleLink:Handlebars.templates["template-hda-titleLink"],hdaSummary:Handlebars.templates["template-hda-hdaSummary"],downloadLinks:Handlebars.templates["template-hda-downloadLinks"],failedMetadata:Handlebars.templates["template-hda-failedMetadata"],displayApps:Handlebars.templates["template-hda-displayApps"]}; \ No newline at end of file diff -r dae89d6c6e82a095cd07d0ffa124f76d659863bc -r b0afeaf880fc9913abc1ddffab25b953f2359fea static/scripts/packed/mvc/history/history-panel.js --- a/static/scripts/packed/mvc/history/history-panel.js +++ b/static/scripts/packed/mvc/history/history-panel.js @@ -1,1 +1,1 @@ -var HistoryPanel=BaseView.extend(LoggableMixin).extend({el:"body.historyPage",HDAView:HDAEditView,events:{"click #history-tag":"loadAndDisplayTags"},initialize:function(a){this.log(this+".initialize:",a);if(!a.urlTemplates){throw (this+" needs urlTemplates on initialize")}if(!a.urlTemplates.history){throw (this+" needs urlTemplates.history on initialize")}if(!a.urlTemplates.hda){throw (this+" needs urlTemplates.hda on initialize")}this.urlTemplates=a.urlTemplates.history;this.hdaUrlTemplates=a.urlTemplates.hda;this._setUpWebStorage(a.initiallyExpanded,a.show_deleted,a.show_hidden);this.model.bind("change:nice_size",this.updateHistoryDiskSize,this);this.model.hdas.bind("add",this.add,this);this.model.hdas.bind("reset",this.addAll,this);this.hdaViews={};this.urls={}},_setUpWebStorage:function(b,a,c){this.storage=new PersistantStorage("HistoryView."+this.model.get("id"),{expandedHdas:{},show_deleted:false,show_hidden:false});this.log("this.storage:",this.storage.get());if(b){this.storage.set("exandedHdas",b)}if((a===true)||(a===false)){this.storage.set("show_deleted",a)}if((c===true)||(c===false)){this.storage.set("show_hidden",c)}this.show_deleted=this.storage.get("show_deleted");this.show_hidden=this.storage.get("show_hidden");this.log("(init'd) this.storage:",this.storage.get())},add:function(a){},addAll:function(){this.render()},render:function(){var b=this,d=b.toString()+".set-up",c=$("<div/>"),a=this.model.toJSON(),e=(this.$el.children().size()===0);a.urls=this._renderUrls(a);c.append(HistoryPanel.templates.historyPanel(a));c.find(".tooltip").tooltip({placement:"bottom"});this._setUpActionButton(c.find("#history-action-popup"));if(!this.model.hdas.length||!this.renderItems(c.find("#"+this.model.get("id")+"-datasets"))){c.find("#emptyHistoryMessage").show()}$(b).queue(d,function(f){b.$el.fadeOut("fast",function(){f()})});$(b).queue(d,function(f){b.$el.html("");b.$el.append(c.children());b.$el.fadeIn("fast",function(){f()})});$(b).queue(d,function(f){this.log(b+" rendered:",b.$el);b._setUpBehaviours();if(e){b.trigger("rendered:initial")}else{b.trigger("rendered")}f()});$(b).dequeue(d);return this},_renderUrls:function(a){var b=this;b.urls={};_.each(this.urlTemplates,function(d,c){b.urls[c]=_.template(d,a)});return b.urls},_setUpActionButton:function(e){var c=this,d=(this.storage.get("show_deleted"))?("Hide deleted"):("Show deleted"),a=(this.storage.get("show_hidden"))?("Hide hidden"):("Show hidden"),b={};b[_l("refresh")]=function(){window.location.reload()};b[_l("collapse all")]=function(){c.hideAllHdaBodies()};b[_l(d)]=function(){c.toggleShowDeleted()};b[_l(a)]=function(){c.toggleShowHidden()};make_popupmenu(e,b)},renderItems:function(b){this.hdaViews={};var a=this,c=this.model.hdas.getVisible(this.storage.get("show_deleted"),this.storage.get("show_hidden"));_.each(c,function(f){var e=f.get("id"),d=a.storage.get("expandedHdas").get(e);a.hdaViews[e]=new a.HDAView({model:f,expanded:d,urlTemplates:a.hdaUrlTemplates});a._setUpHdaListeners(a.hdaViews[e]);b.prepend(a.hdaViews[e].render().$el)});return c.length},_setUpHdaListeners:function(b){var a=this;b.bind("body-expanded",function(c){a.storage.get("expandedHdas").set(c,true)});b.bind("body-collapsed",function(c){a.storage.get("expandedHdas").deleteKey(c)})},_setUpBehaviours:function(){if(!(this.model.get("user")&&this.model.get("user").email)){return}var a=this.$("#history-annotation-area");this.$("#history-annotate").click(function(){if(a.is(":hidden")){a.slideDown("fast")}else{a.slideUp("fast")}return false});async_save_text("history-name-container","history-name",this.urls.rename,"new_name",18);async_save_text("history-annotation-container","history-annotation",this.urls.annotate,"new_annotation",18,true,4)},updateHistoryDiskSize:function(){this.$el.find("#history-size").text(this.model.get("nice_size"))},showQuotaMessage:function(){var a=this.$el.find("#quota-message-container");if(a.is(":hidden")){a.slideDown("fast")}},hideQuotaMessage:function(){var a=this.$el.find("#quota-message-container");if(!a.is(":hidden")){a.slideUp("fast")}},toggleShowDeleted:function(){this.storage.set("show_deleted",!this.storage.get("show_deleted"));this.render()},toggleShowHidden:function(){this.storage.set("show_hidden",!this.storage.get("show_hidden"));this.render()},hideAllHdaBodies:function(){_.each(this.hdaViews,function(a){a.toggleBodyVisibility(null,false)});this.storage.set("expandedHdas",{})},loadAndDisplayTags:function(c){this.log(this+".loadAndDisplayTags",c);var d=this.$el.find("#history-tag-area"),b=d.find(".tag-elt");this.log("\t tagArea",d," tagElt",b);if(d.is(":hidden")){if(!jQuery.trim(b.html())){var a=this;$.ajax({url:a.urls.tag,error:function(){alert(_l("Tagging failed"))},success:function(e){b.html(e);b.find(".tooltip").tooltip();d.slideDown("fast")}})}else{d.slideDown("fast")}}else{d.slideUp("fast")}return false},toString:function(){var a=this.model.get("name")||"";return"HistoryPanel("+a+")"}});HistoryPanel.templates={historyPanel:Handlebars.templates["template-history-historyPanel"]}; \ No newline at end of file +var HistoryPanel=BaseView.extend(LoggableMixin).extend({el:"body.historyPage",HDAView:HDAEditView,events:{"click #history-refresh":function(){window.location.reload()},"click #history-tag":"loadAndDisplayTags"},initialize:function(a){this.log(this+".initialize:",a);if(!a.urlTemplates){throw (this+" needs urlTemplates on initialize")}if(!a.urlTemplates.history){throw (this+" needs urlTemplates.history on initialize")}if(!a.urlTemplates.hda){throw (this+" needs urlTemplates.hda on initialize")}this.urlTemplates=a.urlTemplates.history;this.hdaUrlTemplates=a.urlTemplates.hda;this._setUpWebStorage(a.initiallyExpanded,a.show_deleted,a.show_hidden);this.model.bind("change:nice_size",this.updateHistoryDiskSize,this);this.model.hdas.bind("add",this.add,this);this.model.hdas.bind("reset",this.addAll,this);this.hdaViews={};this.urls={}},_setUpWebStorage:function(b,a,c){this.storage=new PersistantStorage("HistoryView."+this.model.get("id"),{expandedHdas:{},show_deleted:false,show_hidden:false});this.log("this.storage:",this.storage.get());if(b){this.storage.set("exandedHdas",b)}if((a===true)||(a===false)){this.storage.set("show_deleted",a)}if((c===true)||(c===false)){this.storage.set("show_hidden",c)}this.show_deleted=this.storage.get("show_deleted");this.show_hidden=this.storage.get("show_hidden");this.log("(init'd) this.storage:",this.storage.get())},add:function(a){},addAll:function(){this.render()},render:function(){var b=this,d=b.toString()+".set-up",c=$("<div/>"),a=this.model.toJSON(),e=(this.$el.children().size()===0);a.urls=this._renderUrls(a);c.append(HistoryPanel.templates.historyPanel(a));c.find(".tooltip").tooltip({placement:"bottom"});this._setUpActionButton(c.find("#history-action-popup"));if(!this.model.hdas.length||!this.renderItems(c.find("#"+this.model.get("id")+"-datasets"))){c.find("#emptyHistoryMessage").show()}$(b).queue(d,function(f){b.$el.fadeOut("fast",function(){f()})});$(b).queue(d,function(f){b.$el.html("");b.$el.append(c.children());b.$el.fadeIn("fast",function(){f()})});$(b).queue(d,function(f){this.log(b+" rendered:",b.$el);b._setUpBehaviours();if(e){b.trigger("rendered:initial")}else{b.trigger("rendered")}f()});$(b).dequeue(d);return this},_renderUrls:function(a){var b=this;b.urls={};_.each(this.urlTemplates,function(d,c){b.urls[c]=_.template(d,a)});return b.urls},_setUpActionButton:function(e){var c=this,d=(this.storage.get("show_deleted"))?("Hide deleted"):("Show deleted"),a=(this.storage.get("show_hidden"))?("Hide hidden"):("Show hidden"),b={};b[_l("collapse all")]=function(){c.hideAllHdaBodies()};b[_l(d)]=function(){c.toggleShowDeleted()};b[_l(a)]=function(){c.toggleShowHidden()};make_popupmenu(e,b)},renderItems:function(b){this.hdaViews={};var a=this,c=this.model.hdas.getVisible(this.storage.get("show_deleted"),this.storage.get("show_hidden"));_.each(c,function(f){var e=f.get("id"),d=a.storage.get("expandedHdas").get(e);a.hdaViews[e]=new a.HDAView({model:f,expanded:d,urlTemplates:a.hdaUrlTemplates});a._setUpHdaListeners(a.hdaViews[e]);b.prepend(a.hdaViews[e].render().$el)});return c.length},_setUpHdaListeners:function(b){var a=this;b.bind("body-expanded",function(c){a.storage.get("expandedHdas").set(c,true)});b.bind("body-collapsed",function(c){a.storage.get("expandedHdas").deleteKey(c)})},_setUpBehaviours:function(){if(!(this.model.get("user")&&this.model.get("user").email)){return}var a=this.$("#history-annotation-area");this.$("#history-annotate").click(function(){if(a.is(":hidden")){a.slideDown("fast")}else{a.slideUp("fast")}return false});async_save_text("history-name-container","history-name",this.urls.rename,"new_name",18);async_save_text("history-annotation-container","history-annotation",this.urls.annotate,"new_annotation",18,true,4)},updateHistoryDiskSize:function(){this.$el.find("#history-size").text(this.model.get("nice_size"))},showQuotaMessage:function(){var a=this.$el.find("#quota-message-container");if(a.is(":hidden")){a.slideDown("fast")}},hideQuotaMessage:function(){var a=this.$el.find("#quota-message-container");if(!a.is(":hidden")){a.slideUp("fast")}},toggleShowDeleted:function(){this.storage.set("show_deleted",!this.storage.get("show_deleted"));this.render()},toggleShowHidden:function(){this.storage.set("show_hidden",!this.storage.get("show_hidden"));this.render()},hideAllHdaBodies:function(){_.each(this.hdaViews,function(a){a.toggleBodyVisibility(null,false)});this.storage.set("expandedHdas",{})},loadAndDisplayTags:function(c){this.log(this+".loadAndDisplayTags",c);var d=this.$el.find("#history-tag-area"),b=d.find(".tag-elt");this.log("\t tagArea",d," tagElt",b);if(d.is(":hidden")){if(!jQuery.trim(b.html())){var a=this;$.ajax({url:a.urls.tag,error:function(){alert(_l("Tagging failed"))},success:function(e){b.html(e);b.find(".tooltip").tooltip();d.slideDown("fast")}})}else{d.slideDown("fast")}}else{d.slideUp("fast")}return false},toString:function(){var a=this.model.get("name")||"";return"HistoryPanel("+a+")"}});HistoryPanel.templates={historyPanel:Handlebars.templates["template-history-historyPanel"]}; \ No newline at end of file diff -r dae89d6c6e82a095cd07d0ffa124f76d659863bc -r b0afeaf880fc9913abc1ddffab25b953f2359fea static/scripts/packed/templates/compiled/template-hda-failedMetaData.js --- a/static/scripts/packed/templates/compiled/template-hda-failedMetaData.js +++ b/static/scripts/packed/templates/compiled/template-hda-failedMetaData.js @@ -1,1 +1,1 @@ -(function(){var b=Handlebars.template,a=Handlebars.templates=Handlebars.templates||{};a["template-hda-failedMetaData"]=b(function(g,m,f,l,k){f=f||g.helpers;var c,i,o=this,h="function",n=f.blockHelperMissing,j=this.escapeExpression;function e(t,s){var q="",r,p;q+="\n";p=f.local;if(p){r=p.call(t,{hash:{},inverse:o.noop,fn:o.program(2,d,s)})}else{r=t.local;r=typeof r===h?r():r}if(!f.local){r=n.call(t,r,{hash:{},inverse:o.noop,fn:o.program(2,d,s)})}if(r||r===0){q+=r}q+='\nYou may be able to <a href="';r=t.urls;r=r==null||r===false?r:r.edit;r=typeof r===h?r():r;q+=j(r)+'" target="galaxy_main">set it manually or retry auto-detection</a>.\n';return q}function d(q,p){return"An error occurred setting the metadata for this dataset."}i=f.warningmessagesmall;if(i){c=i.call(m,{hash:{},inverse:o.noop,fn:o.program(1,e,k)})}else{c=m.warningmessagesmall;c=typeof c===h?c():c}if(!f.warningmessagesmall){c=n.call(m,c,{hash:{},inverse:o.noop,fn:o.program(1,e,k)})}if(c||c===0){return c}else{return""}})})(); \ No newline at end of file +(function(){var b=Handlebars.template,a=Handlebars.templates=Handlebars.templates||{};a["template-hda-failedMetadata"]=b(function(g,m,f,l,k){f=f||g.helpers;var c,i,o=this,h="function",n=f.blockHelperMissing,j=this.escapeExpression;function e(t,s){var q="",r,p;q+="\n";p=f.local;if(p){r=p.call(t,{hash:{},inverse:o.noop,fn:o.program(2,d,s)})}else{r=t.local;r=typeof r===h?r():r}if(!f.local){r=n.call(t,r,{hash:{},inverse:o.noop,fn:o.program(2,d,s)})}if(r||r===0){q+=r}q+='\nYou may be able to <a href="';r=t.urls;r=r==null||r===false?r:r.edit;r=typeof r===h?r():r;q+=j(r)+'" target="galaxy_main">set it manually or retry auto-detection</a>.\n';return q}function d(q,p){return"An error occurred setting the metadata for this dataset."}i=f.warningmessagesmall;if(i){c=i.call(m,{hash:{},inverse:o.noop,fn:o.program(1,e,k)})}else{c=m.warningmessagesmall;c=typeof c===h?c():c}if(!f.warningmessagesmall){c=n.call(m,c,{hash:{},inverse:o.noop,fn:o.program(1,e,k)})}if(c||c===0){return c}else{return""}})})(); \ No newline at end of file diff -r dae89d6c6e82a095cd07d0ffa124f76d659863bc -r b0afeaf880fc9913abc1ddffab25b953f2359fea static/scripts/packed/templates/compiled/template-history-historyPanel.js --- a/static/scripts/packed/templates/compiled/template-history-historyPanel.js +++ b/static/scripts/packed/templates/compiled/template-history-historyPanel.js @@ -1,1 +1,1 @@ -(function(){var b=Handlebars.template,a=Handlebars.templates=Handlebars.templates||{};a["template-history-historyPanel"]=b(function(k,A,y,q,I){y=y||k.helpers;var z="",n,m,v=this,e="function",c=y.blockHelperMissing,d=this.escapeExpression;function t(N,M){var K="",L,J;K+='\n <div id="history-name" class="tooltip editable-text"\n title="';J=y.local;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(2,s,M)})}else{L=N.local;L=typeof L===e?L():L}if(!y.local){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(2,s,M)})}if(L||L===0){K+=L}K+='">';J=y.name;if(J){L=J.call(N,{hash:{}})}else{L=N.name;L=typeof L===e?L():L}K+=d(L)+"</div>\n ";return K}function s(K,J){return"Click to rename history"}function r(N,M){var K="",L,J;K+='\n <div id="history-name" class="tooltip"\n title="';J=y.local;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(5,p,M)})}else{L=N.local;L=typeof L===e?L():L}if(!y.local){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(5,p,M)})}if(L||L===0){K+=L}K+='">';J=y.name;if(J){L=J.call(N,{hash:{}})}else{L=N.name;L=typeof L===e?L():L}K+=d(L)+"</div>\n ";return K}function p(K,J){return"You must be logged in to edit your history name"}function o(K,J){return"Click to see more actions for this history"}function j(N,M){var K="",L,J;K+='\n <div id="history-secondary-links" style="float: right;">\n <a id="history-tag" title="';J=y.local;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(10,H,M)})}else{L=N.local;L=typeof L===e?L():L}if(!y.local){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(10,H,M)})}if(L||L===0){K+=L}K+='"\n class="icon-button tags tooltip" target="galaxy_main" href="javascript:void(0)"></a>\n <a id="history-annotate" title="';J=y.local;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(12,G,M)})}else{L=N.local;L=typeof L===e?L():L}if(!y.local){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(12,G,M)})}if(L||L===0){K+=L}K+='"\n class="icon-button annotate tooltip" target="galaxy_main" href="javascript:void(0)"></a>\n </div>\n ';return K}function H(K,J){return"Edit history tags"}function G(K,J){return"Edit history annotation"}function F(N,M){var K="",L,J;K+="\n ";J=y.warningmessagesmall;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(15,E,M)})}else{L=N.warningmessagesmall;L=typeof L===e?L():L}if(!y.warningmessagesmall){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(15,E,M)})}if(L||L===0){K+=L}K+="\n ";return K}function E(M,L){var K,J;J=y.local;if(J){K=J.call(M,{hash:{},inverse:v.noop,fn:v.program(16,D,L)})}else{K=M.local;K=typeof K===e?K():K}if(!y.local){K=c.call(M,K,{hash:{},inverse:v.noop,fn:v.program(16,D,L)})}if(K||K===0){return K}else{return""}}function D(K,J){return"You are currently viewing a deleted history!"}function C(N,M){var K="",L,J;K+='\n <div id="history-tag-annotation">\n\n <div id="history-tag-area" style="display: none">\n <strong>';J=y.local;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(19,B,M)})}else{L=N.local;L=typeof L===e?L():L}if(!y.local){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(19,B,M)})}if(L||L===0){K+=L}K+=':</strong>\n <div class="tag-elt"></div>\n </div>\n\n <div id="history-annotation-area" style="display: none">\n <strong>';J=y.local;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(21,l,M)})}else{L=N.local;L=typeof L===e?L():L}if(!y.local){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(21,l,M)})}if(L||L===0){K+=L}K+=':</strong>\n <div id="history-annotation-container">\n <div id="history-annotation" class="tooltip editable-text"\n title="';J=y.local;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(23,i,M)})}else{L=N.local;L=typeof L===e?L():L}if(!y.local){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(23,i,M)})}if(L||L===0){K+=L}K+='">\n ';L=N.annotation;L=y["if"].call(N,L,{hash:{},inverse:v.program(27,g,M),fn:v.program(25,h,M)});if(L||L===0){K+=L}K+="\n </div>\n </div>\n </div>\n </div>\n ";return K}function B(K,J){return"Tags"}function l(K,J){return"Annotation"}function i(K,J){return"Click to edit annotation"}function h(N,M){var K="",L,J;K+="\n ";J=y.annotation;if(J){L=J.call(N,{hash:{}})}else{L=N.annotation;L=typeof L===e?L():L}K+=d(L)+"\n ";return K}function g(N,M){var K="",L,J;K+="\n <em>";J=y.local;if(J){L=J.call(N,{hash:{},inverse:v.noop,fn:v.program(28,f,M)})}else{L=N.local;L=typeof L===e?L():L}if(!y.local){L=c.call(N,L,{hash:{},inverse:v.noop,fn:v.program(28,f,M)})}if(L||L===0){K+=L}K+="</em>\n ";return K}function f(K,J){return"Describe or add notes to history"}function x(N,M){var K="",L,J;K+='\n <div id="message-container">\n <div class="';J=y.status;if(J){L=J.call(N,{hash:{}})}else{L=N.status;L=typeof L===e?L():L}K+=d(L)+'message">\n ';J=y.message;if(J){L=J.call(N,{hash:{}})}else{L=N.message;L=typeof L===e?L():L}K+=d(L)+"\n </div><br />\n </div>\n ";return K}function w(K,J){return"You are over your disk quota.\n Tool execution is on hold until your disk usage drops below your allocated quota."}function u(K,J){return"Your history is empty. Click 'Get Data' on the left pane to start"}z+='<div id="history-controls">\n <div id="history-title-area" class="historyLinks">\n\n ';z+='\n <div id="history-name-container" style="float: left;">\n ';z+="\n ";n=A.user;n=n==null||n===false?n:n.email;n=y["if"].call(A,n,{hash:{},inverse:v.program(4,r,I),fn:v.program(1,t,I)});if(n||n===0){z+=n}z+='\n </div>\n\n <a id="history-action-popup" class="tooltip" title="';m=y.local;if(m){n=m.call(A,{hash:{},inverse:v.noop,fn:v.program(7,o,I)})}else{n=A.local;n=typeof n===e?n():n}if(!y.local){n=c.call(A,n,{hash:{},inverse:v.noop,fn:v.program(7,o,I)})}if(n||n===0){z+=n}z+='"\n href="javascript:void(0);" style="float: right;">\n <span class="ficon cogs large"></span>\n </a>\n <div style="clear: both;"></div>\n </div>\n\n <div id="history-subtitle-area">\n <div id="history-size" style="float:left;">';m=y.nice_size;if(m){n=m.call(A,{hash:{}})}else{n=A.nice_size;n=typeof n===e?n():n}z+=d(n)+"</div>\n ";n=A.user;n=n==null||n===false?n:n.email;n=y["if"].call(A,n,{hash:{},inverse:v.noop,fn:v.program(9,j,I)});if(n||n===0){z+=n}z+='\n <div style="clear: both;"></div>\n </div>\n\n ';n=A.deleted;n=y["if"].call(A,n,{hash:{},inverse:v.noop,fn:v.program(14,F,I)});if(n||n===0){z+=n}z+="\n\n ";z+="\n ";z+="\n ";n=A.user;n=n==null||n===false?n:n.email;n=y["if"].call(A,n,{hash:{},inverse:v.noop,fn:v.program(18,C,I)});if(n||n===0){z+=n}z+="\n\n ";n=A.message;n=y["if"].call(A,n,{hash:{},inverse:v.noop,fn:v.program(30,x,I)});if(n||n===0){z+=n}z+='\n\n <div id="quota-message-container" style="display: none">\n <div id="quota-message" class="errormessage">\n ';m=y.local;if(m){n=m.call(A,{hash:{},inverse:v.noop,fn:v.program(32,w,I)})}else{n=A.local;n=typeof n===e?n():n}if(!y.local){n=c.call(A,n,{hash:{},inverse:v.noop,fn:v.program(32,w,I)})}if(n||n===0){z+=n}z+='\n </div>\n </div>\n</div>\n\n<div id="';m=y.id;if(m){n=m.call(A,{hash:{}})}else{n=A.id;n=typeof n===e?n():n}z+=d(n)+'-datasets" class="history-datasets-list"></div>\n\n<div class="infomessagesmall" id="emptyHistoryMessage" style="display: none;">\n ';m=y.local;if(m){n=m.call(A,{hash:{},inverse:v.noop,fn:v.program(34,u,I)})}else{n=A.local;n=typeof n===e?n():n}if(!y.local){n=c.call(A,n,{hash:{},inverse:v.noop,fn:v.program(34,u,I)})}if(n||n===0){z+=n}z+="\n</div>";return z})})(); \ No newline at end of file +(function(){var b=Handlebars.template,a=Handlebars.templates=Handlebars.templates||{};a["template-history-historyPanel"]=b(function(j,C,A,r,J){A=A||j.helpers;var B="",n,m,x=this,f="function",c=A.blockHelperMissing,e=this.escapeExpression;function u(O,N){var L="",M,K;L+='\n <div id="history-name" class="tooltip editable-text"\n title="';K=A.local;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(2,t,N)})}else{M=O.local;M=typeof M===f?M():M}if(!A.local){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(2,t,N)})}if(M||M===0){L+=M}L+='">';K=A.name;if(K){M=K.call(O,{hash:{}})}else{M=O.name;M=typeof M===f?M():M}L+=e(M)+"</div>\n ";return L}function t(L,K){return"Click to rename history"}function s(O,N){var L="",M,K;L+='\n <div id="history-name" class="tooltip"\n title="';K=A.local;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(5,q,N)})}else{M=O.local;M=typeof M===f?M():M}if(!A.local){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(5,q,N)})}if(M||M===0){L+=M}L+='">';K=A.name;if(K){M=K.call(O,{hash:{}})}else{M=O.name;M=typeof M===f?M():M}L+=e(M)+"</div>\n ";return L}function q(L,K){return"You must be logged in to edit your history name"}function p(O,N){var L="",M,K;L+='\n <a id="history-tag" title="';K=A.local;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(8,l,N)})}else{M=O.local;M=typeof M===f?M():M}if(!A.local){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(8,l,N)})}if(M||M===0){L+=M}L+='"\n class="icon-button tags tooltip" target="galaxy_main" href="javascript:void(0)"></a>\n <a id="history-annotate" title="';K=A.local;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(10,I,N)})}else{M=O.local;M=typeof M===f?M():M}if(!A.local){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(10,I,N)})}if(M||M===0){L+=M}L+='"\n class="icon-button annotate tooltip" target="galaxy_main" href="javascript:void(0)"></a>\n ';return L}function l(L,K){return"Edit history tags"}function I(L,K){return"Edit history annotation"}function H(L,K){return"Refresh this display"}function G(L,K){return"Click to see more actions for this history"}function F(O,N){var L="",M,K;L+="\n ";K=A.warningmessagesmall;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(17,E,N)})}else{M=O.warningmessagesmall;M=typeof M===f?M():M}if(!A.warningmessagesmall){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(17,E,N)})}if(M||M===0){L+=M}L+="\n ";return L}function E(N,M){var L,K;K=A.local;if(K){L=K.call(N,{hash:{},inverse:x.noop,fn:x.program(18,D,M)})}else{L=N.local;L=typeof L===f?L():L}if(!A.local){L=c.call(N,L,{hash:{},inverse:x.noop,fn:x.program(18,D,M)})}if(L||L===0){return L}else{return""}}function D(L,K){return"You are currently viewing a deleted history!"}function o(O,N){var L="",M,K;L+='\n <div id="history-tag-annotation">\n\n <div id="history-tag-area" style="display: none">\n <strong>';K=A.local;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(21,k,N)})}else{M=O.local;M=typeof M===f?M():M}if(!A.local){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(21,k,N)})}if(M||M===0){L+=M}L+=':</strong>\n <div class="tag-elt"></div>\n </div>\n\n <div id="history-annotation-area" style="display: none">\n <strong>';K=A.local;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(23,i,N)})}else{M=O.local;M=typeof M===f?M():M}if(!A.local){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(23,i,N)})}if(M||M===0){L+=M}L+=':</strong>\n <div id="history-annotation-container">\n <div id="history-annotation" class="tooltip editable-text"\n title="';K=A.local;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(25,h,N)})}else{M=O.local;M=typeof M===f?M():M}if(!A.local){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(25,h,N)})}if(M||M===0){L+=M}L+='">\n ';M=O.annotation;M=A["if"].call(O,M,{hash:{},inverse:x.program(29,d,N),fn:x.program(27,g,N)});if(M||M===0){L+=M}L+="\n </div>\n </div>\n </div>\n </div>\n ";return L}function k(L,K){return"Tags"}function i(L,K){return"Annotation"}function h(L,K){return"Click to edit annotation"}function g(O,N){var L="",M,K;L+="\n ";K=A.annotation;if(K){M=K.call(O,{hash:{}})}else{M=O.annotation;M=typeof M===f?M():M}L+=e(M)+"\n ";return L}function d(O,N){var L="",M,K;L+="\n <em>";K=A.local;if(K){M=K.call(O,{hash:{},inverse:x.noop,fn:x.program(30,z,N)})}else{M=O.local;M=typeof M===f?M():M}if(!A.local){M=c.call(O,M,{hash:{},inverse:x.noop,fn:x.program(30,z,N)})}if(M||M===0){L+=M}L+="</em>\n ";return L}function z(L,K){return"Describe or add notes to history"}function y(O,N){var L="",M,K;L+='\n <div id="message-container">\n <div class="';K=A.status;if(K){M=K.call(O,{hash:{}})}else{M=O.status;M=typeof M===f?M():M}L+=e(M)+'message">\n ';K=A.message;if(K){M=K.call(O,{hash:{}})}else{M=O.message;M=typeof M===f?M():M}L+=e(M)+"\n </div><br />\n </div>\n ";return L}function w(L,K){return"You are over your disk quota.\n Tool execution is on hold until your disk usage drops below your allocated quota."}function v(L,K){return"Your history is empty. Click 'Get Data' on the left pane to start"}B+='<div id="history-controls">\n <div id="history-title-area" class="historyLinks">\n\n ';B+='\n <div id="history-name-container">\n ';B+="\n ";n=C.user;n=n==null||n===false?n:n.email;n=A["if"].call(C,n,{hash:{},inverse:x.program(4,s,J),fn:x.program(1,u,J)});if(n||n===0){B+=n}B+='\n </div>\n </div>\n\n <div id="history-subtitle-area">\n <div id="history-size" style="float:left;">';m=A.nice_size;if(m){n=m.call(C,{hash:{}})}else{n=C.nice_size;n=typeof n===f?n():n}B+=e(n)+'</div>\n\n <div id="history-secondary-links" style="float: right;">\n ';n=C.user;n=n==null||n===false?n:n.email;n=A["if"].call(C,n,{hash:{},inverse:x.noop,fn:x.program(7,p,J)});if(n||n===0){B+=n}B+='\n <a id="history-refresh" class="tooltip" title="';m=A.local;if(m){n=m.call(C,{hash:{},inverse:x.noop,fn:x.program(12,H,J)})}else{n=C.local;n=typeof n===f?n():n}if(!A.local){n=c.call(C,n,{hash:{},inverse:x.noop,fn:x.program(12,H,J)})}if(n||n===0){B+=n}B+='"\n href="javascript:void(0);">\n <span class="ficon refresh large"></span>\n </a>\n <a id="history-action-popup" class="tooltip" title="';m=A.local;if(m){n=m.call(C,{hash:{},inverse:x.noop,fn:x.program(14,G,J)})}else{n=C.local;n=typeof n===f?n():n}if(!A.local){n=c.call(C,n,{hash:{},inverse:x.noop,fn:x.program(14,G,J)})}if(n||n===0){B+=n}B+='"\n href="javascript:void(0);">\n <span class="ficon cogs large"></span>\n </a>\n </div>\n <div style="clear: both;"></div>\n </div>\n\n ';n=C.deleted;n=A["if"].call(C,n,{hash:{},inverse:x.noop,fn:x.program(16,F,J)});if(n||n===0){B+=n}B+="\n\n ";B+="\n ";B+="\n ";n=C.user;n=n==null||n===false?n:n.email;n=A["if"].call(C,n,{hash:{},inverse:x.noop,fn:x.program(20,o,J)});if(n||n===0){B+=n}B+="\n\n ";n=C.message;n=A["if"].call(C,n,{hash:{},inverse:x.noop,fn:x.program(32,y,J)});if(n||n===0){B+=n}B+='\n\n <div id="quota-message-container" style="display: none">\n <div id="quota-message" class="errormessage">\n ';m=A.local;if(m){n=m.call(C,{hash:{},inverse:x.noop,fn:x.program(34,w,J)})}else{n=C.local;n=typeof n===f?n():n}if(!A.local){n=c.call(C,n,{hash:{},inverse:x.noop,fn:x.program(34,w,J)})}if(n||n===0){B+=n}B+='\n </div>\n </div>\n</div>\n\n<div id="';m=A.id;if(m){n=m.call(C,{hash:{}})}else{n=C.id;n=typeof n===f?n():n}B+=e(n)+'-datasets" class="history-datasets-list"></div>\n\n<div class="infomessagesmall" id="emptyHistoryMessage" style="display: none;">\n ';m=A.local;if(m){n=m.call(C,{hash:{},inverse:x.noop,fn:x.program(36,v,J)})}else{n=C.local;n=typeof n===f?n():n}if(!A.local){n=c.call(C,n,{hash:{},inverse:x.noop,fn:x.program(36,v,J)})}if(n||n===0){B+=n}B+="\n</div>";return B})})(); \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: tool_runner.py, rerun: allow encoded ids; history panel: move refresh out of popup, rearrange icons.
by Bitbucket 15 Nov '12

15 Nov '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/8e52e426487b/ changeset: 8e52e426487b user: carlfeberhard date: 2012-11-15 19:08:57 summary: tool_runner.py, rerun: allow encoded ids; history panel: move refresh out of popup, rearrange icons. affected #: 8 files diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r 8e52e426487bf732d051e538ac4cd7d352aa5065 lib/galaxy/webapps/galaxy/controllers/tool_runner.py --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -117,8 +117,14 @@ error( "'id' parameter is required" ); try: id = int( id ) + except: - error( "Invalid value for 'id' parameter" ) + # it's not an un-encoded id, try to parse as encoded + try: + id = trans.security.decode_id( id ) + except: + error( "Invalid value for 'id' parameter" ) + # Get the dataset object data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( id ) #only allow rerunning if user is allowed access to the dataset. diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r 8e52e426487bf732d051e538ac4cd7d352aa5065 static/scripts/mvc/dataset/hda-base.js --- a/static/scripts/mvc/dataset/hda-base.js +++ b/static/scripts/mvc/dataset/hda-base.js @@ -523,7 +523,7 @@ titleLink : Handlebars.templates[ 'template-hda-titleLink' ], hdaSummary : Handlebars.templates[ 'template-hda-hdaSummary' ], downloadLinks : Handlebars.templates[ 'template-hda-downloadLinks' ], - failedMetadata : Handlebars.templates[ 'template-hda-failedMetaData' ], + failedMetadata : Handlebars.templates[ 'template-hda-failedMetadata' ], displayApps : Handlebars.templates[ 'template-hda-displayApps' ] }; diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r 8e52e426487bf732d051e538ac4cd7d352aa5065 static/scripts/mvc/history/history-panel.js --- a/static/scripts/mvc/history/history-panel.js +++ b/static/scripts/mvc/history/history-panel.js @@ -127,6 +127,7 @@ /** event map */ events : { + 'click #history-refresh' : function(){ window.location.reload(); }, 'click #history-tag' : 'loadAndDisplayTags' }, @@ -313,7 +314,7 @@ show_deletedText = ( this.storage.get( 'show_deleted' ) )?( 'Hide deleted' ):( 'Show deleted' ), show_hiddenText = ( this.storage.get( 'show_hidden' ) )?( 'Hide hidden' ):( 'Show hidden' ), menuActions = {}; - menuActions[ _l( 'refresh' ) ] = function(){ window.location.reload(); }; + //menuActions[ _l( 'refresh' ) ] = function(){ window.location.reload(); }; menuActions[ _l( 'collapse all' ) ] = function(){ historyPanel.hideAllHdaBodies(); }; menuActions[ _l( show_deletedText ) ] = function(){ historyPanel.toggleShowDeleted(); }; menuActions[ _l( show_hiddenText ) ] = function(){ historyPanel.toggleShowHidden(); }; diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r 8e52e426487bf732d051e538ac4cd7d352aa5065 static/scripts/templates/compiled/template-hda-failedMetaData.js --- a/static/scripts/templates/compiled/template-hda-failedMetaData.js +++ /dev/null @@ -1,33 +0,0 @@ -(function() { - var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; -templates['template-hda-failedMetaData'] = template(function (Handlebars,depth0,helpers,partials,data) { - helpers = helpers || Handlebars.helpers; - var stack1, foundHelper, self=this, functionType="function", blockHelperMissing=helpers.blockHelperMissing, escapeExpression=this.escapeExpression; - -function program1(depth0,data) { - - var buffer = "", stack1, foundHelper; - buffer += "\n"; - foundHelper = helpers.local; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(2, program2, data)}); } - else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(2, program2, data)}); } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\nYou may be able to <a href=\""; - stack1 = depth0.urls; - stack1 = stack1 == null || stack1 === false ? stack1 : stack1.edit; - stack1 = typeof stack1 === functionType ? stack1() : stack1; - buffer += escapeExpression(stack1) + "\" target=\"galaxy_main\">set it manually or retry auto-detection</a>.\n"; - return buffer;} -function program2(depth0,data) { - - - return "An error occurred setting the metadata for this dataset.";} - - foundHelper = helpers.warningmessagesmall; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(1, program1, data)}); } - else { stack1 = depth0.warningmessagesmall; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.warningmessagesmall) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(1, program1, data)}); } - if(stack1 || stack1 === 0) { return stack1; } - else { return ''; }}); -})(); \ No newline at end of file diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r 8e52e426487bf732d051e538ac4cd7d352aa5065 static/scripts/templates/compiled/template-history-historyPanel.js --- a/static/scripts/templates/compiled/template-history-historyPanel.js +++ b/static/scripts/templates/compiled/template-history-historyPanel.js @@ -46,104 +46,109 @@ function program7(depth0,data) { - - return "Click to see more actions for this history";} - -function program9(depth0,data) { - var buffer = "", stack1, foundHelper; - buffer += "\n <div id=\"history-secondary-links\" style=\"float: right;\">\n <a id=\"history-tag\" title=\""; + buffer += "\n <a id=\"history-tag\" title=\""; + foundHelper = helpers.local; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(8, program8, data)}); } + else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(8, program8, data)}); } + if(stack1 || stack1 === 0) { buffer += stack1; } + buffer += "\"\n class=\"icon-button tags tooltip\" target=\"galaxy_main\" href=\"javascript:void(0)\"></a>\n <a id=\"history-annotate\" title=\""; foundHelper = helpers.local; if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(10, program10, data)}); } else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(10, program10, data)}); } if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\"\n class=\"icon-button tags tooltip\" target=\"galaxy_main\" href=\"javascript:void(0)\"></a>\n <a id=\"history-annotate\" title=\""; - foundHelper = helpers.local; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(12, program12, data)}); } - else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(12, program12, data)}); } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\"\n class=\"icon-button annotate tooltip\" target=\"galaxy_main\" href=\"javascript:void(0)\"></a>\n </div>\n "; + buffer += "\"\n class=\"icon-button annotate tooltip\" target=\"galaxy_main\" href=\"javascript:void(0)\"></a>\n "; return buffer;} -function program10(depth0,data) { +function program8(depth0,data) { return "Edit history tags";} -function program12(depth0,data) { +function program10(depth0,data) { return "Edit history annotation";} +function program12(depth0,data) { + + + return "Refresh this display";} + function program14(depth0,data) { + + return "Click to see more actions for this history";} + +function program16(depth0,data) { + var buffer = "", stack1, foundHelper; buffer += "\n "; foundHelper = helpers.warningmessagesmall; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(15, program15, data)}); } + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(17, program17, data)}); } else { stack1 = depth0.warningmessagesmall; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.warningmessagesmall) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(15, program15, data)}); } + if (!helpers.warningmessagesmall) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(17, program17, data)}); } if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "\n "; return buffer;} -function program15(depth0,data) { +function program17(depth0,data) { var stack1, foundHelper; foundHelper = helpers.local; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(16, program16, data)}); } + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(18, program18, data)}); } else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(16, program16, data)}); } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(18, program18, data)}); } if(stack1 || stack1 === 0) { return stack1; } else { return ''; }} -function program16(depth0,data) { +function program18(depth0,data) { return "You are currently viewing a deleted history!";} -function program18(depth0,data) { +function program20(depth0,data) { var buffer = "", stack1, foundHelper; buffer += "\n <div id=\"history-tag-annotation\">\n\n <div id=\"history-tag-area\" style=\"display: none\">\n <strong>"; foundHelper = helpers.local; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(19, program19, data)}); } - else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(19, program19, data)}); } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += ":</strong>\n <div class=\"tag-elt\"></div>\n </div>\n\n <div id=\"history-annotation-area\" style=\"display: none\">\n <strong>"; - foundHelper = helpers.local; if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(21, program21, data)}); } else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(21, program21, data)}); } if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += ":</strong>\n <div id=\"history-annotation-container\">\n <div id=\"history-annotation\" class=\"tooltip editable-text\"\n title=\""; + buffer += ":</strong>\n <div class=\"tag-elt\"></div>\n </div>\n\n <div id=\"history-annotation-area\" style=\"display: none\">\n <strong>"; foundHelper = helpers.local; if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(23, program23, data)}); } else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(23, program23, data)}); } if(stack1 || stack1 === 0) { buffer += stack1; } + buffer += ":</strong>\n <div id=\"history-annotation-container\">\n <div id=\"history-annotation\" class=\"tooltip editable-text\"\n title=\""; + foundHelper = helpers.local; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(25, program25, data)}); } + else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(25, program25, data)}); } + if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "\">\n "; stack1 = depth0.annotation; - stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.program(27, program27, data),fn:self.program(25, program25, data)}); + stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.program(29, program29, data),fn:self.program(27, program27, data)}); if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "\n </div>\n </div>\n </div>\n </div>\n "; return buffer;} -function program19(depth0,data) { +function program21(depth0,data) { return "Tags";} -function program21(depth0,data) { +function program23(depth0,data) { return "Annotation";} -function program23(depth0,data) { +function program25(depth0,data) { return "Click to edit annotation";} -function program25(depth0,data) { +function program27(depth0,data) { var buffer = "", stack1, foundHelper; buffer += "\n "; @@ -153,23 +158,23 @@ buffer += escapeExpression(stack1) + "\n "; return buffer;} -function program27(depth0,data) { +function program29(depth0,data) { var buffer = "", stack1, foundHelper; buffer += "\n <em>"; foundHelper = helpers.local; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(28, program28, data)}); } + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(30, program30, data)}); } else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(28, program28, data)}); } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(30, program30, data)}); } if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "</em>\n "; return buffer;} -function program28(depth0,data) { +function program30(depth0,data) { return "Describe or add notes to history";} -function program30(depth0,data) { +function program32(depth0,data) { var buffer = "", stack1, foundHelper; buffer += "\n <div id=\"message-container\">\n <div class=\""; @@ -183,58 +188,64 @@ buffer += escapeExpression(stack1) + "\n </div><br />\n </div>\n "; return buffer;} -function program32(depth0,data) { +function program34(depth0,data) { return "You are over your disk quota.\n Tool execution is on hold until your disk usage drops below your allocated quota.";} -function program34(depth0,data) { +function program36(depth0,data) { return "Your history is empty. Click 'Get Data' on the left pane to start";} buffer += "<div id=\"history-controls\">\n <div id=\"history-title-area\" class=\"historyLinks\">\n\n "; - buffer += "\n <div id=\"history-name-container\" style=\"float: left;\">\n "; + buffer += "\n <div id=\"history-name-container\">\n "; buffer += "\n "; stack1 = depth0.user; stack1 = stack1 == null || stack1 === false ? stack1 : stack1.email; stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.program(4, program4, data),fn:self.program(1, program1, data)}); if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n </div>\n\n <a id=\"history-action-popup\" class=\"tooltip\" title=\""; - foundHelper = helpers.local; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(7, program7, data)}); } - else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(7, program7, data)}); } - if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\"\n href=\"javascript:void(0);\" style=\"float: right;\">\n <span class=\"ficon cogs large\"></span>\n </a>\n <div style=\"clear: both;\"></div>\n </div>\n\n <div id=\"history-subtitle-area\">\n <div id=\"history-size\" style=\"float:left;\">"; + buffer += "\n </div>\n </div>\n\n <div id=\"history-subtitle-area\">\n <div id=\"history-size\" style=\"float:left;\">"; foundHelper = helpers.nice_size; if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{}}); } else { stack1 = depth0.nice_size; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - buffer += escapeExpression(stack1) + "</div>\n "; + buffer += escapeExpression(stack1) + "</div>\n\n <div id=\"history-secondary-links\" style=\"float: right;\">\n "; stack1 = depth0.user; stack1 = stack1 == null || stack1 === false ? stack1 : stack1.email; - stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(9, program9, data)}); + stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(7, program7, data)}); if(stack1 || stack1 === 0) { buffer += stack1; } - buffer += "\n <div style=\"clear: both;\"></div>\n </div>\n\n "; + buffer += "\n <a id=\"history-refresh\" class=\"tooltip\" title=\""; + foundHelper = helpers.local; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(12, program12, data)}); } + else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(12, program12, data)}); } + if(stack1 || stack1 === 0) { buffer += stack1; } + buffer += "\"\n href=\"javascript:void(0);\">\n <span class=\"ficon refresh large\"></span>\n </a>\n <a id=\"history-action-popup\" class=\"tooltip\" title=\""; + foundHelper = helpers.local; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(14, program14, data)}); } + else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(14, program14, data)}); } + if(stack1 || stack1 === 0) { buffer += stack1; } + buffer += "\"\n href=\"javascript:void(0);\">\n <span class=\"ficon cogs large\"></span>\n </a>\n </div>\n <div style=\"clear: both;\"></div>\n </div>\n\n "; stack1 = depth0.deleted; - stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(14, program14, data)}); + stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(16, program16, data)}); if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "\n\n "; buffer += "\n "; buffer += "\n "; stack1 = depth0.user; stack1 = stack1 == null || stack1 === false ? stack1 : stack1.email; - stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(18, program18, data)}); + stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(20, program20, data)}); if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "\n\n "; stack1 = depth0.message; - stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(30, program30, data)}); + stack1 = helpers['if'].call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(32, program32, data)}); if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "\n\n <div id=\"quota-message-container\" style=\"display: none\">\n <div id=\"quota-message\" class=\"errormessage\">\n "; foundHelper = helpers.local; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(32, program32, data)}); } + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(34, program34, data)}); } else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(32, program32, data)}); } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(34, program34, data)}); } if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "\n </div>\n </div>\n</div>\n\n<div id=\""; foundHelper = helpers.id; @@ -242,9 +253,9 @@ else { stack1 = depth0.id; stack1 = typeof stack1 === functionType ? stack1() : stack1; } buffer += escapeExpression(stack1) + "-datasets\" class=\"history-datasets-list\"></div>\n\n<div class=\"infomessagesmall\" id=\"emptyHistoryMessage\" style=\"display: none;\">\n "; foundHelper = helpers.local; - if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(34, program34, data)}); } + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{},inverse:self.noop,fn:self.program(36, program36, data)}); } else { stack1 = depth0.local; stack1 = typeof stack1 === functionType ? stack1() : stack1; } - if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(34, program34, data)}); } + if (!helpers.local) { stack1 = blockHelperMissing.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(36, program36, data)}); } if(stack1 || stack1 === 0) { buffer += stack1; } buffer += "\n</div>"; return buffer;}); diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r 8e52e426487bf732d051e538ac4cd7d352aa5065 static/scripts/templates/hda-templates.html --- a/static/scripts/templates/hda-templates.html +++ b/static/scripts/templates/hda-templates.html @@ -54,7 +54,7 @@  -<script type="text/template" class="template-hda" id="template-hda-failedMetaData"> +<script type="text/template" class="template-hda" id="template-hda-failedMetadata"> {{#warningmessagesmall}} {{#local}}An error occurred setting the metadata for this dataset.{{/local}} You may be able to <a href="{{ urls.edit }}" target="galaxy_main">set it manually or retry auto-detection</a>. diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r 8e52e426487bf732d051e538ac4cd7d352aa5065 static/scripts/templates/history-templates.html --- a/static/scripts/templates/history-templates.html +++ b/static/scripts/templates/history-templates.html @@ -6,7 +6,7 @@ <div id="history-title-area" class="historyLinks"> {{! history name (if any) }} - <div id="history-name-container" style="float: left;"> + <div id="history-name-container"> {{! TODO: factor out conditional css }} {{#if user.email}} <div id="history-name" class="tooltip editable-text" @@ -16,24 +16,27 @@ title="{{#local}}You must be logged in to edit your history name{{/local}}">{{name}}</div> {{/if}} </div> - - <a id="history-action-popup" class="tooltip" title="{{#local}}Click to see more actions for this history{{/local}}" - href="javascript:void(0);" style="float: right;"> - <span class="ficon cogs large"></span> - </a> - <div style="clear: both;"></div></div><div id="history-subtitle-area"><div id="history-size" style="float:left;">{{nice_size}}</div> - {{#if user.email}} + <div id="history-secondary-links" style="float: right;"> + {{#if user.email}} <a id="history-tag" title="{{#local}}Edit history tags{{/local}}" class="icon-button tags tooltip" target="galaxy_main" href="javascript:void(0)"></a><a id="history-annotate" title="{{#local}}Edit history annotation{{/local}}" class="icon-button annotate tooltip" target="galaxy_main" href="javascript:void(0)"></a> + {{/if}} + <a id="history-refresh" class="tooltip" title="{{#local}}Refresh this display{{/local}}" + href="javascript:void(0);"> + <span class="ficon refresh large"></span> + </a> + <a id="history-action-popup" class="tooltip" title="{{#local}}Click to see more actions for this history{{/local}}" + href="javascript:void(0);"> + <span class="ficon cogs large"></span> + </a></div> - {{/if}} <div style="clear: both;"></div></div> diff -r 0a1db986221074e72131afd6d16320357843c5f8 -r 8e52e426487bf732d051e538ac4cd7d352aa5065 templates/root/alternate_history.mako --- a/templates/root/alternate_history.mako +++ b/templates/root/alternate_history.mako @@ -440,6 +440,15 @@ #history-secondary-links { } + /*why this is getting underlined is beyond me*/ + #history-secondary-links #history-refresh { + text-decoration: none; + } + /*too tweaky*/ + #history-annotate { + margin-right: 3px; + } + #history-tag-area, #history-annotation-area { margin: 10px 0px 10px 0px; } Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0