1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/535d8ed52369/ Changeset: 535d8ed52369 User: greg Date: 2013-09-03 16:44:53 Summary: Fix for rendering various files in the tool shed. Specific files should now correctly and safely render as html or rst. Affected #: 13 files diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -164,14 +164,9 @@ message += "Choose <b>Uninstall this tool dependency</b> from the <b>Repository Actions</b> menu, correct problems " message += "if necessary, and try installing the dependency again." status = "error" - tool_shed_repository = tool_dependency.tool_shed_repository - tool_dependency.name = suc.to_safe_string( tool_dependency.name ) - tool_dependency.version = suc.to_safe_string( tool_dependency.version ) - tool_dependency.type = suc.to_safe_string( tool_dependency.type ) - tool_dependency.status = suc.to_safe_string( tool_dependency.status ) - tool_dependency.error_message = suc.to_safe_string( tool_dependency.error_message ) + repository = tool_dependency.tool_shed_repository return trans.fill_template( '/admin/tool_shed_repository/browse_tool_dependency.mako', - repository=tool_shed_repository, + repository=repository, tool_dependency=tool_dependency, message=message, status=status ) @@ -445,7 +440,7 @@ tool_dependencies=tool_dependencies ) for installed_tool_dependency in installed_tool_dependencies: if installed_tool_dependency.status == trans.app.model.ToolDependency.installation_status.ERROR: - message += ' %s' % suc.to_safe_string( installed_tool_dependency.error_message ) + message += ' %s' % suc.to_html_string( installed_tool_dependency.error_message ) tool_dependency_ids = [ trans.security.encode_id( td.id ) for td in tool_dependencies ] if message: status = 'error' diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1566,7 +1566,9 @@ changeset_revision = kwd[ 'changeset_revision' ] repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + trans.security.encode_id( repository.id ), + changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -2791,7 +2793,7 @@ anchors = modified + added + removed + deleted + unknown + ignored + clean diffs = [] for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node() ): - diffs.append( suc.to_safe_string( diff, to_html=True ) ) + diffs.append( suc.to_html_string( diff ) ) metadata = self.get_metadata( trans, id, ctx_str ) # For rendering the prev button. if ctx_parent: diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b lib/tool_shed/util/readme_util.py --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -1,6 +1,7 @@ import logging import os from galaxy.util import json +from galaxy.util import unicodify import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util @@ -19,9 +20,9 @@ full_path_to_readme_file = os.path.abspath( relative_path_to_readme_file ) try: f = open( full_path_to_readme_file, 'r' ) - text = f.read() + text = unicodify( f.read() ) f.close() - readme_files_dict[ readme_file_name ] = suc.translate_string( text, to_html=False ) + readme_files_dict[ readme_file_name ] = suc.size_string( text ) except Exception, e: log.debug( "Error reading README file '%s' defined in metadata: %s" % ( str( relative_path_to_readme_file ), str( e ) ) ) return readme_files_dict diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -38,8 +38,6 @@ INITIAL_CHANGELOG_HASH = '000000000000' MAX_CONTENT_SIZE = 1048576 MAX_DISPLAY_SIZE = 32768 -VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}&<>|" ) - DATATYPES_CONFIG_FILENAME = 'datatypes_conf.xml' REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = 'data_manager_conf.xml' REPOSITORY_DEPENDENCY_DEFINITION_FILENAME = 'repository_dependencies.xml' @@ -902,31 +900,34 @@ return repository def get_repository_file_contents( file_path ): - """Return the display-safe contents of a repository file.""" + """Return the display-safe contents of a repository file for display in a browser.""" if checkers.is_gzip( file_path ): - safe_str = to_safe_string( '\ngzip compressed file\n' ) + return '<br/>gzip compressed file<br/>' elif checkers.is_bz2( file_path ): - safe_str = to_safe_string( '\nbz2 compressed file\n' ) + return '<br/>bz2 compressed file<br/>' elif checkers.check_zip( file_path ): - safe_str = to_safe_string( '\nzip compressed file\n' ) + return '<br/>zip compressed file<br/>' elif checkers.check_binary( file_path ): - safe_str = to_safe_string( '\nBinary file\n' ) + return '<br/>Binary file<br/>' else: safe_str = '' for i, line in enumerate( open( file_path ) ): - safe_str = '%s%s' % ( safe_str, to_safe_string( line ) ) + safe_str = '%s%s' % ( safe_str, to_html_string( line ) ) # Stop reading after string is larger than MAX_CONTENT_SIZE. if len( safe_str ) > MAX_CONTENT_SIZE: large_str = \ - to_safe_string( '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) ) + '<br/>File contents truncated because file size is larger than maximum viewing size of %s<br/>' % \ + util.nice_size( MAX_CONTENT_SIZE ) safe_str = '%s%s' % ( safe_str, large_str ) break if len( safe_str ) > MAX_DISPLAY_SIZE: - # Eliminate the middle of the file to display a file no larger than MAX_DISPLAY_SIZE. This may not be ideal if the file is larger than MAX_CONTENT_SIZE. + # Eliminate the middle of the file to display a file no larger than MAX_DISPLAY_SIZE. This may not be ideal if the file is larger + # than MAX_CONTENT_SIZE. join_by_str = \ - to_safe_string( "\n\n...some text eliminated here because file size is larger than maximum viewing size of %s...\n\n" % util.nice_size( MAX_DISPLAY_SIZE ) ) + "<br/><br/>...some text eliminated here because file size is larger than maximum viewing size of %s...<br/><br/>" % \ + util.nice_size( MAX_DISPLAY_SIZE ) safe_str = util.shrink_string_by_size( safe_str, MAX_DISPLAY_SIZE, join_by=join_by_str, left_larger=True, beginning_on_size_error=True ) - return safe_str + return safe_str def get_repository_files( trans, folder_path ): """Return the file hierarchy of a tool shed repository.""" @@ -1438,6 +1439,13 @@ return required_rd_tup[ 4 ] return False +def size_string( raw_text, size=MAX_DISPLAY_SIZE ): + """Return a subset of a string (up to MAX_DISPLAY_SIZE) translated to a safe string for display in a browser.""" + if raw_text and len( raw_text ) >= size: + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( size ) + raw_text = '%s%s' % ( raw_text[ 0:size ], large_str ) + return raw_text or '' + def stringify( list ): if list: return ','.join( list ) @@ -1453,40 +1461,17 @@ file_name = fpath return file_name -def to_safe_string( text, to_html=True ): +def to_html_string( text ): """Translates the characters in text to an html string""" if text: - if to_html: - try: - escaped_text = unicodify( text ) - escaped_text = escaped_text.encode( 'ascii', 'ignore' ) - escaped_text = str( markupsafe.escape( escaped_text ) ) - except UnicodeDecodeError, e: - escaped_text = "Error decoding string: %s" % str( e ) - else: - escaped_text = str( text ) - translated = [] - for c in escaped_text: - if c in VALID_CHARS: - translated.append( c ) - elif c in [ '\n' ]: - if to_html: - translated.append( '<br/>' ) - else: - translated.append( c ) - elif c in [ '\r' ]: - continue - elif c in [ ' ', ' ' ]: - if to_html: - if c == ' ': - translated.append( ' ' ) - else: - translated.append( ' ' ) - else: - translated.append( c ) - else: - translated.append( '' ) - return ''.join( translated ) + try: + text = unicodify( text ) + except UnicodeDecodeError, e: + return "Error decoding string: %s" % str( e ) + text = markupsafe.escape( text ) + text = text.replace( '\n', '<br/>' ) + text = text.replace( ' ', ' ' ) + text = text.replace( ' ', ' ' ) return text def tool_shed_from_repository_clone_url( repository_clone_url ): @@ -1497,18 +1482,6 @@ """Determine if a tool shed is the current tool shed.""" return toolshed_base_url.rstrip( '/' ) == str( url_for( '/', qualified=True ) ).rstrip( '/' ) -def translate_string( raw_text, to_html=True ): - """Return a subset of a string (up to MAX_DISPLAY_SIZE) translated to a safe string for display in a browser.""" - if raw_text: - if len( raw_text ) <= MAX_DISPLAY_SIZE: - translated_string = to_safe_string( raw_text, to_html=to_html ) - else: - large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_DISPLAY_SIZE ) - translated_string = to_safe_string( '%s%s' % ( raw_text[ 0:MAX_DISPLAY_SIZE ], large_str ), to_html=to_html ) - else: - translated_string = '' - return translated_string - def update_in_shed_tool_config( app, repository ): """ A tool shed repository is being updated so change the shed_tool_conf file. Parse the config file to generate the entire list diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b templates/webapps/tool_shed/repository/common.mako --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -580,7 +580,7 @@ <%def name="render_readme( readme, pad, parent, row_counter )"><% - from tool_shed.util.shed_util_common import to_safe_string + from tool_shed.util.shed_util_common import to_html_string from galaxy.util import rst_to_html encoded_id = trans.security.encode_id( readme.id ) render_rst = readme.name.endswith( '.rst' ) @@ -605,7 +605,7 @@ %if render_rst: <tr><td>${ rst_to_html( readme.text ) }</td></tr> %else: - <tr><td>${readme.name}<br/>${ to_safe_string( readme.text, to_html=True ) }</td></tr> + <tr><td>${readme.name}<br/>${ to_html_string( readme.text ) }</td></tr> %endif </table></td> diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -355,7 +355,7 @@ %><tr><td>${render_star_rating( name, review.rating, disabled=True )}</td> - <td>${render_review_comment( to_safe_string( review.comment, to_html=True ) )}</td> + <td>${render_review_comment( to_html_string( review.comment ) )}</td><td>${time_ago( review.update_time )}</td><td>${review.user.username | h}</td></tr> diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b templates/webapps/tool_shed/repository/view_repository.mako --- a/templates/webapps/tool_shed/repository/view_repository.mako +++ b/templates/webapps/tool_shed/repository/view_repository.mako @@ -6,7 +6,7 @@ <% from galaxy.web.framework.helpers import time_ago - from tool_shed.util.shed_util_common import to_safe_string + from tool_shed.util.shed_util_common import to_html_string is_new = repository.is_new( trans.app ) is_deprecated = repository.deprecated @@ -106,7 +106,7 @@ ${repository.description | h} </div> %if repository.long_description: - ${render_long_description( to_safe_string( repository.long_description, to_html=True ) )} + ${render_long_description( to_html_string( repository.long_description ) )} %endif <div class="form-row"><label>Revision:</label> @@ -214,7 +214,7 @@ %><tr><td>${render_star_rating( name, review.rating, disabled=True )}</td> - <td>${render_review_comment( to_safe_string( review.comment, to_html=True ) )}</td> + <td>${render_review_comment( to_html_string( review.comment ) )}</td><td>${time_ago( review.update_time )}</td><td>${review.user.username}</td></tr> diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b templates/webapps/tool_shed/repository_review/browse_review.mako --- a/templates/webapps/tool_shed/repository_review/browse_review.mako +++ b/templates/webapps/tool_shed/repository_review/browse_review.mako @@ -6,7 +6,7 @@ <% from galaxy.web.form_builder import CheckboxField from tool_shed.util.container_util import STRSEP - from tool_shed.util.shed_util_common import to_safe_string + from tool_shed.util.shed_util_common import to_html_string %><%def name="stylesheets()"> @@ -84,7 +84,7 @@ <tr><td><div overflow-wrap:normal;overflow:hidden;word-break:keep-all;word-wrap:break-word;line-break:strict;> - ${ to_safe_string( component_review.comment, to_html=True ) } + ${ to_html_string( component_review.comment ) } </div></td></tr> diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b test/tool_shed/base/twilltestcase.py --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -374,7 +374,7 @@ def display_installed_repository_manage_page( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ): url = '/admin_toolshed/manage_repository?id=%s' % self.security.encode_id( installed_repository.id ) self.visit_galaxy_url( url ) - strings_displayed.append( installed_repository.installed_changeset_revision ) + strings_displayed.append( str( installed_repository.installed_changeset_revision ) ) self.check_for_strings( strings_displayed, strings_not_displayed ) def display_installed_workflow_image( self, repository, workflow_name, strings_displayed=[], strings_not_displayed=[] ): diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b test/tool_shed/functional/test_0000_basic_repository_features.py --- a/test/tool_shed/functional/test_0000_basic_repository_features.py +++ b/test/tool_shed/functional/test_0000_basic_repository_features.py @@ -151,7 +151,7 @@ commit_message="Uploaded filtering.txt", strings_displayed=[], strings_not_displayed=[] ) - self.display_manage_repository_page( repository, strings_displayed=[ 'Readme file for filtering 1.1.0' ] ) + self.display_manage_repository_page( repository, strings_displayed=[ 'Readme file for filtering 1.1.0' ] ) def test_0055_upload_filtering_test_data( self ): '''Upload filtering test data.''' @@ -219,19 +219,19 @@ commit_message="Uploaded readme.txt", strings_displayed=[], strings_not_displayed=[] ) - self.display_manage_repository_page( repository, strings_displayed=[ 'This is a readme file.' ] ) + self.display_manage_repository_page( repository, strings_displayed=[ 'This is a readme file.' ] ) # Verify that there is a different readme file for each metadata revision. metadata_revisions = self.get_repository_metadata_revisions( repository ) self.display_manage_repository_page( repository, - strings_displayed=[ 'Readme file for filtering 1.1.0', - 'This is a readme file.' ] ) + strings_displayed=[ 'Readme file for filtering 1.1.0', + 'This is a readme file.' ] ) def test_0075_delete_readme_txt_file( self ): '''Delete the readme.txt file.''' repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) self.delete_files_from_repository( repository, filenames=[ 'readme.txt' ] ) self.check_count_of_metadata_revisions_associated_with_repository( repository, metadata_count=2 ) - self.display_manage_repository_page( repository, strings_displayed=[ 'Readme file for filtering 1.1.0' ] ) + self.display_manage_repository_page( repository, strings_displayed=[ 'Readme file for filtering 1.1.0' ] ) def test_0080_search_for_valid_filter_tool( self ): '''Search for the filtering tool by tool ID, name, and version.''' @@ -335,7 +335,7 @@ commit_message="Uploaded new readme.txt with invalid ascii characters.", strings_displayed=[], strings_not_displayed=[] ) - self.display_manage_repository_page( repository, strings_displayed=[ 'These characters should not' ] ) + self.display_manage_repository_page( repository, strings_displayed=[ 'These characters should not' ] ) def test_0130_verify_handling_of_invalid_characters( self ): '''Load the above changeset in the change log and confirm that there is no server error displayed.''' @@ -353,5 +353,5 @@ # Check for the changeset revision, repository name, owner username, 'repos' in the clone url, and the captured # unicode decoding error message. strings_displayed = [ 'Changeset %d:%s' % ( revision_number, revision_hash ), 'filtering_0000', 'user1', 'repos', 'added:', - '+These characters should not' ] + '+These characters should not' ] self.load_changeset_in_tool_shed( repository_id, changeset_revision, strings_displayed=strings_displayed ) \ No newline at end of file diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b test/tool_shed/functional/test_0410_repository_component_review_access_control.py --- a/test/tool_shed/functional/test_0410_repository_component_review_access_control.py +++ b/test/tool_shed/functional/test_0410_repository_component_review_access_control.py @@ -186,10 +186,10 @@ self.login( email=common.test_user_3_email, username=common.test_user_3_name ) repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) user = test_db_util.get_user( common.test_user_2_email ) - strings_displayed = [ 'A good set of functional tests.', - 'Clear and concise readme file', - 'a true pleasure to read.', - 'Excellent tool, easy to use.' ] + strings_displayed = [ 'A good set of functional tests.', + 'Clear and concise readme file', + 'a true pleasure to read.', + 'Excellent tool, easy to use.' ] changeset_revision = self.get_repository_tip( repository ) review = test_db_util.get_repository_review_by_user_id_changeset_revision( user.id, repository.id, changeset_revision ) self.browse_component_review( review, strings_displayed=strings_displayed ) diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b test/tool_shed/functional/test_1000_install_basic_repository.py --- a/test/tool_shed/functional/test_1000_install_basic_repository.py +++ b/test/tool_shed/functional/test_1000_install_basic_repository.py @@ -99,7 +99,7 @@ "Galaxy's filtering tool", 'user1', self.url.replace( 'http://', '' ), - installed_repository.installed_changeset_revision ] + str( installed_repository.installed_changeset_revision ) ] self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed ) strings_displayed.extend( [ 'Installed tool shed repository', 'Valid tools', 'Filter1' ] ) self.display_installed_repository_manage_page( installed_repository, strings_displayed=strings_displayed ) @@ -119,7 +119,7 @@ "Galaxy's filtering tool", 'user1', self.url.replace( 'http://', '' ), - installed_repository.installed_changeset_revision ] + str( installed_repository.installed_changeset_revision ) ] self.display_installed_repository_manage_page( installed_repository, strings_displayed=strings_displayed ) self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed ) diff -r 76d580b93d86940350bb810cab0c8ef088a2197c -r 535d8ed52369a8bf02ce5e382b488bf94a6f257b test/tool_shed/functional/test_1200_uninstall_and_reinstall_basic_repository.py --- a/test/tool_shed/functional/test_1200_uninstall_and_reinstall_basic_repository.py +++ b/test/tool_shed/functional/test_1200_uninstall_and_reinstall_basic_repository.py @@ -99,7 +99,7 @@ "Galaxy's filtering tool for test 0000", 'user1', self.url.replace( 'http://', '' ), - installed_repository.installed_changeset_revision ] + str( installed_repository.installed_changeset_revision ) ] self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed ) strings_displayed.extend( [ 'Installed tool shed repository', 'Valid tools', 'Filter1' ] ) self.display_installed_repository_manage_page( installed_repository, strings_displayed=strings_displayed ) @@ -120,7 +120,7 @@ "Galaxy's filtering tool for test 0000", 'user1', self.url.replace( 'http://', '' ), - installed_repository.installed_changeset_revision ] + str( installed_repository.installed_changeset_revision ) ] self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed ) strings_displayed.extend( [ 'Installed tool shed repository', 'Valid tools', 'Filter1' ] ) self.display_installed_repository_manage_page( installed_repository, strings_displayed=strings_displayed ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.