- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 2897: add image for trackster, use relative image ur...
by Greg Von Kuster 28 Oct '09

28 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/ed8ea4318e9a changeset: 2897:ed8ea4318e9a user: Kanwei Li <kanwei(a)gmail.com> date: Tue Oct 20 16:53:18 2009 -0400 description: add image for trackster, use relative image url in css 2 file(s) affected in this change: static/images/draggable_horizontal.png static/trackster.css diffs (14 lines): diff -r 52fadf5aabd9 -r ed8ea4318e9a static/images/draggable_horizontal.png Binary file static/images/draggable_horizontal.png has changed diff -r 52fadf5aabd9 -r ed8ea4318e9a static/trackster.css --- a/static/trackster.css Tue Oct 20 16:46:12 2009 -0400 +++ b/static/trackster.css Tue Oct 20 16:53:18 2009 -0400 @@ -63,7 +63,7 @@ position: absolute; margin-top: 1px; height: 16px; - background: #ddd url(/static/images/draggable_horizontal.png) center center no-repeat; + background: #ddd url(images/draggable_horizontal.png) center center no-repeat; border-style: outset; border-width: 1px; }

1 0

[hg] galaxy 2894: Fix setting the peek when a line is already un...
by Greg Von Kuster 28 Oct '09

28 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/ea29f33390d3 changeset: 2894:ea29f33390d3 user: Nate Coraor <nate(a)bx.psu.edu> date: Tue Oct 20 10:39:30 2009 -0400 description: Fix setting the peek when a line is already unicode (when does this happen?) 1 file(s) affected in this change: lib/galaxy/datatypes/data.py diffs (15 lines): diff -r 55d479c8d592 -r ea29f33390d3 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py Mon Oct 19 12:05:19 2009 -0400 +++ b/lib/galaxy/datatypes/data.py Tue Oct 20 10:39:30 2009 -0400 @@ -136,7 +136,10 @@ line = line.strip() if not line: continue - out.append( '<tr><td>%s</td></tr>' % escape( unicode( line, 'utf-8' ) ) ) + if type( line ) is unicode: + out.append( '<tr><td>%s</td></tr>' % escape( line ) ) + else: + out.append( '<tr><td>%s</td></tr>' % escape( unicode( line, 'utf-8' ) ) ) out.append( '</table>' ) out = "".join( out ) except Exception, exc:

1 0

[hg] galaxy 2895: Show encoded URL, while still allowing unencod...
by Greg Von Kuster 28 Oct '09

28 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/61684a4d6b9e changeset: 2895:61684a4d6b9e user: Kanwei Li <kanwei(a)gmail.com> date: Tue Oct 20 15:34:47 2009 -0400 description: Show encoded URL, while still allowing unencoded ids for backward compatibility. Revert to "display" displaying whole file. 6 file(s) affected in this change: lib/galaxy/web/buildapp.py lib/galaxy/web/controllers/dataset.py lib/galaxy/web/controllers/root.py templates/dataset/large_file.mako templates/root/history_common.mako test/base/twilltestcase.py diffs (213 lines): diff -r ea29f33390d3 -r 61684a4d6b9e lib/galaxy/web/buildapp.py --- a/lib/galaxy/web/buildapp.py Tue Oct 20 10:39:30 2009 -0400 +++ b/lib/galaxy/web/buildapp.py Tue Oct 20 15:34:47 2009 -0400 @@ -77,7 +77,7 @@ webapp.add_route( '/async/:tool_id/:data_id/:data_secret', controller='async', action='index', tool_id=None, data_id=None, data_secret=None ) webapp.add_route( '/:controller/:action', action='index' ) webapp.add_route( '/:action', controller='root', action='index' ) - webapp.add_route( '/datasets/:dataset_id/:action/:filename', controller='dataset', action='index', dataset_id=None, filename=None) + webapp.add_route( '/datasets/:encoded_id/:action/:filename', controller='dataset', action='index', encoded_id=None, filename=None) webapp.add_route( '/u/:username/p/:slug', controller='page', action='display_by_username_and_slug' ) webapp.finalize_config() # Wrap the webapp in some useful middleware diff -r ea29f33390d3 -r 61684a4d6b9e lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Tue Oct 20 10:39:30 2009 -0400 +++ b/lib/galaxy/web/controllers/dataset.py Tue Oct 20 15:34:47 2009 -0400 @@ -199,8 +199,14 @@ return 'This link may not be followed from within Galaxy.' @web.expose - def display(self, trans, dataset_id=None, filename=None, show_all=False, **kwd): + def display(self, trans, encoded_id=None, preview=False, filename=None, to_ext=None, **kwd): """Catches the dataset id and displays file contents as directed""" + + # DEPRECATION: We still support unencoded ids for backward compatibility + try: + dataset_id = int( encoded_id ) + except ValueError: + dataset_id = trans.security.decode_id( encoded_id ) data = trans.app.model.HistoryDatasetAssociation.get( dataset_id ) if not data: raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) ) @@ -208,30 +214,47 @@ if trans.app.security_agent.can_access_dataset( roles, data.dataset ): if data.state == trans.model.Dataset.states.UPLOAD: return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." ) - if filename is None or filename.lower() == "index": - file_path = data.file_name - mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) - trans.response.set_content_type(mime) - trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) - - else: + + if filename and filename != "index": file_path = os.path.join( data.extra_files_path, filename ) - mime, encoding = mimetypes.guess_type( file_path ) - if mime is None: - mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] ) - trans.response.set_content_type( mime ) + if os.path.exists( file_path ): + mime, encoding = mimetypes.guess_type( file_path ) + if not mime: + try: + mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] ) + except: + mime = "txt" - if os.path.exists( file_path ): - max_peek_size = 1000000 # 1 MB - if show_all or os.stat( file_path ).st_size < max_peek_size: + trans.response.set_content_type( mime ) return open( file_path ) else: + return "Could not find '%s' on the extra files path." % filename + + mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) + trans.response.set_content_type(mime) + trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) + + if to_ext: # Saving the file + trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size ) + if to_ext[0] != ".": + to_ext = "." + to_ext + valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + fname = data.name + fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] + trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, to_ext) + return open( data.file_name ) + + if os.path.exists( data.file_name ): + max_peek_size = 1000000 # 1 MB + if preview and os.stat( data.file_name ).st_size > max_peek_size: trans.response.set_content_type( "text/html" ) - return trans.fill_template( "/dataset/large_file.mako", - truncated_data = open( file_path ).read(max_peek_size), + return trans.stream_template_mako( "/dataset/large_file.mako", + truncated_data = open( data.file_name ).read(max_peek_size), data = data ) + else: + return open( data.file_name ) else: - raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % ( filename ) ) + raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % data.file_name ) else: return trans.show_error_message( "You are not allowed to access this dataset" ) diff -r ea29f33390d3 -r 61684a4d6b9e lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Tue Oct 20 10:39:30 2009 -0400 +++ b/lib/galaxy/web/controllers/root.py Tue Oct 20 15:34:47 2009 -0400 @@ -139,49 +139,13 @@ @web.expose def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): """ - Returns data directly into the browser. - Sets the mime-type according to the extension + Backward compatibility. """ - if hid is not None: - try: - hid = int( hid ) - except: - return "hid '%s' is invalid" %str( hid ) - history = trans.get_history() - for dataset in history.datasets: - if dataset.hid == hid: - data = dataset - break - else: - raise Exception( "No dataset with hid '%d'" % hid ) + if tofile: + to_ext = toext else: - try: - data = self.app.model.HistoryDatasetAssociation.get( id ) - except: - return "Dataset id '%s' is invalid" %str( id ) - if data: - user, roles = trans.get_user_and_roles() - if trans.app.security_agent.can_access_dataset( roles, data.dataset ): - mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) - trans.response.set_content_type(mime) - if tofile: - fStat = os.stat(data.file_name) - trans.response.headers['Content-Length'] = int(fStat.st_size) - if toext[0:1] != ".": - toext = "." + toext - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = data.name - fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] - trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext) - trans.log_event( "Display dataset id: %s" % str(id) ) - try: - return open( data.file_name ) - except: - return "This dataset contains no content" - else: - return "You are not allowed to access this dataset" - else: - return "No dataset with id '%s'" % str( id ) + to_ext = None + return trans.webapp.controllers['dataset'].display( trans, encoded_id=id, to_ext=to_ext ) @web.expose def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"): diff -r ea29f33390d3 -r 61684a4d6b9e templates/dataset/large_file.mako --- a/templates/dataset/large_file.mako Tue Oct 20 10:39:30 2009 -0400 +++ b/templates/dataset/large_file.mako Tue Oct 20 15:34:47 2009 -0400 @@ -2,8 +2,8 @@ <div class="warningmessagelarge"> This dataset is large and only the first megabyte is shown below.<br /> - <a href="${h.url_for( controller='dataset', action='display', dataset_id=data.id, show_all=True )}">Show all</a> | - <a href="${h.url_for( controller='root', action='display', id=data.id, tofile='yes', toext=data.ext )}">Save</a> + <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), filename='' )}">Show all</a> | + <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), to_ext=data.ext )}">Save</a> </div> <pre> diff -r ea29f33390d3 -r 61684a4d6b9e templates/root/history_common.mako --- a/templates/root/history_common.mako Tue Oct 20 10:39:30 2009 -0400 +++ b/templates/root/history_common.mako Tue Oct 20 15:34:47 2009 -0400 @@ -33,7 +33,7 @@ <img src="${h.url_for('/static/images/pencil_icon_grey.png')}" width='16' height='16' alt='edit attributes' title='edit attributes' class='button edit' border='0'> %endif %else: - <a class="icon-button display" title="display data" href="${h.url_for( controller='dataset', dataset_id=data.id, action='display', filename='index')}" target="galaxy_main"></a> + <a class="icon-button display" title="display data" href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), preview=True, filename='' )}" target="galaxy_main"></a> %if user_owns_dataset: <a class="icon-button edit" title="edit attributes" href="${h.url_for( controller='root', action='edit', id=data.id )}" target="galaxy_main"></a> %endif @@ -87,7 +87,7 @@ <div class="info">${_('Info: ')}${data.display_info()}</div> <div> %if data.has_data: - <a href="${h.url_for( controller='root', action='display', id=data.id, tofile='yes', toext=data.ext )}" target="_blank">save</a> + <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), to_ext=data.ext )}">save</a> %if user_owns_dataset: | <a href="${h.url_for( controller='tool_runner', action='rerun', id=data.id )}" target="galaxy_main">rerun</a> %endif diff -r ea29f33390d3 -r 61684a4d6b9e test/base/twilltestcase.py --- a/test/base/twilltestcase.py Tue Oct 20 10:39:30 2009 -0400 +++ b/test/base/twilltestcase.py Tue Oct 20 15:34:47 2009 -0400 @@ -419,9 +419,9 @@ self.visit_url( "%s/dataset/undelete?id=%s" % ( self.url, hda_id ) ) if check_str: self.check_page_for_string( check_str ) - def display_history_item( self, id, check_str='' ): + def display_history_item( self, hda_id, check_str='' ): """Displays a history item - simulates eye icon click""" - self.visit_url( '%s/datasets/%s/display/index' % ( self.url, id ) ) + self.visit_url( '%s/datasets/%s/display/' % ( self.url, self.security.encode_id( hda_id ) ) ) if check_str: self.check_page_for_string( check_str ) self.home() @@ -554,7 +554,7 @@ def verify_composite_datatype_file_content( self, file_name, hda_id ): local_name = self.get_filename( file_name ) temp_name = self.get_filename( 'temp_%s' % file_name ) - self.visit_url( "%s/datasets/%s/display/%s" % ( self.url, hda_id, file_name ) ) + self.visit_url( "%s/datasets/%s/display/%s" % ( self.url, self.security.encode_id( hda_id ), file_name ) ) data = self.last_page() file( temp_name, 'wb' ).write( data ) try:

1 0

Re: [galaxy-dev] Error in my functional test
by Vipin TS 26 Oct '09

26 Oct '09

Hi Nate, I tried to run the tool with a small input file. Then that case also the test fails and returns the same error message. FAIL: mGenePredict ( mgenepredict ) > Test-1 ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/galaxy/galaxy-2.1.2009/test/functional/test_toolbox.py", line 114, in test_tool self.do_it() File "/home/galaxy/galaxy-2.1.2009/test/functional/test_toolbox.py", line 54, in do_it self.verify_dataset_correctness( file ) File "/home/galaxy/galaxy-2.1.2009/test/base/twilltestcase.py", line 323, in verify_dataset_correctness if wait: self.wait() #wait for job to finish File "/home/galaxy/galaxy-2.1.2009/test/base/twilltestcase.py", line 617, in wait self.assertNotEqual(count, maxiter) AssertionError: 20 == 20 Also I can find that the job is not finished even though I got the fail message. I confirmed the same, when I run the test again it produces an error message stating that the Galaxy port is already in use. I hope my assumption is true, if not please let me know. Many thanks, Vipin T S

2 3

Error in my functional test
by Vipin Thankam Sreedharan 23 Oct '09

23 Oct '09

Hi all, I wrote a functional test for my tool. Here is the test code. <tests> <test> <param name="source" value="history" /> <param name="fasta_input" value="tair9_chr4_10000.fas" ftype="fasta" /> <param name="source" value="pretrained" /> <param name="organism" value="arabidopsis" /> <output name="output" file="tair9_chr4_output.gff3" /> </test> </tests> While running the test I am getting an error message like the following. ERROR: test module functional in /home/galaxy/galaxy-2.1.2009/test ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/galaxy/galaxy-2.1.2009/eggs/py2.5-noplatform/nose-0.9.1.dev_r101-py2.5.egg/nose/suite.py", line 46, in run self.setUp() File "/home/galaxy/galaxy-2.1.2009/eggs/py2.5-noplatform/nose-0.9.1.dev_r101-py2.5.egg/nose/suite.py", line 200, in setUp try_run(self.module, names) File "/home/galaxy/galaxy-2.1.2009/eggs/py2.5-noplatform/nose-0.9.1.dev_r101-py2.5.egg/nose/util.py", line 200, in try_run return func() File "/home/galaxy/galaxy-2.1.2009/test/functional/__init__.py", line 88, in setup server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) File "/home/galaxy/galaxy-2.1.2009/eggs/py2.5-noplatform/Paste-1.5.1-py2.5.egg/paste/httpserver.py", line 1257, in serve threadpool_options=threadpool_options) File "/home/galaxy/galaxy-2.1.2009/eggs/py2.5-noplatform/Paste-1.5.1-py2.5.egg/paste/httpserver.py", line 1107, in __init__ RequestHandlerClass, ssl_context) File "/home/galaxy/galaxy-2.1.2009/eggs/py2.5-noplatform/Paste-1.5.1-py2.5.egg/paste/httpserver.py", line 1087, in __init__ RequestHandlerClass, ssl_context) File "/home/galaxy/galaxy-2.1.2009/eggs/py2.5-noplatform/Paste-1.5.1-py2.5.egg/paste/httpserver.py", line 328, in __init__ HTTPServer.__init__(self, server_address, RequestHandlerClass) File "/usr/lib/python2.5/SocketServer.py", line 330, in __init__ self.server_bind() File "/usr/lib/python2.5/BaseHTTPServer.py", line 101, in server_bind SocketServer.TCPServer.server_bind(self) File "/usr/lib/python2.5/SocketServer.py", line 341, in server_bind self.socket.bind(self.server_address) File "<string>", line 1, in bind error: (98, 'Address already in use') ************************************************************ It will be great if you can give me a suggestion. Thanks in advance, Vipin T S

4 6

[hg] galaxy 2893: removed bogus unit tests from new gg/gff
by Greg Von Kuster 20 Oct '09

20 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/55d479c8d592 changeset: 2893:55d479c8d592 user: Ross Lazarus <ross.lazarus(a)gmail.com> date: Mon Oct 19 12:05:19 2009 -0400 description: removed bogus unit tests from new gg/gff 1 file(s) affected in this change: lib/galaxy/datatypes/genetics.py diffs (55 lines): diff -r 2f249afc66ea -r 55d479c8d592 lib/galaxy/datatypes/genetics.py --- a/lib/galaxy/datatypes/genetics.py Mon Oct 19 09:53:04 2009 -0400 +++ b/lib/galaxy/datatypes/genetics.py Mon Oct 19 12:05:19 2009 -0400 @@ -28,26 +28,6 @@ gal_Log = logging.getLogger(__name__) verbose = False - - -""" -Want to make some wig tracks from each analysis -Best n -log10(p). Make top hit the window. - - -from ucsc wig docs -ll options are placed in a single line separated by spaces: - - track type=wiggle_0 name=track_label description=center_label \ - visibility=display_mode color=r,g,b altColor=r,g,b \ - priority=priority autoScale=on|off \ - gridDefault=on|off maxHeightPixels=max:default:min \ - graphType=bar|points viewLimits=lower:upper \ - yLineMark=real-value yLineOnOff=on|off \ - windowingFunction=maximum|mean|minimum smoothingWindow=off|2-16 - -The track type with version is REQUIRED, and it currently must be wiggle_0: -""" class GenomeGraphs(Interval): @@ -131,7 +111,7 @@ else: # We've spanned a chromosome break - if i > 10: + if i > 10: # span 10 features break except: seqid, start, stop = ( '', '', '' ) @@ -189,15 +169,6 @@ Determines whether the file is in gff format GFF lines have nine required fields that must be tab-separated. - - For complete details see http://genome.ucsc.edu/FAQ/FAQformat#format3 - - >>> fname = get_test_fname( 'gff_version_3.gff' ) - >>> Gff().sniff( fname ) - False - >>> fname = get_test_fname( 'test.gff' ) - >>> Gff().sniff( fname ) - True """ f = open(filename,'r') headers = f.readline().split

1 0

[hg] galaxy 2892: Fixed a form definition editing bug
by Greg Von Kuster 20 Oct '09

20 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/2f249afc66ea changeset: 2892:2f249afc66ea user: rc date: Mon Oct 19 09:53:04 2009 -0400 description: Fixed a form definition editing bug 3 file(s) affected in this change: lib/galaxy/web/controllers/forms.py templates/admin/forms/manage_forms.mako templates/admin/forms/show_form_read_only.mako diffs (69 lines): diff -r 38bbf36f70e8 -r 2f249afc66ea lib/galaxy/web/controllers/forms.py --- a/lib/galaxy/web/controllers/forms.py Sun Oct 18 18:19:12 2009 -0400 +++ b/lib/galaxy/web/controllers/forms.py Mon Oct 19 09:53:04 2009 -0400 @@ -313,10 +313,10 @@ ctr=0 sb_options = [] while True: - option = params.get( 'field_'+str(index)+'_option_'+str(ctr), None ) - ctr = ctr+1 - if option: + if kwd.has_key( 'field_'+str(index)+'_option_'+str(ctr) ): + option = params.get( 'field_'+str(index)+'_option_'+str(ctr), None ) sb_options.append(util.restore_text(option)) + ctr = ctr+1 else: return sb_options def __get_saved_form(self, fd): @@ -342,8 +342,8 @@ layout = [] index = 0 while True: - grid_name = util.restore_text( params.get( 'grid_layout%i' % index, '' ) ) - if grid_name: + if kwd.has_key( 'grid_layout%i' % index ): + grid_name = util.restore_text( params.get( 'grid_layout%i' % index, '' ) ) layout.append( grid_name ) index = index + 1 else: @@ -355,7 +355,7 @@ # get the user entered fields index = 0 while True: - if params.get( 'field_name_%i' % index, False ): + if kwd.has_key( 'field_name_%i' % index ): fields.append( self.__get_field( index, **kwd ) ) index = index + 1 else: diff -r 38bbf36f70e8 -r 2f249afc66ea templates/admin/forms/manage_forms.mako --- a/templates/admin/forms/manage_forms.mako Sun Oct 18 18:19:12 2009 -0400 +++ b/templates/admin/forms/manage_forms.mako Mon Oct 19 09:53:04 2009 -0400 @@ -23,7 +23,7 @@ </div> %else: <div popupmenu="form-${form.id}-popup"> - <a class="action-button" href="${h.url_for( action='edit', form_id=form.id, show_form=True )}">Edit form definition</a> + <a class="action-button" href="${h.url_for( action='edit', form_id=form.id, show_form=True )}">Edit</a> <a class="action-button" confirm="Click OK to delete the form ${form.name}." href="${h.url_for( action='delete', form_id=form.id )}">Delete</a> </div> %endif diff -r 38bbf36f70e8 -r 2f249afc66ea templates/admin/forms/show_form_read_only.mako --- a/templates/admin/forms/show_form_read_only.mako Sun Oct 18 18:19:12 2009 -0400 +++ b/templates/admin/forms/show_form_read_only.mako Mon Oct 19 09:53:04 2009 -0400 @@ -63,14 +63,14 @@ <div class="toolForm"> %if form.desc: - <div class="toolFormTitle">${form.name} - <i>${form.desc}</i> + <div class="toolFormTitle">${form.name} - <i> ${form.desc}</i> (${form.type}) <a id="form-${form.id}-popup" class="popup-arrow" style="display: none;">▼</a> <div popupmenu="form-${form.id}-popup"> <a class="action-button" href="${h.url_for( action='edit', form_id=form.id, show_form=True )}">Edit</a> </div> </div> %else: - <div class="toolFormTitle">${form.name} + <div class="toolFormTitle">${form.name} (${form.type}) <a id="form-${form.id}-popup" class="popup-arrow" style="display: none;">▼</a> <div popupmenu="form-${form.id}-popup"> <a class="action-button" href="${h.url_for( action='edit', form_id=form.id, show_form=True )}">Edit</a>

1 0

[hg] galaxy 2891: Remove Genome Graphs and replace with topTable...
by Greg Von Kuster 20 Oct '09

20 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/38bbf36f70e8 changeset: 2891:38bbf36f70e8 user: Ross Lazarus <ross.lazarus(a)gmail.com> date: Sun Oct 18 18:19:12 2009 -0400 description: Remove Genome Graphs and replace with topTable (eg 1000 best p values) as a ucsc compatible gff 1 file(s) affected in this change: lib/galaxy/datatypes/genetics.py diffs (321 lines): diff -r 16a0c13c94cc -r 38bbf36f70e8 lib/galaxy/datatypes/genetics.py --- a/lib/galaxy/datatypes/genetics.py Fri Oct 16 15:54:43 2009 -0400 +++ b/lib/galaxy/datatypes/genetics.py Sun Oct 18 18:19:12 2009 -0400 @@ -16,126 +16,222 @@ import data from galaxy import util from cgi import escape -import urllib +import urllib, binascii from galaxy.web import url_for from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.data import Text from galaxy.datatypes.tabular import Tabular from galaxy.datatypes.images import Html +from galaxy.datatypes.interval import Interval +from galaxy.util.hash_util import * gal_Log = logging.getLogger(__name__) verbose = False -class GenomeGraphs( Tabular ): - """Tab delimited data containing a marker id and any number of numeric values""" + +""" +Want to make some wig tracks from each analysis +Best n -log10(p). Make top hit the window. + + +from ucsc wig docs +ll options are placed in a single line separated by spaces: + + track type=wiggle_0 name=track_label description=center_label \ + visibility=display_mode color=r,g,b altColor=r,g,b \ + priority=priority autoScale=on|off \ + gridDefault=on|off maxHeightPixels=max:default:min \ + graphType=bar|points viewLimits=lower:upper \ + yLineMark=real-value yLineOnOff=on|off \ + windowingFunction=maximum|mean|minimum smoothingWindow=off|2-16 + +The track type with version is REQUIRED, and it currently must be wiggle_0: +""" + +class GenomeGraphs(Interval): + + """gg version viewable at ucsc of Gff format""" + file_ext = "gg" + column_names = [ 'Seqname', 'Source', 'Feature', 'Start', 'End', 'Score', 'Strand', 'Frame', 'Group' ] """Add metadata elements""" - MetadataElement( name="markerCol", default=1, desc="Marker ID column", param=metadata.ColumnParameter ) - MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True ) - MetadataElement( name="column_types", default=[], desc="Column types", readonly=True, visible=False ) - file_ext = 'gg' + MetadataElement( name="columns", default=9, desc="Number of columns", readonly=True, visible=False ) + MetadataElement( name="column_types", default=['str','str','str','int','int','int','str','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) + MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter ) + MetadataElement( name="startCol", default=4, desc="Start column", param=metadata.ColumnParameter ) + MetadataElement( name="endCol", default=5, desc="End column", param=metadata.ColumnParameter ) + MetadataElement( name="strandCol", desc="Strand column (click box & select)", param=metadata.ColumnParameter, optional=True, no_value=0 ) + ###do we need to repeat these? they are the same as should be inherited from interval type + + def __init__(self, **kwd): + """Initialize datatype, by adding GBrowse display app""" + Interval.__init__(self, **kwd) + self.add_display_app ( 'ucsc', 'display at UCSC', 'as_ucsc_display_file', 'ucsc_links' ) - def __init__(self, **kwd): - """Initialize gg datatype, by adding UCSC display apps""" - Tabular.__init__(self, **kwd) - self.add_display_app ( 'ucsc', 'Genome Graph', 'as_ucsc_display_file', 'ucsc_links' ) - - def set_peek( self, dataset ): - """Set the peek and blurb text""" - if not dataset.dataset.purged: - dataset.peek = data.get_file_peek( dataset.file_name ) - dataset.blurb = util.commaify( str( data.get_line_count( dataset.file_name ) ) ) + " rows" - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + def as_ucsc_display_file( self, dataset, **kwd ): + return open( dataset.file_name ) - def get_mime(self): - """Returns the mime type of the datatype""" - return 'application/vnd.msexcel' + def set_meta( self, dataset, overwrite = True, **kwd ): + i = 0 + for i, line in enumerate( file ( dataset.file_name ) ): + line = line.rstrip('\r\n') + if line and not line.startswith( '#' ): + elems = line.split( '\t' ) + if len(elems) == 9: + try: + int( elems[3] ) + int( elems[4] ) + break + except: + pass + Interval.set_meta( self, dataset, overwrite = overwrite, skip = i ) + + def make_html_table( self, dataset, skipchars=[] ): + """Create HTML table, used for displaying peek""" + out = ['<table cellspacing="0" cellpadding="3">'] + comments = [] + try: + # Generate column header + out.append( '<tr>' ) + for i, name in enumerate( self.column_names ): + out.append( '<th>%s.%s</th>' % ( str( i+1 ), name ) ) + out.append( self.make_html_peek_rows( dataset, skipchars=skipchars ) ) + out.append( '</table>' ) + out = "".join( out ) + except Exception, exc: + out = "Can't create peek %s" % exc + return out def get_estimated_display_viewport( self, dataset ): - """Return a chrom, start, stop tuple for viewing a file.""" - raise notImplemented - - def as_ucsc_display_file( self, dataset, **kwd ): - """Returns file""" - return file(dataset.file_name,'r') - + """ + Return a chrom, start, stop tuple for viewing a file. There are slight differences between gff 2 and gff 3 + formats. This function should correctly handle both... + """ + if True or (dataset.has_data() and dataset.state == dataset.states.OK): + try: + seqid = '' + start = 2147483647 # Maximum value of a signed 32 bit integer ( 2**31 - 1 ) + stop = 0 + for i, line in enumerate( file( dataset.file_name ) ): + if i == 0: # track stuff there + continue + line = line.rstrip( '\r\n' ) + if not line: + continue + if not line.startswith( '#' ): + elems = line.split( '\t' ) + if not seqid: + # We can only set the viewport for a single chromosome + seqid = elems[0] + if seqid == elems[0]: + # Make sure we have not spanned chromosomes + start = min( start, int( elems[3] ) ) + stop = max( stop, int( elems[4] ) ) + else: + # We've spanned a chromosome + break + if i > 10: + break + except: + seqid, start, stop = ( '', '', '' ) + return ( seqid, str( start ), str( stop ) ) + else: + return ( '', '', '' ) + + def gbrowse_links( self, dataset, type, app, base_url ): + ret_val = [] + if dataset.has_data: + viewport_tuple = self.get_estimated_display_viewport( dataset ) + seqid = viewport_tuple[0] + start = viewport_tuple[1] + stop = viewport_tuple[2] + if seqid and start and stop: + for site_name, site_url in util.get_gbrowse_sites_by_build( dataset.dbkey ): + if site_name in app.config.gbrowse_display_sites: + link = "%s?start=%s&stop=%s&ref=%s&dbkey=%s" % ( site_url, start, stop, seqid, dataset.dbkey ) + ret_val.append( ( site_name, link ) ) + return ret_val + def ucsc_links( self, dataset, type, app, base_url ): - """ from the ever-helpful angie hinrichs angie(a)soe.ucsc.edu - a genome graphs call looks like this - http://genome.ucsc.edu/cgi-bin/hgGenome?clade=mammal&org=Human&db=hg18&hgGe… - &hgGenome_dataSetDescription=test&hgGenome_formatType=best%20guess&hgGenome_markerType=best%20guess - &hgGenome_columnLabels=best%20guess&hgGenome_maxVal=&hgGenome_labelVals= - &hgGenome_maxGapToFill=25000000&hgGenome_uploadFile=http://galaxy.esphealth.org/datasets/333/display/index - &hgGenome_doSubmitUpload=submit - Galaxy gives this for an interval file - http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg18&position=chr1:1-1000&hgt.cu… - http%3A%2F%2Fgalaxy.esphealth.org%2Fdisplay_as%3Fid%3D339%26display_app%3Ducsc - """ ret_val = [] - ggtail = '&hgGenome_doSubmitUpload=submit' - if not dataset.dbkey: - dataset.dbkey = 'hg18' # punt! - ret_val = [] - ggtail = '&hgGenome_doSubmitUpload=submit' - if not dataset.dbkey: - dataset.dbkey = 'hg18' # punt! - if dataset.has_data: - for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey): + if dataset.has_data: + viewport_tuple = self.get_estimated_display_viewport(dataset) + if viewport_tuple: + chrom = viewport_tuple[0] + start = viewport_tuple[1] + stop = viewport_tuple[2] + if start == '' or int(start) < 1: + start='1' + if stop == '' or int(stop) <= start: + stop = '%d' % (int(start) + 10000) + for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey): if site_name in app.config.ucsc_display_sites: - site_url = site_url.replace('/hgTracks?','/hgGenome?') # for genome graphs - display_url = urllib.quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) - sl = ["%sdb=%s" % (site_url,dataset.dbkey ),] - sl.append("&hgGenome_dataSetName=%s&hgGenome_dataSetDescription=%s" % (dataset.name, 'GalaxyGG_data')) - sl.append("&hgGenome_formatType=best%20guess&hgGenome_markerType=best%20guess") - sl.append("&hgGenome_columnLabels=first%20row&hgGenome_maxVal=&hgGenome_labelVals=") - sl.append("&hgGenome_maxGapToFill=25000000&hgGenome_uploadFile=") - s = urllib.quote_plus( ''.join(sl) ) - link = "%s%s%s" % (s, display_url, ggtail ) + # HACK: UCSC doesn't support https, so force http even + # if our URL scheme is https. Making this work + # requires additional hackery in your upstream proxy. + # If UCSC ever supports https, remove this hack. + internal_url = "%s" % url_for( controller='dataset', + dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name ) + if base_url.startswith( 'https://' ): + base_url = base_url.replace( 'https', 'http', 1 ) + display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, url_for( controller='root' ), dataset.id, type) ) + redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) ) + link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url ) ret_val.append( (site_name, link) ) + else: + gal_Log.debug('@@@ gg ucsc_links - no viewport_tuple') return ret_val - def validate( self, dataset ): - """Validate a gg file - all numeric after header row""" - errors = list() - infile = open(dataset.file_name, "r") - for i,row in enumerate(infile): #drop header - badvals = [] - if i > 0: - ll = row.strip().split('\t') - for j,x in enumerate(ll): - try: - x = float(x) - except: - badval.append('col%d:%s' % (j+1,x)) - if len(badvals) > 0: - errors.append('row %d, %s' % (' '.join(badvals))) - return errors + def sniff( self, filename ): + """ + Determines whether the file is in gff format - def repair_methods( self, dataset ): - """Return options for removing errors along with a description""" - return [("lines","Remove erroneous lines")] + GFF lines have nine required fields that must be tab-separated. + + For complete details see http://genome.ucsc.edu/FAQ/FAQformat#format3 + + >>> fname = get_test_fname( 'gff_version_3.gff' ) + >>> Gff().sniff( fname ) + False + >>> fname = get_test_fname( 'test.gff' ) + >>> Gff().sniff( fname ) + True + """ + f = open(filename,'r') + headers = f.readline().split + if headers[0].lower() == 'track': + headers = f.readline.split() + #headers = get_headers( filename, '\t' ) + try: + if len(headers) < 2: + return False + for hdr in headers: + if hdr and hdr[0].startswith( '##gff-version' ) and hdr[0].find( '2' ) < 0: + return False + if hdr and hdr[0] and not hdr[0].startswith( '#' ): + if len(hdr) != 9: + return False + try: + int( hdr[3] ) + int( hdr[4] ) + except: + return False + if hdr[5] != '.': + try: + score = int(hdr[5]) + except: + return False + if (score < 0 or score > 1000): + return False + if hdr[6] not in data.valid_strand: + return False + return True + except: + return False + - def sniff(self,filename): - """ - """ - infile = open(dataset.file_name, "r") - header= infile.next() # header - badvals = [] - for i,row in enumerate(infile[:10]): # sample first 10 rows - ll = row.strip().split('\t') - for j,x in enumerate(ll[1:]): # ignore first identifier col - try: - x = float(x) - except: - badval.append('col%d:%s' % (j+1,x)) - if len(badvals) > 0: - return False - else: - return True class rgTabList(Tabular): """ for sampleid and for featureid lists of exclusions or inclusions in the clean tool

1 0

[hg] galaxy 2890: Added anchors--identified by encoded id--to da...
by Greg Von Kuster 20 Oct '09

20 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/16a0c13c94cc changeset: 2890:16a0c13c94cc user: jeremy goecks <jeremy.goecks at emory.edu> date: Fri Oct 16 15:54:43 2009 -0400 description: Added anchors--identified by encoded id--to datasets in a history so that HDAs can be directly referenced. Clicking on a dataset in the datasets (HDAs) grid now goes to the dataset within the history. 4 file(s) affected in this change: lib/galaxy/web/controllers/root.py templates/dataset/grid.mako templates/root/history.mako templates/root/history_common.mako diffs (59 lines): diff -r 15756ebb2b11 -r 16a0c13c94cc lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Thu Oct 15 20:10:32 2009 -0400 +++ b/lib/galaxy/web/controllers/root.py Fri Oct 16 15:54:43 2009 -0400 @@ -55,7 +55,7 @@ return trans.fill_template_mako( "/my_data.mako" ) @web.expose - def history( self, trans, as_xml=False, show_deleted=False ): + def history( self, trans, as_xml=False, show_deleted=False, hda_id=None ): """ Display the current history, creating a new history if necessary. NOTE: No longer accepts "id" or "template" options for security reasons. @@ -78,6 +78,7 @@ return trans.stream_template_mako( "root/history.mako", history = history, datasets = query.all(), + hda_id = hda_id, show_deleted = show_deleted ) @web.expose diff -r 15756ebb2b11 -r 16a0c13c94cc templates/dataset/grid.mako --- a/templates/dataset/grid.mako Thu Oct 15 20:10:32 2009 -0400 +++ b/templates/dataset/grid.mako Fri Oct 16 15:54:43 2009 -0400 @@ -110,7 +110,7 @@ %endif %if 'history' in refresh_frames: if ( parent.frames && parent.frames.galaxy_history ) { - parent.frames.galaxy_history.location.href="${h.url_for( controller='root', action='history')}"; + parent.frames.galaxy_history.location.href="${h.url_for( controller='root', action='history', hda_id=str(ids) )}"; if ( parent.force_right_panel ) { parent.force_right_panel( 'show' ); } diff -r 15756ebb2b11 -r 16a0c13c94cc templates/root/history.mako --- a/templates/root/history.mako Thu Oct 15 20:10:32 2009 -0400 +++ b/templates/root/history.mako Fri Oct 16 15:54:43 2009 -0400 @@ -77,6 +77,10 @@ <% updateable = [data for data in reversed( datasets ) if data.visible and data.state not in [ "deleted", "empty", "error", "ok" ]] %> ${ ",".join( map(lambda data: "\"%s\" : \"%s\"" % (data.id, data.state), updateable) ) } }); + // Navigate to a dataset. + %if hda_id: + self.location = "#${hda_id}"; + %endif }); // Functionized so AJAX'd datasets can call them function initShowHide() { diff -r 15756ebb2b11 -r 16a0c13c94cc templates/root/history_common.mako --- a/templates/root/history_common.mako Thu Oct 15 20:10:32 2009 -0400 +++ b/templates/root/history_common.mako Fri Oct 16 15:54:43 2009 -0400 @@ -20,7 +20,8 @@ </div> %endif - ## Header row for history items (name, state, action buttons) + ## Header row for history items (name, state, action buttons) + <a name="${trans.security.encode_id(data.id)}"></a> <div style="overflow: hidden;" class="historyItemTitleBar"> <div class="historyItemButtons"> %if data_state == "upload":

1 0

[hg] galaxy 2889: Fixes and new functional tests for uploading m...
by Greg Von Kuster 20 Oct '09

20 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/15756ebb2b11 changeset: 2889:15756ebb2b11 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Thu Oct 15 20:10:32 2009 -0400 description: Fixes and new functional tests for uploading multi-byte character files and fixes for displaying multi-byte character dataset peek. 8 file(s) affected in this change: lib/galaxy/datatypes/data.py lib/galaxy/jobs/__init__.py lib/galaxy/model/__init__.py lib/galaxy/tools/actions/upload_common.py lib/galaxy/tools/parameters/grouping.py test-data/asian_chars_1.txt test/functional/test_get_data.py tools/data_source/upload.py diffs (269 lines): diff -r e945dcdfd578 -r 15756ebb2b11 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py Thu Oct 15 19:01:07 2009 -0400 +++ b/lib/galaxy/datatypes/data.py Thu Oct 15 20:10:32 2009 -0400 @@ -136,7 +136,7 @@ line = line.strip() if not line: continue - out.append( '<tr><td>%s</td></tr>' % escape( unicode( line ) ) ) + out.append( '<tr><td>%s</td></tr>' % escape( unicode( line, 'utf-8' ) ) ) out.append( '</table>' ) out = "".join( out ) except Exception, exc: @@ -437,8 +437,8 @@ temp.close() if file_type in [ 'gzipped', 'binary' ]: text = "%s file" % file_type - else: - text = unicode( '\n'.join( lines ), 'utf-8' ) + else: + text = unicode( '\n'.join( lines ), 'utf-8' ) return text def get_line_count(file_name): diff -r e945dcdfd578 -r 15756ebb2b11 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Thu Oct 15 19:01:07 2009 -0400 +++ b/lib/galaxy/jobs/__init__.py Thu Oct 15 20:10:32 2009 -0400 @@ -535,12 +535,12 @@ dataset.metadata.from_JSON_dict( self.external_output_metadata.get_output_filenames_by_dataset( dataset ).filename_out ) try: assert context.get( 'line_count', None ) is not None - if self.tool.is_multi_byte: + if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte: dataset.set_multi_byte_peek( line_count=context['line_count'] ) else: dataset.set_peek( line_count=context['line_count'] ) except: - if self.tool.is_multi_byte: + if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte: dataset.set_multi_byte_peek() else: dataset.set_peek() diff -r e945dcdfd578 -r 15756ebb2b11 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Thu Oct 15 19:01:07 2009 -0400 +++ b/lib/galaxy/model/__init__.py Thu Oct 15 20:10:32 2009 -0400 @@ -5,7 +5,7 @@ the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -import os.path, os, errno, sys +import os.path, os, errno, sys, codecs import galaxy.datatypes from galaxy.util.bunch import Bunch from galaxy import util @@ -418,7 +418,13 @@ return self.get_size() > 0 def mark_deleted( self, include_children=True ): self.deleted = True - + def is_multi_byte( self ): + if not self.has_data(): + return False + try: + return util.is_multi_byte( codecs.open( self.file_name, 'r', 'utf-8' ).read( 100 ) ) + except UnicodeDecodeError, e: + return False # FIXME: sqlalchemy will replace this def _delete(self): """Remove the file that corresponds to this data""" @@ -433,7 +439,7 @@ permitted_actions = Dataset.permitted_actions def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, - parent_id=None, validation_errors=None, visible=True, create_dataset = False ): + parent_id=None, validation_errors=None, visible=True, create_dataset=False ): self.name = name or "Unnamed dataset" self.id = id self.info = info @@ -519,6 +525,9 @@ def get_mime( self ): """Returns the mime type of the data""" return datatypes_registry.get_mimetype_by_extension( self.extension.lower() ) + def is_multi_byte( self ): + """Data consists of multi-byte characters""" + return self.dataset.is_multi_byte() def set_peek( self ): return self.datatype.set_peek( self ) def set_multi_byte_peek( self ): @@ -556,7 +565,7 @@ def get_converter_types(self): return self.datatype.get_converter_types( self, datatypes_registry) def find_conversion_destination( self, accepted_formats, **kwd ): - """Returns ( target_ext, exisiting converted dataset )""" + """Returns ( target_ext, existing converted dataset )""" return self.datatype.find_conversion_destination( self, accepted_formats, datatypes_registry, **kwd ) def add_validation_error( self, validation_error ): self.validation_errors.append( validation_error ) diff -r e945dcdfd578 -r 15756ebb2b11 lib/galaxy/tools/actions/upload_common.py --- a/lib/galaxy/tools/actions/upload_common.py Thu Oct 15 19:01:07 2009 -0400 +++ b/lib/galaxy/tools/actions/upload_common.py Thu Oct 15 20:10:32 2009 -0400 @@ -27,7 +27,7 @@ elif type( f ) == dict and 'filename' and 'local_filename' not in f: raise Exception( 'Uploaded file was encoded in a way not understood by Galaxy.' ) if upload_dataset['url_paste'].strip() != '': - upload_dataset['url_paste'] = datatypes.sniff.stream_to_file( StringIO.StringIO( upload_dataset['url_paste'] ), prefix="strio_url_paste_" )[0] + upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file( StringIO.StringIO( upload_dataset['url_paste'] ), prefix="strio_url_paste_" ) else: upload_dataset['url_paste'] = None new_files.append( upload_dataset ) @@ -135,7 +135,6 @@ folder.refresh() matches = filter( lambda x: x.name == name, active_folders( trans, folder ) ) if matches: - log.debug( 'DEBUGDEBUG: In %s, found a folder name match: %s:%s' % ( folder.name, matches[0].id, matches[0].name ) ) folder = matches[0] else: new_folder = trans.app.model.LibraryFolder( name=name, description='Automatically created by upload tool' ) @@ -143,7 +142,6 @@ folder.add_folder( new_folder ) new_folder.flush() trans.app.security_agent.copy_library_permissions( folder, new_folder ) - log.debug( 'DEBUGDEBUG: In %s, created a new folder: %s:%s' % ( folder.name, new_folder.id, new_folder.name ) ) folder = new_folder if library_bunch.replace_dataset: ld = library_bunch.replace_dataset diff -r e945dcdfd578 -r 15756ebb2b11 lib/galaxy/tools/parameters/grouping.py --- a/lib/galaxy/tools/parameters/grouping.py Thu Oct 15 19:01:07 2009 -0400 +++ b/lib/galaxy/tools/parameters/grouping.py Thu Oct 15 20:10:32 2009 -0400 @@ -240,7 +240,7 @@ url_paste = context['url_paste'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) - space_to_tab = False + space_to_tab = False if context.get( 'space_to_tab', None ) not in ["None", None]: space_to_tab = True warnings = [] @@ -248,7 +248,6 @@ if file_bunch.path: file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) - #rval.append( ( type, temp_name, precreated_name, space_to_tab, dataset_name, dataset_info ) ) for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): if file_bunch.path: file_bunch.space_to_tab = space_to_tab @@ -266,11 +265,6 @@ if d_type.composite_type is not None: #handle uploading of composite datatypes #Only one Dataset can be created - - ''' - dataset = UploadedDataset() - dataset.datatype = d_type - ''' dataset = Bunch() dataset.type = 'composite' dataset.file_type = file_type @@ -279,14 +273,12 @@ dataset.warnings = [] dataset.metadata = {} dataset.composite_files = {} - #load metadata files_metadata = context.get( self.metadata_ref, {} ) for meta_name, meta_spec in d_type.metadata_spec.iteritems(): if meta_spec.set_in_upload: if meta_name in files_metadata: dataset.metadata[ meta_name ] = files_metadata[ meta_name ] - dataset_name = None dataset_info = None if dataset.datatype.composite_type == 'auto_primary_file': diff -r e945dcdfd578 -r 15756ebb2b11 test-data/asian_chars_1.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/asian_chars_1.txt Thu Oct 15 20:10:32 2009 -0400 @@ -0,0 +1,1 @@ +蛋白質核酸酵素:制癌性物質の化学修飾による効果の向上. \ No newline at end of file diff -r e945dcdfd578 -r 15756ebb2b11 test/functional/test_get_data.py --- a/test/functional/test_get_data.py Thu Oct 15 19:01:07 2009 -0400 +++ b/test/functional/test_get_data.py Thu Oct 15 20:10:32 2009 -0400 @@ -93,3 +93,17 @@ self.verify_composite_datatype_file_content( 'rgenetics.bed', str( hda1.id ) ) self.verify_composite_datatype_file_content( 'rgenetics.fam', str( hda1.id ) ) self.delete_history( id=self.security.encode_id( history4.id ) ) + def test_020_upload_multibyte_character_file( self ): + """Test uploading multi-byte character file""" + # Logged in as admin_user + self.check_history_for_string( 'Your history is empty' ) + history5 = galaxy.model.History.filter( and_( galaxy.model.History.table.c.deleted==False, + galaxy.model.History.table.c.user_id==admin_user.id ) ) \ + .order_by( desc( galaxy.model.History.table.c.create_time ) ).first() + self.upload_file( 'asian_chars_1.txt' ) + hda1 = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert hda1 is not None, "Problem retrieving hda1 from database" + self.verify_dataset_correctness( 'asian_chars_1.txt', hid=str( hda1.hid ) ) + self.check_history_for_string( 'uploaded multi-byte char file' ) + self.delete_history( id=self.security.encode_id( history5.id ) ) diff -r e945dcdfd578 -r 15756ebb2b11 tools/data_source/upload.py --- a/tools/data_source/upload.py Thu Oct 15 19:01:07 2009 -0400 +++ b/tools/data_source/upload.py Thu Oct 15 20:10:32 2009 -0400 @@ -4,7 +4,7 @@ # WARNING: Changes in this tool (particularly as related to parsing) may need # to be reflected in galaxy.web.controllers.tool_runner and galaxy.tools -import urllib, sys, os, gzip, tempfile, shutil, re, gzip, zipfile +import urllib, sys, os, gzip, tempfile, shutil, re, gzip, zipfile, codecs from galaxy import eggs # need to import model before sniff to resolve a circular import dependency import galaxy.model @@ -129,13 +129,11 @@ if dataset.type == 'url': try: - temp_name, is_multi_byte = sniff.stream_to_file( urllib.urlopen( dataset.path ), prefix='url_paste' ) + temp_name, dataset.is_multi_byte = sniff.stream_to_file( urllib.urlopen( dataset.path ), prefix='url_paste' ) except Exception, e: file_err( 'Unable to fetch %s\n%s' % ( dataset.path, str( e ) ), dataset, json_file ) return dataset.path = temp_name - dataset.is_multi_byte = is_multi_byte - # See if we have an empty file if not os.path.exists( dataset.path ): file_err( 'Uploaded temporary file (%s) does not exist.' % dataset.path, dataset, json_file ) @@ -143,11 +141,15 @@ if not os.path.getsize( dataset.path ) > 0: file_err( 'The uploaded file is empty', dataset, json_file ) return - if 'is_multi_byte' not in dir( dataset ): - dataset.is_multi_byte = util.is_multi_byte( open( dataset.path, 'r' ).read( 1024 ) ) + if not dataset.type == 'url': + # Already set is_multi_byte above if type == 'url' + try: + dataset.is_multi_byte = util.is_multi_byte( codecs.open( dataset.path, 'r', 'utf-8' ).read( 100 ) ) + except UnicodeDecodeError, e: + dataset.is_multi_byte = False if dataset.is_multi_byte: + data_type = 'multi-byte char' ext = sniff.guess_ext( dataset.path, is_multi_byte=True ) - data_type = ext else: # See if we have a gzipped file, which, if it passes our restrictions, we'll uncompress is_gzipped, is_valid = check_gzip( dataset.path ) @@ -283,24 +285,19 @@ sys.exit( 1 ) output_paths = parse_outputs( sys.argv[2:] ) - json_file = open( 'galaxy.json', 'w' ) - for line in open( sys.argv[1], 'r' ): dataset = from_json_string( line ) dataset = util.bunch.Bunch( **safe_dict( dataset ) ) - try: output_path = output_paths[int( dataset.dataset_id )] except: print >>sys.stderr, 'Output path for dataset %s not found on command line' % dataset.dataset_id sys.exit( 1 ) - if dataset.type == 'composite': add_composite_file( dataset, json_file, output_path ) else: add_file( dataset, json_file, output_path ) - # clean up paramfile try: os.remove( sys.argv[1] )

1 0