October 2009 - galaxy-dev - lists.galaxyproject.org

[hg] galaxy 2863: Proper title for pages
by Greg Von Kuster 13 Oct '09

13 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/5d63e4eee122 changeset: 2863:5d63e4eee122 user: James Taylor <james(a)jamestaylor.org> date: Fri Oct 09 16:02:31 2009 -0400 description: Proper title for pages 1 file(s) affected in this change: templates/page/display.mako diffs (19 lines): diff -r 205a6782d9ae -r 5d63e4eee122 templates/page/display.mako --- a/templates/page/display.mako Fri Oct 09 14:47:54 2009 -0400 +++ b/templates/page/display.mako Fri Oct 09 16:02:31 2009 -0400 @@ -1,4 +1,6 @@ <%inherit file="/base_panels.mako"/> + +<%def name="title()">Galaxy :: ${page.user.username} :: ${page.title}</%def> <%def name="init()"> <% @@ -14,7 +16,7 @@ <div class="unified-panel-header" unselectable="on"> <div class="unified-panel-header-inner"> - ${page.user.username} / ${page.title} + ${page.user.username} :: ${page.title} </div> </div>

1 0

[hg] galaxy 2864: Encode HDA ids when peeking and saving. Fixes ...
by Greg Von Kuster 13 Oct '09

13 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/037374950cc9 changeset: 2864:037374950cc9 user: Kanwei Li <kanwei(a)gmail.com> date: Fri Oct 09 16:59:35 2009 -0400 description: Encode HDA ids when peeking and saving. Fixes #110 5 file(s) affected in this change: lib/galaxy/web/controllers/dataset.py lib/galaxy/web/controllers/root.py templates/dataset/large_file.mako templates/root/history_common.mako test/base/twilltestcase.py diffs (187 lines): diff -r 5d63e4eee122 -r 037374950cc9 lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Fri Oct 09 16:02:31 2009 -0400 +++ b/lib/galaxy/web/controllers/dataset.py Fri Oct 09 16:59:35 2009 -0400 @@ -199,8 +199,9 @@ return 'This link may not be followed from within Galaxy.' @web.expose - def display(self, trans, dataset_id=None, filename=None, show_all=False, **kwd): + def display(self, trans, encoded_id=None, show_all=False, to_ext=False, **kwd): """Catches the dataset id and displays file contents as directed""" + dataset_id = trans.security.decode_id( encoded_id ) data = trans.app.model.HistoryDatasetAssociation.get( dataset_id ) if not data: raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) ) @@ -208,27 +209,29 @@ if trans.app.security_agent.can_access_dataset( roles, data.dataset ): if data.state == trans.model.Dataset.states.UPLOAD: return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." ) - if filename is None or filename.lower() == "index": - file_path = data.file_name - mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) - trans.response.set_content_type(mime) - trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) - - else: - file_path = os.path.join( data.extra_files_path, filename ) - mime, encoding = mimetypes.guess_type( file_path ) - if mime is None: - mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] ) - trans.response.set_content_type( mime ) + + mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) + trans.response.set_content_type(mime) + trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) + + if to_ext: # Saving the file + trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size ) + if to_ext[0] != ".": + to_ext = "." + to_ext + valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + fname = data.name + fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] + trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, to_ext) + return open( data.file_name ) - if os.path.exists( file_path ): + if os.path.exists( data.file_name ): max_peek_size = 1000000 # 1 MB - if show_all or os.stat( file_path ).st_size < max_peek_size: - return open( file_path ) + if show_all or os.stat( data.file_name ).st_size < max_peek_size: + return open( data.file_name ) else: trans.response.set_content_type( "text/html" ) - return trans.fill_template( "/dataset/large_file.mako", - truncated_data = open( file_path ).read(max_peek_size), + return trans.stream_template_mako( "/dataset/large_file.mako", + truncated_data = open( data.file_name ).read(max_peek_size), data = data ) else: raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % ( filename ) ) diff -r 5d63e4eee122 -r 037374950cc9 lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Fri Oct 09 16:02:31 2009 -0400 +++ b/lib/galaxy/web/controllers/root.py Fri Oct 09 16:59:35 2009 -0400 @@ -134,72 +134,6 @@ return rval ## ---- Dataset display / editing ---------------------------------------- - - @web.expose - def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): - """ - Returns data directly into the browser. - Sets the mime-type according to the extension - """ - if hid is not None: - try: - hid = int( hid ) - except: - return "hid '%s' is invalid" %str( hid ) - history = trans.get_history() - for dataset in history.datasets: - if dataset.hid == hid: - data = dataset - break - else: - raise Exception( "No dataset with hid '%d'" % hid ) - else: - try: - data = self.app.model.HistoryDatasetAssociation.get( id ) - except: - return "Dataset id '%s' is invalid" %str( id ) - if data: - user, roles = trans.get_user_and_roles() - if trans.app.security_agent.can_access_dataset( roles, data.dataset ): - mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) - trans.response.set_content_type(mime) - if tofile: - fStat = os.stat(data.file_name) - trans.response.headers['Content-Length'] = int(fStat.st_size) - if toext[0:1] != ".": - toext = "." + toext - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = data.name - fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] - trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext) - trans.log_event( "Display dataset id: %s" % str(id) ) - try: - return open( data.file_name ) - except: - return "This dataset contains no content" - else: - return "You are not allowed to access this dataset" - else: - return "No dataset with id '%s'" % str( id ) - - @web.expose - def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"): - """ - Returns child data directly into the browser, based upon parent_id and designation. - """ - try: - data = self.app.model.HistoryDatasetAssociation.get( parent_id ) - if data: - child = data.get_child_by_designation( designation ) - if child: - user, roles = trans.get_user_and_roles() - if trans.app.security_agent.can_access_dataset( roles, child ): - return self.display( trans, id=child.id, tofile=tofile, toext=toext ) - else: - return "You are not privileged to access this dataset." - except Exception: - pass - return "A child named %s could not be found for data %s" % ( designation, parent_id ) @web.expose def display_as( self, trans, id=None, display_app=None, **kwd ): diff -r 5d63e4eee122 -r 037374950cc9 templates/dataset/large_file.mako --- a/templates/dataset/large_file.mako Fri Oct 09 16:02:31 2009 -0400 +++ b/templates/dataset/large_file.mako Fri Oct 09 16:59:35 2009 -0400 @@ -2,8 +2,8 @@ <div class="warningmessagelarge"> This dataset is large and only the first megabyte is shown below.<br /> - <a href="${h.url_for( controller='dataset', action='display', dataset_id=data.id, show_all=True )}">Show all</a> | - <a href="${h.url_for( controller='root', action='display', id=data.id, tofile='yes', toext=data.ext )}">Save</a> + <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), show_all=True )}">Show all</a> | + <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), to_ext=data.ext )}">Save</a> </div> <pre> diff -r 5d63e4eee122 -r 037374950cc9 templates/root/history_common.mako --- a/templates/root/history_common.mako Fri Oct 09 16:02:31 2009 -0400 +++ b/templates/root/history_common.mako Fri Oct 09 16:59:35 2009 -0400 @@ -32,7 +32,7 @@ <img src="${h.url_for('/static/images/pencil_icon_grey.png')}" width='16' height='16' alt='edit attributes' title='edit attributes' class='button edit' border='0'> %endif %else: - <a class="icon-button display" title="display data" href="${h.url_for( controller='dataset', dataset_id=data.id, action='display', filename='index')}" target="galaxy_main"></a> + <a class="icon-button display" title="display data" href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ))}" target="galaxy_main"></a> %if user_owns_dataset: <a class="icon-button edit" title="edit attributes" href="${h.url_for( controller='root', action='edit', id=data.id )}" target="galaxy_main"></a> %endif @@ -86,7 +86,7 @@ <div class="info">${_('Info: ')}${data.display_info()}</div> <div> %if data.has_data: - <a href="${h.url_for( controller='root', action='display', id=data.id, tofile='yes', toext=data.ext )}" target="_blank">save</a> + <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), to_ext=data.ext )}">save</a> %if user_owns_dataset: | <a href="${h.url_for( controller='tool_runner', action='rerun', id=data.id )}" target="galaxy_main">rerun</a> %endif diff -r 5d63e4eee122 -r 037374950cc9 test/base/twilltestcase.py --- a/test/base/twilltestcase.py Fri Oct 09 16:02:31 2009 -0400 +++ b/test/base/twilltestcase.py Fri Oct 09 16:59:35 2009 -0400 @@ -390,9 +390,9 @@ self.visit_url( "%s/dataset/undelete?id=%s" % ( self.url, hda_id ) ) if check_str: self.check_page_for_string( check_str ) - def display_history_item( self, id, check_str='' ): + def display_history_item( self, hda_id, check_str='' ): """Displays a history item - simulates eye icon click""" - self.visit_url( '%s/datasets/%s/display/index' % ( self.url, id ) ) + self.visit_url( '%s/dataset/display?encoded_id=%s' % ( self.url, self.security.encode_id( hda_id ) ) ) if check_str: self.check_page_for_string( check_str ) self.home()

1 0

[hg] galaxy 2865: Backed out changeset 037374950cc9
by Greg Von Kuster 13 Oct '09

13 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/316b413c933b changeset: 2865:316b413c933b user: James Taylor <james(a)jamestaylor.org> date: Sat Oct 10 14:03:08 2009 -0400 description: Backed out changeset 037374950cc9 5 file(s) affected in this change: lib/galaxy/web/controllers/dataset.py lib/galaxy/web/controllers/root.py templates/dataset/large_file.mako templates/root/history_common.mako test/base/twilltestcase.py diffs (187 lines): diff -r 037374950cc9 -r 316b413c933b lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Fri Oct 09 16:59:35 2009 -0400 +++ b/lib/galaxy/web/controllers/dataset.py Sat Oct 10 14:03:08 2009 -0400 @@ -199,9 +199,8 @@ return 'This link may not be followed from within Galaxy.' @web.expose - def display(self, trans, encoded_id=None, show_all=False, to_ext=False, **kwd): + def display(self, trans, dataset_id=None, filename=None, show_all=False, **kwd): """Catches the dataset id and displays file contents as directed""" - dataset_id = trans.security.decode_id( encoded_id ) data = trans.app.model.HistoryDatasetAssociation.get( dataset_id ) if not data: raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) ) @@ -209,29 +208,27 @@ if trans.app.security_agent.can_access_dataset( roles, data.dataset ): if data.state == trans.model.Dataset.states.UPLOAD: return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." ) - - mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) - trans.response.set_content_type(mime) - trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) - - if to_ext: # Saving the file - trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size ) - if to_ext[0] != ".": - to_ext = "." + to_ext - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = data.name - fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] - trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, to_ext) - return open( data.file_name ) + if filename is None or filename.lower() == "index": + file_path = data.file_name + mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) + trans.response.set_content_type(mime) + trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) + + else: + file_path = os.path.join( data.extra_files_path, filename ) + mime, encoding = mimetypes.guess_type( file_path ) + if mime is None: + mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] ) + trans.response.set_content_type( mime ) - if os.path.exists( data.file_name ): + if os.path.exists( file_path ): max_peek_size = 1000000 # 1 MB - if show_all or os.stat( data.file_name ).st_size < max_peek_size: - return open( data.file_name ) + if show_all or os.stat( file_path ).st_size < max_peek_size: + return open( file_path ) else: trans.response.set_content_type( "text/html" ) - return trans.stream_template_mako( "/dataset/large_file.mako", - truncated_data = open( data.file_name ).read(max_peek_size), + return trans.fill_template( "/dataset/large_file.mako", + truncated_data = open( file_path ).read(max_peek_size), data = data ) else: raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % ( filename ) ) diff -r 037374950cc9 -r 316b413c933b lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Fri Oct 09 16:59:35 2009 -0400 +++ b/lib/galaxy/web/controllers/root.py Sat Oct 10 14:03:08 2009 -0400 @@ -134,6 +134,72 @@ return rval ## ---- Dataset display / editing ---------------------------------------- + + @web.expose + def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): + """ + Returns data directly into the browser. + Sets the mime-type according to the extension + """ + if hid is not None: + try: + hid = int( hid ) + except: + return "hid '%s' is invalid" %str( hid ) + history = trans.get_history() + for dataset in history.datasets: + if dataset.hid == hid: + data = dataset + break + else: + raise Exception( "No dataset with hid '%d'" % hid ) + else: + try: + data = self.app.model.HistoryDatasetAssociation.get( id ) + except: + return "Dataset id '%s' is invalid" %str( id ) + if data: + user, roles = trans.get_user_and_roles() + if trans.app.security_agent.can_access_dataset( roles, data.dataset ): + mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) + trans.response.set_content_type(mime) + if tofile: + fStat = os.stat(data.file_name) + trans.response.headers['Content-Length'] = int(fStat.st_size) + if toext[0:1] != ".": + toext = "." + toext + valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + fname = data.name + fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] + trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext) + trans.log_event( "Display dataset id: %s" % str(id) ) + try: + return open( data.file_name ) + except: + return "This dataset contains no content" + else: + return "You are not allowed to access this dataset" + else: + return "No dataset with id '%s'" % str( id ) + + @web.expose + def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"): + """ + Returns child data directly into the browser, based upon parent_id and designation. + """ + try: + data = self.app.model.HistoryDatasetAssociation.get( parent_id ) + if data: + child = data.get_child_by_designation( designation ) + if child: + user, roles = trans.get_user_and_roles() + if trans.app.security_agent.can_access_dataset( roles, child ): + return self.display( trans, id=child.id, tofile=tofile, toext=toext ) + else: + return "You are not privileged to access this dataset." + except Exception: + pass + return "A child named %s could not be found for data %s" % ( designation, parent_id ) @web.expose def display_as( self, trans, id=None, display_app=None, **kwd ): diff -r 037374950cc9 -r 316b413c933b templates/dataset/large_file.mako --- a/templates/dataset/large_file.mako Fri Oct 09 16:59:35 2009 -0400 +++ b/templates/dataset/large_file.mako Sat Oct 10 14:03:08 2009 -0400 @@ -2,8 +2,8 @@ <div class="warningmessagelarge"> This dataset is large and only the first megabyte is shown below.<br /> - <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), show_all=True )}">Show all</a> | - <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), to_ext=data.ext )}">Save</a> + <a href="${h.url_for( controller='dataset', action='display', dataset_id=data.id, show_all=True )}">Show all</a> | + <a href="${h.url_for( controller='root', action='display', id=data.id, tofile='yes', toext=data.ext )}">Save</a> </div> <pre> diff -r 037374950cc9 -r 316b413c933b templates/root/history_common.mako --- a/templates/root/history_common.mako Fri Oct 09 16:59:35 2009 -0400 +++ b/templates/root/history_common.mako Sat Oct 10 14:03:08 2009 -0400 @@ -32,7 +32,7 @@ <img src="${h.url_for('/static/images/pencil_icon_grey.png')}" width='16' height='16' alt='edit attributes' title='edit attributes' class='button edit' border='0'> %endif %else: - <a class="icon-button display" title="display data" href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ))}" target="galaxy_main"></a> + <a class="icon-button display" title="display data" href="${h.url_for( controller='dataset', dataset_id=data.id, action='display', filename='index')}" target="galaxy_main"></a> %if user_owns_dataset: <a class="icon-button edit" title="edit attributes" href="${h.url_for( controller='root', action='edit', id=data.id )}" target="galaxy_main"></a> %endif @@ -86,7 +86,7 @@ <div class="info">${_('Info: ')}${data.display_info()}</div> <div> %if data.has_data: - <a href="${h.url_for( controller='dataset', action='display', encoded_id=trans.security.encode_id( data.id ), to_ext=data.ext )}">save</a> + <a href="${h.url_for( controller='root', action='display', id=data.id, tofile='yes', toext=data.ext )}" target="_blank">save</a> %if user_owns_dataset: | <a href="${h.url_for( controller='tool_runner', action='rerun', id=data.id )}" target="galaxy_main">rerun</a> %endif diff -r 037374950cc9 -r 316b413c933b test/base/twilltestcase.py --- a/test/base/twilltestcase.py Fri Oct 09 16:59:35 2009 -0400 +++ b/test/base/twilltestcase.py Sat Oct 10 14:03:08 2009 -0400 @@ -390,9 +390,9 @@ self.visit_url( "%s/dataset/undelete?id=%s" % ( self.url, hda_id ) ) if check_str: self.check_page_for_string( check_str ) - def display_history_item( self, hda_id, check_str='' ): + def display_history_item( self, id, check_str='' ): """Displays a history item - simulates eye icon click""" - self.visit_url( '%s/dataset/display?encoded_id=%s' % ( self.url, self.security.encode_id( hda_id ) ) ) + self.visit_url( '%s/datasets/%s/display/index' % ( self.url, id ) ) if check_str: self.check_page_for_string( check_str ) self.home()

1 0

[hg] galaxy 2866: Trimmer fix for fastq processing
by Greg Von Kuster 13 Oct '09

13 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/080b3c44963e changeset: 2866:080b3c44963e user: Anton Nekrutenko <anton(a)bx.psu.edu> date: Sat Oct 10 16:18:08 2009 -0400 description: Trimmer fix for fastq processing 1 file(s) affected in this change: tools/filters/trimmer.py diffs (14 lines): diff -r 316b413c933b -r 080b3c44963e tools/filters/trimmer.py --- a/tools/filters/trimmer.py Sat Oct 10 14:03:08 2009 -0400 +++ b/tools/filters/trimmer.py Sat Oct 10 16:18:08 2009 -0400 @@ -83,9 +83,7 @@ print line continue - if options.fastq and line.startswith('@'): - stop_err('Malformed fastq file: even numbered line starts with @') - + if line[0] not in invalid_starts: if col == 0: if int( options.end ) > 0:

1 0

[hg] galaxy 2862: Increase number of datasets/page on HDA grid.
by Greg Von Kuster 13 Oct '09

13 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/205a6782d9ae changeset: 2862:205a6782d9ae user: jeremy goecks <jeremy.goecks at emory.edu> date: Fri Oct 09 14:47:54 2009 -0400 description: Increase number of datasets/page on HDA grid. 1 file(s) affected in this change: lib/galaxy/web/controllers/dataset.py diffs (21 lines): diff -r c7dd346c20d5 -r 205a6782d9ae lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Thu Oct 08 22:01:36 2009 -0400 +++ b/lib/galaxy/web/controllers/dataset.py Fri Oct 09 14:47:54 2009 -0400 @@ -107,7 +107,7 @@ return accepted_filters # Grid definition - title = "Stored datasets" + title = "Stored Datasets" model_class = model.HistoryDatasetAssociation template='/dataset/grid.mako' default_sort_key = "-create_time" @@ -125,7 +125,7 @@ default_filter = dict( deleted="False", tags="All" ) preserve_state = False use_paging = True - num_rows_per_page = 10 + num_rows_per_page = 50 def apply_default_filter( self, trans, query, **kwargs ): # This is a somewhat obtuse way to join the History and HDA tables. However, it's necessary # because the initial query in build_initial_query is specificied on the HDA table (this is reasonable)

1 0

Re: [galaxy-dev] [galaxy-user] starting workflows with a pool of input files
by Greg Von Kuster 13 Oct '09

13 Oct '09

Hello Jubao, In the future, please direct questions like this to galaxy-dev(a)bx.psu.edu. Some Galaxy tools use 3rd party components. See our Tool Dependencies page at http://bitbucket.org/galaxy/galaxy-central/wiki/ToolDependencies for a list. This page is being updated with new tool dependencies which include: bowtie http://sourceforge.net/projects/bowtie-bio/files/bowtie/ bwa http://sourceforge.net/projects/bio-bwa/files/bwa-0.5.3.tar.bz2/download fastx_toolkit http://hannonlab.cshl.edu/fastx_toolkit/ samtools http://sourceforge.net/projects/samtools/files/ Greg Von Kuster Galaxy Development Team Jubao Duan wrote: > Greg, > > I installed Galaxy locally, everything seemed OK, but some tools are not > there when I run the galaxy, e.g., bowtie mapping. Here was how I > installed it: > > > 1. Get the latest copy from the repository: > > The latest source code can be downloaded from the anonymous†Mercurial > <http://www.selenic.com/mercurial/wiki/>†repository with this command: > > 1 <http://bitbucket.org/galaxy/galaxy-central/wiki/GetGalaxy#cl-1> > > > > % hg clone http://www.bx.psu.edu/hg/galaxy galaxy_dist > > > > Any comments? > > Jubao Duan

1 0

Re: [galaxy-dev] Fwd: [galaxy-user] starting workflows with a pool of input files
by Matthias Dodt 12 Oct '09

12 Oct '09

Okay Thanks;-)! greetings mat Dominic Tolle schrieb: > fyI > > Begin forwarded message: > >> *From: *Greg Von Kuster <ghv2(a)psu.edu <mailto:ghv2@psu.edu>> >> *Date: *October 8, 2009 3:34:35 PM GMT+02:00 >> *To: *Matthias Dodt <matthias.dodt(a)mdc-berlin.de >> <mailto:matthias.dodt@mdc-berlin.de>> >> *Cc: *galaxy <galaxy-user(a)bx.psu.edu <mailto:galaxy-user@bx.psu.edu>> >> *Subject: **Re: [galaxy-user] starting workflows with a pool of input >> files* >> >> Hello Matthias, >> >> I've opened the following ticket for this question - you can "follow" >> the ticket in bitbucket if you want. >> >> http://bitbucket.org/galaxy/galaxy-central/issue/197/starting-workflows-wit… >> >> Greg Von Kuster >> Galaxy Development Team >> >> >> Matthias Dodt wrote: >>> Hi there! >>> >>> I would like to start a workflow several times (with a pool of input >>> files). >>> >>> The reason is: >>> Lets say we process huge FASTA files. To do so, i incorporated a >>> splitting tool into our galaxy, which creates several smaller >>> split-files, which can be more easily processed on a cluster in >>> parallel. Now i would like to start a workflow with these splitfiles as >>> input (without starting it manually by clicking lets say 100 times - >>> for >>> each split-file - the same procedure). >>> >>> Is there an easy way to do that? >>> >>> thanks! >>> >>> greetings >>> >>> mat >>> _______________________________________________ >>> galaxy-user mailing list >>> galaxy-user(a)bx.psu.edu >>> http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-user >>> >>> >> _______________________________________________ >> galaxy-user mailing list >> galaxy-user(a)bx.psu.edu >> http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-user >

1 0

Starting Workflow n-times
by Matthias Dodt 12 Oct '09

12 Oct '09

Hi there! I already posted this question to the user list but didnt get any answer - would be nice;-)! I would like to start a workflow several times (with a pool of input files). The reason is: Lets say we process huge FASTA files. To do so, i incorporated a splitting tool into our galaxy, which creates several smaller split-files, which can be more easily processed on a cluster in parallel. Now i would like to start a workflow with these splitfiles as input (without starting it manually by clicking 100 times - for each split-file - the same procedure). How could that be accomplished? thanks! greetings mat

1 0

NGS impressions (long)
by Gordon, Assaf 09 Oct '09

09 Oct '09

Hello, I'd like to share couple of first impressions after testing the updated NGS version. =============== Biggest issue for me is the support for Solexa formats. (Obiovusly, I'm biased, and think that Solexa machines are much more common/useful than other types). Uploading a FASTQ file automatically detects the type (fastqsolexa or fastqsanger). That's a good start, but there's a catch: If I'm not aware that there are multiple types of FASTQ files - I'm stuck. Bowtie & BWA tools don't display my FASTQ file as possible input dataset. A novice user will need to notice two things: 1. the small warning on the tools' pages: "BWA accepts files in Sanger FASTQ format." "Must have Sanger-scaled quality values with ASCII offset 33 " 2. the fact the the file type is "fastqsolexa" (on the expanded green dataset rectangle). Even if he does notice, there's no way for him to know what to do... The easier (and wrong) solution is to click the pencil icon and change the type to "fastqsanger". It's wrong because the quality scores are not re-scaled, but ironically it kind of works and bowtie even returns pretty good results (just to show you how much those quality scores are worth). Another way (which is also not perfect) is to use the "Quality Format Converter" twice: first from ASCII(64) to Numeric, and then from numeric to ASCII(33). This still doesn't re-scale the quality scores, but at least they are now with offset 33. As the guy who actually wrote the "Quality Format convert" tool, let me tell you that the tool's interface is not understandable at all (that's my fault, of course). It is not clear if the "FASTQ ASCII OFFSET" input parameter refers to the input format or output format. What another user tried to do (seemed very logical from his POV), is to run the tool just ONCE: Select a 'fastqsolexa' file (with ASCII 64) as input dataset, and select ASCII and 33 as the tool's parameters (hoping to convert ASCII-64 to ASCII-33). This obviously doesn't work (the tool returns an error). Basically, if you've uploaded a solexa FASTQ file, you're stuck - but it's worse then just being stuck - you don't know why you're stuck or how to fix it. At least for bowtie, there's a command line argument to accept different scales - so you could build an XML file that adds those command line arguments based on the file type, e.g.: ================= <command> bwa_wrapper.py [ gzillion other options ] #if $input.ext == "fastqsolexa" --solexa-qual #else --pthread33-quals #endif </command> ================ This will not solve the 33/64 issue, or the 1.3 vs 1.4 quality scores, but might still be better. Since I don't trust the quality scores at all, and most of the time prefer to discard them, The thing I'd most like to do is be able to map FASTA (not FASTQ) files. bowtie supports this (with "-f"), and with another "if" in the XML this can be done (I think). =============== Another issue is selecting the organism. Galaxy supports so many of them, that selecting the one you want becomes a bit annoying. The "Upload File" starts with "--Additional Species are Below --" selected (half-way down the list). I need to scroll up if my organism is one of common ones, or down if it's an "additional". Another rant about forgotten usability in the Web2.0 era: on every normal operating system, typing the first couple of letters will scroll the list down to a line starting with those letters. But since we start half-way down the list, typing "hu" (for human) gets me to "Human Herpesvirus 1" instead of "human (hg18)". Right now, the easiest way I found to select 'human' is: 1. move focus to the list box (with TAB, not by clicking the mouse, of course) 2. click the "home" key on the keyboard (if you use a Mac, I don't know what key you need for that...) 3. click "h", then "u" - I get to the "human GRCh37 (hg19)" 4. click "down arrow" once to get to the 'right' human (hg18). And believe me, all you mice-loving people - my keyboard way is faster ;) And after spending so much effort on selecting the right organism at upload time, and with most of the other Galaxy tools respecting (and even requiring) the dbkey to be set - Having a stand-alone list-box in the Bowtie/BWA tools to select a reference organism database is downright insulting. Also, the default for BWA is "hg18" and the default for Bowtie is "hg19" - a bit confusing. Note that even the UCSC Genome Browser still has "hg18" as default (not "hg19"). My greedy suggestion: An edit-box with auto completion. And when you auto-complete, complete both with the short names (e.g. "hg" -> "hg19/hg18/hg17") and with full names (e.g "hu" -> "human") and with mid-string names (e.g "viri" => "Dro. virilis") AND expand the genus name too (because I want to be able to write "dro" and get to the Drosophilas, I don't want to write "D. "). Better yet, also add couple of quick selections (one button per organism) for the most common ones. ================= Another issue is the index database file used for bowtie and bwa. I don't know if you've downloaded the pre-built one or built it yourself, but the index file you have contains useless (IMHO) identifiers. Here's a sample of bowtie mapping result: gi|51511727|ref|NC_000011.8|NC_000011 125559911 25 In short: WTF? Yes, I learned (after a quick search) that "NC_000011.8" is simply "chr11". But please, in the name of all that is holy, rebuild the indexes with the commonly known chromosome names. Same goes for BWA, which is only slightly better: chr17_random.nib:1-2617613 492542 0 Hurrah, I my sequence mapped to "chr17_random.nib:1-2617613" .... This is not just me being spoiled - when I use the "SAM to Interval" tool to get normal mapping results[*], I get: ============ chr17_random.nib:1-2617613 510910 510946 + chr17_random.nib:1-2617613 512402 512438 + chr13.nib:1-114142980 45846690 45846726 - chr17_random.nib:1-2617613 492541 492577 - =========== see http://main.g2.bx.psu.edu/datasets/841134/display/index This will never work with the UCSC Genome browser (or any other sane browser, for that matter), so the interval file is useless. =========================== Another issue is the accuracy of BWA/Bowtie results. Take a look at the following history: http://main.g2.bx.psu.edu/history/imp?id=72116af6496fcec3 It contains just one sequences in fastqsolexa format (dataset 2). It's a real sequences from a real FASTQ file (except for the machine name and flowcell number, which I've changed). This is the sequence: ATCGCTTCTCGGCCTTTTGGCNAAGATCAAGTGTAG Dataset 3 is conversion to numeric values. Dataset 8 is conversion to ASCII-33 values. Dataset 9 is BWA mapping, just one result (chrom 17) with one mismatch: FOOBAR_27_FC42AGAAXX:1:1:2:435 0 chr17_random.nib:1-2617613 510911 0 36M Dataset 10 is Bowtie mapping, just one result (chrom 9) with one mismatch: FOOBAR_27_FC42AGAAXX:1:1:2:435 16 gi|89161216|ref|NC_000009.10|NC_000009 18530316 0 As a sanity check, run it through BLAT on the UCSC Genome browser (with hg18): You'll get 25 (!) results with 1 mismatch, and many more with more mismatches. The blat results include the chr9 (from BWA) and chr17 (from Bowtie). There's no way that the quality scores in the input FASTQ affected the mapping so much that BWA and Bowtie can decide to ignore all coordinates except one (and not even the same one). I've written it before - running bowtie and BWA with the default parameters and without understanding the complexities of those programs is down right misleading (IMHO). A novice user who just wants to get results fast - will be partial results at best (especially when looking for "unique mappers"). The "commonly used" parameters for BWA and Bowtie are not right for everyone (not right for most, even). Offering them as the default without a big flashing red warning is a disservice. ======= Thanks for reading so far. Regardless, galaxy is still the best thing since sliced bread, many people here use it daily and it makes everyone's life a lot easier - so don't take the above too hard. -gordon [*] not sure if it's obvious, but I despise the "SAM" format.

3 3

[hg] galaxy 2860: peeking large datasets only displays a portion...
by Greg Von Kuster 09 Oct '09

09 Oct '09

details: http://www.bx.psu.edu/hg/galaxy/rev/682cde08c3d5 changeset: 2860:682cde08c3d5 user: Kanwei Li <kanwei(a)gmail.com> date: Thu Oct 08 21:02:36 2009 -0400 description: peeking large datasets only displays a portion, but allows one to override or save 2 file(s) affected in this change: lib/galaxy/web/controllers/dataset.py templates/dataset/large_file.mako diffs (64 lines): diff -r e02620ad3f41 -r 682cde08c3d5 lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Thu Oct 08 18:07:02 2009 -0400 +++ b/lib/galaxy/web/controllers/dataset.py Thu Oct 08 21:02:36 2009 -0400 @@ -199,7 +199,7 @@ return 'This link may not be followed from within Galaxy.' @web.expose - def display(self, trans, dataset_id=None, filename=None, **kwd): + def display(self, trans, dataset_id=None, filename=None, show_all=False, **kwd): """Catches the dataset id and displays file contents as directed""" data = trans.app.model.HistoryDatasetAssociation.get( dataset_id ) if not data: @@ -209,23 +209,29 @@ if data.state == trans.model.Dataset.states.UPLOAD: return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." ) if filename is None or filename.lower() == "index": + file_path = data.file_name mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() ) trans.response.set_content_type(mime) trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) - try: - return open( data.file_name ) - except: - raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % ( filename ) ) + else: file_path = os.path.join( data.extra_files_path, filename ) mime, encoding = mimetypes.guess_type( file_path ) if mime is None: mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] ) trans.response.set_content_type( mime ) - try: + + if os.path.exists( file_path ): + max_peek_size = 1000000 # 1 MB + if show_all or os.stat( file_path ).st_size < max_peek_size: return open( file_path ) - except: - raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % ( filename ) ) + else: + trans.response.set_content_type( "text/html" ) + return trans.fill_template( "/dataset/large_file.mako", + truncated_data = open( file_path ).read(max_peek_size), + data = data ) + else: + raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % ( filename ) ) else: return trans.show_error_message( "You are not allowed to access this dataset" ) diff -r e02620ad3f41 -r 682cde08c3d5 templates/dataset/large_file.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/dataset/large_file.mako Thu Oct 08 21:02:36 2009 -0400 @@ -0,0 +1,11 @@ +<%inherit file="/base.mako"/> + +<div class="warningmessagelarge"> + This dataset is large and only the first megabyte is shown below.<br /> + <a href="${h.url_for( controller='dataset', action='display', dataset_id=data.id, show_all=True )}">Show all</a> | + <a href="${h.url_for( controller='root', action='display', id=data.id, tofile='yes', toext=data.ext )}">Save</a> +</div> + +<pre> +${ truncated_data } +</pre>

1 0