- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 1576: Add a memdump page to the admin controller, ma...
by Nate Coraor 27 Oct '08

27 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/8eec48aaca6e changeset: 1576:8eec48aaca6e user: Nate Coraor <nate(a)bx.psu.edu> date: Mon Oct 27 17:03:50 2008 -0400 description: Add a memdump page to the admin controller, makes heapy output slightly browsable and provides an entry point to heapy (via editing the template) while the server is running. 4 file(s) affected in this change: lib/galaxy/app.py lib/galaxy/util/memdump.py lib/galaxy/web/controllers/admin.py templates/admin/memdump.mako diffs (174 lines): diff -r 415cc6dc8e35 -r 8eec48aaca6e lib/galaxy/app.py --- a/lib/galaxy/app.py Mon Oct 27 16:03:43 2008 -0400 +++ b/lib/galaxy/app.py Mon Oct 27 17:03:50 2008 -0400 @@ -35,6 +35,7 @@ self.job_queue = jobs.JobQueue( self, job_dispatcher ) self.job_stop_queue = jobs.JobStopQueue( self, job_dispatcher ) self.heartbeat = None + self.memdump = None # Start the heartbeat process if configured and available if self.config.use_heartbeat: from galaxy.util import heartbeat diff -r 415cc6dc8e35 -r 8eec48aaca6e lib/galaxy/util/memdump.py --- a/lib/galaxy/util/memdump.py Mon Oct 27 16:03:43 2008 -0400 +++ b/lib/galaxy/util/memdump.py Mon Oct 27 17:03:50 2008 -0400 @@ -22,22 +22,29 @@ self.fname = fname signal.signal( signum, self.dump ) self.heapy = guppy.hpy() + self.heap = None def dump( self, signum, stack ): file = open( self.fname, "a" ) print >> file, "Memdump for pid %d at %s" % ( os.getpid(), time.asctime() ) print >> file try: - h = self.heapy.heap() + self.heap = self.heapy.heap() print >> file, "heap():" - print >> file, h + print >> file, self.heap print >> file, "\nbyrcs:" - print >> file, h.byrcs + print >> file, self.heap.byrcs print >> file, "\nbyrcs[0].byid:" - print >> file, h.byrcs[0].byid + print >> file, self.heap.byrcs[0].byid print >> file, "\nget_rp():" - print >> file, h.get_rp() + print >> file, self.heap.get_rp() self.heapy.setref() except AssertionError: pass print >> file, "\nEnd dump\n" file.close() + def setref( self ): + self.heapy.setref() + def get( self, update=False ): + if update: + self.heap = self.heapy.heap() + return self.heap diff -r 415cc6dc8e35 -r 8eec48aaca6e lib/galaxy/web/controllers/admin.py --- a/lib/galaxy/web/controllers/admin.py Mon Oct 27 16:03:43 2008 -0400 +++ b/lib/galaxy/web/controllers/admin.py Mon Oct 27 17:03:50 2008 -0400 @@ -22,3 +22,42 @@ else: msg = 'Invalid password' return msg + @web.expose + def memdump( self, trans, ids = 'None', sorts = 'None', pages = 'None', new_id = None, new_sort = None, **kwd ): + if self.app.memdump is None: + return trans.show_error_message( "Memdump is not enabled (set <code>use_memdump = True</code> in universe_wsgi.ini)" ) + heap = self.app.memdump.get() + p = util.Params( kwd ) + msg = None + if p.dump: + heap = self.app.memdump.get( update = True ) + msg = "Heap dump complete" + elif p.setref: + self.app.memdump.setref() + msg = "Reference point set (dump to see delta from this point)" + ids = ids.split( ',' ) + sorts = sorts.split( ',' ) + if new_id is not None: + ids.append( new_id ) + sorts.append( 'None' ) + elif new_sort is not None: + sorts[-1] = new_sort + breadcrumb = "<a href='%s' class='breadcrumb'>heap</a>" % web.url_for() + # new lists so we can assemble breadcrumb links + new_ids = [] + new_sorts = [] + for id, sort in zip( ids, sorts ): + new_ids.append( id ) + if id != 'None': + breadcrumb += "<a href='%s' class='breadcrumb'>[%s]</a>" % ( web.url_for( ids=','.join( new_ids ), sorts=','.join( new_sorts ) ), id ) + heap = heap[int(id)] + new_sorts.append( sort ) + if sort != 'None': + breadcrumb += "<a href='%s' class='breadcrumb'>.by('%s')</a>" % ( web.url_for( ids=','.join( new_ids ), sorts=','.join( new_sorts ) ), sort ) + heap = heap.by( sort ) + ids = ','.join( new_ids ) + sorts = ','.join( new_sorts ) + if p.theone: + breadcrumb += ".theone" + heap = heap.theone + return trans.fill_template( '/admin/memdump.mako', heap = heap, ids = ids, sorts = sorts, breadcrumb = breadcrumb, msg = msg ) diff -r 415cc6dc8e35 -r 8eec48aaca6e templates/admin/memdump.mako --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/admin/memdump.mako Mon Oct 27 17:03:50 2008 -0400 @@ -0,0 +1,75 @@ +<%inherit file="/base.mako"/> + +<%def name="title()">Memory Profiling</%def> +<% + import re + from xml.sax.saxutils import escape, unescape +%> + +<style type="text/css"> + a.breadcrumb:link, + a.breadcrumb:visited, + a.breadcrumb:active { + text-decoration: none; + } + a.breadcrumb:hover { + text-decoration: underline; + } +</style> + +<h2>Memory Profiling</h2> + +<ul class="manage-table-actions"> + <li><a class="action-button" href="${h.url_for( controller='admin', action='memdump', dump=True )}">Dump memory (warning: hangs server!)</a></li> + <li><a class="action-button" href="${h.url_for( controller='admin', action='memdump', setref=True )}">Set reference point</a></li> +</ul> + +<%def name="htmlize( heap )"> +<% + s = escape( str( heap ) ) + new_s = "" + id_re = re.compile('^(\s+)([0-9]+)') + for line in s.split( '\n' ): + try: + id = id_re.search( line ).group( 2 ) + except: + id = None + new_s += re.sub( id_re, r'\1<a href="' + h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_id=id ) + r'">\2</a>', line ) + if id and heap[int(id)].count == 1: + new_s += " <a href='%s'>theone</a>\n" % h.url_for( ids=ids, sorts=sorts, new_id=id, theone=True ) + else: + new_s += "\n" + return new_s +%> +</%def> + +%if msg: + <div class="donemessage">${msg}</div> +%endif + +%if heap is None: + No memory dump available. Click "Dump memory" to create one. +%else: + <pre> + <br/> +You are here: ${breadcrumb}<br/> + %if breadcrumb.endswith( 'theone' ): + ${heap} + %else: + <nobr> +Sort: + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Class')}">Class</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Clodo' )}">Clodo</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Id' )}">Id</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Idset' )}">Idset</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Module' )}">Module</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Unity' )}">Unity</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Rcs' )}">Rcs</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Size' )}">Size</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Type' )}">Type</a> | + <a href="${h.url_for( controller='admin', action='memdump', ids=ids, sorts=sorts, new_sort='Via' )}">Via</a> + </nobr> + ${htmlize( heap )} + %endif + </pre> +%endif

1 0

[hg] galaxy 1575: Add a new URL_method attribute to data_source ...
by Greg Von Kuster 27 Oct '08

27 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/415cc6dc8e35 changeset: 1575:415cc6dc8e35 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Oct 27 16:03:43 2008 -0400 description: Add a new URL_method attribute to data_source tool types whose value is either "get" or "post" ( some require a get request while others require a post request ). This fixes the Biomart problem ( along with a new, well documented hack that can be eliminated when Biomart encodes the value of URL in the initial response - they'll tell us when they've fixed this ). Also added some requested info to the "send to EpiGRAPH" tool. 12 file(s) affected in this change: lib/galaxy/tools/__init__.py lib/galaxy/util/__init__.py lib/galaxy/web/controllers/tool_runner.py tools/data_destination/epigraph.xml tools/data_source/biomart.xml tools/data_source/biomart_test.xml tools/data_source/data_source.py tools/data_source/epigraph_import.xml tools/data_source/flymine.xml tools/data_source/ucsc_tablebrowser.xml tools/data_source/ucsc_tablebrowser_archaea.xml tools/data_source/ucsc_tablebrowser_test.xml diffs (299 lines): diff -r 36a7ff82faf0 -r 415cc6dc8e35 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Fri Oct 24 12:32:32 2008 -0400 +++ b/lib/galaxy/tools/__init__.py Mon Oct 27 16:03:43 2008 -0400 @@ -228,19 +228,21 @@ self.version = "1.0.0" # Type of tool self.tool_type = root.get( "tool_type", None ) - if self.tool_type is not None: - # data_source tool - if self.tool_type == "data_source": - self.param_trans_dict = {} - req_param_trans = root.find( "request_param_translation" ) - if req_param_trans is not None: - for req_param in req_param_trans.findall( "request_param" ): - # req_param tags must look like <request_param galaxy_name="dbkey" remote_name="GENOME" missing="" /> - trans_list = [] - remote_name = req_param.get( "remote_name" ) - trans_list.append( req_param.get( "galaxy_name" ) ) - trans_list.append( req_param.get( "missing" ) ) - self.param_trans_dict[ remote_name ] = trans_list + # data_source tool + if self.tool_type == "data_source": + self.URL_method = root.get( "URL_method", "get" ) # get is the default + # TODO: Biomart hack - eliminate when they encode URL - they'll let us know when... + self.add_to_URL = root.get( "add_to_URL", None ) + self.param_trans_dict = {} + req_param_trans = root.find( "request_param_translation" ) + if req_param_trans is not None: + for req_param in req_param_trans.findall( "request_param" ): + # req_param tags must look like <request_param galaxy_name="dbkey" remote_name="GENOME" missing="" /> + trans_list = [] + remote_name = req_param.get( "remote_name" ) + trans_list.append( req_param.get( "galaxy_name" ) ) + trans_list.append( req_param.get( "missing" ) ) + self.param_trans_dict[ remote_name ] = trans_list # Command line (template). Optional for tools that do not invoke a local program command = root.find("command") if command is not None and command.text is not None: diff -r 36a7ff82faf0 -r 415cc6dc8e35 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py Fri Oct 24 12:32:32 2008 -0400 +++ b/lib/galaxy/util/__init__.py Mon Oct 27 16:03:43 2008 -0400 @@ -143,7 +143,7 @@ # different parameters can be sanitized in different ways. NEVER_SANITIZE = ['file_data', 'url_paste', 'URL'] - def __init__( self, params, safe=True, sanitize=True, tool_type=None, param_trans_dict={} ): + def __init__( self, params, safe=True, sanitize=True, tool=None ): if safe: for key, value in params.items(): # Check to see if we should translate certain parameter names. For example, @@ -152,21 +152,27 @@ # param_trans_dict looks like { "GENOME" : [ "dbkey" "?" ] } new_key = key new_value = value - if tool_type == 'data_source': - if key in param_trans_dict: - new_key = param_trans_dict[ key ][0] + if tool and tool.tool_type == 'data_source': + if key in tool.param_trans_dict: + new_key = tool.param_trans_dict[ key ][0] if not value: - new_value = param_trans_dict[ key ][1] + new_value = tool.param_trans_dict[ key ][1] if key not in self.NEVER_SANITIZE and sanitize: self.__dict__[ new_key ] = sanitize_param( new_value ) else: self.__dict__[ new_key ] = new_value - for key, value in param_trans_dict.items(): - # Make sure that all translated values used in Galaxy are added to the params - galaxy_name = param_trans_dict[ key ][0] - if galaxy_name not in self.__dict__: - # This will set the galaxy_name to the "missing" value - self.__dict__[ galaxy_name ] = param_trans_dict[ key ][1] + if tool and tool.tool_type == 'data_source': + # Add the tool's URL_method to params + self.__dict__[ 'URL_method' ] = tool.URL_method + # TODO: Biomart hack - eliminate when they encode URL - they'll let us know when... + if tool.add_to_URL is not None: + self.__dict__[ 'add_to_URL' ] = tool.add_to_URL + for key, value in tool.param_trans_dict.items(): + # Make sure that all translated values used in Galaxy are added to the params + galaxy_name = tool.param_trans_dict[ key ][0] + if galaxy_name not in self.__dict__: + # This will set the galaxy_name to the "missing" value + self.__dict__[ galaxy_name ] = tool.param_trans_dict[ key ][1] else: self.__dict__.update(params) diff -r 36a7ff82faf0 -r 415cc6dc8e35 lib/galaxy/web/controllers/tool_runner.py --- a/lib/galaxy/web/controllers/tool_runner.py Fri Oct 24 12:32:32 2008 -0400 +++ b/lib/galaxy/web/controllers/tool_runner.py Mon Oct 27 16:03:43 2008 -0400 @@ -39,11 +39,7 @@ log.error( "index called with tool id '%s' but no such tool exists", tool_id ) trans.log_event( "Tool id '%s' does not exist" % tool_id ) return "Tool '%s' does not exist, kwd=%s " % (tool_id, kwd) - try: - param_trans_dict = tool.param_trans_dict - except: - param_trans_dict = {} - params = util.Params( kwd, sanitize=tool.options.sanitize, tool_type=tool.tool_type, param_trans_dict=param_trans_dict ) + params = util.Params( kwd, sanitize=tool.options.sanitize, tool=tool ) history = trans.get_history() trans.ensure_valid_galaxy_session() template, vars = tool.handle_input( trans, params.__dict__ ) diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_destination/epigraph.xml --- a/tools/data_destination/epigraph.xml Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_destination/epigraph.xml Mon Oct 27 16:03:43 2008 -0400 @@ -1,21 +1,40 @@ <?xml version="1.0"?> -<tool name="Perform genome" id="epigraph_export"> - <description> analysis and prediction with EpiGRAPH</description> - <redirect_url_params>GENOME=${input1.dbkey} NAME=${input1.name} INFO=${input1.info}</redirect_url_params> - <inputs> - <param format="bed" name="input1" type="data" label="Send this dataset to EpiGRAPH"> - <validator type="unspecified_build" /> - </param> - <param name="REDIRECT_URL" type="hidden" value="http://epigraph.mpi-inf.mpg.de/WebGRAPH_Public_Test/faces/DataImport.jsp" /> - <param name="DATA_URL" type="baseurl" value="/datasets" /> - <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=epigraph_import" /> - </inputs> - <outputs/> - <help> +<tool name="Perform genome analysis" id="epigraph_export"> + <description> and prediction with EpiGRAPH</description> + <redirect_url_params>GENOME=${input1.dbkey} NAME=${input1.name} INFO=${input1.info}</redirect_url_params> + <inputs> + <param format="bed" name="input1" type="data" label="Send this dataset to EpiGRAPH"> + <validator type="unspecified_build" /> + </param> + <param name="REDIRECT_URL" type="hidden" value="http://epigraph.mpi-inf.mpg.de/WebGRAPH_Public_Test/faces/DataImport.jsp" /> + <param name="DATA_URL" type="baseurl" value="/datasets" /> + <param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=epigraph_import" /> + </inputs> + <outputs/> + <help> + +.. class:: warningmark + +After clicking the **Execute** button, you will be redirected to the EpiGRAPH website. Please be patient while the dataset is being imported. Inside EpiGRAPH, buttons are available to send the results of the EpiGRAPH analysis back to Galaxy. In addition, you can always abandon an EpiGRAPH session and return to Galaxy by directing your browser to your current Galaxy instance. + +----- + +.. class:: infomark + **What it does** -This tool sends the selected dataset to EpiGRAPH for in-depth analysis and prediction. +This tool sends the selected dataset to EpiGRAPH in order to perform an in-depth analysis with statistical and machine learning methods. - </help> +----- + +.. class:: infomark + +**EpiGRAPH outline** + +The EpiGRAPH_ web service enables biologists to uncover hidden associations in vertebrate genome and epigenome datasets. Users can upload or import sets of genomic regions and EpiGRAPH will test a wide range of attributes (including DNA sequence and structure, gene density, chromatin modifications and evolutionary conservation) for enrichment or depletion among these regions. Furthermore, EpiGRAPH learns to predictively identify genomic regions that exhibit similar properties. + +.. _EpiGRAPH: http://epigraph.mpi-inf.mpg.de/ + + </help> </tool> diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_source/biomart.xml --- a/tools/data_source/biomart.xml Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_source/biomart.xml Mon Oct 27 16:03:43 2008 -0400 @@ -1,5 +1,13 @@ <?xml version="1.0"?> -<tool name="BioMart" id="biomart" tool_type="data_source"> + +<tool name="BioMart" id="biomart" tool_type="data_source" URL_method="get" add_to_URL="biomart_hack"> <description>Central server</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://www.biomart.org/biomart/martview" check_values="false" method="get" target="_top"> diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_source/biomart_test.xml --- a/tools/data_source/biomart_test.xml Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_source/biomart_test.xml Mon Oct 27 16:03:43 2008 -0400 @@ -1,5 +1,13 @@ <?xml version="1.0"?> -<tool name="BioMart" id="biomart" tool_type="data_source"> + +<tool name="BioMart" id="biomart_test" tool_type="data_source" URL_method="get" add_to_URL="biomart_hack"> <description>Test server</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://test.biomart.org/biomart/martview" check_values="false" method="get" target="_top"> diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_source/data_source.py --- a/tools/data_source/data_source.py Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_source/data_source.py Mon Oct 27 16:03:43 2008 -0400 @@ -33,10 +33,20 @@ if not URL: open( filename, 'w' ).write( "" ) stop_err( 'The remote data source application has not sent back a URL parameter in the request.' ) + # TODO: Hack to get biomart to work - this can be eliminated when the Biomart team encodes URL prior to sending, meanwhile + # everything including and beyond the first '&' is truncated from URL. They said they'll let us know when this is fixed + # at their end. + add_to_URL = params.get( 'add_to_URL', None ) + if add_to_URL: + URL += '&_export=1&GALAXY_URL=0' + URL_method = params.get( 'URL_method', None ) out = open( filename, 'w' ) CHUNK_SIZE = 2**20 # 1Mb try: - page = urllib.urlopen( URL, urllib.urlencode( params ) ) + if not URL_method or URL_method == 'get': + page = urllib.urlopen( URL ) + elif URL_method == 'post': + page = urllib.urlopen( URL, urllib.urlencode( params ) ) except: stop_err( 'It appears that the remote data source application is currently off line. Please try again later.' ) while 1: diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_source/epigraph_import.xml --- a/tools/data_source/epigraph_import.xml Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_source/epigraph_import.xml Mon Oct 27 16:03:43 2008 -0400 @@ -1,5 +1,10 @@ <?xml version="1.0"?> -<tool name="EpiGRAPH" id="epigraph_import" tool_type="data_source"> + +<tool name="EpiGRAPH" id="epigraph_import" tool_type="data_source" URL_method="get"> <description> server</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://epigraph.mpi-inf.mpg.de/WebGRAPH_Public_Test/faces/Login.jsp" check_values="false" method="get"> diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_source/flymine.xml --- a/tools/data_source/flymine.xml Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_source/flymine.xml Mon Oct 27 16:03:43 2008 -0400 @@ -1,5 +1,10 @@ <?xml version="1.0"?> -<tool name="Flymine" id="flymine" tool_type="data_source"> + +<tool name="Flymine" id="flymine" tool_type="data_source" URL_method="post"> <description>server</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://preview.flymine.org/preview/begin.do" check_values="false" method="get" target="_top"> diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_source/ucsc_tablebrowser.xml --- a/tools/data_source/ucsc_tablebrowser.xml Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_source/ucsc_tablebrowser.xml Mon Oct 27 16:03:43 2008 -0400 @@ -1,5 +1,10 @@ <?xml version="1.0"?> -<tool name="UCSC Main" id="ucsc_table_direct1" tool_type="data_source"> + +<tool name="UCSC Main" id="ucsc_table_direct1" tool_type="data_source" URL_method="post"> <description>table browser</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://genome.ucsc.edu/cgi-bin/hgTables" check_values="false" method="get"> diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_source/ucsc_tablebrowser_archaea.xml --- a/tools/data_source/ucsc_tablebrowser_archaea.xml Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_source/ucsc_tablebrowser_archaea.xml Mon Oct 27 16:03:43 2008 -0400 @@ -1,5 +1,10 @@ <?xml version="1.0"?> -<tool name="UCSC Archaea" id="ucsc_table_direct_archaea1" tool_type="data_source"> + +<tool name="UCSC Archaea" id="ucsc_table_direct_archaea1" tool_type="data_source" URL_method="post"> <description>table browser</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://archaea.ucsc.edu/cgi-bin/hgTables" check_values="false" method="get"> diff -r 36a7ff82faf0 -r 415cc6dc8e35 tools/data_source/ucsc_tablebrowser_test.xml --- a/tools/data_source/ucsc_tablebrowser_test.xml Fri Oct 24 12:32:32 2008 -0400 +++ b/tools/data_source/ucsc_tablebrowser_test.xml Mon Oct 27 16:03:43 2008 -0400 @@ -1,5 +1,10 @@ <?xml version="1.0"?> -<tool name="UCSC Test" id="ucsc_table_direct_test1" tool_type="data_source"> + +<tool name="UCSC Test" id="ucsc_table_direct_test1" tool_type="data_source" URL_method="post"> <description>table browser</description> <command interpreter="python">data_source.py $output</command> <inputs action="http://genome-test.cse.ucsc.edu/cgi-bin/hgTables" check_values="false" method="get">

1 0

[hg] galaxy 1574: WebHelpers imports simplejson. Import simplej...
by Nate Coraor 24 Oct '08

24 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/36a7ff82faf0 changeset: 1574:36a7ff82faf0 user: Nate Coraor <nate(a)bx.psu.edu> date: Fri Oct 24 12:32:32 2008 -0400 description: WebHelpers imports simplejson. Import simplejson before importing WebHelpers so we can ensure we get our version. 4 file(s) affected in this change: lib/galaxy/eggs/__init__.py lib/galaxy/model/custom_types.py lib/galaxy/web/controllers/workflow.py lib/galaxy/web/framework/__init__.py diffs (67 lines): diff -r c278ca8f9b73 -r 36a7ff82faf0 lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py Fri Oct 24 11:41:23 2008 -0400 +++ b/lib/galaxy/eggs/__init__.py Fri Oct 24 12:32:32 2008 -0400 @@ -528,8 +528,6 @@ else: pkg_resources.working_set.require( "%s==%s" % ( name, egg.get_vertag() ) ) return - except UserWarning, e: - sys.exit( 1 ) except pkg_resources.VersionConflict, e: # there's a conflicting egg on the pythonpath, remove it dist = pkg_resources.get_distribution( name ) diff -r c278ca8f9b73 -r 36a7ff82faf0 lib/galaxy/model/custom_types.py --- a/lib/galaxy/model/custom_types.py Fri Oct 24 11:41:23 2008 -0400 +++ b/lib/galaxy/model/custom_types.py Fri Oct 24 12:32:32 2008 -0400 @@ -1,4 +1,6 @@ from sqlalchemy.types import * +import pkg_resources +pkg_resources.require("simplejson") import simplejson import pickle from galaxy.util.bunch import Bunch diff -r c278ca8f9b73 -r 36a7ff82faf0 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py Fri Oct 24 11:41:23 2008 -0400 +++ b/lib/galaxy/web/controllers/workflow.py Fri Oct 24 12:32:32 2008 -0400 @@ -1,5 +1,7 @@ from galaxy.web.base.controller import * +import pkg_resources +pkg_resources.require( "simplejson" ) import simplejson from galaxy.tools.parameters import * @@ -736,4 +738,4 @@ prefix = "%s|" % ( key ) cleanup( prefix, input.cases[current_case].inputs, group_values ) cleanup( "", inputs, values ) - return associations \ No newline at end of file + return associations diff -r c278ca8f9b73 -r 36a7ff82faf0 lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Fri Oct 24 11:41:23 2008 -0400 +++ b/lib/galaxy/web/framework/__init__.py Fri Oct 24 12:32:32 2008 -0400 @@ -11,18 +11,18 @@ import pickle from galaxy import util +pkg_resources.require( "simplejson" ) +import simplejson + pkg_resources.require( "WebHelpers" ) +import webhelpers + pkg_resources.require( "PasteDeploy" ) - -import webhelpers from paste.deploy.converters import asbool pkg_resources.require( "Mako" ) import mako.template import mako.lookup - -pkg_resources.require( "simplejson" ) -import simplejson pkg_resources.require( "sqlalchemy>=0.3" ) from sqlalchemy import desc

1 0

[hg] galaxy 1573: Add some missing 'url_for's.
by Nate Coraor 24 Oct '08

24 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/c278ca8f9b73 changeset: 1573:c278ca8f9b73 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Fri Oct 24 11:41:23 2008 -0400 description: Add some missing 'url_for's. 4 file(s) affected in this change: lib/galaxy/web/controllers/root.py lib/galaxy/web/controllers/tool_runner.py templates/tool_form.tmpl templates/workflow/editor.mako diffs (80 lines): diff -r f2ce31c74346 -r c278ca8f9b73 lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Thu Oct 23 15:19:48 2008 -0400 +++ b/lib/galaxy/web/controllers/root.py Fri Oct 24 11:41:23 2008 -0400 @@ -114,7 +114,7 @@ ## ---- Dataset display / editing ---------------------------------------- @web.expose - def display(self, trans, id=None, hid=None, tofile=None, toext=".txt"): + def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): """ Returns data directly into the browser. Sets the mime-type according to the extension diff -r f2ce31c74346 -r c278ca8f9b73 lib/galaxy/web/controllers/tool_runner.py --- a/lib/galaxy/web/controllers/tool_runner.py Thu Oct 23 15:19:48 2008 -0400 +++ b/lib/galaxy/web/controllers/tool_runner.py Fri Oct 24 11:41:23 2008 -0400 @@ -30,7 +30,7 @@ def index(self, trans, tool_id=None, from_noframe=None, **kwd): # No tool id passed, redirect to main page if tool_id is None: - return trans.response.send_redirect( "/static/welcome.html" ) + return trans.response.send_redirect( url_for( "/static/welcome.html" ) ) # Load the tool toolbox = self.get_toolbox() tool = toolbox.tools_by_id.get( tool_id, None ) diff -r f2ce31c74346 -r c278ca8f9b73 templates/tool_form.tmpl --- a/templates/tool_form.tmpl Thu Oct 23 15:19:48 2008 -0400 +++ b/templates/tool_form.tmpl Fri Oct 24 11:41:23 2008 -0400 @@ -9,7 +9,7 @@ <title>Galaxy</title> <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" /> <link href="$h.url_for('/static/style/base.css')" rel="stylesheet" type="text/css" /> -<script type='text/javascript' src="/static/scripts/jquery.js"> </script> +<script type='text/javascript' src="$h.url_for('/static/scripts/jquery.js')"> </script> <script type="text/javascript"> jQuery( function() { jQuery( "select[@refresh_on_change='true']").change( function() { diff -r f2ce31c74346 -r c278ca8f9b73 templates/workflow/editor.mako --- a/templates/workflow/editor.mako Thu Oct 23 15:19:48 2008 -0400 +++ b/templates/workflow/editor.mako Fri Oct 24 11:41:23 2008 -0400 @@ -3,7 +3,7 @@ <%def name="title()">Galaxy Workflow Editor</%def> <%def name="late_javascripts()"> - <script type='text/javascript' src="/static/scripts/galaxy.panels.js"> </script> + <script type='text/javascript' src="${h.url_for('/static/scripts/galaxy.panels.js')}"> </script> <script type="text/javascript"> ensure_dd_helper(); make_left_panel( $("#left"), $("#center"), $("#left-border" ) ); @@ -17,16 +17,16 @@ ${parent.javascripts()}  - <script type='text/javascript' src="/static/scripts/jquery.js"> </script> - <script type='text/javascript' src="/static/scripts/jquery.ui.js"> </script> - <script type='text/javascript' src="/static/scripts/galaxy.ui.scrollPanel.js"> </script> - <script type='text/javascript' src="/static/scripts/jquery.hoverIntent.js"> </script> - <script type='text/javascript' src="/static/scripts/jquery.form.js"> </script> - <script type='text/javascript' src="/static/scripts/jquery.json.js"> </script> + <script type='text/javascript' src="${h.url_for('/static/scripts/jquery.js')}"> </script> + <script type='text/javascript' src="${h.url_for('/static/scripts/jquery.ui.js')}"> </script> + <script type='text/javascript' src="${h.url_for('/static/scripts/galaxy.ui.scrollPanel.js')}"> </script> + <script type='text/javascript' src="${h.url_for('/static/scripts/jquery.hoverIntent.js')}"> </script> + <script type='text/javascript' src="${h.url_for('/static/scripts/jquery.form.js')}"> </script> + <script type='text/javascript' src="${h.url_for('/static/scripts/jquery.json.js')}"> </script> - <script type='text/javascript' src="/static/scripts/galaxy.workflow_editor.canvas.js"> </script> + <script type='text/javascript' src="${h.url_for('/static/scripts/galaxy.workflow_editor.canvas.js')}"> </script> <!--[if lt IE 7]> <script type='text/javascript'> @@ -587,4 +587,4 @@ <div class="unified-panel-body" style="overflow: auto;"> <div id="right-content"></div> </div> -</%def> \ No newline at end of file +</%def>

1 0

[hg] galaxy 1572: Fix a bug from the last commit, what I changed...
by Nate Coraor 23 Oct '08

23 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/f2ce31c74346 changeset: 1572:f2ce31c74346 user: Nate Coraor <nate(a)bx.psu.edu> date: Thu Oct 23 15:19:48 2008 -0400 description: Fix a bug from the last commit, what I changed doesn't work right. 2 file(s) affected in this change: lib/galaxy/datatypes/interval.py templates/root/history_common.mako diffs (50 lines): diff -r 6e592c876f4d -r f2ce31c74346 lib/galaxy/datatypes/interval.py --- a/lib/galaxy/datatypes/interval.py Thu Oct 23 15:02:23 2008 -0400 +++ b/lib/galaxy/datatypes/interval.py Thu Oct 23 15:19:48 2008 -0400 @@ -9,6 +9,7 @@ import data from galaxy import util from galaxy.datatypes.sniff import * +from galaxy.web import url_for from cgi import escape import urllib from bx.intervals.io import * @@ -217,7 +218,7 @@ stop = viewport_tuple[2] for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey): if site_name in app.config.ucsc_display_sites: - display_url = urllib.quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) + display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s" % (base_url, url_for( controller='root' ), dataset.id, type) ) link = "%sdb=%s&position=%s:%s-%s&hgt.customText=%s" % (site_url, dataset.dbkey, chrom, start, stop, display_url ) ret_val.append( (site_name, link) ) return ret_val @@ -810,7 +811,7 @@ stop = viewport_tuple[2] for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey): if site_name in app.config.ucsc_display_sites: - display_url = urllib.quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) + display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s" % (base_url, url_for( controller='root' ), dataset.id, type) ) link = "%sdb=%s&position=%s:%s-%s&hgt.customText=%s" % (site_url, dataset.dbkey, chrom, start, stop, display_url ) ret_val.append( (site_name, link) ) return ret_val @@ -889,7 +890,7 @@ stop = viewport_tuple[2] for site_name, site_url in util.get_gbrowse_sites_by_build(dataset.dbkey): if site_name in app.config.gbrowse_display_sites: - display_url = urllib.quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) + display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s" % (base_url, url_for( controller='root' ), dataset.id, type) ) link = "%sname=%s&ref=%s:%s..%s&eurl=%s" % (site_url, dataset.dbkey, chrom, start, stop, display_url ) ret_val.append( (site_name, link) ) return ret_val diff -r 6e592c876f4d -r f2ce31c74346 templates/root/history_common.mako --- a/templates/root/history_common.mako Thu Oct 23 15:02:23 2008 -0400 +++ b/templates/root/history_common.mako Thu Oct 23 15:19:48 2008 -0400 @@ -67,7 +67,7 @@ %if data.has_data: <a href="${h.url_for( action='display', id=data.id, tofile='yes', toext=data.ext )}" target="_blank">save</a> %for display_app in data.datatype.get_display_types(): - <% display_links = data.datatype.get_display_links( data, display_app, app, request.base + h.url_for() ) %> + <% display_links = data.datatype.get_display_links( data, display_app, app, request.base ) %> %if len( display_links ) > 0: | ${data.datatype.get_display_label(display_app)} %for display_name, display_link in display_links:

1 0

[hg] galaxy 1571: When creating display links, use url_for to ad...
by Nate Coraor 23 Oct '08

23 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/6e592c876f4d changeset: 1571:6e592c876f4d user: Nate Coraor <nate(a)bx.psu.edu> date: Thu Oct 23 15:02:23 2008 -0400 description: When creating display links, use url_for to add the proxy prefix, if any. 1 file(s) affected in this change: templates/root/history_common.mako diffs (12 lines): diff -r 3e391f7192f6 -r 6e592c876f4d templates/root/history_common.mako --- a/templates/root/history_common.mako Wed Oct 22 14:37:28 2008 -0400 +++ b/templates/root/history_common.mako Thu Oct 23 15:02:23 2008 -0400 @@ -67,7 +67,7 @@ %if data.has_data: <a href="${h.url_for( action='display', id=data.id, tofile='yes', toext=data.ext )}" target="_blank">save</a> %for display_app in data.datatype.get_display_types(): - <% display_links = data.datatype.get_display_links( data, display_app, app, request.base ) %> + <% display_links = data.datatype.get_display_links( data, display_app, app, request.base + h.url_for() ) %> %if len( display_links ) > 0: | ${data.datatype.get_display_label(display_app)} %for display_name, display_link in display_links:

1 0

[hg] galaxy 1568: Add a new metadata type of Metadata Files.
by Nate Coraor 23 Oct '08

23 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/0b022adfdc34 changeset: 1568:0b022adfdc34 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Wed Oct 22 13:49:22 2008 -0400 description: Add a new metadata type of Metadata Files. These are now used to store the list of chromosomes for species as well as the index for MAF files. MAF tools have been enhanced to make use of index files when available. TODO: When datasets are purged from disk, these files should also be purged. 23 file(s) affected in this change: lib/galaxy/datatypes/data.py lib/galaxy/datatypes/metadata.py lib/galaxy/datatypes/sequence.py lib/galaxy/jobs/__init__.py lib/galaxy/model/__init__.py lib/galaxy/model/mapping.py lib/galaxy/tools/actions/upload.py lib/galaxy/tools/parameters/__init__.py lib/galaxy/tools/parameters/basic.py lib/galaxy/tools/parameters/validation.py lib/galaxy/tools/util/maf_utilities.py lib/galaxy/util/__init__.py templates/dataset/edit_attributes.mako tools/data_source/data_source.py tools/maf/genebed_maf_to_fasta.xml tools/maf/interval2maf.py tools/maf/interval2maf.xml tools/maf/interval_maf_to_merged_fasta.py tools/maf/interval_maf_to_merged_fasta.xml tools/maf/maf_stats.py tools/maf/maf_stats.xml tools/stats/filtering.py tools/visualization/GMAJ.xml diffs (672 lines): diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/datatypes/data.py Wed Oct 22 13:49:22 2008 -0400 @@ -9,7 +9,6 @@ # Valid first column and strand column values vor bed, other formats col1_startswith = ['chr', 'chl', 'groupun', 'reftig_', 'scaffold', 'super_', 'vcho'] valid_strand = ['+', '-', '.'] -gzip_magic = '\037\213' class DataMeta( type ): """ @@ -86,10 +85,19 @@ def set_readonly_meta( self, dataset ): """Unimplemented method, resets the readonly metadata values""" return True - def missing_meta( self, dataset ): - """Checks for empty metadata values, Returns True if non-optional metadata is missing""" - for key, value in dataset.metadata.items(): - if dataset.metadata.spec[key].get("optional"): continue #we skip check for optional values here + def missing_meta( self, dataset, check = [], skip = [] ): + """ + Checks for empty metadata values, Returns True if non-optional metadata is missing + Specifying a list of 'check' values will only check those names provided; when used, optionality is ignored + Specifying a list of 'skip' items will return True even when a named metadata value is missing + """ + if check: + to_check = [ ( to_check, dataset.metadata.get( to_check ) ) for to_check in check ] + else: + to_check = dataset.metadata.items() + for key, value in to_check: + if key in skip or ( not check and dataset.metadata.spec[key].get( "optional" ) ): + continue #we skip check for optional and nonrequested values here if not value: return True return False @@ -328,7 +336,7 @@ line = line[ :WIDTH ] if not data_checked and line: data_checked = True - if line[0:2] == gzip_magic: + if line[0:2] == util.gzip_magic: file_type = 'gzipped' break else: diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/datatypes/metadata.py Wed Oct 22 13:49:22 2008 -0400 @@ -1,8 +1,9 @@ -import sys, logging +import sys, logging, copy, shutil from galaxy.util import string_as_bool from galaxy.util.odict import odict from galaxy.web import form_builder +import galaxy.model log = logging.getLogger( __name__ ) @@ -75,7 +76,13 @@ def get_html_by_name( self, name, **kwd ): if name in self.spec: return self.spec[name].param.get_html( value=getattr( self, name ), context=self, **kwd ) - + def make_dict_copy( self, to_copy ): + """Makes a deep copy of input iterable to_copy according to self.spec""" + rval = {} + for key, value in to_copy.items(): + if key in self.spec: + rval[key] = self.spec[key].param.make_copy( value, target_context=self, source_context=to_copy ) + return rval class MetadataSpecCollection( odict ): """ @@ -121,7 +128,10 @@ def to_string( self, value ): return str( value ) - + + def make_copy( self, value, target_context = None, source_context = None ): + return copy.deepcopy( value ) + @classmethod def marshal ( cls, value ): """ @@ -150,7 +160,6 @@ Turns a value into its usable form. """ return value - class MetadataElementSpec( object ): """ @@ -280,16 +289,14 @@ return ",".join( map( str, value ) ) class PythonObjectParameter( MetadataParameter ): - def __init__( self, spec ): - MetadataParameter.__init__( self, spec ) def to_string( self, value ): if not value: - return self.spec.to_string( self.spec.no_value ) - return self.spec.to_string( value ) + return self.spec._to_string( self.spec.no_value ) + return self.spec._to_string( value ) def get_html_field( self, value=None, context={}, other_values={}, **kwd ): - return form_builder.TextField( self.spec.name, value=self.to_string( value ) ) + return form_builder.TextField( self.spec.name, value=self._to_string( value ) ) def get_html( self, value=None, context={}, other_values={}, **kwd ): return str( self ) @@ -297,3 +304,40 @@ @classmethod def marshal( cls, value ): return value + +class FileParameter( MetadataParameter ): + + def to_string( self, value ): + if not value: + return str( self.spec.no_value ) + return value.file_name + + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): + return form_builder.TextField( self.spec.name, value=str( value.id ) ) + + def get_html( self, value=None, context={}, other_values={}, **kwd ): + return "<div>No display available for Metadata Files</div>" + + def wrap( self, value ): + if isinstance( value, galaxy.model.MetadataFile ): + return value + try: + return galaxy.model.MetadataFile.get( value ) + except: + #value was not a valid id + return None + + def make_copy( self, value, target_context = None, source_context = None ): + value = self.wrap( value ) + if value: + new_value = galaxy.model.MetadataFile( dataset = target_context.parent, name = self.spec.name ) + new_value.flush() + shutil.copy( value.file_name, new_value.file_name ) + return self.unwrap( new_value ) + return None + + @classmethod + def marshal( cls, value ): + if isinstance( value, galaxy.model.MetadataFile ): + value = value.id + return value diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/datatypes/sequence.py --- a/lib/galaxy/datatypes/sequence.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/datatypes/sequence.py Wed Oct 22 13:49:22 2008 -0400 @@ -8,6 +8,7 @@ from cgi import escape from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes import metadata +import galaxy.model from galaxy import util from sniff import * @@ -24,7 +25,6 @@ """Add metadata elements""" MetadataElement( name="species", desc="Species", default=[], param=metadata.SelectParameter, multiple=True, readonly=True, no_value=None ) - MetadataElement( name="species_chromosomes", desc="Species Chromosomes", value={}, param=metadata.PythonObjectParameter, readonly=True, no_value={}, to_string=str, visible=False ) class Fasta( Sequence ): """Class representing a FASTA sequence""" @@ -192,23 +192,30 @@ class Maf( Alignment ): """Class describing a Maf alignment""" file_ext = "maf" + + #Readonly and optional, users can't unset it, but if it is not set, we are generally ok; if required use a metadata validator in the tool definition + MetadataElement( name="species_chromosomes", desc="Species Chromosomes", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) + MetadataElement( name="maf_index", desc="MAF Index File", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) def init_meta( self, dataset, copy_from=None ): Alignment.init_meta( self, dataset, copy_from=copy_from ) def set_meta( self, dataset, overwrite = True, **kwd ): """ - Parses and sets species and chromosomes from MAF files. + Parses and sets species, chromosomes, index from MAF file. """ + #these metadata values are not accessable by users, always overwrite + species = [] species_chromosomes = {} + maf_reader = bx.align.maf.Reader( open( dataset.file_name ) ) + indexes = bx.interval_index_file.Indexes() try: - for i, m in enumerate( bx.align.maf.Reader( open(dataset.file_name) ) ): - for c in m.components: - ## spec,chrom = bx.align.maf.src_split( c.src ) - ## if not spec or not chrom: spec = chrom = c.src - # "src_split" finds the rightmost dot, which is probably - # wrong in general, and certainly here. + while True: + pos = maf_reader.file.tell() + block = maf_reader.next() + if block is None: break + for c in block.components: spec = c.src chrom = None if "." in spec: @@ -218,20 +225,44 @@ species_chromosomes[spec] = [] if chrom and chrom not in species_chromosomes[spec]: species_chromosomes[spec].append( chrom ) - # only check first 100,000 blocks for species - if i > 100000: break - except: + indexes.add( c.src, c.forward_strand_start, c.forward_strand_end, pos, max=c.src_size ) + except: #bad MAF file pass - #these metadata values are not accessable by users, always overwrite dataset.metadata.species = species - dataset.metadata.species_chromosomes = species_chromosomes + #only overwrite the contents if our newly determined chromosomes don't match stored + chrom_file = dataset.metadata.species_chromosomes + compare_chroms = {} + if chrom_file: + try: + for line in open( chrom_file.file_name ): + fields = line.split( "\t" ) + if fields: + spec = fields.pop( 0 ) + if spec: + compare_chroms[spec] = fields + except: + pass + #write out species chromosomes again only if values are different + if not species_chromosomes or compare_chroms != species_chromosomes: + tmp_file = tempfile.TemporaryFile( 'w+b' ) + for spec, chroms in species_chromosomes.items(): + tmp_file.write( "%s\t%s\n" % ( spec, "\t".join( chroms ) ) ) + + if not chrom_file: + chrom_file = galaxy.model.MetadataFile( dataset = dataset, name = "species_chromosomes" ) + chrom_file.flush() + tmp_file.seek( 0 ) + open( chrom_file.file_name, 'wb' ).write( tmp_file.read() ) + dataset.metadata.species_chromosomes = chrom_file + tmp_file.close() + + index_file = dataset.metadata.maf_index + if not index_file: + index_file = galaxy.model.MetadataFile( dataset = dataset, name="maf_index" ) + index_file.flush() + indexes.write( open( index_file.file_name, 'w' ) ) + dataset.metadata.maf_index = index_file - def missing_meta( self, dataset ): - """Checks to see if species is set""" - if dataset.metadata.species in [None, []]: - return True - return False - def display_peek( self, dataset ): """Returns formated html of peek""" return self.make_html_table( dataset ) diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/jobs/__init__.py Wed Oct 22 13:49:22 2008 -0400 @@ -475,7 +475,16 @@ def get_input_fnames( self ): job = model.Job.get( self.job_id ) - return [ da.dataset.file_name for da in job.input_datasets if da.dataset ] + filenames = [] + for da in job.input_datasets: #da is JobToInputDatasetAssociation object + if da.dataset: + filenames.append( da.dataset.file_name ) + #we will need to stage in metadata file names also + #TODO: would be better to only stage in metadata files that are actually needed (found in command line, referenced in config files, etc.) + for key, value in da.dataset.metadata.items(): + if isinstance( value, model.MetadataFile ): + filenames.append( value.file_name ) + return filenames def get_output_fnames( self ): job = model.Job.get( self.job_id ) diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/model/__init__.py Wed Oct 22 13:49:22 2008 -0400 @@ -5,7 +5,7 @@ the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -import os.path, os, errno, copy +import os.path, os, errno import sha import galaxy.datatypes from galaxy.util.bunch import Bunch @@ -165,7 +165,7 @@ return self._metadata_collection def set_metadata( self, bunch ): # Needs to accept a MetadataCollection, a bunch, or a dict - self._metadata = dict( [ ( key, copy.deepcopy( value ) ) for key, value in bunch.items() ] ) + self._metadata = self.metadata.make_dict_copy( bunch ) metadata = property( get_metadata, set_metadata ) """ @@ -223,8 +223,8 @@ return self.datatype.set_meta( self, **kwd ) def set_readonly_meta( self, **kwd ): return self.datatype.set_readonly_meta( self, **kwd ) - def missing_meta( self ): - return self.datatype.missing_meta( self ) + def missing_meta( self, **kwd ): + return self.datatype.missing_meta( self, **kwd ) def as_display_type( self, type, **kwd ): return self.datatype.as_display_type( self, type, **kwd ) def display_peek( self ): @@ -258,8 +258,9 @@ return self.datatype.find_conversion_destination( self, accepted_formats, datatypes_registry, **kwd ) def copy( self, copy_children = False, parent_id = None ): - des = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, blurb=self.blurb, peek=self.peek, extension=self.extension, dbkey=self.dbkey, metadata=self._metadata, dataset = self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id, copied_from_history_dataset_association = self ) + des = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, blurb=self.blurb, peek=self.peek, extension=self.extension, dbkey=self.dbkey, dataset = self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id, copied_from_history_dataset_association = self ) des.flush() + des.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id if copy_children: for child in self.children: child_copy = child.copy( copy_children = copy_children, parent_id = des.id ) @@ -564,6 +565,24 @@ self.user = None self.order_index = None +class MetadataFile( object ): + def __init__( self, dataset = None, name = None ): + self.dataset = dataset + self.name = name + @property + def file_name( self ): + assert self.id is not None, "ID must be set before filename used (commit the object)" + path = os.path.join( Dataset.file_path, '_metadata_files', *directory_hash_id( self.id ) ) + # Create directory if it does not exist + try: + os.makedirs( path ) + except OSError, e: + # File Exists is okay, otherwise reraise + if e.errno != errno.EEXIST: + raise + # Return filename inside hashed directory + return os.path.abspath( os.path.join( path, "metadata_%d.dat" % self.id ) ) + ## ---- Utility methods ------------------------------------------------------- def directory_hash_id( id ): diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/model/mapping.py Wed Oct 22 13:49:22 2008 -0400 @@ -238,6 +238,15 @@ Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "order_index", Integer ) ) +MetadataFile.table = Table( "metadata_file", metadata, + Column( "id", Integer, primary_key=True ), + Column( "name", String ), + Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "purged", Boolean, index=True, default=False ) ) + # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -363,6 +372,9 @@ assign_mapper( context, StoredWorkflowMenuEntry, StoredWorkflowMenuEntry.table, properties=dict( stored_workflow=relation( StoredWorkflow ) ) ) +assign_mapper( context, MetadataFile, MetadataFile.table, + properties=dict( dataset=relation( HistoryDatasetAssociation ) ) ) + def db_next_hid( self ): """ Override __next_hid to generate from the database in a concurrency diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/tools/actions/upload.py Wed Oct 22 13:49:22 2008 -0400 @@ -196,7 +196,7 @@ temp = open( temp_name, "U" ) magic_check = temp.read( 2 ) temp.close() - if magic_check != datatypes.data.gzip_magic: + if magic_check != util.gzip_magic: return ( False, False ) CHUNK_SIZE = 2**15 # 32Kb gzipped_file = gzip.GzipFile( temp_name ) diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/tools/parameters/__init__.py --- a/lib/galaxy/tools/parameters/__init__.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/tools/parameters/__init__.py Wed Oct 22 13:49:22 2008 -0400 @@ -60,4 +60,4 @@ if key in params: value = params[key].value_from_basic( value, app, ignore_errors ) rval[ key ] = value - return rval \ No newline at end of file + return rval diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/tools/parameters/basic.py Wed Oct 22 13:49:22 2008 -0400 @@ -829,6 +829,8 @@ options = [] for filter_key, filter_value in self.filtered.iteritems(): dataset = other_values[filter_key] + if dataset.__class__.__name__.endswith( "DatasetFilenameWrapper" ): #this is a bad way to check for this, but problems importing class ( due to circular imports? ) + dataset = dataset.dataset if dataset: for meta_key, meta_dict in filter_value.iteritems(): if dataset.metadata.spec[meta_key].param.to_string( dataset.metadata.get( meta_key ) ) == meta_dict['value']: diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/tools/parameters/validation.py --- a/lib/galaxy/tools/parameters/validation.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/tools/parameters/validation.py Wed Oct 22 13:49:22 2008 -0400 @@ -163,13 +163,15 @@ """ Validator that checks for missing metadata """ - def __init__( self, message=None ): + def __init__( self, message = None, check = "", skip = "" ): self.message = message + self.check = check.split( "," ) + self.skip = skip.split( "," ) @classmethod def from_element( cls, param, elem ): - return cls( elem.get( 'message', None ) ) + return cls( message=elem.get( 'message', None ), check=elem.get( 'check', "" ), skip=elem.get( 'skip', "" ) ) def validate( self, value, history=None ): - if value and value.missing_meta(): + if value and value.missing_meta( check = self.check, skip = self.skip ): if self.message is None: self.message = "Metadata missing, click the pencil icon in the history item to edit / save the metadata attributes" raise ValueError( self.message ) diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/tools/util/maf_utilities.py --- a/lib/galaxy/tools/util/maf_utilities.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/tools/util/maf_utilities.py Wed Oct 22 13:49:22 2008 -0400 @@ -145,8 +145,16 @@ except: pass return None + +#return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created +def open_or_build_maf_index( maf_file, index_filename, species = None ): + try: + return ( bx.align.maf.Indexed( maf_file, index_filename = index_filename, keep_open = True, parse_e_rows = False ), None ) + except: + return build_maf_index( maf_file, species = species ) + -#builds and returns (index, index_filename) for specified maf_file +#builds and returns ( index, index_filename ) for specified maf_file def build_maf_index( maf_file, species = None ): indexes = bx.interval_index_file.Indexes() try: diff -r b02b8d9196a8 -r 0b022adfdc34 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py Wed Oct 22 12:15:06 2008 -0400 +++ b/lib/galaxy/util/__init__.py Wed Oct 22 13:49:22 2008 -0400 @@ -16,6 +16,8 @@ log = logging.getLogger(__name__) _lock = threading.RLock() + +gzip_magic = '\037\213' def synchronized(func): """This wrapper will serialize access to 'func' to a single thread. Use it as a decorator.""" diff -r b02b8d9196a8 -r 0b022adfdc34 templates/dataset/edit_attributes.mako --- a/templates/dataset/edit_attributes.mako Wed Oct 22 12:15:06 2008 -0400 +++ b/templates/dataset/edit_attributes.mako Wed Oct 22 13:49:22 2008 -0400 @@ -65,6 +65,9 @@ if they are not accurate. </div> </form> + %if data.missing_meta(): + <div class="errormessagesmall">Required metadata values are missing. Some of these values may not be editable by the user. Selecting "Auto-detect" will attempt to fix these values.</div> + %endif </div> </div> diff -r b02b8d9196a8 -r 0b022adfdc34 tools/data_source/data_source.py --- a/tools/data_source/data_source.py Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/data_source/data_source.py Wed Oct 22 13:49:22 2008 -0400 @@ -2,7 +2,8 @@ #Retreives data from UCSC and stores in a file. UCSC parameters are provided in the input/output file. import urllib, sys, os, gzip, tempfile, shutil from galaxy import eggs -from galaxy.datatypes import data +#from galaxy.datatypes import data +from galaxy.util import gzip_magic assert sys.version_info[:2] >= ( 2, 4 ) @@ -14,7 +15,7 @@ temp = open( filename, "U" ) magic_check = temp.read( 2 ) temp.close() - if magic_check != data.gzip_magic: + if magic_check != gzip_magic: return False return True diff -r b02b8d9196a8 -r 0b022adfdc34 tools/maf/genebed_maf_to_fasta.xml --- a/tools/maf/genebed_maf_to_fasta.xml Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/maf/genebed_maf_to_fasta.xml Wed Oct 22 13:49:22 2008 -0400 @@ -1,6 +1,6 @@ <tool id="GeneBed_Maf_Fasta2" name="Stitch Gene blocks"> <description>given a set of coding exon intervals</description> - <command interpreter="python">#if $maf_source_type.maf_source == "user":#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_file --interval_file=$input1 --output_file=$out_file1 --mafSourceType=$maf_source_type.maf_source --geneBED --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} + <command interpreter="python">#if $maf_source_type.maf_source == "user":#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_file --mafIndex=$maf_source_type.maf_file.metadata.maf_index --interval_file=$input1 --output_file=$out_file1 --mafSourceType=$maf_source_type.maf_source --geneBED --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} #else:#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_identifier --interval_file=$input1 --output_file=$out_file1 --mafSourceType=$maf_source_type.maf_source --geneBED --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} #end if </command> diff -r b02b8d9196a8 -r 0b022adfdc34 tools/maf/interval2maf.py --- a/tools/maf/interval2maf.py Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/maf/interval2maf.py Wed Oct 22 13:49:22 2008 -0400 @@ -16,6 +16,7 @@ -S, --strandCol=S: Column of Strand -t, --mafType=t: Type of MAF source to use -m, --mafFile=m: Path of source MAF file, if not using cached version + -I, --mafIndex=I: Path of precomputed source MAF file index, if not using cached version -i, --interval_file=i: Input interval file -o, --output_file=o: Output MAF file -p, --species=p: Species to include in output @@ -92,7 +93,7 @@ print >> sys.stderr, "The MAF source specified (%s) appears to be invalid." % ( options.mafType ) sys.exit() elif options.mafFile: - index, index_filename = maf_utilities.build_maf_index( options.mafFile, species = [dbkey] ) + index, index_filename = maf_utilities.open_or_build_maf_index( options.mafFile, options.mafIndex, species = [dbkey] ) if index is None: print >> sys.stderr, "Your MAF file appears to be malformed." sys.exit() diff -r b02b8d9196a8 -r 0b022adfdc34 tools/maf/interval2maf.xml --- a/tools/maf/interval2maf.xml Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/maf/interval2maf.xml Wed Oct 22 13:49:22 2008 -0400 @@ -1,7 +1,7 @@ <tool id="Interval2Maf1" name="Extract MAF blocks"> <description>given a set of genomic intervals</description> <command interpreter="python"> - #if $maf_source_type.maf_source == "user":#interval2maf.py --dbkey=${input1.dbkey} --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafFile=$maf_source_type.mafFile --interval_file=$input1 --output_file=$out_file1 --mafIndexFile=${GALAXY_DATA_INDEX_DIR}/maf_index.loc + #if $maf_source_type.maf_source == "user":#interval2maf.py --dbkey=${input1.dbkey} --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafFile=$maf_source_type.mafFile --mafIndex=$maf_source_type.mafFile.metadata.maf_index --interval_file=$input1 --output_file=$out_file1 --mafIndexFile=${GALAXY_DATA_INDEX_DIR}/maf_index.loc #else:#interval2maf.py --dbkey=${input1.dbkey} --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafType=$maf_source_type.mafType --interval_file=$input1 --output_file=$out_file1 --mafIndexFile=${GALAXY_DATA_INDEX_DIR}/maf_index.loc #end if </command> diff -r b02b8d9196a8 -r 0b022adfdc34 tools/maf/interval_maf_to_merged_fasta.py --- a/tools/maf/interval_maf_to_merged_fasta.py Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/maf/interval_maf_to_merged_fasta.py Wed Oct 22 13:49:22 2008 -0400 @@ -15,6 +15,7 @@ -G, --geneBED: Input is a Gene BED file, process and join exons as one region -t, --mafSourceType=t: Type of MAF source to use -m, --mafSource=m: Path of source MAF file, if not using cached version + -I, --mafIndex=I: Path of precomputed source MAF file index, if not using cached version -i, --interval_file=i: Input interval file -o, --output_file=o: Output MAF file -p, --species=p: Species to include in output @@ -105,7 +106,7 @@ stop_err( "The MAF source specified (%s) appears to be invalid." % ( options.mafSource ) ) elif options.mafSourceType.lower() in ["user"]: #index maf for use here, need to remove index_file when finished - index, index_filename = maf_utilities.build_maf_index( options.mafSource, species = [primary_species] ) + index, index_filename = maf_utilities.open_or_build_maf_index( options.mafSource, options.mafIndex, species = [primary_species] ) if index is None: stop_err( "Your MAF file appears to be malformed." ) else: diff -r b02b8d9196a8 -r 0b022adfdc34 tools/maf/interval_maf_to_merged_fasta.xml --- a/tools/maf/interval_maf_to_merged_fasta.xml Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/maf/interval_maf_to_merged_fasta.xml Wed Oct 22 13:49:22 2008 -0400 @@ -1,6 +1,6 @@ <tool id="Interval_Maf_Merged_Fasta2" name="Stitch MAF blocks"> <description>given a set of genomic intervals</description> - <command interpreter="python">#if $maf_source_type.maf_source == "user":#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_file --interval_file=$input1 --output_file=$out_file1 --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafSourceType=$maf_source_type.maf_source --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} + <command interpreter="python">#if $maf_source_type.maf_source == "user":#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_file --mafIndex=$maf_source_type.maf_file.metadata.maf_index --interval_file=$input1 --output_file=$out_file1 --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafSourceType=$maf_source_type.maf_source --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} #else:#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_identifier --interval_file=$input1 --output_file=$out_file1 --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafSourceType=$maf_source_type.maf_source --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} #end if </command> diff -r b02b8d9196a8 -r 0b022adfdc34 tools/maf/maf_stats.py --- a/tools/maf/maf_stats.py Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/maf/maf_stats.py Wed Oct 22 13:49:22 2008 -0400 @@ -31,10 +31,14 @@ else: summary = False mafIndexFile = "%s/maf_index.loc" % sys.argv[9] + try: + maf_index_filename = sys.argv[10].strip() + except: + maf_index_filename = None index = index_filename = None if maf_source_type == "user": #index maf for use here - index, index_filename = maf_utilities.build_maf_index( input_maf_filename, species = [dbkey] ) + index, index_filename = maf_utilities.open_or_build_maf_index( input_maf_filename, maf_index_filename, species = [dbkey] ) if index is None: print >>sys.stderr, "Your MAF file appears to be malformed." sys.exit() diff -r b02b8d9196a8 -r 0b022adfdc34 tools/maf/maf_stats.xml --- a/tools/maf/maf_stats.xml Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/maf/maf_stats.xml Wed Oct 22 13:49:22 2008 -0400 @@ -7,7 +7,10 @@ #else: $maf_source_type.maf_source $maf_source_type.mafType $input1 $out_file1 $dbkey ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} $summary #end if - ${GALAXY_DATA_INDEX_DIR} + ${GALAXY_DATA_INDEX_DIR} + #if $maf_source_type.maf_source == "user": + $input2.metadata.maf_index + #end if </command> <inputs> <param format="interval" name="input1" label="Interval File" type="data"> diff -r b02b8d9196a8 -r 0b022adfdc34 tools/stats/filtering.py --- a/tools/stats/filtering.py Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/stats/filtering.py Wed Oct 22 13:49:22 2008 -0400 @@ -4,7 +4,6 @@ import sys, sets, re, os.path from galaxy import eggs -from galaxy.datatypes import metadata assert sys.version_info[:2] >= ( 2, 4 ) diff -r b02b8d9196a8 -r 0b022adfdc34 tools/visualization/GMAJ.xml --- a/tools/visualization/GMAJ.xml Wed Oct 22 12:15:06 2008 -0400 +++ b/tools/visualization/GMAJ.xml Wed Oct 22 13:49:22 2008 -0400 @@ -2,7 +2,9 @@ <description>Multiple Alignment Viewer</description> <command interpreter="python">GMAJ.py $out_file1 $maf_input $gmaj_file $filenames_file</command> <inputs> - <param name="maf_input" type="data" format="maf" label="Alignment File" optional="False"/> + <param name="maf_input" type="data" format="maf" label="Alignment File" optional="False"> + <validator type="metadata" check="species_chromosomes" message="Metadata missing, click the pencil icon in the history item and use the auto-detect feature to correct this issue."/> + </param> <param name="refseq" label="Reference Sequence" type="select"> <option value="first" selected="true">First sequence in each block</option> <option value="any">Any sequence</option> @@ -103,9 +105,19 @@ #set $seq_count = 0 #for $annotation_count, $annotation in $enumerate( $annotations ): -#if $annotation.annotation_style.style == "galaxy": -#if $maf_input.dataset.metadata.species_chromosomes and $annotation.annotation_style['species'].value in $maf_input.dataset.metadata.species_chromosomes and $maf_input.dataset.metadata.species_chromosomes[$annotation.annotation_style['species'].value]: -#set $seq_names = [ "%s.%s" % ( $annotation.annotation_style['species'].value, $chrom ) for $chrom in $maf_input.dataset.metadata.species_chromosomes[$annotation.annotation_style['species'].value]] +#if $annotation.annotation_style.style == "galaxy": +#set $species_chromosomes = {} +#if $maf_input.dataset.metadata.species_chromosomes: +#for $line in open( $maf_input.dataset.metadata.species_chromosomes.file_name ): +#set $fields = $line.split( "\t" ) +#if $fields: +#set $spec = $fields.pop( 0 ) +#set $species_chromosomes[spec] = $fields +#end if +#end for +#end if +#if $species_chromosomes and $annotation.annotation_style['species'].value in $species_chromosomes and $species_chromosomes[$annotation.annotation_style['species'].value]: +#set $seq_names = [ "%s.%s" % ( $annotation.annotation_style['species'].value, $chrom ) for $chrom in $species_chromosomes[$annotation.annotation_style['species'].value]] #else: #set $seq_names = [$annotation.annotation_style['species']] #end if @@ -171,4 +183,4 @@ Gmaj is a tool for viewing and manipulating Generalized Multiple Alignments (GMAs) produced by programs such as TBA (though it can also be used with maf-format alignments from other sources). It can display interactive graphical and text representations of the alignments, a diagram showing the locations of exons and repeats, and other annotations -- all with the user's choice of reference sequence. </help> -</tool> \ No newline at end of file +</tool>

1 0

[hg] galaxy 1570: Record a useful info message when a job fails ...
by Nate Coraor 23 Oct '08

23 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/3e391f7192f6 changeset: 1570:3e391f7192f6 user: Nate Coraor <nate(a)bx.psu.edu> date: Wed Oct 22 14:37:28 2008 -0400 description: Record a useful info message when a job fails to run because inputs are deleted. And clarify log messages. 1 file(s) affected in this change: lib/galaxy/jobs/__init__.py diffs (66 lines): diff -r df07d0c2830f -r 3e391f7192f6 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Wed Oct 22 14:27:57 2008 -0400 +++ b/lib/galaxy/jobs/__init__.py Wed Oct 22 14:37:28 2008 -0400 @@ -15,7 +15,7 @@ log = logging.getLogger( __name__ ) # States for running a job. These are NOT the same as data states -JOB_WAIT, JOB_ERROR, JOB_OK, JOB_READY, JOB_DELETED = 'wait', 'error', 'ok', 'ready', 'deleted' +JOB_WAIT, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_OK, JOB_READY, JOB_DELETED = 'wait', 'input_error', 'input_deleted', 'ok', 'ready', 'deleted' class Sleeper( object ): """ @@ -163,8 +163,10 @@ if job_state == JOB_WAIT: if not self.track_jobs_in_database: new_waiting.append( job ) - elif job_state == JOB_ERROR: - log.info( "job %d ended with an error" % job.job_id ) + elif job_state == JOB_INPUT_ERROR: + log.info( "job %d unable to run: one or more inputs in error state" % job.job_id ) + elif job_state == JOB_INPUT_DELETED: + log.info( "job %d unable to run: one or more inputs deleted" % job.job_id ) elif job_state == JOB_READY: # If special queuing is enabled, put the ready jobs in the special queue if self.use_policy : @@ -320,6 +322,7 @@ dataset.flush() job.state = model.Job.states.ERROR job.command_line = self.command_line + job.info = message # If the failure is due to a Galaxy framework exception, save # the traceback if exception: @@ -356,12 +359,13 @@ def check_if_ready_to_run( self ): """ - Check if a job is ready to run by verifying that each of its input + Check if a job is ready to run by verifying that each of its input datasets is ready (specifically in the OK state). If any input dataset - has an error, fail the job and return JOB_ERROR. If all input datasets - are in OK state, return JOB_READY indicating that the job can be - dispatched. Otherwise, return JOB_WAIT indicating that input datasets - are still being prepared. + has an error, fail the job and return JOB_INPUT_ERROR. If any input + dataset is deleted, fail the job and return JOB_INPUT_DELETED. If all + input datasets are in OK state, return JOB_READY indicating that the + job can be dispatched. Otherwise, return JOB_WAIT indicating that input + datasets are still being prepared. """ job = model.Job.get( self.job_id ) job.refresh() @@ -373,11 +377,11 @@ # don't run jobs for which the input dataset was deleted if idata.deleted == True: self.fail( "input data %d was deleted before this job ran" % idata.hid ) - return JOB_ERROR + return JOB_INPUT_DELETED # an error in the input data causes us to bail immediately elif idata.state == idata.states.ERROR: self.fail( "error in input data %d" % idata.hid ) - return JOB_ERROR + return JOB_INPUT_ERROR elif idata.state != idata.states.OK: # need to requeue return JOB_WAIT

1 0

[hg] galaxy 1569: Fix for mapping unit test. Testing of metadat...
by Nate Coraor 23 Oct '08

23 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/df07d0c2830f changeset: 1569:df07d0c2830f user: Dan Blankenberg <dan(a)bx.psu.edu> date: Wed Oct 22 14:27:57 2008 -0400 description: Fix for mapping unit test. Testing of metadata elements requires a valid datatype. 1 file(s) affected in this change: lib/galaxy/model/mapping_tests.py diffs (12 lines): diff -r 0b022adfdc34 -r df07d0c2830f lib/galaxy/model/mapping_tests.py --- a/lib/galaxy/model/mapping_tests.py Wed Oct 22 13:49:22 2008 -0400 +++ b/lib/galaxy/model/mapping_tests.py Wed Oct 22 14:27:57 2008 -0400 @@ -16,7 +16,7 @@ #h1.queries.append( model.Query( "h1->q2" ) ) h2 = model.History( name=( "H" * 1024 ) ) #q1 = model.Query( "h2->q1" ) - d1 = model.HistoryDatasetAssociation( metadata=dict(chromCol=1,startCol=2,endCol=3 ), history=h2, create_dataset=True ) + d1 = model.HistoryDatasetAssociation( extension="interval", metadata=dict(chromCol=1,startCol=2,endCol=3 ), history=h2, create_dataset=True ) #h2.queries.append( q1 ) #h2.queries.append( model.Query( "h2->q2" ) ) model.context.current.flush()

1 0

[hg] galaxy 1567: Like the local runner, if job preparation fail...
by Nate Coraor 23 Oct '08

23 Oct '08

details: http://www.bx.psu.edu/hg/galaxy/rev/b02b8d9196a8 changeset: 1567:b02b8d9196a8 user: Nate Coraor <nate(a)bx.psu.edu> date: Wed Oct 22 12:15:06 2008 -0400 description: Like the local runner, if job preparation fails, catch and log. 2 file(s) affected in this change: lib/galaxy/jobs/runners/pbs.py lib/galaxy/jobs/runners/sge.py diffs (42 lines): diff -r 510ff36caaf3 -r b02b8d9196a8 lib/galaxy/jobs/runners/pbs.py --- a/lib/galaxy/jobs/runners/pbs.py Tue Oct 21 10:40:08 2008 -0400 +++ b/lib/galaxy/jobs/runners/pbs.py Wed Oct 22 12:15:06 2008 -0400 @@ -114,8 +114,15 @@ def queue_job( self, job_wrapper ): """Create PBS script for a job and submit it to the PBS queue""" - job_wrapper.prepare() - command_line = job_wrapper.get_command_line() + + try: + job_wrapper.prepare() + command_line = job_wrapper.get_command_line() + except: + job_wrapper.fail( "failure preparing job", exception=True ) + log.exception("failure running job %d" % job_wrapper.job_id) + return + runner_url = job_wrapper.tool.job_runner # This is silly, why would we queue a job with no command line? diff -r 510ff36caaf3 -r b02b8d9196a8 lib/galaxy/jobs/runners/sge.py --- a/lib/galaxy/jobs/runners/sge.py Tue Oct 21 10:40:08 2008 -0400 +++ b/lib/galaxy/jobs/runners/sge.py Wed Oct 22 12:15:06 2008 -0400 @@ -103,8 +103,15 @@ def queue_job( self, job_wrapper ): """Create SGE script for a job and submit it to the SGE queue""" - job_wrapper.prepare() - command_line = job_wrapper.get_command_line() + + try: + job_wrapper.prepare() + command_line = job_wrapper.get_command_line() + except: + job_wrapper.fail( "failure preparing job", exception=True ) + log.exception("failure running job %d" % job_wrapper.job_id) + return + runner_url = job_wrapper.tool.job_runner # This is silly, why would we queue a job with no command line?

1 0