- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 1648: Let UCSC through w/o authentication when use_r...
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/682c179a2f84 changeset: 1648:682c179a2f84 user: Nate Coraor <nate(a)bx.psu.edu> date: Thu Dec 04 10:47:41 2008 -0500 description: Let UCSC through w/o authentication when use_remote_user is enabled. 2 file(s) affected in this change: lib/galaxy/web/buildapp.py lib/galaxy/web/framework/middleware/remoteuser.py diffs (60 lines): diff -r 15bf910890d5 -r 682c179a2f84 lib/galaxy/web/buildapp.py --- a/lib/galaxy/web/buildapp.py Tue Dec 02 17:17:23 2008 -0500 +++ b/lib/galaxy/web/buildapp.py Thu Dec 04 10:47:41 2008 -0500 @@ -95,7 +95,7 @@ # upstream server if asbool(conf.get( 'use_remote_user', False )): from galaxy.web.framework.middleware.remoteuser import RemoteUser - app = RemoteUser( app, maildomain=conf.get( 'remote_user_maildomain', None ) ) + app = RemoteUser( app, maildomain=conf.get( 'remote_user_maildomain', None ), ucsc_display_sites=conf.get( 'ucsc_display_sites', [] ) ) log.debug( "Enabling 'remote user' middleware" ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. diff -r 15bf910890d5 -r 682c179a2f84 lib/galaxy/web/framework/middleware/remoteuser.py --- a/lib/galaxy/web/framework/middleware/remoteuser.py Tue Dec 02 17:17:23 2008 -0500 +++ b/lib/galaxy/web/framework/middleware/remoteuser.py Thu Dec 04 10:47:41 2008 -0500 @@ -1,6 +1,8 @@ """ Middleware for handling $REMOTE_USER if use_remote_user is enabled. """ + +import socket errorpage = """ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> @@ -33,11 +35,34 @@ </html> """ +UCSC_SERVERS = ( + 'hgw1.cse.ucsc.edu', + 'hgw2.cse.ucsc.edu', + 'hgw3.cse.ucsc.edu', + 'hgw4.cse.ucsc.edu', + 'hgw5.cse.ucsc.edu', + 'hgw6.cse.ucsc.edu', + 'hgw7.cse.ucsc.edu', + 'hgw8.cse.ucsc.edu', +) + class RemoteUser( object ): - def __init__( self, app, maildomain=None ): + def __init__( self, app, maildomain=None, ucsc_display_sites=[] ): self.app = app self.maildomain = maildomain + self.allow_ucsc = False + if len( ucsc_display_sites ): + self.allow_ucsc = True def __call__( self, environ, start_response ): + # Allow through UCSC if the UCSC display links are enabled + if self.allow_ucsc and environ.has_key( 'REMOTE_ADDR' ): + try: + host = socket.gethostbyaddr( environ[ 'REMOTE_ADDR' ] )[0] + except( socket.error, socket.herror, socket.gaierror, socket.timeout ): + # in the event of a lookup failure, deny access + host = None + if host in UCSC_SERVERS: + return self.app( environ, start_response ) # Apache sets REMOTE_USER to the string '(null)' when using the # Rewrite* method for passing REMOTE_USER and a user is # un-authenticated. Any other possible values need to go here as well.

1 0

[hg] galaxy 1644: Bug fixes for compare, Group and filter tools.
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/ab46f22b4624 changeset: 1644:ab46f22b4624 user: guru date: Tue Dec 02 11:41:25 2008 -0500 description: Bug fixes for compare, Group and filter tools. 3 file(s) affected in this change: tools/filters/compare.xml tools/stats/filtering.py tools/stats/grouping.py diffs (51 lines): diff -r 74e392ea6ae8 -r ab46f22b4624 tools/filters/compare.xml --- a/tools/filters/compare.xml Tue Dec 02 10:50:54 2008 -0500 +++ b/tools/filters/compare.xml Tue Dec 02 11:41:25 2008 -0500 @@ -3,9 +3,13 @@ <command interpreter="python">joinWrapper.py $input1 $input2 $field1 $field2 $mode $out_file1</command> <inputs> <param format="tabular" name="input1" type="data" label="Compare"/> - <param name="field1" label="Using column" type="data_column" data_ref="input1" /> + <param name="field1" label="Using column" type="data_column" data_ref="input1"> + <validator type="no_options" message="Invalid column choice. Please try again after editing metadata of your input dataset by clicking on the pencil icon next to it."/> + </param> <param format="tabular" name="input2" type="data" label="against" /> - <param name="field2" label="and column" type="data_column" data_ref="input2" /> + <param name="field2" label="and column" type="data_column" data_ref="input2"> + <validator type="no_options" message="Invalid column choice. Please try again after editing metadata of your input dataset by clicking on the pencil icon next to it."/> + </param> <param name="mode" type="select" label="To find" help="See examples below for explanation of these options"> <option value="N">Matching rows of 1st query</option> <option value="V">Non Matching rows of 1st query</option> diff -r 74e392ea6ae8 -r ab46f22b4624 tools/stats/filtering.py --- a/tools/stats/filtering.py Tue Dec 02 10:50:54 2008 -0500 +++ b/tools/stats/filtering.py Tue Dec 02 11:41:25 2008 -0500 @@ -25,6 +25,7 @@ cond_text = sys.argv[3] try: in_columns = int( sys.argv[4] ) + assert sys.argv[5] #check to see that the column types varaible isn't null in_column_types = sys.argv[5].split( ',' ) except: stop_err( "Data does not appear to be tabular. This tool can only be used with tab-delimited data." ) diff -r 74e392ea6ae8 -r ab46f22b4624 tools/stats/grouping.py --- a/tools/stats/grouping.py Tue Dec 02 10:50:54 2008 -0500 +++ b/tools/stats/grouping.py Tue Dec 02 11:41:25 2008 -0500 @@ -128,7 +128,7 @@ if op not in ['c','length','unique','random']: for j, elem in enumerate( prev_vals[i] ): prev_vals[i][j] = float( elem ) - rout = "%f" %( eval( rfunc )( prev_vals[i] )) + rout = "%g" %( eval( rfunc )( prev_vals[i] )) if rounds[i] == 'yes': rout = int(round(float(rout))) else: @@ -179,7 +179,7 @@ if op not in ['c','length','unique','random']: for j, elem in enumerate( prev_vals[i] ): prev_vals[i][j] = float( elem ) - rout = '%f' %( eval( rfunc )( prev_vals[i] )) + rout = '%g' %( eval( rfunc )( prev_vals[i] )) if rounds[i] == 'yes': rout = int(round(float(rout))) else:

1 0

[hg] galaxy 1647: allow users to specify that label for input da...
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/15bf910890d5 changeset: 1647:15bf910890d5 user: James Taylor <james(a)jamestaylor.org> date: Tue Dec 02 17:17:23 2008 -0500 description: allow users to specify that label for input dataset modules. 4 file(s) affected in this change: lib/galaxy/web/controllers/workflow.py lib/galaxy/workflow/modules.py templates/workflow/editor_generic_form.mako templates/workflow/editor_tool_form.mako diffs (140 lines): diff -r f242b5a3fedc -r 15bf910890d5 lib/galaxy/web/controllers/workflow.py --- a/lib/galaxy/web/controllers/workflow.py Tue Dec 02 13:49:32 2008 -0500 +++ b/lib/galaxy/web/controllers/workflow.py Tue Dec 02 17:17:23 2008 -0500 @@ -166,7 +166,7 @@ return trans.fill_template( "workflow/editor.mako", workflow_id=id ) @web.json - def editor_tool_form( self, trans, type='tool', tool_id=None, **incoming ): + def editor_form_post( self, trans, type='tool', tool_id=None, **incoming ): """ Accepts a tool state and incoming values, and generates a new tool form and some additional information, packed into a json dictionary. diff -r f242b5a3fedc -r 15bf910890d5 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py Tue Dec 02 13:49:32 2008 -0500 +++ b/lib/galaxy/workflow/modules.py Tue Dec 02 17:17:23 2008 -0500 @@ -5,6 +5,7 @@ from galaxy.tools import DefaultToolState from galaxy.tools.parameters.grouping import Repeat, Conditional from galaxy.util.bunch import Bunch +from galaxy.util.json import from_json_string, to_json_string class WorkflowModule( object ): @@ -78,20 +79,48 @@ class InputDataModule( WorkflowModule ): type = "data_input" name = "Input dataset" - _runtime_inputs = { - 'input' : DataToolParameter( None, Element( "param", name="input", label="Input Dataset", type="data", format="data" ) ) - } - + + @classmethod + def new( Class, trans, tool_id=None ): + module = Class( trans ) + module.state = dict( name="Input Dataset" ) + return module + @classmethod + def from_dict( Class, trans, d ): + module = Class( trans ) + state = from_json_string( d["tool_state"] ) + module.state = dict( name=state.get( "name", "Input Dataset" ) ) + return module + @classmethod + def from_workflow_step( Class, trans, step ): + module = Class( trans ) + module.state = dict( name="Input Dataset" ) + if step.tool_inputs and "name" in step.tool_inputs: + module.state['name'] = step.tool_inputs[ 'name' ] + return module + def save_to_step( self, step ): + step.type = self.type + step.tool_id = None + step.tool_inputs = self.state + def get_data_inputs( self ): return [] def get_data_outputs( self ): return [ dict( name='output', extension='input' ) ] def get_config_form( self ): + form = web.FormBuilder( title=self.name ) \ + .add_text( "name", "Name", value=self.state['name'] ) return self.trans.fill_template( "workflow/editor_generic_form.mako", - form = web.FormBuilder( title=self.name ) ) + module=self, form=form ) + def get_state( self ): + return to_json_string( self.state ) + + def update_state( self, incoming ): + self.state['name'] = incoming.get( 'name', 'Input Dataset' ) def get_runtime_inputs( self ): - return self._runtime_inputs + label = self.state.get( "name", "Input Dataset" ) + return dict( input=DataToolParameter( None, Element( "param", name="input", label=label, type="data", format="data" ) ) ) def get_runtime_state( self ): state = DefaultToolState() state.inputs = dict( input=None ) @@ -106,7 +135,7 @@ return state def update_runtime_state( self, trans, state, values ): errors = {} - for name, param in self._runtime_inputs.iteritems(): + for name, param in self.get_runtime_inputs().iteritems(): value, error = check_param( trans, param, values.get( name, None ), values ) state.inputs[ name ] = value if error: @@ -146,7 +175,6 @@ tool_id = step.tool_id module = Class( trans, tool_id ) module.state = DefaultToolState() - print step.tool_inputs module.state.inputs = module.tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True ) module.errors = step.tool_errors return module @@ -229,4 +257,4 @@ type = step.type return self.module_types[type].from_workflow_step( trans, step ) -module_factory = WorkflowModuleFactory( dict( data_input=InputDataModule, tool=ToolModule ) ) \ No newline at end of file +module_factory = WorkflowModuleFactory( dict( data_input=InputDataModule, tool=ToolModule ) ) diff -r f242b5a3fedc -r 15bf910890d5 templates/workflow/editor_generic_form.mako --- a/templates/workflow/editor_generic_form.mako Tue Dec 02 13:49:32 2008 -0500 +++ b/templates/workflow/editor_generic_form.mako Tue Dec 02 17:17:23 2008 -0500 @@ -1,8 +1,8 @@ <div class="toolForm"> <div class="toolFormTitle">${form.title}</div> <div class="toolFormBody"> - <form name="$form.name" action="${h.url_for(form.action)}" method="post"> - <table cellpadding="0" cellspacing="0" border="0"> + <form name="${form.name}" action="${h.url_for( action='editor_form_post' )}" method="post"> + <input type="hidden" name="type" value="${module.type}" /> %if form.inputs: %for input in form.inputs: <% @@ -33,9 +33,9 @@ </div> %endfor - <tr><td></td><td><input type="submit" value="${form.submit_text}"> + <div class="form-row"><input type="submit" value="${form.submit_text}"></div> %else: - <tr><td colspan="2"><i>No options</i></td></tr> + <div class="form-row"><i>No options</i></div> %endif </table> </form> diff -r f242b5a3fedc -r 15bf910890d5 templates/workflow/editor_tool_form.mako --- a/templates/workflow/editor_tool_form.mako Tue Dec 02 13:49:32 2008 -0500 +++ b/templates/workflow/editor_tool_form.mako Tue Dec 02 17:17:23 2008 -0500 @@ -56,7 +56,7 @@ <div class="toolForm"> <div class="toolFormTitle">Tool: ${tool.name}</div> <div class="toolFormBody"> - <form method="get" action="${h.url_for( action='editor_tool_form' )}"> + <form method="post" action="${h.url_for( action='editor_form_post' )}"> <input type="hidden" name="tool_id" value="${tool.id}" /> %for i, inputs in enumerate( tool.inputs_by_page ): %if tool.has_multiple_pages:

1 0

[hg] galaxy 1643: Call set_size() before set_peek() on datasets, ...
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/74e392ea6ae8 changeset: 1643:74e392ea6ae8 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Tue Dec 02 10:50:54 2008 -0500 description: Call set_size() before set_peek() on datasets, changes to set_peek() to not read entire files for some datatypes. 14 file(s) affected in this change: lib/galaxy/datatypes/images.py lib/galaxy/datatypes/qualityscore.py lib/galaxy/datatypes/registry.py lib/galaxy/datatypes/sequence.py lib/galaxy/model/__init__.py lib/galaxy/tools/__init__.py lib/galaxy/tools/actions/upload.py lib/galaxy/web/controllers/root.py tools/data_source/encode_import_code.py tools/data_source/gbrowse_filter_code.py tools/data_source/hbvar_filter.py tools/data_source/microbial_import_code.py tools/data_source/ucsc_filter.py tools/maf/maf_to_bed_code.py diffs (385 lines): diff -r 499eed864ab3 -r 74e392ea6ae8 lib/galaxy/datatypes/images.py --- a/lib/galaxy/datatypes/images.py Mon Dec 01 14:07:36 2008 -0500 +++ b/lib/galaxy/datatypes/images.py Tue Dec 02 10:50:54 2008 -0500 @@ -15,8 +15,8 @@ file_ext = "ab1" def set_peek( self, dataset ): export_url = "/history_add_to?"+urlencode({'history_id':dataset.history_id,'ext':'ab1','name':'ab1 sequence','info':'Sequence file','dbkey':dataset.dbkey}) - dataset.peek = "Binary ab1 sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) - dataset.blurb = "Binary ab1 sequence file" + dataset.peek = "Binary ab1 sequence file" + dataset.blurb = data.nice_size( dataset.get_size() ) def display_peek(self, dataset): try: return dataset.peek @@ -28,8 +28,8 @@ file_ext = "scf" def set_peek( self, dataset ): export_url = "/history_add_to?"+urlencode({'history_id':dataset.history_id,'ext':'scf','name':'scf sequence','info':'Sequence file','dbkey':dataset.dbkey}) - dataset.peek = "Binary scf sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) - dataset.blurb = "Binary scf sequence file" + dataset.peek = "Binary scf sequence file" + dataset.blurb = data.nice_size( dataset.get_size() ) def display_peek(self, dataset): try: return dataset.peek @@ -42,8 +42,8 @@ def set_peek( self, dataset ): zip_file = zipfile.ZipFile( dataset.file_name, "r" ) num_files = len( zip_file.namelist() ) - dataset.peek = "Binary sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) - dataset.blurb = 'Zip archive of %s binary sequence files' % ( str( num_files ) ) + dataset.peek = "Archive of %s binary sequence files" % ( str( num_files ) ) + dataset.blurb = data.nice_size( dataset.get_size() ) def display_peek(self, dataset): try: return dataset.peek @@ -59,8 +59,8 @@ def set_peek( self, dataset ): zip_file = zipfile.ZipFile( dataset.file_name, "r" ) num_files = len( zip_file.namelist() ) - dataset.peek = "Text sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) - dataset.blurb = 'Zip archive of %s text sequence files' % ( str( num_files ) ) + dataset.peek = "Archive of %s text sequence files" % ( str( num_files ) ) + dataset.blurb = data.nice_size( dataset.get_size() ) def display_peek(self, dataset): try: return dataset.peek @@ -73,8 +73,8 @@ class Image( data.Data ): """Class describing an image""" def set_peek( self, dataset ): - dataset.peek = 'Image in %s format (%s)' % ( dataset.extension, data.nice_size( dataset.get_size() ) ) - dataset.blurb = 'image' + dataset.peek = 'Image in %s format' % dataset.extension + dataset.blurb = data.nice_size( dataset.get_size() ) def create_applet_tag_peek( class_name, archive, params ): text = """ @@ -147,7 +147,7 @@ """Class describing an html file""" file_ext = "html" def set_peek( self, dataset ): - dataset.peek = "HTML file (%s)" % ( data.nice_size( dataset.get_size() ) ) + dataset.peek = "HTML file" dataset.blurb = data.nice_size( dataset.get_size() ) def get_mime(self): """Returns the mime type of the datatype""" diff -r 499eed864ab3 -r 74e392ea6ae8 lib/galaxy/datatypes/qualityscore.py --- a/lib/galaxy/datatypes/qualityscore.py Mon Dec 01 14:07:36 2008 -0500 +++ b/lib/galaxy/datatypes/qualityscore.py Tue Dec 02 10:50:54 2008 -0500 @@ -18,7 +18,7 @@ def set_peek( self, dataset, line_count=None ): dataset.peek = data.get_file_peek( dataset.file_name ) if line_count is None: - dataset.blurb = "%s lines, Quality score file" % util.commaify( str( data.get_line_count( dataset.file_name ) ) ) + dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.blurb = "%s lines, Quality score file" % util.commaify( str( line_count ) ) diff -r 499eed864ab3 -r 74e392ea6ae8 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py Mon Dec 01 14:07:36 2008 -0500 +++ b/lib/galaxy/datatypes/registry.py Tue Dec 02 10:50:54 2008 -0500 @@ -191,6 +191,7 @@ # being converted *to* will handle any metadata copying and # initialization. if data.has_data(): + data.set_size() data.init_meta( copy_from=data ) data.set_meta( overwrite = False ) data.set_peek() diff -r 499eed864ab3 -r 74e392ea6ae8 lib/galaxy/datatypes/sequence.py --- a/lib/galaxy/datatypes/sequence.py Mon Dec 01 14:07:36 2008 -0500 +++ b/lib/galaxy/datatypes/sequence.py Tue Dec 02 10:50:54 2008 -0500 @@ -32,17 +32,7 @@ def set_peek( self, dataset ): dataset.peek = data.get_file_peek( dataset.file_name ) - count = size = 0 - for line in file( dataset.file_name ): - if line and line[0] == ">": - count += 1 - else: - line = line.strip() - size += len(line) - if count == 1: - dataset.blurb = '%d bases' % size - else: - dataset.blurb = '%d sequences' % count + dataset.blurb = data.nice_size( dataset.get_size() ) def sniff(self, filename): """ @@ -96,17 +86,7 @@ def set_peek( self, dataset ): dataset.peek = data.get_file_peek( dataset.file_name ) - count = size = 0 - for line in file( dataset.file_name ): - if line and line[0] == ">": - count += 1 - else: - line = line.strip() - size += len(line) - if count == 1: - dataset.blurb = '%d bases' % size - else: - dataset.blurb = '%d sequences' % count + dataset.blurb = data.nice_size( dataset.get_size() ) def sniff( self, filename ): """ @@ -128,18 +108,7 @@ def set_peek( self, dataset ): dataset.peek = data.get_file_peek( dataset.file_name ) - count = size = 0 - bases_regexp = re.compile("^[NGTAC]*$") - for i, line in enumerate(file( dataset.file_name )): - if line and line[0] == "@" and i % 4 == 0: - count += 1 - elif bases_regexp.match(line): - line = line.strip() - size += len(line) - if count == 1: - dataset.blurb = '%d bases' % size - else: - dataset.blurb = '%d sequences' % count + dataset.blurb = data.nice_size( dataset.get_size() ) def sniff( self, filename ): """ diff -r 499eed864ab3 -r 74e392ea6ae8 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Mon Dec 01 14:07:36 2008 -0500 +++ b/lib/galaxy/model/__init__.py Tue Dec 02 10:50:54 2008 -0500 @@ -289,6 +289,7 @@ def copy( self, copy_children = False, parent_id = None ): des = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, blurb=self.blurb, peek=self.peek, extension=self.extension, dbkey=self.dbkey, dataset = self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id, copied_from_history_dataset_association = self ) des.flush() + des.set_size() des.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id if copy_children: for child in self.children: @@ -467,7 +468,8 @@ def set_size( self ): """Returns the size of the data on disk""" try: - self.file_size = os.path.getsize( self.file_name ) + if not self.file_size: + self.file_size = os.path.getsize( self.file_name ) except OSError: self.file_size = 0 def has_data( self ): diff -r 499eed864ab3 -r 74e392ea6ae8 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Mon Dec 01 14:07:36 2008 -0500 +++ b/lib/galaxy/tools/__init__.py Tue Dec 02 10:50:54 2008 -0500 @@ -1189,6 +1189,7 @@ # tag set in the tool config. if self.tool_type == 'data_source': name, data = out_data.items()[0] + data.set_size() if data.state == data.states.OK: data.name = param_dict.get( 'name', data.name ) data.info = param_dict.get( 'info', data.name ) @@ -1202,7 +1203,6 @@ if data.missing_meta(): data = app.datatypes_registry.change_datatype( data, 'tabular' ) data.set_peek() - data.set_size() data.flush() def collect_associated_files( self, output ): @@ -1234,12 +1234,12 @@ # Move data from temp location to dataset location shutil.move( filename, child_dataset.file_name ) child_dataset.flush() + child_dataset.set_size() child_dataset.name = "Secondary Dataset (%s)" % ( designation ) child_dataset.state = child_dataset.states.OK child_dataset.init_meta() child_dataset.set_meta() child_dataset.set_peek() - child_dataset.set_size() child_dataset.flush() # Add child to return dict children[name][designation] = child_dataset @@ -1270,13 +1270,13 @@ primary_data.flush() # Move data from temp location to dataset location shutil.move( filename, primary_data.file_name ) + primary_data.set_size() primary_data.name = dataset.name primary_data.info = dataset.info primary_data.state = primary_data.states.OK primary_data.init_meta( copy_from=dataset ) primary_data.set_meta() primary_data.set_peek() - primary_data.set_size() primary_data.flush() outdata.history.add_dataset( primary_data ) # Add dataset to return dict diff -r 499eed864ab3 -r 74e392ea6ae8 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Mon Dec 01 14:07:36 2008 -0500 +++ b/lib/galaxy/tools/actions/upload.py Tue Dec 02 10:50:54 2008 -0500 @@ -210,6 +210,7 @@ data.flush() shutil.move( temp_name, data.file_name ) data.state = data.states.OK + data.set_size() data.init_meta() if self.line_count is not None: try: @@ -218,7 +219,6 @@ data.set_peek() else: data.set_peek() - data.set_size() # validate incomming data """ diff -r 499eed864ab3 -r 74e392ea6ae8 lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Mon Dec 01 14:07:36 2008 -0500 +++ b/lib/galaxy/web/controllers/root.py Tue Dec 02 10:50:54 2008 -0500 @@ -647,13 +647,13 @@ data_file.write( file_data.file.read() ) data_file.close() data.state = data.states.OK + data.set_size() data.init_meta() data.set_meta() data.flush() history.add_dataset( data ) history.flush() data.set_peek() - data.set_size() data.flush() trans.log_event("Added dataset %d to history %d" %(data.id, trans.history.id)) if self.app.memory_usage: diff -r 499eed864ab3 -r 74e392ea6ae8 tools/data_source/encode_import_code.py --- a/tools/data_source/encode_import_code.py Mon Dec 01 14:07:36 2008 -0500 +++ b/tools/data_source/encode_import_code.py Tue Dec 02 10:50:54 2008 -0500 @@ -19,6 +19,7 @@ dbkey = fields[2] file_type = fields[3] name, data = out_data.items()[0] + data.set_size() basic_name = data.name data.name = data.name + " (" + description + ")" data.dbkey = dbkey @@ -26,7 +27,6 @@ data = app.datatypes_registry.change_datatype( data, file_type ) data.init_meta() data.set_peek() - data.set_size() app.model.flush() elif fields[0] == "#NewFile": description = fields[1] @@ -34,6 +34,7 @@ filepath = fields[3] file_type = fields[4] newdata = app.model.HistoryDatasetAssociation( create_dataset = True ) #This import should become a library + newdata.set_size() newdata.extension = file_type newdata.name = basic_name + " (" + description + ")" history.add_dataset( newdata ) @@ -48,5 +49,4 @@ newdata.dbkey = dbkey newdata.set_meta() newdata.set_peek() - newdata.set_size() app.model.flush() diff -r 499eed864ab3 -r 74e392ea6ae8 tools/data_source/gbrowse_filter_code.py --- a/tools/data_source/gbrowse_filter_code.py Mon Dec 01 14:07:36 2008 -0500 +++ b/tools/data_source/gbrowse_filter_code.py Tue Dec 02 10:50:54 2008 -0500 @@ -24,11 +24,11 @@ def exec_after_process( app, inp_data, out_data, param_dict, tool=None, stdout=None, stderr=None ): """Verifies the data after the run""" name, data = out_data.items()[0] + data.set_size() if data.state == data.states.OK: data.info = data.name if data.extension == 'txt': data_type = sniff.guess_ext( data.file_name, sniff_order=app.datatypes_registry.sniff_order ) data = app.datatypes_registry.change_datatype( data, data_type ) data.set_peek() - data.set_size() data.flush() diff -r 499eed864ab3 -r 74e392ea6ae8 tools/data_source/hbvar_filter.py --- a/tools/data_source/hbvar_filter.py Mon Dec 01 14:07:36 2008 -0500 +++ b/tools/data_source/hbvar_filter.py Tue Dec 02 10:50:54 2008 -0500 @@ -71,6 +71,6 @@ else: data = app.datatypes_registry.change_datatype(data, 'tabular') + data.set_size() data.set_peek() - data.set_size() data.flush() diff -r 499eed864ab3 -r 74e392ea6ae8 tools/data_source/microbial_import_code.py --- a/tools/data_source/microbial_import_code.py Mon Dec 01 14:07:36 2008 -0500 +++ b/tools/data_source/microbial_import_code.py Tue Dec 02 10:50:54 2008 -0500 @@ -114,6 +114,7 @@ dbkey = fields[3] file_type = fields[4] name, data = out_data.items()[0] + data.set_size() basic_name = data.name data.name = data.name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] +" for " + microbe_info[kingdom][org]['name'] + ":" + chr + ")" data.dbkey = dbkey @@ -121,7 +122,6 @@ data = app.datatypes_registry.change_datatype( data, file_type ) data.init_meta() data.set_peek() - data.set_size() app.model.flush() elif fields[0] == "#NewFile": description = fields[1] @@ -130,6 +130,7 @@ filepath = fields[4] file_type = fields[5] newdata = app.model.HistoryDatasetAssociation( create_dataset = True ) #This import should become a library + newdata.set_size() newdata.extension = file_type newdata.name = basic_name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] +" for "+microbe_info[kingdom][org]['name']+":"+chr + ")" newdata.flush() @@ -145,5 +146,4 @@ newdata.dbkey = dbkey newdata.init_meta() newdata.set_peek() - newdata.set_size() app.model.flush() diff -r 499eed864ab3 -r 74e392ea6ae8 tools/data_source/ucsc_filter.py --- a/tools/data_source/ucsc_filter.py Mon Dec 01 14:07:36 2008 -0500 +++ b/tools/data_source/ucsc_filter.py Tue Dec 02 10:50:54 2008 -0500 @@ -43,6 +43,7 @@ """Verifies the data after the run""" items = out_data.items() for name, data in items: + data.set_size() try: err_msg, err_flag = 'Errors:', False line_count = 0 @@ -65,4 +66,3 @@ except Exception, exc: data.info = data.info + "\n" + str(exc) data.blurb = "error" - data.set_size() diff -r 499eed864ab3 -r 74e392ea6ae8 tools/maf/maf_to_bed_code.py --- a/tools/maf/maf_to_bed_code.py Mon Dec 01 14:07:36 2008 -0500 +++ b/tools/maf/maf_to_bed_code.py Tue Dec 02 10:50:54 2008 -0500 @@ -29,6 +29,7 @@ filepath = fields[2] newdata = app.model.HistoryDatasetAssociation( create_dataset = True ) + newdata.set_size() newdata.extension = "bed" newdata.name = basic_name + " (" + dbkey + ")" newdata.flush() @@ -46,7 +47,6 @@ newdata.dbkey = dbkey newdata.init_meta() newdata.set_peek() - newdata.set_size() app.model.flush() output_data_list.append(newdata) else:

1 0

[hg] galaxy 1641: Improve template for error reporting from users.
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/e2e7c33f587d changeset: 1641:e2e7c33f587d user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Dec 01 09:57:59 2008 -0500 description: Improve template for error reporting from users. 1 file(s) affected in this change: lib/galaxy/web/controllers/dataset.py diffs (63 lines): diff -r 20683fe3bfcb -r e2e7c33f587d lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Sun Nov 30 08:03:21 2008 -0500 +++ b/lib/galaxy/web/controllers/dataset.py Mon Dec 01 09:57:59 2008 -0500 @@ -21,25 +21,23 @@ GALAXY TOOL ERROR REPORT ------------------------ -This error report is in reference to dataset ${dataset_id}. The user -(${email}) provided the following information: - +This error report is in reference to output dataset ${dataset_id}. ----------------------------------------------------------------------------- +The user '${email}' provided the following information: ${message} ----------------------------------------------------------------------------- - -The tool error was: - +job id: ${job_id} +tool id: ${tool_id} ----------------------------------------------------------------------------- +job stderr: ${stderr} ----------------------------------------------------------------------------- - -And the tool output was: - ------------------------------------------------------------------------------ +job stdout: ${stdout} ----------------------------------------------------------------------------- - +job info: +${info} +----------------------------------------------------------------------------- (This is an automated message). """ @@ -62,7 +60,8 @@ smtp_server = trans.app.config.smtp_server if smtp_server is None: return trans.show_error_message( "Sorry, mail is not configured for this galaxy instance" ) - to_address = trans.app.config.error_email_to + #to_address = trans.app.config.error_email_to + to_address = 'greg(a)bx.psu.edu' if to_address is None: return trans.show_error_message( "Sorry, error reporting has been disabled for this galaxy instance" ) # Get the dataset and associated job @@ -72,9 +71,12 @@ msg = MIMEText( string.Template( error_report_template ) .safe_substitute( dataset_id=dataset.id, email=email, - message=message, + message=message, + job_id=job.id, + tool_id=job.tool_id, stderr=job.stderr, - stdout=job.stdout ) ) + stdout=job.stdout, + info=job.info ) ) frm = to_address # Check email a bit email = email.strip()

1 0

[hg] galaxy 1642: Refix cleanup_dataset.py script.
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/499eed864ab3 changeset: 1642:499eed864ab3 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Mon Dec 01 14:07:36 2008 -0500 description: Refix cleanup_dataset.py script. 1 file(s) affected in this change: scripts/cleanup_datasets/cleanup_datasets.py diffs (16 lines): diff -r e2e7c33f587d -r 499eed864ab3 scripts/cleanup_datasets/cleanup_datasets.py --- a/scripts/cleanup_datasets/cleanup_datasets.py Mon Dec 01 09:57:59 2008 -0500 +++ b/scripts/cleanup_datasets/cleanup_datasets.py Mon Dec 01 14:07:36 2008 -0500 @@ -8,6 +8,12 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path + +from galaxy import eggs +import galaxy.model.mapping +import pkg_resources + +pkg_resources.require( "SQLAlchemy >= 0.4" ) from galaxy.model.orm import *

1 0

[hg] galaxy 1639: Fix for setting metadata on a history item des...
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/bcd158679985 changeset: 1639:bcd158679985 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 24 11:46:45 2008 -0500 description: Fix for setting metadata on a history item designated as a MAF file, but is not a valid MAF. 1 file(s) affected in this change: lib/galaxy/datatypes/sequence.py diffs (14 lines): diff -r 66ea0a1ac929 -r bcd158679985 lib/galaxy/datatypes/sequence.py --- a/lib/galaxy/datatypes/sequence.py Mon Nov 24 11:19:48 2008 -0500 +++ b/lib/galaxy/datatypes/sequence.py Mon Nov 24 11:46:45 2008 -0500 @@ -208,9 +208,9 @@ species = [] species_chromosomes = {} - maf_reader = bx.align.maf.Reader( open( dataset.file_name ) ) indexes = bx.interval_index_file.Indexes() try: + maf_reader = bx.align.maf.Reader( open( dataset.file_name ) ) while True: pos = maf_reader.file.tell() block = maf_reader.next()

1 0

[hg] galaxy 1640: Fixes, code cleanup in cleanup_datasets.py.
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/20683fe3bfcb changeset: 1640:20683fe3bfcb user: Greg Von Kuster <greg(a)bx.psu.edu> date: Sun Nov 30 08:03:21 2008 -0500 description: Fixes, code cleanup in cleanup_datasets.py. 1 file(s) affected in this change: scripts/cleanup_datasets/cleanup_datasets.py diffs (141 lines): diff -r bcd158679985 -r 20683fe3bfcb scripts/cleanup_datasets/cleanup_datasets.py --- a/scripts/cleanup_datasets/cleanup_datasets.py Mon Nov 24 11:46:45 2008 -0500 +++ b/scripts/cleanup_datasets/cleanup_datasets.py Sun Nov 30 08:03:21 2008 -0500 @@ -9,12 +9,7 @@ new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs -import galaxy.model.mapping -import pkg_resources - -pkg_resources.require( "SQLAlchemy >= 0.4" ) -from sqlalchemy.orm import eagerload +from galaxy.model.orm import * assert sys.version_info[:2] >= ( 2, 4 ) @@ -79,8 +74,10 @@ # Provide info about the histories and datasets that will be affected if the delete_userless_histories function is executed. history_count = 0 dataset_count = 0 - where = ( h.table.c.user_id==None ) & ( h.table.c.deleted==False ) & ( h.table.c.update_time < cutoff_time ) - histories = h.query().filter( where ).options( eagerload( 'active_datasets' ) ).all() + histories = h.filter( and_( h.table.c.user_id==None, + h.table.c.deleted==False, + h.table.c.update_time < cutoff_time ) ) \ + .options( eagerload( 'active_datasets' ) ).all() print '# The following datasets and associated userless histories will be deleted' start = time.clock() @@ -102,17 +99,18 @@ # The datasets associated with each history are also deleted. Nothing is removed from disk. history_count = 0 dataset_count = 0 - h_where = ( h.table.c.user_id==None ) & ( h.table.c.deleted==False ) & ( h.table.c.update_time < cutoff_time ) print '# The following datasets and associated userless histories have been deleted' start = time.clock() - histories = h.query().filter( h_where ).options( eagerload( 'active_datasets' ) ).all() + histories = h.filter( and_( h.table.c.user_id==None, + h.table.c.deleted==False, + h.table.c.update_time < cutoff_time ) ) \ + .options( eagerload( 'active_datasets' ) ).all() for history in histories: for dataset_assoc in history.active_datasets: if not dataset_assoc.deleted: # Mark all datasets as deleted - d_where = ( d.table.c.id==dataset_assoc.dataset_id ) - datasets = d.query().filter( d_where ).all() + datasets = d.filter( d.table.c.id==dataset_assoc.dataset_id ).all() for dataset in datasets: if not dataset.deleted: dataset.deleted = True @@ -136,17 +134,17 @@ history_count = 0 dataset_count = 0 disk_space = 0 - h_where = ( h.table.c.deleted==True ) & ( h.table.c.purged==False ) & ( h.table.c.update_time < cutoff_time ) - print '# The following datasets and associated deleted histories will be purged' start = time.clock() - histories = h.query().filter( h_where ).options( eagerload( 'datasets' ) ).all() + histories = h.filter( and_( h.table.c.deleted==True, + h.table.c.purged==False, + h.table.c.update_time < cutoff_time ) ) \ + .options( eagerload( 'datasets' ) ).all() for history in histories: for dataset_assoc in history.datasets: # Datasets can only be purged if their HistoryDatasetAssociation has been deleted. if dataset_assoc.deleted: - d_where = ( d.table.c.id==dataset_assoc.dataset_id ) - datasets = d.query().filter( d_where ).all() + datasets = d.filter( d.table.c.id==dataset_assoc.dataset_id ).all() for dataset in datasets: if dataset.purgable and not dataset.purged: print "%s" % dataset.file_name @@ -169,17 +167,17 @@ disk_space = 0 file_size = 0 errors = False - h_where = ( h.table.c.deleted==True ) & ( h.table.c.purged==False ) & ( h.table.c.update_time < cutoff_time ) - print '# The following datasets and associated deleted histories have been purged' start = time.clock() - histories = h.query().filter( h_where ).options( eagerload( 'datasets' ) ).all() + histories = h.filter( and_( h.table.c.deleted==True, + h.table.c.purged==False, + h.table.c.update_time < cutoff_time ) ) \ + .options( eagerload( 'datasets' ) ).all() for history in histories: errors = False for dataset_assoc in history.datasets: if dataset_assoc.deleted: - d_where = ( d.table.c.id==dataset_assoc.dataset_id ) - datasets = d.query().filter( d_where ).all() + datasets = d.filter( d.table.c.id==dataset_assoc.dataset_id ).all() for dataset in datasets: if dataset.purgable and not dataset.purged: file_size = dataset.file_size @@ -221,11 +219,12 @@ # Provide info about the datasets that will be affected if the purge_datasets function is executed. dataset_count = 0 disk_space = 0 - where = ( d.table.c.deleted==True ) & ( d.table.c.purgable==True ) & ( d.table.c.purged==False ) & ( d.table.c.update_time < cutoff_time ) - print '# The following deleted datasets will be purged' start = time.clock() - datasets = d.query().filter( where ).all() + datasets = d.filter( and_( d.table.c.deleted==True, + d.table.c.purgable==True, + d.table.c.purged==False, + d.table.c.update_time < cutoff_time ) ).all() for dataset in datasets: print "%s" % dataset.file_name dataset_count += 1 @@ -243,11 +242,12 @@ dataset_count = 0 disk_space = 0 file_size = 0 - where = ( d.table.c.deleted==True ) & ( d.table.c.purgable==True ) & ( d.table.c.purged==False ) & ( d.table.c.update_time < cutoff_time ) - print '# The following deleted datasets have been purged' start = time.clock() - datasets = d.query().filter( where ).all() + datasets = d.filter( and_( d.table.c.deleted==True, + d.table.c.purgable==True, + d.table.c.purged==False, + d.table.c.update_time < cutoff_time ) ).all() for dataset in datasets: file_size = dataset.file_size if remove_from_disk: @@ -289,7 +289,8 @@ # See if the dataset has been shared if dataset.external_filename: # This check handles the pre-history_dataset_association approach to sharing. - shared_data = d.filter( and_( d.table.c.external_filename==dataset.external_filename, d.table.c.deleted==False ) ).all() + shared_data = d.filter( and_( d.table.c.external_filename==dataset.external_filename, + d.table.c.deleted==False ) ).all() if shared_data: purgable = False if purgable:

1 0

[hg] galaxy 1638: Update GMAJ to latest beta.
by Nate Coraor 04 Dec '08

04 Dec '08

details: http://www.bx.psu.edu/hg/galaxy/rev/66ea0a1ac929 changeset: 1638:66ea0a1ac929 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Nov 24 11:19:48 2008 -0500 description: Update GMAJ to latest beta. 2 file(s) affected in this change: static/gmaj/gmaj.jar tools/visualization/GMAJ.xml diffs (30 lines): diff -r c141cb89aa45 -r 66ea0a1ac929 static/gmaj/gmaj.jar Binary file static/gmaj/gmaj.jar has changed diff -r c141cb89aa45 -r 66ea0a1ac929 tools/visualization/GMAJ.xml --- a/tools/visualization/GMAJ.xml Thu Nov 20 16:15:39 2008 -0500 +++ b/tools/visualization/GMAJ.xml Mon Nov 24 11:19:48 2008 -0500 @@ -46,6 +46,7 @@ <option name="MAF File" value="maf"> <option name="Invalid MAF version (maf_version)" value="maf_version"/> <option name="Skipping unsupported paragraph (maf_paragraph)" value="maf_paragraph"/> + <option name="Unrecognized character found in alignment (bad_char_all)" value="bad_char_all"/> <option name="Skipping all reconstruction scores: no species specified (recon_noseq)" value="recon_noseq"/> <option name="Skipping reconstruction scores in blocks with missing row (recon_missing)" value="recon_missing"/> <option name="The first row in some blocks is not the specified reference sequence (refseq_not_first)" value="refseq_not_first"/> @@ -75,7 +76,7 @@ <option name="Score shading is not yet supported (score_shading)" value="score_shading"/> </option> <option name="Red Flags" value="red"> - <option name="Sequence name in annotation file does not match name in MAF (seqname_mismatch)" value="seqname_mismatch"/> + <option name="Assuming that annotations in file ___ are for species ___ (seqname_fix_all)" value="seqname_fix_all"/> <option name="BED start or end < 0 (bed_coord)" value="bed_coord"/> <option name="GFF start or end < 1 (gff_coord)" value="gff_coord"/> <option name="Missing item name for URL substitution (url_subst)" value="url_subst"/> @@ -87,6 +88,7 @@ <option name="Skipping blocks for export where reference sequence is hidden or all gaps (export_skip)" value="export_skip"/> <option name="Possible parse error: token ends with an escaped quote (escaped_quote)" value="escaped_quote"/> <option name="Draggable panel dividers will not be sticky (no_sticky)" value="no_sticky"/> + <option name="Selecting a large block may be very slow (big_block)" value="big_block"/> </option> </option> </options>

1 0

[hg] galaxy 1637: Fix job recovery
by Nate Coraor 20 Nov '08

20 Nov '08

details: http://www.bx.psu.edu/hg/galaxy/rev/c141cb89aa45 changeset: 1637:c141cb89aa45 user: Nate Coraor <nate(a)bx.psu.edu> date: Thu Nov 20 16:15:39 2008 -0500 description: Fix job recovery 2 file(s) affected in this change: lib/galaxy/jobs/__init__.py universe_wsgi.ini.sample diffs (27 lines): diff -r 8412e0cbe7f0 -r c141cb89aa45 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py Tue Nov 18 16:04:41 2008 -0500 +++ b/lib/galaxy/jobs/__init__.py Thu Nov 20 16:15:39 2008 -0500 @@ -107,7 +107,7 @@ self.monitor_thread = threading.Thread( target=self.monitor ) self.monitor_thread.start() log.info( "job manager started" ) - if self.track_jobs_in_database: + if app.config.get_bool( 'enable_job_recovery', True ): self.check_jobs_at_startup() def check_jobs_at_startup( self ): diff -r 8412e0cbe7f0 -r c141cb89aa45 universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample Tue Nov 18 16:04:41 2008 -0500 +++ b/universe_wsgi.ini.sample Thu Nov 20 16:15:39 2008 -0500 @@ -24,6 +24,11 @@ # Should jobs be tracked through the database, rather than in memory ## track_jobs_in_database = true + +# Enable job recovery (if Galaxy is restarted while cluster jobs are running, +# it can "recover" them when it starts). This is not safe to use if you are +# running more than one Galaxy server using the same database. +#enable_job_recovery = True # Number of concurrent jobs to run (local runner) local_job_queue_workers = 5

1 0