July 2009 - galaxy-dev - lists.galaxyproject.org

[hg] galaxy 2494: Allow composite files to be defined as binary ...
by Greg Von Kuster 23 Jul '09

23 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/3dae3393e275 changeset: 2494:3dae3393e275 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Thu Jul 23 12:38:11 2009 -0400 description: Allow composite files to be defined as binary to allow proper uploading. This is done on a per component basis. To set a particular composite file to be binary, set is_binary = True. If the primary file is to be binary, set the datatype attribute is_binary = True. 4 file(s) affected in this change: lib/galaxy/datatypes/data.py lib/galaxy/datatypes/genetics.py lib/galaxy/tools/actions/upload.py lib/galaxy/tools/parameters/grouping.py diffs (221 lines): diff -r d51e2ea0daf9 -r 3dae3393e275 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py Thu Jul 23 12:24:12 2009 -0400 +++ b/lib/galaxy/datatypes/data.py Thu Jul 23 12:38:11 2009 -0400 @@ -252,12 +252,14 @@ def after_edit( self, dataset ): """This function is called on the dataset after metadata is edited.""" dataset.clear_associated_files( metadata_safe = True ) - def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, **kwds ): + def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = True, **kwds ): kwds[ 'name' ] = name kwds[ 'optional' ] = optional kwds[ 'mimetype' ] = mimetype kwds[ 'description' ] = description kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata + kwds[ 'is_binary' ] = is_binary + kwds[ 'space_to_tab' ] = space_to_tab return Bunch( **kwds ) def add_composite_file( self, name, **kwds ): #self.composite_files = self.composite_files.copy() diff -r d51e2ea0daf9 -r 3dae3393e275 lib/galaxy/datatypes/genetics.py --- a/lib/galaxy/datatypes/genetics.py Thu Jul 23 12:24:12 2009 -0400 +++ b/lib/galaxy/datatypes/genetics.py Thu Jul 23 12:38:11 2009 -0400 @@ -160,8 +160,8 @@ def __init__( self, **kwd ): Rgenetics.__init__( self, **kwd ) - self.add_composite_file( '%s.ped', description = 'Pedigree File', substitute_name_with_metadata = 'base_name' ) - self.add_composite_file( '%s.map', description = 'Map File', substitute_name_with_metadata = 'base_name' ) + self.add_composite_file( '%s.ped', description = 'Pedigree File', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s.map', description = 'Map File', substitute_name_with_metadata = 'base_name', is_binary = True ) class Pphe(Rgenetics): @@ -198,10 +198,10 @@ def __init__( self, **kwd ): Rgenetics.__init__( self, **kwd ) - self.add_composite_file( '%s.bim', substitute_name_with_metadata = 'base_name' ) - self.add_composite_file( '%s.bed', substitute_name_with_metadata = 'base_name' ) - self.add_composite_file( '%s.fam', substitute_name_with_metadata = 'base_name' ) - self.add_composite_file( '%s.map', substitute_name_with_metadata = 'base_name' ) + self.add_composite_file( '%s.bim', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s.bed', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s.fam', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s.map', substitute_name_with_metadata = 'base_name', is_binary = True ) class Eigenstratgeno(Rgenetics): @@ -211,15 +211,15 @@ def __init__( self, **kwd ): Rgenetics.__init__( self, **kwd ) - self.add_composite_file( '%s.eigenstratgeno', substitute_name_with_metadata = 'base_name' ) - self.add_composite_file( '%s.ind', substitute_name_with_metadata = 'base_name' ) - self.add_composite_file( '%s.map', substitute_name_with_metadata = 'base_name' ) - self.add_composite_file( '%s_fo.eigenstratgeno', substitute_name_with_metadata = 'base_name', optional = 'True' ) - self.add_composite_file( '%s_fo.ind', substitute_name_with_metadata = 'base_name', optional = 'True' ) - self.add_composite_file( '%s_fo.map', substitute_name_with_metadata = 'base_name', optional = 'True' ) - self.add_composite_file( '%s_oo.eigenstratgeno', substitute_name_with_metadata = 'base_name', optional = 'True' ) - self.add_composite_file( '%s_oo.ind', substitute_name_with_metadata = 'base_name', optional = 'True' ) - self.add_composite_file( '%s_oo.map', substitute_name_with_metadata = 'base_name', optional = 'True' ) + self.add_composite_file( '%s.eigenstratgeno', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s.ind', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s.map', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s_fo.eigenstratgeno', substitute_name_with_metadata = 'base_name', optional = 'True', is_binary = True ) + self.add_composite_file( '%s_fo.ind', substitute_name_with_metadata = 'base_name', optional = 'True', is_binary = True ) + self.add_composite_file( '%s_fo.map', substitute_name_with_metadata = 'base_name', optional = 'True', is_binary = True ) + self.add_composite_file( '%s_oo.eigenstratgeno', substitute_name_with_metadata = 'base_name', optional = 'True', is_binary = True ) + self.add_composite_file( '%s_oo.ind', substitute_name_with_metadata = 'base_name', optional = 'True', is_binary = True ) + self.add_composite_file( '%s_oo.map', substitute_name_with_metadata = 'base_name', optional = 'True', is_binary = True ) @@ -259,7 +259,7 @@ def __init__( self, **kwd ): Html.__init__( self, **kwd ) - self.add_composite_file( '%s.phenodata', substitute_name_with_metadata = 'base_name' ) + self.add_composite_file( '%s.phenodata', substitute_name_with_metadata = 'base_name', is_binary = True ) def set_peek( self, dataset ): """expects a .pheno file in the extra_files_dir - ugh diff -r d51e2ea0daf9 -r 3dae3393e275 lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Thu Jul 23 12:24:12 2009 -0400 +++ b/lib/galaxy/tools/actions/upload.py Thu Jul 23 12:38:11 2009 -0400 @@ -46,26 +46,8 @@ uploaded_datasets = dataset_upload_input.get_uploaded_datasets( trans, incoming ) for uploaded_dataset in uploaded_datasets: precreated_dataset = self.get_precreated_dataset( uploaded_dataset.precreated_name ) - dataset = self.add_file( trans, uploaded_dataset.primary_file, uploaded_dataset.name, uploaded_dataset.file_type, uploaded_dataset.is_multi_byte, uploaded_dataset.dbkey, space_to_tab = uploaded_dataset.space_to_tab, info = uploaded_dataset.info, precreated_dataset = precreated_dataset, metadata = uploaded_dataset.metadata ) - composite_files = dataset.datatype.get_composite_files( dataset ) - if composite_files: - os.mkdir( dataset.extra_files_path ) #make extra files path - for name, value in composite_files.iteritems(): - #what about binary files here, need to skip converting newlines - if uploaded_dataset.composite_files[ value.name ] is None and not value.optional: - dataset.info = "A required composite data file was not provided (%s)" % name - dataset.state = dataset.states.ERROR - break - elif uploaded_dataset.composite_files[ value.name] is not None: - if uploaded_dataset.composite_files[ value.name ].space_to_tab: - sniff.convert_newlines_sep2tabs( uploaded_dataset.composite_files[ value.name ].filename ) - else: - sniff.convert_newlines( uploaded_dataset.composite_files[ value.name ].filename ) - shutil.move( uploaded_dataset.composite_files[ value.name ].filename, os.path.join( dataset.extra_files_path, name ) ) - if dataset.datatype.composite_type == 'auto_primary_file': - #now that metadata is set, we should create the primary file as required - open( dataset.file_name, 'wb+' ).write( dataset.datatype.generate_primary_file( dataset = dataset ) ) - + dataset = self.add_file( trans, uploaded_dataset.primary_file, uploaded_dataset.name, uploaded_dataset.file_type, uploaded_dataset.is_multi_byte, uploaded_dataset.dbkey, space_to_tab = uploaded_dataset.space_to_tab, info = uploaded_dataset.info, precreated_dataset = precreated_dataset, metadata = uploaded_dataset.metadata, uploaded_dataset = uploaded_dataset ) + #dataset state is now set, we should not do anything else to this dataset data_list.append( dataset ) #clean up extra temp names uploaded_dataset.clean_up_temp_files() @@ -130,7 +112,7 @@ trans.log_event( 'job id %d ended with errors, err_msg: %s' % ( job.id, err_msg ), tool_id=job.tool_id ) return dict( output=data ) - def add_file( self, trans, temp_name, file_name, file_type, is_multi_byte, dbkey, info=None, space_to_tab=False, precreated_dataset=None, metadata = {} ): + def add_file( self, trans, temp_name, file_name, file_type, is_multi_byte, dbkey, info=None, space_to_tab=False, precreated_dataset=None, metadata = {}, uploaded_dataset = None ): def dataset_no_data_error( data, message = 'there was an error uploading your file' ): data.info = "No data: %s." % message data.state = data.states.ERROR @@ -152,7 +134,7 @@ if is_multi_byte: ext = sniff.guess_ext( temp_name, is_multi_byte=True ) else: - if not data_type: + if not data_type: #at this point data_type is always None (just initialized above), so this is always True...lots of cleanup needed here # See if we have a gzipped file, which, if it passes our restrictions, # we'll decompress on the fly. is_gzipped, is_valid = self.check_gzip( temp_name ) @@ -204,19 +186,24 @@ ext = file_type if not data_type: if self.check_binary( temp_name ): - parts = file_name.split( "." ) - if len( parts ) > 1: - ext = parts[1].strip().lower() - if not( ext == 'ab1' or ext == 'scf' ): - return dataset_no_data_error( data, message = "you attempted to upload an inappropriate file" ) - #raise BadFileException( "you attempted to upload an inappropriate file." ) - if ext == 'ab1' and file_type != 'ab1': - return dataset_no_data_error( data, message = "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files" ) - #raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." ) - elif ext == 'scf' and file_type != 'scf': - return dataset_no_data_error( data, message = "you must manually set the 'File Format' to 'Scf' when uploading scf files" ) - #raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." ) - data_type = 'binary' + if uploaded_dataset and uploaded_dataset.datatype and uploaded_dataset.datatype.is_binary: + #we need a more generalized way of checking if a binary upload is of the right format for a datatype...magic number, etc + data_type = 'binary' + ext = uploaded_dataset.file_type + else: + parts = file_name.split( "." ) + if len( parts ) > 1: + ext = parts[1].strip().lower() + if not( ext == 'ab1' or ext == 'scf' ): + return dataset_no_data_error( data, message = "you attempted to upload an inappropriate file" ) + #raise BadFileException( "you attempted to upload an inappropriate file." ) + if ext == 'ab1' and file_type != 'ab1': + return dataset_no_data_error( data, message = "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files" ) + #raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." ) + elif ext == 'scf' and file_type != 'scf': + return dataset_no_data_error( data, message = "you must manually set the 'File Format' to 'Scf' when uploading scf files" ) + #raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." ) + data_type = 'binary' if not data_type: # We must have a text file if trans.app.datatypes_registry.get_datatype_by_extension( file_type ).composite_type != 'auto_primary_file' and self.check_html( temp_name ): @@ -241,9 +228,7 @@ data.info = info data.flush() shutil.move( temp_name, data.file_name ) - ## FIXME - data.state = data.states.OK ##THIS SHOULD BE THE LAST THING DONE - #### its bad to set other things after this point, i.e. metadata and composite files...this creates a race condition where a dataset could be pushed into a job before its metadata, etc is set + dataset_state = data.states.OK #don't set actual state here, only set to OK when finished setting attributes of the dataset data.set_size() data.init_meta() #need to set metadata, has to be done after extention is set @@ -280,6 +265,27 @@ trans.history.genome_build = dbkey_to_store else: trans.history.add_dataset( data, genome_build=dbkey_to_store ) + #set up composite files + if uploaded_dataset is not None: + composite_files = data.datatype.get_composite_files( data ) + if composite_files: + os.mkdir( data.extra_files_path ) #make extra files path + for name, value in composite_files.iteritems(): + if uploaded_dataset.composite_files[ value.name ] is None and not value.optional: + data.info = "A required composite data file was not provided (%s)" % name + dataset_state = data.states.ERROR + break + elif uploaded_dataset.composite_files[ value.name] is not None: + if not value.is_binary: + if uploaded_dataset.composite_files[ value.name ].space_to_tab: + sniff.convert_newlines_sep2tabs( uploaded_dataset.composite_files[ value.name ].filename ) + else: + sniff.convert_newlines( uploaded_dataset.composite_files[ value.name ].filename ) + shutil.move( uploaded_dataset.composite_files[ value.name ].filename, os.path.join( data.extra_files_path, name ) ) + if data.datatype.composite_type == 'auto_primary_file': + #now that metadata was set above, we should create the primary file as required + open( data.file_name, 'wb+' ).write( data.datatype.generate_primary_file( dataset = data ) ) + data.state = dataset_state #Always set dataset state LAST trans.app.model.flush() trans.log_event( "Added dataset %d to history %d" %( data.id, trans.history.id ), tool_id="upload" ) return data diff -r d51e2ea0daf9 -r 3dae3393e275 lib/galaxy/tools/parameters/grouping.py --- a/lib/galaxy/tools/parameters/grouping.py Thu Jul 23 12:24:12 2009 -0400 +++ b/lib/galaxy/tools/parameters/grouping.py Thu Jul 23 12:38:11 2009 -0400 @@ -356,6 +356,8 @@ precreated_name = dataset_name = 'Uploaded Composite Dataset (%s)' % ( file_type ) else: temp_name, precreated_name, is_multi_byte, space_to_tab, dataset_name, dataset_info, warnings = get_one_filename( groups_incoming[ 0 ] ) + if dataset.datatype.composite_type: + precreated_name = 'Uploaded Composite Dataset (%s)' % ( file_type ) writable_files_offset = 1 if temp_name is None:#remove this before finish, this should create an empty dataset raise Exception( 'No primary dataset file was available for composite upload' )

1 0

[hg] galaxy 2492: Performance improvements for fasta_filter_by_l...
by Greg Von Kuster 23 Jul '09

23 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/2177d6f7bf80 changeset: 2492:2177d6f7bf80 user: Kelly Vincent <kpvincent(a)bx.psu.edu> date: Thu Jul 23 12:16:44 2009 -0400 description: Performance improvements for fasta_filter_by_length and fasta_to_tabular 2 file(s) affected in this change: tools/fasta_tools/fasta_filter_by_length.py tools/fasta_tools/fasta_to_tabular.py diffs (143 lines): diff -r cd366f615220 -r 2177d6f7bf80 tools/fasta_tools/fasta_filter_by_length.py --- a/tools/fasta_tools/fasta_filter_by_length.py Thu Jul 23 11:11:41 2009 -0400 +++ b/tools/fasta_tools/fasta_filter_by_length.py Thu Jul 23 12:16:44 2009 -0400 @@ -14,8 +14,7 @@ sys.exit() def __main__(): - - infile = sys.argv[1] + input_filename = sys.argv[1] try: min_length = int( sys.argv[2] ) except: @@ -24,65 +23,30 @@ max_length = int( sys.argv[3] ) except: stop_err( "Maximum length of the return sequence requires a numerical value." ) - outfile = sys.argv[4] - fasta_title = fasta_seq = '' + output_filename = sys.argv[4] + output_handle = open( output_filename, 'w' ) + tmp_size = 0 #-1 + tmp_buf = '' at_least_one = 0 - - out = open( outfile, 'w' ) - - for i, line in enumerate( file( infile ) ): - line = line.rstrip( '\r\n' ) - if not line or line.startswith( '#' ): + for line in file(input_filename): + if not line or line.startswith('#'): continue - if line[0] == '>': - if len( fasta_seq ) > 0: - - if max_length <= 0: - compare_max_length = len( fasta_seq ) + 1 - else: - compare_max_length = max_length - - l = len( fasta_seq ) - - if l >= min_length and l <= compare_max_length: - at_least_one += 1 - out.write( "%s\n" % fasta_title ) - c = 0 - s = fasta_seq - while c < l: - b = min( c + 50, l ) - out.write( "%s\n" % s[ c:b ] ) - c = b - - fasta_title = line - fasta_seq = '' + if min_length <= tmp_size <= max_length or (min_length <= tmp_size and max_length == 0): + output_handle.write(tmp_buf) + at_least_one = 1 + tmp_buf = line + tmp_size = 0 else: - fasta_seq = "%s%s" % ( fasta_seq, line ) - - if len( fasta_seq ) > 0: - - if max_length <= 0: - compare_max_length = len( fasta_seq ) + 1 - else: - compare_max_length = max_length - - l = len( fasta_seq ) - - if l >= min_length and l <= compare_max_length: - at_least_one += 1 - out.write( "%s\n" % fasta_title ) - c = 0 - s = fasta_seq - while c < l: - b = min( c + 50, l ) - out.write( "%s\n" % s[ c:b ] ) - c = b - - out.close() - + if max_length == 0 or tmp_size < max_length: + tmp_size += len(line.rstrip('\r\n')) + tmp_buf += line + # final flush of buffer + if min_length <= tmp_size <= max_length or (min_length <= tmp_size and max_length == 0): + output_handle.write(tmp_buf.rstrip('\r\n')) + at_least_one = 1 + output_handle.close() if at_least_one == 0: print "There is no sequence that falls within your range." - -if __name__ == "__main__" : __main__() \ No newline at end of file +if __name__ == "__main__" : __main__() diff -r cd366f615220 -r 2177d6f7bf80 tools/fasta_tools/fasta_to_tabular.py --- a/tools/fasta_tools/fasta_to_tabular.py Thu Jul 23 11:11:41 2009 -0400 +++ b/tools/fasta_tools/fasta_to_tabular.py Thu Jul 23 12:16:44 2009 -0400 @@ -15,30 +15,23 @@ outfile = sys.argv[2] keep_first = int( sys.argv[3] ) fasta_title = fasta_seq = '' - if keep_first == 0: keep_first = None else: keep_first += 1 - out = open( outfile, 'w' ) - for i, line in enumerate( open( infile ) ): line = line.rstrip( '\r\n' ) if not line or line.startswith( '#' ): continue if line.startswith( '>' ): - if fasta_seq: - out.write( "%s\t%s\n" %( fasta_title[ 1:keep_first ], fasta_seq ) ) - fasta_title = line - fasta_seq = '' + if i: + out.write('\n') + out.write(line[1:keep_first]) + out.write('\t') else: - if line: - fasta_seq = "%s%s" % ( fasta_seq, line ) - - if fasta_seq: - out.write( "%s\t%s\n" %( fasta_title[ 1:keep_first ], fasta_seq ) ) - + out.write(line) out.close() -if __name__ == "__main__" : __main__() \ No newline at end of file +if __name__ == "__main__" : __main__() + \ No newline at end of file

1 0

[hg] galaxy 2491: No longer guess the name column in interval se...
by Greg Von Kuster 23 Jul '09

23 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/cd366f615220 changeset: 2491:cd366f615220 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Thu Jul 23 11:11:41 2009 -0400 description: No longer guess the name column in interval set_meta(). Name column must now always be set manually. 1 file(s) affected in this change: lib/galaxy/datatypes/interval.py diffs (24 lines): diff -r 3851029eda94 -r cd366f615220 lib/galaxy/datatypes/interval.py --- a/lib/galaxy/datatypes/interval.py Wed Jul 22 21:28:26 2009 -0400 +++ b/lib/galaxy/datatypes/interval.py Thu Jul 23 11:11:41 2009 -0400 @@ -115,12 +115,14 @@ dataset.metadata.endCol = 3 except: pass # Metadata default will be used - if len( elems ) > 3: - try: - int( elems[3] ) - except: - if overwrite or not dataset.metadata.element_is_set( 'nameCol' ): - dataset.metadata.nameCol = 4 + #we no longer want to guess that this column is the 'name', name must now be set manually for interval files + #we will still guess at the strand, as we can make a more educated guess + #if len( elems ) > 3: + # try: + # int( elems[3] ) + # except: + # if overwrite or not dataset.metadata.element_is_set( 'nameCol' ): + # dataset.metadata.nameCol = 4 if len( elems ) < 6 or elems[5] not in data.valid_strand: if overwrite or not dataset.metadata.element_is_set( 'strandCol' ): dataset.metadata.strandCol = 0

1 0

[hg] galaxy 2493: Only search up to the first 100 lines in an in...
by Greg Von Kuster 23 Jul '09

23 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/d51e2ea0daf9 changeset: 2493:d51e2ea0daf9 user: Dan Blankenberg <dan(a)bx.psu.edu> date: Thu Jul 23 12:24:12 2009 -0400 description: Only search up to the first 100 lines in an interval file when setting metadata. 1 file(s) affected in this change: lib/galaxy/datatypes/interval.py diffs (25 lines): diff -r 2177d6f7bf80 -r d51e2ea0daf9 lib/galaxy/datatypes/interval.py --- a/lib/galaxy/datatypes/interval.py Thu Jul 23 12:16:44 2009 -0400 +++ b/lib/galaxy/datatypes/interval.py Thu Jul 23 12:24:12 2009 -0400 @@ -75,6 +75,8 @@ """Tries to guess from the line the location number of the column for the chromosome, region start-end and strand""" if dataset.has_data(): + empty_line_count = 0 + num_check_lines = 100 # only check up to this many non empty lines for i, line in enumerate( file( dataset.file_name ) ): line = line.rstrip( '\r\n' ) if line: @@ -131,8 +133,10 @@ dataset.metadata.strandCol = 6 metadata_is_set = True break - if metadata_is_set: - break # Our metadata is set, so break out of the outer loop + if metadata_is_set or ( i - empty_line_count ) > num_check_lines: + break # Our metadata is set or we examined 100 non-empty lines, so break out of the outer loop + else: + empty_line_count += 1 def get_estimated_display_viewport( self, dataset ): """Return a chrom, start, stop tuple for viewing a file."""

1 0

Stopping Jobs
by SHAUN WEBB 23 Jul '09

23 Jul '09

It appears that I have a "stuck" job on our local galaxy server that is causing problems. Galaxy seems unable to upload new files or run tools. I have checked the manage jobs tab in the admin section and there is a job (an upload) that has not been updated for an hour. I have tried to stop this job using the button on this page but that hasn't worked. Is there any other way I can stop or delete this? Thanks Shaun -- The University of Edinburgh is a charitable body, registered in Scotland, with registration number SC005336.

2 1

[hg] galaxy 2490: Fix for editing attributes of library datasets.
by Greg Von Kuster 22 Jul '09

22 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/3851029eda94 changeset: 2490:3851029eda94 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Jul 22 21:28:26 2009 -0400 description: Fix for editing attributes of library datasets. 2 file(s) affected in this change: templates/admin/library/ldda_edit_info.mako templates/library/ldda_edit_info.mako diffs (24 lines): diff -r 98ae7a5023c1 -r 3851029eda94 templates/admin/library/ldda_edit_info.mako --- a/templates/admin/library/ldda_edit_info.mako Wed Jul 22 19:45:03 2009 -0400 +++ b/templates/admin/library/ldda_edit_info.mako Wed Jul 22 21:28:26 2009 -0400 @@ -72,7 +72,7 @@ <div class="form-row"> <label>${spec.desc}:</label> <div style="float: left; width: 250px; margin-right: 10px;"> - ${ldda.metadata.get_html_by_name( name )} + ${ldda.metadata.get_html_by_name( name, trans=trans )} </div> <div style="clear: both"></div> </div> diff -r 98ae7a5023c1 -r 3851029eda94 templates/library/ldda_edit_info.mako --- a/templates/library/ldda_edit_info.mako Wed Jul 22 19:45:03 2009 -0400 +++ b/templates/library/ldda_edit_info.mako Wed Jul 22 21:28:26 2009 -0400 @@ -72,7 +72,7 @@ <div class="form-row"> <label>${spec.desc}:</label> <div style="float: left; width: 250px; margin-right: 10px;"> - ${ldda.metadata.get_html_by_name( name )} + ${ldda.metadata.get_html_by_name( name, trans=trans )} </div> <div style="clear: both"></div> </div>

1 0

[hg] galaxy 2489: Fix for rendering a form field from a metadata...
by Greg Von Kuster 22 Jul '09

22 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/98ae7a5023c1 changeset: 2489:98ae7a5023c1 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Jul 22 19:45:03 2009 -0400 description: Fix for rendering a form field from a metadata spec. 2 file(s) affected in this change: lib/galaxy/datatypes/metadata.py lib/galaxy/web/form_builder.py diffs (86 lines): diff -r ecd5fb7ec59b -r 98ae7a5023c1 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py Mon Jul 20 16:57:55 2009 -0400 +++ b/lib/galaxy/datatypes/metadata.py Wed Jul 22 19:45:03 2009 -0400 @@ -1,6 +1,6 @@ import sys, logging, copy, shutil, weakref, cPickle, tempfile, os -from galaxy.util import string_as_bool, relpath, stringify_dictionary_keys +from galaxy.util import string_as_bool, relpath, stringify_dictionary_keys, listify from galaxy.util.odict import odict from galaxy.web import form_builder import galaxy.model @@ -138,7 +138,7 @@ def __init__( self, spec ): self.spec = spec - def get_html_field( self, value=None, context={}, other_values={}, **kwd ): + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=value ) def get_html( self, value, context={}, other_values={}, **kwd ): @@ -256,9 +256,17 @@ value = [value] return ",".join( map( str, value ) ) - def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ): + def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ): field = form_builder.SelectField( self.spec.name, multiple=self.multiple, display=self.spec.get("display") ) - for val, label in self.values or values or [(val2, val2) for val2 in value]: + if self.values: + value_list = self.values + elif values: + value_list = values + elif value: + value_list = [ ( v, v ) for v in listify( value )] + else: + value_list = [] + for val, label in value_list: try: if ( self.multiple and val in value ) or ( not self.multiple and val == value ): field.add_option( label, val, selected=True ) @@ -266,7 +274,6 @@ field.add_option( label, val, selected=False ) except TypeError: field.add_option( val, label, selected=False ) - return field def get_html( self, value, context={}, other_values={}, values=None, **kwd ): @@ -299,13 +306,12 @@ except KeyError: pass return super(DBKeyParameter, self).get_html_field( value, context, other_values, values, **kwd) - def get_html( self, value=None, context={}, other_values={}, values=None, **kwd): + def get_html( self, value=None, context={}, other_values={}, values=None, **kwd): try: values = kwd['trans'].db_builds except KeyError: pass return super(DBKeyParameter, self).get_html( value, context, other_values, values, **kwd) - class RangeParameter( SelectParameter ): def __init__( self, spec ): diff -r ecd5fb7ec59b -r 98ae7a5023c1 lib/galaxy/web/form_builder.py --- a/lib/galaxy/web/form_builder.py Mon Jul 20 16:57:55 2009 -0400 +++ b/lib/galaxy/web/form_builder.py Wed Jul 22 19:45:03 2009 -0400 @@ -12,9 +12,7 @@ @staticmethod def form_field_types(): return ['TextField', 'TextArea', 'SelectField', 'CheckboxField'] - - - + class TextField(BaseField): """ A standard text input box. @@ -235,8 +233,6 @@ if self.options: return self.options[0] return None - - class DrillDownField( BaseField ): """

1 0

[hg] galaxy 2488: Committing changes to 'extract genomic dna' to...
by Greg Von Kuster 22 Jul '09

22 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/ecd5fb7ec59b changeset: 2488:ecd5fb7ec59b user: guru date: Mon Jul 20 16:57:55 2009 -0400 description: Committing changes to 'extract genomic dna' tool - twobit file will be opened only once per job. (Courtesy: Gordon). 2 file(s) affected in this change: tools/extract/extract_genomic_dna.py tools/extract/extract_genomic_dna.xml diffs (38 lines): diff -r 46bd94b12a0c -r ecd5fb7ec59b tools/extract/extract_genomic_dna.py --- a/tools/extract/extract_genomic_dna.py Thu Jul 16 15:35:58 2009 -0400 +++ b/tools/extract/extract_genomic_dna.py Mon Jul 20 16:57:55 2009 -0400 @@ -70,7 +70,8 @@ fout = open( output_filename, "w" ) warnings = [] warning = '' - + twobitfile = None + for i, line in enumerate( open( input_filename ) ): line = line.rstrip( '\r\n' ) if line and not line.startswith( "#" ): @@ -118,12 +119,10 @@ invalid_line = line continue elif seq_path and os.path.isfile( seq_path ): - if chrom in twobits: - t = twobits[chrom] - else: - twobits[chrom] = t = bx.seq.twobit.TwoBitFile( file( seq_path ) ) + if not(twobitfile): + twobitfile = bx.seq.twobit.TwoBitFile( file( seq_path ) ) try: - sequence = t[chrom][start:end] + sequence = twobitfile[chrom][start:end] except: warning = "Unable to fetch the sequence from '%d' to '%d' for build '%s'. " %( start, end-start, dbkey ) warnings.append( warning ) diff -r 46bd94b12a0c -r ecd5fb7ec59b tools/extract/extract_genomic_dna.xml --- a/tools/extract/extract_genomic_dna.xml Thu Jul 16 15:35:58 2009 -0400 +++ b/tools/extract/extract_genomic_dna.xml Mon Jul 20 16:57:55 2009 -0400 @@ -1,4 +1,4 @@ -<tool id="Extract genomic DNA 1" name="Extract Genomic DNA" version="2.1.0"> +<tool id="Extract genomic DNA 1" name="Extract Genomic DNA" version="2.2.0"> <description>using coordinates from assembled/unassembled genomes</description> <command interpreter="python">extract_genomic_dna.py $input $out_file1 -1 ${input.metadata.chromCol},${input.metadata.startCol},${input.metadata.endCol},${input.metadata.strandCol} -d $dbkey -o $out_format -g ${GALAXY_DATA_INDEX_DIR}</command> <inputs>

1 0

Simplified security model
by Assaf Gordon 22 Jul '09

22 Jul '09

Hello, Continuing the security discussion, here's a suggestion for a simplified security model: 1. no anonymous users (login required) 2. no roles, or groups. 3. no public data. 4. if a user has the dataset in one of his histories, he can do anything with it (including sharing it). 5. if a user doesn't have the dataset in one of his histories, he can't access it. 6. Administrators can access everything. This model fits a small lab (I hope), and assumes the following: 1. no security administration what-so-ever. 2. each user can access only his own data. nothing is public. 3. users are reasonably responsible: if they decide to share their files with somebody, they know what they are doing. 4. If users share history with somebody else, they don't have control over it any more (the other user can share it with more users) - so they should exercise caution before sharing. note: 'libraries' are not really incorporated into this model. Below is the patch with the new code. A new class is added to ./lib/galaxy/security/__init__.py, which does the permission checking. The added 'app' parameter is needed to access the config object and find out who the administrators are. Comments are welcomed, -Gordon. ================================== diff -r 46bd94b12a0c lib/galaxy/app.py --- a/lib/galaxy/app.py Thu Jul 16 15:35:58 2009 -0400 +++ b/lib/galaxy/app.py Wed Jul 22 20:40:56 2009 -0400 @@ -28,7 +28,8 @@ create_or_verify_database( db_url, self.config.database_engine_options ) # Setup the database engine and ORM from galaxy.model import mapping - self.model = mapping.init( self.config.file_path, + self.model = mapping.init( self, + self.config.file_path, db_url, self.config.database_engine_options ) # Security helper diff -r 46bd94b12a0c lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Thu Jul 16 15:35:58 2009 -0400 +++ b/lib/galaxy/model/mapping.py Wed Jul 22 20:40:56 2009 -0400 @@ -14,6 +14,7 @@ from galaxy.model.custom_types import * from galaxy.util.bunch import Bunch from galaxy.security import GalaxyRBACAgent +from galaxy.security import GalaxyRBACAgent_Gordon metadata = MetaData() context = Session = scoped_session( sessionmaker( autoflush=False, transactional=False ) ) @@ -1046,7 +1047,7 @@ # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) -def init( file_path, url, engine_options={}, create_tables=False ): +def init( app, file_path, url, engine_options={}, create_tables=False ): """Connect mappings to the database""" # Connect dataset to the file path Dataset.file_path = file_path @@ -1072,7 +1073,7 @@ result.context = Session result.create_tables = create_tables #load local galaxy security policy - result.security_agent = GalaxyRBACAgent( result ) + result.security_agent = GalaxyRBACAgent_Gordon( app, result ) return result def get_suite(): diff -r 46bd94b12a0c lib/galaxy/security/__init__.py --- a/lib/galaxy/security/__init__.py Thu Jul 16 15:35:58 2009 -0400 +++ b/lib/galaxy/security/__init__.py Wed Jul 22 20:40:56 2009 -0400 @@ -5,6 +5,11 @@ import logging from galaxy.util.bunch import Bunch from galaxy.model.orm import * +# gordon: these are needed for new security model +import sys +import sqlalchemy as sa +import galaxy.model +import galaxy.web log = logging.getLogger(__name__) @@ -484,6 +489,122 @@ else: raise 'Passed an illegal object to check_folder_contents: %s' % type( entry ) + +#Hack by gordon(a)cshl.edu: +# new, simplified security model +# +# There are no public datasets. +# If the user has a dataset in one of his histories, he can do everything to it (including sharing) +# If the user doesn't have the dataset in onw of his histories, the dataset is blocked. +# +# roles and groups are ignored. +class GalaxyRBACAgent_Gordon( RBACAgent ): + def __init__( self, app, model, permitted_actions=None ): + self.model = model + self.app = app + if permitted_actions: + self.permitted_actions = permitted_actions + def allow_action( self, user, action, **kwd ): + if 'dataset' in kwd: + return self.allow_dataset_action( user, action, kwd[ 'dataset' ] ) + raise 'No valid method of checking action (%s) for user %s using kwd %s' % ( action, str( user ), str( kwd ) ) + def allow_dataset_action( self, user, action, dataset ): + """Returns true when user has permission to perform an action""" + # This security model requires user to be logged on. no user - no access! + if not user: + return False + # Administrators can access everything (that's why they're administators) + if self.app.config.is_admin_user(user): + return True + if not isinstance( dataset, self.model.Dataset ): + dataset = dataset.dataset + """ + Perform the equivalent of the following SQL query: + select 'yes' from + history_dataset_association, + history + where + history_dataset_association.dataset_id = [dataset.id] + and + history_dataset_association.history_id = history.id + and + history.user_id = [user.id] + If we get a 'yes' - then this user has this dataset in one of his histories, + and access is allowed. + """ + q = sa.select ( ( galaxy.model.HistoryDatasetAssociation.table.c.id, galaxy.model.History.c.user_id ), + from_obj = [ galaxy.model.HistoryDatasetAssociation.table.join ( + galaxy.model.History.table, + galaxy.model.HistoryDatasetAssociation.history_id == galaxy.model.History.id ) ], + whereclause = ( and_( + galaxy.model.HistoryDatasetAssociation.dataset_id == dataset.id, + galaxy.model.History.user_id == user.id )) + ) + #Surely there's a better way to check if we got results or not... + found = False + for row in q.execute(): + found = True + break + #sys.stderr.write( "!!! user_id = %d\nhda_id = %d\n" % (user.id, dataset.id) ) + #sys.stderr.write( "!!! Query = %s\nFound = %s\n" % (str(q), str(found)) ) + # if the user has this dataset in one of his histories - allow him access + if found: + return True + return False + def allow_library_item_action( self, user, action, library_item ): + return False + def guess_derived_permissions_for_datasets( self, datasets=[] ): + perms = {} + return perms + def associate_components( self, **kwd ): + raise 'No valid method of associating provided components: %s' % kwd + def create_private_user_role( self, user ): + return None + def get_private_user_role( self, user, auto_create=False ): + return None + def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False, bypass_manage_permission=False ): + return None + def user_get_default_permissions( self, user ): + permissions = {} + return permissions + def history_set_default_permissions( self, history, permissions={}, dataset=False, bypass_manage_permission=False ): + return + def history_get_default_permissions( self, history ): + permissions = {} + return permissions + def set_all_dataset_permissions( self, dataset, permissions={} ): + return + def set_dataset_permission( self, dataset, permission={} ): + return + def dataset_is_public( self, dataset ): + return True + def make_dataset_public( self, dataset ): + return + def get_dataset_permissions( self, dataset ): + permissions = {} + return permissions + def copy_dataset_permissions( self, src, dst ): + return + def privately_share_dataset( self, dataset, users = [] ): + return + def set_all_library_permissions( self, library_item, permissions={} ): + return + def get_library_dataset_permissions( self, library_dataset ): + permissions = {} + return permissions + def copy_library_permissions( self, source_library_item, target_library_item, user=None ): + return + def show_library_item( self, user, library_item ): + return True + def set_entity_user_associations( self, users=[], roles=[], groups=[], delete_existing_assocs=True ): + return + def set_entity_group_associations( self, groups=[], users=[], roles=[], delete_existing_assocs=True ): + return + def set_entity_role_associations( self, roles=[], users=[], groups=[], delete_existing_assocs=True ): + return + def check_folder_contents( self, user, entry ): + return True + def get_permitted_actions( filter=None ): '''Utility method to return a subset of RBACAgent's permitted actions''' if filter is None:

1 0

Re: [galaxy-dev] [galaxy-user] Experience with Loading NGS data on standalone instance of galaxy
by Assaf Gordon 21 Jul '09

21 Jul '09

Hello, If I may suggest a workaround for getting big files fast into galaxy: For a local galaxy server, and local files (on the same server as the running galaxy), there is indeed no need to upload the files. What we did is to create a simple tool, which accepts a local path, and copies the file from the local path into the galaxy database. Local file copy is much quicker than uploading the file through HTTP. We need a full file copy because the source files are routinely deleted, but if the source files are kept 'forever', you can modify the tool to create a soft/hard link to the local file - that would be almost instantaneous. Here's an example of such tool: ### ( scifi.cshl.edu is mounted as SMB, so for all practice purposes it behaves like local files ) ### ### The XML file: import_scifi_file.xml ### <tool id="cshl_import_scifi_file" name="Import SciFi file"> <command interpreter="sh">import_scifi_file.sh '$filepath' $output</command> <inputs> <param name="filepath" type="text" size="100" label="File path (on \\Scifi.cshl.edu\hannon )" /> </inputs> <outputs> <data format="txt" name="output" label="Scifi Import: $filepath" /> </outputs> </tool> ### ### The shell script: import_scifi_file.sh ### ### The script basically copies the source file (param 1) ### to the destination file (param 2, which is galaxy's dataset_NNNNN.dat). ### ### The extra code tries to make it as safe as possible, allowing imports only ### from /media/scifi ### ### The script can be changed from copying to linking - would be even faster. ### #!/bin/sh SCIFI_BASE_DIR="/media/scifi" BASE_DIR_LEN=${#SCIFI_BASE_DIR} INPUT="$1" OUTPUT="$2" if [ -z "$OUTPUT" ]; then echo "Usage: $0 [INPUT] [OUTPUT]" >&2 exit 1 fi if [ ! -d "${SCIFI_BASE_DIR}" ]; then echo "Internal Error: Scifi is not mounted on '${SCIFI_BASE_DIR}'" >&2 exit 1 fi FULLPATH="$INPUT" # Convert backslashes (possibly pasted from windows machines) into forward slashes FULLPATH=${FULLPATH//\\/\/} # Remove server prefix (possibly pasted by the user on a windows machine) FULLPATH=${FULLPATH/\/\/scifi\.cshl\.edu\/hannon\//} FULLPATH=${FULLPATH/\/\/scifi\/hannon\//} #Construct full path with "/media/scifi" prefix FULLPATH="${SCIFI_BASE_DIR}/${FULLPATH}" # Safety check - # change to the directory of the requested file. # It should begin with "/media/scifi". # If it doesn't, it means somebody tried to pull a trick by using # a bad mixture of "../../../../.." in the file path. DIRECTORY=$(dirname "$FULLPATH") pushd "$DIRECTORY" > /dev/null REALDIR=$(pwd) popd > /dev/null #extract the prefix from the 'real' directory path REAL_DIR_PREFIX=${REALDIR:0:$BASE_DIR_LEN} #DEBUG #echo "FULLPATH = $FULLPATH" #echo "DIRECTORY= $DIRECTORY" #echo "REALDIR = $REALDIR" #echo "REAL_DIR_PREFIX = $REAL_DIR_PREFIX" #echo "BASE_DIR = $SCIFI_BASE_DIR" # Probably foul play: # the real path of the requested input file does not start with the prefix # of '/media/scifi' - maybe somebody's trying to get a file outside 'scifi' ?? if [ "$REAL_DIR_PREFIX" != "$SCIFI_BASE_DIR" ]; then echo "Error: invalid input file ($INPUT)" >&2 exit 1 fi # If we got here, the $FULLPATH is at least in a valid location under the '/media/scifi' directory. if [ ! -r "$FULLPATH" ]; then echo "Error: input file ($INPUT) is not a valid file." >&2 exit 1 fi cp "$FULLPATH" "$OUTPUT" if [ $? != 0 ]; then echo "Error: failed to copy \"$INPUT\"!" >&2 exit 1 fi echo "File \"$INPUT\" Imported." exit 0 ### ### ### A further improvement is not to allow free-text file path, but instead use dynamic options to select from a list of files, as so (in the XML file): <param name="localfile" type="select" label="Solexa Data File"> <options from_file="cshl_import_files_hannon.txt"> <column name="name" index="1"/> <column name="value" index="0"/> </options> </param> And then have a cron job to create 'cshl_import_files_hannon.txt' with files which can be uploaded. Hope this helps, -gordon. Greg Von Kuster wrote, On 07/21/2009 10:44 AM: > Hello Abhi, > > Can you clarify the steps you took that produced the behavior? See my > comments below. > > Anton Nekrutenko wrote: >> Abhishek: >> >> Let talk. This is the area of active current development. We are >> looking at implementing a universal fastq-like format or supporting >> multiple formats. Perhaps we should join efforts in ironing out >> specifications. >> >> anton >> galaxy team >> >> >> On Jul 20, 2009, at 5:18 PM, Abhishek Pratap wrote: >> >>> Hi All >>> >>> >>> I recently came to know about NGS analysis on galaxy during ISMB. >>> Getting excited I tried couple of things basically to play with it. >>> >>> Few comments : I may have interepretted something described below in a >>> wrong way. My apologies before hand. >>> >>> >>> >>> On a standalone installation of galaxy while I was trying to explore >>> one FASTQ(sequence) file. It takes considerable (> 20 min) for a fastq >>> file to get uploaded (2 GB). > > Are you using the Galaxy upload utility to create an item in your > history that points to the dataset file on disk? > > I am not sure what is the rationale >>> behind that. Ideally I think there should be no need to upload such >>> heavy files into the workspace. > > A data file that originates from a place external to Galaxy must be > uploaded into Galaxy so that the disk file can be placed in the location > configured in the Galaxy config file. Also, when data is uploaded to > Galaxy ( either to a history or a library ), several database table > settings are created that are used by various Galaxy features. > > They could actually be used straight >>> away by the path specified. > > What do you mean by "the path specified"? > > Also is there any way to access the >>> scripts for analysis on the command line. I know this undermines the >>> main aim of working with galaxy but rite now I am concerned about the >>> performance/time. > > You should be able to run any Galaxy tool from the command line as long > as you have all of the tool's required binaries in your path. However, > running a tool from within Galaxy should generally not be any slower > than running it outside of Galaxy, depending, of course, on what you are > doing. > > >>> I will be happy to discuss more about this in case you have some >>> comments/questions for me. >>> >>> >>> >>> Best, >>> -Abhi >>> >>> >>> >>> ----------------------------- >>> >>> Abhishek Pratap >>> >>> Bioinformatics Software Engineer >>> >>> Institute for Genome Sciences >>> >>> School of Medicine, Univ of Maryland >>> >>> 801, W. Baltimore Street, Baltimore, MD 21209 >>> >>> Ph: (+1)-410-706-2296 >>> >>> www.igs.umaryland.edu/ >>> _______________________________________________ >>> galaxy-user mailing list >>> galaxy-user(a)bx.psu.edu >>> http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-user >> Anton Nekrutenko >> http://nekrut.bx.psu.edu >> http://galaxyproject.org >> >> _______________________________________________ >> galaxy-user mailing list >> galaxy-user(a)bx.psu.edu >> http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-user >> >> > _______________________________________________ > galaxy-user mailing list > galaxy-user(a)bx.psu.edu > http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-user

2 1