- galaxy-dev - lists.galaxyproject.org

[hg] galaxy 2492: Performance improvements for fasta_filter_by_l...
by Greg Von Kuster 23 Jul '09

23 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/2177d6f7bf80 changeset: 2492:2177d6f7bf80 user: Kelly Vincent <kpvincent(a)bx.psu.edu> date: Thu Jul 23 12:16:44 2009 -0400 description: Performance improvements for fasta_filter_by_length and fasta_to_tabular 2 file(s) affected in this change: tools/fasta_tools/fasta_filter_by_length.py tools/fasta_tools/fasta_to_tabular.py diffs (143 lines): diff -r cd366f615220 -r 2177d6f7bf80 tools/fasta_tools/fasta_filter_by_length.py --- a/tools/fasta_tools/fasta_filter_by_length.py Thu Jul 23 11:11:41 2009 -0400 +++ b/tools/fasta_tools/fasta_filter_by_length.py Thu Jul 23 12:16:44 2009 -0400 @@ -14,8 +14,7 @@ sys.exit() def __main__(): - - infile = sys.argv[1] + input_filename = sys.argv[1] try: min_length = int( sys.argv[2] ) except: @@ -24,65 +23,30 @@ max_length = int( sys.argv[3] ) except: stop_err( "Maximum length of the return sequence requires a numerical value." ) - outfile = sys.argv[4] - fasta_title = fasta_seq = '' + output_filename = sys.argv[4] + output_handle = open( output_filename, 'w' ) + tmp_size = 0 #-1 + tmp_buf = '' at_least_one = 0 - - out = open( outfile, 'w' ) - - for i, line in enumerate( file( infile ) ): - line = line.rstrip( '\r\n' ) - if not line or line.startswith( '#' ): + for line in file(input_filename): + if not line or line.startswith('#'): continue - if line[0] == '>': - if len( fasta_seq ) > 0: - - if max_length <= 0: - compare_max_length = len( fasta_seq ) + 1 - else: - compare_max_length = max_length - - l = len( fasta_seq ) - - if l >= min_length and l <= compare_max_length: - at_least_one += 1 - out.write( "%s\n" % fasta_title ) - c = 0 - s = fasta_seq - while c < l: - b = min( c + 50, l ) - out.write( "%s\n" % s[ c:b ] ) - c = b - - fasta_title = line - fasta_seq = '' + if min_length <= tmp_size <= max_length or (min_length <= tmp_size and max_length == 0): + output_handle.write(tmp_buf) + at_least_one = 1 + tmp_buf = line + tmp_size = 0 else: - fasta_seq = "%s%s" % ( fasta_seq, line ) - - if len( fasta_seq ) > 0: - - if max_length <= 0: - compare_max_length = len( fasta_seq ) + 1 - else: - compare_max_length = max_length - - l = len( fasta_seq ) - - if l >= min_length and l <= compare_max_length: - at_least_one += 1 - out.write( "%s\n" % fasta_title ) - c = 0 - s = fasta_seq - while c < l: - b = min( c + 50, l ) - out.write( "%s\n" % s[ c:b ] ) - c = b - - out.close() - + if max_length == 0 or tmp_size < max_length: + tmp_size += len(line.rstrip('\r\n')) + tmp_buf += line + # final flush of buffer + if min_length <= tmp_size <= max_length or (min_length <= tmp_size and max_length == 0): + output_handle.write(tmp_buf.rstrip('\r\n')) + at_least_one = 1 + output_handle.close() if at_least_one == 0: print "There is no sequence that falls within your range." - -if __name__ == "__main__" : __main__() \ No newline at end of file +if __name__ == "__main__" : __main__() diff -r cd366f615220 -r 2177d6f7bf80 tools/fasta_tools/fasta_to_tabular.py --- a/tools/fasta_tools/fasta_to_tabular.py Thu Jul 23 11:11:41 2009 -0400 +++ b/tools/fasta_tools/fasta_to_tabular.py Thu Jul 23 12:16:44 2009 -0400 @@ -15,30 +15,23 @@ outfile = sys.argv[2] keep_first = int( sys.argv[3] ) fasta_title = fasta_seq = '' - if keep_first == 0: keep_first = None else: keep_first += 1 - out = open( outfile, 'w' ) - for i, line in enumerate( open( infile ) ): line = line.rstrip( '\r\n' ) if not line or line.startswith( '#' ): continue if line.startswith( '>' ): - if fasta_seq: - out.write( "%s\t%s\n" %( fasta_title[ 1:keep_first ], fasta_seq ) ) - fasta_title = line - fasta_seq = '' + if i: + out.write('\n') + out.write(line[1:keep_first]) + out.write('\t') else: - if line: - fasta_seq = "%s%s" % ( fasta_seq, line ) - - if fasta_seq: - out.write( "%s\t%s\n" %( fasta_title[ 1:keep_first ], fasta_seq ) ) - + out.write(line) out.close() -if __name__ == "__main__" : __main__() \ No newline at end of file +if __name__ == "__main__" : __main__() + \ No newline at end of file

1 0

Stopping Jobs
by SHAUN WEBB 23 Jul '09

23 Jul '09

It appears that I have a "stuck" job on our local galaxy server that is causing problems. Galaxy seems unable to upload new files or run tools. I have checked the manage jobs tab in the admin section and there is a job (an upload) that has not been updated for an hour. I have tried to stop this job using the button on this page but that hasn't worked. Is there any other way I can stop or delete this? Thanks Shaun -- The University of Edinburgh is a charitable body, registered in Scotland, with registration number SC005336.

2 1

[hg] galaxy 2490: Fix for editing attributes of library datasets.
by Greg Von Kuster 23 Jul '09

23 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/3851029eda94 changeset: 2490:3851029eda94 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Jul 22 21:28:26 2009 -0400 description: Fix for editing attributes of library datasets. 2 file(s) affected in this change: templates/admin/library/ldda_edit_info.mako templates/library/ldda_edit_info.mako diffs (24 lines): diff -r 98ae7a5023c1 -r 3851029eda94 templates/admin/library/ldda_edit_info.mako --- a/templates/admin/library/ldda_edit_info.mako Wed Jul 22 19:45:03 2009 -0400 +++ b/templates/admin/library/ldda_edit_info.mako Wed Jul 22 21:28:26 2009 -0400 @@ -72,7 +72,7 @@ <div class="form-row"> <label>${spec.desc}:</label> <div style="float: left; width: 250px; margin-right: 10px;"> - ${ldda.metadata.get_html_by_name( name )} + ${ldda.metadata.get_html_by_name( name, trans=trans )} </div> <div style="clear: both"></div> </div> diff -r 98ae7a5023c1 -r 3851029eda94 templates/library/ldda_edit_info.mako --- a/templates/library/ldda_edit_info.mako Wed Jul 22 19:45:03 2009 -0400 +++ b/templates/library/ldda_edit_info.mako Wed Jul 22 21:28:26 2009 -0400 @@ -72,7 +72,7 @@ <div class="form-row"> <label>${spec.desc}:</label> <div style="float: left; width: 250px; margin-right: 10px;"> - ${ldda.metadata.get_html_by_name( name )} + ${ldda.metadata.get_html_by_name( name, trans=trans )} </div> <div style="clear: both"></div> </div>

1 0

[hg] galaxy 2489: Fix for rendering a form field from a metadata...
by Greg Von Kuster 23 Jul '09

23 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/98ae7a5023c1 changeset: 2489:98ae7a5023c1 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Wed Jul 22 19:45:03 2009 -0400 description: Fix for rendering a form field from a metadata spec. 2 file(s) affected in this change: lib/galaxy/datatypes/metadata.py lib/galaxy/web/form_builder.py diffs (86 lines): diff -r ecd5fb7ec59b -r 98ae7a5023c1 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py Mon Jul 20 16:57:55 2009 -0400 +++ b/lib/galaxy/datatypes/metadata.py Wed Jul 22 19:45:03 2009 -0400 @@ -1,6 +1,6 @@ import sys, logging, copy, shutil, weakref, cPickle, tempfile, os -from galaxy.util import string_as_bool, relpath, stringify_dictionary_keys +from galaxy.util import string_as_bool, relpath, stringify_dictionary_keys, listify from galaxy.util.odict import odict from galaxy.web import form_builder import galaxy.model @@ -138,7 +138,7 @@ def __init__( self, spec ): self.spec = spec - def get_html_field( self, value=None, context={}, other_values={}, **kwd ): + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=value ) def get_html( self, value, context={}, other_values={}, **kwd ): @@ -256,9 +256,17 @@ value = [value] return ",".join( map( str, value ) ) - def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ): + def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ): field = form_builder.SelectField( self.spec.name, multiple=self.multiple, display=self.spec.get("display") ) - for val, label in self.values or values or [(val2, val2) for val2 in value]: + if self.values: + value_list = self.values + elif values: + value_list = values + elif value: + value_list = [ ( v, v ) for v in listify( value )] + else: + value_list = [] + for val, label in value_list: try: if ( self.multiple and val in value ) or ( not self.multiple and val == value ): field.add_option( label, val, selected=True ) @@ -266,7 +274,6 @@ field.add_option( label, val, selected=False ) except TypeError: field.add_option( val, label, selected=False ) - return field def get_html( self, value, context={}, other_values={}, values=None, **kwd ): @@ -299,13 +306,12 @@ except KeyError: pass return super(DBKeyParameter, self).get_html_field( value, context, other_values, values, **kwd) - def get_html( self, value=None, context={}, other_values={}, values=None, **kwd): + def get_html( self, value=None, context={}, other_values={}, values=None, **kwd): try: values = kwd['trans'].db_builds except KeyError: pass return super(DBKeyParameter, self).get_html( value, context, other_values, values, **kwd) - class RangeParameter( SelectParameter ): def __init__( self, spec ): diff -r ecd5fb7ec59b -r 98ae7a5023c1 lib/galaxy/web/form_builder.py --- a/lib/galaxy/web/form_builder.py Mon Jul 20 16:57:55 2009 -0400 +++ b/lib/galaxy/web/form_builder.py Wed Jul 22 19:45:03 2009 -0400 @@ -12,9 +12,7 @@ @staticmethod def form_field_types(): return ['TextField', 'TextArea', 'SelectField', 'CheckboxField'] - - - + class TextField(BaseField): """ A standard text input box. @@ -235,8 +233,6 @@ if self.options: return self.options[0] return None - - class DrillDownField( BaseField ): """

1 0

[hg] galaxy 2488: Committing changes to 'extract genomic dna' to...
by Greg Von Kuster 23 Jul '09

23 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/ecd5fb7ec59b changeset: 2488:ecd5fb7ec59b user: guru date: Mon Jul 20 16:57:55 2009 -0400 description: Committing changes to 'extract genomic dna' tool - twobit file will be opened only once per job. (Courtesy: Gordon). 2 file(s) affected in this change: tools/extract/extract_genomic_dna.py tools/extract/extract_genomic_dna.xml diffs (38 lines): diff -r 46bd94b12a0c -r ecd5fb7ec59b tools/extract/extract_genomic_dna.py --- a/tools/extract/extract_genomic_dna.py Thu Jul 16 15:35:58 2009 -0400 +++ b/tools/extract/extract_genomic_dna.py Mon Jul 20 16:57:55 2009 -0400 @@ -70,7 +70,8 @@ fout = open( output_filename, "w" ) warnings = [] warning = '' - + twobitfile = None + for i, line in enumerate( open( input_filename ) ): line = line.rstrip( '\r\n' ) if line and not line.startswith( "#" ): @@ -118,12 +119,10 @@ invalid_line = line continue elif seq_path and os.path.isfile( seq_path ): - if chrom in twobits: - t = twobits[chrom] - else: - twobits[chrom] = t = bx.seq.twobit.TwoBitFile( file( seq_path ) ) + if not(twobitfile): + twobitfile = bx.seq.twobit.TwoBitFile( file( seq_path ) ) try: - sequence = t[chrom][start:end] + sequence = twobitfile[chrom][start:end] except: warning = "Unable to fetch the sequence from '%d' to '%d' for build '%s'. " %( start, end-start, dbkey ) warnings.append( warning ) diff -r 46bd94b12a0c -r ecd5fb7ec59b tools/extract/extract_genomic_dna.xml --- a/tools/extract/extract_genomic_dna.xml Thu Jul 16 15:35:58 2009 -0400 +++ b/tools/extract/extract_genomic_dna.xml Mon Jul 20 16:57:55 2009 -0400 @@ -1,4 +1,4 @@ -<tool id="Extract genomic DNA 1" name="Extract Genomic DNA" version="2.1.0"> +<tool id="Extract genomic DNA 1" name="Extract Genomic DNA" version="2.2.0"> <description>using coordinates from assembled/unassembled genomes</description> <command interpreter="python">extract_genomic_dna.py $input $out_file1 -1 ${input.metadata.chromCol},${input.metadata.startCol},${input.metadata.endCol},${input.metadata.strandCol} -d $dbkey -o $out_format -g ${GALAXY_DATA_INDEX_DIR}</command> <inputs>

1 0

Simplified security model
by Assaf Gordon 23 Jul '09

23 Jul '09

Hello, Continuing the security discussion, here's a suggestion for a simplified security model: 1. no anonymous users (login required) 2. no roles, or groups. 3. no public data. 4. if a user has the dataset in one of his histories, he can do anything with it (including sharing it). 5. if a user doesn't have the dataset in one of his histories, he can't access it. 6. Administrators can access everything. This model fits a small lab (I hope), and assumes the following: 1. no security administration what-so-ever. 2. each user can access only his own data. nothing is public. 3. users are reasonably responsible: if they decide to share their files with somebody, they know what they are doing. 4. If users share history with somebody else, they don't have control over it any more (the other user can share it with more users) - so they should exercise caution before sharing. note: 'libraries' are not really incorporated into this model. Below is the patch with the new code. A new class is added to ./lib/galaxy/security/__init__.py, which does the permission checking. The added 'app' parameter is needed to access the config object and find out who the administrators are. Comments are welcomed, -Gordon. ================================== diff -r 46bd94b12a0c lib/galaxy/app.py --- a/lib/galaxy/app.py Thu Jul 16 15:35:58 2009 -0400 +++ b/lib/galaxy/app.py Wed Jul 22 20:40:56 2009 -0400 @@ -28,7 +28,8 @@ create_or_verify_database( db_url, self.config.database_engine_options ) # Setup the database engine and ORM from galaxy.model import mapping - self.model = mapping.init( self.config.file_path, + self.model = mapping.init( self, + self.config.file_path, db_url, self.config.database_engine_options ) # Security helper diff -r 46bd94b12a0c lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py Thu Jul 16 15:35:58 2009 -0400 +++ b/lib/galaxy/model/mapping.py Wed Jul 22 20:40:56 2009 -0400 @@ -14,6 +14,7 @@ from galaxy.model.custom_types import * from galaxy.util.bunch import Bunch from galaxy.security import GalaxyRBACAgent +from galaxy.security import GalaxyRBACAgent_Gordon metadata = MetaData() context = Session = scoped_session( sessionmaker( autoflush=False, transactional=False ) ) @@ -1046,7 +1047,7 @@ # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) -def init( file_path, url, engine_options={}, create_tables=False ): +def init( app, file_path, url, engine_options={}, create_tables=False ): """Connect mappings to the database""" # Connect dataset to the file path Dataset.file_path = file_path @@ -1072,7 +1073,7 @@ result.context = Session result.create_tables = create_tables #load local galaxy security policy - result.security_agent = GalaxyRBACAgent( result ) + result.security_agent = GalaxyRBACAgent_Gordon( app, result ) return result def get_suite(): diff -r 46bd94b12a0c lib/galaxy/security/__init__.py --- a/lib/galaxy/security/__init__.py Thu Jul 16 15:35:58 2009 -0400 +++ b/lib/galaxy/security/__init__.py Wed Jul 22 20:40:56 2009 -0400 @@ -5,6 +5,11 @@ import logging from galaxy.util.bunch import Bunch from galaxy.model.orm import * +# gordon: these are needed for new security model +import sys +import sqlalchemy as sa +import galaxy.model +import galaxy.web log = logging.getLogger(__name__) @@ -484,6 +489,122 @@ else: raise 'Passed an illegal object to check_folder_contents: %s' % type( entry ) + +#Hack by gordon(a)cshl.edu: +# new, simplified security model +# +# There are no public datasets. +# If the user has a dataset in one of his histories, he can do everything to it (including sharing) +# If the user doesn't have the dataset in onw of his histories, the dataset is blocked. +# +# roles and groups are ignored. +class GalaxyRBACAgent_Gordon( RBACAgent ): + def __init__( self, app, model, permitted_actions=None ): + self.model = model + self.app = app + if permitted_actions: + self.permitted_actions = permitted_actions + def allow_action( self, user, action, **kwd ): + if 'dataset' in kwd: + return self.allow_dataset_action( user, action, kwd[ 'dataset' ] ) + raise 'No valid method of checking action (%s) for user %s using kwd %s' % ( action, str( user ), str( kwd ) ) + def allow_dataset_action( self, user, action, dataset ): + """Returns true when user has permission to perform an action""" + # This security model requires user to be logged on. no user - no access! + if not user: + return False + # Administrators can access everything (that's why they're administators) + if self.app.config.is_admin_user(user): + return True + if not isinstance( dataset, self.model.Dataset ): + dataset = dataset.dataset + """ + Perform the equivalent of the following SQL query: + select 'yes' from + history_dataset_association, + history + where + history_dataset_association.dataset_id = [dataset.id] + and + history_dataset_association.history_id = history.id + and + history.user_id = [user.id] + If we get a 'yes' - then this user has this dataset in one of his histories, + and access is allowed. + """ + q = sa.select ( ( galaxy.model.HistoryDatasetAssociation.table.c.id, galaxy.model.History.c.user_id ), + from_obj = [ galaxy.model.HistoryDatasetAssociation.table.join ( + galaxy.model.History.table, + galaxy.model.HistoryDatasetAssociation.history_id == galaxy.model.History.id ) ], + whereclause = ( and_( + galaxy.model.HistoryDatasetAssociation.dataset_id == dataset.id, + galaxy.model.History.user_id == user.id )) + ) + #Surely there's a better way to check if we got results or not... + found = False + for row in q.execute(): + found = True + break + #sys.stderr.write( "!!! user_id = %d\nhda_id = %d\n" % (user.id, dataset.id) ) + #sys.stderr.write( "!!! Query = %s\nFound = %s\n" % (str(q), str(found)) ) + # if the user has this dataset in one of his histories - allow him access + if found: + return True + return False + def allow_library_item_action( self, user, action, library_item ): + return False + def guess_derived_permissions_for_datasets( self, datasets=[] ): + perms = {} + return perms + def associate_components( self, **kwd ): + raise 'No valid method of associating provided components: %s' % kwd + def create_private_user_role( self, user ): + return None + def get_private_user_role( self, user, auto_create=False ): + return None + def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False, bypass_manage_permission=False ): + return None + def user_get_default_permissions( self, user ): + permissions = {} + return permissions + def history_set_default_permissions( self, history, permissions={}, dataset=False, bypass_manage_permission=False ): + return + def history_get_default_permissions( self, history ): + permissions = {} + return permissions + def set_all_dataset_permissions( self, dataset, permissions={} ): + return + def set_dataset_permission( self, dataset, permission={} ): + return + def dataset_is_public( self, dataset ): + return True + def make_dataset_public( self, dataset ): + return + def get_dataset_permissions( self, dataset ): + permissions = {} + return permissions + def copy_dataset_permissions( self, src, dst ): + return + def privately_share_dataset( self, dataset, users = [] ): + return + def set_all_library_permissions( self, library_item, permissions={} ): + return + def get_library_dataset_permissions( self, library_dataset ): + permissions = {} + return permissions + def copy_library_permissions( self, source_library_item, target_library_item, user=None ): + return + def show_library_item( self, user, library_item ): + return True + def set_entity_user_associations( self, users=[], roles=[], groups=[], delete_existing_assocs=True ): + return + def set_entity_group_associations( self, groups=[], users=[], roles=[], delete_existing_assocs=True ): + return + def set_entity_role_associations( self, roles=[], users=[], groups=[], delete_existing_assocs=True ): + return + def check_folder_contents( self, user, entry ): + return True + def get_permitted_actions( filter=None ): '''Utility method to return a subset of RBACAgent's permitted actions''' if filter is None:

1 0

Re: [galaxy-dev] [galaxy-user] Experience with Loading NGS data on standalone instance of galaxy
by Assaf Gordon 21 Jul '09

21 Jul '09

Hello, If I may suggest a workaround for getting big files fast into galaxy: For a local galaxy server, and local files (on the same server as the running galaxy), there is indeed no need to upload the files. What we did is to create a simple tool, which accepts a local path, and copies the file from the local path into the galaxy database. Local file copy is much quicker than uploading the file through HTTP. We need a full file copy because the source files are routinely deleted, but if the source files are kept 'forever', you can modify the tool to create a soft/hard link to the local file - that would be almost instantaneous. Here's an example of such tool: ### ( scifi.cshl.edu is mounted as SMB, so for all practice purposes it behaves like local files ) ### ### The XML file: import_scifi_file.xml ### <tool id="cshl_import_scifi_file" name="Import SciFi file"> <command interpreter="sh">import_scifi_file.sh '$filepath' $output</command> <inputs> <param name="filepath" type="text" size="100" label="File path (on \\Scifi.cshl.edu\hannon )" /> </inputs> <outputs> <data format="txt" name="output" label="Scifi Import: $filepath" /> </outputs> </tool> ### ### The shell script: import_scifi_file.sh ### ### The script basically copies the source file (param 1) ### to the destination file (param 2, which is galaxy's dataset_NNNNN.dat). ### ### The extra code tries to make it as safe as possible, allowing imports only ### from /media/scifi ### ### The script can be changed from copying to linking - would be even faster. ### #!/bin/sh SCIFI_BASE_DIR="/media/scifi" BASE_DIR_LEN=${#SCIFI_BASE_DIR} INPUT="$1" OUTPUT="$2" if [ -z "$OUTPUT" ]; then echo "Usage: $0 [INPUT] [OUTPUT]" >&2 exit 1 fi if [ ! -d "${SCIFI_BASE_DIR}" ]; then echo "Internal Error: Scifi is not mounted on '${SCIFI_BASE_DIR}'" >&2 exit 1 fi FULLPATH="$INPUT" # Convert backslashes (possibly pasted from windows machines) into forward slashes FULLPATH=${FULLPATH//\\/\/} # Remove server prefix (possibly pasted by the user on a windows machine) FULLPATH=${FULLPATH/\/\/scifi\.cshl\.edu\/hannon\//} FULLPATH=${FULLPATH/\/\/scifi\/hannon\//} #Construct full path with "/media/scifi" prefix FULLPATH="${SCIFI_BASE_DIR}/${FULLPATH}" # Safety check - # change to the directory of the requested file. # It should begin with "/media/scifi". # If it doesn't, it means somebody tried to pull a trick by using # a bad mixture of "../../../../.." in the file path. DIRECTORY=$(dirname "$FULLPATH") pushd "$DIRECTORY" > /dev/null REALDIR=$(pwd) popd > /dev/null #extract the prefix from the 'real' directory path REAL_DIR_PREFIX=${REALDIR:0:$BASE_DIR_LEN} #DEBUG #echo "FULLPATH = $FULLPATH" #echo "DIRECTORY= $DIRECTORY" #echo "REALDIR = $REALDIR" #echo "REAL_DIR_PREFIX = $REAL_DIR_PREFIX" #echo "BASE_DIR = $SCIFI_BASE_DIR" # Probably foul play: # the real path of the requested input file does not start with the prefix # of '/media/scifi' - maybe somebody's trying to get a file outside 'scifi' ?? if [ "$REAL_DIR_PREFIX" != "$SCIFI_BASE_DIR" ]; then echo "Error: invalid input file ($INPUT)" >&2 exit 1 fi # If we got here, the $FULLPATH is at least in a valid location under the '/media/scifi' directory. if [ ! -r "$FULLPATH" ]; then echo "Error: input file ($INPUT) is not a valid file." >&2 exit 1 fi cp "$FULLPATH" "$OUTPUT" if [ $? != 0 ]; then echo "Error: failed to copy \"$INPUT\"!" >&2 exit 1 fi echo "File \"$INPUT\" Imported." exit 0 ### ### ### A further improvement is not to allow free-text file path, but instead use dynamic options to select from a list of files, as so (in the XML file): <param name="localfile" type="select" label="Solexa Data File"> <options from_file="cshl_import_files_hannon.txt"> <column name="name" index="1"/> <column name="value" index="0"/> </options> </param> And then have a cron job to create 'cshl_import_files_hannon.txt' with files which can be uploaded. Hope this helps, -gordon. Greg Von Kuster wrote, On 07/21/2009 10:44 AM: > Hello Abhi, > > Can you clarify the steps you took that produced the behavior? See my > comments below. > > Anton Nekrutenko wrote: >> Abhishek: >> >> Let talk. This is the area of active current development. We are >> looking at implementing a universal fastq-like format or supporting >> multiple formats. Perhaps we should join efforts in ironing out >> specifications. >> >> anton >> galaxy team >> >> >> On Jul 20, 2009, at 5:18 PM, Abhishek Pratap wrote: >> >>> Hi All >>> >>> >>> I recently came to know about NGS analysis on galaxy during ISMB. >>> Getting excited I tried couple of things basically to play with it. >>> >>> Few comments : I may have interepretted something described below in a >>> wrong way. My apologies before hand. >>> >>> >>> >>> On a standalone installation of galaxy while I was trying to explore >>> one FASTQ(sequence) file. It takes considerable (> 20 min) for a fastq >>> file to get uploaded (2 GB). > > Are you using the Galaxy upload utility to create an item in your > history that points to the dataset file on disk? > > I am not sure what is the rationale >>> behind that. Ideally I think there should be no need to upload such >>> heavy files into the workspace. > > A data file that originates from a place external to Galaxy must be > uploaded into Galaxy so that the disk file can be placed in the location > configured in the Galaxy config file. Also, when data is uploaded to > Galaxy ( either to a history or a library ), several database table > settings are created that are used by various Galaxy features. > > They could actually be used straight >>> away by the path specified. > > What do you mean by "the path specified"? > > Also is there any way to access the >>> scripts for analysis on the command line. I know this undermines the >>> main aim of working with galaxy but rite now I am concerned about the >>> performance/time. > > You should be able to run any Galaxy tool from the command line as long > as you have all of the tool's required binaries in your path. However, > running a tool from within Galaxy should generally not be any slower > than running it outside of Galaxy, depending, of course, on what you are > doing. > > >>> I will be happy to discuss more about this in case you have some >>> comments/questions for me. >>> >>> >>> >>> Best, >>> -Abhi >>> >>> >>> >>> ----------------------------- >>> >>> Abhishek Pratap >>> >>> Bioinformatics Software Engineer >>> >>> Institute for Genome Sciences >>> >>> School of Medicine, Univ of Maryland >>> >>> 801, W. Baltimore Street, Baltimore, MD 21209 >>> >>> Ph: (+1)-410-706-2296 >>> >>> www.igs.umaryland.edu/ >>> _______________________________________________ >>> galaxy-user mailing list >>> galaxy-user(a)bx.psu.edu >>> http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-user >> Anton Nekrutenko >> http://nekrut.bx.psu.edu >> http://galaxyproject.org >> >> _______________________________________________ >> galaxy-user mailing list >> galaxy-user(a)bx.psu.edu >> http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-user >> >> > _______________________________________________ > galaxy-user mailing list > galaxy-user(a)bx.psu.edu > http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-user

2 1

Problem with Galaxy Installation
by Pankaj Narang 21 Jul '09

21 Jul '09

Dear Sir, I am installing Galaxy using setup.sh script on my 64 bit RED HAT Enterprises Linux system but it gives an error . I have Python 2.4.It works well on my other 32 bit RHEL system in the same network.Could you please tell me whether current version of Galaxy is compatible with 64 bit linux system or not? Thanx in advance, -- Regards, Pankaj Narang

2 1

Small improvement for Extract-Genomic-Sequences tool
by Assaf Gordon 20 Jul '09

20 Jul '09

Hello, We've encountered a small issue when using the extract-genomic-sequences tool with a TwoBit file and lot's of sequences: memory and CPU usage increases unnecessarily (and processing time, too). Background: a TwoBit (2bit) file is a binary representation of a FASTA file. Unlike nbit files, TwoBit files contain multiple sequences per file. For example, The Drosophila Virilis 'genome' contains 13530 scaffolds. Converting the droVir3 FASTA file into a 2bit file creates a 52MB file. Details: When trying to extract sequences (using extract_genomic_dna.py), Each time a new chromosome/scaffold is encountered, the *entire* 2bit file is loaded and stored. This is unnecessary - the same 2bit file is used for all the scaffolds. (Imagine a BED file with 1000 different scaffolds - memory usage goes to 1000 * 52MB... ). The offending code is (extract_genomic_dna.py) : ------------------- elif twobit_path and os.path.exists( twobit_path ): if chrom in twobits: t = twobits[chrom] else: twobits[chrom] = t = bx.seq.twobit.TwoBitFile( file( twobit_path ) ) try: sequence = t[chrom][start:end] ------------------- the hash 'twobits' contains multiple copies of the same 2bit file, over and over again (one per chromosome/scaffold), but only one chromosome/scaffold is read from it. An improvement would be to load the file only once, and each time take the chromosome/scaffold from the already-loaded hash. Here's a possible patch: ======================== --- extract_genomic_dna.py 2009-07-17 12:55:53.000000000 -0400 +++ extract_genomic_dna2.py 2009-07-17 12:55:18.000000000 -0400 @@ -84,6 +84,7 @@ def __main__(): fout = open( output_filename, "w" ) warnings = [] warning = '' + twobitfile = None for i, line in enumerate( open( input_filename ) ): line = line.rstrip( '\r\n' ) @@ -132,12 +133,10 @@ def __main__(): invalid_line = line continue elif twobit_path and os.path.exists( twobit_path ): - if chrom in twobits: - t = twobits[chrom] - else: - twobits[chrom] = t = bx.seq.twobit.TwoBitFile( file( twobit_path ) ) + if not twobitfile: + twobitfile = bx.seq.twobit.TwoBitFile( file( twobit_path ) ) try: - sequence = t[chrom][start:end] + sequence = twobitfile[chrom][start:end] except: warning = "Unable to fetch the sequence from '%d' to '%d' for build '%s'. " %( start, end-start, dbkey ) warnings.append( warning ) ======================== Technical note: This code and patch refers to an older revision (before changeset 2457:8dfe971fcc27), but I think the new code still behaves this way. Kudos to Ted Karginov who had to learn python to find and fix this bug. -Gordon.

2 1

[hg] galaxy 2487: Bugfix for history refresh in IE
by Greg Von Kuster 16 Jul '09

16 Jul '09

details: http://www.bx.psu.edu/hg/galaxy/rev/46bd94b12a0c changeset: 2487:46bd94b12a0c user: James Taylor <james(a)jamestaylor.org> date: Thu Jul 16 15:35:58 2009 -0400 description: Bugfix for history refresh in IE 1 file(s) affected in this change: templates/root/history.mako diffs (18 lines): diff -r 079bf1a8422b -r 46bd94b12a0c templates/root/history.mako --- a/templates/root/history.mako Thu Jul 16 09:45:43 2009 -0400 +++ b/templates/root/history.mako Thu Jul 16 15:35:58 2009 -0400 @@ -73,9 +73,12 @@ }); // Updater updater({ - %for data in reversed( datasets ): + %for i, data in enumerate( reversed( datasets ) ): %if data.visible and data.state not in [ "deleted", "empty", "error", "ok" ]: - "${data.id}": "${data.state}"; + %if i > 0: + , + %endif + "${data.id}": "${data.state}" %endif %endfor });

1 0