- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 2 new changesets
by Bitbucket 28 Apr '11

28 Apr '11

2 new changesets in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/871ae4a9e95c/ changeset: r5470:871ae4a9e95c user: fubar date: 2011-04-28 17:48:37 summary: missing quote in picard_wrapper affected #: 1 file (1 byte) --- a/tools/picard/picard_wrapper.py Thu Apr 28 11:10:41 2011 -0400 +++ b/tools/picard/picard_wrapper.py Thu Apr 28 11:48:37 2011 -0400 @@ -527,7 +527,7 @@ s,stdouts = pic.runCL(cl=cl2,output_dir=opts.outdir) else: s = 'Unable to find expected pdf file %s<br/>\n' % pdfpath - s += This <b>always happens if single ended data was provided</b> to this tool,\n' + s += 'This <b>always happens if single ended data was provided</b> to this tool,\n' s += 'so please double check that your input data really is paired-end NGS data.<br/>\n' s += 'If your input was paired data this may be a bug worth reporting to the galaxy-bugs list\n<br/>' stdouts = '' http://bitbucket.org/galaxy/galaxy-central/changeset/ea9a898b73be/ changeset: r5471:ea9a898b73be user: fubar date: 2011-04-28 17:48:52 summary: branch merge affected #: 5 files (3.6 KB) --- a/lib/galaxy/web/controllers/user.py Thu Apr 28 11:48:37 2011 -0400 +++ b/lib/galaxy/web/controllers/user.py Thu Apr 28 11:48:52 2011 -0400 @@ -735,6 +735,38 @@ webapp=webapp, message=message, status=status ) + # For REMOTE_USER, we need the ability to just edit the username + @web.expose + @web.require_login( "to manage the public name" ) + def edit_username( self, trans, cntrller, **kwd ): + params = util.Params( kwd ) + is_admin = cntrller == 'admin' and trans.user_is_admin() + webapp = params.get( 'webapp', 'galaxy' ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + user_id = params.get( 'user_id', None ) + if user_id and is_admin: + user = trans.sa_session.query( trans.app.model.User ).get( trans.security.decode_id( user_id ) ) + else: + user = trans.user + if user and params.get( 'change_username_button', False ): + username = kwd.get( 'username', '' ) + if username: + message = self.__validate_username( trans, username, user ) + if message: + status = 'error' + else: + user.username = username + trans.sa_session.add( user ) + trans.sa_session.flush() + message = 'The username has been updated with the changes.' + return trans.fill_template( '/user/username.mako', + cntrller=cntrller, + user=user, + username=user.username, + webapp=webapp, + message=message, + status=status ) @web.expose def edit_info( self, trans, cntrller, **kwd ): params = util.Params( kwd ) @@ -743,12 +775,15 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) user_id = params.get( 'user_id', None ) - #new_kwd = dict( webapp=webapp ) - if user_id: + if user_id and is_admin: user = trans.sa_session.query( trans.app.model.User ).get( trans.security.decode_id( user_id ) ) + elif user_id and ( not trans.user or trans.user.id != trans.security.decode_id( user_id ) ): + message = 'Invalid user id' + status = 'error' + user = None else: user = trans.user - if params.get( 'login_info_button', False ): + if user and params.get( 'login_info_button', False ): # Editing email and username email = util.restore_text( params.get( 'email', '' ) ) username = util.restore_text( params.get( 'username', '' ) ).lower() @@ -769,7 +804,7 @@ trans.sa_session.add_all( ( user, private_role ) ) trans.sa_session.flush() message = 'The login information has been updated with the changes.' - elif params.get( 'change_password_button', False ): + elif user and params.get( 'change_password_button', False ): # Editing password. Do not sanitize passwords, so get from kwd # and not params (which were sanitized). password = kwd.get( 'password', '' ) @@ -794,7 +829,7 @@ trans.sa_session.flush() trans.log_event( "User change password" ) message = 'The password has been changed.' - elif params.get( 'edit_user_info_button', False ): + elif user and params.get( 'edit_user_info_button', False ): # Edit user information - webapp MUST BE 'galaxy' user_type_fd_id = params.get( 'user_type_fd_id', 'none' ) if user_type_fd_id not in [ 'none' ]: @@ -809,7 +844,7 @@ else: values = {} flush_needed = False - if user.values: + if user.values: # Editing the user info of an existing user with existing user info user.values.content = values trans.sa_session.add( user.values ) @@ -823,7 +858,7 @@ trans.sa_session.add( user ) trans.sa_session.flush() message = "The user information has been updated with the changes." - if webapp == 'galaxy' and is_admin: + if user and webapp == 'galaxy' and is_admin: kwd[ 'user_id' ] = trans.security.encode_id( user.id ) kwd[ 'id' ] = user_id if message: --- a/lib/galaxy/web/framework/__init__.py Thu Apr 28 11:48:37 2011 -0400 +++ b/lib/galaxy/web/framework/__init__.py Thu Apr 28 11:48:52 2011 -0400 @@ -437,10 +437,18 @@ if not user.default_permissions: self.app.security_agent.user_set_default_permissions( user, history=True, dataset=True ) elif user is None: + username = remote_user_email.split( '@', 1 )[0] random.seed() user = self.app.model.User( email=remote_user_email ) user.set_password_cleartext( ''.join( random.sample( string.letters + string.digits, 12 ) ) ) user.external = True + # Find a unique username - user can change it later + if ( self.sa_session.query( self.app.model.User ).filter_by( username=username ).first() ): + i = 0 + while ( self.sa_session.query( self.app.model.User ).filter_by( username=(username + '-' + str(i) ) ).first() ): + i += 1 + username += '-' + str(i) + user.username = username self.sa_session.add( user ) self.sa_session.flush() self.app.security_agent.create_private_user_role( user ) --- a/lib/galaxy/web/framework/middleware/remoteuser.py Thu Apr 28 11:48:37 2011 -0400 +++ b/lib/galaxy/web/framework/middleware/remoteuser.py Thu Apr 28 11:48:52 2011 -0400 @@ -97,6 +97,8 @@ pass # admins can create users elif path_info.startswith( '/user/api_keys' ): pass # api keys can be managed when remote_user is in use + elif path_info.startswith( '/user/edit_username' ): + pass # username can be managed when remote_user is in use elif path_info.startswith( '/user/dbkeys' ): pass # dbkeys can be managed when remote_user is in use elif path_info.startswith( '/user' ): --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/templates/user/username.mako Thu Apr 28 11:48:52 2011 -0400 @@ -0,0 +1,23 @@ +<%inherit file="/base.mako"/> + +<% is_admin = cntrller == 'admin' and trans.user_is_admin() %> + +<h2>Manage Public Name</h2> +<div class="toolForm"> + <form name="username" id="username" action="${h.url_for( controller='user', action='edit_username', cntrller=cntrller, user_id=trans.security.encode_id( user.id ) )}" method="post" > + <input type="hidden" name="webapp" value="${webapp}" size="40"/> + <div class="toolFormTitle">Login Information</div> + <div class="form-row"> + <label>Public name:</label> + <input type="text" name="username" size="40" value="${username}"/> + <div class="toolParamHelp" style="clear: both;"> + Your public name is an optional identifier that will be used to generate addresses for information + you share publicly. Public names must be at least four characters in length and contain only lower-case + letters, numbers, and the '-' character. + </div> + </div> + <div class="form-row"> + <input type="submit" name="change_username_button" value="Save"/> + </div> + </form> +</div> --- a/templates/webapps/galaxy/base_panels.mako Thu Apr 28 11:48:37 2011 -0400 +++ b/templates/webapps/galaxy/base_panels.mako Thu Apr 28 11:48:52 2011 -0400 @@ -149,6 +149,8 @@ menu_options.append( [ 'Saved Pages', h.url_for( controller='/page', action='list' ), "_top" ] ) if app.config.enable_api: menu_options.append( [ 'API Keys', h.url_for( controller='/user', action='api_keys', cntrller='user' ), "galaxy_main" ] ) + if app.config.use_remote_user: + menu_options.append( [ 'Public Name', h.url_for( controller='/user', action='edit_username', cntrller='user' ), "galaxy_main" ] ) extra_class = "loggedin-only" visible = ( trans.user != None ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Set and allow modification of the public username if use_remote_user = True
by Bitbucket 28 Apr '11

28 Apr '11

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/7c101a11d261/ changeset: r5469:7c101a11d261 user: natefoo date: 2011-04-28 17:33:07 summary: Set and allow modification of the public username if use_remote_user = True affected #: 5 files (2.5 KB) --- a/lib/galaxy/web/controllers/user.py Thu Apr 28 11:10:41 2011 -0400 +++ b/lib/galaxy/web/controllers/user.py Thu Apr 28 11:33:07 2011 -0400 @@ -735,6 +735,38 @@ webapp=webapp, message=message, status=status ) + # For REMOTE_USER, we need the ability to just edit the username + @web.expose + @web.require_login( "to manage the public name" ) + def edit_username( self, trans, cntrller, **kwd ): + params = util.Params( kwd ) + is_admin = cntrller == 'admin' and trans.user_is_admin() + webapp = params.get( 'webapp', 'galaxy' ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + user_id = params.get( 'user_id', None ) + if user_id and is_admin: + user = trans.sa_session.query( trans.app.model.User ).get( trans.security.decode_id( user_id ) ) + else: + user = trans.user + if user and params.get( 'change_username_button', False ): + username = kwd.get( 'username', '' ) + if username: + message = self.__validate_username( trans, username, user ) + if message: + status = 'error' + else: + user.username = username + trans.sa_session.add( user ) + trans.sa_session.flush() + message = 'The username has been updated with the changes.' + return trans.fill_template( '/user/username.mako', + cntrller=cntrller, + user=user, + username=user.username, + webapp=webapp, + message=message, + status=status ) @web.expose def edit_info( self, trans, cntrller, **kwd ): params = util.Params( kwd ) @@ -743,12 +775,15 @@ message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) user_id = params.get( 'user_id', None ) - #new_kwd = dict( webapp=webapp ) - if user_id: + if user_id and is_admin: user = trans.sa_session.query( trans.app.model.User ).get( trans.security.decode_id( user_id ) ) + elif user_id and ( not trans.user or trans.user.id != trans.security.decode_id( user_id ) ): + message = 'Invalid user id' + status = 'error' + user = None else: user = trans.user - if params.get( 'login_info_button', False ): + if user and params.get( 'login_info_button', False ): # Editing email and username email = util.restore_text( params.get( 'email', '' ) ) username = util.restore_text( params.get( 'username', '' ) ).lower() @@ -769,7 +804,7 @@ trans.sa_session.add_all( ( user, private_role ) ) trans.sa_session.flush() message = 'The login information has been updated with the changes.' - elif params.get( 'change_password_button', False ): + elif user and params.get( 'change_password_button', False ): # Editing password. Do not sanitize passwords, so get from kwd # and not params (which were sanitized). password = kwd.get( 'password', '' ) @@ -794,7 +829,7 @@ trans.sa_session.flush() trans.log_event( "User change password" ) message = 'The password has been changed.' - elif params.get( 'edit_user_info_button', False ): + elif user and params.get( 'edit_user_info_button', False ): # Edit user information - webapp MUST BE 'galaxy' user_type_fd_id = params.get( 'user_type_fd_id', 'none' ) if user_type_fd_id not in [ 'none' ]: @@ -809,7 +844,7 @@ else: values = {} flush_needed = False - if user.values: + if user.values: # Editing the user info of an existing user with existing user info user.values.content = values trans.sa_session.add( user.values ) @@ -823,7 +858,7 @@ trans.sa_session.add( user ) trans.sa_session.flush() message = "The user information has been updated with the changes." - if webapp == 'galaxy' and is_admin: + if user and webapp == 'galaxy' and is_admin: kwd[ 'user_id' ] = trans.security.encode_id( user.id ) kwd[ 'id' ] = user_id if message: --- a/lib/galaxy/web/framework/__init__.py Thu Apr 28 11:10:41 2011 -0400 +++ b/lib/galaxy/web/framework/__init__.py Thu Apr 28 11:33:07 2011 -0400 @@ -437,10 +437,18 @@ if not user.default_permissions: self.app.security_agent.user_set_default_permissions( user, history=True, dataset=True ) elif user is None: + username = remote_user_email.split( '@', 1 )[0] random.seed() user = self.app.model.User( email=remote_user_email ) user.set_password_cleartext( ''.join( random.sample( string.letters + string.digits, 12 ) ) ) user.external = True + # Find a unique username - user can change it later + if ( self.sa_session.query( self.app.model.User ).filter_by( username=username ).first() ): + i = 0 + while ( self.sa_session.query( self.app.model.User ).filter_by( username=(username + '-' + str(i) ) ).first() ): + i += 1 + username += '-' + str(i) + user.username = username self.sa_session.add( user ) self.sa_session.flush() self.app.security_agent.create_private_user_role( user ) --- a/lib/galaxy/web/framework/middleware/remoteuser.py Thu Apr 28 11:10:41 2011 -0400 +++ b/lib/galaxy/web/framework/middleware/remoteuser.py Thu Apr 28 11:33:07 2011 -0400 @@ -97,6 +97,8 @@ pass # admins can create users elif path_info.startswith( '/user/api_keys' ): pass # api keys can be managed when remote_user is in use + elif path_info.startswith( '/user/edit_username' ): + pass # username can be managed when remote_user is in use elif path_info.startswith( '/user/dbkeys' ): pass # dbkeys can be managed when remote_user is in use elif path_info.startswith( '/user' ): --- a/templates/webapps/galaxy/base_panels.mako Thu Apr 28 11:10:41 2011 -0400 +++ b/templates/webapps/galaxy/base_panels.mako Thu Apr 28 11:33:07 2011 -0400 @@ -149,6 +149,8 @@ menu_options.append( [ 'Saved Pages', h.url_for( controller='/page', action='list' ), "_top" ] ) if app.config.enable_api: menu_options.append( [ 'API Keys', h.url_for( controller='/user', action='api_keys', cntrller='user' ), "galaxy_main" ] ) + if app.config.use_remote_user: + menu_options.append( [ 'Public Name', h.url_for( controller='/user', action='edit_username', cntrller='user' ), "galaxy_main" ] ) extra_class = "loggedin-only" visible = ( trans.user != None ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: fubar: clean up of accidentally added test detritus
by Bitbucket 28 Apr '11

28 Apr '11

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/9c63ae1f7cef/ changeset: r5468:9c63ae1f7cef user: fubar date: 2011-04-28 17:10:41 summary: clean up of accidentally added test detritus affected #: 1 file (0 bytes) Diff too large to display. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: fubar: an errant == in tool_conf.xml.sample fixed
by Bitbucket 28 Apr '11

28 Apr '11

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/f53f589efb1e/ changeset: r5467:f53f589efb1e user: fubar date: 2011-04-28 15:33:19 summary: an errant == in tool_conf.xml.sample fixed affected #: 1 file (1 byte) --- a/tool_conf.xml.sample Thu Apr 28 09:18:13 2011 -0400 +++ b/tool_conf.xml.sample Thu Apr 28 09:33:19 2011 -0400 @@ -244,7 +244,7 @@ <tool file="picard/rgPicardASMetrics.xml" /><tool file="picard/rgPicardGCBiasMetrics.xml" /><tool file="picard/rgPicardInsertSize.xml" /> - <label text=="bam/sam Cleaning" id="picard-tools" /> + <label text="bam/sam Cleaning" id="picard-tools" /><tool file="picard/picard_AddOrReplaceReadGroups.xml" /><tool file="picard/picard_ReorderSam.xml" /><tool file="picard/picard_ReplaceSamHeader.xml" /> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 28 Apr '11

28 Apr '11

2 new changesets in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/725deb060e8e/ changeset: r5465:725deb060e8e user: fubar date: 2011-04-28 15:17:50 summary: missing slash in tool_conf.xml.sample fixed affected #: 1 file (2 bytes) --- a/tool_conf.xml.sample Wed Apr 27 17:06:14 2011 -0400 +++ b/tool_conf.xml.sample Thu Apr 28 09:17:50 2011 -0400 @@ -244,7 +244,7 @@ <tool file="picard/rgPicardASMetrics.xml" /><tool file="picard/rgPicardGCBiasMetrics.xml" /><tool file="picard/rgPicardInsertSize.xml" /> - <label text=="bam/sam Cleaning" id="picard-tools"> + <label text=="bam/sam Cleaning" id="picard-tools" /><tool file="picard/picard_AddOrReplaceReadGroups.xml" /><tool file="picard/picard_ReorderSam.xml" /><tool file="picard/picard_ReplaceSamHeader.xml" /> http://bitbucket.org/galaxy/galaxy-central/changeset/c28dc46ede98/ changeset: r5466:c28dc46ede98 user: fubar date: 2011-04-28 15:18:13 summary: branch merge affected #: 3 files (2.1 KB) --- a/datatypes_conf.xml.sample Thu Apr 28 09:17:50 2011 -0400 +++ b/datatypes_conf.xml.sample Thu Apr 28 09:18:13 2011 -0400 @@ -66,8 +66,7 @@ </datatype><datatype extension="gff" type="galaxy.datatypes.interval:Gff" display_in_upload="true"><converter file="gff_to_bed_converter.xml" target_datatype="bed"/> - <converter file="gff_to_bgzip_converter.xml" target_datatype="bgzip"/> - <converter file="gff_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> + <converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/><converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/><display file="ensembl/ensembl_gff.xml" inherit="True"/> @@ -75,10 +74,7 @@ <datatype extension="gff3" type="galaxy.datatypes.interval:Gff3" display_in_upload="true"/><datatype extension="gif" type="galaxy.datatypes.images:Image" mimetype="image/gif"/><datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/> - <datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"> - <converter file="gff_to_bgzip_converter.xml" target_datatype="bgzip"/> - <converter file="gff_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - </datatype> + <datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"/><datatype extension="html" type="galaxy.datatypes.images:Html" mimetype="text/html"/><datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true"><converter file="interval_to_bed_converter.xml" target_datatype="bed"/> --- a/lib/galaxy/datatypes/interval.py Thu Apr 28 09:17:50 2011 -0400 +++ b/lib/galaxy/datatypes/interval.py Thu Apr 28 09:18:13 2011 -0400 @@ -772,7 +772,7 @@ return False def get_track_type( self ): - return "FeatureTrack", {"data": "tabix", "index": "summary_tree"} + return "FeatureTrack", {"data": "interval_index", "index": "summary_tree"} class Gff3( Gff ): --- a/lib/galaxy/visualization/tracks/data_providers.py Thu Apr 28 09:17:50 2011 -0400 +++ b/lib/galaxy/visualization/tracks/data_providers.py Thu Apr 28 09:18:13 2011 -0400 @@ -360,13 +360,7 @@ f = open( self.original_dataset.file_name ) return f, BigWigFile(file=f) -class TabixDataProvider( TracksDataProvider ): - """ - Tabix index data provider for the Galaxy track browser. - """ - - col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } - +class FilterableMixin: def get_filters( self ): """ Returns a dataset's filters. """ @@ -422,12 +416,19 @@ filters = [ { 'name': 'Score', 'type': 'int', 'index': filter_col } ] return filters + +class TabixDataProvider( TracksDataProvider, FilterableMixin ): + """ + Tabix index data provider for the Galaxy track browser. + """ + + col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } def get_iterator( self, chrom, start, end ): - if end >= 2<<29: + start, end = int(start), int(end) + if end >= (2<<29): end = (2<<29 - 1) # Tabix-enforced maximum - start, end = int(start), int(end) - + bgzip_fname = self.dependencies['bgzip'].file_name # if os.path.getsize(self.converted_dataset.file_name) == 0: @@ -447,24 +448,64 @@ iterator = self.get_iterator( chrom, start, end ) return self.process_data(iterator, **kwargs) -class GffDataProvider( TabixDataProvider ): - # FIXME: Doesn't implement write_data_to_file - def process_data( self, iterator, **kwargs ): +class GffDataProvider( TracksDataProvider, FilterableMixin ): + col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } + + def write_data_to_file( self, chrom, start, end, filename ): + source = open( self.original_dataset.file_name ) + index = Indexes( self.converted_dataset.file_name ) + out = open( filename, 'w' ) + for start, end, offset in index.find(chrom, start, end): + source.seek( offset ) + + reader = GFFReaderWrapper( source, fix_strand=True ) + feature = reader.next() + for interval in feature.intervals: + out.write(interval.raw_line + '\n') + out.close() + + def get_data( self, chrom, start, end, **kwargs ): + start, end = int(start), int(end) + source = open( self.original_dataset.file_name ) + index = Indexes( self.converted_dataset.file_name ) + results = [] count = 0 - rval = [] message = None - for feature in GFFReaderWrapper( iterator, fix_strand=True ): + + # If chrom is not found in indexes, try removing the first three + # characters (e.g. 'chr') and see if that works. This enables the + # provider to handle chrome names defined as chrXXX and as XXX. + chrom = str(chrom) + if chrom not in index.indexes and chrom[3:] in index.indexes: + chrom = chrom[3:] + + # + # Build data to return. Payload format is: + # [ <guid/offset>, <start>, <end>, <name>, <score>, <strand>, <thick_start>, + # <thick_end>, <blocks> ] + # + # First three entries are mandatory, others are optional. + # + filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + no_detail = ( "no_detail" in kwargs ) + for start, end, offset in index.find(chrom, start, end): if count >= MAX_VALS: message = ERROR_MAX_VALS % "features" break count += 1 + source.seek( offset ) + # TODO: can we use column metadata to fill out payload? + # TODO: use function to set payload data - payload = package_gff_feature( feature ) - hashed = hash("\t".join(feature.fields)) - payload.insert( 0, hashed ) - rval.append( payload ) - - return { 'data': rval, 'message': message } + # GFF dataset. + reader = GFFReaderWrapper( source, fix_strand=True ) + feature = reader.next() + payload = package_gff_feature( feature, no_detail, filter_cols ) + payload.insert( 0, offset ) + + results.append( payload ) + + return { 'data': results, 'message': message } class BedDataProvider( TabixDataProvider ): """ @@ -611,7 +652,8 @@ # type. First key is converted dataset type; if result is another dict, second key # is original dataset type. TODO: This needs to be more flexible. dataset_type_name_to_data_provider = { - "tabix": { Vcf: VcfDataProvider, Bed: BedDataProvider, Gff: GffDataProvider, Gtf: GffDataProvider, "default" : TabixDataProvider }, + "tabix": { Vcf: VcfDataProvider, Bed: BedDataProvider, "default" : TabixDataProvider }, + "interval_index": { Gff: GffDataProvider, Gtf: GffDataProvider }, "bai": BamDataProvider, "summary_tree": SummaryTreeDataProvider, "bigwig": BigWigDataProvider, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: kanwei: trackster: Restore interval_index as Gff indexer for now due to sorting issues
by Bitbucket 27 Apr '11

27 Apr '11

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/760d34e6c692/ changeset: r5464:760d34e6c692 user: kanwei date: 2011-04-28 01:08:31 summary: trackster: Restore interval_index as Gff indexer for now due to sorting issues affected #: 3 files (2.1 KB) --- a/datatypes_conf.xml.sample Wed Apr 27 17:33:08 2011 -0400 +++ b/datatypes_conf.xml.sample Wed Apr 27 19:08:31 2011 -0400 @@ -66,8 +66,7 @@ </datatype><datatype extension="gff" type="galaxy.datatypes.interval:Gff" display_in_upload="true"><converter file="gff_to_bed_converter.xml" target_datatype="bed"/> - <converter file="gff_to_bgzip_converter.xml" target_datatype="bgzip"/> - <converter file="gff_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> + <converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/><converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/><display file="ensembl/ensembl_gff.xml" inherit="True"/> @@ -75,10 +74,7 @@ <datatype extension="gff3" type="galaxy.datatypes.interval:Gff3" display_in_upload="true"/><datatype extension="gif" type="galaxy.datatypes.images:Image" mimetype="image/gif"/><datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/> - <datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"> - <converter file="gff_to_bgzip_converter.xml" target_datatype="bgzip"/> - <converter file="gff_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - </datatype> + <datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"/><datatype extension="html" type="galaxy.datatypes.images:Html" mimetype="text/html"/><datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true"><converter file="interval_to_bed_converter.xml" target_datatype="bed"/> --- a/lib/galaxy/datatypes/interval.py Wed Apr 27 17:33:08 2011 -0400 +++ b/lib/galaxy/datatypes/interval.py Wed Apr 27 19:08:31 2011 -0400 @@ -772,7 +772,7 @@ return False def get_track_type( self ): - return "FeatureTrack", {"data": "tabix", "index": "summary_tree"} + return "FeatureTrack", {"data": "interval_index", "index": "summary_tree"} class Gff3( Gff ): --- a/lib/galaxy/visualization/tracks/data_providers.py Wed Apr 27 17:33:08 2011 -0400 +++ b/lib/galaxy/visualization/tracks/data_providers.py Wed Apr 27 19:08:31 2011 -0400 @@ -360,13 +360,7 @@ f = open( self.original_dataset.file_name ) return f, BigWigFile(file=f) -class TabixDataProvider( TracksDataProvider ): - """ - Tabix index data provider for the Galaxy track browser. - """ - - col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } - +class FilterableMixin: def get_filters( self ): """ Returns a dataset's filters. """ @@ -422,6 +416,13 @@ filters = [ { 'name': 'Score', 'type': 'int', 'index': filter_col } ] return filters + +class TabixDataProvider( TracksDataProvider, FilterableMixin ): + """ + Tabix index data provider for the Galaxy track browser. + """ + + col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } def get_iterator( self, chrom, start, end ): start, end = int(start), int(end) @@ -447,24 +448,64 @@ iterator = self.get_iterator( chrom, start, end ) return self.process_data(iterator, **kwargs) -class GffDataProvider( TabixDataProvider ): - # FIXME: Doesn't implement write_data_to_file - def process_data( self, iterator, **kwargs ): +class GffDataProvider( TracksDataProvider, FilterableMixin ): + col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } + + def write_data_to_file( self, chrom, start, end, filename ): + source = open( self.original_dataset.file_name ) + index = Indexes( self.converted_dataset.file_name ) + out = open( filename, 'w' ) + for start, end, offset in index.find(chrom, start, end): + source.seek( offset ) + + reader = GFFReaderWrapper( source, fix_strand=True ) + feature = reader.next() + for interval in feature.intervals: + out.write(interval.raw_line + '\n') + out.close() + + def get_data( self, chrom, start, end, **kwargs ): + start, end = int(start), int(end) + source = open( self.original_dataset.file_name ) + index = Indexes( self.converted_dataset.file_name ) + results = [] count = 0 - rval = [] message = None - for feature in GFFReaderWrapper( iterator, fix_strand=True ): + + # If chrom is not found in indexes, try removing the first three + # characters (e.g. 'chr') and see if that works. This enables the + # provider to handle chrome names defined as chrXXX and as XXX. + chrom = str(chrom) + if chrom not in index.indexes and chrom[3:] in index.indexes: + chrom = chrom[3:] + + # + # Build data to return. Payload format is: + # [ <guid/offset>, <start>, <end>, <name>, <score>, <strand>, <thick_start>, + # <thick_end>, <blocks> ] + # + # First three entries are mandatory, others are optional. + # + filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + no_detail = ( "no_detail" in kwargs ) + for start, end, offset in index.find(chrom, start, end): if count >= MAX_VALS: message = ERROR_MAX_VALS % "features" break count += 1 + source.seek( offset ) + # TODO: can we use column metadata to fill out payload? + # TODO: use function to set payload data - payload = package_gff_feature( feature ) - hashed = hash("\t".join(feature.fields)) - payload.insert( 0, hashed ) - rval.append( payload ) - - return { 'data': rval, 'message': message } + # GFF dataset. + reader = GFFReaderWrapper( source, fix_strand=True ) + feature = reader.next() + payload = package_gff_feature( feature, no_detail, filter_cols ) + payload.insert( 0, offset ) + + results.append( payload ) + + return { 'data': results, 'message': message } class BedDataProvider( TabixDataProvider ): """ @@ -611,7 +652,8 @@ # type. First key is converted dataset type; if result is another dict, second key # is original dataset type. TODO: This needs to be more flexible. dataset_type_name_to_data_provider = { - "tabix": { Vcf: VcfDataProvider, Bed: BedDataProvider, Gff: GffDataProvider, Gtf: GffDataProvider, "default" : TabixDataProvider }, + "tabix": { Vcf: VcfDataProvider, Bed: BedDataProvider, "default" : TabixDataProvider }, + "interval_index": { Gff: GffDataProvider, Gtf: GffDataProvider }, "bai": BamDataProvider, "summary_tree": SummaryTreeDataProvider, "bigwig": BigWigDataProvider, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: kanwei: trackster: Fix incorrect "end" calculation
by Bitbucket 27 Apr '11

27 Apr '11

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/cacc1f179af3/ changeset: r5463:cacc1f179af3 user: kanwei date: 2011-04-27 23:33:08 summary: trackster: Fix incorrect "end" calculation affected #: 1 file (14 bytes) --- a/lib/galaxy/visualization/tracks/data_providers.py Wed Apr 27 17:06:14 2011 -0400 +++ b/lib/galaxy/visualization/tracks/data_providers.py Wed Apr 27 17:33:08 2011 -0400 @@ -424,10 +424,10 @@ return filters def get_iterator( self, chrom, start, end ): - if end >= 2<<29: + start, end = int(start), int(end) + if end >= (2<<29): end = (2<<29 - 1) # Tabix-enforced maximum - start, end = int(start), int(end) - + bgzip_fname = self.dependencies['bgzip'].file_name # if os.path.getsize(self.converted_dataset.file_name) == 0: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: fubar: uncommented new tools in tool_conf.xml.sample so buildbot can find them and explanation when insertsizemetrics fails
by Bitbucket 27 Apr '11

27 Apr '11

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/0d4ece99a476/ changeset: r5462:0d4ece99a476 user: fubar date: 2011-04-27 23:06:14 summary: uncommented new tools in tool_conf.xml.sample so buildbot can find them and explanation when insertsizemetrics fails affected #: 2 files (556 bytes) --- a/tool_conf.xml.sample Wed Apr 27 15:41:25 2011 -0400 +++ b/tool_conf.xml.sample Wed Apr 27 17:06:14 2011 -0400 @@ -236,44 +236,57 @@ <tool file="ncbi_blast_plus/ncbi_tblastx_wrapper.xml" /><tool file="ncbi_blast_plus/blastxml_to_tabular.xml" /></section> - <section name="NGS: QC and manipulation" id="cshl_library_information"> - <label text="Illumina fastq" id="illumina" /> - <tool file="fastq/fastq_groomer.xml" /> - <tool file="fastq/fastq_paired_end_splitter.xml" /> - <tool file="fastq/fastq_paired_end_joiner.xml" /> - <tool file="fastq/fastq_stats.xml" /> - <label text="Roche-454 data" id="454" /> - <tool file="metag_tools/short_reads_figure_score.xml" /> - <tool file="metag_tools/short_reads_trim_seq.xml" /> - <tool file="fastq/fastq_combiner.xml" /> - <label text="AB-SOLiD data" id="solid" /> - <tool file="next_gen_conversion/solid2fastq.xml" /> - <tool file="solid_tools/solid_qual_stats.xml" /> - <tool file="solid_tools/solid_qual_boxplot.xml" /> - <label text="Generic FASTQ manipulation" id="generic_fastq" /> - <tool file="fastq/fastq_filter.xml" /> - <tool file="fastq/fastq_trimmer.xml" /> - <tool file="fastq/fastq_trimmer_by_quality.xml" /> - <tool file="fastq/fastq_masker_by_quality.xml" /> - <tool file="fastq/fastq_manipulation.xml" /> - <tool file="fastq/fastq_to_fasta.xml" /> - <tool file="fastq/fastq_to_tabular.xml" /> - <tool file="fastq/tabular_to_fastq.xml" /> - <label text="FASTX-Toolkit for FASTQ data" id="fastx_toolkit" /> - <tool file="fastx_toolkit/fastq_quality_converter.xml" /> - <tool file="fastx_toolkit/fastx_quality_statistics.xml" /> - <tool file="fastx_toolkit/fastq_quality_boxplot.xml" /> - <tool file="fastx_toolkit/fastx_nucleotides_distribution.xml" /> - <tool file="fastx_toolkit/fastq_to_fasta.xml" /> - <tool file="fastx_toolkit/fastq_quality_filter.xml" /> - <tool file="fastx_toolkit/fastq_to_fasta.xml" /> - <tool file="fastx_toolkit/fastx_artifacts_filter.xml" /> - <tool file="fastx_toolkit/fastx_barcode_splitter.xml" /> - <tool file="fastx_toolkit/fastx_clipper.xml" /> - <tool file="fastx_toolkit/fastx_collapser.xml" /> - <tool file="fastx_toolkit/fastx_renamer.xml" /> - <tool file="fastx_toolkit/fastx_reverse_complement.xml" /> - <tool file="fastx_toolkit/fastx_trimmer.xml" /> + <section name="NGS: QC and manipulation" id="NGS_QC"> + <label text="FastQC: fastq/sam/bam" id="fastqcsambam" /> + <tool file="rgenetics/rgFastQC.xml" /> + <label text="QC/Metrics for sam/bam" id="qcsambam"/> + <tool file="picard/picard_BamIndexStats.xml" /> + <tool file="picard/rgPicardASMetrics.xml" /> + <tool file="picard/rgPicardGCBiasMetrics.xml" /> + <tool file="picard/rgPicardInsertSize.xml" /> + <label text=="bam/sam Cleaning" id="picard-tools"> + <tool file="picard/picard_AddOrReplaceReadGroups.xml" /> + <tool file="picard/picard_ReorderSam.xml" /> + <tool file="picard/picard_ReplaceSamHeader.xml" /> + <tool file="picard/rgPicardFixMate.xml" /> + <tool file="picard/rgPicardMarkDups.xml" /> + <label text="Illumina fastq" id="illumina" /> + <tool file="fastq/fastq_groomer.xml" /> + <tool file="fastq/fastq_paired_end_splitter.xml" /> + <tool file="fastq/fastq_paired_end_joiner.xml" /> + <tool file="fastq/fastq_stats.xml" /> + <label text="Roche-454 data" id="454" /> + <tool file="metag_tools/short_reads_figure_score.xml" /> + <tool file="metag_tools/short_reads_trim_seq.xml" /> + <tool file="fastq/fastq_combiner.xml" /> + <label text="AB-SOLiD data" id="solid" /> + <tool file="next_gen_conversion/solid2fastq.xml" /> + <tool file="solid_tools/solid_qual_stats.xml" /> + <tool file="solid_tools/solid_qual_boxplot.xml" /> + <label text="Generic FASTQ manipulation" id="generic_fastq" /> + <tool file="fastq/fastq_filter.xml" /> + <tool file="fastq/fastq_trimmer.xml" /> + <tool file="fastq/fastq_trimmer_by_quality.xml" /> + <tool file="fastq/fastq_masker_by_quality.xml" /> + <tool file="fastq/fastq_manipulation.xml" /> + <tool file="fastq/fastq_to_fasta.xml" /> + <tool file="fastq/fastq_to_tabular.xml" /> + <tool file="fastq/tabular_to_fastq.xml" /> + <label text="FASTX-Toolkit for FASTQ data" id="fastx_toolkit" /> + <tool file="fastx_toolkit/fastq_quality_converter.xml" /> + <tool file="fastx_toolkit/fastx_quality_statistics.xml" /> + <tool file="fastx_toolkit/fastq_quality_boxplot.xml" /> + <tool file="fastx_toolkit/fastx_nucleotides_distribution.xml" /> + <tool file="fastx_toolkit/fastq_to_fasta.xml" /> + <tool file="fastx_toolkit/fastq_quality_filter.xml" /> + <tool file="fastx_toolkit/fastq_to_fasta.xml" /> + <tool file="fastx_toolkit/fastx_artifacts_filter.xml" /> + <tool file="fastx_toolkit/fastx_barcode_splitter.xml" /> + <tool file="fastx_toolkit/fastx_clipper.xml" /> + <tool file="fastx_toolkit/fastx_collapser.xml" /> + <tool file="fastx_toolkit/fastx_renamer.xml" /> + <tool file="fastx_toolkit/fastx_reverse_complement.xml" /> + <tool file="fastx_toolkit/fastx_trimmer.xml" /></section> + <!-- - <section name="NGS: QC and manipulation" id="cshl_library_information"> - <label text="FastQC: fastq/sam/bam" /> - <tool file="rgenetics/rgFastQC.xml" /> - <label text="QC/Metrics for sam/bam" /> - <tool file="picard/picard_BamIndexStats.xml" /> - <tool file="picard/rgPicardASMetrics.xml" /> - <tool file="picard/rgPicardGCBiasMetrics.xml" /> - <tool file="picard/rgPicardInsertSize.xml" /> - </section> - <section name="NGS: bam/sam Cleaning" id="picard-tools"> - <tool file="picard/picard_AddOrReplaceReadGroups.xml" /> - <tool file="picard/picard_ReorderSam.xml" /> - <tool file="picard/picard_ReplaceSamHeader.xml" /> - <tool file="picard/rgPicardFixMate.xml" /> - <tool file="picard/rgPicardMarkDups.xml" /> - </section> - <section name="Genome Diversity" id="gd"><tool file="genome_diversity/extract_primers.xml" /> --- a/tools/picard/picard_wrapper.py Wed Apr 27 15:41:25 2011 -0400 +++ b/tools/picard/picard_wrapper.py Wed Apr 27 17:06:14 2011 -0400 @@ -526,7 +526,10 @@ cl2 = ['mogrify', '-format jpg -resize x400 %s' % pdfpath] s,stdouts = pic.runCL(cl=cl2,output_dir=opts.outdir) else: - s = 'Unable to find expected pdf file %s\n' % pdfpath + s = 'Unable to find expected pdf file %s<br/>\n' % pdfpath + s += This <b>always happens if single ended data was provided</b> to this tool,\n' + s += 'so please double check that your input data really is paired-end NGS data.<br/>\n' + s += 'If your input was paired data this may be a bug worth reporting to the galaxy-bugs list\n<br/>' stdouts = '' lf = open(pic.log_filename,'a') lf.write(s) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: kanwei: Fix GTF for trackster
by Bitbucket 27 Apr '11

27 Apr '11

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/ed5aea827a26/ changeset: r5461:ed5aea827a26 user: kanwei date: 2011-04-27 21:41:25 summary: Fix GTF for trackster affected #: 2 files (226 bytes) --- a/datatypes_conf.xml.sample Wed Apr 27 12:41:57 2011 -0400 +++ b/datatypes_conf.xml.sample Wed Apr 27 15:41:25 2011 -0400 @@ -75,7 +75,10 @@ <datatype extension="gff3" type="galaxy.datatypes.interval:Gff3" display_in_upload="true"/><datatype extension="gif" type="galaxy.datatypes.images:Image" mimetype="image/gif"/><datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/> - <datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"/> + <datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"> + <converter file="gff_to_bgzip_converter.xml" target_datatype="bgzip"/> + <converter file="gff_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> + </datatype><datatype extension="html" type="galaxy.datatypes.images:Html" mimetype="text/html"/><datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true"><converter file="interval_to_bed_converter.xml" target_datatype="bed"/> --- a/lib/galaxy/visualization/tracks/data_providers.py Wed Apr 27 12:41:57 2011 -0400 +++ b/lib/galaxy/visualization/tracks/data_providers.py Wed Apr 27 15:41:25 2011 -0400 @@ -611,7 +611,7 @@ # type. First key is converted dataset type; if result is another dict, second key # is original dataset type. TODO: This needs to be more flexible. dataset_type_name_to_data_provider = { - "tabix": { Vcf: VcfDataProvider, Bed: BedDataProvider, Gff: GffDataProvider, "default" : TabixDataProvider }, + "tabix": { Vcf: VcfDataProvider, Bed: BedDataProvider, Gff: GffDataProvider, Gtf: GffDataProvider, "default" : TabixDataProvider }, "bai": BamDataProvider, "summary_tree": SummaryTreeDataProvider, "bigwig": BigWigDataProvider, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: fubar: Large push with modifications to the Picard tools recently added plus 5 new ones
by Bitbucket 27 Apr '11

27 Apr '11

1 new changeset in galaxy-central: http://bitbucket.org/galaxy/galaxy-central/changeset/cef47f0ab8f2/ changeset: r5460:cef47f0ab8f2 user: fubar date: 2011-04-27 18:41:57 summary: Large push with modifications to the Picard tools recently added plus 5 new ones and tests Also includes fastqc which is a very comprehensive QC report generator for sam/bam and fastq affected #: 42 files (22.5 KB) Diff too large to display. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0