galaxy-dev August 2009

galaxy-dev@lists.galaxyproject.org

12 participants
156 discussions

[hg] galaxy 2566: Provide better bi-directional Python version c...
by Greg Von Kuster 24 Aug '09

24 Aug '09

details: http://www.bx.psu.edu/hg/galaxy/rev/32c76fdcacd2 changeset: 2566:32c76fdcacd2 user: Greg Von Kuster <greg(a)bx.psu.edu> date: Sat Aug 15 16:26:37 2009 -0400 description: Provide better bi-directional Python version compatibility for encryption. 5 file(s) affected in this change: lib/galaxy/model/__init__.py lib/galaxy/tools/__init__.py lib/galaxy/util/hash_util.py lib/galaxy/web/controllers/async.py lib/galaxy/web/controllers/genetrack.py diffs (189 lines): diff -r 2ae19c12114c -r 32c76fdcacd2 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Fri Aug 14 15:37:31 2009 -0400 +++ b/lib/galaxy/model/__init__.py Sat Aug 15 16:26:37 2009 -0400 @@ -13,11 +13,7 @@ import galaxy.datatypes.registry from galaxy.datatypes.metadata import MetadataCollection from galaxy.security import RBACAgent, get_permitted_actions -using_24 = sys.version_info[:2] < ( 2, 5 ) -if using_24: - import sha -else: - import hashlib +from galaxy.util.hash_util import * import logging log = logging.getLogger( __name__ ) @@ -42,16 +38,10 @@ def set_password_cleartext( self, cleartext ): """Set 'self.password' to the digest of 'cleartext'.""" - if using_24: - self.password = sha.new( cleartext ).hexdigest() - else: - self.password = hashlib.sha1( cleartext ).hexdigest() + self.password = new_secure_hash( text_type=cleartext ) def check_password( self, cleartext ): """Check if 'cleartext' matches 'self.password' when hashed.""" - if using_24: - return self.password == sha.new( cleartext ).hexdigest() - else: - return self.password == hashlib.sha1( cleartext ).hexdigest() + return self.password == new_secure_hash( text_type=cleartext ) def all_roles( self ): roles = [ ura.role for ura in self.roles ] for group in [ uga.group for uga in self.groups ]: diff -r 2ae19c12114c -r 32c76fdcacd2 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Fri Aug 14 15:37:31 2009 -0400 +++ b/lib/galaxy/tools/__init__.py Sat Aug 15 16:26:37 2009 -0400 @@ -7,7 +7,7 @@ import logging, os, string, sys, tempfile, glob, shutil import simplejson -import hmac, binascii +import binascii from UserDict import DictMixin from galaxy.util.odict import odict from galaxy.util.bunch import Bunch @@ -24,12 +24,7 @@ from galaxy.util.none_like import NoneDataset from galaxy.datatypes import sniff from cgi import FieldStorage - -using_24 = sys.version_info[:2] < ( 2, 5 ) -if using_24: - import sha -else: - import hashlib +from galaxy.util.hash_util import * log = logging.getLogger( __name__ ) @@ -215,10 +210,7 @@ value["__page__"] = self.page value = simplejson.dumps( value ) # Make it secure - if using_24: - a = hmac.new( app.config.tool_secret, value, sha ).hexdigest() - else: - a = hmac.new( app.config.tool_secret, value, hashlib.sha1 ).hexdigest() + a = hmac_new( app.config.tool_secret, value ) b = binascii.hexlify( value ) return "%s:%s" % ( a, b ) def decode( self, value, tool, app ): @@ -228,10 +220,7 @@ # Extract and verify hash a, b = value.split( ":" ) value = binascii.unhexlify( b ) - if using_24: - test = hmac.new( app.config.tool_secret, value, sha ).hexdigest() - else: - test = hmac.new( app.config.tool_secret, value, hashlib.sha1 ).hexdigest() + test = hmac_new( app.config.tool_secret, value ) assert a == test # Restore from string values = json_fix( simplejson.loads( value ) ) diff -r 2ae19c12114c -r 32c76fdcacd2 lib/galaxy/util/hash_util.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/util/hash_util.py Sat Aug 15 16:26:37 2009 -0400 @@ -0,0 +1,28 @@ +import sys, logging +using_24 = sys.version_info[:2] < ( 2, 5 ) +if using_24: + import sha +else: + import hashlib +import hmac + +log = logging.getLogger( __name__ ) + +""" +Utility functions for bi-directional Python version compatibility. Python 2.5 +introduced hashlib which replaced sha in Python 2.4 and previous versions. +""" +def new_secure_hash( text_type=None ): + if using_24: + if text_type: + return sha.new( text_type ).hexdigest() + return sha.new() + else: + if text_type: + return hashlib.sha1( text_type ).hexdigest() + return hashlib.sha1() +def hmac_new( key, value ): + if using_24: + return hmac.new( key, value, sha ).hexdigest() + else: + return hmac.new( key, value, hashlib.sha1 ).hexdigest() diff -r 2ae19c12114c -r 32c76fdcacd2 lib/galaxy/web/controllers/async.py --- a/lib/galaxy/web/controllers/async.py Fri Aug 14 15:37:31 2009 -0400 +++ b/lib/galaxy/web/controllers/async.py Sat Aug 15 16:26:37 2009 -0400 @@ -6,13 +6,8 @@ from galaxy import jobs, util, datatypes, web -import logging, urllib, hmac, sys - -using_24 = sys.version_info[:2] < ( 2, 5 ) -if using_24: - import sha -else: - import hashlib +import logging, urllib, sys +from galaxy.util.hash_util import * log = logging.getLogger( __name__ ) @@ -63,10 +58,7 @@ return "Data %s does not exist or has already been deleted" % data_id if STATUS == 'OK': - if using_24: - key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), sha ).hexdigest() - else: - key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), hashlib.sha1 ).hexdigest() + key = hmac_new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id ) ) if key != data_secret: return "You do not have permission to alter data %s." % data_id # push the job into the queue @@ -124,10 +116,7 @@ trans.log_event( "Added dataset %d to history %d" %(data.id, trans.history.id ), tool_id=tool_id ) try: - if using_24: - key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), sha ).hexdigest() - else: - key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), hashlib.sha1 ).hexdigest() + key = hmac_new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id ) ) galaxy_url = trans.request.base + '/async/%s/%s/%s' % ( tool_id, data.id, key ) params.update( { 'GALAXY_URL' :galaxy_url } ) params.update( { 'data_id' :data.id } ) diff -r 2ae19c12114c -r 32c76fdcacd2 lib/galaxy/web/controllers/genetrack.py --- a/lib/galaxy/web/controllers/genetrack.py Fri Aug 14 15:37:31 2009 -0400 +++ b/lib/galaxy/web/controllers/genetrack.py Sat Aug 15 16:26:37 2009 -0400 @@ -4,12 +4,7 @@ from mako.template import Template from mako.lookup import TemplateLookup from galaxy.web.base.controller import * - -using_24 = sys.version_info[:2] < ( 2, 5 ) -if using_24: - import sha -else: - import hashlib +from galaxy.util.hash_util import * try: import pkg_resources @@ -269,10 +264,7 @@ tmpl_name, track_maker = conf.PLOT_MAPPER[param.plot] # check against a hash, display an image that already exists if it was previously created. - if using_24: - hash = sha.new() - else: - hash = hashlib.sha1() + hash = new_secure_hash() hash.update(str(dataset_id)) for key in sorted(kwds.keys()): hash.update(str(kwds[key]))

1 0

[hg] galaxy 2567: Change import from . to __init__.
by Greg Von Kuster 24 Aug '09

24 Aug '09

details: http://www.bx.psu.edu/hg/galaxy/rev/b56291fad13d changeset: 2567:b56291fad13d user: Dan Blankenberg <dan(a)bx.psu.edu> date: Mon Aug 17 09:54:57 2009 -0400 description: Change import from . to __init__. 2 file(s) affected in this change: lib/galaxy/tools/actions/metadata.py lib/galaxy/tools/actions/upload.py diffs (19 lines): diff -r 32c76fdcacd2 -r b56291fad13d lib/galaxy/tools/actions/metadata.py --- a/lib/galaxy/tools/actions/metadata.py Sat Aug 15 16:26:37 2009 -0400 +++ b/lib/galaxy/tools/actions/metadata.py Mon Aug 17 09:54:57 2009 -0400 @@ -1,4 +1,4 @@ -from . import ToolAction +from __init__ import ToolAction from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper import logging diff -r 32c76fdcacd2 -r b56291fad13d lib/galaxy/tools/actions/upload.py --- a/lib/galaxy/tools/actions/upload.py Sat Aug 15 16:26:37 2009 -0400 +++ b/lib/galaxy/tools/actions/upload.py Mon Aug 17 09:54:57 2009 -0400 @@ -1,5 +1,5 @@ import os, shutil, urllib, StringIO, re, gzip, tempfile, shutil, zipfile -from . import ToolAction +from __init__ import ToolAction from galaxy import datatypes, jobs from galaxy.datatypes import sniff from galaxy import model, util

1 0

[hg] galaxy 2565: Deprecated code corrections for supporting Pyt...
by Greg Von Kuster 24 Aug '09

24 Aug '09

details: http://www.bx.psu.edu/hg/galaxy/rev/2ae19c12114c changeset: 2565:2ae19c12114c user: Greg Von Kuster <greg(a)bx.psu.edu> date: Fri Aug 14 15:37:31 2009 -0400 description: Deprecated code corrections for supporting Python 2.6. Many of the Python 2.6 eggs are still throwing DeprecationWarning messages, so some things still won't work. Eggs for 2.6 need to be re-scrambled after corrections are made. 26 file(s) affected in this change: lib/galaxy/datatypes/coverage.py lib/galaxy/datatypes/data.py lib/galaxy/datatypes/genetics.py lib/galaxy/datatypes/interval.py lib/galaxy/model/__init__.py lib/galaxy/tools/__init__.py lib/galaxy/web/controllers/admin.py lib/galaxy/web/controllers/async.py lib/galaxy/web/controllers/dataset.py lib/galaxy/web/controllers/genetrack.py lib/galaxy/web/controllers/root.py lib/galaxy/web/controllers/tool_runner.py lib/galaxy/web/framework/__init__.py lib/galaxy/webapps/reports/controllers/root.py tools/data_source/genbank.py tools/data_source/ucsc_proxy.py tools/new_operations/get_flanks.py tools/new_operations/operation_filter.py tools/new_operations/subtract_query.py tools/regVariation/windowSplitter.py tools/stats/column_maker.py tools/stats/filtering.py tools/stats/grouping.py tools/stats/gsummary.py tools/visualization/build_ucsc_custom_track_code.py tools/visualization/genetrack_code.py diffs (587 lines): diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/datatypes/coverage.py --- a/lib/galaxy/datatypes/coverage.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/datatypes/coverage.py Fri Aug 14 15:37:31 2009 -0400 @@ -5,7 +5,7 @@ import pkg_resources pkg_resources.require( "bx-python" ) -import logging, os, sys, time, sets, tempfile, shutil +import logging, os, sys, time, tempfile, shutil import data from galaxy import util from galaxy.datatypes.sniff import * diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/datatypes/data.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,4 +1,4 @@ -import logging, os, sys, time, sets, tempfile +import logging, os, sys, time, tempfile from galaxy import util from galaxy.util.odict import odict from galaxy.util.bunch import Bunch diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/datatypes/genetics.py --- a/lib/galaxy/datatypes/genetics.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/datatypes/genetics.py Fri Aug 14 15:37:31 2009 -0400 @@ -12,7 +12,7 @@ august 20 2007 """ -import logging, os, sys, time, sets, tempfile, shutil +import logging, os, sys, time, tempfile, shutil import data from galaxy import util from cgi import escape diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/datatypes/interval.py --- a/lib/galaxy/datatypes/interval.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/datatypes/interval.py Fri Aug 14 15:37:31 2009 -0400 @@ -5,7 +5,7 @@ import pkg_resources pkg_resources.require( "bx-python" ) -import logging, os, sys, time, sets, tempfile, shutil +import logging, os, sys, time, tempfile, shutil import data from galaxy import util from galaxy.datatypes.sniff import * diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/model/__init__.py Fri Aug 14 15:37:31 2009 -0400 @@ -5,8 +5,7 @@ the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -import os.path, os, errno -import sha +import os.path, os, errno, sys import galaxy.datatypes from galaxy.util.bunch import Bunch from galaxy import util @@ -14,8 +13,11 @@ import galaxy.datatypes.registry from galaxy.datatypes.metadata import MetadataCollection from galaxy.security import RBACAgent, get_permitted_actions - - +using_24 = sys.version_info[:2] < ( 2, 5 ) +if using_24: + import sha +else: + import hashlib import logging log = logging.getLogger( __name__ ) @@ -40,10 +42,16 @@ def set_password_cleartext( self, cleartext ): """Set 'self.password' to the digest of 'cleartext'.""" - self.password = sha.new( cleartext ).hexdigest() + if using_24: + self.password = sha.new( cleartext ).hexdigest() + else: + self.password = hashlib.sha1( cleartext ).hexdigest() def check_password( self, cleartext ): """Check if 'cleartext' matches 'self.password' when hashed.""" - return self.password == sha.new( cleartext ).hexdigest() + if using_24: + return self.password == sha.new( cleartext ).hexdigest() + else: + return self.password == hashlib.sha1( cleartext ).hexdigest() def all_roles( self ): roles = [ ura.role for ura in self.roles ] for group in [ uga.group for uga in self.groups ]: diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/tools/__init__.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,15 +1,13 @@ """ Classes encapsulating galaxy tools and tool configuration. """ - import pkg_resources; pkg_resources.require( "simplejson" ) import logging, os, string, sys, tempfile, glob, shutil import simplejson -import sha, hmac, binascii - +import hmac, binascii from UserDict import DictMixin from galaxy.util.odict import odict from galaxy.util.bunch import Bunch @@ -26,6 +24,12 @@ from galaxy.util.none_like import NoneDataset from galaxy.datatypes import sniff from cgi import FieldStorage + +using_24 = sys.version_info[:2] < ( 2, 5 ) +if using_24: + import sha +else: + import hashlib log = logging.getLogger( __name__ ) @@ -211,7 +215,10 @@ value["__page__"] = self.page value = simplejson.dumps( value ) # Make it secure - a = hmac.new( app.config.tool_secret, value, sha ).hexdigest() + if using_24: + a = hmac.new( app.config.tool_secret, value, sha ).hexdigest() + else: + a = hmac.new( app.config.tool_secret, value, hashlib.sha1 ).hexdigest() b = binascii.hexlify( value ) return "%s:%s" % ( a, b ) def decode( self, value, tool, app ): @@ -221,7 +228,10 @@ # Extract and verify hash a, b = value.split( ":" ) value = binascii.unhexlify( b ) - test = hmac.new( app.config.tool_secret, value, sha ).hexdigest() + if using_24: + test = hmac.new( app.config.tool_secret, value, sha ).hexdigest() + else: + test = hmac.new( app.config.tool_secret, value, hashlib.sha1 ).hexdigest() assert a == test # Restore from string values = json_fix( simplejson.loads( value ) ) @@ -453,7 +463,6 @@ self.tests = None # Determine if this tool can be used in workflows self.is_workflow_compatible = self.check_workflow_compatible() - def parse_inputs( self, root ): """ diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/web/controllers/admin.py --- a/lib/galaxy/web/controllers/admin.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/web/controllers/admin.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,9 +1,14 @@ -import shutil, StringIO, operator, urllib, gzip, tempfile, sets, string, sys +import shutil, StringIO, operator, urllib, gzip, tempfile, string, sys from datetime import datetime, timedelta from galaxy import util, datatypes from galaxy.web.base.controller import * from galaxy.model.orm import * from galaxy.web.controllers.forms import get_all_forms, get_form_widgets +# Older py compatibility +try: + set() +except: + from sets import Set as set import logging log = logging.getLogger( __name__ ) @@ -1236,16 +1241,16 @@ if v == trans.app.security_agent.permitted_actions.DATASET_ACCESS: if len( in_roles ) > 1: # Get the set of all users that are being associated with the dataset - in_roles_set = sets.Set() + in_roles_set = set() for role in in_roles: in_roles_set.add( role ) - users_set = sets.Set() + users_set = set() for role in in_roles: for ura in role.users: users_set.add( ura.user ) # Make sure that at least 1 user has every role being associated with the dataset for user in users_set: - user_roles_set = sets.Set() + user_roles_set = set() for ura in user.roles: user_roles_set.add( ura.role ) if in_roles_set.issubset( user_roles_set ): @@ -1421,16 +1426,16 @@ if v == trans.app.security_agent.permitted_actions.DATASET_ACCESS: if len( in_roles ) > 1: # Get the set of all users that are being associated with the dataset - in_roles_set = sets.Set() + in_roles_set = set() for role in in_roles: in_roles_set.add( role ) - users_set = sets.Set() + users_set = set() for role in in_roles: for ura in role.users: users_set.add( ura.user ) # Make sure that at least 1 user has every role being associated with the dataset for user in users_set: - user_roles_set = sets.Set() + user_roles_set = set() for ura in user.roles: user_roles_set.add( ura.role ) if in_roles_set.issubset( user_roles_set ): diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/web/controllers/async.py --- a/lib/galaxy/web/controllers/async.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/web/controllers/async.py Fri Aug 14 15:37:31 2009 -0400 @@ -6,8 +6,13 @@ from galaxy import jobs, util, datatypes, web -import logging, urllib -import sha, hmac +import logging, urllib, hmac, sys + +using_24 = sys.version_info[:2] < ( 2, 5 ) +if using_24: + import sha +else: + import hashlib log = logging.getLogger( __name__ ) @@ -58,7 +63,10 @@ return "Data %s does not exist or has already been deleted" % data_id if STATUS == 'OK': - key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), sha ).hexdigest() + if using_24: + key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), sha ).hexdigest() + else: + key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), hashlib.sha1 ).hexdigest() if key != data_secret: return "You do not have permission to alter data %s." % data_id # push the job into the queue @@ -116,7 +124,10 @@ trans.log_event( "Added dataset %d to history %d" %(data.id, trans.history.id ), tool_id=tool_id ) try: - key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), sha ).hexdigest() + if using_24: + key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), sha ).hexdigest() + else: + key = hmac.new( trans.app.config.tool_secret, "%d:%d" % ( data.id, data.history_id), hashlib.sha1 ).hexdigest() galaxy_url = trans.request.base + '/async/%s/%s/%s' % ( tool_id, data.id, key ) params.update( { 'GALAXY_URL' :galaxy_url } ) params.update( { 'data_id' :data.id } ) diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/web/controllers/dataset.py --- a/lib/galaxy/web/controllers/dataset.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/web/controllers/dataset.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,4 +1,4 @@ -import logging, os, sets, string, shutil, re, socket, mimetypes, smtplib, urllib +import logging, os, string, shutil, re, socket, mimetypes, smtplib, urllib from galaxy.web.base.controller import * from galaxy import util, datatypes, jobs, web, model diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/web/controllers/genetrack.py --- a/lib/galaxy/web/controllers/genetrack.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/web/controllers/genetrack.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,11 +1,15 @@ -import time, glob, os +import time, glob, os, sys from itertools import cycle -import sha - from mako import exceptions from mako.template import Template from mako.lookup import TemplateLookup from galaxy.web.base.controller import * + +using_24 = sys.version_info[:2] < ( 2, 5 ) +if using_24: + import sha +else: + import hashlib try: import pkg_resources @@ -265,7 +269,10 @@ tmpl_name, track_maker = conf.PLOT_MAPPER[param.plot] # check against a hash, display an image that already exists if it was previously created. - hash = sha.new() + if using_24: + hash = sha.new() + else: + hash = hashlib.sha1() hash.update(str(dataset_id)) for key in sorted(kwds.keys()): hash.update(str(kwds[key])) diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/web/controllers/root.py --- a/lib/galaxy/web/controllers/root.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/web/controllers/root.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,7 +1,7 @@ """ Contains the main interface in the Universe class """ -import logging, os, sets, string, shutil, urllib, re, socket +import logging, os, string, shutil, urllib, re, socket from cgi import escape, FieldStorage from galaxy import util, datatypes, jobs, web, util from galaxy.web.base.controller import * @@ -60,7 +60,6 @@ trans.response.set_content_type('text/xml') return trans.fill_template_mako( "root/history_as_xml.mako", history=history, show_deleted=util.string_as_bool( show_deleted ) ) else: - template = "root/history.mako" show_deleted = util.string_as_bool( show_deleted ) query = trans.sa_session.query( model.HistoryDatasetAssociation ) \ .filter( model.HistoryDatasetAssociation.history == history ) \ diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/web/controllers/tool_runner.py --- a/lib/galaxy/web/controllers/tool_runner.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/web/controllers/tool_runner.py Fri Aug 14 15:37:31 2009 -0400 @@ -117,7 +117,6 @@ tool_state_string = util.object_to_string(state.encode(tool, trans.app)) # Setup context for template history = trans.get_history() - template = "tool_form.mako" vars = dict( tool_state=state, errors = {} ) # Is the "add frame" stuff neccesary here? add_frame = AddFrameData() @@ -125,17 +124,13 @@ if from_noframe is not None: add_frame.wiki_url = trans.app.config.wiki_url add_frame.from_noframe = True - return trans.fill_template( template, history=history, toolbox=toolbox, tool=tool, util=util, add_frame=add_frame, **vars ) - - + return trans.fill_template( "tool_form.mako", history=history, toolbox=toolbox, tool=tool, util=util, add_frame=add_frame, **vars ) @web.expose def redirect( self, trans, redirect_url=None, **kwd ): if not redirect_url: return trans.show_error_message( "Required URL for redirection missing" ) trans.log_event( "Redirecting to: %s" % redirect_url ) return trans.fill_template( 'root/redirect.mako', redirect_url=redirect_url ) - - @web.json def upload_async_create( self, trans, tool_id=None, **kwd ): """ diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/web/framework/__init__.py --- a/lib/galaxy/web/framework/__init__.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/web/framework/__init__.py Fri Aug 14 15:37:31 2009 -0400 @@ -93,8 +93,8 @@ """ Exception to make throwing errors from deep in controllers easier """ - def __init__( self, message, type="info" ): - self.message = message + def __init__( self, err_msg, type="info" ): + self.err_msg = err_msg self.type = type def error( message ): @@ -117,7 +117,7 @@ self.security = galaxy_app.security def handle_controller_exception( self, e, trans, **kwargs ): if isinstance( e, MessageException ): - return trans.show_message( e.message, e.type ) + return trans.show_message( e.err_msg, e.type ) def make_body_iterable( self, trans, body ): if isinstance( body, FormBuilder ): body = trans.show_form( body ) diff -r eb1244477b90 -r 2ae19c12114c lib/galaxy/webapps/reports/controllers/root.py --- a/lib/galaxy/webapps/reports/controllers/root.py Fri Aug 14 12:17:45 2009 -0400 +++ b/lib/galaxy/webapps/reports/controllers/root.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,8 +1,8 @@ -import sys, os, operator, sets, string, shutil, re, socket, urllib +import sys, os, operator, string, shutil, re, socket, urllib, time from galaxy import web from cgi import escape, FieldStorage from galaxy.webapps.reports.base.controller import * -import logging, sets, time +import logging log = logging.getLogger( __name__ ) class Report( BaseController ): diff -r eb1244477b90 -r 2ae19c12114c tools/data_source/genbank.py --- a/tools/data_source/genbank.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/data_source/genbank.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,6 +1,6 @@ #!/usr/bin/env python from Bio import GenBank -import sys, os, sets, textwrap +import sys, os, textwrap assert sys.version_info[:2] >= ( 2, 4 ) diff -r eb1244477b90 -r 2ae19c12114c tools/data_source/ucsc_proxy.py --- a/tools/data_source/ucsc_proxy.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/data_source/ucsc_proxy.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,6 +1,6 @@ #!/usr/bin/env python import urllib -import sys, os, sets +import sys, os assert sys.version_info[:2] >= ( 2, 4 ) diff -r eb1244477b90 -r 2ae19c12114c tools/new_operations/get_flanks.py --- a/tools/new_operations/get_flanks.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/new_operations/get_flanks.py Fri Aug 14 15:37:31 2009 -0400 @@ -9,7 +9,7 @@ -o, --off=N: Offset """ -import sys, sets, re, os +import sys, re, os from galaxy import eggs import pkg_resources; pkg_resources.require( "bx-python" ) from bx.cookbook import doc_optparse diff -r eb1244477b90 -r 2ae19c12114c tools/new_operations/operation_filter.py --- a/tools/new_operations/operation_filter.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/new_operations/operation_filter.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,8 +1,13 @@ # runs after the job (and after the default post-filter) -import sets, os +import os from galaxy import eggs from galaxy import jobs from galaxy.tools.parameters import DataToolParameter +# Older py compatibility +try: + set() +except: + from sets import Set as set #def exec_before_process(app, inp_data, out_data, param_dict, tool=None): # """Sets the name of the data""" @@ -11,8 +16,8 @@ # raise Exception, '<p><font color="yellow">Both Queries must be from the same genome build</font></p>' def validate_input( trans, error_map, param_values, page_param_map ): - dbkeys = sets.Set() - data_param_names = sets.Set() + dbkeys = set() + data_param_names = set() data_params = 0 for name, param in page_param_map.iteritems(): if isinstance( param, DataToolParameter ): diff -r eb1244477b90 -r 2ae19c12114c tools/new_operations/subtract_query.py --- a/tools/new_operations/subtract_query.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/new_operations/subtract_query.py Fri Aug 14 15:37:31 2009 -0400 @@ -5,12 +5,16 @@ Subtract an entire query from another query usage: %prog in_file_1 in_file_2 begin_col end_col output """ - -import sys, sets, re - +import sys, re from galaxy import eggs import pkg_resources; pkg_resources.require( "bx-python" ) from bx.cookbook import doc_optparse + +# Older py compatibility +try: + set() +except: + from sets import Set as set assert sys.version_info[:2] >= ( 2, 4 ) diff -r eb1244477b90 -r 2ae19c12114c tools/regVariation/windowSplitter.py --- a/tools/regVariation/windowSplitter.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/regVariation/windowSplitter.py Fri Aug 14 15:37:31 2009 -0400 @@ -7,7 +7,7 @@ -l, --cols=N,N,N,N: Columns for chrom, start, end, strand in file """ -import sys, sets, re, os +import sys, re, os from galaxy import eggs import pkg_resources; pkg_resources.require( "bx-python" ) diff -r eb1244477b90 -r 2ae19c12114c tools/stats/column_maker.py --- a/tools/stats/column_maker.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/stats/column_maker.py Fri Aug 14 15:37:31 2009 -0400 @@ -2,7 +2,7 @@ # This tool takes a tab-delimited textfile as input and creates another column in the file which is the result of # a computation performed on every row in the original file. The tool will skip over invalid lines within the file, # informing the user about the number of lines skipped. -import sys, sets, re, os.path +import sys, re, os.path from galaxy import eggs from galaxy.tools import validation from galaxy.datatypes import metadata diff -r eb1244477b90 -r 2ae19c12114c tools/stats/filtering.py --- a/tools/stats/filtering.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/stats/filtering.py Fri Aug 14 15:37:31 2009 -0400 @@ -2,8 +2,13 @@ # This tool takes a tab-delimited text file as input and creates filters on columns based on certain properties. # The tool will skip over invalid lines within the file, informing the user about the number of lines skipped. -import sys, sets, re, os.path +import sys, re, os.path from galaxy import eggs +# Older py compatibility +try: + set() +except: + from sets import Set as set assert sys.version_info[:2] >= ( 2, 4 ) @@ -13,7 +18,7 @@ for item in items_to_strip: if filter_condition.find( item ) >= 0: filter_condition = filter_condition.replace( item, ' ' ) - operands = sets.Set( filter_condition.split( ' ' ) ) + operands = set( filter_condition.split( ' ' ) ) return operands def stop_err( msg ): diff -r eb1244477b90 -r 2ae19c12114c tools/stats/grouping.py --- a/tools/stats/grouping.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/stats/grouping.py Fri Aug 14 15:37:31 2009 -0400 @@ -3,7 +3,7 @@ """ This tool provides the SQL "group by" functionality. """ -import sys, string, re, commands, tempfile, random, sets +import sys, string, re, commands, tempfile, random from rpy import * def stop_err(msg): diff -r eb1244477b90 -r 2ae19c12114c tools/stats/gsummary.py --- a/tools/stats/gsummary.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/stats/gsummary.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,7 +1,12 @@ #!/usr/bin/python -import sys, sets, re, tempfile +import sys, re, tempfile from rpy import * +# Older py compatibility +try: + set() +except: + from sets import Set as set assert sys.version_info[:2] >= ( 2, 4 ) @@ -33,7 +38,7 @@ for word in re.compile( '[a-zA-Z]+' ).findall( expression ): if word and not word in math_allowed: stop_err( "Invalid expression '%s': term '%s' is not recognized or allowed" %( expression, word ) ) - symbols = sets.Set() + symbols = set() for symbol in re.compile( '[^a-z0-9\s]+' ).findall( expression ): if symbol and not symbol in ops_allowed: stop_err( "Invalid expression '%s': operator '%s' is not recognized or allowed" % ( expression, symbol ) ) diff -r eb1244477b90 -r 2ae19c12114c tools/visualization/build_ucsc_custom_track_code.py --- a/tools/visualization/build_ucsc_custom_track_code.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/visualization/build_ucsc_custom_track_code.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,6 +1,10 @@ # runs after the job (and after the default post-filter) -from sets import Set as set +# Older py compatibility +try: + set() +except: + from sets import Set as set def validate_input( trans, error_map, param_values, page_param_map ): dbkeys = set() diff -r eb1244477b90 -r 2ae19c12114c tools/visualization/genetrack_code.py --- a/tools/visualization/genetrack_code.py Fri Aug 14 12:17:45 2009 -0400 +++ b/tools/visualization/genetrack_code.py Fri Aug 14 15:37:31 2009 -0400 @@ -1,4 +1,4 @@ -import sets, os +import os from galaxy import eggs from galaxy import jobs from galaxy.tools.parameters import DataToolParameter

1 0

[hg] galaxy 2568: Added two new datatypes (sam and bam) and an a...
by Greg Von Kuster 24 Aug '09

24 Aug '09

details: http://www.bx.psu.edu/hg/galaxy/rev/670f8800a2bf changeset: 2568:670f8800a2bf user: Kelly Vincent <kpvincent(a)bx.psu.edu> date: Wed Aug 19 09:13:21 2009 -0400 description: Added two new datatypes (sam and bam) and an associated test 7 file(s) affected in this change: datatypes_conf.xml.sample lib/galaxy/datatypes/images.py lib/galaxy/datatypes/registry.py lib/galaxy/datatypes/tabular.py lib/galaxy/datatypes/test/1.sam test-data/1.sam test/functional/test_sniffing_and_metadata_settings.py diffs (362 lines): diff -r b56291fad13d -r 670f8800a2bf datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample Mon Aug 17 09:54:57 2009 -0400 +++ b/datatypes_conf.xml.sample Wed Aug 19 09:13:21 2009 -0400 @@ -3,6 +3,7 @@ <registration converters_path="lib/galaxy/datatypes/converters"> <datatype extension="ab1" type="galaxy.datatypes.images:Ab1" mimetype="application/octet-stream" display_in_upload="true"/> <datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true"/> + <datatype extension="bam" type="galaxy.datatypes.images:Bam" mimetype="application/octet-stream"/> <datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true"> <converter file="bed_to_gff_converter.xml" target_datatype="gff"/> <converter file="interval_to_coverage.xml" target_datatype="coverage"/> @@ -49,6 +50,7 @@ <datatype extension="qualsolexa" type="galaxy.datatypes.qualityscore:QualityScoreSolexa" display_in_upload="true"/> <datatype extension="qualsolid" type="galaxy.datatypes.qualityscore:QualityScoreSOLiD" display_in_upload="true"/> <datatype extension="qual454" type="galaxy.datatypes.qualityscore:QualityScore454" display_in_upload="true"/> + <datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true"/> <datatype extension="scf" type="galaxy.datatypes.images:Scf" mimetype="application/octet-stream" display_in_upload="true"/> <datatype extension="taxonomy" type="galaxy.datatypes.tabular:Taxonomy" display_in_upload="true"/> <datatype extension="tabular" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true"/> @@ -205,5 +207,6 @@ <sniffer type="galaxy.datatypes.interval:Gff"/> <sniffer type="galaxy.datatypes.interval:Gff3"/> <sniffer type="galaxy.datatypes.interval:Interval"/> + <sniffer type="galaxy.datatypes.tabular:Sam"/> </sniffers> </datatypes> diff -r b56291fad13d -r 670f8800a2bf lib/galaxy/datatypes/images.py --- a/lib/galaxy/datatypes/images.py Mon Aug 17 09:54:57 2009 -0400 +++ b/lib/galaxy/datatypes/images.py Wed Aug 19 09:13:21 2009 -0400 @@ -4,6 +4,8 @@ import data import logging +from galaxy.datatypes.metadata import MetadataElement +from galaxy.datatypes import metadata from galaxy.datatypes.sniff import * from urllib import urlencode, quote_plus import zipfile @@ -187,7 +189,7 @@ return 'text/html' def sniff( self, filename ): """ - Determines wether the file is in html format + Determines whether the file is in html format >>> fname = get_test_fname( 'complete.bed' ) >>> Html().sniff( fname ) @@ -233,3 +235,25 @@ return dataset.peek except: return "peek unavailable" + +class Bam( data.Binary ): + """Class describing a BAM binary file""" + file_ext = "bam" + MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) + def set_peek( self, dataset ): + if not dataset.dataset.purged: + export_url = "/history_add_to?" + urlencode({'history_id':dataset.history_id,'ext':'bam','name':'bam alignments','info':'Alignments file','dbkey':dataset.dbkey}) + dataset.peek = "Binary bam alignments file" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Binary bam alignments file (%s)" % ( data.nice_size( dataset.get_size() ) ) + def get_mime(self): + """Returns the mime type of the datatype""" + return 'application/octet-stream' + \ No newline at end of file diff -r b56291fad13d -r 670f8800a2bf lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py Mon Aug 17 09:54:57 2009 -0400 +++ b/lib/galaxy/datatypes/registry.py Wed Aug 19 09:13:21 2009 -0400 @@ -111,6 +111,7 @@ self.datatypes_by_extension = { 'ab1' : images.Ab1(), 'axt' : sequence.Axt(), + 'bam' : images.Bam(), 'bed' : interval.Bed(), 'binseq.zip' : images.Binseq(), 'blastxml' : xml.BlastXml(), @@ -130,6 +131,7 @@ 'qualsolid' : qualityscore.QualityScoreSOLiD(), 'qualsolexa' : qualityscore.QualityScoreSolexa(), 'qual454' : qualityscore.QualityScore454(), + 'sam' : tabular.Sam(), 'scf' : images.Scf(), 'tabular' : tabular.Tabular(), 'taxonomy' : tabular.Taxonomy(), @@ -140,6 +142,7 @@ self.mimetypes_by_extension = { 'ab1' : 'application/octet-stream', 'axt' : 'text/plain', + 'bam' : 'application/octet-stream', 'bed' : 'text/plain', 'binseq.zip' : 'application/zip', 'blastxml' : 'text/plain', @@ -157,6 +160,7 @@ 'qualsolid' : 'text/plain', 'qualsolexa' : 'text/plain', 'qual454' : 'text/plain', + 'sam' : 'text/plain', 'scf' : 'application/octet-stream', 'tabular' : 'text/plain', 'taxonomy' : 'text/plain', @@ -184,7 +188,8 @@ interval.CustomTrack(), interval.Gff(), interval.Gff3(), - interval.Interval() + interval.Interval(), + tabular.Sam() ] def append_to_sniff_order(): # Just in case any supported data types are not included in the config's sniff_order section. diff -r b56291fad13d -r 670f8800a2bf lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py Mon Aug 17 09:54:57 2009 -0400 +++ b/lib/galaxy/datatypes/tabular.py Wed Aug 19 09:13:21 2009 -0400 @@ -236,3 +236,84 @@ out = "Can't create peek %s" % exc return out +class Sam( Tabular ): + file_ext = 'sam' + def __init__(self, **kwd): + """Initialize taxonomy datatype""" + Tabular.__init__( self, **kwd ) + self.column_names = ['QNAME', 'FLAG', 'RNAME', 'POS', 'MAPQ', 'CIGAR', + 'MRNM', 'MPOS', 'ISIZE', 'SEQ', 'QUAL', 'OPT' + ] + def make_html_table( self, dataset, skipchars=[] ): + """Create HTML table, used for displaying peek""" + out = ['<table cellspacing="0" cellpadding="3">'] + try: + # Generate column header + out.append( '<tr>' ) + for i, name in enumerate( self.column_names ): + out.append( '<th>%s.%s</th>' % ( str( i+1 ), name ) ) + # This data type requires at least 11 columns in the data + if dataset.metadata.columns - len( self.column_names ) > 0: + for i in range( len( self.column_names ), dataset.metadata.columns ): + out.append( '<th>%s</th>' % str( i+1 ) ) + out.append( '</tr>' ) + out.append( self.make_html_peek_rows( dataset, skipchars=skipchars ) ) + out.append( '</table>' ) + out = "".join( out ) + except Exception, exc: + out = "Can't create peek %s" % exc + return out + def sniff( self, filename ): + """ + Determines whether the file is in SAM format + + A file in SAM format consists of lines of tab-separated data. + The following header line may be the first line: + @QNAME FLAG RNAME POS MAPQ CIGAR MRNM MPOS ISIZE SEQ QUAL + or + @QNAME FLAG RNAME POS MAPQ CIGAR MRNM MPOS ISIZE SEQ QUAL OPT + Data in the OPT column is optional and can consist of tab-separated data + + For complete details see http://samtools.sourceforge.net/SAM1.pdf + + Rules for sniffing as True: + There must be 11 or more columns of data on each line + Columns 2 (FLAG), 4(POS), 5 (MAPQ), 8 (MPOS), and 9 (ISIZE) must be numbers (9 can be negative) + We will only check that up to the first 5 alignments are correctly formatted. + + >>> fname = get_test_fname( 'sequence.maf' ) + >>> Sam().sniff( fname ) + False + >>> fname = get_test_fname( '1.sam' ) + >>> Sam().sniff( fname ) + True + """ + try: + fh = open( filename ) + count = 0 + while True: + line = fh.readline() + line = line.strip() + if not line: + break #EOF + if line: + if line[0] != '@': + linePieces = line.split('\t') + if len(linePieces) < 11: + return False + try: + check = int(linePieces[1]) + check = int(linePieces[3]) + check = int(linePieces[4]) + check = int(linePieces[7]) + check = int(linePieces[8]) + except ValueError: + return False + count += 1 + if count == 5: + return True + if count < 5 and count > 0: + return True + except: + pass + return False diff -r b56291fad13d -r 670f8800a2bf lib/galaxy/datatypes/test/1.sam --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lib/galaxy/datatypes/test/1.sam Wed Aug 19 09:13:21 2009 -0400 @@ -0,0 +1,97 @@ +@QNAME FLAG RNAME POS MAPQ CIGAR MRNM MPOS ISIZE SEQ QUAL OPT +1378_11_329 69 * 0 0 * * 0 0 AGACCGGGCGGGGTGGCGTTCGGT %##+'#######%###$#$##$(# +1378_11_329 133 * 0 0 * * 0 0 GTTCGTGGCCGGTGGGTGTTTGGG ###$$#$#$&#####$'$#$###$ +1378_17_1788 69 * 0 0 * * 0 0 TGCCGTGTCTTGCTAACGCCGATT #'#$$#$###%%##$$$$###### +1378_17_1788 133 * 0 0 * * 0 0 TGGGTGGATGTGTTGTCGTTCATG #$#$###$#$#######$#$#### +1378_25_2035 69 * 0 0 * * 0 0 CTGCGTGTTGGTGTCTACTGGGGT #%#'##$#$##&%#%$$$%#%#'# +1378_25_2035 133 * 0 0 * * 0 0 GTGCGTCGGGGAGGGTGCTGTCGG ######%#$%#$$###($###&&% +1378_28_770 89 chr11.nib:1-134452384 72131356 37 17M1I5M = 72131356 0 CACACTGTGACAGACAGCGCAGC 00/02!!0//1200210!!44/1 XT:A:U CM:i:2 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_28_770 181 chr11.nib:1-134452384 72131356 0 24M = 72131356 0 TTGGTGCGCGCGGTTGAGGGTTGG $$(#%%#$%#%####$%%##$### +1378_33_1945 113 chr2.nib:1-242951149 181247988 0 23M chr12.nib:1-132349534 41710908 0 GAGAGAGAGAGAGAGAGAGAGAG PQRVUMNXYRPUXYXWXSOSZ]M XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:163148 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_33_1945 177 chr12.nib:1-132349534 41710908 0 23M chr2.nib:1-242951149 181247988 0 AGAGAGAGAGAGAGAGAGAGAGA SQQWZYURVYWX]]YXTSY]]ZM XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:163148 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_34_789 69 * 0 0 * * 0 0 ATGGTGGCTGACGCGTTTGACTGT #$##%#$##$&$#%##$##$###$ +1378_34_789 133 * 0 0 * * 0 0 GGGCTTGCGTTAGTGAGAGGTTGT ###%$%$%%###$####$###$#& +1378_35_263 115 chr16.nib:1-88827254 19671878 0 23M = 19671877 -1 AGAGAGAGAGAGAGAGAGAGTCT 77543:<55#"4!&=964518A> XT:A:R CM:i:2 SM:i:0 AM:i:0 X0:i:4 X1:i:137 XM:i:2 XO:i:0 XG:i:0 MD:Z:23 +1378_35_263 179 chr16.nib:1-88827254 19671877 0 23M = 19671878 1 GAGAGAGAGAGAGAGAGAGAGTC LE7402DD34FL:27AKE>;432 XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:265 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_43_186 69 * 0 0 * * 0 0 ATACTAGTTGGGACGCGTTGTGCT #$(4%$########$#$###$$$# +1378_43_186 133 * 0 0 * * 0 0 GCTAGGGTTTGGGTTTGCGGTGGG $%#$########%##%#$###'#' +1378_51_1671 117 chr2.nib:1-242951149 190342418 0 24M = 190342418 0 CTGGCGTTCTCGGCGTGGATGGGT #####$$##$#%#%%###%$#$## +1378_51_1671 153 chr2.nib:1-242951149 190342418 37 16M1I6M = 190342418 0 TCTAACTTAGCCTCATAATAGCT /<<!"0///////00/!!0121/ XT:A:U CM:i:2 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_56_324 117 chr2.nib:1-242951149 80324999 0 24M = 80324999 0 TCCAGTCGCGTTGTTAGGTTCGGA #$#$$$#####%##%%###**#+/ +1378_56_324 153 chr2.nib:1-242951149 80324999 37 8M1I14M = 80324999 0 TTTAGCCCGAAATGCCTAGAGCA 4;6//11!"11100110////00 XT:A:U CM:i:2 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_56_773 69 * 0 0 * * 0 0 TGTCGTGAGGTCACTTATCCCCAT &%#%##%%#####&#$%##$%##$ +1378_56_773 133 * 0 0 * * 0 0 TCTGGTCGGTTTCGGGGAGTGGAA ##%%#&$###$#$##%$####%%$ +1378_62_2027 69 * 0 0 * * 0 0 CTTCCACGATCTGCTCGCTGTGGT (#&&$##$$#$%#%$$$#$###'# +1378_62_2027 133 * 0 0 * * 0 0 GTTGGCCTGGCCTGCCGTGCTGCG *##),/%##$)#%##1$#'%.#&# +1378_62_2029 69 * 0 0 * * 0 0 TCTGGGCTGTCTTCGGGTCGGTGT $%$$####$##$$#)##%%#$### +1378_62_2029 133 * 0 0 * * 0 0 GGCGGTGTGTGGTGCGGCTGTGCG /$$$=(####%####)$$%$-&%# +1378_67_1795 81 chr16.nib:1-88827254 26739130 0 23M chrY.nib:1-57772954 57401793 0 TGGCATTCCTGTAGGCAGAGAGG AZWWZS]!"QNXZ]VQ]]]/2]] XT:A:R CM:i:2 SM:i:0 AM:i:0 X0:i:3 X1:i:0 XM:i:2 XO:i:0 XG:i:0 MD:Z:23 +1378_67_1795 161 chrY.nib:1-57772954 57401793 37 23M chr16.nib:1-88827254 26739130 0 GATCACCCAGGTGATGTAACTCC ]WV]]]]WW]]]]]]]]]]PU]] XT:A:U CM:i:0 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_68_466 69 * 0 0 * * 0 0 GTGATCGTCGGTGCCAGTCCCTGT #(%)+##$#$#%#+$%##$##### +1378_68_466 133 * 0 0 * * 0 0 GTGTCATCTGAGGTAAAGCATTGT /##$09#$#.=$#$76+$%1'### +1378_68_1692 117 chr13.nib:1-114142980 36365609 0 24M = 36365609 0 TTGAACCGGGCACGGGTCTTCTGG #$#######%###$##%&'%)### +1378_68_1692 153 chr13.nib:1-114142980 36365609 37 10M1D13M = 36365609 0 CTGCACATACAGAATATTCATAG 0010/!"0/!!021/132231// XT:A:U CM:i:2 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:10^T13 +1378_80_664 69 * 0 0 * * 0 0 CTGCTTTGATCCCCGGTGGAGCAC 7#%###$$6#######$##$$$## +1378_80_664 133 * 0 0 * * 0 0 TGTCTGCGTTGTATCTCTGGTGTA %##%,%$$#&$$###$#$%##'%# +1378_85_1786 69 * 0 0 * * 0 0 ATACTATGTCGATCTGTAAAAAAA )&.)#3%(a)$&%-,2#&+.-%0&./ +1378_85_1786 133 * 0 0 * * 0 0 CCCTAGGAGCGTATACCGGACGAG ,'&/%/@,&1,&'/)&,6&&1)(( +1378_86_1011 69 * 0 0 * * 0 0 CTACGTTATTGCTCTGTTTGTCCT ######$%##$$$%###%#$#### +1378_86_1011 133 * 0 0 * * 0 0 AGGCGATGGGATATTATTTTACTT :$###)%##$9$###1$$#$2### +1378_86_1789 89 chr12.nib:1-132349534 39007065 37 23M = 39007065 0 GCTTTCCATAGATGTGTAATTTC J2K]]Z5!GN?@U]]]VX]UYYP XT:A:U CM:i:1 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:0 XG:i:0 MD:Z:23 +1378_86_1789 181 chr12.nib:1-132349534 39007065 0 24M = 39007065 0 ACAACTTAAATAATCATGGACCGG 02,5$$0&6#%?*,$'#%&/15.1 +1378_91_1596 69 * 0 0 * * 0 0 TTAGCGGTTGACTATCTGCTGACA *&+'#9'(%*'#//,&<),/)'*# +1378_91_1596 133 * 0 0 * * 0 0 GCTTTTTCATTCGGTGCCTTTGGA '>%/3%=()8'#.%?50$&5>%)% +1378_94_1595 69 chr7.nib:1-158821424 127518258 0 24M = 127518258 0 CGTGCGACAGCCCATGTTTTCAGA -=..5,3826&*+.+#+#%%6;%# +1378_94_1595 137 chr7.nib:1-158821424 127518258 37 23M = 127518258 0 TGAGATAAACACCTAACATGCTC M]]FN]]\V]]]Q>T]KIG:LVN XT:A:U CM:i:0 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_95_1039 69 * 0 0 * * 0 0 CGGCGTCCATCTTCGCCTTGAGAT $##.#$##$$#%$#$%%$###$)$ +1378_95_1039 133 * 0 0 * * 0 0 GTTCTGTGCCAGGTGAGGTACGGA &##,./#$&)6##+,'#$$0(##$ +1378_95_1767 65 chr11.nib:1-134452384 65333552 25 23M chr3.nib:1-199501827 123725482 0 CAACTGGTGGCATCTGGACAAAC W[[TZYY]]RO<BI7!!:!!>@2 XT:A:U CM:i:2 SM:i:25 AM:i:25 X0:i:1 X1:i:0 XM:i:2 XO:i:0 XG:i:0 MD:Z:23 +1378_95_1767 129 chr3.nib:1-199501827 123725482 37 6M1I16M chr11.nib:1-134452384 65333552 0 ATTTATCTGTCTCATTCATTATT <AGB8B"!V]]UO/&JB4DE88E XT:A:U CM:i:2 SM:i:37 AM:i:25 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_96_1037 69 * 0 0 * * 0 0 ATCCCCCAAGATGCCTGTTGATTG $#$'##$$$#%$$#%###+##$#$ +1378_96_1037 133 * 0 0 * * 0 0 CTGCTGGGCCATTTGACTTACTCA '$#+#(##-%5##+*&###-.$$$ +1378_96_1764 81 chr15.nib:1-100338915 89251272 25 23M chr7.nib:1-158821424 19412615 0 AGAAATGGTCGCACCCTCTGGTT E*2ZEHX\SN]O>SYRL):LIOL XT:A:U CM:i:2 SM:i:25 AM:i:25 X0:i:1 X1:i:0 XM:i:2 XO:i:0 XG:i:0 MD:Z:23 +1378_96_1764 161 chr7.nib:1-158821424 19412615 37 23M chr15.nib:1-100338915 89251272 0 GTATAGCCCACAACGCCTAATAT ZMBS]UW]UYR\]QPZ[SMYL7C XT:A:U CM:i:0 SM:i:37 AM:i:25 X0:i:1 X1:i:0 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_98_1574 69 * 0 0 * * 0 0 GTTCTGCCGGTGTCTGTGGCGGGC $$#+&$$####%$$$###$%#%%# +1378_98_1574 133 * 0 0 * * 0 0 AGGCGAGTGTGGGGGTTGTTTGAG +%%$#)##%##$####%###$%$# +1378_107_1647 69 * 0 0 * * 0 0 AGGCCTACTACGCGTCATTGATAG &#$$#$(.#%#$$####&$%##($ +1378_107_1647 133 * 0 0 * * 0 0 GGTCTGGTTCTATGTTGGTCGACT ###'$$#$$$(#%###(#$##$%# +1378_111_829 69 chr9.nib:1-140273252 82506894 0 24M = 82506894 0 TGCGGCACTTGCTTCTTCGTATTT %#%##%#$%#$#%###$$##&#$$ +1378_111_829 137 chr9.nib:1-140273252 82506894 37 4M1I18M = 82506894 0 GATGCGTAATCTAGTAAAATAAG 0/362//00/5516500210451 XT:A:U CM:i:2 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_111_1900 69 * 0 0 * * 0 0 TCCCCTCGCTCGGCTCTGTGCTGT $&%*$#(#)##$#'##%(##$#$% +1378_111_1900 133 * 0 0 * * 0 0 GCACGCCTTTGGGCTAAGCCGTAA )$)'#%$########$'#&%$#(# +1378_112_1483 69 * 0 0 * * 0 0 TGTCCAGCTATGCGGCTTCCTCCT %#$+#%#&#$#####%####%$## +1378_112_1483 133 * 0 0 * * 0 0 TGGAGTGGTGTGTTTGCTGAGCCA #$#)#############$#%#%'% +1378_125_1287 69 * 0 0 * * 0 0 TGTCTCTGGGGGGCCTGGTTAGGT $##13$'%#$###$$###$$$#&# +1378_125_1287 133 * 0 0 * * 0 0 TGACGTGGGTTGTCCCGTGAGATT ##$%%#$###$##$$#&%##$(%% +1378_126_468 117 chr11.nib:1-134452384 72541052 0 24M = 72541052 0 TGCCTCTATACAGATTAGTCCTCT )7,7..?97594@8=,=?813@>7 +1378_126_468 153 chr11.nib:1-134452384 72541052 0 23M = 72541052 0 AGGCAAGACTCTGTCTCAAAAAA PK5G]]PDT\]SEXY[]]]]]]] XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:4 X1:i:15713 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_127_664 69 * 0 0 * * 0 0 AGAGGTTGGTGTCTTGTCGCAGCT ##'#$######$$%######$$$# +1378_127_664 133 * 0 0 * * 0 0 TCGCTTTGCCTATGTTTGTTCGGA #%$%#&##$%#%%###$$###)-' +1378_129_463 97 chr8.nib:1-146274826 29931771 37 23M chr19.nib:1-63811651 5702213 0 GTAGCTCTGTTTCACATTAGGGG J>AQ[G>C?NM:GD=)*PLORIF XT:A:U CM:i:1 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:0 XG:i:0 MD:Z:23 +1378_129_463 145 chr19.nib:1-63811651 5702213 0 23M chr8.nib:1-146274826 29931771 0 AAAAAAAAAAAAAAAAAAAAAAA JOI:AHGD==@KQB78HF>KA8> XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:583698 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_129_875 69 * 0 0 * * 0 0 TTTCTATGGCTTACGCTGTCTGCC #$($##%####%$#$#####$### +1378_129_875 133 * 0 0 * * 0 0 GACCTTTACGTATTGGGGGTTGGC ###)###+###$##$#&%##$,#$ +1378_140_1251 69 * 0 0 * * 0 0 ATCCTAGCGCGGTGTCTTGGGGAC #$%1#$$$##$##$#$#$##$%$$ +1378_140_1251 133 * 0 0 * * 0 0 TTTCCTTCGTGTGCGTGCGGAGTG #%#%$##$$$######.$$$%#%( +1378_141_809 69 * 0 0 * * 0 0 TGTCCTCCAGTGTCTGTTGGGTGT %&,-##$$#(%###$#$$'###'# +1378_141_809 133 * 0 0 * * 0 0 TCTCGTGGTTTCTTTTTTATGTGT ##%)##$$#####%$#$#%%#'## +1378_144_983 69 * 0 0 * * 0 0 AGCGCCCGGTTGGTGCGGCTCGTC -$(&%*$#*#))#$$$#%%$#$## +1378_144_983 133 * 0 0 * * 0 0 GTTCGTTCGTGGTGTACGAGGGTG #(#%#####($#%##$$#%##%#) +1378_153_270 69 * 0 0 * * 0 0 AGTCCTTGTCCCCTGGGTTTTCCC +''$#&%$%#$##&$$($#&#$$# +1378_153_270 133 * 0 0 * * 0 0 GGCCGTGTGCGGGTGTAGATTGGA %$##($######&##$&$$$$%## +1378_155_1689 65 chrX.nib:1-154913754 106941539 37 23M = 106940385 -1154 ATCTCCTCTTCCTTCCATTCCAC \]]]Y]]]]]UV]]]ZYZZ]]RV XT:A:U CM:i:0 SM:i:37 AM:i:37 X0:i:1 X1:i:0 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_155_1689 129 chrX.nib:1-154913754 106940385 37 23M = 106941539 1154 GACTATGAGGTTTTCATTCAACA ]]]]\\]]]YW]]]WRZ]]WIOK XT:A:U CM:i:0 SM:i:37 AM:i:37 X0:i:1 X1:i:0 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_157_1580 69 * 0 0 * * 0 0 TGGGCCTCGGTGCCCTTGGTCTGT #%)$##'#$$$&#####%#$#$## +1378_157_1580 133 * 0 0 * * 0 0 GGGATTGAAGGGATGTATGCTAGG #%$&%#$$'%$%#$##*#%$$$$# +1378_161_317 69 * 0 0 * * 0 0 TTGGCCGGCAACCCCGGTACCTAA 7<,<'@)@>.)2@/')'&(?/-<( +1378_161_317 133 * 0 0 * * 0 0 AATCCATACCCACAAAAGCAGGCC .&%','(@''?7//+&)+2.+)0) +1378_177_735 113 chr2.nib:1-242951149 222173182 25 23M = 222173882 700 TTGTTCAGCGCCGATTGTCAATC KPNICFMS]]]Z]]]]Y]]]]]] XT:A:U CM:i:2 SM:i:25 AM:i:25 X0:i:1 X1:i:0 XM:i:2 XO:i:0 XG:i:0 MD:Z:1G21 +1378_177_735 177 chr2.nib:1-242951149 222173882 37 23M = 222173182 -700 AGAATTCCTAACAAAATGTGAAG ES6-]]]]]]]]]]]]]]]]]]] XT:A:U CM:i:1 SM:i:37 AM:i:25 X0:i:1 X1:i:0 XM:i:1 XO:i:0 XG:i:0 MD:Z:23 +1378_181_1684 69 * 0 0 * * 0 0 CGACTCCCGCATTCACGGTCAAGT &*#,##$#&$*$$#$#$$$#%$## +1378_181_1684 133 * 0 0 * * 0 0 TTTCTGTTGTGGTTTTGTTGGGGT $##'$%'##%##$%$#$$####$* +1378_187_1407 69 * 0 0 * * 0 0 TGGCGTCCACTCGTGGGTCTATCG $#$'%#$%$%&$%#####$#$#%# +1378_187_1407 133 * 0 0 * * 0 0 TTGGGTGAAATCTTGTCGAGTGGA ####&##$$###$#####%##%%) +1378_203_721 97 chr1.nib:1-247249719 245680524 25 23M chr2.nib:1-242951149 213173999 0 GTAAAATTTGTGGAGATTTAAGT ]VEFFEZ]XPW]TOVINQ,;T!! XT:A:U CM:i:2 SM:i:25 AM:i:25 X0:i:1 X1:i:0 XM:i:2 XO:i:0 XG:i:0 MD:Z:23 +1378_203_721 145 chr2.nib:1-242951149 213173999 37 4M1I18M chr1.nib:1-247249719 245680524 0 ACCTAACAAAATTGTTCAATATG F>8AWT<AV]Q9B"+]O@IF=K] XT:A:U CM:i:2 SM:i:37 AM:i:25 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_206_2039 113 chr4.nib:1-191273063 103793427 0 23M chr18.nib:1-76117153 57165542 0 ACACACACACACACACACACACA NKWZVWZ]]XV[]]]]]]]]]]] XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:1292040 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_206_2039 177 chr18.nib:1-76117153 57165542 0 23M chr4.nib:1-191273063 103793427 0 CACACACACACACACACACACAC NAJ[SPT[]]]W[]]]]]]]]]] XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:1292040 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 diff -r b56291fad13d -r 670f8800a2bf test-data/1.sam --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/1.sam Wed Aug 19 09:13:21 2009 -0400 @@ -0,0 +1,29 @@ +@QNAME FLAG RNAME POS MAPQ CIGAR MRNM MPOS ISIZE SEQ QUAL OPT +1378_11_329 69 * 0 0 * * 0 0 AGACCGGGCGGGGTGGCGTTCGGT %##+'#######%###$#$##$(# +1378_11_329 133 * 0 0 * * 0 0 GTTCGTGGCCGGTGGGTGTTTGGG ###$$#$#$&#####$'$#$###$ +1378_17_1788 69 * 0 0 * * 0 0 TGCCGTGTCTTGCTAACGCCGATT #'#$$#$###%%##$$$$###### +1378_17_1788 133 * 0 0 * * 0 0 TGGGTGGATGTGTTGTCGTTCATG #$#$###$#$#######$#$#### +1378_25_2035 69 * 0 0 * * 0 0 CTGCGTGTTGGTGTCTACTGGGGT #%#'##$#$##&%#%$$$%#%#'# +1378_25_2035 133 * 0 0 * * 0 0 GTGCGTCGGGGAGGGTGCTGTCGG ######%#$%#$$###($###&&% +1378_28_770 89 chr11.nib:1-134452384 72131356 37 17M1I5M = 72131356 0 CACACTGTGACAGACAGCGCAGC 00/02!!0//1200210!!44/1 XT:A:U CM:i:2 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_28_770 181 chr11.nib:1-134452384 72131356 0 24M = 72131356 0 TTGGTGCGCGCGGTTGAGGGTTGG $$(#%%#$%#%####$%%##$### +1378_33_1945 113 chr2.nib:1-242951149 181247988 0 23M chr12.nib:1-132349534 41710908 0 GAGAGAGAGAGAGAGAGAGAGAG PQRVUMNXYRPUXYXWXSOSZ]M XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:163148 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_33_1945 177 chr12.nib:1-132349534 41710908 0 23M chr2.nib:1-242951149 181247988 0 AGAGAGAGAGAGAGAGAGAGAGA SQQWZYURVYWX]]YXTSY]]ZM XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:163148 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_34_789 69 * 0 0 * * 0 0 ATGGTGGCTGACGCGTTTGACTGT #$##%#$##$&$#%##$##$###$ +1378_34_789 133 * 0 0 * * 0 0 GGGCTTGCGTTAGTGAGAGGTTGT ###%$%$%%###$####$###$#& +1378_35_263 115 chr16.nib:1-88827254 19671878 0 23M = 19671877 -1 AGAGAGAGAGAGAGAGAGAGTCT 77543:<55#"4!&=964518A> XT:A:R CM:i:2 SM:i:0 AM:i:0 X0:i:4 X1:i:137 XM:i:2 XO:i:0 XG:i:0 MD:Z:23 +1378_35_263 179 chr16.nib:1-88827254 19671877 0 23M = 19671878 1 GAGAGAGAGAGAGAGAGAGAGTC LE7402DD34FL:27AKE>;432 XT:A:R CM:i:0 SM:i:0 AM:i:0 X0:i:265 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 +1378_43_186 69 * 0 0 * * 0 0 ATACTAGTTGGGACGCGTTGTGCT #$(4%$########$#$###$$$# +1378_43_186 133 * 0 0 * * 0 0 GCTAGGGTTTGGGTTTGCGGTGGG $%#$########%##%#$###'#' +1378_51_1671 117 chr2.nib:1-242951149 190342418 0 24M = 190342418 0 CTGGCGTTCTCGGCGTGGATGGGT #####$$##$#%#%%###%$#$## +1378_51_1671 153 chr2.nib:1-242951149 190342418 37 16M1I6M = 190342418 0 TCTAACTTAGCCTCATAATAGCT /<<!"0///////00/!!0121/ XT:A:U CM:i:2 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_56_324 117 chr2.nib:1-242951149 80324999 0 24M = 80324999 0 TCCAGTCGCGTTGTTAGGTTCGGA #$#$$$#####%##%%###**#+/ +1378_56_324 153 chr2.nib:1-242951149 80324999 37 8M1I14M = 80324999 0 TTTAGCCCGAAATGCCTAGAGCA 4;6//11!"11100110////00 XT:A:U CM:i:2 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:1 XO:i:1 XG:i:1 MD:Z:22 +1378_56_773 69 * 0 0 * * 0 0 TGTCGTGAGGTCACTTATCCCCAT &%#%##%%#####&#$%##$%##$ +1378_56_773 133 * 0 0 * * 0 0 TCTGGTCGGTTTCGGGGAGTGGAA ##%%#&$###$#$##%$####%%$ +1378_62_2027 69 * 0 0 * * 0 0 CTTCCACGATCTGCTCGCTGTGGT (#&&$##$$#$%#%$$$#$###'# +1378_62_2027 133 * 0 0 * * 0 0 GTTGGCCTGGCCTGCCGTGCTGCG *##),/%##$)#%##1$#'%.#&# +1378_62_2029 69 * 0 0 * * 0 0 TCTGGGCTGTCTTCGGGTCGGTGT $%$$####$##$$#)##%%#$### +1378_62_2029 133 * 0 0 * * 0 0 GGCGGTGTGTGGTGCGGCTGTGCG /$$$=(####%####)$$%$-&%# +1378_67_1795 81 chr16.nib:1-88827254 26739130 0 23M chrY.nib:1-57772954 57401793 0 TGGCATTCCTGTAGGCAGAGAGG AZWWZS]!"QNXZ]VQ]]]/2]] XT:A:R CM:i:2 SM:i:0 AM:i:0 X0:i:3 X1:i:0 XM:i:2 XO:i:0 XG:i:0 MD:Z:23 +1378_67_1795 161 chrY.nib:1-57772954 57401793 37 23M chr16.nib:1-88827254 26739130 0 GATCACCCAGGTGATGTAACTCC ]WV]]]]WW]]]]]]]]]]PU]] XT:A:U CM:i:0 SM:i:37 AM:i:0 X0:i:1 X1:i:0 XM:i:0 XO:i:0 XG:i:0 MD:Z:23 \ No newline at end of file diff -r b56291fad13d -r 670f8800a2bf test/functional/test_sniffing_and_metadata_settings.py --- a/test/functional/test_sniffing_and_metadata_settings.py Mon Aug 17 09:54:57 2009 -0400 +++ b/test/functional/test_sniffing_and_metadata_settings.py Wed Aug 19 09:13:21 2009 -0400 @@ -226,6 +226,16 @@ assert latest_hda is not None, "Problem retrieving fastqsanger hda from the database" if not latest_hda.name == '1.fastqsanger' and not latest_hda.extension == 'fastqsanger': raise AssertionError, "fastqsanger data type was not correctly sniffed." + def test_090_sam_datatype( self ): + """Testing correctly sniffing sam format upon upload""" + self.upload_file( '1.sam' ) + self.verify_dataset_correctness( '1.sam' ) + self.check_history_for_string( '1.sam format: <span class="sam">sam</span>, database: \? Info: uploaded sam file' ) + latest_hda = galaxy.model.HistoryDatasetAssociation.query() \ + .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ).first() + assert latest_hda is not None, "Problem retrieving sam hda from the database" + if not latest_hda.name == '1.sam' and not latest_hda.extension == 'sam': + raise AssertionError, "sam data type was not correctly sniffed." def test_9999_clean_up( self ): self.delete_history( id=self.security.encode_id( history1.id ) ) self.logout()

1 0

Database Connection
by SHAUN WEBB 21 Aug '09

21 Aug '09

Hi, I'm trying to switch from sqlite to mySQL and need a little assistance with what goes in the database connection field in universe_wsgi.ini. For instance, if I set up the following mySQL database: database = galaxy_test user = galaxy password = password what would I use to connect to this. I have tried using mysql:///galaxy_test?unix_socket=/var/run/mysqld/mysqld.sock but I cannot connect as it looks like a username and password are expected. Thanks for any help Shaun Webb -- The University of Edinburgh is a charitable body, registered in Scotland, with registration number SC005336.

2 2

Re: [galaxy-dev] Database locked errors
by Nate Coraor 20 Aug '09

20 Aug '09

SHAUN WEBB wrote: > Yes we are using SQLite. I will look in to switching to MySQL. I know a > few people have been discussing this lately, is there a wiki page > devoted to this? There are a few pages that contain most of the information, but I'm trying to consolidate all of this right now. > I found a few errors in the log from todays activity. I've attached a > few examples in a text file. The final error relates to the job that I > stopped in the admin section. As described, some of these "queued" > datasets are not associated with job ids. > > I am concerned that if I transfer all the information as is to a mySQL > database that I will still have the same problem. The locking issues will go away with a real database server, and thus these uncaught exceptions will go away. The ultimate result is that Galaxy won't lose track of jobs because of this. --nate

1 0

Database locked errors
by SHAUN WEBB 20 Aug '09

20 Aug '09

Hi, I'm having the following problems with our Galaxy server: This morning one of our users complained that Galaxy was crashing a lot, either: -The main page was coming up as an operational error page: Database Locked -The output history item returned an error "database is locked" -The output history item would appear to be waiting to run (grey box) but never complete This error is intermittent. I took a look at the manage jobs section on the admin page and found that all the jobs had a "new" status except one which had not been updated for over an hour and had a status of queued. I stopped this job from the admin page and after this the remaining jobs completed. However this doesn't seem to have solved the problem. I've taken a look at the database and as of today only, there are 11 datasets in the dataset table with a "queued" status. Some of these are associated with histories (and marked as deleted in hda table) and jobs (with an error status). The remaining 6 are associated with a history but not marked as deleted and there is no entry in the job_to_output_dataset table to associate these datasets with a job. I wonder if these datasets are causing the problem and galaxy is continually trying to resolve their queued status resulting in the "locked database" error. I'd be grateful if anyone can help with this or even shed some light on how the problem might have occurred and the processes galaxy uses to initiate jobs. Thanks Shaun Webb -- The University of Edinburgh is a charitable body, registered in Scotland, with registration number SC005336.

2 1

SGE Cluster suggestion
by Assaf Gordon 17 Aug '09

17 Aug '09

Hello, I'm trying to move our Galaxy to the our SGE cluster, and I need couple of fine-tuning tweaks. 1. set a project name (in addition to the queue) - globally, and per tool. 2. set extra QSUB options per tool 3. Specify the actual user who submitted the job - in the "email" field and as the job name. Details: The project name is needed for accounting (requested by our SGE admins). To specify a project name, add a fourth slash to the sge URL in universe_wsgi.ini, as so: --- ## Global SGE project name , for all tools default_cluster_job_runner = sge:////GalaxyProject/ ## Tool specific project name Cut1 = sge:////OtherGalaxyProject/ ---- The extra QSUB options per tool allow requesting nodes with more memory or request more cores (=threads) per job. To specify extra options, add a fifth slash to the sge tool runner URL in univer_wsge.ini, as so: ---- ## Tool specific parameters - ## run 'cut' on a node with at least 7GB of ram Cut1 = sge:////GalaxyProject/-l virtual_free=7G/ ## run 'select first lines' on a node with 4 cores, and reserve all the cores for the tool ## allowing multi-threaded 'select lines' operation ;) Show beginning1 = sge:////GalaxyProject/-R y -pe threads 4/ ---- Note: specifying extra parameters ALWAYS overrides the project name (but the project name can be an empty string). The actual user is needed to quickly see (with qstat) who runs what (currently, the job names are "galaxy_NNNN.sh"). Comments are welcomed, -gordon.

2 4

Re: [galaxy-dev] galaxy-dev Digest, Vol 44, Issue 22
by Ross 15 Aug '09

15 Aug '09

On Fri, Aug 14, 2009 at 1:14 PM, <galaxy-dev-request(a)bx.psu.edu> wrote: This is excellent - it seems very natural - another extremely compelling feature. Congratulations everyone! > Message: 4 > Date: Fri, 14 Aug 2009 13:14:02 -0400 > From: Greg Von Kuster <greg(a)bx.psu.edu> > Subject: [galaxy-dev] [hg] galaxy 2558: Integrate 'rerun' support from > Assaf Gordan. N... > To: galaxy-dev(a)bx.psu.edu > Message-ID: > <hg.52ad5635f01a.1250270042.-331200333(a)scofield.bx.psu.edu> > > details: http://www.bx.psu.edu/hg/galaxy/rev/52ad5635f01a > changeset: 2558:52ad5635f01a > user: James Taylor <james(a)jamestaylor.org> > date: Thu Aug 13 11:27:45 2009 -0400 > description: > Integrate 'rerun' support from Assaf Gordan. Not supported for all tools, but checking whether the link should be displayed is expensive currently, this can be improved ghopefully. > > 2 file(s) affected in this change: > > lib/galaxy/web/controllers/tool_runner.py > templates/root/history_common.mako > > diffs (118 lines): > > diff -r ad5232a4dbc5 -r 52ad5635f01a lib/galaxy/web/controllers/tool_runner.py > --- a/lib/galaxy/web/controllers/tool_runner.py Thu Aug 13 11:00:00 2009 -0400 > +++ b/lib/galaxy/web/controllers/tool_runner.py Thu Aug 13 11:27:45 2009 -0400 > @@ -5,6 +5,7 @@ > from galaxy.web.base.controller import * > from galaxy.util.bunch import Bunch > from galaxy.tools import DefaultToolState > +from galaxy.tools.parameters.basic import UnvalidatedValue > > import logging > log = logging.getLogger( __name__ ) > @@ -52,6 +53,80 @@ > add_frame.wiki_url = trans.app.config.wiki_url > add_frame.from_noframe = True > return trans.fill_template( template, history=history, toolbox=toolbox, tool=tool, util=util, add_frame=add_frame, **vars ) > + > + @web.expose > + def rerun( self, trans, id=None, from_noframe=None, **kwd ): > + """ > + Given a HistoryDatasetAssociation id, find the job and that created > + the dataset, extract the parameters, and display the appropriate tool > + form with parameters already filled in. > + """ > + if not id: > + error( "'id' parameter is required" ); > + try: > + id = int( id ) > + except: > + error( "Invalid value for 'id' parameter" ) > + # Get the dataset object > + data = trans.app.model.HistoryDatasetAssociation.get( id ) > + # Get the associated job, if any. If this hda was copied from another, > + # we need to find the job that created the origial hda > + job_hda = data > + while job_hda.copied_from_history_dataset_association: > + job_hda = job_hda.copied_from_history_dataset_association > + if not job_hda.creating_job_associations: > + error( "Could not find the job for this dataset" ) > + # Get the job object > + job = None > + for assoc in job_hda.creating_job_associations: > + job = assoc.job > + break > + if not job: > + raise Exception("Failed to get job information for dataset hid %d" % hid) > + # Get the tool object > + tool_id = job.tool_id > + try: > + # Load the tool > + toolbox = self.get_toolbox() > + tool = toolbox.tools_by_id.get( tool_id, None ) > + except: > + #this is expected, so not an exception > + error( "This dataset was created by an obsolete tool (%s). Can't re-run." % tool_id ) > + # Can't rerun upload, external data sources, et cetera. Workflow > + # compatible will proxy this for now > + if not tool.is_workflow_compatible: > + error( "The '%s' tool does not currently support rerunning." % tool.name ) > + # Get the job's parameters > + try: > + params_objects = job.get_param_values( trans.app ) > + except: > + raise Exception( "Failed to get paramemeters for dataset id %d " % hid ) > + # Unpack unvalidated values to strings, they'll be validated when the > + # form is submitted (this happens when re-running a job that was > + # initially run by a workflow) > + validated_params = {} > + for name, value in params_objects.items(): > + if isinstance( value, UnvalidatedValue ): > + validated_params [ str(name) ] = str(value) > + else: > + validated_params [ str(name) ] = value > + params_objects = validated_params > + # Create a fake tool_state for the tool, with the parameters values > + state = tool.new_state( trans ) > + state.inputs = params_objects > + tool_state_string = util.object_to_string(state.encode(tool, trans.app)) > + # Setup context for template > + history = trans.get_history() > + template = "tool_form.mako" > + vars = dict( tool_state=state, errors = {} ) > + # Is the "add frame" stuff neccesary here? > + add_frame = AddFrameData() > + add_frame.debug = trans.debug > + if from_noframe is not None: > + add_frame.wiki_url = trans.app.config.wiki_url > + add_frame.from_noframe = True > + return trans.fill_template( template, history=history, toolbox=toolbox, tool=tool, util=util, add_frame=add_frame, **vars ) > + > > @web.expose > def redirect( self, trans, redirect_url=None, **kwd ): > diff -r ad5232a4dbc5 -r 52ad5635f01a templates/root/history_common.mako > --- a/templates/root/history_common.mako Thu Aug 13 11:00:00 2009 -0400 > +++ b/templates/root/history_common.mako Thu Aug 13 11:27:45 2009 -0400 > @@ -51,9 +51,12 @@ > <div>${_('Job is currently running')}</div> > %elif data_state == "error": > <div> > - An error occurred running this job: <i>${data.display_info().strip()}</i>, > - <a href="${h.url_for( controller='dataset', action='errors', id=data.id )}" target="galaxy_main">report this error</a> > + An error occurred running this job: <i>${data.display_info().strip()}</i> > </div> > + <div> > + <a href="${h.url_for( controller='dataset', action='errors', id=data.id )}" target="galaxy_main">report this error</a> > + | <a href="${h.url_for( controller='tool_runner', action='rerun', id=data.id )}" target="galaxy_main">rerun</a> > + </div> > %elif data_state == "discarded": > <div> > The job creating this dataset was cancelled before completion. > @@ -77,6 +80,7 @@ > <div> > %if data.has_data: > <a href="${h.url_for( action='display', id=data.id, tofile='yes', toext=data.ext )}" target="_blank">save</a> > + | <a href="${h.url_for( controller='tool_runner', action='rerun', id=data.id )}" target="galaxy_main">rerun</a> > %for display_app in data.datatype.get_display_types(): > <% display_links = data.datatype.get_display_links( data, display_app, app, request.base ) %> > %if len( display_links ) > 0: > > > ------------------------------ > > _______________________________________________ > galaxy-dev mailing list > galaxy-dev(a)bx.psu.edu > http://mail.bx.psu.edu/cgi-bin/mailman/listinfo/galaxy-dev > > > End of galaxy-dev Digest, Vol 44, Issue 22 > ****************************************** > -- Ross Lazarus, Associate Professor, Department of Population Medicine, Harvard Medical School; Director of Bioinformatics, Channing Laboratory, BWH 181 Longwood Ave., Boston MA 02115, USA.

1 0

Possible Bug in SGE Runner when recovering jobs
by Assaf Gordon 15 Aug '09

15 Aug '09

Hi, Not sure if this is a bug or my configuration problem, but: 1. universe_wsgi.ini has: track_jobs_in_database = True enable_job_recovery = True 2. A tool (which uses the SGE runner) is in 'queued' state: psql# select id,tool_id,state from job where state = 'queued'; id | tool_id | state ----+-----------------+-------- 39 | Show beginning1 | queued (1 row) 3. Starting galaxy with "sh run.sh" gives: Traceback (most recent call last): File "/bh/data/hannon/gordon/galaxy/galaxy_bh_dev/lib/galaxy/web/buildapp.py", line 61, in app_factory app = UniverseApplication( global_conf = global_conf, **kwargs ) File "/bh/data/hannon/gordon/galaxy/galaxy_bh_dev/lib/galaxy/app.py", line 64, in __init__ self.job_manager = jobs.JobManager( self ) File "/bh/data/hannon/gordon/galaxy/galaxy_bh_dev/lib/galaxy/jobs/__init__.py", line 35, in __init__ self.job_queue = JobQueue( app, self.dispatcher ) File "/bh/data/hannon/gordon/galaxy/galaxy_bh_dev/lib/galaxy/jobs/__init__.py", line 112, in __init__ self.__check_jobs_at_startup() File "/bh/data/hannon/gordon/galaxy/galaxy_bh_dev/lib/galaxy/jobs/__init__.py", line 137, in __check_jobs_at_startup self.dispatcher.recover( job, job_wrapper ) File "/bh/data/hannon/gordon/galaxy/galaxy_bh_dev/lib/galaxy/jobs/__init__.py", line 661, in recover self.job_runners[runner_name].recover( job, job_wrapper ) File "/data/hannon/gordon/galaxy/galaxy_bh_dev/lib/galaxy/jobs/runners/sge.py", line 358, in recover sge_job_state.old_state = DRMAA.Session.QUEUED AttributeError: type object 'Session' has no attribute 'QUEUED' ------------- The immediate cause is that the DRMAA Session class does not have a "QUEUED" state. The closest thing is "QUEUED_ACTIVE" state, as evident from both sge.py lines 18-28, and from DRMAA.py line 363. On a related note, I'm not quite sure how job recovery works with SGE: Let's say a job is queued with QSUB, then galaxy is stopped. The database still says "queued", but in the meantime, the SGE cluster might run the job, and even complete it - So the job can in fact be 'running' or 'ok' or 'error', but Galaxy thinks it is queued. What will happen when Galaxy is restarted ? Is the job automatically restarted, or is the new state queried from the SGE ? Thanks, Gordon.

1 0

← Newer
1
...
7
8
9
10
11
12
13
...
16
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

galaxy-dev August 2009