- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: dan: Fixes for dictify to to_dict change.
by commits-noreply＠bitbucket.org 31 Aug '13

31 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0470feeb593f/ Changeset: 0470feeb593f User: dan Date: 2013-08-31 00:06:52 Summary: Fixes for dictify to to_dict change. Affected #: 8 files diff -r c59cc42b279385f0d351d31f919d29bb7c012877 -r 0470feeb593f9797fa9fd19dfdac4751a7ca788b config/plugins/visualizations/scatterplot/templates/scatterplot.mako --- a/config/plugins/visualizations/scatterplot/templates/scatterplot.mako +++ b/config/plugins/visualizations/scatterplot/templates/scatterplot.mako @@ -41,7 +41,7 @@ <script type="text/javascript"> $(function(){ - var hda = ${h.to_json_string( trans.security.encode_dict_ids( hda.dictify() ) )}, + var hda = ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )}, querySettings = ${h.to_json_string( query_args )}, chartConfig = _.extend( querySettings, { containerSelector : '#chart', diff -r c59cc42b279385f0d351d31f919d29bb7c012877 -r 0470feeb593f9797fa9fd19dfdac4751a7ca788b templates/webapps/galaxy/base_panels.mako --- a/templates/webapps/galaxy/base_panels.mako +++ b/templates/webapps/galaxy/base_panels.mako @@ -16,7 +16,7 @@ """Bootstrapping user API JSON""" #TODO: move into common location (poss. BaseController) if trans.user: - user_dict = trans.user.dictify( view='element', value_mapper={ 'id': trans.security.encode_id, + user_dict = trans.user.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'total_disk_usage': float } ) user_dict['quota_percent'] = trans.app.quota_agent.get_percent( trans=trans ) else: diff -r c59cc42b279385f0d351d31f919d29bb7c012877 -r 0470feeb593f9797fa9fd19dfdac4751a7ca788b templates/webapps/galaxy/dataset/display.mako --- a/templates/webapps/galaxy/dataset/display.mako +++ b/templates/webapps/galaxy/dataset/display.mako @@ -20,7 +20,7 @@ require(['mvc/data'], function(data) { data.createTabularDatasetChunkedView( // Dataset config. TODO: encode id. - _.extend( ${h.to_json_string( item.dictify() )}, + _.extend( ${h.to_json_string( item.to_dict() )}, { chunk_url: "${h.url_for( controller='/dataset', action='display', dataset_id=trans.security.encode_id( item.id ))}", diff -r c59cc42b279385f0d351d31f919d29bb7c012877 -r 0470feeb593f9797fa9fd19dfdac4751a7ca788b templates/webapps/galaxy/dataset/tabular_chunked.mako --- a/templates/webapps/galaxy/dataset/tabular_chunked.mako +++ b/templates/webapps/galaxy/dataset/tabular_chunked.mako @@ -19,7 +19,7 @@ require(['mvc/data'], function(data) { data.createTabularDatasetChunkedView( - _.extend( ${h.to_json_string( trans.security.encode_dict_ids( dataset.dictify() ) )}, + _.extend( ${h.to_json_string( trans.security.encode_dict_ids( dataset.to_dict() ) )}, { url_viz: "${h.url_for( controller='/visualization')}", chunk_url: "${h.url_for( controller='/dataset', action='display', diff -r c59cc42b279385f0d351d31f919d29bb7c012877 -r 0470feeb593f9797fa9fd19dfdac4751a7ca788b templates/webapps/galaxy/galaxy.masthead.mako --- a/templates/webapps/galaxy/galaxy.masthead.mako +++ b/templates/webapps/galaxy/galaxy.masthead.mako @@ -4,7 +4,7 @@ """Bootstrapping user API JSON""" #TODO: move into common location (poss. BaseController) if trans.user: - user_dict = trans.user.dictify( view='element', value_mapper={ 'id': trans.security.encode_id, + user_dict = trans.user.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'total_disk_usage': float } ) user_dict['quota_percent'] = trans.app.quota_agent.get_percent( trans=trans ) else: diff -r c59cc42b279385f0d351d31f919d29bb7c012877 -r 0470feeb593f9797fa9fd19dfdac4751a7ca788b templates/webapps/galaxy/root/tool_menu.mako --- a/templates/webapps/galaxy/root/tool_menu.mako +++ b/templates/webapps/galaxy/root/tool_menu.mako @@ -25,12 +25,12 @@ hidden: false }), tools = new tools_mod.ToolCollection( - ${ h.to_json_string( trans.app.toolbox.dictify( trans, in_panel=False ) ) } + ${ h.to_json_string( trans.app.toolbox.to_dict( trans, in_panel=False ) ) } ), tool_panel = new tools_mod.ToolPanel({ tool_search: tool_search, tools: tools, - layout: ${h.to_json_string( trans.app.toolbox.dictify( trans ) )} + layout: ${h.to_json_string( trans.app.toolbox.to_dict( trans ) )} }), tool_panel_view = new tools_mod.ToolPanelView({ model: tool_panel }); diff -r c59cc42b279385f0d351d31f919d29bb7c012877 -r 0470feeb593f9797fa9fd19dfdac4751a7ca788b templates/webapps/galaxy/visualization/scatterplot.mako --- a/templates/webapps/galaxy/visualization/scatterplot.mako +++ b/templates/webapps/galaxy/visualization/scatterplot.mako @@ -222,7 +222,7 @@ <script type="text/javascript"> $(function(){ - var hda = ${h.to_json_string( trans.security.encode_dict_ids( hda.dictify() ) )}, + var hda = ${h.to_json_string( trans.security.encode_dict_ids( hda.to_dict() ) )}, querySettings = ${h.to_json_string( query_args )}, chartConfig = _.extend( querySettings, { containerSelector : '#chart', diff -r c59cc42b279385f0d351d31f919d29bb7c012877 -r 0470feeb593f9797fa9fd19dfdac4751a7ca788b templates/webapps/galaxy/visualization/v_fwork_test.mako --- a/templates/webapps/galaxy/visualization/v_fwork_test.mako +++ b/templates/webapps/galaxy/visualization/v_fwork_test.mako @@ -19,7 +19,7 @@ <%def name="process_hda( hda )"><% - hda_dict = hda.dictify() + hda_dict = hda.to_dict() hda_dict[ 'id' ] = trans.security.encode_id( hda_dict[ 'id' ] ) hda_dict[ 'history_id' ] = trans.security.encode_id( hda_dict[ 'history_id' ] ) del hda_dict[ 'peek' ] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ce4c7a3fb0d8/ Changeset: ce4c7a3fb0d8 User: jgoecks Date: 2013-08-30 20:09:18 Summary: Use super() to call dictify() rather than calling directly. Affected #: 1 file diff -r 9345e15ec4b7f81271a071f0703e89fff0c5b3ac -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2342,7 +2342,7 @@ self.tags.append(new_swta) def dictify( self, view='collection', value_mapper = None ): - rval = DictifiableMixin.dictify(self, view=view, value_mapper = value_mapper) + rval = super( StoredWorkflow, self ).dictify(self, view=view, value_mapper = value_mapper) tags_str_list = [] for tag in self.tags: tag_str = tag.user_tname @@ -3831,7 +3831,7 @@ return [ tool_version.tool_id for tool_version in self.get_versions( app ) ] def dictify( self, view='element' ): - rval = DictifiableMixin.dictify(self, view) + rval = super( ToolVersion, self ).dictify( self, view ) rval['tool_name'] = self.tool_id for a in self.parent_tool_association: rval['parent_tool_id'] = a.parent_id https://bitbucket.org/galaxy/galaxy-central/commits/a4259a97c287/ Changeset: a4259a97c287 User: jgoecks Date: 2013-08-30 20:17:23 Summary: DictifiableMixin cleanup: (a) remove 'Mixin' from name and (b) rename dictify to to_dict. Affected #: 32 files diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -23,7 +23,7 @@ import galaxy.datatypes.registry import galaxy.security.passwords from galaxy.datatypes.metadata import MetadataCollection -from galaxy.model.item_attrs import DictifiableMixin, UsesAnnotations +from galaxy.model.item_attrs import Dictifiable, UsesAnnotations from galaxy.security import get_permitted_actions from galaxy.util import is_multi_byte, nice_size, Params, restore_text, send_mail from galaxy.util.bunch import Bunch @@ -61,15 +61,15 @@ datatypes_registry = d_registry -class User( object, DictifiableMixin ): +class User( object, Dictifiable ): use_pbkdf2 = True """ Data for a Galaxy user or admin and relations to their histories, credentials, and roles. """ - # attributes that will be accessed and returned when calling dictify( view='collection' ) + # attributes that will be accessed and returned when calling to_dict( view='collection' ) dict_collection_visible_keys = ( 'id', 'email' ) - # attributes that will be accessed and returned when calling dictify( view='element' ) + # attributes that will be accessed and returned when calling to_dict( view='element' ) dict_element_visible_keys = ( 'id', 'email', 'username', 'total_disk_usage', 'nice_total_disk_usage' ) def __init__( self, email=None, password=None ): @@ -157,7 +157,7 @@ return total -class Job( object, DictifiableMixin ): +class Job( object, Dictifiable ): dict_collection_visible_keys = [ 'id' ] dict_element_visible_keys = [ 'id' ] @@ -363,8 +363,8 @@ dataset.blurb = 'deleted' dataset.peek = 'Job deleted' dataset.info = 'Job output deleted by user before job completed' - def dictify( self, view='collection' ): - rval = super( Job, self ).dictify( view=view ) + def to_dict( self, view='collection' ): + rval = super( Job, self ).to_dict( view=view ) rval['tool_name'] = self.tool_id param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) rval['params'] = param_dict @@ -649,7 +649,7 @@ else: return False -class Group( object, DictifiableMixin ): +class Group( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name' ) dict_element_visible_keys = ( 'id', 'name' ) @@ -662,7 +662,7 @@ self.user = user self.group = group -class History( object, DictifiableMixin, UsesAnnotations ): +class History( object, Dictifiable, UsesAnnotations ): dict_collection_visible_keys = ( 'id', 'name', 'published', 'deleted' ) dict_element_visible_keys = ( 'id', 'name', 'published', 'deleted', 'genome_build', 'purged' ) @@ -780,10 +780,10 @@ history_name = unicode(history_name, 'utf-8') return history_name - def dictify( self, view='collection', value_mapper = None ): + def to_dict( self, view='collection', value_mapper = None ): # Get basic value. - rval = super( History, self ).dictify( view=view, value_mapper=value_mapper ) + rval = super( History, self ).to_dict( view=view, value_mapper=value_mapper ) # Add tags. tags_str_list = [] @@ -869,7 +869,7 @@ self.group = group self.role = role -class Role( object, DictifiableMixin ): +class Role( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name' ) dict_element_visible_keys = ( 'id', 'name', 'description', 'type' ) private_id = None @@ -886,19 +886,19 @@ self.type = type self.deleted = deleted -class UserQuotaAssociation( object, DictifiableMixin ): +class UserQuotaAssociation( object, Dictifiable ): dict_element_visible_keys = ( 'user', ) def __init__( self, user, quota ): self.user = user self.quota = quota -class GroupQuotaAssociation( object, DictifiableMixin ): +class GroupQuotaAssociation( object, Dictifiable ): dict_element_visible_keys = ( 'group', ) def __init__( self, group, quota ): self.group = group self.quota = quota -class Quota( object, DictifiableMixin ): +class Quota( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name' ) dict_element_visible_keys = ( 'id', 'name', 'description', 'bytes', 'operation', 'display_amount', 'default', 'users', 'groups' ) valid_operations = ( '+', '-', '=' ) @@ -927,7 +927,7 @@ else: return nice_size( self.bytes ) -class DefaultQuotaAssociation( Quota, DictifiableMixin ): +class DefaultQuotaAssociation( Quota, Dictifiable ): dict_element_visible_keys = ( 'type', ) types = Bunch( UNREGISTERED = 'unregistered', @@ -1508,7 +1508,7 @@ return msg -class HistoryDatasetAssociation( DatasetInstance, DictifiableMixin, UsesAnnotations ): +class HistoryDatasetAssociation( DatasetInstance, Dictifiable, UsesAnnotations ): """ Resource class that creates a relation between a dataset and a user history. """ @@ -1680,7 +1680,7 @@ rval += child.get_disk_usage( user ) return rval - def dictify( self, view='collection' ): + def to_dict( self, view='collection' ): """ Return attributes of this HDA that are exposed using the API. """ @@ -1759,7 +1759,7 @@ self.subset = subset self.location = location -class Library( object, DictifiableMixin ): +class Library( object, Dictifiable ): permitted_actions = get_permitted_actions( filter='LIBRARY' ) dict_collection_visible_keys = ( 'id', 'name' ) dict_element_visible_keys = ( 'id', 'deleted', 'name', 'description', 'synopsis' ) @@ -1828,7 +1828,7 @@ name = unicode( name, 'utf-8' ) return name -class LibraryFolder( object, DictifiableMixin ): +class LibraryFolder( object, Dictifiable ): dict_element_visible_keys = ( 'id', 'parent_id', 'name', 'description', 'item_count', 'genome_build' ) def __init__( self, name=None, description=None, item_count=0, order_id=None ): self.name = name or "Unnamed folder" @@ -1900,8 +1900,8 @@ if isinstance( name, str ): name = unicode( name, 'utf-8' ) return name - def dictify( self, view='collection' ): - rval = super( LibraryFolder, self ).dictify( view=view ) + def to_dict( self, view='collection' ): + rval = super( LibraryFolder, self ).to_dict( view=view ) info_association, inherited = self.get_info_association() if info_association: if inherited: @@ -1966,7 +1966,7 @@ name = property( get_name, set_name ) def display_name( self ): self.library_dataset_dataset_association.display_name() - def dictify( self, view='collection' ): + def to_dict( self, view='collection' ): # Since this class is a proxy to rather complex attributes we want to # display in other objects, we can't use the simpler method used by # other model classes. @@ -2096,7 +2096,7 @@ if restrict: return None, inherited return self.library_dataset.folder.get_info_association( inherited=True ) - def dictify( self, view='collection' ): + def to_dict( self, view='collection' ): # Since this class is a proxy to rather complex attributes we want to # display in other objects, we can't use the simpler method used by # other model classes. @@ -2323,7 +2323,7 @@ self.id = None self.user = None -class StoredWorkflow( object, DictifiableMixin): +class StoredWorkflow( object, Dictifiable): dict_collection_visible_keys = ( 'id', 'name', 'published' ) dict_element_visible_keys = ( 'id', 'name', 'published' ) def __init__( self ): @@ -2341,8 +2341,8 @@ new_swta.user = target_user self.tags.append(new_swta) - def dictify( self, view='collection', value_mapper = None ): - rval = super( StoredWorkflow, self ).dictify(self, view=view, value_mapper = value_mapper) + def to_dict( self, view='collection', value_mapper = None ): + rval = super( StoredWorkflow, self ).to_dict(self, view=view, value_mapper = value_mapper) tags_str_list = [] for tag in self.tags: tag_str = tag.user_tname @@ -2434,7 +2434,7 @@ return os.path.abspath( os.path.join( path, "metadata_%d.dat" % self.id ) ) -class FormDefinition( object, DictifiableMixin ): +class FormDefinition( object, Dictifiable ): # The following form_builder classes are supported by the FormDefinition class. supported_field_types = [ AddressField, CheckboxField, PasswordField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField, HistoryField ] types = Bunch( REQUEST = 'Sequencing Request Form', @@ -2562,7 +2562,7 @@ self.form_definition = form_def self.content = content -class Request( object, DictifiableMixin ): +class Request( object, Dictifiable ): states = Bunch( NEW = 'New', SUBMITTED = 'In Progress', REJECTED = 'Rejected', @@ -2753,7 +2753,7 @@ def populate_actions( self, trans, item, param_dict=None ): return self.get_external_service_type( trans ).actions.populate( self, item, param_dict=param_dict ) -class RequestType( object, DictifiableMixin ): +class RequestType( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name', 'desc' ) dict_element_visible_keys = ( 'id', 'name', 'desc', 'request_form_id', 'sample_form_id' ) rename_dataset_options = Bunch( NO = 'Do not rename', @@ -2839,7 +2839,7 @@ self.request_type = request_type self.role = role -class Sample( object, DictifiableMixin ): +class Sample( object, Dictifiable ): # The following form_builder classes are supported by the Sample class. supported_field_types = [ CheckboxField, SelectField, TextField, WorkflowField, WorkflowMappingField, HistoryField ] bulk_operations = Bunch( CHANGE_STATE = 'Change state', @@ -3169,7 +3169,7 @@ def __str__ ( self ): return "Tag(id=%s, type=%i, parent_id=%s, name=%s)" % ( self.id, self.type, self.parent_id, self.name ) -class ItemTagAssociation ( object, DictifiableMixin ): +class ItemTagAssociation ( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'user_tname', 'user_value' ) dict_element_visible_keys = dict_collection_visible_keys @@ -3350,7 +3350,7 @@ self.error_message = error_message def as_dict( self, value_mapper=None ): - return self.dictify( view='element', value_mapper=value_mapper ) + return self.to_dict( view='element', value_mapper=value_mapper ) @property def can_install( self ): @@ -3372,7 +3372,7 @@ def can_reinstall_or_activate( self ): return self.deleted - def dictify( self, view='collection', value_mapper=None ): + def to_dict( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} rval = {} @@ -3772,7 +3772,7 @@ self.tool_shed_repository.name, self.tool_shed_repository.installed_changeset_revision ) -class ToolVersion( object, DictifiableMixin ): +class ToolVersion( object, Dictifiable ): dict_element_visible_keys = ( 'id', 'tool_shed_repository' ) def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repository=None ): self.id = id @@ -3830,8 +3830,8 @@ return version_ids return [ tool_version.tool_id for tool_version in self.get_versions( app ) ] - def dictify( self, view='element' ): - rval = super( ToolVersion, self ).dictify( self, view ) + def to_dict( self, view='element' ): + rval = super( ToolVersion, self ).to_dict( self, view ) rval['tool_name'] = self.tool_id for a in self.parent_tool_association: rval['parent_tool_id'] = a.parent_id diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/model/item_attrs.py --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -158,12 +158,12 @@ class_name = '%sAnnotationAssociation' % item.__class__.__name__ return getattr( galaxy.model, class_name, None ) -class DictifiableMixin: +class Dictifiable: """ Mixin that enables objects to be converted to dictionaries. This is useful when for sharing objects across boundaries, such as the API, tool scripts, and JavaScript code. """ - def dictify( self, view='collection', value_mapper=None ): + def to_dict( self, view='collection', value_mapper=None ): """ Return item dictionary. """ @@ -176,9 +176,9 @@ Recursive helper function to get item values. """ # FIXME: why use exception here? Why not look for key in value_mapper - # first and then default to dictify? + # first and then default to to_dict? try: - return item.dictify( view=view, value_mapper=value_mapper ) + return item.to_dict( view=view, value_mapper=value_mapper ) except: if key in value_mapper: return value_mapper.get( key )( item ) @@ -193,7 +193,7 @@ try: visible_keys = self.__getattribute__( 'dict_' + view + '_visible_keys' ) except AttributeError: - raise Exception( 'Unknown DictifiableMixin view: %s' % view ) + raise Exception( 'Unknown Dictifiable view: %s' % view ) for key in visible_keys: try: item = self.__getattribute__( key ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/model/search.py --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -560,7 +560,7 @@ return self.view.get_results(True) def item_to_api_value(self, item): - r = item.dictify( view='element' ) + r = item.to_dict( view='element' ) if self.query.field_list.count("*"): return r o = {} diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -59,7 +59,7 @@ from galaxy.util.template import fill_template from galaxy.web import url_for from galaxy.web.form_builder import SelectField -from galaxy.model.item_attrs import DictifiableMixin +from galaxy.model.item_attrs import Dictifiable from tool_shed.util import shed_util_common from .loader import load_tool, template_macro_params @@ -93,16 +93,16 @@ class ToolNotFoundException( Exception ): pass -def dictify_helper( obj, kwargs ): - """ Helper function that provides the appropriate kwargs to dictify an object. """ +def to_dict_helper( obj, kwargs ): + """ Helper function that provides the appropriate kwargs to to_dict an object. """ - # Label.dictify cannot have kwargs. + # Label.to_dict cannot have kwargs. if isinstance( obj, ToolSectionLabel ): kwargs = {} - return obj.dictify( **kwargs ) + return obj.to_dict( **kwargs ) -class ToolBox( object, DictifiableMixin ): +class ToolBox( object, Dictifiable ): """Container for a collection of tools""" def __init__( self, config_filenames, tool_root_dir, app ): @@ -711,9 +711,9 @@ """ return self.app.model.context - def dictify( self, trans, in_panel=True, **kwds ): + def to_dict( self, trans, in_panel=True, **kwds ): """ - Dictify toolbox. + to_dict toolbox. """ context = Bunch( toolbox=self, trans=trans, **kwds ) @@ -736,11 +736,11 @@ link_details = True ) for elt in panel_elts: - rval.append( dictify_helper( elt, kwargs ) ) + rval.append( to_dict_helper( elt, kwargs ) ) else: tools = [] for id, tool in self.tools_by_id.items(): - tools.append( tool.dictify( trans, link_details=True ) ) + tools.append( tool.to_dict( trans, link_details=True ) ) rval = tools return rval @@ -801,7 +801,7 @@ -class ToolSection( object, DictifiableMixin ): +class ToolSection( object, Dictifiable ): """ A group of tools with similar type/purpose that will be displayed as a group in the user interface. @@ -824,22 +824,22 @@ copy.elems = self.elems.copy() return copy - def dictify( self, trans, link_details=False ): + def to_dict( self, trans, link_details=False ): """ Return a dict that includes section's attributes. """ - section_dict = super( ToolSection, self ).dictify() + section_dict = super( ToolSection, self ).to_dict() section_elts = [] kwargs = dict( trans = trans, link_details = link_details ) for elt in self.elems.values(): - section_elts.append( dictify_helper( elt, kwargs ) ) + section_elts.append( to_dict_helper( elt, kwargs ) ) section_dict[ 'elems' ] = section_elts return section_dict -class ToolSectionLabel( object, DictifiableMixin ): +class ToolSectionLabel( object, Dictifiable ): """ A label for a set of tools that can be displayed above groups of tools and sections in the user interface @@ -898,7 +898,7 @@ self.rerun_remap_job_id = None self.inputs = params_from_strings( tool.inputs, values, app, ignore_errors=True ) -class ToolOutput( object, DictifiableMixin ): +class ToolOutput( object, Dictifiable ): """ Represents an output datasets produced by a tool. For backward compatibility this behaves as if it were the tuple:: @@ -948,7 +948,7 @@ self.type = type self.version = version -class Tool( object, DictifiableMixin ): +class Tool( object, Dictifiable ): """ Represents a computational tool that can be executed through Galaxy. """ @@ -2962,11 +2962,11 @@ self.sa_session.flush() return primary_datasets - def dictify( self, trans, link_details=False, io_details=False ): + def to_dict( self, trans, link_details=False, io_details=False ): """ Returns dict of tool. """ # Basic information - tool_dict = super( Tool, self ).dictify() + tool_dict = super( Tool, self ).to_dict() # Add link details. if link_details: @@ -2983,8 +2983,8 @@ # Add input and output details. if io_details: - tool_dict[ 'inputs' ] = [ input.dictify( trans ) for input in self.inputs.values() ] - tool_dict[ 'outputs' ] = [ output.dictify() for output in self.outputs.values() ] + tool_dict[ 'inputs' ] = [ input.to_dict( trans ) for input in self.inputs.values() ] + tool_dict[ 'outputs' ] = [ output.to_dict() for output in self.outputs.values() ] return tool_dict diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -12,12 +12,12 @@ import validation, dynamic_options # For BaseURLToolParameter from galaxy.web import url_for -from galaxy.model.item_attrs import DictifiableMixin +from galaxy.model.item_attrs import Dictifiable import galaxy.model log = logging.getLogger(__name__) -class ToolParameter( object, DictifiableMixin ): +class ToolParameter( object, Dictifiable ): """ Describes a parameter accepted by a tool. This is just a simple stub at the moment but in the future should encapsulate more complex parameters (lists @@ -170,10 +170,10 @@ for validator in self.validators: validator.validate( value, history ) - def dictify( self, trans, view='collection', value_mapper=None ): - """ Dictify tool parameter. This can be overridden by subclasses. """ + def to_dict( self, trans, view='collection', value_mapper=None ): + """ to_dict tool parameter. This can be overridden by subclasses. """ - tool_dict = super( ToolParameter, self ).dictify() + tool_dict = super( ToolParameter, self ).to_dict() tool_dict[ 'html' ] = urllib.quote( self.get_html( trans ) ) if hasattr( self, 'value' ): tool_dict[ 'value' ] = self.value @@ -872,8 +872,8 @@ else: return [] - def dictify( self, trans, view='collection', value_mapper=None ): - d = super( SelectToolParameter, self ).dictify( trans ) + def to_dict( self, trans, view='collection', value_mapper=None ): + d = super( SelectToolParameter, self ).to_dict( trans ) # Get options, value. options = self.get_options( trans, [] ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/tools/parameters/grouping.py --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -15,9 +15,9 @@ from galaxy.util import sanitize_for_filename from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext -from galaxy.model.item_attrs import DictifiableMixin +from galaxy.model.item_attrs import Dictifiable -class Group( object, DictifiableMixin ): +class Group( object, Dictifiable ): dict_collection_visible_keys = ( 'name', 'type' ) @@ -48,9 +48,9 @@ """ raise TypeError( "Not implemented" ) - def dictify( self, trans, view='collection', value_mapper=None ): - # TODO: need to dictify conditions. - group_dict = super( Group, self ).dictify( view=view, value_mapper=value_mapper ) + def to_dict( self, trans, view='collection', value_mapper=None ): + # TODO: need to to_dict conditions. + group_dict = super( Group, self ).to_dict( view=view, value_mapper=value_mapper ) return group_dict class Repeat( Group ): diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/visualization/genomes.py --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -73,7 +73,7 @@ self.len_file = len_file self.twobit_file = twobit_file - def dictify( self, num=None, chrom=None, low=None ): + def to_dict( self, num=None, chrom=None, low=None ): """ Returns representation of self as a dictionary. """ @@ -289,7 +289,7 @@ # Set up return value or log exception if genome not found for key. rval = None if genome: - rval = genome.dictify( num=num, chrom=chrom, low=low ) + rval = genome.to_dict( num=num, chrom=chrom, low=low ) else: log.exception( 'genome not found for key %s' % dbkey ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -32,7 +32,7 @@ from galaxy.model.orm import eagerload, eagerload_all from galaxy.security.validate_user_input import validate_publicname from galaxy.util.sanitize_html import sanitize_html -from galaxy.model.item_attrs import DictifiableMixin +from galaxy.model.item_attrs import Dictifiable from galaxy.datatypes.interval import ChromatinInteractions from galaxy.datatypes.data import Text @@ -422,7 +422,7 @@ def get_history_dict( self, trans, history, hda_dictionaries=None ): """Returns history data in the form of a dictionary. """ - history_dict = history.dictify( view='element', value_mapper={ 'id':trans.security.encode_id }) + history_dict = history.to_dict( view='element', value_mapper={ 'id':trans.security.encode_id }) history_dict[ 'nice_size' ] = history.get_disk_size( nice_size=True ) history_dict[ 'annotation' ] = history.get_item_annotation_str( trans.sa_session, trans.user, history ) @@ -583,7 +583,7 @@ """ #precondition: the user's access to this hda has already been checked #TODO:?? postcondition: all ids are encoded (is this really what we want at this level?) - hda_dict = hda.dictify( view='element' ) + hda_dict = hda.to_dict( view='element' ) hda_dict[ 'api_type' ] = "file" # Add additional attributes that depend on trans can hence must be added here rather than at the model level. @@ -594,7 +594,7 @@ # ---- return here if deleted AND purged OR can't access purged = ( hda.purged or hda.dataset.purged ) if ( hda.deleted and purged ): - #TODO: dictify should really go AFTER this - only summary data + #TODO: to_dict should really go AFTER this - only summary data return trans.security.encode_dict_ids( hda_dict ) if trans.user_is_admin() or trans.app.config.expose_dataset_path: @@ -920,7 +920,7 @@ return query return query.all() - #TODO: move into model (dictify) + #TODO: move into model (to_dict) def get_visualization_summary_dict( self, visualization ): """ Return a set of summary attributes for a visualization in dictionary form. @@ -949,7 +949,7 @@ 'user_id' : visualization.user.id, 'dbkey' : visualization.dbkey, 'slug' : visualization.slug, - # dictify only the latest revision (allow older to be fetched elsewhere) + # to_dict only the latest revision (allow older to be fetched elsewhere) 'latest_revision' : self.get_visualization_revision_dict( visualization.latest_revision ), 'revisions' : [ r.id for r in visualization.revisions ], } @@ -1129,7 +1129,7 @@ return None # Get tool definition and add input values from job. - tool_dict = tool.dictify( trans, io_details=True ) + tool_dict = tool.to_dict( trans, io_details=True ) inputs_dict = tool_dict[ 'inputs' ] tool_param_values = dict( [ ( p.name, p.value ) for p in job.parameters ] ) tool_param_values = tool.params_from_strings( tool_param_values, trans.app, ignore_errors=True ) @@ -1139,8 +1139,8 @@ name = t_input[ 'name' ] if name in tool_param_values: value = tool_param_values[ name ] - if isinstance( value, DictifiableMixin ): - value = value.dictify() + if isinstance( value, Dictifiable ): + value = value.to_dict() t_input[ 'value' ] = value return tool_dict @@ -1172,7 +1172,7 @@ source='data' ) return { "track_type": dataset.datatype.track_type, - "dataset": trans.security.encode_dict_ids( dataset.dictify() ), + "dataset": trans.security.encode_dict_ids( dataset.to_dict() ), "name": track_dict['name'], "prefs": prefs, "mode": track_dict.get( 'mode', 'Auto' ), @@ -1252,7 +1252,7 @@ return { "track_type": dataset.datatype.track_type, "name": dataset.name, - "dataset": trans.security.encode_dict_ids( dataset.dictify() ), + "dataset": trans.security.encode_dict_ids( dataset.to_dict() ), "prefs": {}, "filters": { 'filters' : track_data_provider.get_filters() }, "tool": self.get_tool_def( trans, dataset ), diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/datasets.py --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -60,7 +60,7 @@ rval[ 'display_types' ] = self.get_old_display_applications( trans, dataset ) rval[ 'display_apps' ] = self.get_display_apps( trans, dataset ) else: - rval = dataset.dictify() + rval = dataset.to_dict() except Exception, e: rval = "Error in dataset API at listing contents: " + str( e ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/folders.py --- a/lib/galaxy/webapps/galaxy/api/folders.py +++ b/lib/galaxy/webapps/galaxy/api/folders.py @@ -35,7 +35,7 @@ # check_ownership=false since we are only displaying it. content = self.get_library_folder( trans, id, check_ownership=False, check_accessible=True ) - return self.encode_all_ids( trans, content.dictify( view='element' ) ) + return self.encode_all_ids( trans, content.to_dict( view='element' ) ) @web.expose_api def create( self, trans, payload, **kwd ): diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/forms.py --- a/lib/galaxy/webapps/galaxy/api/forms.py +++ b/lib/galaxy/webapps/galaxy/api/forms.py @@ -23,7 +23,7 @@ query = trans.sa_session.query( trans.app.model.FormDefinition )#.filter( trans.app.model.FormDefinition.table.c.deleted == False ) rval = [] for form_definition in query: - item = form_definition.dictify( value_mapper={ 'id': trans.security.encode_id, 'form_definition_current_id': trans.security.encode_id } ) + item = form_definition.to_dict( value_mapper={ 'id': trans.security.encode_id, 'form_definition_current_id': trans.security.encode_id } ) item['url'] = url_for( 'form', id=trans.security.encode_id( form_definition.id ) ) rval.append( item ) return rval @@ -47,7 +47,7 @@ if not form_definition or not trans.user_is_admin(): trans.response.status = 400 return "Invalid form definition id ( %s ) specified." % str( form_definition_id ) - item = form_definition.dictify( view='element', value_mapper={ 'id': trans.security.encode_id, 'form_definition_current_id': trans.security.encode_id } ) + item = form_definition.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'form_definition_current_id': trans.security.encode_id } ) item['url'] = url_for( 'form', id=form_definition_id ) return item @@ -69,6 +69,6 @@ trans.sa_session.add( form_definition ) trans.sa_session.flush() encoded_id = trans.security.encode_id( form_definition.id ) - item = form_definition.dictify( view='element', value_mapper={ 'id': trans.security.encode_id, 'form_definition_current_id': trans.security.encode_id } ) + item = form_definition.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'form_definition_current_id': trans.security.encode_id } ) item['url'] = url_for( 'form', id=encoded_id ) return [ item ] diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/groups.py --- a/lib/galaxy/webapps/galaxy/api/groups.py +++ b/lib/galaxy/webapps/galaxy/api/groups.py @@ -21,7 +21,7 @@ rval = [] for group in trans.sa_session.query( trans.app.model.Group ).filter( trans.app.model.Group.table.c.deleted == False ): if trans.user_is_admin(): - item = group.dictify( value_mapper={ 'id': trans.security.encode_id } ) + item = group.to_dict( value_mapper={ 'id': trans.security.encode_id } ) encoded_id = trans.security.encode_id( group.id ) item['url'] = url_for( 'group', id=encoded_id ) rval.append( item ) @@ -65,7 +65,7 @@ """ trans.sa_session.flush() encoded_id = trans.security.encode_id( group.id ) - item = group.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) + item = group.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id } ) item['url'] = url_for( 'group', id=encoded_id ) return [ item ] @@ -89,7 +89,7 @@ if not group: trans.response.status = 400 return "Invalid group id ( %s ) specified." % str( group_id ) - item = group.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) + item = group.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id } ) item['url'] = url_for( 'group', id=group_id ) item['users_url'] = url_for( 'group_users', group_id=group_id ) item['roles_url'] = url_for( 'group_roles', group_id=group_id ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -46,14 +46,14 @@ .order_by( desc( trans.app.model.History.table.c.update_time ) ) .all() ) for history in query: - item = history.dictify(value_mapper={'id':trans.security.encode_id}) + item = history.to_dict(value_mapper={'id':trans.security.encode_id}) item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) rval.append( item ) elif trans.galaxy_session.current_history: #No user, this must be session authentication with an anonymous user. history = trans.galaxy_session.current_history - item = history.dictify(value_mapper={'id':trans.security.encode_id}) + item = history.to_dict(value_mapper={'id':trans.security.encode_id}) item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) rval.append(item) @@ -139,7 +139,7 @@ trans.sa_session.add( new_history ) trans.sa_session.flush() - item = new_history.dictify(view='element', value_mapper={'id':trans.security.encode_id}) + item = new_history.to_dict(view='element', value_mapper={'id':trans.security.encode_id}) item['url'] = url_for( 'history', id=item['id'] ) #TODO: copy own history @@ -254,7 +254,7 @@ :param id: the encoded id of the history to undelete :type payload: dict :param payload: a dictionary containing any or all the - fields in :func:`galaxy.model.History.dictify` and/or the following: + fields in :func:`galaxy.model.History.to_dict` and/or the following: * annotation: an annotation for the history diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -190,7 +190,7 @@ hda = ld.library_dataset_dataset_association.to_history_dataset_association( history, add_to_history=True ) trans.sa_session.flush() - return hda.dictify() + return hda.to_dict() else: # TODO: implement other "upload" methods here. @@ -210,7 +210,7 @@ :param id: the encoded id of the history to undelete :type payload: dict :param payload: a dictionary containing any or all the - fields in :func:`galaxy.model.HistoryDatasetAssociation.dictify` + fields in :func:`galaxy.model.HistoryDatasetAssociation.to_dict` and/or the following: * annotation: an annotation for the HDA diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/item_tags.py --- a/lib/galaxy/webapps/galaxy/api/item_tags.py +++ b/lib/galaxy/webapps/galaxy/api/item_tags.py @@ -49,7 +49,7 @@ return 'OK' def _api_value( self, tag, trans, view='element' ): - return tag.dictify( view=view, value_mapper={ 'id': trans.security.encode_id } ) + return tag.to_dict( view=view, value_mapper={ 'id': trans.security.encode_id } ) class HistoryContentTagsController( BaseItemTagsController ): controller_name = "history_content_tags" diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/libraries.py --- a/lib/galaxy/webapps/galaxy/api/libraries.py +++ b/lib/galaxy/webapps/galaxy/api/libraries.py @@ -49,7 +49,7 @@ trans.model.Library.table.c.id.in_( accessible_restricted_library_ids ) ) ) rval = [] for library in query: - item = library.dictify() + item = library.to_dict() item['url'] = url_for( route, id=trans.security.encode_id( library.id ) ) item['id'] = trans.security.encode_id( item['id'] ) rval.append( item ) @@ -87,7 +87,7 @@ library = None if not library or not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( trans.get_current_user_roles(), library ) ): raise HTTPBadRequest( detail='Invalid library id ( %s ) specified.' % id ) - item = library.dictify( view='element' ) + item = library.to_dict( view='element' ) #item['contents_url'] = url_for( 'contents', library_id=library_id ) item['contents_url'] = url_for( 'library_contents', library_id=library_id ) return item @@ -162,4 +162,4 @@ library.deleted = True trans.sa_session.add( library ) trans.sa_session.flush() - return library.dictify( view='element', value_mapper={ 'id' : trans.security.encode_id } ) + return library.to_dict( view='element', value_mapper={ 'id' : trans.security.encode_id } ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/library_contents.py --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -105,7 +105,7 @@ :rtype: dict :returns: detailed library item information .. seealso:: - :func:`galaxy.model.LibraryDataset.dictify` and + :func:`galaxy.model.LibraryDataset.to_dict` and :attr:`galaxy.model.LibraryFolder.dict_element_visible_keys` """ class_name, content_id = self.__decode_library_content_id( trans, id ) @@ -113,7 +113,7 @@ content = self.get_library_folder( trans, content_id, check_ownership=False, check_accessible=True ) else: content = self.get_library_dataset( trans, content_id, check_ownership=False, check_accessible=True ) - return self.encode_all_ids( trans, content.dictify( view='element' ) ) + return self.encode_all_ids( trans, content.to_dict( view='element' ) ) @web.expose_api def create( self, trans, library_id, payload, **kwd ): @@ -266,7 +266,7 @@ return { 'error' : 'user has no permission to add to library folder (%s)' %( folder_id ) } ldda = self.copy_hda_to_library_folder( trans, hda, folder, ldda_message=ldda_message ) - ldda_dict = ldda.dictify() + ldda_dict = ldda.to_dict() rval = trans.security.encode_dict_ids( ldda_dict ) except Exception, exc: diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/permissions.py --- a/lib/galaxy/webapps/galaxy/api/permissions.py +++ b/lib/galaxy/webapps/galaxy/api/permissions.py @@ -47,6 +47,6 @@ trans.app.security_agent.copy_library_permissions( trans, library, library.root_folder ) message = "Permissions updated for library '%s'." % library.name - item = library.dictify( view='element' ) + item = library.to_dict( view='element' ) return item diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/quotas.py --- a/lib/galaxy/webapps/galaxy/api/quotas.py +++ b/lib/galaxy/webapps/galaxy/api/quotas.py @@ -34,7 +34,7 @@ route = 'quota' query = query.filter( trans.app.model.Quota.table.c.deleted == False ) for quota in query: - item = quota.dictify( value_mapper={ 'id': trans.security.encode_id } ) + item = quota.to_dict( value_mapper={ 'id': trans.security.encode_id } ) encoded_id = trans.security.encode_id( quota.id ) item['url'] = url_for( route, id=encoded_id ) rval.append( item ) @@ -49,7 +49,7 @@ Displays information about a quota. """ quota = self.get_quota( trans, id, deleted=util.string_as_bool( deleted ) ) - return quota.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) + return quota.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id } ) @web.expose_api @web.require_admin @@ -67,7 +67,7 @@ quota, message = self._create_quota( params ) except ActionInputError, e: raise HTTPBadRequest( detail=str( e ) ) - item = quota.dictify( value_mapper={ 'id': trans.security.encode_id } ) + item = quota.to_dict( value_mapper={ 'id': trans.security.encode_id } ) item['url'] = url_for( 'quota', id=trans.security.encode_id( quota.id ) ) item['message'] = message return item diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/request_types.py --- a/lib/galaxy/webapps/galaxy/api/request_types.py +++ b/lib/galaxy/webapps/galaxy/api/request_types.py @@ -19,7 +19,7 @@ """ rval = [] for request_type in trans.app.security_agent.get_accessible_request_types( trans, trans.user ): - item = request_type.dictify( value_mapper={ 'id': trans.security.encode_id, 'request_form_id': trans.security.encode_id, 'sample_form_id': trans.security.encode_id } ) + item = request_type.to_dict( value_mapper={ 'id': trans.security.encode_id, 'request_form_id': trans.security.encode_id, 'sample_form_id': trans.security.encode_id } ) encoded_id = trans.security.encode_id( request_type.id ) item['url'] = url_for( 'request_type', id=encoded_id ) rval.append( item ) @@ -47,7 +47,7 @@ if not trans.app.security_agent.can_access_request_type( trans.user.all_roles(), request_type ): trans.response.status = 400 return "No permission to access request_type ( %s )." % str( request_type_id ) - item = request_type.dictify( view='element', value_mapper={ 'id': trans.security.encode_id, 'request_form_id': trans.security.encode_id, 'sample_form_id': trans.security.encode_id } ) + item = request_type.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'request_form_id': trans.security.encode_id, 'sample_form_id': trans.security.encode_id } ) item['url'] = url_for( 'request_type', id=request_type_id ) return item @@ -97,6 +97,6 @@ trans.sa_session.add( request_type ) trans.sa_session.flush() encoded_id = trans.security.encode_id( request_type.id ) - item = request_type.dictify( view='element', value_mapper={ 'id': trans.security.encode_id, 'request_form_id': trans.security.encode_id, 'sample_form_id': trans.security.encode_id } ) + item = request_type.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'request_form_id': trans.security.encode_id, 'sample_form_id': trans.security.encode_id } ) item['url'] = url_for( 'request_type', id=encoded_id ) return [ item ] diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/requests.py --- a/lib/galaxy/webapps/galaxy/api/requests.py +++ b/lib/galaxy/webapps/galaxy/api/requests.py @@ -30,7 +30,7 @@ .all() rval = [] for request in query: - item = request.dictify() + item = request.to_dict() item['url'] = url_for( 'requests', id=trans.security.encode_id( request.id ) ) item['id'] = trans.security.encode_id( item['id'] ) if trans.user_is_admin(): @@ -55,7 +55,7 @@ if not request or not ( trans.user_is_admin() or request.user.id == trans.user.id ): trans.response.status = 400 return "Invalid request id ( %s ) specified." % str( request_id ) - item = request.dictify() + item = request.to_dict() item['url'] = url_for( 'requests', id=trans.security.encode_id( request.id ) ) item['id'] = trans.security.encode_id( item['id'] ) item['user'] = request.user.email diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/roles.py --- a/lib/galaxy/webapps/galaxy/api/roles.py +++ b/lib/galaxy/webapps/galaxy/api/roles.py @@ -18,7 +18,7 @@ rval = [] for role in trans.sa_session.query( trans.app.model.Role ).filter( trans.app.model.Role.table.c.deleted == False ): if trans.user_is_admin() or trans.app.security_agent.ok_to_display( trans.user, role ): - item = role.dictify( value_mapper={ 'id': trans.security.encode_id } ) + item = role.to_dict( value_mapper={ 'id': trans.security.encode_id } ) encoded_id = trans.security.encode_id( role.id ) item['url'] = url_for( 'role', id=encoded_id ) rval.append( item ) @@ -43,7 +43,7 @@ if not role or not (trans.user_is_admin() or trans.app.security_agent.ok_to_display( trans.user, role )): trans.response.status = 400 return "Invalid role id ( %s ) specified." % str( role_id ) - item = role.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) + item = role.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id } ) item['url'] = url_for( 'role', id=role_id ) return item @@ -81,6 +81,6 @@ trans.app.security_agent.associate_group_role( group, role ) trans.sa_session.flush() encoded_id = trans.security.encode_id( role.id ) - item = role.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) + item = role.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id } ) item['url'] = url_for( 'role', id=encoded_id ) return [ item ] diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/samples.py --- a/lib/galaxy/webapps/galaxy/api/samples.py +++ b/lib/galaxy/webapps/galaxy/api/samples.py @@ -35,7 +35,7 @@ return "Invalid request id ( %s ) specified." % str( request_id ) rval = [] for sample in request.samples: - item = sample.dictify() + item = sample.to_dict() item['url'] = url_for( 'samples', request_id=trans.security.encode_id( request_id ), id=trans.security.encode_id( sample.id ) ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -45,7 +45,7 @@ .order_by( trans.app.model.ToolShedRepository.table.c.name ) \ .all() for tool_shed_repository in query: - tool_shed_repository_dict = tool_shed_repository.dictify( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict = tool_shed_repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( tool_shed_repository.id ) ) @@ -402,7 +402,7 @@ repair_dict = repository_util.repair_tool_shed_repository( trans, repository, encoding_util.tool_shed_encode( repo_info_dict ) ) - repository_dict = repository.dictify( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) ) + repository_dict = repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) ) repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', action='show', id=trans.security.encode_id( repository.id ) ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -32,7 +32,7 @@ # Create return value. try: - return self.app.toolbox.dictify( trans, in_panel=in_panel, trackster=trackster ) + return self.app.toolbox.to_dict( trans, in_panel=in_panel, trackster=trackster ) except Exception, exc: log.error( 'could not convert toolbox to dictionary: %s', str( exc ), exc_info=True ) trans.response.status = 500 @@ -45,7 +45,7 @@ Returns tool information, including parameters and inputs. """ try: - return self.app.toolbox.tools_by_id[ id ].dictify( trans, for_display=True ) + return self.app.toolbox.tools_by_id[ id ].to_dict( trans, for_display=True ) except Exception, exc: log.error( 'could not convert tool (%s) to dictionary: %s', id, str( exc ), exc_info=True ) trans.response.status = 500 @@ -105,7 +105,7 @@ outputs = rval[ "outputs" ] #TODO:?? poss. only return ids? for output in output_datasets: - output_dict = output.dictify() + output_dict = output.to_dict() outputs.append( trans.security.encode_dict_ids( output_dict ) ) return rval @@ -411,7 +411,7 @@ if joda.name == output_name: output_dataset = joda.dataset - dataset_dict = output_dataset.dictify() + dataset_dict = output_dataset.to_dict() dataset_dict[ 'id' ] = trans.security.encode_id( dataset_dict[ 'id' ] ) dataset_dict[ 'track_config' ] = self.get_new_track_config( trans, output_dataset ); return dataset_dict diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/users.py --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -33,13 +33,13 @@ query = query.filter( trans.app.model.User.table.c.deleted == False ) # special case: user can see only their own user if not trans.user_is_admin(): - item = trans.user.dictify( value_mapper={ 'id': trans.security.encode_id } ) + item = trans.user.to_dict( value_mapper={ 'id': trans.security.encode_id } ) item['url'] = url_for( route, id=item['id'] ) item['quota_percent'] = trans.app.quota_agent.get_percent( trans=trans ) return [item] for user in query: - item = user.dictify( value_mapper={ 'id': trans.security.encode_id } ) + item = user.to_dict( value_mapper={ 'id': trans.security.encode_id } ) #TODO: move into api_values item['quota_percent'] = trans.app.quota_agent.get_percent( trans=trans ) item['url'] = url_for( route, id=item['id'] ) @@ -77,7 +77,7 @@ raise else: raise HTTPBadRequest( detail='Invalid user id ( %s ) specified' % id ) - item = user.dictify( view='element', value_mapper={ 'id': trans.security.encode_id, + item = user.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'total_disk_usage': float } ) #TODO: move into api_values (needs trans, tho - can we do that with api_keys/@property??) #TODO: works with other users (from admin)?? @@ -94,7 +94,7 @@ raise HTTPNotImplemented( detail='User creation is not allowed in this Galaxy instance' ) if trans.app.config.use_remote_user and trans.user_is_admin(): user = trans.get_or_create_remote_user(remote_user_email=payload['remote_user_email']) - item = user.dictify( view='element', value_mapper={ 'id': trans.security.encode_id, + item = user.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'total_disk_usage': float } ) else: raise HTTPNotImplemented() diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/visualizations.py --- a/lib/galaxy/webapps/galaxy/api/visualizations.py +++ b/lib/galaxy/webapps/galaxy/api/visualizations.py @@ -212,7 +212,7 @@ # this allows PUT'ing an entire model back to the server without attribute errors on uneditable attrs valid_but_uneditable_keys = ( 'id', 'model_class' - #TODO: fill out when we create dictify, get_dict, whatevs + #TODO: fill out when we create to_dict, get_dict, whatevs ) #TODO: deleted #TODO: importable diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/api/workflows.py --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -31,7 +31,7 @@ for wf in trans.sa_session.query(trans.app.model.StoredWorkflow).filter_by( user=trans.user, deleted=False).order_by( desc(trans.app.model.StoredWorkflow.table.c.update_time)).all(): - item = wf.dictify(value_mapper={'id':trans.security.encode_id}) + item = wf.to_dict(value_mapper={'id':trans.security.encode_id}) encoded_id = trans.security.encode_id(wf.id) item['url'] = url_for('workflow', id=encoded_id) rval.append(item) @@ -39,7 +39,7 @@ user=trans.user ).join( 'stored_workflow' ).filter( trans.app.model.StoredWorkflow.deleted == False ).order_by( desc( trans.app.model.StoredWorkflow.update_time ) ).all(): - item = wf_sa.stored_workflow.dictify(value_mapper={'id':trans.security.encode_id}) + item = wf_sa.stored_workflow.to_dict(value_mapper={'id':trans.security.encode_id}) encoded_id = trans.security.encode_id(wf_sa.stored_workflow.id) item['url'] = url_for('workflow', id=encoded_id) rval.append(item) @@ -67,7 +67,7 @@ except: trans.response.status = 400 return "That workflow does not exist." - item = stored_workflow.dictify(view='element', value_mapper={'id':trans.security.encode_id}) + item = stored_workflow.to_dict(view='element', value_mapper={'id':trans.security.encode_id}) item['url'] = url_for('workflow', id=workflow_id) latest_workflow = stored_workflow.latest_workflow inputs = {} @@ -329,7 +329,7 @@ # return list rval= []; - item = workflow.dictify(value_mapper={'id':trans.security.encode_id}) + item = workflow.to_dict(value_mapper={'id':trans.security.encode_id}) item['url'] = url_for('workflow', id=encoded_id) rval.append(item); diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/galaxy/controllers/visualization.py --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -873,8 +873,8 @@ # Add tool, dataset attributes to config based on id. tool = trans.app.toolbox.get_tool( viz_config[ 'tool_id' ] ) - viz_config[ 'tool' ] = tool.dictify( trans, io_details=True ) - viz_config[ 'dataset' ] = trans.security.encode_dict_ids( dataset.dictify() ) + viz_config[ 'tool' ] = tool.to_dict( trans, io_details=True ) + viz_config[ 'dataset' ] = trans.security.encode_dict_ids( dataset.to_dict() ) return trans.fill_template_mako( "visualization/sweepster.mako", config=viz_config ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/tool_shed/api/repositories.py --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -113,7 +113,7 @@ # Get the repository information. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) encoded_repository_id = trans.security.encode_id( repository.id ) - repository_dict = repository.dictify( view='element', value_mapper=default_repository_value_mapper( trans, repository ) ) + repository_dict = repository.to_dict( view='element', value_mapper=default_repository_value_mapper( trans, repository ) ) repository_dict[ 'url' ] = web.url_for( controller='repositories', action='show', id=encoded_repository_id ) @@ -129,7 +129,7 @@ changeset_revision = new_changeset_revision if repository_metadata: encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id ) - repository_metadata_dict = repository_metadata.dictify( view='collection', + repository_metadata_dict = repository_metadata.to_dict( view='collection', value_mapper=default_repository_metadata_value_mapper( trans, repository_metadata ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', @@ -164,7 +164,7 @@ .order_by( trans.app.model.Repository.table.c.name ) \ .all() for repository in query: - repository_dict = repository.dictify( view='collection', value_mapper=default_repository_value_mapper( trans, repository ) ) + repository_dict = repository.to_dict( view='collection', value_mapper=default_repository_value_mapper( trans, repository ) ) repository_dict[ 'url' ] = web.url_for( controller='repositories', action='show', id=trans.security.encode_id( repository.id ) ) @@ -187,7 +187,7 @@ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 try: repository = suc.get_repository_in_tool_shed( trans, id ) - repository_dict = repository.dictify( view='element', value_mapper=default_repository_value_mapper( trans, repository ) ) + repository_dict = repository.to_dict( view='element', value_mapper=default_repository_value_mapper( trans, repository ) ) repository_dict[ 'url' ] = web.url_for( controller='repositories', action='show', id=trans.security.encode_id( repository.id ) ) diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/tool_shed/api/repository_revisions.py --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -127,7 +127,7 @@ .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id ) \ .all() for repository_metadata in query: - repository_metadata_dict = repository_metadata.dictify( view='collection', + repository_metadata_dict = repository_metadata.to_dict( view='collection', value_mapper=default_value_mapper( trans, repository_metadata ) ) repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', diff -r ce4c7a3fb0d847633db4e77d6e90cc94c7d55aa3 -r a4259a97c287927d6a377092bc649579a88cb433 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -4,7 +4,7 @@ from galaxy import util from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash -from galaxy.model.item_attrs import DictifiableMixin +from galaxy.model.item_attrs import Dictifiable import tool_shed.repository_types.util as rt_util from galaxy import eggs @@ -19,7 +19,7 @@ pass -class User( object, DictifiableMixin ): +class User( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'email' ) dict_element_visible_keys = ( 'id', 'email', 'username' ) @@ -61,7 +61,7 @@ return 0 -class Group( object, DictifiableMixin ): +class Group( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name' ) dict_element_visible_keys = ( 'id', 'name' ) @@ -70,7 +70,7 @@ self.deleted = False -class Role( object, DictifiableMixin ): +class Role( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name' ) dict_element_visible_keys = ( 'id', 'name', 'description', 'type' ) private_id = None @@ -130,7 +130,7 @@ self.prev_session_id = prev_session_id -class Repository( object, DictifiableMixin ): +class Repository( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name', 'type', 'description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' ) dict_element_visible_keys = ( 'id', 'name', 'type', 'description', 'long_description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' ) @@ -155,7 +155,7 @@ self.deprecated = deprecated def as_dict( self, value_mapper=None ): - return self.dictify( view='element', value_mapper=value_mapper ) + return self.to_dict( view='element', value_mapper=value_mapper ) def can_change_type( self, app ): # Allow changing the type only if the repository has no contents, has never been installed, or has never been changed from @@ -175,7 +175,7 @@ return True return False - def dictify( self, view='collection', value_mapper=None ): + def to_dict( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} rval = {} @@ -244,7 +244,7 @@ fp.close() -class RepositoryMetadata( object, DictifiableMixin ): +class RepositoryMetadata( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) dict_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', 'do_not_test', @@ -284,9 +284,9 @@ return False def as_dict( self, value_mapper=None ): - return self.dictify( view='element', value_mapper=value_mapper ) + return self.to_dict( view='element', value_mapper=value_mapper ) - def dictify( self, view='collection', value_mapper=None ): + def to_dict( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} rval = {} @@ -304,7 +304,7 @@ return rval -class SkipToolTest( object, DictifiableMixin ): +class SkipToolTest( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision' ) dict_element_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision', 'comment' ) @@ -315,9 +315,9 @@ self.comment = comment def as_dict( self, value_mapper=None ): - return self.dictify( view='element', value_mapper=value_mapper ) + return self.to_dict( view='element', value_mapper=value_mapper ) - def dictify( self, view='collection', value_mapper=None ): + def to_dict( self, view='collection', value_mapper=None ): if value_mapper is None: value_mapper = {} rval = {} @@ -335,7 +335,7 @@ return rval -class RepositoryReview( object, DictifiableMixin ): +class RepositoryReview( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) dict_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) approved_states = Bunch( NO='no', YES='yes' ) @@ -347,7 +347,7 @@ self.rating = rating self.deleted = deleted -class ComponentReview( object, DictifiableMixin ): +class ComponentReview( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'repository_review_id', 'component_id', 'private', 'approved', 'rating', 'deleted' ) dict_element_visible_keys = ( 'id', 'repository_review_id', 'component_id', 'private', 'approved', 'rating', 'deleted' ) approved_states = Bunch( NO='no', YES='yes', NA='not_applicable' ) @@ -389,7 +389,7 @@ self.repository = repository -class Category( object, DictifiableMixin ): +class Category( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'name', 'description', 'deleted' ) dict_element_visible_keys = ( 'id', 'name', 'description', 'deleted' ) https://bitbucket.org/galaxy/galaxy-central/commits/c59cc42b2793/ Changeset: c59cc42b2793 User: jgoecks Date: 2013-08-30 20:23:42 Summary: Remove unneeded to_dict code/methods. Affected #: 1 file diff -r a4259a97c287927d6a377092bc649579a88cb433 -r c59cc42b279385f0d351d31f919d29bb7c012877 lib/galaxy/webapps/tool_shed/model/__init__.py --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -176,20 +176,7 @@ return False def to_dict( self, view='collection', value_mapper=None ): - if value_mapper is None: - value_mapper = {} - rval = {} - try: - visible_keys = self.__getattribute__( 'dict_' + view + '_visible_keys' ) - except AttributeError: - raise Exception( 'Unknown API view: %s' % view ) - for key in visible_keys: - try: - rval[ key ] = self.__getattribute__( key ) - if key in value_mapper: - rval[ key ] = value_mapper.get( key, rval[ key ] ) - except AttributeError: - rval[ key ] = None + rval = super( Repository, self ).to_dict( view=view, value_mapper=value_mapper ) if 'user_id' in rval: rval[ 'owner' ] = self.user.username return rval @@ -286,23 +273,6 @@ def as_dict( self, value_mapper=None ): return self.to_dict( view='element', value_mapper=value_mapper ) - def to_dict( self, view='collection', value_mapper=None ): - if value_mapper is None: - value_mapper = {} - rval = {} - try: - visible_keys = self.__getattribute__( 'dict_' + view + '_visible_keys' ) - except AttributeError: - raise Exception( 'Unknown API view: %s' % view ) - for key in visible_keys: - try: - rval[ key ] = self.__getattribute__( key ) - if key in value_mapper: - rval[ key ] = value_mapper.get( key, rval[ key ] ) - except AttributeError: - rval[ key ] = None - return rval - class SkipToolTest( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'repository_metadata_id', 'initial_changeset_revision' ) @@ -317,23 +287,6 @@ def as_dict( self, value_mapper=None ): return self.to_dict( view='element', value_mapper=value_mapper ) - def to_dict( self, view='collection', value_mapper=None ): - if value_mapper is None: - value_mapper = {} - rval = {} - try: - visible_keys = self.__getattribute__( 'dict_' + view + '_visible_keys' ) - except AttributeError: - raise Exception( 'Unknown API view: %s' % view ) - for key in visible_keys: - try: - rval[ key ] = self.__getattribute__( key ) - if key in value_mapper: - rval[ key ] = value_mapper.get( key, rval[ key ] ) - except AttributeError: - rval[ key ] = None - return rval - class RepositoryReview( object, Dictifiable ): dict_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Fix for zero-based comparison in the tool shed.
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/9345e15ec4b7/ Changeset: 9345e15ec4b7 User: greg Date: 2013-08-30 19:08:22 Summary: Fix for zero-based comparison in the tool shed. Affected #: 1 file diff -r 55bbaa8f5017e32dbe7968514a4f87b941ed2a0d -r 9345e15ec4b7f81271a071f0703e89fff0c5b3ac lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2695,7 +2695,7 @@ tool_shed_status_dict[ 'revision_upgrade' ] = 'False' break if metadata_changeset_revision == changeset_revision: - if num_metadata_revisions - index > 0: + if num_metadata_revisions - index > 1: tool_shed_status_dict[ 'revision_upgrade' ] = 'True' else: tool_shed_status_dict[ 'revision_upgrade' ] = 'False' Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Re-apply changes that got reverted in 10492:8f6f926f912e
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/55bbaa8f5017/ Changeset: 55bbaa8f5017 User: dannon Date: 2013-08-30 18:06:10 Summary: Re-apply changes that got reverted in 10492:8f6f926f912e Affected #: 2 files diff -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 -r 55bbaa8f5017e32dbe7968514a4f87b941ed2a0d lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2664,7 +2664,7 @@ if repository: repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) - tool_shed_status_dict = {} + tool_shed_status_dict = {} # Handle repository deprecation. tool_shed_status_dict[ 'repository_deprecated' ] = str( repository.deprecated ) # Handle latest installable revision. @@ -2680,7 +2680,7 @@ if changeset_revision == repository.tip( trans.app ): tool_shed_status_dict[ 'revision_update' ] = 'False' else: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: diff -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 -r 55bbaa8f5017e32dbe7968514a4f87b941ed2a0d lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -31,7 +31,7 @@ eggs.require( 'markupsafe' ) import markupsafe - + log = logging.getLogger( __name__ ) CHUNK_SIZE = 2**20 # 1Mb @@ -209,7 +209,7 @@ return tool_shed_url.split( ':' )[ 0 ] return tool_shed_url.rstrip( '/' ) -def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): +def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" try: commands.clone( get_configured_ui(), @@ -263,7 +263,7 @@ # was later uninstalled, this value should be received as the value of that change set to which the repository had been updated just prior # to it being uninstalled. current_changeset_revision = installed_changeset_revision - sa_session = app.model.context.current + sa_session = app.model.context.current tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) if not owner: owner = get_repository_owner_from_clone_url( repository_clone_url ) @@ -351,7 +351,7 @@ new_elem = XmlET.SubElement( elem, key ) new_elem.text = value return elem - + def generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=None, **kwd ): return generate_repository_info_elem( tool_shed_repository.tool_shed, tool_shed_repository.name, @@ -394,7 +394,7 @@ def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for - the tool in the Galaxy tool shed from which it is being installed. The form of the guid is + the tool in the Galaxy tool shed from which it is being installed. The form of the guid is <tool shed host>/repos/<repository owner>/<repository name>/<tool id>/<tool version> """ tmp_url = clean_repository_clone_url( repository_clone_url ) @@ -417,7 +417,7 @@ tool_config = tool_dict[ 'tool_config' ] file_name = strip_path( tool_config ) guids_and_configs[ guid ] = file_name - # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. + # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. tree, error_message = xml_util.parse_xml( shed_tool_conf ) if tree is None: return tool_panel_dict @@ -535,7 +535,7 @@ Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the combination of a name, owner and changeset revision. """ - url = url_join( tool_shed_url, + url = url_join( tool_shed_url, 'repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % ( name, owner, changeset_revision ) ) ctx_rev = common_util.tool_shed_get( app, tool_shed_url, url ) return ctx_rev @@ -1385,7 +1385,7 @@ def reset_previously_installed_repository( trans, repository ): """ - Reset the atrributes of a tool_shed_repository that was previsouly installed. The repository will be in some state other than with a + Reset the atrributes of a tool_shed_repository that was previsouly installed. The repository will be in some state other than with a status of INSTALLED, so all atributes will be set to the default (NEW( state. This will enable the repository to be freshly installed. """ repository.deleted = False Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the update_available feature for installed tool shed repositories to enable any type of status from the tool shed for each repository. The Installed tool shed repositories grid now displays whether there are revision updates available, revision upgrades available, whether the revision is the latest installable revision, and whether the repository has been deprecated in the tool shed. This change set includes a 0016 db table migration script for which it
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8f6f926f912e/ Changeset: 8f6f926f912e User: greg Date: 2013-08-30 17:47:27 Summary: Enhance the update_available feature for installed tool shed repositories to enable any type of status from the tool shed for each repository. The Installed tool shed repositories grid now displays whether there are revision updates available, revision upgrades available, whether the revision is the latest installable revision, and whether the repository has been deprecated in the tool shed. This change set includes a 0016 db table migration script for which it has been discovered that it is no longer possible to drop a column from a database table using a migration script if the database is sqlite. Affected #: 12 files diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3306,11 +3306,12 @@ class APIKeys( object ): pass + class ToolShedRepository( object ): dict_collection_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'update_available', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) dict_element_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', - 'update_available', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) + 'tool_shed_status', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) installation_status = Bunch( NEW='New', CLONING='Cloning', SETTING_TOOL_VERSIONS='Setting tool versions', @@ -3326,8 +3327,9 @@ WARNING = 'queued', ERROR = 'error', UNINSTALLED = 'deleted_new' ) + def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, installed_changeset_revision=None, - changeset_revision=None, ctx_rev=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False, + changeset_revision=None, ctx_rev=None, metadata=None, includes_datatypes=False, tool_shed_status=None, deleted=False, uninstalled=False, dist_to_shed=False, status=None, error_message=None ): self.id = id self.create_time = create_time @@ -3340,38 +3342,72 @@ self.ctx_rev = ctx_rev self.metadata = metadata self.includes_datatypes = includes_datatypes - self.update_available = update_available + self.tool_shed_status = tool_shed_status self.deleted = deleted self.uninstalled = uninstalled self.dist_to_shed = dist_to_shed self.status = status self.error_message = error_message + def as_dict( self, value_mapper=None ): return self.dictify( view='element', value_mapper=value_mapper ) - def repo_files_directory( self, app ): - repo_path = self.repo_path( app ) - if repo_path: - return os.path.join( repo_path, self.name ) - return None - def repo_path( self, app ): - tool_shed_url = self.tool_shed - if tool_shed_url.find( ':' ) > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - tool_shed_url = tool_shed_url.split( ':' )[ 0 ] - tool_shed = tool_shed_url.rstrip( '/' ) - for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): - tool_path = shed_tool_conf_dict[ 'tool_path' ] - relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) - if os.path.exists( relative_path ): - return relative_path - return None + @property - def tool_shed_path_name( self ): - tool_shed_url = self.tool_shed - if tool_shed_url.find( ':' ) > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - tool_shed_url = tool_shed_url.split( ':' )[ 0 ] - return tool_shed_url.rstrip( '/' ) + def can_install( self ): + return self.status == self.installation_status.NEW + + @property + def can_reset_metadata( self ): + return self.status == self.installation_status.INSTALLED + + @property + def can_uninstall( self ): + return self.status != self.installation_status.UNINSTALLED + + @property + def can_deactivate( self ): + return self.status not in [ self.installation_status.DEACTIVATED, self.installation_status.UNINSTALLED ] + + @property + def can_reinstall_or_activate( self ): + return self.deleted + + def dictify( self, view='collection', value_mapper=None ): + if value_mapper is None: + value_mapper = {} + rval = {} + try: + visible_keys = self.__getattribute__( 'dict_' + view + '_visible_keys' ) + except AttributeError: + raise Exception( 'Unknown API view: %s' % view ) + for key in visible_keys: + try: + rval[ key ] = self.__getattribute__( key ) + if key in value_mapper: + rval[ key ] = value_mapper.get( key, rval[ key ] ) + except AttributeError: + rval[ key ] = None + return rval + + def get_shed_config_filename( self ): + shed_config_filename = None + if self.metadata: + shed_config_filename = self.metadata.get( 'shed_config_filename', shed_config_filename ) + return shed_config_filename + + def get_shed_config_dict( self, app, default=None ): + """ + Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry + in the shed_tool_conf_dict. + """ + if not self.shed_config_filename: + self.guess_shed_config( app, default=default ) + if self.shed_config_filename: + for shed_tool_conf_dict in app.toolbox.shed_tool_confs: + if self.shed_config_filename == shed_tool_conf_dict[ 'config_filename' ]: + return shed_tool_conf_dict + return default + def get_tool_relative_path( self, app ): shed_conf_dict = self.get_shed_config_dict( app ) tool_path = None @@ -3380,14 +3416,7 @@ tool_path = shed_conf_dict[ 'tool_path' ] relative_path = os.path.join( self.tool_shed_path_name, 'repos', self.owner, self.name, self.installed_changeset_revision ) return tool_path, relative_path - def get_shed_config_filename( self ): - shed_config_filename = None - if self.metadata: - shed_config_filename = self.metadata.get( 'shed_config_filename', shed_config_filename ) - return shed_config_filename - def set_shed_config_filename( self, value ): - self.metadata[ 'shed_config_filename' ] = value - shed_config_filename = property( get_shed_config_filename, set_shed_config_filename ) + def guess_shed_config( self, app, default=None ): tool_ids = [] metadata = self.metadata or {} @@ -3423,59 +3452,163 @@ self.shed_config_filename = shed_tool_conf_dict[ 'config_filename' ] return shed_tool_conf_dict return default - def get_shed_config_dict( self, app, default=None ): - """ - Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry - in the shed_tool_conf_dict. - """ - if not self.shed_config_filename: - self.guess_shed_config( app, default=default ) - if self.shed_config_filename: - for shed_tool_conf_dict in app.toolbox.shed_tool_confs: - if self.shed_config_filename == shed_tool_conf_dict[ 'config_filename' ]: - return shed_tool_conf_dict - return default - def dictify( self, view='collection', value_mapper=None ): - if value_mapper is None: - value_mapper = {} - rval = {} - try: - visible_keys = self.__getattribute__( 'dict_' + view + '_visible_keys' ) - except AttributeError: - raise Exception( 'Unknown API view: %s' % view ) - for key in visible_keys: - try: - rval[ key ] = self.__getattribute__( key ) - if key in value_mapper: - rval[ key ] = value_mapper.get( key, rval[ key ] ) - except AttributeError: - rval[ key ] = None - return rval - @property - def can_install( self ): - return self.status == self.installation_status.NEW - @property - def can_reset_metadata( self ): - return self.status == self.installation_status.INSTALLED - @property - def can_uninstall( self ): - return self.status != self.installation_status.UNINSTALLED - @property - def can_deactivate( self ): - return self.status not in [ self.installation_status.DEACTIVATED, self.installation_status.UNINSTALLED ] - @property - def can_reinstall_or_activate( self ): - return self.deleted + @property def has_readme_files( self ): if self.metadata: return 'readme_files' in self.metadata return False + @property def has_repository_dependencies( self ): if self.metadata: return 'repository_dependencies' in self.metadata return False + + @property + def in_error_state( self ): + return self.status == self.installation_status.ERROR + + @property + def includes_data_managers( self ): + if self.metadata: + return bool( len( self.metadata.get( 'data_manager', {} ).get( 'data_managers', {} ) ) ) + return False + + @property + def includes_tools( self ): + if self.metadata: + return 'tools' in self.metadata + return False + + @property + def includes_tools_for_display_in_tool_panel( self ): + if self.includes_tools: + tool_dicts = self.metadata[ 'tools' ] + for tool_dict in tool_dicts: + if tool_dict.get( 'add_to_tool_panel', True ): + return True + return False + + @property + def includes_tool_dependencies( self ): + if self.metadata: + return 'tool_dependencies' in self.metadata + return False + + @property + def includes_workflows( self ): + if self.metadata: + return 'workflows' in self.metadata + return False + + @property + def installed_repository_dependencies( self ): + """Return the repository's repository dependencies that are currently installed.""" + installed_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.INSTALLED: + installed_required_repositories.append( required_repository ) + return installed_required_repositories + + @property + def installed_tool_dependencies( self ): + """Return the repository's tool dependencies that are currently installed.""" + installed_dependencies = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status in [ ToolDependency.installation_status.INSTALLED, ToolDependency.installation_status.ERROR ]: + installed_dependencies.append( tool_dependency ) + return installed_dependencies + + @property + def is_deprecated_in_tool_shed( self ): + if self.tool_shed_status: + return galaxy.util.asbool( self.tool_shed_status.get( 'repository_deprecated', False ) ) + return False + + @property + def is_latest_installable_revision( self ): + if self.tool_shed_status: + return galaxy.util.asbool( self.tool_shed_status.get( 'latest_installable_revision', False ) ) + return False + + @property + def missing_repository_dependencies( self ): + """Return the repository's repository dependencies that are not currently installed, and may not ever have been installed.""" + missing_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status not in [ self.installation_status.INSTALLED ]: + missing_required_repositories.append( required_repository ) + return missing_required_repositories + + @property + def missing_tool_dependencies( self ): + """Return the repository's tool dependencies that are not currently installed, and may not ever have been installed.""" + missing_dependencies = [] + for tool_dependency in self.tool_dependencies: + if tool_dependency.status not in [ ToolDependency.installation_status.INSTALLED ]: + missing_dependencies.append( tool_dependency ) + return missing_dependencies + + def repo_files_directory( self, app ): + repo_path = self.repo_path( app ) + if repo_path: + return os.path.join( repo_path, self.name ) + return None + + def repo_path( self, app ): + tool_shed_url = self.tool_shed + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + tool_shed_url = tool_shed_url.split( ':' )[ 0 ] + tool_shed = tool_shed_url.rstrip( '/' ) + for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): + tool_path = shed_tool_conf_dict[ 'tool_path' ] + relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision ) + if os.path.exists( relative_path ): + return relative_path + return None + + @property + def repository_dependencies( self ): + required_repositories = [] + for rrda in self.required_repositories: + repository_dependency = rrda.repository_dependency + required_repository = repository_dependency.repository + if required_repository: + required_repositories.append( required_repository ) + return required_repositories + + @property + def repository_dependencies_being_installed( self ): + """Return the repository's repository dependencies that are currently being installed.""" + required_repositories_being_installed = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.INSTALLING: + required_repositories_being_installed.append( required_repository ) + return required_repositories_being_installed + + @property + def repository_dependencies_missing_or_being_installed( self ): + """Return the repository's repository dependencies that are either missing or currently being installed.""" + required_repositories_missing_or_being_installed = [] + for required_repository in self.repository_dependencies: + if required_repository.status in [ self.installation_status.ERROR, + self.installation_status.INSTALLING, + self.installation_status.NEVER_INSTALLED, + self.installation_status.UNINSTALLED ]: + required_repositories_missing_or_being_installed.append( required_repository ) + return required_repositories_missing_or_being_installed + + @property + def repository_dependencies_with_installation_errors( self ): + """Return the repository's repository dependencies that have installation errors.""" + required_repositories_with_installation_errors = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.ERROR: + required_repositories_with_installation_errors.append( required_repository ) + return required_repositories_with_installation_errors + @property def requires_prior_installation_of( self ): """ @@ -3501,113 +3634,22 @@ if prior_installation_required: required_rd_tups_that_must_be_installed.append( ( tool_shed, name, owner, changeset_revision, prior_installation_required ) ) return required_rd_tups_that_must_be_installed + @property - def includes_data_managers( self ): - if self.metadata: - return bool( len( self.metadata.get( 'data_manager', {} ).get( 'data_managers', {} ) ) ) + def revision_update_available( self ): + # This method should be named update_available, but since it is no longer possible to drop a table column using migration scripts + # with the sqlite database (see ~/galaxy/model/migrate/versions/0016_drop_update_available_col_add_tool_shed_status_col.py), we + # have to name it in such a way that it will not conflict with the eliminated tool_shed_repository.update_available column (which + # cannot be eliminated if using the sqlite database). + if self.tool_shed_status: + return galaxy.util.asbool( self.tool_shed_status.get( 'revision_update', False ) ) return False - @property - def includes_tools( self ): - if self.metadata: - return 'tools' in self.metadata - return False - @property - def includes_tools_for_display_in_tool_panel( self ): - if self.includes_tools: - tool_dicts = self.metadata[ 'tools' ] - for tool_dict in tool_dicts: - if tool_dict.get( 'add_to_tool_panel', True ): - return True - return False - @property - def includes_tool_dependencies( self ): - if self.metadata: - return 'tool_dependencies' in self.metadata - return False - @property - def includes_workflows( self ): - if self.metadata: - return 'workflows' in self.metadata - return False - @property - def in_error_state( self ): - return self.status == self.installation_status.ERROR - @property - def repository_dependencies( self ): - required_repositories = [] - for rrda in self.required_repositories: - repository_dependency = rrda.repository_dependency - required_repository = repository_dependency.repository - if required_repository: - required_repositories.append( required_repository ) - return required_repositories - @property - def installed_repository_dependencies( self ): - """Return the repository's repository dependencies that are currently installed.""" - installed_required_repositories = [] - for required_repository in self.repository_dependencies: - if required_repository.status == self.installation_status.INSTALLED: - installed_required_repositories.append( required_repository ) - return installed_required_repositories - @property - def missing_repository_dependencies( self ): - """Return the repository's repository dependencies that are not currently installed, and may not ever have been installed.""" - missing_required_repositories = [] - for required_repository in self.repository_dependencies: - if required_repository.status not in [ self.installation_status.INSTALLED ]: - missing_required_repositories.append( required_repository ) - return missing_required_repositories - @property - def repository_dependencies_being_installed( self ): - """Return the repository's repository dependencies that are currently being installed.""" - required_repositories_being_installed = [] - for required_repository in self.repository_dependencies: - if required_repository.status == self.installation_status.INSTALLING: - required_repositories_being_installed.append( required_repository ) - return required_repositories_being_installed - @property - def repository_dependencies_missing_or_being_installed( self ): - """Return the repository's repository dependencies that are either missing or currently being installed.""" - required_repositories_missing_or_being_installed = [] - for required_repository in self.repository_dependencies: - if required_repository.status in [ self.installation_status.ERROR, - self.installation_status.INSTALLING, - self.installation_status.NEVER_INSTALLED, - self.installation_status.UNINSTALLED ]: - required_repositories_missing_or_being_installed.append( required_repository ) - return required_repositories_missing_or_being_installed - @property - def repository_dependencies_with_installation_errors( self ): - """Return the repository's repository dependencies that have installation errors.""" - required_repositories_with_installation_errors = [] - for required_repository in self.repository_dependencies: - if required_repository.status == self.installation_status.ERROR: - required_repositories_with_installation_errors.append( required_repository ) - return required_repositories_with_installation_errors - @property - def uninstalled_repository_dependencies( self ): - """Return the repository's repository dependencies that have been uninstalled.""" - uninstalled_required_repositories = [] - for required_repository in self.repository_dependencies: - if required_repository.status == self.installation_status.UNINSTALLED: - uninstalled_required_repositories.append( required_repository ) - return uninstalled_required_repositories - @property - def installed_tool_dependencies( self ): - """Return the repository's tool dependencies that are currently installed.""" - installed_dependencies = [] - for tool_dependency in self.tool_dependencies: - if tool_dependency.status in [ ToolDependency.installation_status.INSTALLED, ToolDependency.installation_status.ERROR ]: - installed_dependencies.append( tool_dependency ) - return installed_dependencies - @property - def missing_tool_dependencies( self ): - """Return the repository's tool dependencies that are not currently installed, and may not ever have been installed.""" - missing_dependencies = [] - for tool_dependency in self.tool_dependencies: - if tool_dependency.status not in [ ToolDependency.installation_status.INSTALLED ]: - missing_dependencies.append( tool_dependency ) - return missing_dependencies + + def set_shed_config_filename( self, value ): + self.metadata[ 'shed_config_filename' ] = value + + shed_config_filename = property( get_shed_config_filename, set_shed_config_filename ) + @property def tool_dependencies_being_installed( self ): dependencies_being_installed = [] @@ -3615,6 +3657,7 @@ if tool_dependency.status == ToolDependency.installation_status.INSTALLING: dependencies_being_installed.append( tool_dependency ) return dependencies_being_installed + @property def tool_dependencies_missing_or_being_installed( self ): dependencies_missing_or_being_installed = [] @@ -3625,6 +3668,7 @@ ToolDependency.installation_status.UNINSTALLED ]: dependencies_missing_or_being_installed.append( tool_dependency ) return dependencies_missing_or_being_installed + @property def tool_dependencies_with_installation_errors( self ): dependencies_with_installation_errors = [] @@ -3632,6 +3676,24 @@ if tool_dependency.status == ToolDependency.installation_status.ERROR: dependencies_with_installation_errors.append( tool_dependency ) return dependencies_with_installation_errors + + @property + def tool_shed_path_name( self ): + tool_shed_url = self.tool_shed + if tool_shed_url.find( ':' ) > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + tool_shed_url = tool_shed_url.split( ':' )[ 0 ] + return tool_shed_url.rstrip( '/' ) + + @property + def uninstalled_repository_dependencies( self ): + """Return the repository's repository dependencies that have been uninstalled.""" + uninstalled_required_repositories = [] + for required_repository in self.repository_dependencies: + if required_repository.status == self.installation_status.UNINSTALLED: + uninstalled_required_repositories.append( required_repository ) + return uninstalled_required_repositories + @property def uninstalled_tool_dependencies( self ): """Return the repository's tool dependencies that have been uninstalled.""" @@ -3641,11 +3703,22 @@ uninstalled_tool_dependencies.append( tool_dependency ) return uninstalled_tool_dependencies + @property + def upgrade_available( self ): + if self.tool_shed_status: + if self.is_deprecated_in_tool_shed: + # Only allow revision upgrades if the repository is not deprecated in the tool shed. + return False + return galaxy.util.asbool( self.tool_shed_status.get( 'revision_upgrade', False ) ) + return False + + class RepositoryRepositoryDependencyAssociation( object ): def __init__( self, tool_shed_repository_id=None, repository_dependency_id=None ): self.tool_shed_repository_id = tool_shed_repository_id self.repository_dependency_id = repository_dependency_id + class RepositoryDependency( object ): def __init__( self, tool_shed_repository_id=None ): self.tool_shed_repository_id = tool_shed_repository_id diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/galaxy/model/mapping.py --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -405,7 +405,7 @@ Column( "ctx_rev", TrimmedString( 10 ) ), Column( "metadata", JSONType, nullable=True ), Column( "includes_datatypes", Boolean, index=True, default=False ), - Column( "update_available", Boolean, default=False ), + Column( "tool_shed_status", JSONType, nullable=True ), Column( "deleted", Boolean, index=True, default=False ), Column( "uninstalled", Boolean, default=False ), Column( "dist_to_shed", Boolean, default=False ), diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/galaxy/model/migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py @@ -0,0 +1,77 @@ +""" +Migration script to drop the update_available Boolean column and replace it with the tool_shed_status JSONType column in the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +import sys, logging +from galaxy.model.custom_types import * +from sqlalchemy.exc import * +import datetime +now = datetime.datetime.utcnow + +log = logging.getLogger( __name__ ) +log.setLevel( logging.DEBUG ) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData() + +def default_false( migrate_engine ): + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + return "0" + elif migrate_engine.name in [ 'postgresql', 'postgres' ]: + return "false" + +def upgrade( migrate_engine ): + metadata.bind = migrate_engine + print __doc__ + metadata.reflect() + try: + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + except NoSuchTableError: + ToolShedRepository_table = None + log.debug( "Failed loading table tool_shed_repository" ) + if ToolShedRepository_table is not None: + # For some unknown reason it is no longer possible to drop a column in a migration script if using the sqlite database. + if migrate_engine.name != 'sqlite': + try: + col = ToolShedRepository_table.c.update_available + col.drop() + except Exception, e: + print "Dropping column update_available from the tool_shed_repository table failed: %s" % str( e ) + c = Column( "tool_shed_status", JSONType, nullable=True ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.tool_shed_status + except Exception, e: + print "Adding tool_shed_status column to the tool_shed_repository table failed: %s" % str( e ) + +def downgrade( migrate_engine ): + metadata.bind = migrate_engine + metadata.reflect() + try: + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + except NoSuchTableError: + ToolShedRepository_table = None + log.debug( "Failed loading table tool_shed_repository" ) + if ToolShedRepository_table is not None: + # For some unknown reason it is no longer possible to drop a column in a migration script if using the sqlite database. + if migrate_engine.name != 'sqlite': + try: + col = ToolShedRepository_table.c.tool_shed_status + col.drop() + except Exception, e: + print "Dropping column tool_shed_status from the tool_shed_repository table failed: %s" % str( e ) + c = Column( "update_available", Boolean, default=False ) + try: + c.create( ToolShedRepository_table ) + assert c is ToolShedRepository_table.c.update_available + migrate_engine.execute( "UPDATE tool_shed_repository SET update_available=%s" % default_false( migrate_engine ) ) + except Exception, e: + print "Adding column update_available to the tool_shed_repository table failed: %s" % str( e ) diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/galaxy/web/framework/helpers/grids.py --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -39,6 +39,7 @@ cur_filter_pref_name = ".filter" cur_sort_key_pref_name = ".sort_key" pass_through_operations = {} + legend = None def __init__( self ): # Determine if any multiple row operations are defined self.has_multiple_item_operations = False diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -94,6 +94,10 @@ return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='check_for_updates', **kwd ) ) + if operation == "update tool shed status": + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='update_tool_shed_status_for_installed_repository', + **kwd ) ) if operation == "reset to install": kwd[ 'reset_repository' ] = True return trans.response.send_redirect( web.url_for( controller='admin_toolshed', @@ -146,8 +150,6 @@ return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='deactivate_or_uninstall_repository', **kwd ) ) - if 'message' not in kwd or not kwd[ 'message' ]: - kwd[ 'message' ] = 'Names of repositories for which updates are available are highlighted in yellow.' return self.installed_repository_grid( trans, **kwd ) @web.expose @@ -1627,10 +1629,15 @@ updating_installed_repository=True, persist=True ) repository.metadata = metadata_dict - # Update the repository changeset_revision in the database. + # Update the repository.changeset_revision column in the database. repository.changeset_revision = latest_changeset_revision repository.ctx_rev = latest_ctx_rev - repository.update_available = False + # Update the repository.tool_shed_status column in the database. + tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( trans.app, repository ) + if tool_shed_status_dict: + repository.tool_shed_status = tool_shed_status_dict + else: + repository.tool_shed_status = None trans.sa_session.add( repository ) trans.sa_session.flush() if 'tools' in metadata_dict: @@ -1678,6 +1685,46 @@ @web.expose @web.require_admin + def update_tool_shed_status_for_installed_repository( self, trans, all_installed_repositories=False, **kwd ): + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + if all_installed_repositories: + success_count = 0 + repository_names_not_updated = [] + updated_count = 0 + for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.deleted == False ): + ok, updated = suc.check_or_update_tool_shed_status_for_installed_repository( trans, repository ) + if ok: + success_count += 1 + else: + repository_names_not_updated.append( '<b>%s</b>' % str( repository.name ) ) + if updated: + updated_count += 1 + message = "Checked the status in the tool shed for %d repositories. " % success_count + message += "Updated the tool shed status for %d repositories. " % updated_count + if repository_names_not_updated: + message += "Unable to retrieve status from the tool shed for the following repositories:\n" + message += ", ".join( repository_names_not_updated ) + else: + repository_id = kwd.get( 'id', None ) + repository = suc.get_tool_shed_repository_by_id( trans, repository_id ) + ok, updated = suc.check_or_update_tool_shed_status_for_installed_repository( trans, repository ) + if ok: + if updated: + message = "The tool shed status for repository <b>%s</b> has been updated." % str( repository.name ) + else: + message = "The status has not changed in the tool shed for repository <b>%s</b>." % str( repository.name ) + else: + message = "Unable to retrieve status from the tool shed for repository <b>%s</b>." % str( repository.name ) + status = 'error' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) + + @web.expose + @web.require_admin def view_tool_metadata( self, trans, repository_id, tool_id, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2651,6 +2651,59 @@ owner=owner ) ) @web.expose + def status_for_installed_repository( self, trans, **kwd ): + """ + Handle a request from a local Galaxy instance, returning a dictionary with boolean values for whether there are updates available + for the repository revision, newer installable revisions available, the revision is the latest installable revision, or if the repository + is deprecated. + """ + name = kwd.get( 'name', None ) + owner = kwd.get( 'owner', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + if repository: + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) + tool_shed_status_dict = {} + # Handle repository deprecation. + tool_shed_status_dict[ 'repository_deprecated' ] = str( repository.deprecated ) + # Handle latest installable revision. + if changeset_revision == repository.tip( trans.app ): + tool_shed_status_dict[ 'latest_installable_revision' ] = 'True' + else: + next_installable_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + if next_installable_revision: + tool_shed_status_dict[ 'latest_installable_revision' ] = 'False' + else: + tool_shed_status_dict[ 'latest_installable_revision' ] = 'True' + # Handle revision updates. + if changeset_revision == repository.tip( trans.app ): + tool_shed_status_dict[ 'revision_update' ] = 'False' + else: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + trans.security.encode_id( repository.id ), + changeset_revision ) + if repository_metadata: + tool_shed_status_dict[ 'revision_update' ] = 'False' + else: + tool_shed_status_dict[ 'revision_update' ] = 'True' + # Handle revision upgrades. + ordered_metadata_changeset_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True ) + num_metadata_revisions = len( ordered_metadata_changeset_revisions ) + for index, metadata_changeset_revision in enumerate( ordered_metadata_changeset_revisions ): + if index == num_metadata_revisions: + tool_shed_status_dict[ 'revision_upgrade' ] = 'False' + break + if metadata_changeset_revision == changeset_revision: + if num_metadata_revisions - index > 0: + tool_shed_status_dict[ 'revision_upgrade' ] = 'True' + else: + tool_shed_status_dict[ 'revision_upgrade' ] = 'False' + break + return encoding_util.tool_shed_encode( tool_shed_status_dict ) + return encoding_util.tool_shed_encode( {} ) + + @web.expose def updated_changeset_revisions( self, trans, **kwd ): """ Handle a request from a local Galaxy instance to retrieve the list of changeset revisions to which an installed repository can be updated. This diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py --- a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py +++ b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py @@ -3,38 +3,90 @@ from galaxy import model, util from galaxy.web.framework.helpers import iff, grids from galaxy.model.orm import or_ +import tool_shed.util.shed_util_common as suc from tool_shed.util import tool_dependency_util log = logging.getLogger( __name__ ) +def generate_deprecated_repository_img_str( include_mouse_over=False ): + if include_mouse_over: + deprecated_tip_str = 'class="icon-button tooltip" title="This repository is deprecated in the Tool Shed"' + else: + deprecated_tip_str = '' + return '<img src="/static/images/icon_error_sml.gif" %s/>' % deprecated_tip_str + +def generate_latest_revision_img_str( include_mouse_over=False ): + if include_mouse_over: + latest_revision_tip_str = 'class="icon-button tooltip" title="This is the latest installable revision of this repository"' + else: + latest_revision_tip_str = '' + return '<img src="/static/june_2007_style/blue/ok_small.png" %s/>' % latest_revision_tip_str + +def generate_revision_updates_img_str( include_mouse_over=False ): + if include_mouse_over: + revision_updates_tip_str = 'class="icon-button tooltip" title="Updates are available in the Tool Shed for this revision"' + else: + revision_updates_tip_str = '' + return '<img src="/static/images/icon_warning_sml.gif" %s/>' % revision_updates_tip_str + +def generate_revision_upgrades_img_str( include_mouse_over=False ): + if include_mouse_over: + revision_upgrades_tip_str = 'class="icon-button tooltip" title="A newer installable revision is available for this repository"' + else: + revision_upgrades_tip_str = '' + return '<img src="/static/images/up.gif" %s/>' % revision_upgrades_tip_str + +def generate_unknown_img_str( include_mouse_over=False ): + if include_mouse_over: + unknown_tip_str = 'class="icon-button tooltip" title="Unable to get information from the Tool Shed"' + else: + unknown_tip_str = '' + return '<img src="/static/june_2007_style/blue/question-octagon-frame.png" %s/>' % unknown_tip_str + class InstalledRepositoryGrid( grids.Grid ): + class ToolShedStatusColumn( grids.TextColumn ): + + def get_value( self, trans, grid, tool_shed_repository ): + if tool_shed_repository.tool_shed_status: + tool_shed_status_str = '' + if tool_shed_repository.is_deprecated_in_tool_shed: + tool_shed_status_str += generate_deprecated_repository_img_str( include_mouse_over=True ) + if tool_shed_repository.is_latest_installable_revision: + tool_shed_status_str += generate_latest_revision_img_str( include_mouse_over=True ) + if tool_shed_repository.revision_update_available: + tool_shed_status_str += generate_revision_updates_img_str( include_mouse_over=True ) + if tool_shed_repository.upgrade_available: + tool_shed_status_str += generate_revision_upgrades_img_str( include_mouse_over=True ) + else: + tool_shed_status_str = generate_unknown_img_str( include_mouse_over=True ) + return tool_shed_status_str + + class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): - if tool_shed_repository.update_available: - return '<div class="count-box state-color-running">%s</div>' % tool_shed_repository.name - return tool_shed_repository.name + return str( tool_shed_repository.name ) class DescriptionColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): - return tool_shed_repository.description + return util.unicodify( tool_shed_repository.description ) class OwnerColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): - return tool_shed_repository.owner + return str( tool_shed_repository.owner ) class RevisionColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): - return tool_shed_repository.changeset_revision + return str( tool_shed_repository.changeset_revision ) class StatusColumn( grids.TextColumn ): @@ -91,6 +143,8 @@ template='/admin/tool_shed_repository/grid.mako' default_sort_key = "name" columns = [ + ToolShedStatusColumn( "", + attach_popup=False ), NameColumn( "Name", key="name", link=( lambda item: iff( item.status in [ model.ToolShedRepository.installation_status.CLONING ], @@ -114,10 +168,18 @@ key="free-text-search", visible=False, filterable="standard" ) ) - global_actions = [] - operations = [ grids.GridOperation( "Get updates", + global_actions = [ + grids.GridAction( "Update tool shed status", + dict( controller='admin_toolshed', action='update_tool_shed_status_for_installed_repository', all_installed_repositories=True ) ) + ] + operations = [ grids.GridOperation( "Update tool shed status", allow_multiple=False, - condition=( lambda item: not item.deleted and item.status not in \ + condition=( lambda item: not item.deleted ), + async_compatible=False, + url_args=dict( controller='admin_toolshed', action='browse_repositories', operation='update tool shed status' ) ), + grids.GridOperation( "Get updates", + allow_multiple=False, + condition=( lambda item: not item.deleted and item.revision_update_available and item.status not in \ [ model.ToolShedRepository.installation_status.ERROR, model.ToolShedRepository.installation_status.NEW ] ), async_compatible=False, url_args=dict( controller='admin_toolshed', action='browse_repositories', operation='get updates' ) ), @@ -151,6 +213,15 @@ def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( self.model_class ) + @property + def legend( self ): + legend_str = '%s  Updates are available in the Tool Shed for this revision<br/>' % generate_revision_updates_img_str() + legend_str += '%s  A newer installable revision is available for this repository<br/>' % generate_revision_upgrades_img_str() + legend_str += '%s  This is the latest installable revision of this repository<br/>' % generate_latest_revision_img_str() + legend_str += '%s  This repository is deprecated in the Tool Shed<br/>' % generate_deprecated_repository_img_str() + legend_str += '%s  Unable to get information from the Tool Shed<br/>' % generate_unknown_img_str() + return legend_str + class RepositoryInstallationGrid( grids.Grid ): diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -14,8 +14,15 @@ def clean_tool_shed_url( base_url ): if base_url: - protocol, base = base_url.split( '://' ) - return base.rstrip( '/' ) + if base_url.find( '://' ) > -1: + try: + protocol, base = base_url.split( '://' ) + except ValueError, e: + # The received base_url must be an invalid url. + log.debug( "Returning unchanged invalid base_url from td_common_util.clean_tool_shed_url: %s" % str( base_url ) ) + return base_url + return base.rstrip( '/' ) + return base_url.rstrip( '/' ) return base_url def create_env_var_dict( elem, tool_dependency_install_dir=None, tool_shed_repository_install_dir=None ): diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/tool_shed/galaxy_install/update_manager.py --- a/lib/tool_shed/galaxy_install/update_manager.py +++ b/lib/tool_shed/galaxy_install/update_manager.py @@ -26,29 +26,24 @@ def __restarter( self ): log.info( 'Update manager restarter starting up...' ) while self.running: - flush_needed = False + # Make a call to the tool shed for each installed repository to get the latest status information in the tool shed for the + # repository. This information includes items like newer installable repository revisions, current revision updates, whether + # the repository revision is the latest installable revision, and whether the repository has been deprecated in the tool shed. for repository in self.sa_session.query( self.app.model.ToolShedRepository ) \ - .filter( and_( self.app.model.ToolShedRepository.table.c.update_available == False, - self.app.model.ToolShedRepository.table.c.deleted == False ) ): - if self.check_for_update( repository ): - repository.update_available = True - self.sa_session.add( repository ) - flush_needed = True - if flush_needed: - self.sa_session.flush() + .filter( self.app.model.ToolShedRepository.table.c.deleted == False ): + tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, repository ) + if tool_shed_status_dict: + if tool_shed_status_dict != repository.tool_shed_status: + repository.tool_shed_status = tool_shed_status_dict + self.sa_session.flush() + else: + # The received tool_shed_status_dict is an empty dictionary, so coerce to None. + tool_shed_status_dict = None + if tool_shed_status_dict != repository.tool_shed_status: + repository.tool_shed_status = tool_shed_status_dict + self.sa_session.flush() self.sleeper.sleep( self.seconds_to_sleep ) - log.info( 'Transfer job restarter shutting down...' ) - - def check_for_update( self, repository ): - tool_shed_url = suc.get_url_from_tool_shed( self.app, repository.tool_shed ) - url = '%s/repository/check_for_updates?name=%s&owner=%s&changeset_revision=%s&from_update_manager=True' % \ - ( tool_shed_url, repository.name, repository.owner, repository.changeset_revision ) - try: - text = common_util.tool_shed_get( self.app, tool_shed_url, url ) - except Exception, e: - # The required tool shed may be unavailable. - text = 'False' - return string_as_bool( text ) + log.info( 'Update manager restarter shutting down...' ) def shutdown( self ): self.running = False diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/tool_shed/util/repository_dependency_util.py --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -182,7 +182,7 @@ debug_msg = "Resetting tool_shed_repository '%s' for installation.\n" % str( installed_tool_shed_repository.name ) debug_msg += "The current state of the tool_shed_repository is:\n" debug_msg += "deleted: %s\n" % str( installed_tool_shed_repository.deleted ) - debug_msg += "update_available: %s\n" % str( installed_tool_shed_repository.update_available ) + debug_msg += "tool_shed_status: %s\n" % str( installed_tool_shed_repository.tool_shed_status ) debug_msg += "uninstalled: %s\n" % str( installed_tool_shed_repository.uninstalled ) debug_msg += "status: %s\n" % str( installed_tool_shed_repository.status ) debug_msg += "error_message: %s\n" % str( installed_tool_shed_repository.error_message ) diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 lib/tool_shed/util/shed_util_common.py --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -17,6 +17,7 @@ from galaxy.model.orm import or_ import sqlalchemy.orm.exc from tool_shed.util import common_util +from tool_shed.util import encoding_util from tool_shed.util import xml_util from xml.etree import ElementTree as XmlET from galaxy import eggs @@ -30,7 +31,7 @@ eggs.require( 'markupsafe' ) import markupsafe - + log = logging.getLogger( __name__ ) CHUNK_SIZE = 2**20 # 1Mb @@ -171,6 +172,20 @@ return True return False +def check_or_update_tool_shed_status_for_installed_repository( trans, repository ): + updated = False + tool_shed_status_dict = get_tool_shed_status_for_installed_repository( trans.app, repository ) + if tool_shed_status_dict: + ok = True + if tool_shed_status_dict != repository.tool_shed_status: + repository.tool_shed_status = tool_shed_status_dict + trans.sa_session.add( repository ) + trans.sa_session.flush() + updated = True + else: + ok = False + return ok, updated + def clean_repository_clone_url( repository_clone_url ): """Return a URL that can be used to clone a tool shed repository, eliminating the protocol and user if either exists.""" if repository_clone_url.find( '@' ) > 0: @@ -194,7 +209,7 @@ return tool_shed_url.split( ':' )[ 0 ] return tool_shed_url.rstrip( '/' ) -def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): +def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" try: commands.clone( get_configured_ui(), @@ -248,7 +263,7 @@ # was later uninstalled, this value should be received as the value of that change set to which the repository had been updated just prior # to it being uninstalled. current_changeset_revision = installed_changeset_revision - sa_session = app.model.context.current + sa_session = app.model.context.current tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) if not owner: owner = get_repository_owner_from_clone_url( repository_clone_url ) @@ -322,7 +337,6 @@ elem = XmlET.Element( 'tool_shed_repository' ) else: elem = XmlET.SubElement( parent_elem, 'tool_shed_repository' ) - tool_shed_elem = XmlET.SubElement( elem, 'tool_shed' ) tool_shed_elem.text = tool_shed repository_name_elem = XmlET.SubElement( elem, 'repository_name' ) @@ -337,10 +351,14 @@ new_elem = XmlET.SubElement( elem, key ) new_elem.text = value return elem - + def generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=None, **kwd ): - return generate_repository_info_elem( tool_shed_repository.tool_shed, tool_shed_repository.name, tool_shed_repository.installed_changeset_revision, tool_shed_repository.owner, parent_elem=parent_elem, **kwd ) - + return generate_repository_info_elem( tool_shed_repository.tool_shed, + tool_shed_repository.name, + tool_shed_repository.installed_changeset_revision, + tool_shed_repository.owner, + parent_elem=parent_elem, + **kwd ) def generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" @@ -376,7 +394,7 @@ def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for - the tool in the Galaxy tool shed from which it is being installed. The form of the guid is + the tool in the Galaxy tool shed from which it is being installed. The form of the guid is <tool shed host>/repos/<repository owner>/<repository name>/<tool id>/<tool version> """ tmp_url = clean_repository_clone_url( repository_clone_url ) @@ -399,7 +417,7 @@ tool_config = tool_dict[ 'tool_config' ] file_name = strip_path( tool_config ) guids_and_configs[ guid ] = file_name - # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. + # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. tree, error_message = xml_util.parse_xml( shed_tool_conf ) if tree is None: return tool_panel_dict @@ -517,7 +535,7 @@ Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the combination of a name, owner and changeset revision. """ - url = url_join( tool_shed_url, + url = url_join( tool_shed_url, 'repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % ( name, owner, changeset_revision ) ) ctx_rev = common_util.tool_shed_get( app, tool_shed_url, url ) return ctx_rev @@ -1105,6 +1123,10 @@ return shed_tool_conf_dict[ 'tool_path' ] return None +def get_tool_shed_from_clone_url( repository_clone_url ): + tmp_url = clean_repository_clone_url( repository_clone_url ) + return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) + def get_tool_shed_repository_by_id( trans, repository_id ): """Return a tool shed repository database record defined by the id.""" # This method is used only in Galaxy, not the tool shed. @@ -1140,9 +1162,23 @@ app.model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ .first() -def get_tool_shed_from_clone_url( repository_clone_url ): - tmp_url = clean_repository_clone_url( repository_clone_url ) - return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) +def get_tool_shed_status_for_installed_repository( app, repository ): + """ + Send a request to the tool shed to retrieve information about newer installable repository revisions, current revision updates, + whether the repository revision is the latest downloadable revision, and whether the repository has been deprecated in the tool shed. + The received repository is a ToolShedRepository object from Galaxy. + """ + tool_shed_url = get_url_from_tool_shed( app, repository.tool_shed ) + url = url_join( tool_shed_url, + 'repository/status_for_installed_repository?name=%s&owner=%s&changeset_revision=%s' % \ + ( repository.name, repository.owner, repository.changeset_revision ) ) + try: + encoded_tool_shed_status_dict = common_util.tool_shed_get( app, tool_shed_url, url ) + tool_shed_status_dict = encoding_util.tool_shed_decode( encoded_tool_shed_status_dict ) + except Exception, e: + log.exception( "Error attemtping to get tool shed status for installed repository %s: %s" % ( str( repository.name ), str( e ) ) ) + return {} + return tool_shed_status_dict def get_url_from_tool_shed( app, tool_shed ): """ @@ -1349,11 +1385,11 @@ def reset_previously_installed_repository( trans, repository ): """ - Reset the atrributes of a tool_shed_repository that was previsouly installed. The repository will be in some state other than with a - status of INSTALLED, so all atributes will be set to the default NEW state. This will enable the repository to be freshly installed. + Reset the atrributes of a tool_shed_repository that was previsouly installed. The repository will be in some state other than with a + status of INSTALLED, so all atributes will be set to the default (NEW( state. This will enable the repository to be freshly installed. """ repository.deleted = False - repository.update_available = False + repository.tool_shed_status = None repository.uninstalled = False repository.status = trans.model.ToolShedRepository.installation_status.NEW repository.error_message = None diff -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 -r 8f6f926f912e2aea3f11905498fd5f04f857ec97 templates/grid_base.mako --- a/templates/grid_base.mako +++ b/templates/grid_base.mako @@ -454,5 +454,12 @@ </td></tr> %endif + %if grid.legend: + <tr> + <td colspan="100"> + ${grid.legend} + </td> + </tr> + %endif </%def> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Strip trailing whitespace (and windows line endings) from all python files in lib
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/be3b0358acc9/ Changeset: be3b0358acc9 User: dannon Date: 2013-08-30 05:39:52 Summary: Strip trailing whitespace (and windows line endings) from all python files in lib Affected #: 256 files diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -186,6 +186,6 @@ def configure_fluent_log( self ): if self.config.fluent_log: from galaxy.util.log.fluent_log import FluentTraceLogger - self.trace_logger = FluentTraceLogger( 'galaxy', self.config.fluent_host, self.config.fluent_port ) + self.trace_logger = FluentTraceLogger( 'galaxy', self.config.fluent_host, self.config.fluent_port ) else: self.trace_logger = None diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -282,7 +282,7 @@ self.biostar_url = kwargs.get( 'biostar_url', None ) self.biostar_key_name = kwargs.get( 'biostar_key_name', None ) self.biostar_key = kwargs.get( 'biostar_key', None ) - # Experimental: This will not be enabled by default and will hide + # Experimental: This will not be enabled by default and will hide # nonproduction code. # The api_folders refers to whether the API exposes the /folders section. self.api_folders = string_as_bool( kwargs.get( 'api_folders', False ) ) @@ -302,7 +302,7 @@ @property def sentry_dsn_public( self ): """ - Sentry URL with private key removed for use in client side scripts, + Sentry URL with private key removed for use in client side scripts, sentry server will need to be configured to accept events """ if self.sentry_dsn: @@ -436,8 +436,8 @@ """ # Get root logger root = logging.getLogger() - # PasteScript will have already configured the logger if the - # 'loggers' section was found in the config file, otherwise we do + # PasteScript will have already configured the logger if the + # 'loggers' section was found in the config file, otherwise we do # some simple setup using the 'log_*' values from the config. if not config.global_conf_parser.has_section( "loggers" ): format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" ) diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/assembly.py --- a/lib/galaxy/datatypes/assembly.py +++ b/lib/galaxy/datatypes/assembly.py @@ -168,7 +168,7 @@ def regenerate_primary_file(self,dataset): """ - cannot do this until we are setting metadata + cannot do this until we are setting metadata """ log.debug( "Velvet log info %s" % 'JJ regenerate_primary_file') gen_msg = '' diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -203,7 +203,7 @@ stderr = open( stderr_name ).read().strip() if stderr: if exit_code != 0: - shutil.rmtree( tmp_dir) #clean up + shutil.rmtree( tmp_dir) #clean up raise Exception, "Error Grooming BAM file contents: %s" % stderr else: print stderr @@ -231,7 +231,7 @@ stderr = open( stderr_name ).read().strip() if stderr: if exit_code != 0: - os.unlink( stderr_name ) #clean up + os.unlink( stderr_name ) #clean up raise Exception, "Error Setting BAM Metadata: %s" % stderr else: print stderr @@ -240,7 +240,7 @@ os.unlink( stderr_name ) def sniff( self, filename ): # BAM is compressed in the BGZF format, and must not be uncompressed in Galaxy. - # The first 4 bytes of any bam file is 'BAM\1', and the file is binary. + # The first 4 bytes of any bam file is 'BAM\1', and the file is binary. try: header = gzip.open( filename ).read(4) if binascii.b2a_hex( header ) == binascii.hexlify( 'BAM\1' ): @@ -250,7 +250,7 @@ return False def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: - dataset.peek = "Binary bam alignments file" + dataset.peek = "Binary bam alignments file" dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' @@ -278,7 +278,7 @@ samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset ) settings[ 'comment_char' ] = '@' return dataproviders.line.RegexLineDataProvider( samtools_source, **settings ) - + @dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings ) def column_dataprovider( self, dataset, **settings ): samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset ) @@ -352,7 +352,7 @@ def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: - dataset.peek = "Binary h5 file" + dataset.peek = "Binary h5 file" dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' @@ -372,7 +372,7 @@ def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: - dataset.peek = "Binary scf sequence file" + dataset.peek = "Binary scf sequence file" dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' @@ -404,7 +404,7 @@ return False def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: - dataset.peek = "Binary sff file" + dataset.peek = "Binary sff file" dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' @@ -451,7 +451,7 @@ return dataset.peek except: return "Binary UCSC %s file (%s)" % ( self._name, data.nice_size( dataset.get_size() ) ) - + Binary.register_sniffable_binary_format("bigwig", "bigwig", BigWig) @@ -470,9 +470,9 @@ class TwoBit (Binary): """Class describing a TwoBit format nucleotide file""" - + file_ext = "twobit" - + def sniff(self, filename): try: # All twobit files start with a 16-byte header. If the file is smaller than 16 bytes, it's obviously not a valid twobit file. diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py --- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py @@ -23,13 +23,13 @@ if not line: raise StopIteration() if line.isspace(): - continue + continue if line[0] == "#": continue if line[0].isalpha(): if line.startswith( "track" ) or line.startswith( "browser" ): continue - + feature = line.strip().split() chrom = feature[0] chrom_start = int(feature[1]) @@ -37,19 +37,19 @@ score = float(feature[3]) return chrom, chrom_start, chrom_end, None, score def main(): - + input_fname = sys.argv[1] out_fname = sys.argv[2] - + reader = BedGraphReader( open( input_fname ) ) - + # Fill array from reader d = array_tree_dict_from_reader( reader, {}, block_size = BLOCK_SIZE ) - + for array_tree in d.itervalues(): array_tree.root.build_summary() - + FileArrayTreeDict.dict_to_file( d, open( out_fname, "w" ) ) -if __name__ == "__main__": +if __name__ == "__main__": main() \ No newline at end of file diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/bgzip.py --- a/lib/galaxy/datatypes/converters/bgzip.py +++ b/lib/galaxy/datatypes/converters/bgzip.py @@ -19,13 +19,13 @@ parser.add_option( '-P', '--preset', dest='preset' ) (options, args) = parser.parse_args() input_fname, output_fname = args - + tmpfile = tempfile.NamedTemporaryFile() sort_params = None - + if options.chrom_col and options.start_col and options.end_col: - sort_params = ["sort", - "-k%(i)s,%(i)s" % { 'i': options.chrom_col }, + sort_params = ["sort", + "-k%(i)s,%(i)s" % { 'i': options.chrom_col }, "-k%(i)i,%(i)in" % { 'i': options.start_col }, "-k%(i)i,%(i)in" % { 'i': options.end_col } ] @@ -40,9 +40,8 @@ after_sort = subprocess.Popen(sort_params, stdin=grepped.stdout, stderr=subprocess.PIPE, stdout=tmpfile ) grepped.stdout.close() output, err = after_sort.communicate() - + ctabix.tabix_compress(tmpfile.name, output_fname, force=True) - -if __name__ == "__main__": + +if __name__ == "__main__": main() - \ No newline at end of file diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/fasta_to_len.py --- a/lib/galaxy/datatypes/converters/fasta_to_len.py +++ b/lib/galaxy/datatypes/converters/fasta_to_len.py @@ -10,7 +10,7 @@ assert sys.version_info[:2] >= ( 2, 4 ) def compute_fasta_length( fasta_file, out_file, keep_first_char, keep_first_word=False ): - + infile = fasta_file out = open( out_file, 'w') keep_first_char = int( keep_first_char ) diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/fastq_to_fqtoc.py --- a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py +++ b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py @@ -38,11 +38,11 @@ chunk_end = in_file.tell() out_file.write('{"start":"%s","end":"%s","sequences":"%s"},' % (chunk_begin, chunk_end, sequences)) chunk_begin = chunk_end - + chunk_end = in_file.tell() out_file.write('{"start":"%s","end":"%s","sequences":"%s"}' % (chunk_begin, chunk_end, (current_line % lines_per_chunk) / 4)) out_file.write(']}\n') - -if __name__ == "__main__": + +if __name__ == "__main__": main() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py --- a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py @@ -7,7 +7,7 @@ 1st line: @title_of_seq 2nd line: nucleotides 3rd line: +title_of_qualityscore (might be skipped) -4th line: quality scores +4th line: quality scores (in three forms: a. digits, b. ASCII codes, the first char as the coding base, c. ASCII codes without the first char.) Usage: @@ -52,4 +52,4 @@ outfile.close() -if __name__ == "__main__": __main__() \ No newline at end of file +if __name__ == "__main__": __main__() \ No newline at end of file diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py --- a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py +++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py @@ -7,7 +7,7 @@ 1st line: @title_of_seq 2nd line: nucleotides 3rd line: +title_of_qualityscore (might be skipped) -4th line: quality scores +4th line: quality scores (in three forms: a. digits, b. ASCII codes, the first char as the coding base, c. ASCII codes without the first char.) Usage: @@ -30,7 +30,7 @@ seq_title_startswith = '' default_coding_value = 64 fastq_block_lines = 0 - + for i, line in enumerate( file( infile_name ) ): line = line.rstrip() if not line or line.startswith( '#' ): @@ -52,7 +52,7 @@ if not qual_title_startswith: qual_title_startswith = line_startswith if line_startswith != qual_title_startswith: - stop_err( 'Invalid fastqsolexa format at line %d: %s.' % ( i + 1, line ) ) + stop_err( 'Invalid fastqsolexa format at line %d: %s.' % ( i + 1, line ) ) quality_title = line[1:] if quality_title and read_title != quality_title: stop_err( 'Invalid fastqsolexa format at line %d: sequence title "%s" differes from score title "%s".' % ( i + 1, read_title, quality_title ) ) @@ -67,15 +67,15 @@ # peek: ascii or digits? val = line.split()[0] - try: + try: check = int( val ) fastq_integer = True except: fastq_integer = False - + if fastq_integer: # digits qual = line - else: + else: # ascii quality_score_length = len( line ) if quality_score_length == read_length + 1: @@ -89,8 +89,7 @@ score = ord( char ) - quality_score_startswith # 64 qual = "%s%s " % ( qual, str( score ) ) outfile_score.write( '%s\n' % qual ) - + outfile_score.close() -if __name__ == "__main__": __main__() - \ No newline at end of file +if __name__ == "__main__": __main__() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py --- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py @@ -18,23 +18,22 @@ def main(): # Arguments input_fname, out_fname = sys.argv[1:] - + # Do conversion. index = Indexes() offset = 0 reader_wrapper = GFFReaderWrapper( fileinput.FileInput( input_fname ), fix_strand=True ) - for feature in list( reader_wrapper ): + for feature in list( reader_wrapper ): # Add feature; index expects BED coordinates. if isinstance( feature, GenomicInterval ): convert_gff_coords_to_bed( feature ) index.add( feature.chrom, feature.start, feature.end, offset ) - + # Always increment offset, even if feature is not an interval and hence # not included in the index. offset += feature.raw_size index.write( open(out_fname, "w") ) - -if __name__ == "__main__": + +if __name__ == "__main__": main() - \ No newline at end of file diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/interval_to_bed_converter.py --- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py @@ -1,62 +1,62 @@ -#!/usr/bin/env python -#Dan Blankenberg - -import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -import bx.intervals.io - -assert sys.version_info[:2] >= ( 2, 4 ) - -def stop_err( msg ): - sys.stderr.write( msg ) - sys.exit() - -def __main__(): - output_name = sys.argv[1] - input_name = sys.argv[2] - try: - chromCol = int( sys.argv[3] ) - 1 - except: - stop_err( "'%s' is an invalid chrom column, correct the column settings before attempting to convert the data format." % str( sys.argv[3] ) ) - try: - startCol = int( sys.argv[4] ) - 1 - except: - stop_err( "'%s' is an invalid start column, correct the column settings before attempting to convert the data format." % str( sys.argv[4] ) ) - try: - endCol = int( sys.argv[5] ) - 1 - except: - stop_err( "'%s' is an invalid end column, correct the column settings before attempting to convert the data format." % str( sys.argv[5] ) ) - try: - strandCol = int( sys.argv[6] ) - 1 - except: - strandCol = -1 - try: - nameCol = int( sys.argv[7] ) - 1 - except: - nameCol = -1 - skipped_lines = 0 - first_skipped_line = 0 - out = open( output_name,'w' ) - count = 0 - for count, region in enumerate( bx.intervals.io.NiceReaderWrapper( open( input_name, 'r' ), chrom_col=chromCol, start_col=startCol, end_col=endCol, strand_col=strandCol, fix_strand=True, return_header=False, return_comments=False ) ): - try: - if nameCol >= 0: - name = region.fields[nameCol] - else: - raise IndexError - except: - name = "region_%i" % count - try: - - out.write( "%s\t%i\t%i\t%s\t%i\t%s\n" % ( region.chrom, region.start, region.end, name, 0, region.strand ) ) - except: - skipped_lines += 1 - if not first_skipped_line: - first_skipped_line = count + 1 - out.close() - print "%i regions converted to BED." % ( count + 1 - skipped_lines ) - if skipped_lines > 0: - print "Skipped %d blank or invalid lines starting with line # %d." % ( skipped_lines, first_skipped_line ) - -if __name__ == "__main__": __main__() +#!/usr/bin/env python +#Dan Blankenberg + +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +import bx.intervals.io + +assert sys.version_info[:2] >= ( 2, 4 ) + +def stop_err( msg ): + sys.stderr.write( msg ) + sys.exit() + +def __main__(): + output_name = sys.argv[1] + input_name = sys.argv[2] + try: + chromCol = int( sys.argv[3] ) - 1 + except: + stop_err( "'%s' is an invalid chrom column, correct the column settings before attempting to convert the data format." % str( sys.argv[3] ) ) + try: + startCol = int( sys.argv[4] ) - 1 + except: + stop_err( "'%s' is an invalid start column, correct the column settings before attempting to convert the data format." % str( sys.argv[4] ) ) + try: + endCol = int( sys.argv[5] ) - 1 + except: + stop_err( "'%s' is an invalid end column, correct the column settings before attempting to convert the data format." % str( sys.argv[5] ) ) + try: + strandCol = int( sys.argv[6] ) - 1 + except: + strandCol = -1 + try: + nameCol = int( sys.argv[7] ) - 1 + except: + nameCol = -1 + skipped_lines = 0 + first_skipped_line = 0 + out = open( output_name,'w' ) + count = 0 + for count, region in enumerate( bx.intervals.io.NiceReaderWrapper( open( input_name, 'r' ), chrom_col=chromCol, start_col=startCol, end_col=endCol, strand_col=strandCol, fix_strand=True, return_header=False, return_comments=False ) ): + try: + if nameCol >= 0: + name = region.fields[nameCol] + else: + raise IndexError + except: + name = "region_%i" % count + try: + + out.write( "%s\t%i\t%i\t%s\t%i\t%s\n" % ( region.chrom, region.start, region.end, name, 0, region.strand ) ) + except: + skipped_lines += 1 + if not first_skipped_line: + first_skipped_line = count + 1 + out.close() + print "%i regions converted to BED." % ( count + 1 - skipped_lines ) + if skipped_lines > 0: + print "Skipped %d blank or invalid lines starting with line # %d." % ( skipped_lines, first_skipped_line ) + +if __name__ == "__main__": __main__() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py --- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py @@ -64,7 +64,7 @@ force_num_columns = int( sys.argv[9] ) except: force_num_columns = None - + skipped_lines = 0 first_skipped_line = None out = open( output_name,'w' ) @@ -88,32 +88,32 @@ break #name (fields[3]) can be anything, no verification needed if len( fields ) > 4: - float( fields[4] ) #score - A score between 0 and 1000. If the track line useScore attribute is set to 1 for this annotation data set, the score value will determine the level of gray in which this feature is displayed (higher numbers = darker gray). + float( fields[4] ) #score - A score between 0 and 1000. If the track line useScore attribute is set to 1 for this annotation data set, the score value will determine the level of gray in which this feature is displayed (higher numbers = darker gray). if len( fields ) > 5: - assert fields[5] in [ '+', '-' ], 'Invalid strand' #strand - Defines the strand - either '+' or '-'. + assert fields[5] in [ '+', '-' ], 'Invalid strand' #strand - Defines the strand - either '+' or '-'. if len( fields ) > 6: - int( fields[6] ) #thickStart - The starting position at which the feature is drawn thickly (for example, the start codon in gene displays). + int( fields[6] ) #thickStart - The starting position at which the feature is drawn thickly (for example, the start codon in gene displays). if len( fields ) > 7: - int( fields[7] ) #thickEnd - The ending position at which the feature is drawn thickly (for example, the stop codon in gene displays). - if len( fields ) > 8: + int( fields[7] ) #thickEnd - The ending position at which the feature is drawn thickly (for example, the stop codon in gene displays). + if len( fields ) > 8: if fields[8] != '0': #itemRgb - An RGB value of the form R,G,B (e.g. 255,0,0). If the track line itemRgb attribute is set to "On", this RBG value will determine the display color of the data contained in this BED line. NOTE: It is recommended that a simple color scheme (eight colors or less) be used with this attribute to avoid overwhelming the color resources of the Genome Browser and your Internet browser. fields2 = fields[8].split( ',' ) assert len( fields2 ) == 3, 'RGB value must be 0 or have length of 3' for field in fields2: int( field ) #rgb values are integers if len( fields ) > 9: - int( fields[9] ) #blockCount - The number of blocks (exons) in the BED line. + int( fields[9] ) #blockCount - The number of blocks (exons) in the BED line. if len( fields ) > 10: - if fields[10] != ',': #blockSizes - A comma-separated list of the block sizes. The number of items in this list should correspond to blockCount. + if fields[10] != ',': #blockSizes - A comma-separated list of the block sizes. The number of items in this list should correspond to blockCount. fields2 = fields[10].rstrip( "," ).split( "," ) #remove trailing comma and split on comma - for field in fields2: + for field in fields2: int( field ) if len( fields ) > 11: - if fields[11] != ',': #blockStarts - A comma-separated list of block starts. All of the blockStart positions should be calculated relative to chromStart. The number of items in this list should correspond to blockCount. + if fields[11] != ',': #blockStarts - A comma-separated list of block starts. All of the blockStart positions should be calculated relative to chromStart. The number of items in this list should correspond to blockCount. fields2 = fields[11].rstrip( "," ).split( "," ) #remove trailing comma and split on comma for field in fields2: int( field ) - except: + except: strict_bed = False break if force_num_columns is not None and len( fields ) != force_num_columns: @@ -122,7 +122,7 @@ else: strict_bed = False out.close() - + if not strict_bed: skipped_lines = 0 first_skipped_line = None diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/interval_to_coverage.py --- a/lib/galaxy/datatypes/converters/interval_to_coverage.py +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -50,12 +50,12 @@ forward = forward_covs[partition] reverse = reverse_covs[partition] if forward+reverse > 0: - coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), + coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), forward=forward, reverse=reverse) partitions = [] forward_covs = [] reverse_covs = [] - + start_index = bisect(partitions, record.start) forward = int(record.strand == "+") reverse = int(record.strand == "-") @@ -74,43 +74,43 @@ partitions.insert(end_index, record.end) forward_covs.insert(end_index, forward_covs[end_index-1] - forward ) reverse_covs.insert(end_index, reverse_covs[end_index-1] - reverse ) - + if partitions: for partition in xrange(0, start_index): forward = forward_covs[partition] reverse = reverse_covs[partition] if forward+reverse > 0: - coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), + coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), forward=forward, reverse=reverse) partitions = partitions[start_index:] forward_covs = forward_covs[start_index:] reverse_covs = reverse_covs[start_index:] - + lastchrom = chrom - + # Finish the last chromosome if partitions: for partition in xrange(0, len(partitions)-1): forward = forward_covs[partition] reverse = reverse_covs[partition] if forward+reverse > 0: - coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), + coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), forward=forward, reverse=reverse) - + class CoverageWriter( object ): def __init__( self, out_stream=None, chromCol=0, positionCol=1, forwardCol=2, reverseCol=3 ): self.out_stream = out_stream self.reverseCol = reverseCol self.nlines = 0 - positions = {str(chromCol):'%(chrom)s', - str(positionCol):'%(position)d', - str(forwardCol):'%(forward)d', + positions = {str(chromCol):'%(chrom)s', + str(positionCol):'%(position)d', + str(forwardCol):'%(forward)d', str(reverseCol):'%(reverse)d'} - if reverseCol < 0: + if reverseCol < 0: self.template = "%(0)s\t%(1)s\t%(2)s\n" % positions else: self.template = "%(0)s\t%(1)s\t%(2)s\t%(3)s\n" % positions - + def write(self, **kwargs ): if self.reverseCol < 0: kwargs['forward'] += kwargs['reverse'] posgen = kwargs['position'] @@ -121,12 +121,12 @@ def close(self): self.out_stream.flush() self.out_stream.close() - + if __name__ == "__main__": options, args = doc_optparse.parse( __doc__ ) try: chr_col_1, start_col_1, end_col_1, strand_col_1 = [int(x)-1 for x in options.cols1.split(',')] - chr_col_2, position_col_2, forward_col_2, reverse_col_2 = [int(x)-1 for x in options.cols2.split(',')] + chr_col_2, position_col_2, forward_col_2, reverse_col_2 = [int(x)-1 for x in options.cols2.split(',')] in_fname, out_fname = args except: doc_optparse.exception() @@ -141,7 +141,7 @@ chromCol = chr_col_2, positionCol = position_col_2, forwardCol = forward_col_2, reverseCol = reverse_col_2, ) temp_file.seek(0) - interval = io.NiceReaderWrapper( temp_file, + interval = io.NiceReaderWrapper( temp_file, chrom_col=chr_col_1, start_col=start_col_1, end_col=end_col_1, diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/interval_to_fli.py --- a/lib/galaxy/datatypes/converters/interval_to_fli.py +++ b/lib/galaxy/datatypes/converters/interval_to_fli.py @@ -78,13 +78,13 @@ if len( fields ) < 4: continue - # Process line + # Process line name_loc_dict[ fields[3] ] = { 'contig': fields[0], 'start': int( fields[1] ), 'end': int ( fields[2] ) } - + # Create sorted list of entries. out = open( out_fname, 'w' ) max_len = 0 @@ -95,7 +95,7 @@ if len( entry ) > max_len: max_len = len( entry ) entries.append( entry ) - + # Write padded entries. out.write( str( max_len + 1 ).ljust( max_len ) + '\n' ) for entry in entries: diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/interval_to_tabix_converter.py --- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py @@ -20,20 +20,19 @@ parser.add_option( '-P', '--preset', dest='preset' ) (options, args) = parser.parse_args() input_fname, index_fname, out_fname = args - + # Create index. if options.preset: # Preset type. - ctabix.tabix_index(filename=index_fname, preset=options.preset, keep_original=True, + ctabix.tabix_index(filename=index_fname, preset=options.preset, keep_original=True, already_compressed=True, index_filename=out_fname) else: # For interval files; column indices are 0-based. - ctabix.tabix_index(filename=index_fname, seq_col=(options.chrom_col - 1), - start_col=(options.start_col - 1), end_col=(options.end_col - 1), + ctabix.tabix_index(filename=index_fname, seq_col=(options.chrom_col - 1), + start_col=(options.start_col - 1), end_col=(options.end_col - 1), keep_original=True, already_compressed=True, index_filename=out_fname) if os.path.getsize(index_fname) == 0: sys.stderr.write("The converted tabix index file is empty, meaning the input data is invalid.") - -if __name__ == "__main__": + +if __name__ == "__main__": main() - \ No newline at end of file diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/lped_to_fped_converter.py --- a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py @@ -1,110 +1,110 @@ -# for rgenetics - lped to fbat -# recode to numeric fbat version -# much slower so best to always -# use numeric alleles internally - -import sys,os,time - - -prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' - -galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> -<head> -<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> -<meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> -<title></title> -<link rel="stylesheet" href="/static/style/base.css" type="text/css" /> -</head> -<body> -<div class="document"> -""" - -def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) - - -def rgConv(inpedfilepath,outhtmlname,outfilepath): - """convert linkage ped/map to fbat""" - recode={'A':'1','C':'2','G':'3','T':'4','N':'0','0':'0','1':'1','2':'2','3':'3','4':'4'} - basename = os.path.split(inpedfilepath)[-1] # get basename - inmap = '%s.map' % inpedfilepath - inped = '%s.ped' % inpedfilepath - outf = '%s.ped' % basename # note the fbat exe insists that this is the extension for the ped data - outfpath = os.path.join(outfilepath,outf) # where to write the fbat format file to - try: - mf = file(inmap,'r') - except: - sys.stderr.write('%s cannot open inmap file %s - do you have permission?\n' % (prog,inmap)) - sys.exit(1) - try: - rsl = [x.split()[1] for x in mf] - except: - sys.stderr.write('## cannot parse %s' % inmap) - sys.exit(1) - try: - os.makedirs(outfilepath) - except: - pass # already exists - head = ' '.join(rsl) # list of rs numbers - # TODO add anno to rs but fbat will prolly barf? - pedf = file(inped,'r') - o = file(outfpath,'w',2**20) - o.write(head) - o.write('\n') - for i,row in enumerate(pedf): - if i == 0: - lrow = row.split() - try: - x = [int(x) for x in lrow[10:50]] # look for non numeric codes - except: - dorecode = 1 - if dorecode: - lrow = row.strip().split() - p = lrow[:6] - g = lrow[6:] - gc = [recode.get(x,'0') for x in g] - lrow = p+gc - row = '%s\n' % ' '.join(lrow) - o.write(row) - o.close() - - -def main(): - """call fbater - need to work with rgenetics composite datatypes - so in and out are html files with data in extrafiles path - <command interpreter="python">rg_convert_lped_fped.py '$input1/$input1.metadata.base_name' - '$output1' '$output1.extra_files_path' - </command> - """ - nparm = 3 - if len(sys.argv) < nparm: - sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog,sys.argv,nparm)) - sys.exit(1) - inpedfilepath = sys.argv[1] - outhtmlname = sys.argv[2] - outfilepath = sys.argv[3] - try: - os.makedirs(outfilepath) - except: - pass - rgConv(inpedfilepath,outhtmlname,outfilepath) - f = file(outhtmlname,'w') - f.write(galhtmlprefix % prog) - flist = os.listdir(outfilepath) - print '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info - f.write('<div>## Rgenetics: http://rgenetics.org Galaxy Tools %s %s\n<ol>' % (prog,timenow())) - for i, data in enumerate( flist ): - f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1],os.path.split(data)[-1])) - f.write("</div></body></html>") - f.close() - - - -if __name__ == "__main__": - main() +# for rgenetics - lped to fbat +# recode to numeric fbat version +# much slower so best to always +# use numeric alleles internally + +import sys,os,time + + +prog = os.path.split(sys.argv[0])[-1] +myversion = 'Oct 10 2009' + +galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> +<title></title> +<link rel="stylesheet" href="/static/style/base.css" type="text/css" /> +</head> +<body> +<div class="document"> +""" + +def timenow(): + """return current time as a string + """ + return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + + +def rgConv(inpedfilepath,outhtmlname,outfilepath): + """convert linkage ped/map to fbat""" + recode={'A':'1','C':'2','G':'3','T':'4','N':'0','0':'0','1':'1','2':'2','3':'3','4':'4'} + basename = os.path.split(inpedfilepath)[-1] # get basename + inmap = '%s.map' % inpedfilepath + inped = '%s.ped' % inpedfilepath + outf = '%s.ped' % basename # note the fbat exe insists that this is the extension for the ped data + outfpath = os.path.join(outfilepath,outf) # where to write the fbat format file to + try: + mf = file(inmap,'r') + except: + sys.stderr.write('%s cannot open inmap file %s - do you have permission?\n' % (prog,inmap)) + sys.exit(1) + try: + rsl = [x.split()[1] for x in mf] + except: + sys.stderr.write('## cannot parse %s' % inmap) + sys.exit(1) + try: + os.makedirs(outfilepath) + except: + pass # already exists + head = ' '.join(rsl) # list of rs numbers + # TODO add anno to rs but fbat will prolly barf? + pedf = file(inped,'r') + o = file(outfpath,'w',2**20) + o.write(head) + o.write('\n') + for i,row in enumerate(pedf): + if i == 0: + lrow = row.split() + try: + x = [int(x) for x in lrow[10:50]] # look for non numeric codes + except: + dorecode = 1 + if dorecode: + lrow = row.strip().split() + p = lrow[:6] + g = lrow[6:] + gc = [recode.get(x,'0') for x in g] + lrow = p+gc + row = '%s\n' % ' '.join(lrow) + o.write(row) + o.close() + + +def main(): + """call fbater + need to work with rgenetics composite datatypes + so in and out are html files with data in extrafiles path + <command interpreter="python">rg_convert_lped_fped.py '$input1/$input1.metadata.base_name' + '$output1' '$output1.extra_files_path' + </command> + """ + nparm = 3 + if len(sys.argv) < nparm: + sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog,sys.argv,nparm)) + sys.exit(1) + inpedfilepath = sys.argv[1] + outhtmlname = sys.argv[2] + outfilepath = sys.argv[3] + try: + os.makedirs(outfilepath) + except: + pass + rgConv(inpedfilepath,outhtmlname,outfilepath) + f = file(outhtmlname,'w') + f.write(galhtmlprefix % prog) + flist = os.listdir(outfilepath) + print '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info + f.write('<div>## Rgenetics: http://rgenetics.org Galaxy Tools %s %s\n<ol>' % (prog,timenow())) + for i, data in enumerate( flist ): + f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1],os.path.split(data)[-1])) + f.write("</div></body></html>") + f.close() + + + +if __name__ == "__main__": + main() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/lped_to_pbed_converter.py --- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py @@ -1,110 +1,110 @@ -# for rgenetics - lped to pbed -# where to stop with converters -# pbed might be central -# eg lped/eigen/fbat/snpmatrix all to pbed -# and pbed to lped/eigen/fbat/snpmatrix ? -# that's a lot of converters -import sys,os,time,subprocess - - -prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' - -galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> -<head> -<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> -<meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> -<title></title> -<link rel="stylesheet" href="/static/style/base.css" type="text/css" /> -</head> -<body> -<div class="document"> -""" - -def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) - -def getMissval(inped=''): - """ - read some lines...ugly hack - try to guess missing value - should be N or 0 but might be . or - - """ - commonmissvals = {'N':'N','0':'0','n':'n','9':'9','-':'-','.':'.'} - try: - f = file(inped,'r') - except: - return None # signal no in file - missval = None - while missval == None: # doggedly continue until we solve the mystery - try: - l = f.readline() - except: - break - ll = l.split()[6:] # ignore pedigree stuff - for c in ll: - if commonmissvals.get(c,None): - missval = c - f.close() - return missval - if not missval: - missval = 'N' # punt - close(f) - return missval - -def rgConv(inpedfilepath,outhtmlname,outfilepath,plink): - """ - """ - pedf = '%s.ped' % inpedfilepath - basename = os.path.split(inpedfilepath)[-1] # get basename - outroot = os.path.join(outfilepath,basename) - missval = getMissval(inped = pedf) - if not missval: - print '### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf - missval = '0' - cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink,inpedfilepath,outroot,missval) - p = subprocess.Popen(cl,shell=True,cwd=outfilepath) - retval = p.wait() # run plink - - - - -def main(): - """ - need to work with rgenetics composite datatypes - so in and out are html files with data in extrafiles path - <command interpreter="python">lped_to_pbed_converter.py '$input1/$input1.metadata.base_name' - '$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' - </command> - """ - nparm = 4 - if len(sys.argv) < nparm: - sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog,sys.argv,nparm)) - sys.exit(1) - inpedfilepath = sys.argv[1] - outhtmlname = sys.argv[2] - outfilepath = sys.argv[3] - try: - os.makedirs(outfilepath) - except: - pass - plink = sys.argv[4] - rgConv(inpedfilepath,outhtmlname,outfilepath,plink) - f = file(outhtmlname,'w') - f.write(galhtmlprefix % prog) - flist = os.listdir(outfilepath) - s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info - print s - f.write('<div>%s\n<ol>' % (s)) - for i, data in enumerate( flist ): - f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1],os.path.split(data)[-1])) - f.write("</div></body></html>") - f.close() - - - -if __name__ == "__main__": - main() +# for rgenetics - lped to pbed +# where to stop with converters +# pbed might be central +# eg lped/eigen/fbat/snpmatrix all to pbed +# and pbed to lped/eigen/fbat/snpmatrix ? +# that's a lot of converters +import sys,os,time,subprocess + + +prog = os.path.split(sys.argv[0])[-1] +myversion = 'Oct 10 2009' + +galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> +<title></title> +<link rel="stylesheet" href="/static/style/base.css" type="text/css" /> +</head> +<body> +<div class="document"> +""" + +def timenow(): + """return current time as a string + """ + return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + +def getMissval(inped=''): + """ + read some lines...ugly hack - try to guess missing value + should be N or 0 but might be . or - + """ + commonmissvals = {'N':'N','0':'0','n':'n','9':'9','-':'-','.':'.'} + try: + f = file(inped,'r') + except: + return None # signal no in file + missval = None + while missval == None: # doggedly continue until we solve the mystery + try: + l = f.readline() + except: + break + ll = l.split()[6:] # ignore pedigree stuff + for c in ll: + if commonmissvals.get(c,None): + missval = c + f.close() + return missval + if not missval: + missval = 'N' # punt + close(f) + return missval + +def rgConv(inpedfilepath,outhtmlname,outfilepath,plink): + """ + """ + pedf = '%s.ped' % inpedfilepath + basename = os.path.split(inpedfilepath)[-1] # get basename + outroot = os.path.join(outfilepath,basename) + missval = getMissval(inped = pedf) + if not missval: + print '### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf + missval = '0' + cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink,inpedfilepath,outroot,missval) + p = subprocess.Popen(cl,shell=True,cwd=outfilepath) + retval = p.wait() # run plink + + + + +def main(): + """ + need to work with rgenetics composite datatypes + so in and out are html files with data in extrafiles path + <command interpreter="python">lped_to_pbed_converter.py '$input1/$input1.metadata.base_name' + '$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' + </command> + """ + nparm = 4 + if len(sys.argv) < nparm: + sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog,sys.argv,nparm)) + sys.exit(1) + inpedfilepath = sys.argv[1] + outhtmlname = sys.argv[2] + outfilepath = sys.argv[3] + try: + os.makedirs(outfilepath) + except: + pass + plink = sys.argv[4] + rgConv(inpedfilepath,outhtmlname,outfilepath,plink) + f = file(outhtmlname,'w') + f.write(galhtmlprefix % prog) + flist = os.listdir(outfilepath) + s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info + print s + f.write('<div>%s\n<ol>' % (s)) + for i, data in enumerate( flist ): + f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1],os.path.split(data)[-1])) + f.write("</div></body></html>") + f.close() + + + +if __name__ == "__main__": + main() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/maf_to_fasta_converter.py --- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py @@ -1,32 +1,32 @@ -#!/usr/bin/env python -#Dan Blankenberg - -import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -import bx.align.maf -from galaxy.tools.util import maf_utilities - -assert sys.version_info[:2] >= ( 2, 4 ) - -def __main__(): - output_name = sys.argv.pop(1) - input_name = sys.argv.pop(1) - out = open( output_name, 'w' ) - count = 0 - for count, block in enumerate( bx.align.maf.Reader( open( input_name, 'r' ) ) ): +#!/usr/bin/env python +#Dan Blankenberg + +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +import bx.align.maf +from galaxy.tools.util import maf_utilities + +assert sys.version_info[:2] >= ( 2, 4 ) + +def __main__(): + output_name = sys.argv.pop(1) + input_name = sys.argv.pop(1) + out = open( output_name, 'w' ) + count = 0 + for count, block in enumerate( bx.align.maf.Reader( open( input_name, 'r' ) ) ): spec_counts = {} - for c in block.components: + for c in block.components: spec, chrom = maf_utilities.src_split( c.src ) if spec not in spec_counts: spec_counts[ spec ] = 0 else: - spec_counts[ spec ] += 1 - out.write( "%s\n" % maf_utilities.get_fasta_header( c, { 'block_index' : count, 'species' : spec, 'sequence_index' : spec_counts[ spec ] }, suffix = "%s_%i_%i" % ( spec, count, spec_counts[ spec ] ) ) ) - out.write( "%s\n" % c.text ) - out.write( "\n" ) - out.close() - print "%i MAF blocks converted to FASTA." % ( count ) - - -if __name__ == "__main__": __main__() + spec_counts[ spec ] += 1 + out.write( "%s\n" % maf_utilities.get_fasta_header( c, { 'block_index' : count, 'species' : spec, 'sequence_index' : spec_counts[ spec ] }, suffix = "%s_%i_%i" % ( spec, count, spec_counts[ spec ] ) ) ) + out.write( "%s\n" % c.text ) + out.write( "\n" ) + out.close() + print "%i MAF blocks converted to FASTA." % ( count ) + + +if __name__ == "__main__": __main__() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/maf_to_interval_converter.py --- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py @@ -1,32 +1,32 @@ -#!/usr/bin/env python -#Dan Blankenberg - -import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) +#!/usr/bin/env python +#Dan Blankenberg + +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) import bx.align.maf -from galaxy.tools.util import maf_utilities - -assert sys.version_info[:2] >= ( 2, 4 ) - -def __main__(): - output_name = sys.argv.pop(1) - input_name = sys.argv.pop(1) - species = sys.argv.pop(1) - out = open(output_name,'w') - count = 0 - #write interval header line - out.write( "#chrom\tstart\tend\tstrand\n" ) - try: - for block in bx.align.maf.Reader( open( input_name, 'r' ) ): - for c in maf_utilities.iter_components_by_src_start( block, species ): - if c is not None: - out.write( "%s\t%i\t%i\t%s\n" % ( maf_utilities.src_split( c.src )[-1], c.get_forward_strand_start(), c.get_forward_strand_end(), c.strand ) ) - count += 1 - except Exception, e: - print >> sys.stderr, "There was a problem processing your input: %s" % e - out.close() - print "%i MAF blocks converted to Genomic Intervals for species %s." % ( count, species ) - - -if __name__ == "__main__": __main__() +from galaxy.tools.util import maf_utilities + +assert sys.version_info[:2] >= ( 2, 4 ) + +def __main__(): + output_name = sys.argv.pop(1) + input_name = sys.argv.pop(1) + species = sys.argv.pop(1) + out = open(output_name,'w') + count = 0 + #write interval header line + out.write( "#chrom\tstart\tend\tstrand\n" ) + try: + for block in bx.align.maf.Reader( open( input_name, 'r' ) ): + for c in maf_utilities.iter_components_by_src_start( block, species ): + if c is not None: + out.write( "%s\t%i\t%i\t%s\n" % ( maf_utilities.src_split( c.src )[-1], c.get_forward_strand_start(), c.get_forward_strand_end(), c.strand ) ) + count += 1 + except Exception, e: + print >> sys.stderr, "There was a problem processing your input: %s" % e + out.close() + print "%i MAF blocks converted to Genomic Intervals for species %s." % ( count, species ) + + +if __name__ == "__main__": __main__() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -21,7 +21,7 @@ <div class="document"> """ -plinke = 'plink' +plinke = 'plink' def timenow(): @@ -51,7 +51,7 @@ except: alog.append('### %s Strange - no std out from plink when running command line\n%s\n' % (timenow(),' '.join(vcl))) return alog - + def makeLDreduced(basename,infpath=None,outfpath=None,plinke='plink',forcerebuild=False,returnFname=False, winsize="60", winmove="40", r2thresh="0.1" ): @@ -79,11 +79,11 @@ need to work with rgenetics composite datatypes so in and out are html files with data in extrafiles path - .. raw:: xml + .. raw:: xml <command interpreter="python"> - pbed_ldreduced_converter.py '$input1.extra_files_path/$input1.metadata.base_name' '$winsize' '$winmove' '$r2thresh' - '$output1' '$output1.files_path' 'plink' + pbed_ldreduced_converter.py '$input1.extra_files_path/$input1.metadata.base_name' '$winsize' '$winmove' '$r2thresh' + '$output1' '$output1.files_path' 'plink' </command> """ @@ -116,7 +116,7 @@ f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1],os.path.split(data)[-1])) f.write("</div></body></html>") f.close() - + if __name__ == "__main__": main() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/pbed_to_lped_converter.py --- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py @@ -1,80 +1,80 @@ -# for rgenetics - lped to pbed -# where to stop with converters -# pbed might be central -# eg lped/eigen/fbat/snpmatrix all to pbed -# and pbed to lped/eigen/fbat/snpmatrix ? -# that's a lot of converters -import sys,os,time,subprocess - - -prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' - -galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> -<head> -<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> -<meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> -<title></title> -<link rel="stylesheet" href="/static/style/base.css" type="text/css" /> -</head> -<body> -<div class="document"> -""" - -def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) +# for rgenetics - lped to pbed +# where to stop with converters +# pbed might be central +# eg lped/eigen/fbat/snpmatrix all to pbed +# and pbed to lped/eigen/fbat/snpmatrix ? +# that's a lot of converters +import sys,os,time,subprocess - + +prog = os.path.split(sys.argv[0])[-1] +myversion = 'Oct 10 2009' + +galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> +<title></title> +<link rel="stylesheet" href="/static/style/base.css" type="text/css" /> +</head> +<body> +<div class="document"> +""" + +def timenow(): + """return current time as a string + """ + return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + + def rgConv(inpedfilepath,outhtmlname,outfilepath,plink): """ """ - - basename = os.path.split(inpedfilepath)[-1] # get basename + + basename = os.path.split(inpedfilepath)[-1] # get basename outroot = os.path.join(outfilepath,basename) cl = '%s --noweb --bfile %s --recode --out %s ' % (plink,inpedfilepath,outroot) p = subprocess.Popen(cl,shell=True,cwd=outfilepath) retval = p.wait() # run plink - - - -def main(): - """ - need to work with rgenetics composite datatypes - so in and out are html files with data in extrafiles path - <command interpreter="python">pbed_to_lped_converter.py '$input1/$input1.metadata.base_name' - '$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' - </command> - """ - nparm = 4 - if len(sys.argv) < nparm: - sys.stderr.write('## %s called with %s - needs %d parameters \n' % (myname,sys.argv,nparm)) - sys.exit(1) - inpedfilepath = sys.argv[1] - outhtmlname = sys.argv[2] - outfilepath = sys.argv[3] - try: - os.makedirs(outfilepath) - except: - pass - plink = sys.argv[4] - rgConv(inpedfilepath,outhtmlname,outfilepath,plink) - f = file(outhtmlname,'w') - f.write(galhtmlprefix % prog) - flist = os.listdir(outfilepath) - s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info - print s - f.write('<div>%s\n<ol>' % (s)) - for i, data in enumerate( flist ): - f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1],os.path.split(data)[-1])) - f.write("</div></body></html>") - f.close() - - - -if __name__ == "__main__": - main() + + + +def main(): + """ + need to work with rgenetics composite datatypes + so in and out are html files with data in extrafiles path + <command interpreter="python">pbed_to_lped_converter.py '$input1/$input1.metadata.base_name' + '$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' + </command> + """ + nparm = 4 + if len(sys.argv) < nparm: + sys.stderr.write('## %s called with %s - needs %d parameters \n' % (myname,sys.argv,nparm)) + sys.exit(1) + inpedfilepath = sys.argv[1] + outhtmlname = sys.argv[2] + outfilepath = sys.argv[3] + try: + os.makedirs(outfilepath) + except: + pass + plink = sys.argv[4] + rgConv(inpedfilepath,outhtmlname,outfilepath,plink) + f = file(outhtmlname,'w') + f.write(galhtmlprefix % prog) + flist = os.listdir(outfilepath) + s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info + print s + f.write('<div>%s\n<ol>' % (s)) + for i, data in enumerate( flist ): + f.write('<li><a href="%s">%s</a></li>\n' % (os.path.split(data)[-1],os.path.split(data)[-1])) + f.write("</div></body></html>") + f.close() + + + +if __name__ == "__main__": + main() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py --- a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py +++ b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py @@ -17,7 +17,7 @@ for i, line in enumerate( open( input_name ) ): complete_interval = False line = line.rstrip( '\r\n' ) - if line: + if line: if line.startswith( HEADER_STARTS_WITH ): header_lines += 1 else: diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/sam_to_bam.py --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -19,12 +19,12 @@ #Parse Command Line parser = optparse.OptionParser() (options, args) = parser.parse_args() - + assert len( args ) == 2, 'You must specify the input and output filenames' input_filename, output_filename = args - + tmp_dir = tempfile.mkdtemp( prefix='tmp-sam_to_bam_converter-' ) - + #convert to SAM unsorted_bam_filename = os.path.join( tmp_dir, 'unsorted.bam' ) unsorted_stderr_filename = os.path.join( tmp_dir, 'unsorted.stderr' ) @@ -43,14 +43,14 @@ else: break stderr.close() - + #sort sam, so indexing will not fail sorted_stderr_filename = os.path.join( tmp_dir, 'sorted.stderr' ) sorting_prefix = os.path.join( tmp_dir, 'sorted_bam' ) cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) proc = subprocess.Popen( args=cmd, stderr=open( sorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) return_code = proc.wait() - + if return_code: stderr_target = sys.stderr else: @@ -63,7 +63,7 @@ else: break stderr.close() - + cleanup_before_exit( tmp_dir ) if __name__=="__main__": __main__() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py --- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py @@ -16,20 +16,19 @@ # Read options, args. parser = optparse.OptionParser() (options, args) = parser.parse_args() - in_file, out_file = args - + in_file, out_file = args + # Do conversion. index = Indexes() - reader = galaxy_utils.sequence.vcf.Reader( open( in_file ) ) + reader = galaxy_utils.sequence.vcf.Reader( open( in_file ) ) offset = reader.metadata_len for vcf_line in reader: - # VCF format provides a chrom and 1-based position for each variant. + # VCF format provides a chrom and 1-based position for each variant. # IntervalIndex expects 0-based coordinates. index.add( vcf_line.chrom, vcf_line.pos-1, vcf_line.pos, offset ) offset += len( vcf_line.raw_line ) - + index.write( open( out_file, "w" ) ) -if __name__ == "__main__": +if __name__ == "__main__": main() - \ No newline at end of file diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py --- a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py +++ b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ -Uses pysam to bgzip a vcf file as-is. +Uses pysam to bgzip a vcf file as-is. Headers, which are important, are kept. Original ordering, which may be specifically needed by tools or external display applications, is also maintained. @@ -17,8 +17,8 @@ parser = optparse.OptionParser() (options, args) = parser.parse_args() input_fname, output_fname = args - + ctabix.tabix_compress(input_fname, output_fname, force=True) - -if __name__ == "__main__": + +if __name__ == "__main__": main() diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py --- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py @@ -11,19 +11,19 @@ BLOCK_SIZE = 100 def main(): - + input_fname = sys.argv[1] out_fname = sys.argv[2] - + reader = WiggleReader( open( input_fname ) ) - + # Fill array from reader d = array_tree_dict_from_reader( reader, {}, block_size = BLOCK_SIZE ) - + for array_tree in d.itervalues(): array_tree.root.build_summary() - + FileArrayTreeDict.dict_to_file( d, open( out_fname, "w" ) ) -if __name__ == "__main__": +if __name__ == "__main__": main() \ No newline at end of file diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py --- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py @@ -17,16 +17,16 @@ sys.exit() def main(): - if len( sys.argv ) > 1: + if len( sys.argv ) > 1: in_file = open( sys.argv[1] ) - else: + else: in_file = open( sys.stdin ) - + if len( sys.argv ) > 2: out_file = open( sys.argv[2], "w" ) else: out_file = sys.stdout - + try: for fields in bx.wiggle.IntervalReader( UCSCOutWrapper( in_file ) ): out_file.write( "%s\n" % "\t".join( map( str, fields ) ) ) diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/coverage.py --- a/lib/galaxy/datatypes/coverage.py +++ b/lib/galaxy/datatypes/coverage.py @@ -15,7 +15,7 @@ class LastzCoverage( Tabular ): file_ext = "coverage" - + MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter ) MetadataElement( name="positionCol", default=2, desc="Position column", param=metadata.ColumnParameter ) MetadataElement( name="forwardCol", default=3, desc="Forward or aggregate read column", param=metadata.ColumnParameter ) @@ -44,7 +44,7 @@ t_end = math.ceil( end / resolution ) x = numpy.arange( t_start, t_end ) * resolution y = data[ t_start : t_end ] - + return zip(x.tolist(), y.tolist()) def get_track_resolution( self, dataset, start, end): diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -282,14 +282,14 @@ tmpfh = open( tmpf ) # CANNOT clean up - unlink/rmdir was always failing because file handle retained to return - must rely on a cron job to clean up tmp trans.response.set_content_type( "application/x-zip-compressed" ) - trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.zip"' % outfname + trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.zip"' % outfname return tmpfh else: trans.response.set_content_type( "application/x-tar" ) outext = 'tgz' if params.do_action == 'tbz': outext = 'tbz' - trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % (outfname,outext) + trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % (outfname,outext) archive.wsgi_status = trans.response.wsgi_status() archive.wsgi_headeritems = trans.response.wsgi_headeritems() return archive.stream diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/dataproviders/base.py --- a/lib/galaxy/datatypes/dataproviders/base.py +++ b/lib/galaxy/datatypes/dataproviders/base.py @@ -304,7 +304,7 @@ self.source = self.validate_source( source ) except exceptions.InvalidDataProviderSource, invalid_source: continue - + parent_gen = super( MultiSourceDataProvider, self ).__iter__() for datum in parent_gen: yield datum diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/dataproviders/line.py --- a/lib/galaxy/datatypes/dataproviders/line.py +++ b/lib/galaxy/datatypes/dataproviders/line.py @@ -262,7 +262,7 @@ """ if self.limit != None and self.num_data_returned >= self.limit: return None - + last_block = self.assemble_current_block() self.num_data_read += 1 diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/display_applications/application.py --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -128,7 +128,7 @@ self.data = data self.dataset_hash = dataset_hash self.user_hash = user_hash - self.trans = trans + self.trans = trans self.ready, self.parameters = self.link.build_parameter_dict( self.data, self.dataset_hash, self.user_hash, trans, app_kwds ) def display_ready( self ): return self.ready diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/display_applications/parameters.py --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -10,9 +10,9 @@ class DisplayApplicationParameter( object ): """ Abstract Class for Display Application Parameters """ - + type = None - + @classmethod def from_elem( cls, elem, link ): param_type = elem.get( 'type', None ) @@ -42,9 +42,9 @@ class DisplayApplicationDataParameter( DisplayApplicationParameter ): """ Parameter that returns a file_name containing the requested content """ - + type = 'data' - + def __init__( self, elem, link ): DisplayApplicationParameter.__init__( self, elem, link ) self.extensions = elem.get( 'format', None ) @@ -113,7 +113,7 @@ return False def ready( self, other_values ): value = self._get_dataset_like_object( other_values ) - if value: + if value: if value.state == value.states.OK: return True elif value.state == value.states.ERROR: @@ -122,9 +122,9 @@ class DisplayApplicationTemplateParameter( DisplayApplicationParameter ): """ Parameter that returns a string containing the requested content """ - + type = 'template' - + def __init__( self, elem, link ): DisplayApplicationParameter.__init__( self, elem, link ) self.text = elem.text or '' @@ -154,7 +154,7 @@ if self.parameter.guess_mime_type: mime, encoding = mimetypes.guess_type( self._url ) if not mime: - mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( self._url )[ -1 ], None ) + mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( self._url )[ -1 ], None ) if mime: return mime return 'text/plain' @@ -193,7 +193,7 @@ if self.parameter.guess_mime_type: mime, encoding = mimetypes.guess_type( self._url ) if not mime: - mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( self._url )[ -1 ], None ) + mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( self._url )[ -1 ], None ) if mime: return mime if hasattr( self.value, 'get_mime' ): diff -r f33687b0e59013a924a696f1c9630cae960b74a2 -r be3b0358acc9f251a8c8d66beb87f6b4684c1c10 lib/galaxy/datatypes/display_applications/util.py --- a/lib/galaxy/datatypes/display_applications/util.py +++ b/lib/galaxy/datatypes/display_applications/util.py @@ -10,7 +10,7 @@ user_hash = 'None' else: user_hash = str( user.id ) - # Pad to a multiple of 8 with leading "!" + # Pad to a multiple of 8 with leading "!" user_hash = ( "!" * ( 8 - len( user_hash ) % 8 ) ) + user_hash cipher = Blowfish.new( str( dataset.create_time ) ) user_hash = cipher.encrypt( user_hash ).encode( 'hex' ) This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Strip trailing whitespace in data_providers
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f33687b0e590/ Changeset: f33687b0e590 User: dannon Date: 2013-08-30 05:32:37 Summary: Strip trailing whitespace in data_providers Affected #: 1 file diff -r c05e0e9714f5bd2ecfa96cfd6b6f7152b815e82f -r f33687b0e59013a924a696f1c9630cae960b74a2 lib/galaxy/visualization/data_providers/basic.py --- a/lib/galaxy/visualization/data_providers/basic.py +++ b/lib/galaxy/visualization/data_providers/basic.py @@ -15,37 +15,37 @@ self.original_dataset = original_dataset self.dependencies = dependencies self.error_max_vals = error_max_vals - + def has_data( self, **kwargs ): """ Returns true if dataset has data in the specified genome window, false otherwise. """ raise Exception( "Unimplemented Function" ) - + def get_iterator( self, **kwargs ): """ Returns an iterator that provides data in the region chrom:start-end """ raise Exception( "Unimplemented Function" ) - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Process data from an iterator to a format that can be provided to client. """ - raise Exception( "Unimplemented Function" ) - + raise Exception( "Unimplemented Function" ) + def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ): - """ - Returns data as specified by kwargs. start_val is the first element to + """ + Returns data as specified by kwargs. start_val is the first element to return and max_vals indicates the number of values to return. - + Return value must be a dictionary with the following attributes: dataset_type, data """ iterator = self.get_iterator( chrom, start, end ) return self.process_data( iterator, start_val, max_vals, **kwargs ) - + def write_data_to_file( self, filename, **kwargs ): """ Write data in region defined by chrom, start, and end to a file. @@ -56,20 +56,20 @@ class ColumnDataProvider( BaseDataProvider ): """ Data provider for columnar data """ MAX_LINES_RETURNED = 30000 - + def __init__( self, original_dataset, max_lines_returned=MAX_LINES_RETURNED ): # Compatibility check. if not isinstance( original_dataset.datatype, Tabular ): raise Exception( "Data provider can only be used with tabular data" ) - + # Attribute init. self.original_dataset = original_dataset # allow throttling self.max_lines_returned = max_lines_returned - + def get_data( self, columns=None, start_val=0, max_vals=None, skip_comments=True, **kwargs ): """ - Returns data from specified columns in dataset. Format is list of lists + Returns data from specified columns in dataset. Format is list of lists where each list is a line of data. """ if not columns: @@ -81,20 +81,20 @@ max_vals = min([ max_vals, self.max_lines_returned ]) except ( ValueError, TypeError ): max_vals = self.max_lines_returned - + try: start_val = int( start_val ) start_val = max([ start_val, 0 ]) except ( ValueError, TypeError ): start_val = 0 - + # skip comment lines (if any/avail) # pre: should have original_dataset and if( skip_comments and self.original_dataset.metadata.comment_lines and start_val < self.original_dataset.metadata.comment_lines ): start_val = int( self.original_dataset.metadata.comment_lines ) - + # columns is an array of ints for now (should handle column names later) columns = from_json_string( columns ) for column in columns: @@ -103,7 +103,7 @@ "column index (%d) must be positive and less" % ( column ) + " than the number of columns: %d" % ( self.original_dataset.metadata.columns ) ) #print columns, start_val, max_vals, skip_comments, kwargs - + # set up the response, column lists response = {} response[ 'data' ] = data = [ [] for column in columns ] @@ -113,9 +113,9 @@ 'count' : 0, 'sum' : 0 } for column in columns ] - + column_types = [ self.original_dataset.metadata.column_types[ column ] for column in columns ] - + # function for casting by column_types def cast_val( val, type ): """ Cast value based on type. Return None if can't be cast """ @@ -126,12 +126,12 @@ try: val = float( val ) except: return None return val - + returning_data = False f = open( self.original_dataset.file_name ) #TODO: add f.seek if given fptr in kwargs for count, line in enumerate( f ): - + # check line v. desired start, end if count < start_val: continue @@ -139,7 +139,7 @@ break returning_data = True - + fields = line.split() fields_len = len( fields ) #NOTE: this will return None/null for abberrant column values (including bad indeces) @@ -149,39 +149,39 @@ if column < fields_len: column_val = cast_val( fields[ column ], column_type ) if column_val != None: - + # if numeric, maintain min, max, sum if( column_type == 'float' or column_type == 'int' ): if( ( meta[ index ][ 'min' ] == None ) or ( column_val < meta[ index ][ 'min' ] ) ): meta[ index ][ 'min' ] = column_val - + if( ( meta[ index ][ 'max' ] == None ) or ( column_val > meta[ index ][ 'max' ] ) ): meta[ index ][ 'max' ] = column_val - + meta[ index ][ 'sum' ] += column_val - + # maintain a count - for other stats meta[ index ][ 'count' ] += 1 data[ index ].append( column_val ) - + response[ 'endpoint' ] = dict( last_line=( count - 1 ), file_ptr=f.tell() ) f.close() if not returning_data: return None - + for index, meta in enumerate( response[ 'meta' ] ): column_type = column_types[ index ] count = meta[ 'count' ] - + if( ( column_type == 'float' or column_type == 'int' ) and count ): meta[ 'mean' ] = float( meta[ 'sum' ] ) / count - + sorted_data = sorted( response[ 'data' ][ index ] ) middle_index = ( count / 2 ) - 1 if count % 2 == 0: meta[ 'median' ] = ( ( sorted_data[ middle_index ] + sorted_data[( middle_index + 1 )] ) / 2.0 ) - + else: meta[ 'median' ] = sorted_data[ middle_index ] Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Realizing it's abstract, still require appropriate inputs to get_data in BaseDataProvider
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c05e0e9714f5/ Changeset: c05e0e9714f5 User: dannon Date: 2013-08-30 05:31:55 Summary: Realizing it's abstract, still require appropriate inputs to get_data in BaseDataProvider Affected #: 1 file diff -r c9b77dbc13165b020dfbe67b88faa99e5bf4d9fb -r c05e0e9714f5bd2ecfa96cfd6b6f7152b815e82f lib/galaxy/visualization/data_providers/basic.py --- a/lib/galaxy/visualization/data_providers/basic.py +++ b/lib/galaxy/visualization/data_providers/basic.py @@ -35,7 +35,7 @@ """ raise Exception( "Unimplemented Function" ) - def get_data( self, start_val=0, max_vals=sys.maxint, **kwargs ): + def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ): """ Returns data as specified by kwargs. start_val is the first element to return and max_vals indicates the number of values to return. Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7038567aedc2/ Changeset: 7038567aedc2 User: dannon Date: 2013-08-30 05:24:24 Summary: Add missing os import to ngsindex.py Affected #: 1 file diff -r 068acf051f9acfb8058f2bc50b0361d9a59d8cdb -r 7038567aedc2978d43386838441d3536a3d34650 lib/galaxy/datatypes/ngsindex.py --- a/lib/galaxy/datatypes/ngsindex.py +++ b/lib/galaxy/datatypes/ngsindex.py @@ -1,7 +1,9 @@ """ NGS indexes """ +import os import logging + from metadata import MetadataElement from images import Html https://bitbucket.org/galaxy/galaxy-central/commits/c9b77dbc1316/ Changeset: c9b77dbc1316 User: dannon Date: 2013-08-30 05:26:57 Summary: Data manager element loading assertion would fail if executed; use elem.tag and not root.tag Affected #: 1 file diff -r 7038567aedc2978d43386838441d3536a3d34650 -r c9b77dbc13165b020dfbe67b88faa99e5bf4d9fb lib/galaxy/tools/data_manager/manager.py --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -2,7 +2,7 @@ pkg_resources.require( "simplejson" ) -import os, shutil, errno +import os, errno import simplejson from galaxy import util @@ -112,7 +112,7 @@ if elem is not None: self.load_from_element( elem, tool_path or self.data_managers.tool_path ) def load_from_element( self, elem, tool_path ): - assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( root.tag ) + assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag ) self.declared_id = elem.get( 'id', None ) self.guid = elem.get( 'guid', None ) path = elem.get( 'tool_file', None ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 5 new changesets
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/fcabfb819232/ Changeset: fcabfb819232 User: dannon Date: 2013-08-30 05:06:53 Summary: Explicitly require source argument to RawBedDataProvider.get_iterator instead of relying on it in kwargs Affected #: 1 file diff -r facc879fe0543f25e6b4d65e3e5d5efe716ff455 -r fcabfb81923220651c0dd95181eed24d1b21ac68 lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -568,7 +568,7 @@ for large datasets. """ - def get_iterator( self, chrom=None, start=None, end=None, **kwargs ): + def get_iterator( self, source, chrom=None, start=None, end=None, **kwargs ): # Read first line in order to match chrom naming format. line = source.readline() dataset_chrom = line.split()[0] https://bitbucket.org/galaxy/galaxy-central/commits/3314e402ebaf/ Changeset: 3314e402ebaf User: dannon Date: 2013-08-30 05:10:02 Summary: Add missing import parse_gff_attributes to genome data provider Affected #: 1 file diff -r fcabfb81923220651c0dd95181eed24d1b21ac68 -r 3314e402ebaf326d57665615ba0e2e7b83dcc330 lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -9,7 +9,7 @@ pkg_resources.require( "pysam" ) pkg_resources.require( "numpy" ) import numpy -from galaxy.datatypes.util.gff_util import GFFReaderWrapper, GFFInterval, GFFFeature, convert_gff_coords_to_bed +from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFFeature, GFFInterval, GFFReaderWrapper, parse_gff_attributes from galaxy.util.json import from_json_string from bx.interval_index_file import Indexes from bx.bbi.bigwig_file import BigWigFile https://bitbucket.org/galaxy/galaxy-central/commits/4b86e65ee645/ Changeset: 4b86e65ee645 User: dannon Date: 2013-08-30 05:14:17 Summary: Strip unused imports (and trailing whitespace) from genome data provider Affected #: 1 file diff -r 3314e402ebaf326d57665615ba0e2e7b83dcc330 -r 4b86e65ee645caa6b5923b05e759a06a9f06113f lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -3,7 +3,6 @@ """ import os, sys, re -from math import ceil, log import pkg_resources pkg_resources.require( "bx-python" ) pkg_resources.require( "pysam" ) @@ -14,7 +13,6 @@ from bx.interval_index_file import Indexes from bx.bbi.bigwig_file import BigWigFile from bx.bbi.bigbed_file import BigBedFile -from galaxy.util.lrucache import LRUCache from galaxy.visualization.data_providers.basic import BaseDataProvider from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and_cigar from galaxy.datatypes.interval import Bed, Gff, Gtf @@ -33,7 +31,7 @@ return None else: return float(n) - + def get_bounds( reads, start_pos_index, end_pos_index ): ''' Returns the minimum and maximum position for a set of reads. @@ -76,7 +74,7 @@ line_len = int( textloc_file.readline() ) file_len = os.path.getsize( self.converted_dataset.file_name ) query = query.lower() - + # Find query in file using binary search. low = 0 high = file_len / line_len @@ -91,42 +89,42 @@ low = mid + 1 else: high = mid - + position = low * line_len - + # At right point in file, generate hits. result = [] while True: line = textloc_file.readline() - if not line.startswith( query ): + if not line.startswith( query ): break - if line[ -1: ] == '\n': + if line[ -1: ] == '\n': line = line[ :-1 ] result.append( line.split()[1:] ) - textloc_file.close() + textloc_file.close() return result - + class GenomeDataProvider( BaseDataProvider ): - """ - Base class for genome data providers. All genome providers use BED coordinate + """ + Base class for genome data providers. All genome providers use BED coordinate format (0-based, half-open coordinates) for both queries and returned data. """ dataset_type = None - - """ + + """ Mapping from column name to payload data; this mapping is used to create - filters. Key is column name, value is a dict with mandatory key 'index' and + filters. Key is column name, value is a dict with mandatory key 'index' and optional key 'name'. E.g. this defines column 4 col_name_data_attr_mapping = {4 : { index: 5, name: 'Score' } } """ col_name_data_attr_mapping = {} - + def __init__( self, converted_dataset=None, original_dataset=None, dependencies=None, error_max_vals="Only the first %i %s in this region are displayed." ): - super( GenomeDataProvider, self ).__init__( converted_dataset=converted_dataset, + super( GenomeDataProvider, self ).__init__( converted_dataset=converted_dataset, original_dataset=original_dataset, dependencies=dependencies, error_max_vals=error_max_vals ) @@ -135,44 +133,44 @@ # queries, such as is necessary for genome-wide data. # TODO: add functions to (a) create data_file and (b) clean up data_file. self.data_file = None - + def write_data_to_file( self, regions, filename ): """ Write data in region defined by chrom, start, and end to a file. """ raise Exception( "Unimplemented Function" ) - + def valid_chroms( self ): """ Returns chroms/contigs that the dataset contains """ return None # by default - + def has_data( self, chrom, start, end, **kwargs ): """ Returns true if dataset has data in the specified genome window, false otherwise. """ raise Exception( "Unimplemented Function" ) - + def get_iterator( self, chrom, start, end, **kwargs ): """ Returns an iterator that provides data in the region chrom:start-end """ raise Exception( "Unimplemented Function" ) - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Process data from an iterator to a format that can be provided to client. """ - raise Exception( "Unimplemented Function" ) - + raise Exception( "Unimplemented Function" ) + def get_data( self, chrom=None, low=None, high=None, start_val=0, max_vals=sys.maxint, **kwargs ): - """ + """ Returns data in region defined by chrom, start, and end. start_val and - max_vals are used to denote the data to return: start_val is the first element to + max_vals are used to denote the data to return: start_val is the first element to return and max_vals indicates the number of values to return. - + Return value must be a dictionary with the following attributes: dataset_type, data """ @@ -204,12 +202,12 @@ 'dataset_type': self.dataset_type } - + def get_filters( self ): - """ - Returns filters for provider's data. Return value is a list of + """ + Returns filters for provider's data. Return value is a list of filters; each filter is a dictionary with the keys 'name', 'index', 'type'. - NOTE: This method uses the original dataset's datatype and metadata to + NOTE: This method uses the original dataset's datatype and metadata to create the filters. """ # Get column names. @@ -220,18 +218,18 @@ column_names = range( self.original_dataset.metadata.columns ) except: # Give up return [] - + # Dataset must have column types; if not, cannot create filters. try: column_types = self.original_dataset.metadata.column_types except AttributeError: return [] - + # Create and return filters. filters = [] if self.original_dataset.metadata.viz_filter_cols: for viz_col_index in self.original_dataset.metadata.viz_filter_cols: - # Some columns are optional, so can't assume that a filter + # Some columns are optional, so can't assume that a filter # column is in dataset. if viz_col_index >= len( column_names ): continue; @@ -248,7 +246,7 @@ def get_default_max_vals( self ): return 5000 - + # # -- Base mixins and providers -- # @@ -256,26 +254,26 @@ class FilterableMixin: def get_filters( self ): """ Returns a dataset's filters. """ - + # is_ functions taken from Tabular.set_meta def is_int( column_text ): try: int( column_text ) return True - except: + except: return False def is_float( column_text ): try: float( column_text ) return True - except: + except: if column_text.strip().lower() == 'na': return True #na is special cased to be a float return False - + # # Get filters. - # TODOs: + # TODOs: # (a) might be useful to move this into each datatype's set_meta method; # (b) could look at first N lines to ensure GTF attribute types are consistent. # @@ -284,9 +282,9 @@ filter_col = 8 if isinstance( self.original_dataset.datatype, Gff ): # Can filter by score and GTF attributes. - filters = [ { 'name': 'Score', - 'type': 'number', - 'index': filter_col, + filters = [ { 'name': 'Score', + 'type': 'number', + 'index': filter_col, 'tool_id': 'Filter1', 'tool_exp_name': 'c6' } ] filter_col += 1 @@ -294,10 +292,10 @@ # Create filters based on dataset metadata. for name, a_type in self.original_dataset.metadata.attribute_types.items(): if a_type in [ 'int', 'float' ]: - filters.append( + filters.append( { 'name': name, - 'type': 'number', - 'index': filter_col, + 'type': 'number', + 'index': filter_col, 'tool_id': 'gff_filter_by_attribute', 'tool_exp_name': name } ) filter_col += 1 @@ -324,9 +322,9 @@ ''' elif isinstance( self.original_dataset.datatype, Bed ): # Can filter by score column only. - filters = [ { 'name': 'Score', - 'type': 'number', - 'index': filter_col, + filters = [ { 'name': 'Score', + 'type': 'number', + 'index': filter_col, 'tool_id': 'Filter1', 'tool_exp_name': 'c5' } ] @@ -340,19 +338,19 @@ """ Tabix index data provider for the Galaxy track browser. """ - + col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } - + def get_iterator( self, chrom, start, end, **kwargs ): start, end = int(start), int(end) if end >= (2<<29): end = (2<<29 - 1) # Tabix-enforced maximum - + bgzip_fname = self.dependencies['bgzip'].file_name - + if not self.data_file: self.data_file = ctabix.Tabixfile(bgzip_fname, index_filename=self.converted_dataset.file_name) - + # Get iterator using either naming scheme. iterator = iter( [] ) if chrom in self.data_file.contigs: @@ -365,10 +363,10 @@ return iterator - + def write_data_to_file( self, regions, filename ): out = open( filename, "w" ) - + for region in regions: # Write data in region. chrom = region.chrom @@ -377,7 +375,7 @@ iterator = self.get_iterator( chrom, start, end ) for line in iterator: out.write( "%s\n" % line ) - + out.close() # @@ -389,20 +387,20 @@ """ Processes interval data from native format to payload format. - + Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ] """ - + def get_iterator( self, chrom, start, end, **kwargs ): raise Exception( "Unimplemented Function" ) - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Provides """ # Build data to return. Payload format is: # [ <guid/offset>, <start>, <end>, <name>, <strand> ] - # + # # First three entries are mandatory, others are optional. # filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) @@ -421,7 +419,7 @@ if max_vals and count-start_val >= max_vals: message = self.error_max_vals % ( max_vals, "features" ) break - + feature = line.split() length = len(feature) # Unique id is just a hash of the line @@ -439,7 +437,7 @@ if not name_col: payload.append( "" ) payload.append( feature[strand_col] ) - # Score (filter data) + # Score (filter data) if length >= 5 and filter_cols and filter_cols[0] == "Score": try: payload.append( float( feature[4] ) ) @@ -467,23 +465,23 @@ class BedDataProvider( GenomeDataProvider ): """ Processes BED data from native format to payload format. - + Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ] """ dataset_type = 'interval_index' - + def get_iterator( self, chrom, start, end, **kwargs ): raise Exception( "Unimplemented Method" ) - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Provides """ # Build data to return. Payload format is: - # [ <guid/offset>, <start>, <end>, <name>, <strand>, <thick_start>, + # [ <guid/offset>, <start>, <end>, <name>, <strand>, <thick_start>, # <thick_end>, <blocks> ] - # + # # First three entries are mandatory, others are optional. # filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) @@ -524,10 +522,10 @@ blocks = zip( block_sizes, block_starts ) payload.append( [ ( int(feature[1]) + block[1], int(feature[1]) + block[1] + block[0] ) for block in blocks ] ) - # Score (filter data) + # Score (filter data) if length >= 5 and filter_cols and filter_cols[0] == "Score": - # If dataset doesn't have name/strand/thick start/thick end/blocks, - # add placeholders. There should be 8 entries if all attributes + # If dataset doesn't have name/strand/thick start/thick end/blocks, + # add placeholders. There should be 8 entries if all attributes # are present. payload.extend( [ None for i in range( 8 - len( payload ) ) ] ) @@ -542,7 +540,7 @@ def write_data_to_file( self, regions, filename ): out = open( filename, "w" ) - + for region in regions: # Write data in region. chrom = region.chrom @@ -551,15 +549,15 @@ iterator = self.get_iterator( chrom, start, end ) for line in iterator: out.write( "%s\n" % line ) - + out.close() - + class BedTabixDataProvider( TabixDataProvider, BedDataProvider ): """ Provides data from a BED file indexed via tabix. """ pass - + class RawBedDataProvider( BedDataProvider ): """ Provide data from BED file. @@ -590,7 +588,7 @@ or ( end is not None and feature_end < start ): continue yield line - + return line_filter_iter() # @@ -601,10 +599,10 @@ """ Abstract class that processes VCF data from native format to payload format. - Payload format: An array of entries for each locus in the file. Each array + Payload format: An array of entries for each locus in the file. Each array has the following entries: 1. GUID (unused) - 2. location (0-based) + 2. location (0-based) 3. reference base(s) 4. alternative base(s) 5. quality score @@ -613,20 +611,20 @@ denotes the reference genotype 8-end: allele counts for each alternative """ - + col_name_data_attr_mapping = { 'Qual' : { 'index': 6 , 'name' : 'Qual' } } dataset_type = 'variant' - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Returns a dict with the following attributes:: - data - a list of variants with the format + data - a list of variants with the format .. raw:: text - [<guid>, <start>, <end>, <name>, cigar, seq] + [<guid>, <start>, <end>, <name>, cigar, seq] message - error/informative message @@ -636,8 +634,8 @@ def get_mapping( ref, alt ): """ - Returns ( offset, new_seq, cigar ) tuple that defines mapping of - alt to ref. Cigar format is an array of [ op_index, length ] pairs + Returns ( offset, new_seq, cigar ) tuple that defines mapping of + alt to ref. Cigar format is an array of [ op_index, length ] pairs where op_index is the 0-based index into the string "MIDNSHP=X" """ @@ -676,7 +674,7 @@ samples_data = feature [ 9: ] # VCF is 1-based. pos = int( pos ) - 1 - + # FIXME: OK to skip? if alt == '.': count -= 1 @@ -707,7 +705,7 @@ has_alleles = True except ValueError: pass - + # If no alleles, use empty string as proxy. if not has_alleles: genotype = '' @@ -732,7 +730,7 @@ def write_data_to_file( self, regions, filename ): out = open( filename, "w" ) - + for region in regions: # Write data in region. chrom = region.chrom @@ -747,7 +745,7 @@ """ Provides data from a VCF file indexed via tabix. """ - + dataset_type = 'variant' class RawVcfDataProvider( VcfDataProvider ): @@ -797,17 +795,17 @@ for data_line in source: if line_in_region( data_line, chrom, start, end ): yield data_line - + return line_filter_iter() class BamDataProvider( GenomeDataProvider, FilterableMixin ): """ - Provides access to intervals from a sorted indexed BAM file. Coordinate + Provides access to intervals from a sorted indexed BAM file. Coordinate data is reported in BED format: 0-based, half-open. """ dataset_type = 'bai' - + def get_filters( self ): """ Returns filters for dataset. @@ -815,31 +813,31 @@ # HACK: first 7 fields are for drawing, so start filter column index at 7. filter_col = 7 filters = [] - filters.append( { 'name': 'Mapping Quality', - 'type': 'number', + filters.append( { 'name': 'Mapping Quality', + 'type': 'number', 'index': filter_col } ) return filters - - + + def write_data_to_file( self, regions, filename ): """ Write reads in regions to file. """ - + # Open current BAM file using index. bamfile = csamtools.Samfile( filename=self.original_dataset.file_name, mode='rb', \ index_filename=self.converted_dataset.file_name ) # TODO: write headers as well? new_bamfile = csamtools.Samfile( template=bamfile, filename=filename, mode='wb' ) - + for region in regions: # Write data from region. chrom = region.chrom start = region.start end = region.end - + try: data = bamfile.fetch(start=start, end=end, reference=chrom) except ValueError, e: @@ -853,11 +851,11 @@ # Write reads in region. for i, read in enumerate( data ): new_bamfile.write( read ) - + # Cleanup. new_bamfile.close() bamfile.close() - + def get_iterator( self, chrom, start, end, **kwargs ): """ Returns an iterator that provides data in the region chrom:start-end @@ -865,7 +863,7 @@ start, end = int( start ), int( end ) orig_data_filename = self.original_dataset.file_name index_filename = self.converted_dataset.file_name - + # Attempt to open the BAM file with index bamfile = csamtools.Samfile( filename=orig_data_filename, mode='rb', index_filename=index_filename ) try: @@ -878,12 +876,12 @@ except ValueError: return None return data - + def process_data( self, iterator, start_val=0, max_vals=None, ref_seq=None, start=0, **kwargs ): """ Returns a dict with the following attributes:: - data - a list of reads with the format + data - a list of reads with the format [<guid>, <start>, <end>, <name>, <read_1>, <read_2>, [empty], <mapq_scores>] where <read_1> has the format @@ -895,10 +893,10 @@ Field 7 is empty so that mapq scores' location matches that in single-end reads. For single-end reads, read has format: [<guid>, <start>, <end>, <name>, <cigar>, <strand>, <seq>, <mapq_score>] - + NOTE: read end and sequence data are not valid for reads outside of requested region and should not be used. - + max_low - lowest coordinate for the returned reads max_high - highest coordinate for the returned reads message - error/informative message @@ -919,7 +917,7 @@ return "+" else: return "-" - + # # Encode reads as list of lists. # @@ -933,13 +931,13 @@ if ( count - start_val - unmapped ) >= max_vals: message = self.error_max_vals % ( max_vals, "reads" ) break - + # If not mapped, skip read. is_mapped = ( read.flag & 0x0004 == 0 ) if not is_mapped: unmapped += 1 continue - + qname = read.qname seq = read.seq strand = decode_strand( read.flag, 0x0010 ) @@ -951,11 +949,11 @@ if read.is_proper_pair: if qname in paired_pending: # one in dict is always first pair = paired_pending[qname] - results.append( [ "%i_%s" % ( pair['start'], qname ), - pair['start'], - read.pos + read_len, - qname, - [ pair['start'], pair['end'], pair['cigar'], pair['strand'], pair['seq'] ], + results.append( [ "%i_%s" % ( pair['start'], qname ), + pair['start'], + read.pos + read_len, + qname, + [ pair['start'], pair['end'], pair['cigar'], pair['strand'], pair['seq'] ], [ read.pos, read.pos + read_len, read.cigar, strand, seq ], None, [ pair['mapq'], read.mapq ] ] ) @@ -964,10 +962,10 @@ paired_pending[qname] = { 'start': read.pos, 'end': read.pos + read_len, 'seq': seq, 'mate_start': read.mpos, 'rlen': read_len, 'strand': strand, 'cigar': read.cigar, 'mapq': read.mapq } else: - results.append( [ "%i_%s" % ( read.pos, qname ), - read.pos, read.pos + read_len, qname, + results.append( [ "%i_%s" % ( read.pos, qname ), + read.pos, read.pos + read_len, qname, read.cigar, strand, read.seq, read.mapq ] ) - + # Take care of reads whose mates are out of range. # TODO: count paired reads when adhering to max_vals? for qname, read in paired_pending.iteritems(): @@ -989,7 +987,7 @@ r2 = [ read['mate_start'], read['mate_start'] ] results.append( [ "%i_%s" % ( read_start, qname ), read_start, read_end, qname, r1, r2, [read[ 'mapq' ], 125] ] ) - + # Clean up. TODO: is this needed? If so, we'll need a cleanup function after processing the data. # bamfile.close() @@ -999,10 +997,10 @@ ''' Process a read using the designated fields. ''' - read_seq, read_cigar = get_ref_based_read_seq_and_cigar( read[ seq_field ].upper(), - read[ start_field ], - ref_seq, - start, + read_seq, read_cigar = get_ref_based_read_seq_and_cigar( read[ seq_field ].upper(), + read[ start_field ], + ref_seq, + start, read[ cigar_field ] ) read[ seq_field ] = read_seq read[ cigar_field ] = read_cigar @@ -1012,7 +1010,7 @@ Process single-end read. ''' process_read( read, 1, 4, 6) - + def process_pe_read( read ): ''' Process paired-end read. @@ -1034,28 +1032,28 @@ process_se_read( read ) max_low, max_high = get_bounds( results, 1, 2 ) - + return { 'data': results, 'message': message, 'max_low': max_low, 'max_high': max_high } - + class SamDataProvider( BamDataProvider ): dataset_type = 'bai' - + def __init__( self, converted_dataset=None, original_dataset=None, dependencies=None ): """ Create SamDataProvider. """ super( SamDataProvider, self ).__init__( converted_dataset=converted_dataset, original_dataset=original_dataset, dependencies=dependencies ) - - # To use BamDataProvider, original dataset must be BAM and + + # To use BamDataProvider, original dataset must be BAM and # converted dataset must be BAI. Use BAI from BAM metadata. if converted_dataset: self.original_dataset = converted_dataset self.converted_dataset = converted_dataset.metadata.bam_index - + class BBIDataProvider( GenomeDataProvider ): """ - BBI data provider for the Galaxy track browser. + BBI data provider for the Galaxy track browser. """ dataset_type = 'bigwig' @@ -1063,7 +1061,7 @@ def valid_chroms( self ): # No way to return this info as of now return None - + def has_data( self, chrom ): f, bbi = self._get_dataset() all_dat = bbi.query( chrom, 0, 2147483647, 1 ) or \ @@ -1081,18 +1079,18 @@ return bbi.summarize( chrom, start, end, num_points ) or \ bbi.summarize( _convert_between_ucsc_and_ensemble_naming( chrom ) , start, end, num_points ) - # Bigwig can be a standalone bigwig file, in which case we use - # original_dataset, or coming from wig->bigwig conversion in + # Bigwig can be a standalone bigwig file, in which case we use + # original_dataset, or coming from wig->bigwig conversion in # which we use converted_dataset f, bbi = self._get_dataset() - + # If stats requested, compute overall summary data for the range - # start:endbut no reduced data. This is currently used by client + # start:endbut no reduced data. This is currently used by client # to determine the default range. if 'stats' in kwargs: summary = _summarize_bbi( bbi, chrom, start, end, 1 ) f.close() - + min_val = 0 max_val = 0 mean = 0 @@ -1127,12 +1125,12 @@ summary = _summarize_bbi( bbi, chrom, start, end, num_points ) if summary: #mean = summary.sum_data / summary.valid_count - + ## Standard deviation by bin, not yet used ## var = summary.sum_squares - mean ## var /= minimum( valid_count - 1, 1 ) ## sd = sqrt( var ) - + pos = start step_size = (end - start) / num_points @@ -1150,34 +1148,34 @@ num_points = end - start + 1 end += 1 else: - # - # The goal is to sample the region between start and end uniformly - # using ~N (num_samples) data points. The challenge is that the size of - # sampled intervals rarely is full bases, so sampling using N points - # will leave the end of the region unsampled due to remainders for - # each interval. To recitify this, a new N is calculated based on the + # + # The goal is to sample the region between start and end uniformly + # using ~N (num_samples) data points. The challenge is that the size of + # sampled intervals rarely is full bases, so sampling using N points + # will leave the end of the region unsampled due to remainders for + # each interval. To recitify this, a new N is calculated based on the # step size that covers as much of the region as possible. # - # However, this still leaves some of the region unsampled. This - # could be addressed by repeatedly sampling remainder using a - # smaller and smaller step_size, but that would require iteratively + # However, this still leaves some of the region unsampled. This + # could be addressed by repeatedly sampling remainder using a + # smaller and smaller step_size, but that would require iteratively # going to BBI, which could be time consuming. # # Start with N samples. num_points = num_samples step_size = ( end - start ) / num_points - # Add additional points to sample in the remainder not covered by + # Add additional points to sample in the remainder not covered by # the initial N samples. remainder_start = start + step_size * num_points additional_points = ( end - remainder_start ) / step_size num_points += additional_points - + result = summarize_region( bbi, chrom, start, end, num_points ) - + # Cleanup and return. f.close() - return { + return { 'data': result, 'dataset_type': self.dataset_type } @@ -1190,7 +1188,7 @@ class BigWigDataProvider ( BBIDataProvider ): """ - Provides data from BigWig files; position data is reported in 1-based + Provides data from BigWig files; position data is reported in 1-based coordinate system, i.e. wiggle format. """ def _get_dataset( self ): @@ -1199,7 +1197,7 @@ else: f = open( self.original_dataset.file_name ) return f, BigWigFile(file=f) - + class IntervalIndexDataProvider( FilterableMixin, GenomeDataProvider ): """ Interval index files used for GFF, Pileup files. @@ -1207,7 +1205,7 @@ col_name_data_attr_mapping = { 4 : { 'index': 4 , 'name' : 'Score' } } dataset_type = 'interval_index' - + def write_data_to_file( self, regions, filename ): source = open( self.original_dataset.file_name ) index = Indexes( self.converted_dataset.file_name ) @@ -1230,10 +1228,10 @@ feature = reader.next() for interval in feature.intervals: out.write( '\t'.join( interval.fields ) + '\n' ) - + source.close() out.close() - + def get_iterator( self, chrom, start, end, **kwargs ): """ Returns an array with values: (a) source file and (b) an iterator that @@ -1246,7 +1244,7 @@ if chrom not in index.indexes: # Try alternative naming. chrom = _convert_between_ucsc_and_ensemble_naming( chrom ) - + return index.find(chrom, start, end) def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): @@ -1258,7 +1256,7 @@ # Build data to return. Payload format is: # [ <guid/offset>, <start>, <end>, <name>, <score>, <strand>, <thick_start>, # <thick_end>, <blocks> ] - # + # # First three entries are mandatory, others are optional. # filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) @@ -1272,7 +1270,7 @@ break source.seek( offset ) # TODO: can we use column metadata to fill out payload? - + # GFF dataset. reader = GFFReaderWrapper( source, fix_strand=True ) feature = reader.next() @@ -1286,13 +1284,13 @@ class RawGFFDataProvider( GenomeDataProvider ): """ Provide data from GFF file that has not been indexed. - + NOTE: this data provider does not use indices, and hence will be very slow for large datasets. """ dataset_type = 'interval_index' - + def get_iterator( self, chrom, start, end, **kwargs ): """ Returns an iterator that provides data in the region chrom:start-end as well as @@ -1302,18 +1300,18 @@ # Read first line in order to match chrom naming format. line = source.readline() - + # If line empty, assume file is empty and return empty iterator. if len( line ) == 0: return iter([]) - + # Determine chromosome naming format. dataset_chrom = line.split()[0] if not _chrom_naming_matches( chrom, dataset_chrom ): chrom = _convert_between_ucsc_and_ensemble_naming( chrom ) # Undo read. source.seek( 0 ) - + def features_in_region_iter(): offset = 0 for feature in GFFReaderWrapper( source, fix_strand=True ): @@ -1324,7 +1322,7 @@ offset += feature.raw_size return features_in_region_iter() - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Process data from an iterator to a format that can be provided to client. @@ -1340,22 +1338,22 @@ if count-start_val >= max_vals: message = self.error_max_vals % ( max_vals, "reads" ) break - + payload = package_gff_feature( feature, no_detail=no_detail, filter_cols=filter_cols ) payload.insert( 0, offset ) results.append( payload ) - + return { 'data': results, 'dataset_type': self.dataset_type, 'message': message } - + class GtfTabixDataProvider( TabixDataProvider ): """ Returns data from GTF datasets that are indexed via tabix. """ - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): # Loop through lines and group by transcript_id; each group is a feature. - + # TODO: extend this code or use code in gff_util to process GFF/3 as well # and then create a generic GFFDataProvider that can be used with both # raw and tabix datasets. @@ -1369,7 +1367,7 @@ feature = [] features[ transcript_id ] = feature feature.append( GFFInterval( None, line.split( '\t') ) ) - + # Process data. filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) no_detail = ( "no_detail" in kwargs ) @@ -1382,12 +1380,12 @@ if count-start_val >= max_vals: message = self.error_max_vals % ( max_vals, "reads" ) break - - feature = GFFFeature( None, intervals=intervals ) + + feature = GFFFeature( None, intervals=intervals ) payload = package_gff_feature( feature, no_detail=no_detail, filter_cols=filter_cols ) payload.insert( 0, feature.intervals[ 0 ].attributes[ 'transcript_id' ] ) results.append( payload ) - + return { 'data': results, 'message': message } # @@ -1397,26 +1395,26 @@ class ENCODEPeakDataProvider( GenomeDataProvider ): """ Abstract class that processes ENCODEPeak data from native format to payload format. - + Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ] """ - + def get_iterator( self, chrom, start, end, **kwargs ): raise "Unimplemented Method" - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Provides """ - + ## FIXMEs: # (1) should be able to unify some of this code with BedDataProvider.process_data # (2) are optional number of parameters supported? - + # Build data to return. Payload format is: # [ <guid/offset>, <start>, <end>, <name>, <strand>, <thick_start>, # <thick_end>, <blocks> ] - # + # # First three entries are mandatory, others are optional. # no_detail = ( "no_detail" in kwargs ) @@ -1431,16 +1429,16 @@ feature = line.split() length = len( feature ) - + # Feature initialization. payload = [ # GUID is just a hash of the line hash( line ), # Add start, end. - int( feature[1] ), + int( feature[1] ), int( feature[2] ) ] - + if no_detail: rval.append( payload ) continue @@ -1448,7 +1446,7 @@ # Extend with additional data. payload.extend( [ # Add name, strand. - feature[3], + feature[3], feature[5], # Thick start, end are feature start, end for now. int( feature[1] ), @@ -1465,12 +1463,12 @@ rval.append( payload ) return { 'data': rval, 'message': message } - + class ENCODEPeakTabixDataProvider( TabixDataProvider, ENCODEPeakDataProvider ): """ Provides data from an ENCODEPeak dataset indexed via tabix. """ - + def get_filters( self ): """ Returns filters for dataset. @@ -1478,26 +1476,26 @@ # HACK: first 8 fields are for drawing, so start filter column index at 9. filter_col = 8 filters = [] - filters.append( { 'name': 'Score', - 'type': 'number', + filters.append( { 'name': 'Score', + 'type': 'number', 'index': filter_col, 'tool_id': 'Filter1', 'tool_exp_name': 'c6' } ) filter_col += 1 - filters.append( { 'name': 'Signal Value', - 'type': 'number', + filters.append( { 'name': 'Signal Value', + 'type': 'number', 'index': filter_col, 'tool_id': 'Filter1', 'tool_exp_name': 'c7' } ) filter_col += 1 - filters.append( { 'name': 'pValue', - 'type': 'number', + filters.append( { 'name': 'pValue', + 'type': 'number', 'index': filter_col, 'tool_id': 'Filter1', 'tool_exp_name': 'c8' } ) filter_col += 1 - filters.append( { 'name': 'qValue', - 'type': 'number', + filters.append( { 'name': 'qValue', + 'type': 'number', 'index': filter_col, 'tool_id': 'Filter1', 'tool_exp_name': 'c9' } ) @@ -1523,7 +1521,7 @@ feature = line.split() length = len( feature ) - + s1 = int( feature[1] ) e1 = int( feature[2] ) c = feature[3] @@ -1538,14 +1536,14 @@ # Add start1, end1, chr2, start2, end2, value. s1, e1, c, s2, e2, v ] - + rval.append( payload ) return { 'data': rval, 'message': message } def get_default_max_vals( self ): return 100000; - + class ChromatinInteractionsTabixDataProvider( TabixDataProvider, ChromatinInteractionsDataProvider ): def get_iterator( self, chrom, start=0, end=sys.maxint, interchromosomal=False, **kwargs ): """ @@ -1556,7 +1554,7 @@ def filter( iter ): for line in iter: feature = line.split() - s1 = int( feature[1] ) + s1 = int( feature[1] ) e1 = int( feature[2] ) c = feature[3] s2 = int( feature[4] ) @@ -1568,22 +1566,22 @@ if interchromosomal and c != chrom: yield line return filter( TabixDataProvider.get_iterator( self, chrom, filter_start, end ) ) - -# + +# # -- Helper methods. -- # def package_gff_feature( feature, no_detail=False, filter_cols=[] ): """ Package a GFF feature in an array for data providers. """ feature = convert_gff_coords_to_bed( feature ) - + # No detail means only start, end. if no_detail: return [ feature.start, feature.end ] - + # Return full feature. - payload = [ feature.start, - feature.end, + payload = [ feature.start, + feature.end, feature.name(), feature.strand, # No notion of thick start, end in GFF, so make everything @@ -1591,9 +1589,9 @@ feature.start, feature.end ] - - # HACK: ignore interval with name 'transcript' from feature. - # Cufflinks puts this interval in each of its transcripts, + + # HACK: ignore interval with name 'transcript' from feature. + # Cufflinks puts this interval in each of its transcripts, # and they mess up trackster by covering the feature's blocks. # This interval will always be a feature's first interval, # and the GFF's third column is its feature name. @@ -1605,7 +1603,7 @@ block_starts = [ ( interval.start - feature.start ) for interval in feature_intervals ] blocks = zip( block_sizes, block_starts ) payload.append( [ ( feature.start + block[1], feature.start + block[1] + block[0] ) for block in blocks ] ) - + # Add filter data to payload. for col in filter_cols: if col == "Score": https://bitbucket.org/galaxy/galaxy-central/commits/d2a624fd6dc2/ Changeset: d2a624fd6dc2 User: dannon Date: 2013-08-30 05:17:36 Summary: dataprovider dataset cleanup, add missing bx.bbi import Affected #: 1 file diff -r 4b86e65ee645caa6b5923b05e759a06a9f06113f -r d2a624fd6dc2fecdc319848f3d35c2f4b66a389e lib/galaxy/datatypes/dataproviders/dataset.py --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -6,19 +6,18 @@ (e.g. parsing genomic regions from their source) """ -from galaxy import eggs -import pkg_resources -pkg_resources.require( 'bx-python' ) -from bx import seq as bx_seq -from bx import wiggle as bx_wig - -import exceptions import base import line import column import external +from galaxy import eggs +eggs.require( 'bx-python' ) +from bx import seq as bx_seq +from bx import wiggle as bx_wig +from bx import bbi as bx_bbi + _TODO = """ use bx as much as possible gff3 hierarchies https://bitbucket.org/galaxy/galaxy-central/commits/068acf051f9a/ Changeset: 068acf051f9a User: dannon Date: 2013-08-30 05:20:05 Summary: Variable confusion in dataproviders/dataset -- clarify and use correct indices var Affected #: 1 file diff -r d2a624fd6dc2fecdc319848f3d35c2f4b66a389e -r 068acf051f9acfb8058f2bc50b0361d9a59d8cdb lib/galaxy/datatypes/dataproviders/dataset.py --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -145,10 +145,10 @@ :returns: list of column indeces for the named columns. """ region_column_names = ( 'chromCol', 'startCol', 'endCol' ) - region_indeces = [ self.get_metadata_column_index_by_name( name ) for name in region_column_names ] - if check and not all( map( lambda i: i != None, indeces ) ): - raise ValueError( "Could not determine proper column indeces for chrom, start, end: %s" %( str( indeces ) ) ) - return region_indeces + region_indices = [ self.get_metadata_column_index_by_name( name ) for name in region_column_names ] + if check and not all( map( lambda i: i != None, region_indices) ): + raise ValueError( "Could not determine proper column indices for chrom, start, end: %s" %( str( region_indices ) ) ) + return region_indices class ConvertedDatasetDataProvider( DatasetDataProvider ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Add missing os import to cli_shell/rsh
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/facc879fe054/ Changeset: facc879fe054 User: dannon Date: 2013-08-30 05:00:04 Summary: Add missing os import to cli_shell/rsh Affected #: 1 file diff -r 5e3567cdc08984012b6b26162a4ebe75e77e942b -r facc879fe0543f25e6b4d65e3e5d5efe716ff455 lib/galaxy/jobs/runners/cli_shell/rsh.py --- a/lib/galaxy/jobs/runners/cli_shell/rsh.py +++ b/lib/galaxy/jobs/runners/cli_shell/rsh.py @@ -2,10 +2,11 @@ Interface for remote shell commands (rsh, rcp) and derivatives that use the same syntax (ssh, scp) """ +import logging +import os +import subprocess +import tempfile import time -import logging -import tempfile -import subprocess from galaxy.util.bunch import Bunch from galaxy.jobs.runners.cli_shell import BaseShellExec Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: data_transfer workflow execute was using replacement_dict (which didn't exist in scope) instead of specifying it empty
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5e3567cdc089/ Changeset: 5e3567cdc089 User: dannon Date: 2013-08-30 04:58:30 Summary: data_transfer workflow execute was using replacement_dict (which didn't exist in scope) instead of specifying it empty Affected #: 1 file diff -r 1cd27d43ab14111326ab9601c7d818e6b15a3e74 -r 5e3567cdc08984012b6b26162a4ebe75e77e942b lib/galaxy/jobs/deferred/data_transfer.py --- a/lib/galaxy/jobs/deferred/data_transfer.py +++ b/lib/galaxy/jobs/deferred/data_transfer.py @@ -367,7 +367,7 @@ outputs[ step.id ] = out_data for pja in step.post_job_actions: if pja.action_type in ActionBox.immediate_actions: - ActionBox.execute(self.app, self.sa_session, pja, job, replacement_dict) + ActionBox.execute(self.app, self.sa_session, pja, job, replacement_dict=None) else: job.add_post_job_action(pja) else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Broken exception in transfer manager using error vs response['error']
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1cd27d43ab14/ Changeset: 1cd27d43ab14 User: dannon Date: 2013-08-30 04:55:56 Summary: Broken exception in transfer manager using error vs response['error'] Affected #: 1 file diff -r a1abf7b75d3e28ee0dc800a2ba13498e45d70ca2 -r 1cd27d43ab14111326ab9601c7d818e6b15a3e74 lib/galaxy/jobs/transfer_manager.py --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -90,7 +90,7 @@ raise Exception( dict( code=128, message='Did not receive valid response from transfer daemon for state' ) ) if 'error' in response: # Response was valid but Request resulted in an error - raise Exception( error ) + raise Exception( response['error']) else: # Request was valid response['result']['transfer_job_id'] = tj.id Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5623c1027c64/ Changeset: 5623c1027c64 User: dannon Date: 2013-08-30 04:35:20 Summary: Cleanup imports in sniff.py Affected #: 1 file diff -r ce2528c40e4d89642bc98c2aff7bdc6f524e4f03 -r 5623c1027c6490b84fb35a6dd3897337153f8ec5 lib/galaxy/datatypes/sniff.py --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -1,12 +1,21 @@ """ File format detector """ -import logging, sys, os, csv, tempfile, shutil, re, zipfile, gzip +import gzip +import logging +import os +import re import registry +import shutil +import sys +import tempfile +import zipfile + +from encodings import search_function as encodings_search_function + from galaxy import util -from galaxy.datatypes.checkers import * -from encodings import search_function as encodings_search_function -from binary import Binary +from galaxy.datatypes.checkers import check_binary, check_html, is_gzip +from galaxy.datatypes.binary import Binary log = logging.getLogger(__name__) @@ -405,5 +414,5 @@ pass if __name__ == '__main__': - import doctest, sys + import doctest doctest.testmod(sys.modules[__name__]) https://bitbucket.org/galaxy/galaxy-central/commits/53f78e82f398/ Changeset: 53f78e82f398 User: dannon Date: 2013-08-30 04:36:33 Summary: Flag duplicated code in workflows API (basically copied straight from controllers/workflow.py) -- needs refactor. Affected #: 1 file diff -r 5623c1027c6490b84fb35a6dd3897337153f8ec5 -r 53f78e82f398067349d2c37c1e0281846be72a31 lib/galaxy/webapps/galaxy/api/workflows.py --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -453,6 +453,11 @@ ### ----------------------------------- ### ## RPARK EDIT ## + + # TODO: This is duplicated from + # lib/galaxy/webapps/controllres/workflow.py -- refactor and + # eliminate copied code. + # Get user annotation. step_annotation = self.get_item_annotation_obj(trans.sa_session, trans.user, step ) annotation_str = "" https://bitbucket.org/galaxy/galaxy-central/commits/a1abf7b75d3e/ Changeset: a1abf7b75d3e User: dannon Date: 2013-08-30 04:38:22 Summary: Clarify in maf_utilities that there now no longer exists a duplicate method in datatypes/sequence.py. Affected #: 1 file diff -r 53f78e82f398067349d2c37c1e0281846be72a31 -r a1abf7b75d3e28ee0dc800a2ba13498e45d70ca2 lib/galaxy/tools/util/maf_utilities.py --- a/lib/galaxy/tools/util/maf_utilities.py +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -187,7 +187,6 @@ except: return build_maf_index( maf_file, species = species ) -#*** ANYCHANGE TO THIS METHOD HERE OR IN galaxy.datatypes.sequences MUST BE PROPAGATED *** def build_maf_index_species_chromosomes( filename, index_species = None ): species = [] species_chromosomes = {} Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Remove unused 'chunks' variable calculation from get_sequences_per_file
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ce2528c40e4d/ Changeset: ce2528c40e4d User: dannon Date: 2013-08-30 04:32:20 Summary: Remove unused 'chunks' variable calculation from get_sequences_per_file Affected #: 1 file diff -r 31b4f32c38443343612b54cc879739bf7bc019f4 -r ce2528c40e4d89642bc98c2aff7bdc6f524e4f03 lib/galaxy/datatypes/sequence.py --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -116,8 +116,6 @@ elif split_params['split_mode'] == 'to_size': # loop through the sections and calculate the number of sequences chunk_size = long(split_params['split_size']) - - chunks = total_sequences / chunk_size rem = total_sequences % chunk_size sequences_per_file = [chunk_size for i in range(total_sequences / chunk_size)] # TODO: Should we invest the time in a better way to handle small remainders? Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Import cleanup in sequence.py, remove import *.
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/31b4f32c3844/ Changeset: 31b4f32c3844 User: dannon Date: 2013-08-30 04:31:42 Summary: Import cleanup in sequence.py, remove import *. Affected #: 1 file diff -r d265a8a713092d3ba3fbca4d8db2cb93006d502f -r 31b4f32c38443343612b54cc879739bf7bc019f4 lib/galaxy/datatypes/sequence.py --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -2,18 +2,19 @@ Sequence classes """ +import data import gzip -import data import logging +import os import re import string -import os from cgi import escape + +from galaxy import eggs, util +from galaxy.datatypes import metadata +from galaxy.datatypes.checkers import is_gzip +from galaxy.datatypes.sniff import get_test_fname, get_headers from galaxy.datatypes.metadata import MetadataElement -from galaxy.datatypes import metadata -import galaxy.model -from galaxy import eggs, util -from sniff import * eggs.require("simplejson") import simplejson @@ -30,7 +31,7 @@ class SequenceSplitLocations( data.Text ): """ Class storing information about a sequence file composed of multiple gzip files concatenated as - one OR an uncompressed file. In the GZIP case, each sub-file's location is stored in start and end. + one OR an uncompressed file. In the GZIP case, each sub-file's location is stored in start and end. The format of the file is JSON:: @@ -174,7 +175,7 @@ directories.append(dir) return dir - # we know how many splits and how many sequences in each. What remains is to write out instructions for the + # we know how many splits and how many sequences in each. What remains is to write out instructions for the # splitting of all the input files. To decouple the format of those instructions from this code, the exact format of # those instructions is delegated to scripts start_sequence=0 @@ -197,7 +198,7 @@ start_sequence += sequences_per_file[part_no] return directories write_split_files = classmethod(write_split_files) - + def split( cls, input_datasets, subdir_generator_function, split_params): """Split a generic sequence file (not sensible or possible, see subclasses).""" if split_params is None: @@ -217,7 +218,7 @@ return None raise NotImplementedError("Can't split generic alignment files") - + class Fasta( Sequence ): """Class representing a FASTA sequence""" file_ext = "fasta" @@ -225,13 +226,13 @@ def sniff( self, filename ): """ Determines whether the file is in fasta format - - A sequence in FASTA format consists of a single-line description, followed by lines of sequence data. - The first character of the description line is a greater-than (">") symbol in the first column. + + A sequence in FASTA format consists of a single-line description, followed by lines of sequence data. + The first character of the description line is a greater-than (">") symbol in the first column. All lines should be shorter than 80 characters - + For complete details see http://www.ncbi.nlm.nih.gov/blast/fasta.shtml - + Rules for sniffing as True: We don't care about line length (other than empty lines). @@ -247,7 +248,7 @@ This should be done through sniff order, where csfasta (currently has a null sniff function) is detected for first (stricter definition) followed sometime after by fasta We will only check that the first purported sequence is correctly formatted. - + >>> fname = get_test_fname( 'sequence.maf' ) >>> Fasta().sniff( fname ) False @@ -255,7 +256,7 @@ >>> Fasta().sniff( fname ) True """ - + try: fh = open( filename ) while True: @@ -410,7 +411,7 @@ def sniff( self, filename ): """ - Color-space sequence: + Color-space sequence: >2_15_85_F3 T213021013012303002332212012112221222112212222 @@ -444,7 +445,7 @@ except: pass return False - + def set_meta( self, dataset, **kwd ): if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: dataset.metadata.data_lines = None @@ -474,7 +475,7 @@ if line and line.startswith( '#' ) and not sequences: # We don't count comment lines for sequence data types continue - if line and line.startswith( '@' ): + if line and line.startswith( '@' ): if seq_counter >= 4: # count previous block # blocks should be 4 lines long @@ -515,7 +516,7 @@ # Check the sequence line, make sure it contains only G/C/A/T/N if not bases_regexp.match( headers[1][0] ): return False - return True + return True return False except: return False @@ -556,7 +557,7 @@ output_name = data['output_name'] start_sequence = long(args['start_sequence']) sequence_count = long(args['num_sequences']) - + if 'toc_file' in args: toc_file = simplejson.load(open(args['toc_file'], 'r')) commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) @@ -588,7 +589,7 @@ class Maf( Alignment ): """Class describing a Maf alignment""" file_ext = "maf" - + #Readonly and optional, users can't unset it, but if it is not set, we are generally ok; if required use a metadata validator in the tool definition MetadataElement( name="blocks", default=0, desc="Number of blocks", readonly=True, optional=True, visible=False, no_value=0 ) MetadataElement( name="species_chromosomes", desc="Species Chromosomes", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) @@ -608,7 +609,7 @@ return #this is not a MAF file dataset.metadata.species = species dataset.metadata.blocks = blocks - + #write species chromosomes to a file chrom_file = dataset.metadata.species_chromosomes if not chrom_file: @@ -618,7 +619,7 @@ chrom_out.write( "%s\t%s\n" % ( spec, "\t".join( chroms ) ) ) chrom_out.close() dataset.metadata.species_chromosomes = chrom_file - + index_file = dataset.metadata.maf_index if not index_file: index_file = dataset.metadata.spec['maf_index'].param.new_file( dataset = dataset ) @@ -665,18 +666,18 @@ def sniff( self, filename ): """ Determines wether the file is in maf format - - The .maf format is line-oriented. Each multiple alignment ends with a blank line. - Each sequence in an alignment is on a single line, which can get quite long, but - there is no length limit. Words in a line are delimited by any white space. - Lines starting with # are considered to be comments. Lines starting with ## can + + The .maf format is line-oriented. Each multiple alignment ends with a blank line. + Each sequence in an alignment is on a single line, which can get quite long, but + there is no length limit. Words in a line are delimited by any white space. + Lines starting with # are considered to be comments. Lines starting with ## can be ignored by most programs, but contain meta-data of one form or another. - - The first line of a .maf file begins with ##maf. This word is followed by white-space-separated + + The first line of a .maf file begins with ##maf. This word is followed by white-space-separated variable=value pairs. There should be no white space surrounding the "=". - + For complete details see http://genome.ucsc.edu/FAQ/FAQformat#format5 - + >>> fname = get_test_fname( 'sequence.maf' ) >>> Maf().sniff( fname ) True @@ -696,11 +697,11 @@ class MafCustomTrack( data.Text ): file_ext = "mafcustomtrack" - + MetadataElement( name="vp_chromosome", default='chr1', desc="Viewport Chromosome", readonly=True, optional=True, visible=False, no_value='' ) MetadataElement( name="vp_start", default='1', desc="Viewport Start", readonly=True, optional=True, visible=False, no_value='' ) MetadataElement( name="vp_end", default='100', desc="Viewport End", readonly=True, optional=True, visible=False, no_value='' ) - + def set_meta( self, dataset, overwrite = True, **kwd ): """ Parses and sets viewport metadata from MAF file. @@ -723,7 +724,7 @@ forward_strand_end = max( forward_strand_end, ref_comp.forward_strand_end ) if i > max_block_check: break - + if forward_strand_end > forward_strand_start: dataset.metadata.vp_chromosome = chrom dataset.metadata.vp_start = forward_strand_start @@ -734,7 +735,7 @@ class Axt( data.Text ): """Class describing an axt alignment""" - + # gvk- 11/19/09 - This is really an alignment, but we no longer have tools that use this data type, and it is # here simply for backward compatibility ( although it is still in the datatypes registry ). Subclassing # from data.Text eliminates managing metadata elements inherited from the Alignemnt class. @@ -744,21 +745,21 @@ def sniff( self, filename ): """ Determines whether the file is in axt format - - axt alignment files are produced from Blastz, an alignment tool available from Webb Miller's lab + + axt alignment files are produced from Blastz, an alignment tool available from Webb Miller's lab at Penn State University. - + Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. - + The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields. - + The sequence lines contain the sequence of the primary assembly (line 2) and aligning assembly (line 3) with inserts. Repeats are indicated by lower-case letters. - + For complete details see http://genome.ucsc.edu/goldenPath/help/axt.html - + >>> fname = get_test_fname( 'alignment.axt' ) >>> Axt().sniff( fname ) True @@ -797,12 +798,12 @@ def sniff( self, filename ): """ Determines whether the file is in lav format - + LAV is an alignment format developed by Webb Miller's group. It is the primary output format for BLASTZ. The first line of a .lav file begins with #:lav. - + For complete details see http://www.bioperl.org/wiki/LAV_alignment_format - + >>> fname = get_test_fname( 'alignment.lav' ) >>> Lav().sniff( fname ) True Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ce558696ab74/ Changeset: ce558696ab74 User: dannon Date: 2013-08-30 04:28:04 Summary: Strip COPIED_FROM method in datatypes/sequence.py from maf_utilities; avoid circular import. Affected #: 1 file diff -r 638e011bd72d029f08d4f51ea32f6f47ec80b87f -r ce558696ab749b5500e027d3b601a5b715fdac3c lib/galaxy/datatypes/sequence.py --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -12,13 +12,19 @@ from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes import metadata import galaxy.model -from galaxy import util +from galaxy import eggs, util from sniff import * -import pkg_resources -pkg_resources.require("simplejson") +eggs.require("simplejson") import simplejson +try: + eggs.require( "bx-python" ) + import bx.align.maf +except: + pass + + log = logging.getLogger(__name__) class SequenceSplitLocations( data.Text ): @@ -579,90 +585,6 @@ """Class representing a Color Space FASTQ sequence ( e.g a SOLiD variant )""" file_ext = "fastqcssanger" -try: - from galaxy import eggs - import pkg_resources; pkg_resources.require( "bx-python" ) - import bx.align.maf -except: - pass - -#trying to import maf_utilities here throws an ImportError due to a circular import between jobs and tools: -#from galaxy.tools.util.maf_utilities import build_maf_index_species_chromosomes -#Traceback (most recent call last): -# File "./scripts/paster.py", line 27, in <module> -# command.run() -# File "build/bdist.solaris-2.11-i86pc/egg/paste/script/command.py", line 78, in run -# File "build/bdist.solaris-2.11-i86pc/egg/paste/script/command.py", line 117, in invoke -# File "build/bdist.solaris-2.11-i86pc/egg/paste/script/command.py", line 212, in run -# File "build/bdist.solaris-2.11-i86pc/egg/paste/script/serve.py", line 227, in command -# File "build/bdist.solaris-2.11-i86pc/egg/paste/script/serve.py", line 250, in loadapp -# File "build/bdist.solaris-2.11-i86pc/egg/paste/deploy/loadwsgi.py", line 193, in loadapp -# File "build/bdist.solaris-2.11-i86pc/egg/paste/deploy/loadwsgi.py", line 213, in loadobj -# File "build/bdist.solaris-2.11-i86pc/egg/paste/deploy/loadwsgi.py", line 237, in loadcontext -# File "build/bdist.solaris-2.11-i86pc/egg/paste/deploy/loadwsgi.py", line 267, in _loadconfig -# File "build/bdist.solaris-2.11-i86pc/egg/paste/deploy/loadwsgi.py", line 397, in get_context -# File "build/bdist.solaris-2.11-i86pc/egg/paste/deploy/loadwsgi.py", line 439, in _context_from_explicit -# File "build/bdist.solaris-2.11-i86pc/egg/paste/deploy/loadwsgi.py", line 18, in import_string -# File "/afs/bx.psu.edu/home/dan/galaxy/central/lib/pkg_resources.py", line 1912, in load -# entry = __import__(self.module_name, globals(),globals(), ['__name__']) -# File "/afs/bx.psu.edu/home/dan/galaxy/central/lib/galaxy/web/buildapp.py", line 18, in <module> -# from galaxy import config, jobs, util, tools -# File "/afs/bx.psu.edu/home/dan/galaxy/central/lib/galaxy/jobs/__init__.py", line 3, in <module> -# from galaxy import util, model -# File "/afs/bx.psu.edu/home/dan/galaxy/central/lib/galaxy/model/__init__.py", line 13, in <module> -# import galaxy.datatypes.registry -# File "/afs/bx.psu.edu/home/dan/galaxy/central/lib/galaxy/datatypes/registry.py", line 6, in <module> -# import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo -# File "/afs/bx.psu.edu/home/dan/galaxy/central/lib/galaxy/datatypes/sequence.py", line 344, in <module> -# from galaxy.tools.util.maf_utilities import build_maf_index_species_chromosomes -# File "/afs/bx.psu.edu/home/dan/galaxy/central/lib/galaxy/tools/__init__.py", line 15, in <module> -# from galaxy import util, jobs, model -#ImportError: cannot import name jobs -#so we'll copy and paste for now...terribly icky -#*** ANYCHANGE TO THIS METHOD HERE OR IN maf_utilities MUST BE PROPAGATED *** -def COPIED_build_maf_index_species_chromosomes( filename, index_species = None ): - species = [] - species_chromosomes = {} - indexes = bx.interval_index_file.Indexes() - blocks = 0 - try: - maf_reader = bx.align.maf.Reader( open( filename ) ) - while True: - pos = maf_reader.file.tell() - block = maf_reader.next() - if block is None: - break - blocks += 1 - for c in block.components: - spec = c.src - chrom = None - if "." in spec: - spec, chrom = spec.split( ".", 1 ) - if spec not in species: - species.append( spec ) - species_chromosomes[spec] = [] - if chrom and chrom not in species_chromosomes[spec]: - species_chromosomes[spec].append( chrom ) - if index_species is None or spec in index_species: - forward_strand_start = c.forward_strand_start - forward_strand_end = c.forward_strand_end - try: - forward_strand_start = int( forward_strand_start ) - forward_strand_end = int( forward_strand_end ) - except ValueError: - continue #start and end are not integers, can't add component to index, goto next component - #this likely only occurs when parse_e_rows is True? - #could a species exist as only e rows? should the - if forward_strand_end > forward_strand_start: - #require positive length; i.e. certain lines have start = end = 0 and cannot be indexed - indexes.add( c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size ) - except Exception, e: - #most likely a bad MAF - log.debug( 'Building MAF index on %s failed: %s' % ( filename, e ) ) - return ( None, [], {}, 0 ) - return ( indexes, species, species_chromosomes, blocks ) - - class Maf( Alignment ): """Class describing a Maf alignment""" file_ext = "maf" @@ -679,7 +601,9 @@ Parses and sets species, chromosomes, index from MAF file. """ #these metadata values are not accessable by users, always overwrite - indexes, species, species_chromosomes, blocks = COPIED_build_maf_index_species_chromosomes( dataset.file_name ) + #Imported here to avoid circular dependency + from galaxy.tools.util.maf_utilities import build_maf_index_species_chromosomes + indexes, species, species_chromosomes, blocks = build_maf_index_species_chromosomes( dataset.file_name ) if indexes is None: return #this is not a MAF file dataset.metadata.species = species https://bitbucket.org/galaxy/galaxy-central/commits/d265a8a71309/ Changeset: d265a8a71309 User: dannon Date: 2013-08-30 04:28:23 Summary: Whitespace cleanup in datatypes/sequence.py Affected #: 1 file diff -r ce558696ab749b5500e027d3b601a5b715fdac3c -r d265a8a713092d3ba3fbca4d8db2cb93006d502f lib/galaxy/datatypes/sequence.py --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -30,7 +30,7 @@ class SequenceSplitLocations( data.Text ): """ Class storing information about a sequence file composed of multiple gzip files concatenated as - one OR an uncompressed file. In the GZIP case, each sub-file's location is stored in start and end. + one OR an uncompressed file. In the GZIP case, each sub-file's location is stored in start and end. The format of the file is JSON:: @@ -174,7 +174,7 @@ directories.append(dir) return dir - # we know how many splits and how many sequences in each. What remains is to write out instructions for the + # we know how many splits and how many sequences in each. What remains is to write out instructions for the # splitting of all the input files. To decouple the format of those instructions from this code, the exact format of # those instructions is delegated to scripts start_sequence=0 @@ -197,7 +197,7 @@ start_sequence += sequences_per_file[part_no] return directories write_split_files = classmethod(write_split_files) - + def split( cls, input_datasets, subdir_generator_function, split_params): """Split a generic sequence file (not sensible or possible, see subclasses).""" if split_params is None: @@ -217,7 +217,7 @@ return None raise NotImplementedError("Can't split generic alignment files") - + class Fasta( Sequence ): """Class representing a FASTA sequence""" file_ext = "fasta" @@ -225,13 +225,13 @@ def sniff( self, filename ): """ Determines whether the file is in fasta format - - A sequence in FASTA format consists of a single-line description, followed by lines of sequence data. - The first character of the description line is a greater-than (">") symbol in the first column. + + A sequence in FASTA format consists of a single-line description, followed by lines of sequence data. + The first character of the description line is a greater-than (">") symbol in the first column. All lines should be shorter than 80 characters - + For complete details see http://www.ncbi.nlm.nih.gov/blast/fasta.shtml - + Rules for sniffing as True: We don't care about line length (other than empty lines). @@ -247,7 +247,7 @@ This should be done through sniff order, where csfasta (currently has a null sniff function) is detected for first (stricter definition) followed sometime after by fasta We will only check that the first purported sequence is correctly formatted. - + >>> fname = get_test_fname( 'sequence.maf' ) >>> Fasta().sniff( fname ) False @@ -255,7 +255,7 @@ >>> Fasta().sniff( fname ) True """ - + try: fh = open( filename ) while True: @@ -410,7 +410,7 @@ def sniff( self, filename ): """ - Color-space sequence: + Color-space sequence: >2_15_85_F3 T213021013012303002332212012112221222112212222 @@ -444,7 +444,7 @@ except: pass return False - + def set_meta( self, dataset, **kwd ): if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: dataset.metadata.data_lines = None @@ -474,7 +474,7 @@ if line and line.startswith( '#' ) and not sequences: # We don't count comment lines for sequence data types continue - if line and line.startswith( '@' ): + if line and line.startswith( '@' ): if seq_counter >= 4: # count previous block # blocks should be 4 lines long @@ -515,7 +515,7 @@ # Check the sequence line, make sure it contains only G/C/A/T/N if not bases_regexp.match( headers[1][0] ): return False - return True + return True return False except: return False @@ -556,7 +556,7 @@ output_name = data['output_name'] start_sequence = long(args['start_sequence']) sequence_count = long(args['num_sequences']) - + if 'toc_file' in args: toc_file = simplejson.load(open(args['toc_file'], 'r')) commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) @@ -588,7 +588,7 @@ class Maf( Alignment ): """Class describing a Maf alignment""" file_ext = "maf" - + #Readonly and optional, users can't unset it, but if it is not set, we are generally ok; if required use a metadata validator in the tool definition MetadataElement( name="blocks", default=0, desc="Number of blocks", readonly=True, optional=True, visible=False, no_value=0 ) MetadataElement( name="species_chromosomes", desc="Species Chromosomes", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) @@ -608,7 +608,7 @@ return #this is not a MAF file dataset.metadata.species = species dataset.metadata.blocks = blocks - + #write species chromosomes to a file chrom_file = dataset.metadata.species_chromosomes if not chrom_file: @@ -618,7 +618,7 @@ chrom_out.write( "%s\t%s\n" % ( spec, "\t".join( chroms ) ) ) chrom_out.close() dataset.metadata.species_chromosomes = chrom_file - + index_file = dataset.metadata.maf_index if not index_file: index_file = dataset.metadata.spec['maf_index'].param.new_file( dataset = dataset ) @@ -665,18 +665,18 @@ def sniff( self, filename ): """ Determines wether the file is in maf format - - The .maf format is line-oriented. Each multiple alignment ends with a blank line. - Each sequence in an alignment is on a single line, which can get quite long, but - there is no length limit. Words in a line are delimited by any white space. - Lines starting with # are considered to be comments. Lines starting with ## can + + The .maf format is line-oriented. Each multiple alignment ends with a blank line. + Each sequence in an alignment is on a single line, which can get quite long, but + there is no length limit. Words in a line are delimited by any white space. + Lines starting with # are considered to be comments. Lines starting with ## can be ignored by most programs, but contain meta-data of one form or another. - - The first line of a .maf file begins with ##maf. This word is followed by white-space-separated + + The first line of a .maf file begins with ##maf. This word is followed by white-space-separated variable=value pairs. There should be no white space surrounding the "=". - + For complete details see http://genome.ucsc.edu/FAQ/FAQformat#format5 - + >>> fname = get_test_fname( 'sequence.maf' ) >>> Maf().sniff( fname ) True @@ -696,11 +696,11 @@ class MafCustomTrack( data.Text ): file_ext = "mafcustomtrack" - + MetadataElement( name="vp_chromosome", default='chr1', desc="Viewport Chromosome", readonly=True, optional=True, visible=False, no_value='' ) MetadataElement( name="vp_start", default='1', desc="Viewport Start", readonly=True, optional=True, visible=False, no_value='' ) MetadataElement( name="vp_end", default='100', desc="Viewport End", readonly=True, optional=True, visible=False, no_value='' ) - + def set_meta( self, dataset, overwrite = True, **kwd ): """ Parses and sets viewport metadata from MAF file. @@ -723,7 +723,7 @@ forward_strand_end = max( forward_strand_end, ref_comp.forward_strand_end ) if i > max_block_check: break - + if forward_strand_end > forward_strand_start: dataset.metadata.vp_chromosome = chrom dataset.metadata.vp_start = forward_strand_start @@ -734,7 +734,7 @@ class Axt( data.Text ): """Class describing an axt alignment""" - + # gvk- 11/19/09 - This is really an alignment, but we no longer have tools that use this data type, and it is # here simply for backward compatibility ( although it is still in the datatypes registry ). Subclassing # from data.Text eliminates managing metadata elements inherited from the Alignemnt class. @@ -744,21 +744,21 @@ def sniff( self, filename ): """ Determines whether the file is in axt format - - axt alignment files are produced from Blastz, an alignment tool available from Webb Miller's lab + + axt alignment files are produced from Blastz, an alignment tool available from Webb Miller's lab at Penn State University. - + Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. - + The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields. - + The sequence lines contain the sequence of the primary assembly (line 2) and aligning assembly (line 3) with inserts. Repeats are indicated by lower-case letters. - + For complete details see http://genome.ucsc.edu/goldenPath/help/axt.html - + >>> fname = get_test_fname( 'alignment.axt' ) >>> Axt().sniff( fname ) True @@ -797,12 +797,12 @@ def sniff( self, filename ): """ Determines whether the file is in lav format - + LAV is an alignment format developed by Webb Miller's group. It is the primary output format for BLASTZ. The first line of a .lav file begins with #:lav. - + For complete details see http://www.bioperl.org/wiki/LAV_alignment_format - + >>> fname = get_test_fname( 'alignment.lav' ) >>> Lav().sniff( fname ) True Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Fix maf_utilities formatting, removing windows terminators.
by commits-noreply＠bitbucket.org 30 Aug '13

30 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/638e011bd72d/ Changeset: 638e011bd72d User: dannon Date: 2013-08-30 04:08:18 Summary: Fix maf_utilities formatting, removing windows terminators. Affected #: 1 file diff -r bec1baa9d2db009abd16778093fb15b96bde0950 -r 638e011bd72d029f08d4f51ea32f6f47ec80b87f lib/galaxy/tools/util/maf_utilities.py --- a/lib/galaxy/tools/util/maf_utilities.py +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -1,17 +1,17 @@ -#!/usr/bin/env python -""" -Provides wrappers and utilities for working with MAF files and alignments. -""" -#Dan Blankenberg -import pkg_resources; pkg_resources.require( "bx-python" ) -import bx.align.maf -import bx.intervals -import bx.interval_index_file +#!/usr/bin/env python +""" +Provides wrappers and utilities for working with MAF files and alignments. +""" +#Dan Blankenberg +import pkg_resources; pkg_resources.require( "bx-python" ) +import bx.align.maf +import bx.intervals +import bx.interval_index_file import sys, os, string, tempfile import logging -from copy import deepcopy - -assert sys.version_info[:2] >= ( 2, 4 ) +from copy import deepcopy + +assert sys.version_info[:2] >= ( 2, 4 ) log = logging.getLogger(__name__) @@ -26,7 +26,7 @@ chrom = fields.pop( 0 ) else: chrom = spec - return spec, chrom + return spec, chrom def src_merge( spec, chrom, contig = None ): if None in [ spec, chrom ]: @@ -45,140 +45,140 @@ print >> sys.stderr, "Fatal Error: %s" % msg sys.exit() -#an object corresponding to a reference layered alignment -class RegionAlignment( object ): - - DNA_COMPLEMENT = string.maketrans( "ACGTacgt", "TGCAtgca" ) - MAX_SEQUENCE_SIZE = sys.maxint #Maximum length of sequence allowed - - def __init__( self, size, species = [] ): - assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % ( size, self.MAX_SEQUENCE_SIZE ) - self.size = size - self.sequences = {} - if not isinstance( species, list ): - species = [species] - for spec in species: - self.add_species( spec ) - - #add a species to the alignment - def add_species( self, species ): - #make temporary sequence files - self.sequences[species] = tempfile.TemporaryFile() - self.sequences[species].write( "-" * self.size ) - - #returns the names for species found in alignment, skipping names as requested - def get_species_names( self, skip = [] ): - if not isinstance( skip, list ): skip = [skip] - names = self.sequences.keys() - for name in skip: - try: names.remove( name ) - except: pass - return names - - #returns the sequence for a species - def get_sequence( self, species ): - self.sequences[species].seek( 0 ) - return self.sequences[species].read() - - #returns the reverse complement of the sequence for a species - def get_sequence_reverse_complement( self, species ): - complement = [base for base in self.get_sequence( species ).translate( self.DNA_COMPLEMENT )] - complement.reverse() - return "".join( complement ) - - #sets a position for a species - def set_position( self, index, species, base ): - if len( base ) != 1: raise Exception( "A genomic position can only have a length of 1." ) - return self.set_range( index, species, base ) - #sets a range for a species - def set_range( self, index, species, bases ): - if index >= self.size or index < 0: raise Exception( "Your index (%i) is out of range (0 - %i)." % ( index, self.size - 1 ) ) - if len( bases ) == 0: raise Exception( "A set of genomic positions can only have a positive length." ) - if species not in self.sequences.keys(): self.add_species( species ) - self.sequences[species].seek( index ) - self.sequences[species].write( bases ) - - #Flush temp file of specified species, or all species - def flush( self, species = None ): - if species is None: - species = self.sequences.keys() - elif not isinstance( species, list ): - species = [species] - for spec in species: - self.sequences[spec].flush() - -class GenomicRegionAlignment( RegionAlignment ): - - def __init__( self, start, end, species = [] ): - RegionAlignment.__init__( self, end - start, species ) - self.start = start - self.end = end - -class SplicedAlignment( object ): - - DNA_COMPLEMENT = string.maketrans( "ACGTacgt", "TGCAtgca" ) - - def __init__( self, exon_starts, exon_ends, species = [] ): - if not isinstance( exon_starts, list ): - exon_starts = [exon_starts] - if not isinstance( exon_ends, list ): - exon_ends = [exon_ends] - assert len( exon_starts ) == len( exon_ends ), "The number of starts does not match the number of sizes." - self.exons = [] - for i in range( len( exon_starts ) ): - self.exons.append( GenomicRegionAlignment( exon_starts[i], exon_ends[i], species ) ) - - #returns the names for species found in alignment, skipping names as requested - def get_species_names( self, skip = [] ): - if not isinstance( skip, list ): skip = [skip] - names = [] - for exon in self.exons: - for name in exon.get_species_names( skip = skip ): - if name not in names: - names.append( name ) - return names - - #returns the sequence for a species - def get_sequence( self, species ): - sequence = tempfile.TemporaryFile() - for exon in self.exons: - if species in exon.get_species_names(): - sequence.write( exon.get_sequence( species ) ) - else: - sequence.write( "-" * exon.size ) - sequence.seek( 0 ) - return sequence.read() - - #returns the reverse complement of the sequence for a species - def get_sequence_reverse_complement( self, species ): - complement = [base for base in self.get_sequence( species ).translate( self.DNA_COMPLEMENT )] - complement.reverse() - return "".join( complement ) - - #Start and end of coding region - @property - def start( self ): - return self.exons[0].start - @property - def end( self ): - return self.exons[-1].end - -#Open a MAF index using a UID -def maf_index_by_uid( maf_uid, index_location_file ): - for line in open( index_location_file ): - try: - #read each line, if not enough fields, go to next line - if line[0:1] == "#" : continue - fields = line.split('\t') - if maf_uid == fields[1]: - try: - maf_files = fields[4].replace( "\n", "" ).replace( "\r", "" ).split( "," ) - return bx.align.maf.MultiIndexed( maf_files, keep_open = True, parse_e_rows = False ) - except Exception, e: - raise Exception( 'MAF UID (%s) found, but configuration appears to be malformed: %s' % ( maf_uid, e ) ) - except: - pass - return None +#an object corresponding to a reference layered alignment +class RegionAlignment( object ): + + DNA_COMPLEMENT = string.maketrans( "ACGTacgt", "TGCAtgca" ) + MAX_SEQUENCE_SIZE = sys.maxint #Maximum length of sequence allowed + + def __init__( self, size, species = [] ): + assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % ( size, self.MAX_SEQUENCE_SIZE ) + self.size = size + self.sequences = {} + if not isinstance( species, list ): + species = [species] + for spec in species: + self.add_species( spec ) + + #add a species to the alignment + def add_species( self, species ): + #make temporary sequence files + self.sequences[species] = tempfile.TemporaryFile() + self.sequences[species].write( "-" * self.size ) + + #returns the names for species found in alignment, skipping names as requested + def get_species_names( self, skip = [] ): + if not isinstance( skip, list ): skip = [skip] + names = self.sequences.keys() + for name in skip: + try: names.remove( name ) + except: pass + return names + + #returns the sequence for a species + def get_sequence( self, species ): + self.sequences[species].seek( 0 ) + return self.sequences[species].read() + + #returns the reverse complement of the sequence for a species + def get_sequence_reverse_complement( self, species ): + complement = [base for base in self.get_sequence( species ).translate( self.DNA_COMPLEMENT )] + complement.reverse() + return "".join( complement ) + + #sets a position for a species + def set_position( self, index, species, base ): + if len( base ) != 1: raise Exception( "A genomic position can only have a length of 1." ) + return self.set_range( index, species, base ) + #sets a range for a species + def set_range( self, index, species, bases ): + if index >= self.size or index < 0: raise Exception( "Your index (%i) is out of range (0 - %i)." % ( index, self.size - 1 ) ) + if len( bases ) == 0: raise Exception( "A set of genomic positions can only have a positive length." ) + if species not in self.sequences.keys(): self.add_species( species ) + self.sequences[species].seek( index ) + self.sequences[species].write( bases ) + + #Flush temp file of specified species, or all species + def flush( self, species = None ): + if species is None: + species = self.sequences.keys() + elif not isinstance( species, list ): + species = [species] + for spec in species: + self.sequences[spec].flush() + +class GenomicRegionAlignment( RegionAlignment ): + + def __init__( self, start, end, species = [] ): + RegionAlignment.__init__( self, end - start, species ) + self.start = start + self.end = end + +class SplicedAlignment( object ): + + DNA_COMPLEMENT = string.maketrans( "ACGTacgt", "TGCAtgca" ) + + def __init__( self, exon_starts, exon_ends, species = [] ): + if not isinstance( exon_starts, list ): + exon_starts = [exon_starts] + if not isinstance( exon_ends, list ): + exon_ends = [exon_ends] + assert len( exon_starts ) == len( exon_ends ), "The number of starts does not match the number of sizes." + self.exons = [] + for i in range( len( exon_starts ) ): + self.exons.append( GenomicRegionAlignment( exon_starts[i], exon_ends[i], species ) ) + + #returns the names for species found in alignment, skipping names as requested + def get_species_names( self, skip = [] ): + if not isinstance( skip, list ): skip = [skip] + names = [] + for exon in self.exons: + for name in exon.get_species_names( skip = skip ): + if name not in names: + names.append( name ) + return names + + #returns the sequence for a species + def get_sequence( self, species ): + sequence = tempfile.TemporaryFile() + for exon in self.exons: + if species in exon.get_species_names(): + sequence.write( exon.get_sequence( species ) ) + else: + sequence.write( "-" * exon.size ) + sequence.seek( 0 ) + return sequence.read() + + #returns the reverse complement of the sequence for a species + def get_sequence_reverse_complement( self, species ): + complement = [base for base in self.get_sequence( species ).translate( self.DNA_COMPLEMENT )] + complement.reverse() + return "".join( complement ) + + #Start and end of coding region + @property + def start( self ): + return self.exons[0].start + @property + def end( self ): + return self.exons[-1].end + +#Open a MAF index using a UID +def maf_index_by_uid( maf_uid, index_location_file ): + for line in open( index_location_file ): + try: + #read each line, if not enough fields, go to next line + if line[0:1] == "#" : continue + fields = line.split('\t') + if maf_uid == fields[1]: + try: + maf_files = fields[4].replace( "\n", "" ).replace( "\r", "" ).split( "," ) + return bx.align.maf.MultiIndexed( maf_files, keep_open = True, parse_e_rows = False ) + except Exception, e: + raise Exception( 'MAF UID (%s) found, but configuration appears to be malformed: %s' % ( maf_uid, e ) ) + except: + pass + return None #return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created def open_or_build_maf_index( maf_file, index_filename, species = None ): @@ -186,27 +186,27 @@ return ( bx.align.maf.Indexed( maf_file, index_filename = index_filename, keep_open = True, parse_e_rows = False ), None ) except: return build_maf_index( maf_file, species = species ) - + #*** ANYCHANGE TO THIS METHOD HERE OR IN galaxy.datatypes.sequences MUST BE PROPAGATED *** def build_maf_index_species_chromosomes( filename, index_species = None ): species = [] species_chromosomes = {} - indexes = bx.interval_index_file.Indexes() + indexes = bx.interval_index_file.Indexes() blocks = 0 try: maf_reader = bx.align.maf.Reader( open( filename ) ) while True: pos = maf_reader.file.tell() block = maf_reader.next() - if block is None: - break + if block is None: + break blocks += 1 for c in block.components: spec = c.src chrom = None if "." in spec: spec, chrom = spec.split( ".", 1 ) - if spec not in species: + if spec not in species: species.append( spec ) species_chromosomes[spec] = [] if chrom and chrom not in species_chromosomes[spec]: @@ -229,17 +229,17 @@ log.debug( 'Building MAF index on %s failed: %s' % ( filename, e ) ) return ( None, [], {}, 0 ) return ( indexes, species, species_chromosomes, blocks ) - -#builds and returns ( index, index_filename ) for specified maf_file -def build_maf_index( maf_file, species = None ): + +#builds and returns ( index, index_filename ) for specified maf_file +def build_maf_index( maf_file, species = None ): indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes( maf_file, species ) if indexes is not None: - fd, index_filename = tempfile.mkstemp() - out = os.fdopen( fd, 'w' ) - indexes.write( out ) - out.close() + fd, index_filename = tempfile.mkstemp() + out = os.fdopen( fd, 'w' ) + indexes.write( out ) + out.close() return ( bx.align.maf.Indexed( maf_file, index_filename = index_filename, keep_open = True, parse_e_rows = False ), index_filename ) - return ( None, None ) + return ( None, None ) def component_overlaps_region( c, region ): if c is None: return False @@ -276,7 +276,7 @@ start = end - slice_len slice_start = min( start, slice_start ) slice_end = max( end, slice_end ) - + if slice_start < slice_end: block = block.slice( slice_start, slice_end ) if block.text_size > mincols: @@ -287,7 +287,7 @@ block.remove_all_gap_columns() return block return None - + def orient_block_by_region( block, src, region, force_strand = None ): #loop through components matching src, #make sure each of these components overlap region @@ -295,12 +295,12 @@ #if force_strand / region.strand not in strand cache, reverse complement ### we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing strands = [ c.strand for c in iter_components_by_src( block, src ) if component_overlaps_region( c, region ) ] - if strands and ( force_strand is None and region.strand not in strands ) or ( force_strand is not None and force_strand not in strands ): + if strands and ( force_strand is None and region.strand not in strands ) or ( force_strand is not None and force_strand not in strands ): block = block.reverse_complement() return block def get_oriented_chopped_blocks_for_region( index, src, region, species = None, mincols = 0, force_strand = None ): - for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region( index, src, region, species, mincols, force_strand ): + for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region( index, src, region, species, mincols, force_strand ): yield block def get_oriented_chopped_blocks_with_index_offset_for_region( index, src, region, species = None, mincols = 0, force_strand = None ): for block, idx, offset in get_chopped_blocks_with_index_offset_for_region( index, src, region, species, mincols ): @@ -331,8 +331,8 @@ else: #no more components to add, yield this block yield new_block - - #divide components by species + + #divide components by species spec_dict = {} if not species: species = [] @@ -347,7 +347,7 @@ spec_dict[ spec ] = [] for c in iter_components_by_src_start( block, spec ): spec_dict[ spec ].append( c ) - + empty_block = bx.align.Alignment( score=block.score, attributes=deepcopy( block.attributes ) ) #should we copy attributes? empty_block.text_size = block.text_size #call recursive function to split into each combo of spec/blocks @@ -356,68 +356,69 @@ yield value -#generator yielding only chopped and valid blocks for a specified region -def get_chopped_blocks_for_region( index, src, region, species = None, mincols = 0 ): - for block, idx, offset in get_chopped_blocks_with_index_offset_for_region( index, src, region, species, mincols ): - yield block -def get_chopped_blocks_with_index_offset_for_region( index, src, region, species = None, mincols = 0 ): - for block, idx, offset in index.get_as_iterator_with_index_and_offset( src, region.start, region.end ): - block = chop_block_by_region( block, src, region, species, mincols ) - if block is not None: - yield block, idx, offset - -#returns a filled region alignment for specified regions -def get_region_alignment( index, primary_species, chrom, start, end, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True ): - if species is not None: alignment = RegionAlignment( end - start, species ) - else: alignment = RegionAlignment( end - start, primary_species ) - return fill_region_alignment( alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps ) - -#reduces a block to only positions exisiting in the src provided -def reduce_block_by_primary_genome( block, species, chromosome, region_start ): - #returns ( startIndex, {species:texts} - #where texts' contents are reduced to only positions existing in the primary genome - src = "%s.%s" % ( species, chromosome ) - ref = block.get_component_by_src( src ) - start_offset = ref.start - region_start - species_texts = {} - for c in block.components: - species_texts[ c.src.split( '.' )[0] ] = list( c.text ) - #remove locations which are gaps in the primary species, starting from the downstream end - for i in range( len( species_texts[ species ] ) - 1, -1, -1 ): - if species_texts[ species ][i] == '-': - for text in species_texts.values(): - text.pop( i ) - for spec, text in species_texts.items(): - species_texts[spec] = ''.join( text ) - return ( start_offset, species_texts ) - -#fills a region alignment -def fill_region_alignment( alignment, index, primary_species, chrom, start, end, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True ): - region = bx.intervals.Interval( start, end ) - region.chrom = chrom - region.strand = strand - primary_src = "%s.%s" % ( primary_species, chrom ) - - #Order blocks overlaping this position by score, lowest first - blocks = [] - for block, idx, offset in index.get_as_iterator_with_index_and_offset( primary_src, start, end ): - score = float( block.score ) - for i in range( 0, len( blocks ) ): - if score < blocks[i][0]: - blocks.insert( i, ( score, idx, offset ) ) - break - else: - blocks.append( ( score, idx, offset ) ) - +#generator yielding only chopped and valid blocks for a specified region +def get_chopped_blocks_for_region( index, src, region, species = None, mincols = 0 ): + for block, idx, offset in get_chopped_blocks_with_index_offset_for_region( index, src, region, species, mincols ): + yield block +def get_chopped_blocks_with_index_offset_for_region( index, src, region, species = None, mincols = 0 ): + for block, idx, offset in index.get_as_iterator_with_index_and_offset( src, region.start, region.end ): + block = chop_block_by_region( block, src, region, species, mincols ) + if block is not None: + yield block, idx, offset + +#returns a filled region alignment for specified regions +def get_region_alignment( index, primary_species, chrom, start, end, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True ): + if species is not None: alignment = RegionAlignment( end - start, species ) + else: alignment = RegionAlignment( end - start, primary_species ) + return fill_region_alignment( alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps ) + +#reduces a block to only positions exisiting in the src provided +def reduce_block_by_primary_genome( block, species, chromosome, region_start ): + #returns ( startIndex, {species:texts} + #where texts' contents are reduced to only positions existing in the primary genome + src = "%s.%s" % ( species, chromosome ) + ref = block.get_component_by_src( src ) + start_offset = ref.start - region_start + species_texts = {} + for c in block.components: + species_texts[ c.src.split( '.' )[0] ] = list( c.text ) + #remove locations which are gaps in the primary species, starting from the downstream end + for i in range( len( species_texts[ species ] ) - 1, -1, -1 ): + if species_texts[ species ][i] == '-': + for text in species_texts.values(): + text.pop( i ) + for spec, text in species_texts.items(): + species_texts[spec] = ''.join( text ) + return ( start_offset, species_texts ) + +#fills a region alignment +def fill_region_alignment( alignment, index, primary_species, chrom, start, end, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True ): + region = bx.intervals.Interval( start, end ) + region.chrom = chrom + region.strand = strand + primary_src = "%s.%s" % ( primary_species, chrom ) + + #Order blocks overlaping this position by score, lowest first + blocks = [] + for block, idx, offset in index.get_as_iterator_with_index_and_offset( primary_src, start, end ): + score = float( block.score ) + for i in range( 0, len( blocks ) ): + if score < blocks[i][0]: + blocks.insert( i, ( score, idx, offset ) ) + break + else: + blocks.append( ( score, idx, offset ) ) + #gap_chars_tuple = tuple( GAP_CHARS ) - gap_chars_str = ''.join( GAP_CHARS ) - #Loop through ordered blocks and layer by increasing score - for block_dict in blocks: for block in iter_blocks_split_by_species( block_dict[1].get_at_offset( block_dict[2] ) ): #need to handle each occurance of sequence in block seperately + gap_chars_str = ''.join( GAP_CHARS ) + #Loop through ordered blocks and layer by increasing score + for block_dict in blocks: + for block in iter_blocks_split_by_species( block_dict[1].get_at_offset( block_dict[2] ) ): #need to handle each occurance of sequence in block seperately if component_overlaps_region( block.get_component_by_src( primary_src ), region ): block = chop_block_by_region( block, primary_src, region, species, mincols ) #chop block block = orient_block_by_region( block, primary_src, region ) #orient block - start_offset, species_texts = reduce_block_by_primary_genome( block, primary_species, chrom, start ) - for spec, text in species_texts.items(): + start_offset, species_texts = reduce_block_by_primary_genome( block, primary_species, chrom, start ) + for spec, text in species_texts.items(): #we should trim gaps from both sides, since these are not positions in this species genome (sequence) text = text.rstrip( gap_chars_str ) gap_offset = 0 @@ -433,169 +434,169 @@ else: for i, char in enumerate( text ): if char not in GAP_CHARS: - alignment.set_position( start_offset + gap_offset + i, spec, char ) - return alignment - -#returns a filled spliced region alignment for specified region with start and end lists -def get_spliced_region_alignment( index, primary_species, chrom, starts, ends, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True ): - #create spliced alignment object - if species is not None: alignment = SplicedAlignment( starts, ends, species ) - else: alignment = SplicedAlignment( starts, ends, [primary_species] ) - for exon in alignment.exons: - fill_region_alignment( exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps ) - return alignment - -#loop through string array, only return non-commented lines -def line_enumerator( lines, comment_start = '#' ): - i = 0 - for line in lines: - if not line.startswith( comment_start ): - i += 1 - yield ( i, line ) - -#read a GeneBed file, return list of starts, ends, raw fields -def get_starts_ends_fields_from_gene_bed( line ): - #Starts and ends for exons - starts = [] - ends = [] - - fields = line.split() - #Requires atleast 12 BED columns - if len(fields) < 12: - raise Exception( "Not a proper 12 column BED line (%s)." % line ) - chrom = fields[0] - tx_start = int( fields[1] ) - tx_end = int( fields[2] ) - name = fields[3] - strand = fields[5] - if strand != '-': strand='+' #Default strand is + - cds_start = int( fields[6] ) - cds_end = int( fields[7] ) - - #Calculate and store starts and ends of coding exons - region_start, region_end = cds_start, cds_end - exon_starts = map( int, fields[11].rstrip( ',\n' ).split( ',' ) ) - exon_starts = map( ( lambda x: x + tx_start ), exon_starts ) - exon_ends = map( int, fields[10].rstrip( ',' ).split( ',' ) ) - exon_ends = map( ( lambda x, y: x + y ), exon_starts, exon_ends ); - for start, end in zip( exon_starts, exon_ends ): - start = max( start, region_start ) - end = min( end, region_end ) - if start < end: - starts.append( start ) - ends.append( end ) - return ( starts, ends, fields ) - -def iter_components_by_src( block, src ): - for c in block.components: - if c.src == src: - yield c - -def get_components_by_src( block, src ): - return [ value for value in iter_components_by_src( block, src ) ] - -def iter_components_by_src_start( block, src ): - for c in block.components: - if c.src.startswith( src ): - yield c - -def get_components_by_src_start( block, src ): - return [ value for value in iter_components_by_src_start( block, src ) ] + alignment.set_position( start_offset + gap_offset + i, spec, char ) + return alignment + +#returns a filled spliced region alignment for specified region with start and end lists +def get_spliced_region_alignment( index, primary_species, chrom, starts, ends, strand = '+', species = None, mincols = 0, overwrite_with_gaps = True ): + #create spliced alignment object + if species is not None: alignment = SplicedAlignment( starts, ends, species ) + else: alignment = SplicedAlignment( starts, ends, [primary_species] ) + for exon in alignment.exons: + fill_region_alignment( exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps ) + return alignment + +#loop through string array, only return non-commented lines +def line_enumerator( lines, comment_start = '#' ): + i = 0 + for line in lines: + if not line.startswith( comment_start ): + i += 1 + yield ( i, line ) + +#read a GeneBed file, return list of starts, ends, raw fields +def get_starts_ends_fields_from_gene_bed( line ): + #Starts and ends for exons + starts = [] + ends = [] + + fields = line.split() + #Requires atleast 12 BED columns + if len(fields) < 12: + raise Exception( "Not a proper 12 column BED line (%s)." % line ) + chrom = fields[0] + tx_start = int( fields[1] ) + tx_end = int( fields[2] ) + name = fields[3] + strand = fields[5] + if strand != '-': strand='+' #Default strand is + + cds_start = int( fields[6] ) + cds_end = int( fields[7] ) + + #Calculate and store starts and ends of coding exons + region_start, region_end = cds_start, cds_end + exon_starts = map( int, fields[11].rstrip( ',\n' ).split( ',' ) ) + exon_starts = map( ( lambda x: x + tx_start ), exon_starts ) + exon_ends = map( int, fields[10].rstrip( ',' ).split( ',' ) ) + exon_ends = map( ( lambda x, y: x + y ), exon_starts, exon_ends ); + for start, end in zip( exon_starts, exon_ends ): + start = max( start, region_start ) + end = min( end, region_end ) + if start < end: + starts.append( start ) + ends.append( end ) + return ( starts, ends, fields ) + +def iter_components_by_src( block, src ): + for c in block.components: + if c.src == src: + yield c + +def get_components_by_src( block, src ): + return [ value for value in iter_components_by_src( block, src ) ] + +def iter_components_by_src_start( block, src ): + for c in block.components: + if c.src.startswith( src ): + yield c + +def get_components_by_src_start( block, src ): + return [ value for value in iter_components_by_src_start( block, src ) ] def sort_block_components_by_block( block1, block2 ): #orders the components in block1 by the index of the component in block2 #block1 must be a subset of block2 #occurs in-place return block1.components.sort( cmp = lambda x, y: block2.components.index( x ) - block2.components.index( y ) ) - + def get_species_in_maf( maf_filename ): species = [] for block in bx.align.maf.Reader( open( maf_filename ) ): for spec in get_species_in_block( block ): if spec not in species: species.append( spec ) - return species - -def parse_species_option( species ): - if species: - species = species.split( ',' ) - if 'None' not in species: - return species - return None #provided species was '', None, or had 'None' in it - -def remove_temp_index_file( index_filename ): - try: os.unlink( index_filename ) - except: pass - -#Below are methods to deal with FASTA files - -def get_fasta_header( component, attributes = {}, suffix = None ): - header = ">%s(%s):%i-%i|" % ( component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end() ) - for key, value in attributes.iteritems(): - header = "%s%s=%s|" % ( header, key, value ) - if suffix: - header = "%s%s" % ( header, suffix ) - else: - header = "%s%s" % ( header, src_split( component.src )[ 0 ] ) - return header - -def get_attributes_from_fasta_header( header ): - if not header: return {} - attributes = {} - header = header.lstrip( '>' ) - header = header.strip() - fields = header.split( '|' ) - try: - region = fields[0] - region = region.split( '(', 1 ) - temp = region[0].split( '.', 1 ) - attributes['species'] = temp[0] - if len( temp ) == 2: - attributes['chrom'] = temp[1] - else: - attributes['chrom'] = temp[0] - region = region[1].split( ')', 1 ) - attributes['strand'] = region[0] - region = region[1].lstrip( ':' ).split( '-' ) - attributes['start'] = int( region[0] ) - attributes['end'] = int( region[1] ) - except: - #fields 0 is not a region coordinate - pass - if len( fields ) > 2: - for i in xrange( 1, len( fields ) - 1 ): - prop = fields[i].split( '=', 1 ) - if len( prop ) == 2: - attributes[ prop[0] ] = prop[1] - if len( fields ) > 1: - attributes['__suffix__'] = fields[-1] - return attributes - -def iter_fasta_alignment( filename ): - class fastaComponent: - def __init__( self, species, text = "" ): - self.species = species - self.text = text - def extend( self, text ): - self.text = self.text + text.replace( '\n', '' ).replace( '\r', '' ).strip() - #yields a list of fastaComponents for a FASTA file - f = open( filename, 'rb' ) - components = [] - #cur_component = None - while True: - line = f.readline() - if not line: - if components: - yield components - return - line = line.strip() - if not line: - if components: - yield components - components = [] - elif line.startswith( '>' ): - attributes = get_attributes_from_fasta_header( line ) - components.append( fastaComponent( attributes['species'] ) ) - elif components: - components[-1].extend( line ) - + return species + +def parse_species_option( species ): + if species: + species = species.split( ',' ) + if 'None' not in species: + return species + return None #provided species was '', None, or had 'None' in it + +def remove_temp_index_file( index_filename ): + try: os.unlink( index_filename ) + except: pass + +#Below are methods to deal with FASTA files + +def get_fasta_header( component, attributes = {}, suffix = None ): + header = ">%s(%s):%i-%i|" % ( component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end() ) + for key, value in attributes.iteritems(): + header = "%s%s=%s|" % ( header, key, value ) + if suffix: + header = "%s%s" % ( header, suffix ) + else: + header = "%s%s" % ( header, src_split( component.src )[ 0 ] ) + return header + +def get_attributes_from_fasta_header( header ): + if not header: return {} + attributes = {} + header = header.lstrip( '>' ) + header = header.strip() + fields = header.split( '|' ) + try: + region = fields[0] + region = region.split( '(', 1 ) + temp = region[0].split( '.', 1 ) + attributes['species'] = temp[0] + if len( temp ) == 2: + attributes['chrom'] = temp[1] + else: + attributes['chrom'] = temp[0] + region = region[1].split( ')', 1 ) + attributes['strand'] = region[0] + region = region[1].lstrip( ':' ).split( '-' ) + attributes['start'] = int( region[0] ) + attributes['end'] = int( region[1] ) + except: + #fields 0 is not a region coordinate + pass + if len( fields ) > 2: + for i in xrange( 1, len( fields ) - 1 ): + prop = fields[i].split( '=', 1 ) + if len( prop ) == 2: + attributes[ prop[0] ] = prop[1] + if len( fields ) > 1: + attributes['__suffix__'] = fields[-1] + return attributes + +def iter_fasta_alignment( filename ): + class fastaComponent: + def __init__( self, species, text = "" ): + self.species = species + self.text = text + def extend( self, text ): + self.text = self.text + text.replace( '\n', '' ).replace( '\r', '' ).strip() + #yields a list of fastaComponents for a FASTA file + f = open( filename, 'rb' ) + components = [] + #cur_component = None + while True: + line = f.readline() + if not line: + if components: + yield components + return + line = line.strip() + if not line: + if components: + yield components + components = [] + elif line.startswith( '>' ): + attributes = get_attributes_from_fasta_header( line ) + components.append( fastaComponent( attributes['species'] ) ) + elif components: + components[-1].extend( line ) + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Irods get_data was using wrong varname for reading(count)
by commits-noreply＠bitbucket.org 29 Aug '13

29 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/bec1baa9d2db/ Changeset: bec1baa9d2db User: dannon Date: 2013-08-29 18:21:50 Summary: Irods get_data was using wrong varname for reading(count) Affected #: 1 file diff -r 444c5b5b451b50eacba3ba1d08d83e16c1027fa5 -r bec1baa9d2db009abd16778093fb15b96bde0950 lib/galaxy/objectstore/rods.py --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -210,7 +210,7 @@ if count == -1: return h.read() else: - return f.read( count ) + return h.read( count ) # TODO: make sure implicit close is okay, DiskObjectStore actually # reads data into a var, closes, and returns the var Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Log, then raise the IOError exception that occur in DiskObjectStore.update_from_file() as per discussion 08/05/2013-08/12/2013.
by commits-noreply＠bitbucket.org 29 Aug '13

29 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/444c5b5b451b/ Changeset: 444c5b5b451b User: dan Date: 2013-08-29 17:52:41 Summary: Log, then raise the IOError exception that occur in DiskObjectStore.update_from_file() as per discussion 08/05/2013-08/12/2013. Affected #: 1 file diff -r 533fb8fcc330c52646529659a246e1c50dc18e6a -r 444c5b5b451b50eacba3ba1d08d83e16c1027fa5 lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -344,6 +344,7 @@ except IOError, ex: log.critical('Error copying %s to %s: %s' % (file_name, self._get_filename(obj, **kwargs), ex)) + raise ex def get_object_url(self, obj, **kwargs): return None Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: UI and Trackster: (a) restore trackster tool save functionality; (b) add toggle functionality to visibility mixin; (c) extend tool class to create trackster tool rather than create new class.
by commits-noreply＠bitbucket.org 29 Aug '13

29 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/533fb8fcc330/ Changeset: 533fb8fcc330 User: jgoecks Date: 2013-08-28 23:46:03 Summary: UI and Trackster: (a) restore trackster tool save functionality; (b) add toggle functionality to visibility mixin; (c) extend tool class to create trackster tool rather than create new class. Affected #: 2 files diff -r fc4e1a162195e45f69573724adec564ffe1ea9ee -r 533fb8fcc330c52646529659a246e1c50dc18e6a static/scripts/mvc/tools.js --- a/static/scripts/mvc/tools.js +++ b/static/scripts/mvc/tools.js @@ -18,6 +18,10 @@ hide: function() { this.set("hidden", true); }, + + toggle: function() { + this.set("hidden", !this.get("hidden")); + }, is_visible: function() { return !this.attributes.hidden; @@ -600,8 +604,8 @@ query_changed: function( evData ) { // check for the 'clear key' (ESC) first - if( ( this.model.attributes.clear_key ) - && ( this.model.attributes.clear_key === evData.which ) ){ + if( ( this.model.attributes.clear_key ) && + ( this.model.attributes.clear_key === evData.which ) ){ this.clear(); return false; } diff -r fc4e1a162195e45f69573724adec564ffe1ea9ee -r 533fb8fcc330c52646529659a246e1c50dc18e6a static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1622,25 +1622,25 @@ /** * Encapsulation of a tool that users can apply to tracks/datasets. */ -var TracksterTool = Backbone.RelationalModel.extend({ +var TracksterTool = tools_mod.Tool.extend({ defaults: { - track: null, - tool: null, + track: null }, - relations: [ - { - type: Backbone.HasOne, - key: 'tool', - relatedModel: tools_mod.Tool + initialize: function(options) { + // Restore tool visibility from state. + if (options.tool_state !== undefined) { + this.set('hidden', options.tool_state.hidden) } - ], - - initialize: function(options) { + + // FIXME: need to restore tool values from options.tool_state + // HACK: remove some inputs because Trackster does yet not work with them. - this.get('tool').remove_inputs( [ 'data', 'hidden_data', 'conditional' ] ); - - // FIXME: need to restore tool values/visibility. + this.remove_inputs( [ 'data', 'hidden_data', 'conditional' ] ); + }, + + state_dict: function(options) { + return _.extend( this.get_inputs_dict(), { hidden: !this.is_visible() } ); } }); @@ -1678,12 +1678,16 @@ */ var TracksterToolView = Backbone.View.extend({ + initialize: function(options) { + this.model.on('change:hidden', this.set_visible, this); + }, + /** * Render tool UI. */ render: function() { var self = this; - tool = this.model.get('tool'), + tool = this.model, parent_div = this.$el.addClass("dynamic-tool").hide(); // Prevent div events from propogating to other elements. @@ -1719,11 +1723,21 @@ self.run_on_dataset(); }); - /* - if ('visible' in tool_state_dict && tool_state_dict.visible) { - this.parent_div.show(); + if (tool.is_visible()) { + this.$el.show(); } - */ + }, + + /** + * Show or hide tool depending on tool visibility state. + */ + set_visible: function() { + if (this.model.is_visible()) { + this.$el.show(); + } + else { + this.$el.hide(); + } }, /** @@ -1736,23 +1750,10 @@ }, /** - * Returns a dict with tool state information. - */ - state_dict: function() { - // Save parameter values. - var tool_state = this.model.get('tool').get_param_values_dict(); - - // Save visibility. - tool_state.visible = this.parent_div.is(":visible"); - - return tool_state; - }, - - /** * Run tool on dataset. Output is placed in dataset's history and no changes to viz are made. */ run_on_dataset: function() { - var tool = this.model.get('tool'); + var tool = this.model; this.run( // URL params. { @@ -1779,7 +1780,7 @@ // Create track for tool's output immediately to provide user feedback. // var track = this.model.get('track'), - tool = this.model.get('tool'), + tool = this.model, region = new visualization.GenomeRegion({ chrom: track.view.chrom, start: track.view.low, @@ -1850,7 +1851,7 @@ */ run: function(url_params, new_track, success_callback) { // Run tool. - url_params.inputs = this.model.get('tool').get_inputs_dict(); + url_params.inputs = this.model.get_inputs_dict(); var ss_deferred = new util.ServerStateDeferred({ ajax_settings: { url: galaxy_config.root + "api/tools", @@ -2405,10 +2406,10 @@ on_click_fn: function(track) { // TODO: update tipsy text. - track.dynamic_tool_div.toggle(); + track.tool.toggle(); // Update track name. - if (track.dynamic_tool_div.is(":visible")) { + if (track.tool.is_visible()) { track.set_name(track.name + track.tool_region_and_parameters_str()); } else { @@ -2790,11 +2791,10 @@ // FIXME: prolly need function to set filters and update data_manager reference. this.data_manager.set('filters_manager', this.filters_manager); this.filters_available = false; - this.tool = (obj_dict.tool ? new TracksterTool({ - 'track': this, - 'tool': obj_dict.tool, + this.tool = (obj_dict.tool ? new TracksterTool( _.extend( obj_dict.tool, { + 'track': this, 'tool_state': obj_dict.tool_state - }) + } ) ) : null); this.tile_cache = new visualization.Cache(TILE_CACHE_SIZE); this.left_offset = 0; @@ -3333,9 +3333,7 @@ * @param w_scale pixels per base * @param ref_seq reference sequence data */ - draw_tile: function(result, ctx, mode, resolution, region, w_scale, ref_seq) { - console.log("Warning: TiledTrack.draw_tile() not implemented."); - }, + draw_tile: function(result, ctx, mode, resolution, region, w_scale, ref_seq) {}, /** * Show track tile and perform associated actions. Showing tile may actually move @@ -3406,7 +3404,7 @@ tool_region_and_parameters_str: function(region) { var track = this, region_str = (region !== undefined ? region.toString() : "all"), - param_str = _.values( track.tool.get('tool').get_inputs_dict()).join(', '); + param_str = _.values( track.tool.get_inputs_dict()).join(', '); return " - region=[" + region_str + "], parameters=[" + param_str + "]"; }, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 29 Aug '13

29 Aug '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d1593c067833/ Changeset: d1593c067833 User: natefoo Date: 2013-08-28 22:08:21 Summary: Reorganize object store options in sample config file. Affected #: 1 file diff -r 4fea98291cbce9a0e8d580bfd9c4ef4a5184c680 -r d1593c067833c8dc4c85bdcc7b4d90b79d8ab01c universe_wsgi.ini.sample --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -203,6 +203,41 @@ # breaking existing tools. #collect_outputs_from = new_file_path,job_working_directory +# -- Data Storage (Object Store) + +# Object store backend module (valid options are: disk, s3, swift, irods, +# distributed, hierarchical) +#object_store = disk + +# Credentials used by certain (s3, swift) object store backends +#os_access_key = <your cloud object store access key> +#os_secret_key = <your cloud object store secret key> +#os_bucket_name = <name of an existing object store bucket or container> + +# If using 'swift' object store, you must specify the following connection +# properties +#os_host = swift.rc.nectar.org.au +#os_port = 8888 +#os_is_secure = False +#os_conn_path = / + +# Reduced redundancy can be used only with the 's3' object store +#os_use_reduced_redundancy = False + +# Path to cache directory for object store backends that utilize a cache (s3, +# swift, irods) +#object_store_cache_path = database/files/ + +# Size (in GB) that the cache used by object store should be limited to. +# If the value is not specified, the cache size will be limited only by the +# file system size. +#object_store_cache_size = 100 + +# Configuration file for the distributed object store, if object_store = +# distributed. See the sample at distributed_object_store_conf.xml.sample +#distributed_object_store_config_file = None + + # -- Mail and notification # Galaxy sends mail for various things: Subscribing users to the mailing list @@ -524,28 +559,6 @@ # -- Beta features -# Object store mode (valid options are: disk, s3, swift, distributed, hierarchical) -#object_store = disk -#os_access_key = <your cloud object store access key> -#os_secret_key = <your cloud object store secret key> -#os_bucket_name = <name of an existing object store bucket or container> -# If using 'swift' object store, you must specify the following connection properties -#os_host = swift.rc.nectar.org.au -#os_port = 8888 -#os_is_secure = False -#os_conn_path = / -# Reduced redundancy can be used only with the 's3' object store -#os_use_reduced_redundancy = False -# Size (in GB) that the cache used by object store should be limited to. -# If the value is not specified, the cache size will be limited only by the -# file system size. The file system location of the cache is considered the -# configuration of the ``file_path`` directive defined above. -#object_store_cache_size = 100 - -# Configuration file for the distributed object store, if object_store = -# distributed. See the sample at distributed_object_store_conf.xml.sample -#distributed_object_store_config_file = None - # Enable Galaxy to communicate directly with a sequencer #enable_sequencer_communication = False https://bitbucket.org/galaxy/galaxy-central/commits/75bcfb029e8c/ Changeset: 75bcfb029e8c User: natefoo Date: 2013-08-28 22:11:29 Summary: Implement a local_extra_dirs decorator in the object store that object store plugins (e.g. cloud object stores) can use to cause things like the job_working_directory to be stored on local disk. Affected #: 1 file diff -r d1593c067833c8dc4c85bdcc7b4d90b79d8ab01c -r 75bcfb029e8c5dd8e3d78829469a8583c73055ae lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -25,7 +25,7 @@ """ ObjectStore abstract interface """ - def __init__(self): + def __init__(self, config, **kwargs): self.running = True self.extra_dirs = {} @@ -195,7 +195,7 @@ >>> assert s.get_filename(obj) == file_path + '/000/dataset_1.dat' """ def __init__(self, config, file_path=None, extra_dirs=None): - super(DiskObjectStore, self).__init__() + super(DiskObjectStore, self).__init__(config, file_path=file_path, extra_dirs=extra_dirs) self.file_path = file_path or config.file_path self.config = config self.extra_dirs['job_work'] = config.job_working_directory @@ -213,6 +213,7 @@ if not os.path.exists(path): return self._construct_path(obj, base_dir=base_dir, dir_only=dir_only, extra_dir=extra_dir, extra_dir_at_root=extra_dir_at_root, alt_name=alt_name) + # TODO: rename to _disk_path or something like that to avoid conflicts with children that'll use the local_extra_dirs decorator, e.g. S3 def _construct_path(self, obj, old_style=False, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, **kwargs): """ Construct the expected absolute path for accessing the object identified by `obj`.id. @@ -544,6 +545,19 @@ else: log.error("Unrecognized object store definition: {0}".format(store)) +def local_extra_dirs( func ): + """ A decorator for non-local plugins to utilize local directories for their extra_dirs (job_working_directory and temp). + """ + def wraps( self, *args, **kwargs ): + if kwargs.get( 'base_dir', None ) is None: + return func( self, *args, **kwargs ) + else: + for c in self.__class__.__mro__: + if c.__name__ == 'DiskObjectStore': + return getattr( c, func.__name__ )( self, *args, **kwargs ) + raise Exception( "Could not call DiskObjectStore's %s method, does your Object Store plugin inherit from DiskObjectStore?" % func.__name__ ) + return wraps + def convert_bytes(bytes): """ A helper function used for pretty printing disk usage """ if bytes is None: https://bitbucket.org/galaxy/galaxy-central/commits/fc4e1a162195/ Changeset: fc4e1a162195 User: natefoo Date: 2013-08-28 22:20:19 Summary: Experimental/alpha/in-development/incomplete object store plugin for iRODS. This plugin will probably set your iRODS instance on fire: do not use except in development/testing. Affected #: 6 files diff -r 75bcfb029e8c5dd8e3d78829469a8583c73055ae -r fc4e1a162195e45f69573724adec564ffe1ea9ee eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -18,6 +18,7 @@ MarkupSafe = 0.12 mercurial = 2.2.3 MySQL_python = 1.2.3c1 +PyRods = 3.2.4 numpy = 1.6.0 pbs_python = 4.3.5 psycopg2 = 2.0.13 diff -r 75bcfb029e8c5dd8e3d78829469a8583c73055ae -r fc4e1a162195e45f69573724adec564ffe1ea9ee lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -209,6 +209,7 @@ if self.nginx_upload_store: self.nginx_upload_store = os.path.abspath( self.nginx_upload_store ) self.object_store = kwargs.get( 'object_store', 'disk' ) + self.object_store_cache_path = resolve_path( kwargs.get( "object_store_cache_path", "database/object_store_cache" ), self.root ) # Handle AWS-specific config options for backward compatibility if kwargs.get( 'aws_access_key', None) is not None: self.os_access_key= kwargs.get( 'aws_access_key', None ) @@ -228,6 +229,8 @@ self.distributed_object_store_config_file = kwargs.get( 'distributed_object_store_config_file', None ) if self.distributed_object_store_config_file is not None: self.distributed_object_store_config_file = resolve_path( self.distributed_object_store_config_file, self.root ) + self.irods_root_collection_path = kwargs.get( 'irods_root_collection_path', None ) + self.irods_default_resource = kwargs.get( 'irods_default_resource', None ) # Parse global_conf and save the parser global_conf = kwargs.get( 'global_conf', None ) global_conf_parser = ConfigParser.ConfigParser() @@ -369,6 +372,7 @@ self.nginx_upload_store, \ './static/genetrack/plots', \ self.whoosh_index_dir, \ + self.object_store_cache_path, \ os.path.join( self.tool_data_path, 'shared', 'jars' ): if path not in [ None, False ] and not os.path.isdir( path ): try: diff -r 75bcfb029e8c5dd8e3d78829469a8583c73055ae -r fc4e1a162195e45f69573724adec564ffe1ea9ee lib/galaxy/eggs/__init__.py --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -387,7 +387,8 @@ "guppy": lambda: self.config.get( "app:main", "use_memdump" ), "python_openid": lambda: self.config.get( "app:main", "enable_openid" ), "python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ), - "pysam": lambda: check_pysam() + "pysam": lambda: check_pysam(), + "PyRods": lambda: self.config.get( "app:main", "object_store" ) == "irods" }.get( egg_name, lambda: True )() except: return False diff -r 75bcfb029e8c5dd8e3d78829469a8583c73055ae -r fc4e1a162195e45f69573724adec564ffe1ea9ee lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -542,6 +542,9 @@ return DistributedObjectStore(config=config, fsmon=fsmon) elif store == 'hierarchical': return HierarchicalObjectStore() + elif store == 'irods': + from galaxy.objectstore.rods import IRODSObjectStore + return IRODSObjectStore(config=config) else: log.error("Unrecognized object store definition: {0}".format(store)) diff -r 75bcfb029e8c5dd8e3d78829469a8583c73055ae -r fc4e1a162195e45f69573724adec564ffe1ea9ee lib/galaxy/objectstore/rods.py --- /dev/null +++ b/lib/galaxy/objectstore/rods.py @@ -0,0 +1,332 @@ +""" +Object Store plugin for the Integrated Rule-Oriented Data Store (iRODS) + +The module is named rods to avoid conflicting with the PyRods module, irods +""" + +import os +import time +import errno +import logging +#import traceback + +from posixpath import join as path_join +from posixpath import basename as path_basename +from posixpath import dirname as path_dirname + +from galaxy.objectstore import DiskObjectStore, ObjectStore, local_extra_dirs +from galaxy.exceptions import ObjectNotFound, ObjectInvalid + +import galaxy.eggs +galaxy.eggs.require( 'PyRods' ) +import irods + +log = logging.getLogger( __name__ ) + + +class IRODSObjectStore( DiskObjectStore, ObjectStore ): + """ + Galaxy object store based on iRODS + """ + def __init__( self, config, file_path=None, extra_dirs=None ): + super( IRODSObjectStore, self ).__init__( config, file_path=file_path, extra_dirs=extra_dirs ) + self.cache_path = config.object_store_cache_path + self.default_resource = config.irods_default_resource or None + + # Connect to iRODS (AssertionErrors will be raised if anything goes wrong) + self.rods_env, self.rods_conn = rods_connect() + + # if the root collection path in the config is unset or relative, try to use a sensible default + if config.irods_root_collection_path is None or ( config.irods_root_collection_path is not None and not config.irods_root_collection_path.startswith( '/' ) ): + rods_home = self.rods_env.rodsHome + assert rods_home != '', "Unable to initialize iRODS Object Store: rodsHome cannot be determined and irods_root_collection_path in Galaxy config is unset or not absolute." + if config.irods_root_collection_path is None: + self.root_collection_path = path_join( rods_home, 'galaxy_data' ) + else: + self.root_collection_path = path_join( rods_home, config.irods_root_collection_path ) + else: + self.root_collection_path = config.irods_root_collection_path + + # will return a collection object regardless of whether it exists + self.root_collection = irods.irodsCollection( self.rods_conn, self.root_collection_path ) + + if self.root_collection.getId() == -1: + log.warning( "iRODS root collection does not exist, will attempt to create: %s", self.root_collection_path ) + self.root_collection.upCollection() + assert self.root_collection.createCollection( os.path.basename( self.root_collection_path ) ) == 0, "iRODS root collection creation failed: %s" % self.root_collection_path + self.root_collection = irods.irodsCollection( self.rods_conn, self.root_collection_path ) + assert self.root_collection.getId() != -1, "iRODS root collection creation claimed success but still does not exist" + + if self.default_resource is None: + self.default_resource = self.rods_env.rodsDefResource + + log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource ) + + def __get_rods_path( self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, strip_dat=True, **kwargs ): + path = "" + if extra_dir is not None: + path = extra_dir + + # extra_dir_at_root is ignored - since the iRODS plugin does not use + # the directory hash, there is only one level of subdirectory. + + if not dir_only: + # the .dat extension is stripped when stored in iRODS + # TODO: is the strip_dat kwarg the best way to implement this? + if strip_dat and alt_name and alt_name.endswith( '.dat' ): + alt_name = os.path.splitext( alt_name )[0] + default_name = 'dataset_%s' % obj.id + if not strip_dat: + default_name += '.dat' + path = path_join( path, alt_name if alt_name else default_name ) + + path = path_join( self.root_collection_path, path ) + + #log.debug( 'iRODS path for %s %s is %s', obj.__class__.__name__, obj.id, path ) + + return path + + def __get_cache_path( self, obj, **kwargs ): + # FIXME: does not handle collections + # FIXME: collisions could occur here + return os.path.join( self.cache_path, path_basename( self.__get_rods_path( obj, strip_dat=False, **kwargs ) ) ) + + def __clean_cache_entry( self, obj, **kwargs ): + # FIXME: does not handle collections + try: + os.unlink( self.__get_cache_path( obj, **kwargs ) ) + except OSError: + # it is expected that we'll call this method a lot regardless of + # whether we think the cached file exists + pass + + def __get_rods_handle( self, obj, mode='r', **kwargs ): + if kwargs.get( 'dir_only', False ): + return irods.irodsCollection( self.rods_conn, self.__get_rods_path( obj, **kwargs ) ) + else: + return irods.irodsOpen( self.rods_conn, self.__get_rods_path( obj, **kwargs ), mode ) + + def __mkcolls( self, rods_path ): + """ + An os.makedirs() for iRODS collections. `rods_path` is the desired collection to create. + """ + assert rods_path.startswith( self.root_collection_path + '/' ), '__mkcolls(): Creating collections outside the root collection is not allowed (requested path was: %s)' % rods_path + mkcolls = [] + c = irods.irodsCollection( self.rods_conn, rods_path ) + while c.getId() == -1: + assert c.getCollName().startswith( self.root_collection_path + '/' ), '__mkcolls(): Attempted to move above the root collection: %s' % c.getCollName() + mkcolls.append( c.getCollName() ) + c.upCollection() + for collname in reversed( mkcolls ): + log.debug( 'Creating collection %s' % collname ) + ci = irods.collInp_t() + ci.collName = collname + status = rcCollCreate( self.rods_conn, ci ) + assert status == 0, '__mkcolls(): Failed to create collection: %s' % collname + + @local_extra_dirs + def exists( self, obj, **kwargs ): + doi = irods.dataObjInp_t() + doi.objPath = self.__get_rods_path( obj, **kwargs ) + log.debug( 'exists(): checking: %s', doi.objPath ) + return irods.rcObjStat( self.rods_conn, doi ) is not None + + @local_extra_dirs + def create(self, obj, **kwargs): + if not self.exists( obj, **kwargs ): + rods_path = self.__get_rods_path( obj, **kwargs ) + log.debug( 'create(): %s', rods_path ) + dir_only = kwargs.get( 'dir_only', False ) + # short circuit collection creation since most of the time it will + # be the root collection which already exists + collection_path = rods_path if dir_only else path_dirname( rods_path ) + if collection_path != self.root_collection_path: + self.__mkcolls( collection_path ) + if not dir_only: + # rcDataObjCreate is used instead of the irodsOpen wrapper so + # that we can prevent overwriting + doi = irods.dataObjInp_t() + doi.objPath = rods_path + doi.createMode = 0640 + doi.dataSize = 0 # 0 actually means "unknown", although literally 0 would be preferable + irods.addKeyVal( doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource ) + status = irods.rcDataObjCreate( self.rods_conn, doi ) + assert status >= 0, 'create(): rcDataObjCreate() failed: %s: %s: %s' % ( rods_path, status, irods.strerror( status ) ) + + @local_extra_dirs + def empty( self, obj, **kwargs ): + assert 'dir_only' not in kwargs, 'empty(): `dir_only` parameter is invalid here' + h = self.__get_rods_handle( obj, **kwargs ) + try: + return h.getSize() == 0 + except AttributeError: + # h is None + raise ObjectNotFound() + + def size( self, obj, **kwargs ): + assert 'dir_only' not in kwargs, 'size(): `dir_only` parameter is invalid here' + h = self.__get_rods_handle( obj, **kwargs ) + try: + return h.getSize() + except AttributeError: + # h is None + return 0 + + @local_extra_dirs + def delete( self, obj, entire_dir=False, **kwargs ): + assert 'dir_only' not in kwargs, 'delete(): `dir_only` parameter is invalid here' + rods_path = self.__get_rods_path( obj, **kwargs ) + # __get_rods_path prepends self.root_collection_path but we are going + # to ensure that it's valid anyway for safety's sake + assert rods_path.startswith( self.root_collection_path + '/' ), 'ERROR: attempt to delete object outside root collection (path was: %s)' % rods_path + if entire_dir: + # TODO + raise NotImplementedError() + h = self.__get_rods_handle( obj, **kwargs ) + try: + # note: PyRods' irodsFile.delete() does not set force + status = h.delete() + assert status == 0, '%d: %s' % ( status, irods.strerror( status ) ) + return True + except AttributeError: + log.warning( 'delete(): operation failed: object does not exist: %s', rods_path ) + except AssertionError, e: + # delete() does not raise on deletion failure + log.error( 'delete(): operation failed: %s', e ) + finally: + # remove the cached entry (finally is executed even when the try + # contains a return) + self.__clean_cache_entry( self, obj, **kwargs ) + return False + + @local_extra_dirs + def get_data( self, obj, start=0, count=-1, **kwargs ): + log.debug( 'get_data(): %s' ) + h = self.__get_rods_handle( obj, **kwargs ) + try: + h.seek( start ) + except AttributeError: + raise ObjectNotFound() + if count == -1: + return h.read() + else: + return f.read( count ) + # TODO: make sure implicit close is okay, DiskObjectStore actually + # reads data into a var, closes, and returns the var + + @local_extra_dirs + def get_filename( self, obj, **kwargs ): + log.debug( "get_filename(): called on %s %s. For better performance, avoid this method and use get_data() instead.", obj.__class__.__name__, obj.id ) + + # For finding all places where get_filename is called... + #log.debug( ''.join( traceback.format_stack() ) ) + + cached_path = self.__get_cache_path( obj, **kwargs ) + + if not self.exists( obj, **kwargs ): + raise ObjectNotFound() + + # TODO: implement or define whether dir_only is valid + if 'dir_only' in kwargs: + raise NotImplementedError() + + # cache hit + if os.path.exists( cached_path ): + return os.path.abspath( cached_path ) + + # cache miss + # TODO: thread this + incoming_path = os.path.join( os.path.dirname( cached_path ), "__incoming_%s" % os.path.basename( cached_path ) ) + doi = irods.dataObjInp_t() + doi.objPath = self.__get_rods_path( obj, **kwargs ) + doi.dataSize = 0 # TODO: does this affect performance? should we get size? + doi.numThreads = 0 + # TODO: might want to VERIFY_CHKSUM_KW + log.debug( 'get_filename(): caching %s to %s', doi.objPath, incoming_path ) + + # do the iget + status = irods.rcDataObjGet( self.rods_conn, doi, incoming_path ) + + # if incoming already exists, we'll wait for another process or thread + # to finish caching + if status != irods.OVERWRITE_WITHOUT_FORCE_FLAG: + assert status == 0, 'get_filename(): iget %s failed (%s): %s' % ( doi.objPath, status, irods.strerror( status ) ) + # POSIX rename is atomic + # TODO: rename without clobbering + os.rename( incoming_path, cached_path ) + log.debug( 'get_filename(): cached %s to %s', doi.objPath, cached_path ) + + # another process or thread is caching, wait for it + while not os.path.exists( cached_path ): + # TODO: force restart after mod time > some configurable, or + # otherwise deal with this potential deadlock and interrupted + # transfers + time.sleep( 5 ) + log.debug( "get_filename(): waiting on incoming '%s' for %s %s", incoming_path, obj.__class__.__name__, obj.id ) + + return os.path.abspath( cached_path ) + + @local_extra_dirs + def update_from_file(self, obj, file_name=None, create=False, **kwargs): + assert 'dir_only' not in kwargs, 'update_from_file(): `dir_only` parameter is invalid here' + + # do not create if not requested + if create and not self.exists( obj, **kwargs ): + raise ObjectNotFound() + + if file_name is None: + file_name = self.__get_cache_path( obj, **kwargs ) + + # put will create if necessary + doi = irods.dataObjInp_t() + doi.objPath = self.__get_rods_path( obj, **kwargs ) + doi.createMode = 0640 + doi.dataSize = os.stat( file_name ).st_size + doi.numThreads = 0 + irods.addKeyVal( doi.condInput, irods.DEST_RESC_NAME_KW, self.default_resource ) + irods.addKeyVal( doi.condInput, irods.FORCE_FLAG_KW, '' ) + # TODO: might want to VERIFY_CHKSUM_KW + log.debug( 'update_from_file(): updating %s to %s', file_name, doi.objPath ) + + # do the iput + status = irods.rcDataObjPut( self.rods_conn, doi, file_name ) + assert status == 0, 'update_from_file(): iput %s failed (%s): %s' % ( doi.objPath, status, irods.strerror( status ) ) + + def get_object_url(self, obj, **kwargs): + return None + + def get_store_usage_percent(self): + return 0.0 + +# monkeypatch an strerror method into the irods module +def _rods_strerror( errno ): + """ + The missing `strerror` for iRODS error codes + """ + if not hasattr( irods, '__rods_strerror_map' ): + irods.__rods_strerror_map = {} + for name in dir( irods ): + v = getattr( irods, name ) + if type( v ) == int and v < 0: + irods.__rods_strerror_map[ v ] = name + return irods.__rods_strerror_map.get( errno, 'GALAXY_NO_ERRNO_MAPPING_FOUND' ) + +irods.strerror = _rods_strerror + +def rods_connect(): + """ + A basic iRODS connection mechanism that connects using the current iRODS + environment + """ + status, env = irods.getRodsEnv() + assert status == 0, 'connect(): getRodsEnv() failed (%s): %s' % ( status, irods.strerror( status ) ) + conn, err = irods.rcConnect( env.rodsHost, + env.rodsPort, + env.rodsUserName, + env.rodsZone ) + assert err.status == 0, 'connect(): rcConnect() failed (%s): %s' % ( err.status, err.msg ) + status, pw = irods.obfGetPw() + assert status == 0, 'connect(): getting password with obfGetPw() failed (%s): %s' % ( status, irods.strerror( status ) ) + status = irods.clientLoginWithObfPassword( conn, pw ) + assert status == 0, 'connect(): logging in with clientLoginWithObfPassword() failed (%s): %s' % ( status, irods.strerror( status ) ) + return env, conn diff -r 75bcfb029e8c5dd8e3d78829469a8583c73055ae -r fc4e1a162195e45f69573724adec564ffe1ea9ee scripts/scramble/scripts/PyRods.py --- /dev/null +++ b/scripts/scramble/scripts/PyRods.py @@ -0,0 +1,33 @@ +import os, sys +import subprocess + +# change back to the build dir +if os.path.dirname( sys.argv[0] ) != "": + os.chdir( os.path.dirname( sys.argv[0] ) ) + +# find setuptools +sys.path.append( os.path.join( '..', '..', '..', 'lib' ) ) +from scramble_lib import * + +tag = get_tag() # get the tag +get_deps() # require any dependent eggs +clean() # clean up any existing stuff (could happen if you run scramble.py by hand) + +subprocess.check_call( "./scripts/configure", shell=True ) +subprocess.check_call( "CFLAGS='-fPIC' make clients", shell=True ) + +# reset args for distutils +me = sys.argv[0] +sys.argv = [ me ] +sys.argv.append( "egg_info" ) +if tag is not None: + sys.argv.append( "--tag-build=%s" %tag ) +# svn revision (if any) is handled directly in tag-build +sys.argv.append( "--no-svn-revision" ) +sys.argv.append( "bdist_egg" ) + +# apply patches (if any) +apply_patches() + +# do it +execfile( "setup.py", globals(), locals() ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 29 Aug '13

29 Aug '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ee6d553f7847/ Changeset: ee6d553f7847 Branch: stable User: natefoo Date: 2013-08-28 22:02:16 Summary: Logic on DiskObjectStore's empty() method was reversed (previously returned True when non-empty). Affected #: 1 file diff -r de04c251c6cae5760fd13c83030a6aab78ae0c4e -r ee6d553f7847ec47cca0cf205c767d1407eccbab lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -297,7 +297,7 @@ util.umask_fix_perms(path, self.config.umask, 0666) def empty(self, obj, **kwargs): - return os.path.getsize(self.get_filename(obj, **kwargs)) > 0 + return os.path.getsize(self.get_filename(obj, **kwargs)) == 0 def size(self, obj, **kwargs): if self.exists(obj, **kwargs): https://bitbucket.org/galaxy/galaxy-central/commits/f2677acb7d77/ Changeset: f2677acb7d77 User: natefoo Date: 2013-08-28 22:02:35 Summary: merge stable. Affected #: 2 files diff -r a60d756aeb5d40c62854fda8a5f5be274762cd86 -r f2677acb7d775a4d9ca26fc78a003dab060a0e8b lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -286,7 +286,7 @@ util.umask_fix_perms(path, self.config.umask, 0666) def empty(self, obj, **kwargs): - return os.path.getsize(self.get_filename(obj, **kwargs)) > 0 + return os.path.getsize(self.get_filename(obj, **kwargs)) == 0 def size(self, obj, **kwargs): if self.exists(obj, **kwargs): https://bitbucket.org/galaxy/galaxy-central/commits/4fea98291cbc/ Changeset: 4fea98291cbc User: natefoo Date: 2013-08-28 22:04:59 Summary: Remove S3 connection method from objectstore, missed when S3 was moved to its own module. Affected #: 1 file diff -r f2677acb7d775a4d9ca26fc78a003dab060a0e8b -r 4fea98291cbce9a0e8d580bfd9c4ef4a5184c680 lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -565,25 +565,3 @@ else: size = '%.2fb' % bytes return size - -def get_OS_connection(config): - """ - Get a connection object for a cloud Object Store specified in the config. - Currently, this is a ``boto`` connection object. - """ - log.debug("Getting a connection object for '{0}' object store".format(config.object_store)) - a_key = config.os_access_key - s_key = config.os_secret_key - if config.object_store == 's3': - return S3Connection(a_key, s_key) - else: - # Establish the connection now - calling_format = boto.s3.connection.OrdinaryCallingFormat() - s3_conn = boto.connect_s3(aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=config.os_is_secure, - host=config.os_host, - port=int(config.os_port), - calling_format=calling_format, - path=config.os_conn_path) - return s3_conn Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Datatypes: temporarily remove graph datatype sniffers (f3a985005267) until they can be made more discriminating
by commits-noreply＠bitbucket.org 28 Aug '13

28 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a60d756aeb5d/ Changeset: a60d756aeb5d User: carlfeberhard Date: 2013-08-28 20:22:27 Summary: Datatypes: temporarily remove graph datatype sniffers (f3a985005267) until they can be made more discriminating Affected #: 1 file diff -r 22a4f03c1fe40dedcb7e6f11510d58ada8e9c458 -r a60d756aeb5d40c62854fda8a5f5be274762cd86 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -285,9 +285,6 @@ <sniffer type="galaxy.datatypes.tabular:Sam"/><sniffer type="galaxy.datatypes.data:Newick"/><sniffer type="galaxy.datatypes.data:Nexus"/> - <sniffer type="galaxy.datatypes.graph:Xgmml"/> - <sniffer type="galaxy.datatypes.graph:Sif"/> - <sniffer type="galaxy.datatypes.graph:Rdf"/><sniffer type="galaxy.datatypes.images:Jpg"/><sniffer type="galaxy.datatypes.images:Png"/><sniffer type="galaxy.datatypes.images:Tiff"/> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Handle the case where the repository being installed has a newer downloadable changeset revision available. (Generally restricted to the new tool_dependency_definition repository type)
by commits-noreply＠bitbucket.org 28 Aug '13

28 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/22a4f03c1fe4/ Changeset: 22a4f03c1fe4 User: Dave Bouvier Date: 2013-08-28 15:54:45 Summary: Handle the case where the repository being installed has a newer downloadable changeset revision available. (Generally restricted to the new tool_dependency_definition repository type) Affected #: 1 file diff -r 29110e49c311339c521525f0b0884ce8eb172129 -r 22a4f03c1fe40dedcb7e6f11510d58ada8e9c458 lib/tool_shed/galaxy_install/repository_util.py --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -214,24 +214,39 @@ def get_repo_info_dict( trans, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( trans, repository_id ) + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) - metadata = repository_metadata.metadata - if 'tools' in metadata: - includes_tools = True + if not repository_metadata: + # The received changeset_revision is no longer installable, so get the next changeset_revision in the repository's changelog. + # This generally occurs only with repositories of type tool_dependency_definition. + next_downloadable_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + if next_downloadable_changeset_revision: + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, next_downloadable_changeset_revision ) + if repository_metadata: + # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption is not valid we'll have to + # enhance the callers to handle repository_metadata values of None in the returned repo_info_dict. + metadata = repository_metadata.metadata + if 'tools' in metadata: + includes_tools = True + else: + includes_tools = False + includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel + if 'repository_dependencies' in metadata: + has_repository_dependencies = True + else: + has_repository_dependencies = False + if 'tool_dependencies' in metadata: + includes_tool_dependencies = True + else: + includes_tool_dependencies = False else: + # Here's where we may have to handle enhancements to the callers. See above comment. includes_tools = False - includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel - if 'repository_dependencies' in metadata: - has_repository_dependencies = True - else: has_repository_dependencies = False - if 'tool_dependencies' in metadata: - includes_tool_dependencies = True - else: includes_tool_dependencies = False - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( suc.get_configured_ui(), repo_dir ) + includes_tools_for_display_in_tool_panel = False ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = create_repo_info_dict( trans=trans, repository_clone_url=repository_clone_url, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0