May 2013 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 22 May '13

22 May '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7228d0eac7b1/ Changeset: 7228d0eac7b1 User: dan Date: 2013-05-22 15:51:47 Summary: Fix for RawObjectWrapper when None values are provided. Affected #: 1 file diff -r feaab01c738a077719dd2bc90734e6ecde72d68e -r 7228d0eac7b183dca69c41725dc73bf5dedf8d0b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3264,7 +3264,11 @@ def __nonzero__( self ): return bool( self.obj ) #FIXME: would it be safe/backwards compatible to rename .obj to .value, so that we can just inherit this method? def __str__( self ): - return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + try: + return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + except: + #Most likely None, which lacks __module__. + return str( self.obj ) def __getattr__( self, key ): return getattr( self.obj, key ) https://bitbucket.org/galaxy/galaxy-central/commits/6f6ea9a765f6/ Changeset: 6f6ea9a765f6 Branch: next-stable User: dan Date: 2013-05-22 15:51:47 Summary: Fix for RawObjectWrapper when None values are provided. Affected #: 1 file diff -r 6b766dac9faec6e47fca620b514306149954811b -r 6f6ea9a765f6a0824fe0500e9b656f86143aa44c lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3264,7 +3264,11 @@ def __nonzero__( self ): return bool( self.obj ) #FIXME: would it be safe/backwards compatible to rename .obj to .value, so that we can just inherit this method? def __str__( self ): - return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + try: + return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + except: + #Most likely None, which lacks __module__. + return str( self.obj ) def __getattr__( self, key ): return getattr( self.obj, key ) https://bitbucket.org/galaxy/galaxy-central/commits/9d2b0932e6b4/ Changeset: 9d2b0932e6b4 User: dan Date: 2013-05-22 15:53:04 Summary: merge Affected #: 1 file diff -r b45cefc2901c6434acb7d9f5ab841e0554e81f17 -r 9d2b0932e6b43f6e8c5fa7525e10d70b799fdf17 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3264,7 +3264,11 @@ def __nonzero__( self ): return bool( self.obj ) #FIXME: would it be safe/backwards compatible to rename .obj to .value, so that we can just inherit this method? def __str__( self ): - return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + try: + return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + except: + #Most likely None, which lacks __module__. + return str( self.obj ) def __getattr__( self, key ): return getattr( self.obj, key ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Eliminate links in the tool shed menu when they are not necessary.
by commits-noreply＠bitbucket.org 22 May '13

22 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b45cefc2901c/ Changeset: b45cefc2901c User: greg Date: 2013-05-22 15:52:03 Summary: Eliminate links in the tool shed menu when they are not necessary. Affected #: 1 file diff -r feaab01c738a077719dd2bc90734e6ecde72d68e -r b45cefc2901c6434acb7d9f5ab841e0554e81f17 templates/webapps/tool_shed/index.mako --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -92,51 +92,54 @@ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories' )}">Browse by category</a></div> %if trans.user: - <div class="toolSectionPad"></div> - <div class="toolSectionTitle"> - Repositories I Can Change - </div> - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_i_own' )}">Repositories I own</a> - </div> - %if has_reviewed_repositories: + %if trans.user.active_repositories: + <div class="toolSectionPad"></div> + <div class="toolSectionTitle"> + Repositories I Can Change + </div><div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', operation='reviewed_repositories_i_own' )}">Reviewed repositories I own</a> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories_i_own' )}">Repositories I own</a> + </div> + %if has_reviewed_repositories: + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', operation='reviewed_repositories_i_own' )}">Reviewed repositories I own</a> + </div> + %endif + %if has_deprecated_repositories: + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_deprecated_repositories_i_own' )}">Deprecated repositories I own</a> + </div> + %endif + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories' )}">My writable repositories</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_missing_tool_test_components' )}">Latest revision: missing tool tests</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_install_errors' )}">Latest revision: installation errors</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_failing_tool_tests' )}">Latest revision: failing tool tests</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_skip_tool_test_checked' )}">Latest revision: skip tool tests</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_no_failing_tool_tests' )}">Latest revision: all tool tests pass</a> + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_invalid_tools' )}">Latest revision: invalid tools</a> + </div> + %else: + <div class="toolSectionPad"></div> + <div class="toolSectionTitle"> + Available Actions + </div> + <div class="toolTitle"> + <a target="galaxy_main" href="${h.url_for( controller='repository', action='create_repository' )}">Create new repository</a></div> %endif - %if has_deprecated_repositories: - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_deprecated_repositories_i_own' )}">Deprecated repositories I own</a> - </div> - %endif - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories' )}">My writable repositories</a> - </div> - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_missing_tool_test_components' )}">Latest revision: missing tool tests</a> - </div> - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_install_errors' )}">Latest revision: installation errors</a> - </div> - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_failing_tool_tests' )}">Latest revision: failing tool tests</a> - </div> - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_skip_tool_test_checked' )}">Latest revision: skip tool tests</a> - </div> - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_no_failing_tool_tests' )}">Latest revision: all tool tests pass</a> - </div> - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_my_writable_repositories_with_invalid_tools' )}">Latest revision: invalid tools</a> - </div> - <div class="toolSectionPad"></div> - <div class="toolSectionTitle"> - Available Actions - </div> - <div class="toolTitle"> - <a target="galaxy_main" href="${h.url_for( controller='repository', action='create_repository' )}">Create new repository</a> - </div> %if can_review_repositories: <div class="toolSectionPad"></div><div class="toolSectionTitle"> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Error handling and logging for the install and test exclude list.
by commits-noreply＠bitbucket.org 22 May '13

22 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/feaab01c738a/ Changeset: feaab01c738a User: Dave Bouvier Date: 2013-05-22 15:20:31 Summary: Error handling and logging for the install and test exclude list. Affected #: 1 file diff -r f5fe2f0940609a46b6105a2db66da821420f4f99 -r feaab01c738a077719dd2bc90734e6ecde72d68e test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -368,24 +368,34 @@ ] ''' exclude_list = [] + exclude_verbose = [] xml_tree = parse_xml( xml_filename ) tool_sheds = xml_tree.findall( 'repositories' ) - xml_element = None + xml_element = [] + exclude_count = 0 for tool_shed in tool_sheds: if galaxy_tool_shed_url != tool_shed.attrib[ 'tool_shed' ]: continue else: xml_element = tool_shed for reason_section in xml_element: - print reason_section reason_text = reason_section.find( 'text' ).text repositories = reason_section.findall( 'repository' ) exclude_dict = dict( reason=reason_text, repositories=[] ) for repository in repositories: repository_tuple = get_repository_tuple_from_elem( repository ) if repository_tuple not in exclude_dict[ 'repositories' ]: + exclude_verbose.append( repository_tuple ) + exclude_count += 1 exclude_dict[ 'repositories' ].append( repository_tuple ) exclude_list.append( exclude_dict ) + log.debug( '%d repositories excluded from testing...', exclude_count ) + if '-list_repositories' in sys.argv: + for name, owner, changeset_revision in exclude_verbose: + if changeset_revision: + log.debug( 'Repository %s owned by %s, changeset revision %s.', name, owner, changeset_revision ) + else: + log.debug( 'Repository %s owned by %s, all revisions.', name, owner ) return exclude_list def register_test_result( url, metadata_id, test_results_dict, repository_info_dict, params ): @@ -620,8 +630,8 @@ repositories_failed_install = [] exclude_list = [] if os.path.exists( exclude_list_file ): + log.info( 'Loading the list of repositories excluded from testing from the file %s...', exclude_list_file ) exclude_list = parse_exclude_list( exclude_list_file ) - log.info( exclude_list ) try: # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. log.info( "Retrieving repositories to install from the URL:\n%s\n", str( galaxy_tool_shed_url ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 21 May '13

21 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6b766dac9fae/ Changeset: 6b766dac9fae Branch: next-stable User: greg Date: 2013-05-21 22:46:16 Summary: Fix for defining set_environment_for_install tags in multiple tag sets instead of multiple repositories per tag set. Affected #: 2 files diff -r 206050841f67bfc31dfc67773bf815c1506ffe77 -r 6b766dac9faec6e47fca620b514306149954811b lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -131,7 +131,14 @@ tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) installation_directory = tool_dependency.installation_directory( app ) env_shell_file_path = get_env_shell_file_path( installation_directory ) - env_shell_file_paths.append( env_shell_file_path ) + if env_shell_file_path: + env_shell_file_paths.append( env_shell_file_path ) + else: + error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue else: error_message = "Skipping tool dependency definition because unable to locate tool dependency " error_message += "type %s, name %s, version %s for repository %s" % \ diff -r 206050841f67bfc31dfc67773bf815c1506ffe77 -r 6b766dac9faec6e47fca620b514306149954811b lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -346,6 +346,7 @@ if package_name: actions_dict[ 'package_name' ] = package_name actions = [] + all_env_shell_file_paths = [] for action_elem in actions_elem: action_dict = {} action_type = action_elem.get( 'type', 'shell_command' ) @@ -415,7 +416,6 @@ # that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be # supported. The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current # tool dependency package. See the package_matplotlib_1_2 repository in the test tool shed for a real-world example. - all_env_shell_file_paths = [] for env_elem in action_elem: if env_elem.tag == 'repository': env_shell_file_paths = common_util.get_env_shell_file_paths( app, env_elem ) https://bitbucket.org/galaxy/galaxy-central/commits/f5fe2f094060/ Changeset: f5fe2f094060 User: greg Date: 2013-05-21 22:46:57 Summary: Merged from next-stable Affected #: 2 files diff -r 542f2ba1bc575ab80168280159af642b7caace6f -r f5fe2f0940609a46b6105a2db66da821420f4f99 lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -131,7 +131,14 @@ tool_dependency_key = '%s/%s' % ( tool_dependency_name, tool_dependency_version ) installation_directory = tool_dependency.installation_directory( app ) env_shell_file_path = get_env_shell_file_path( installation_directory ) - env_shell_file_paths.append( env_shell_file_path ) + if env_shell_file_path: + env_shell_file_paths.append( env_shell_file_path ) + else: + error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " + error_message += "type %s, name %s, version %s for repository %s" % \ + ( str( tool_dependency_type ), str( tool_dependency_name ), str( tool_dependency_version ), str( repository.name ) ) + log.debug( error_message ) + continue else: error_message = "Skipping tool dependency definition because unable to locate tool dependency " error_message += "type %s, name %s, version %s for repository %s" % \ diff -r 542f2ba1bc575ab80168280159af642b7caace6f -r f5fe2f0940609a46b6105a2db66da821420f4f99 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -346,6 +346,7 @@ if package_name: actions_dict[ 'package_name' ] = package_name actions = [] + all_env_shell_file_paths = [] for action_elem in actions_elem: action_dict = {} action_type = action_elem.get( 'type', 'shell_command' ) @@ -415,7 +416,6 @@ # that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be # supported. The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current # tool dependency package. See the package_matplotlib_1_2 repository in the test tool shed for a real-world example. - all_env_shell_file_paths = [] for env_elem in action_elem: if env_elem.tag == 'repository': env_shell_file_paths = common_util.get_env_shell_file_paths( app, env_elem ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 21 May '13

21 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/206050841f67/ Changeset: 206050841f67 Branch: next-stable User: natefoo Date: 2013-05-21 22:28:44 Summary: Pass a db session in to MetadataParameters' wrap() method so that MetadataFiles can be returned as real mapped objects and passed to the object store. Affected #: 1 file diff -r 3836fc62d3a24a1ad432bc8888dc0f92dbf32292 -r 206050841f67bfc31dfc67773bf815c1506ffe77 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -85,8 +85,8 @@ def __getattr__( self, name ): if name in self.spec: if name in self.parent._metadata: - return self.spec[name].wrap( self.parent._metadata[name] ) - return self.spec[name].wrap( self.spec[name].default ) + return self.spec[name].wrap( self.parent._metadata[name], object_session( self.parent ) ) + return self.spec[name].wrap( self.spec[name].default, object_session( self.parent ) ) if name in self.parent._metadata: return self.parent._metadata[name] def __setattr__( self, name, value ): @@ -213,7 +213,7 @@ self.validate( value ) return value - def wrap( self, value ): + def wrap( self, value, session ): """ Turns a value into its usable form. """ @@ -256,11 +256,11 @@ def get( self, name, default=None ): return self.__dict__.get(name, default) - def wrap( self, value ): + def wrap( self, value, session ): """ Turns a stored value into its usable form. """ - return self.param.wrap( value ) + return self.param.wrap( value, session ) def unwrap( self, value ): """ @@ -323,7 +323,7 @@ return ", ".join( map( str, value ) ) return MetadataParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd ) - def wrap( self, value ): + def wrap( self, value, session ): value = self.marshal( value ) #do we really need this (wasteful)? - yes because we are not sure that all existing selects have been stored previously as lists. Also this will handle the case where defaults/no_values are specified and are single non-list values. if self.multiple: return value @@ -435,17 +435,16 @@ def get_html( self, value=None, context={}, other_values={}, **kwd ): return "<div>No display available for Metadata Files</div>" - def wrap( self, value ): + def wrap( self, value, session ): if value is None: return None if isinstance( value, galaxy.model.MetadataFile ) or isinstance( value, MetadataTempFile ): return value - mf = galaxy.model.MetadataFile() - mf.id = value #we assume this is a valid id, since we cannot check it + mf = session.query( galaxy.model.MetadataFile ).get( value ) return mf def make_copy( self, value, target_context, source_context ): - value = self.wrap( value ) + value = self.wrap( value, object_session( target_context.parent ) ) if value: new_value = galaxy.model.MetadataFile( dataset = target_context.parent, name = self.spec.name ) object_session( target_context.parent ).add( new_value ) https://bitbucket.org/galaxy/galaxy-central/commits/542f2ba1bc57/ Changeset: 542f2ba1bc57 User: natefoo Date: 2013-05-21 22:29:04 Summary: Merge next-stable Affected #: 1 file diff -r d8259d7fcba011f1d97eb808d7812346d0ef8739 -r 542f2ba1bc575ab80168280159af642b7caace6f lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -85,8 +85,8 @@ def __getattr__( self, name ): if name in self.spec: if name in self.parent._metadata: - return self.spec[name].wrap( self.parent._metadata[name] ) - return self.spec[name].wrap( self.spec[name].default ) + return self.spec[name].wrap( self.parent._metadata[name], object_session( self.parent ) ) + return self.spec[name].wrap( self.spec[name].default, object_session( self.parent ) ) if name in self.parent._metadata: return self.parent._metadata[name] def __setattr__( self, name, value ): @@ -213,7 +213,7 @@ self.validate( value ) return value - def wrap( self, value ): + def wrap( self, value, session ): """ Turns a value into its usable form. """ @@ -256,11 +256,11 @@ def get( self, name, default=None ): return self.__dict__.get(name, default) - def wrap( self, value ): + def wrap( self, value, session ): """ Turns a stored value into its usable form. """ - return self.param.wrap( value ) + return self.param.wrap( value, session ) def unwrap( self, value ): """ @@ -323,7 +323,7 @@ return ", ".join( map( str, value ) ) return MetadataParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd ) - def wrap( self, value ): + def wrap( self, value, session ): value = self.marshal( value ) #do we really need this (wasteful)? - yes because we are not sure that all existing selects have been stored previously as lists. Also this will handle the case where defaults/no_values are specified and are single non-list values. if self.multiple: return value @@ -435,17 +435,16 @@ def get_html( self, value=None, context={}, other_values={}, **kwd ): return "<div>No display available for Metadata Files</div>" - def wrap( self, value ): + def wrap( self, value, session ): if value is None: return None if isinstance( value, galaxy.model.MetadataFile ) or isinstance( value, MetadataTempFile ): return value - mf = galaxy.model.MetadataFile() - mf.id = value #we assume this is a valid id, since we cannot check it + mf = session.query( galaxy.model.MetadataFile ).get( value ) return mf def make_copy( self, value, target_context, source_context ): - value = self.wrap( value ) + value = self.wrap( value, object_session( target_context.parent ) ) if value: new_value = galaxy.model.MetadataFile( dataset = target_context.parent, name = self.spec.name ) object_session( target_context.parent ).add( new_value ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for detecting Data Manager tool dependencies when installed from a tool shed.
by commits-noreply＠bitbucket.org 21 May '13

21 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3836fc62d3a2/ Changeset: 3836fc62d3a2 Branch: next-stable User: dan Date: 2013-05-21 22:21:17 Summary: Fix for detecting Data Manager tool dependencies when installed from a tool shed. Affected #: 2 files diff -r 00e6a69ed1c4f34e68ab89ca03462e47c23b71c4 -r 3836fc62d3a24a1ad432bc8888dc0f92dbf32292 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -952,6 +952,8 @@ self.version = None # Enable easy access to this tool's version lineage. self.lineage_ids = [] + #populate toolshed repository info, if available + self.populate_tool_shed_info() # Parse XML element containing configuration self.parse( root, guid=guid ) self.external_runJob_script = app.config.drmaa_external_runjob_script @@ -1775,6 +1777,17 @@ version = requirement_elem.get( "version", None ) requirement = ToolRequirement( name=name, type=type, version=version ) self.requirements.append( requirement ) + + def populate_tool_shed_info( self ): + if self.repository_id is not None and 'ToolShedRepository' in self.app.model: + repository_id = self.app.security.decode_id( self.repository_id ) + tool_shed_repository = self.sa_session.query( self.app.model.ToolShedRepository ).get( repository_id ) + if tool_shed_repository: + self.tool_shed = tool_shed_repository.tool_shed + self.repository_name = tool_shed_repository.name + self.repository_owner = tool_shed_repository.owner + self.installed_changeset_revision = tool_shed_repository.installed_changeset_revision + def check_workflow_compatible( self, root ): """ Determine if a tool can be used in workflows. External tools and the diff -r 00e6a69ed1c4f34e68ab89ca03462e47c23b71c4 -r 3836fc62d3a24a1ad432bc8888dc0f92dbf32292 lib/galaxy/tools/data_manager/manager.py --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -9,6 +9,7 @@ from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.tools.data import TabularToolDataTable +import tool_shed.util.shed_util_common as suc #set up logger import logging @@ -113,12 +114,20 @@ self.guid = elem.get( 'guid', None ) path = elem.get( 'tool_file', None ) self.version = elem.get( 'version', self.version ) + tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find( 'tool' ) assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) ) path = tool_elem.get( "file", None ) tool_guid = tool_elem.get( "guid", None ) + #need to determine repository info so that dependencies will work correctly + tool_shed = tool_elem.find( 'tool_shed' ).text + repository_name = tool_elem.find( 'repository_name' ).text + repository_owner = tool_elem.find( 'repository_owner' ).text + installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text + tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.data_managers.app, tool_shed, repository_name, repository_owner, installed_changeset_revision ) + tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) #use shed_conf_file to determine tool_path shed_conf_file = elem.get( "shed_conf_file", None ) if shed_conf_file: @@ -126,7 +135,7 @@ if shed_conf: tool_path = shed_conf.get( "tool_path", tool_path ) assert path is not None, "A tool file path could not be determined:\n%s" % ( util.xml_to_string( elem ) ) - self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id ) + self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id, tool_shed_repository_id=tool_shed_repository_id ) self.name = elem.get( 'name', self.tool.name ) self.description = elem.get( 'description', self.tool.description ) @@ -180,8 +189,8 @@ @property def id( self ): return self.guid or self.declared_id #if we have a guid, we will use that as the data_manager id - def load_tool( self, tool_filename, guid=None, data_manager_id=None ): - tool = self.data_managers.app.toolbox.load_tool( tool_filename, guid=guid, data_manager_id=data_manager_id ) + def load_tool( self, tool_filename, guid=None, data_manager_id=None, tool_shed_repository_id=None ): + tool = self.data_managers.app.toolbox.load_tool( tool_filename, guid=guid, data_manager_id=data_manager_id, repository_id=tool_shed_repository_id ) self.data_managers.app.toolbox.data_manager_tools[ tool.id ] = tool self.data_managers.app.toolbox.tools_by_id[ tool.id ] = tool self.tool = tool Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for detecting Data Manager tool dependencies when installed from a tool shed.
by commits-noreply＠bitbucket.org 21 May '13

21 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d8259d7fcba0/ Changeset: d8259d7fcba0 User: dan Date: 2013-05-21 22:21:17 Summary: Fix for detecting Data Manager tool dependencies when installed from a tool shed. Affected #: 2 files diff -r 856800d4c91546d51948176f4b78a7bd4b1a1b8b -r d8259d7fcba011f1d97eb808d7812346d0ef8739 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -952,6 +952,8 @@ self.version = None # Enable easy access to this tool's version lineage. self.lineage_ids = [] + #populate toolshed repository info, if available + self.populate_tool_shed_info() # Parse XML element containing configuration self.parse( root, guid=guid ) self.external_runJob_script = app.config.drmaa_external_runjob_script @@ -1775,6 +1777,17 @@ version = requirement_elem.get( "version", None ) requirement = ToolRequirement( name=name, type=type, version=version ) self.requirements.append( requirement ) + + def populate_tool_shed_info( self ): + if self.repository_id is not None and 'ToolShedRepository' in self.app.model: + repository_id = self.app.security.decode_id( self.repository_id ) + tool_shed_repository = self.sa_session.query( self.app.model.ToolShedRepository ).get( repository_id ) + if tool_shed_repository: + self.tool_shed = tool_shed_repository.tool_shed + self.repository_name = tool_shed_repository.name + self.repository_owner = tool_shed_repository.owner + self.installed_changeset_revision = tool_shed_repository.installed_changeset_revision + def check_workflow_compatible( self, root ): """ Determine if a tool can be used in workflows. External tools and the diff -r 856800d4c91546d51948176f4b78a7bd4b1a1b8b -r d8259d7fcba011f1d97eb808d7812346d0ef8739 lib/galaxy/tools/data_manager/manager.py --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -9,6 +9,7 @@ from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.tools.data import TabularToolDataTable +import tool_shed.util.shed_util_common as suc #set up logger import logging @@ -113,12 +114,20 @@ self.guid = elem.get( 'guid', None ) path = elem.get( 'tool_file', None ) self.version = elem.get( 'version', self.version ) + tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find( 'tool' ) assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) ) path = tool_elem.get( "file", None ) tool_guid = tool_elem.get( "guid", None ) + #need to determine repository info so that dependencies will work correctly + tool_shed = tool_elem.find( 'tool_shed' ).text + repository_name = tool_elem.find( 'repository_name' ).text + repository_owner = tool_elem.find( 'repository_owner' ).text + installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text + tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.data_managers.app, tool_shed, repository_name, repository_owner, installed_changeset_revision ) + tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) #use shed_conf_file to determine tool_path shed_conf_file = elem.get( "shed_conf_file", None ) if shed_conf_file: @@ -126,7 +135,7 @@ if shed_conf: tool_path = shed_conf.get( "tool_path", tool_path ) assert path is not None, "A tool file path could not be determined:\n%s" % ( util.xml_to_string( elem ) ) - self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id ) + self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id, tool_shed_repository_id=tool_shed_repository_id ) self.name = elem.get( 'name', self.tool.name ) self.description = elem.get( 'description', self.tool.description ) @@ -180,8 +189,8 @@ @property def id( self ): return self.guid or self.declared_id #if we have a guid, we will use that as the data_manager id - def load_tool( self, tool_filename, guid=None, data_manager_id=None ): - tool = self.data_managers.app.toolbox.load_tool( tool_filename, guid=guid, data_manager_id=data_manager_id ) + def load_tool( self, tool_filename, guid=None, data_manager_id=None, tool_shed_repository_id=None ): + tool = self.data_managers.app.toolbox.load_tool( tool_filename, guid=guid, data_manager_id=data_manager_id, repository_id=tool_shed_repository_id ) self.data_managers.app.toolbox.data_manager_tools[ tool.id ] = tool self.data_managers.app.toolbox.tools_by_id[ tool.id ] = tool self.tool = tool Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Add the ability to define a list of repositories or changeset revisions of repositories that should not be tested by the automated testing framework.
by commits-noreply＠bitbucket.org 21 May '13

21 May '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/856800d4c915/ Changeset: 856800d4c915 User: Dave Bouvier Date: 2013-05-21 22:02:21 Summary: Add the ability to define a list of repositories or changeset revisions of repositories that should not be tested by the automated testing framework. Affected #: 1 file diff -r fe0905ea52874c451303f5e60d5158387695aaef -r 856800d4c91546d51948176f4b78a7bd4b1a1b8b test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -129,6 +129,7 @@ galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', None ) tool_shed_api_key = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY', None ) +exclude_list_file = os.environ.get( 'GALAXY_INSTALL_TEST_EXCLUDE_REPOSITORIES', 'install_test_exclude.xml' ) if tool_shed_api_key is None: print "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty." @@ -243,6 +244,13 @@ extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) return extended_dict +def get_repository_tuple_from_elem( elem ): + attributes = elem.attrib + name = attributes.get( 'name', None ) + owner = attributes.get( 'owner', None ) + changeset_revision = attributes.get( 'changeset_revision', None ) + return ( name, owner, changeset_revision ) + def get_repositories_to_install( tool_shed_url, latest_revision_only=True ): ''' Get a list of repository info dicts to install. This method expects a json list of dicts with the following structure: @@ -266,6 +274,7 @@ skip_tool_test='false' ) ) api_url = get_api_url( base=tool_shed_url, parts=[ 'repository_revisions' ], params=params ) base_repository_list = json_from_url( api_url ) + log.info( 'The api returned %d metadata revisions.', len( base_repository_list ) ) known_repository_ids = {} detailed_repository_list = [] for repository_to_install_dict in base_repository_list: @@ -336,6 +345,49 @@ raise return parsed_json +def parse_exclude_list( xml_filename ): + ''' + This method should return a list with the following structure: + [ + { + 'reason': The default reason or the reason specified in this section, + 'repositories': + [ + ( name, owner, changeset revision if changeset revision else None ), + ( name, owner, changeset revision if changeset revision else None ) + ] + }, + { + 'reason': The default reason or the reason specified in this section, + 'repositories': + [ + ( name, owner, changeset revision if changeset revision else None ), + ( name, owner, changeset revision if changeset revision else None ) + ] + }, + ] + ''' + exclude_list = [] + xml_tree = parse_xml( xml_filename ) + tool_sheds = xml_tree.findall( 'repositories' ) + xml_element = None + for tool_shed in tool_sheds: + if galaxy_tool_shed_url != tool_shed.attrib[ 'tool_shed' ]: + continue + else: + xml_element = tool_shed + for reason_section in xml_element: + print reason_section + reason_text = reason_section.find( 'text' ).text + repositories = reason_section.findall( 'repository' ) + exclude_dict = dict( reason=reason_text, repositories=[] ) + for repository in repositories: + repository_tuple = get_repository_tuple_from_elem( repository ) + if repository_tuple not in exclude_dict[ 'repositories' ]: + exclude_dict[ 'repositories' ].append( repository_tuple ) + exclude_list.append( exclude_dict ) + return exclude_list + def register_test_result( url, metadata_id, test_results_dict, repository_info_dict, params ): ''' Update the repository metadata tool_test_results and appropriate flags using the API. @@ -566,6 +618,10 @@ repositories_passed = [] repositories_failed = [] repositories_failed_install = [] + exclude_list = [] + if os.path.exists( exclude_list_file ): + exclude_list = parse_exclude_list( exclude_list_file ) + log.info( exclude_list ) try: # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. log.info( "Retrieving repositories to install from the URL:\n%s\n", str( galaxy_tool_shed_url ) ) @@ -623,8 +679,43 @@ # Get the name and owner out of the repository info dict. name = str( repository_info_dict[ 'name' ] ) owner = str( repository_info_dict[ 'owner' ] ) - log.info( "Installing and testing revision %s of repository id %s (%s/%s)...", - str( changeset_revision ), str( repository_id ), owner, name ) + # Populate the repository_status dict now. + repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) + if 'test_environment' not in repository_status: + repository_status[ 'test_environment' ] = {} + test_environment = get_test_environment( repository_status[ 'test_environment' ] ) + test_environment[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_environment' ] = test_environment + repository_status[ 'passed_tests' ] = [] + repository_status[ 'failed_tests' ] = [] + repository_status[ 'skip_reason' ] = None + # Iterate through the list of repositories defined not to be installed. This should be a list of dicts in the following format: + # { + # 'reason': The default reason or the reason specified in this section, + # 'repositories': + # [ + # ( name, owner, changeset revision if changeset revision else None ), + # ( name, owner, changeset revision if changeset revision else None ) + # ] + # }, + # If changeset revision is None, that means the entire repository is excluded from testing, otherwise only the specified + # revision should be skipped. + # TODO: When a repository is selected to be skipped, use the API to update the tool shed with the defined skip reason. + skip_this_repository = False + skip_because = None + for exclude_by_reason in exclude_list: + reason = exclude_by_reason[ 'reason' ] + exclude_repositories = exclude_by_reason[ 'repositories' ] + if ( name, owner, changeset_revision ) in exclude_repositories or ( name, owner, None ) in exclude_repositories: + skip_this_repository = True + skip_because = reason + break + if skip_this_repository: + log.info( "Not testing revision %s of repository %s owned by %s.", changeset_revision, name, owner ) + continue + else: + log.info( "Installing and testing revision %s of repository %s owned by %s...", changeset_revision, name, owner ) # Explicitly clear tests from twill's test environment. remove_generated_tests( app ) # Use the repository information dict to generate an install method that will install the repository into the embedded @@ -638,15 +729,6 @@ # repository, with tool and repository dependencies also selected for installation. result, _ = run_tests( test_config ) success = result.wasSuccessful() - repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) - if 'test_environment' not in repository_status: - repository_status[ 'test_environment' ] = {} - test_environment = get_test_environment( repository_status[ 'test_environment' ] ) - test_environment[ 'galaxy_database_version' ] = get_database_version( app ) - test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) - repository_status[ 'test_environment' ] = test_environment - repository_status[ 'passed_tests' ] = [] - repository_status[ 'failed_tests' ] = [] repository_status[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) try: repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) @@ -750,16 +832,19 @@ # If the repository does not have a test-data directory, any functional tests in the tool configuration will # fail. Mark the repository as failed and skip installation. log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' ) - # Record the lack of test data. - for tool in repository.metadata[ 'tools' ]: - tool_id = tool[ 'id' ] - tool_version = tool[ 'version' ] - tool_guid = tool[ 'guid' ] - # In keeping with the standard display layout, add the error message to the dict for each tool individually. - missing_components = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, - missing_components="Repository %s is missing a test-data directory." % name ) - if missing_components not in repository_status[ 'missing_test_components' ]: - repository_status[ 'missing_test_components' ].append( missing_components ) + # Record the lack of test data if the repository metadata defines tools. + if 'tools' in repository.metadata: + for tool in repository.metadata[ 'tools' ]: + tool_id = tool[ 'id' ] + tool_version = tool[ 'version' ] + tool_guid = tool[ 'guid' ] + # In keeping with the standard display layout, add the error message to the dict for each tool individually. + missing_components = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, + missing_components="Repository %s is missing a test-data directory." % name ) + if missing_components not in repository_status[ 'missing_test_components' ]: + repository_status[ 'missing_test_components' ].append( missing_components ) + else: + continue # Record the status of this repository in the tool shed. set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) params[ 'tools_functionally_correct' ] = False Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 21 May '13

21 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/00e6a69ed1c4/ Changeset: 00e6a69ed1c4 Branch: next-stable User: Dave Bouvier Date: 2013-05-21 21:53:42 Summary: Fix for repositories incorrectly being displayed in the list of repositories with failing tool tests. Affected #: 1 file diff -r ea8d92515d42321592e77842387290dcc1190a2b -r 00e6a69ed1c4f34e68ab89ca03462e47c23b71c4 lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -1650,6 +1650,7 @@ tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) if repository_metadata \ + and repository_metadata.tool_test_results is not None \ and not repository_metadata.missing_test_components \ and not repository_metadata.tools_functionally_correct \ and not repository_metadata.test_install_error: https://bitbucket.org/galaxy/galaxy-central/commits/fe0905ea5287/ Changeset: fe0905ea5287 User: Dave Bouvier Date: 2013-05-21 21:54:20 Summary: Merge in fix from next-stable. Affected #: 1 file diff -r 342617fcad86f6948dabaf136fe0c04a83212aad -r fe0905ea52874c451303f5e60d5158387695aaef lib/tool_shed/grids/repository_grids.py --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -1650,6 +1650,7 @@ tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) repository_metadata = get_latest_installable_repository_metadata_if_it_includes_tools( trans, repository ) if repository_metadata \ + and repository_metadata.tool_test_results is not None \ and not repository_metadata.missing_test_components \ and not repository_metadata.tools_functionally_correct \ and not repository_metadata.test_install_error: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 21 May '13

21 May '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/ea8d92515d42/ Changeset: ea8d92515d42 Branch: next-stable User: carlfeberhard Date: 2013-05-21 21:19:33 Summary: History: add simple profiling method Affected #: 3 files diff -r 6b59809541ce4766efd1d044fbdee86a954746c4 -r ea8d92515d42321592e77842387290dcc1190a2b lib/galaxy/util/debugging.py --- a/lib/galaxy/util/debugging.py +++ b/lib/galaxy/util/debugging.py @@ -1,6 +1,7 @@ import inspect import pprint +import time import logging log = logging.getLogger( __name__ ) @@ -27,3 +28,39 @@ stack_list.append( line_format.format( **caller_data ) ) return '\n'.join( stack_list ) + + +class SimpleProfiler( object ): + """ + Simple profiler that captures the duration between calls to `report` + and stores the results in a list. + """ + REPORT_FORMAT = '%20f: %s' + + def __init__( self, log=None ): + self.log = log + self.start_time = 0 + self.reports = [] + + def start( self, msg=None ): + msg = msg or 'Start' + self.start_time = time.time() + report = self.REPORT_FORMAT %( self.start_time, msg ) + self.reports.append( report ) + if self.log: + self.log( report ) + + def report( self, msg ): + if not self.start_time: + self.start() + duration = time.time() - self.start_time + report = self.REPORT_FORMAT %( duration, msg ) + self.reports.append( report ) + if self.log: + self.log( report ) + + def get_reports( self ): + return self.reports + + def __str__( self ): + return '\n'.join( self.reports ) diff -r 6b59809541ce4766efd1d044fbdee86a954746c4 -r ea8d92515d42321592e77842387290dcc1190a2b lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -2,7 +2,6 @@ Contains the main interface in the Universe class """ import cgi -import logging import os import urllib @@ -15,6 +14,9 @@ from galaxy.web.base.controller import BaseUIController, UsesHistoryDatasetAssociationMixin, UsesHistoryMixin from galaxy.util.json import to_json_string +from galaxy.util.debugging import SimpleProfiler + +import logging log = logging.getLogger( __name__ ) class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, UsesAnnotations ): @@ -176,6 +178,73 @@ history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ), show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status ) + @web.expose + def profile_history( self, trans, as_xml=False, show_deleted=None, show_hidden=None, hda_id=None, **kwd ): + """ + Same as above but adds SimpleProfiler to get some + profiling times for the operations done. + """ + if as_xml: + return self.history_as_xml( trans, + show_deleted=string_as_bool( show_deleted ), show_hidden=string_as_bool( show_hidden ) ) + + # get all datasets server-side, client-side will get flags and render appropriately + show_deleted = string_as_bool_or_none( show_deleted ) + show_purged = show_deleted + show_hidden = string_as_bool_or_none( show_hidden ) + params = Params( kwd ) + message = params.get( 'message', '' ) + #TODO: ugh... + message = message if message != 'None' else '' + status = params.get( 'status', 'done' ) + + if trans.app.config.require_login and not trans.user: + return trans.fill_template( '/no_access.mako', message = 'Please log in to access Galaxy histories.' ) + + def err_msg( where=None ): + where = where if where else 'getting the history data from the server' + err_msg = ( 'An error occurred %s. ' + + 'Please contact a Galaxy administrator if the problem persists.' ) %( where ) + return err_msg, 'error' + + profiler = SimpleProfiler() + profiler.start() + + history_dictionary = {} + hda_dictionaries = [] + try: + history = trans.get_history( create=True ) + profiler.report( 'trans.get_history' ) + hdas = self.get_history_datasets( trans, history, + show_deleted=True, show_hidden=True, show_purged=True ) + profiler.report( 'get_history_datasets' ) + + for hda in hdas: + try: + hda_dictionaries.append( self.get_hda_dict( trans, hda ) ) + profiler.report( '\t hda -> dictionary (%s)' %( hda.name ) ) + + except Exception, exc: + # don't fail entire list if hda err's, record and move on + log.error( 'Error bootstrapping hda %d: %s', hda.id, str( exc ), exc_info=True ) + hda_dictionaries.append( self.get_hda_dict_with_error( trans, hda, str( exc ) ) ) + profiler.report( 'hdas -> dictionaries' ) + + # re-use the hdas above to get the history data... + history_dictionary = self.get_history_dict( trans, history, hda_dictionaries=hda_dictionaries ) + profiler.report( 'history -> dictionary' ) + + except Exception, exc: + user_id = str( trans.user.id ) if trans.user else '(anonymous)' + log.error( 'Error bootstrapping history for user %s: %s', user_id, str( exc ), exc_info=True ) + message, status = err_msg() + history_dictionary[ 'error' ] = message + + return trans.stream_template_mako( "root/history.mako", + history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ), + show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status, + profiling=profiler.get_reports() ) + ## ---- Dataset display / editing ---------------------------------------- @web.expose def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): diff -r 6b59809541ce4766efd1d044fbdee86a954746c4 -r ea8d92515d42321592e77842387290dcc1190a2b templates/webapps/galaxy/root/history.mako --- a/templates/webapps/galaxy/root/history.mako +++ b/templates/webapps/galaxy/root/history.mako @@ -213,6 +213,10 @@ galaxyPageSetUp(); // ostensibly, this is the App + %if 'profiling' in self.context.kwargs: + Galaxy.profiling = ${h.to_json_string( profiling )}.join( '\n' ); + %endif + //NOTE: for debugging on non-local instances (main/test) // 1. load history panel in own tab // 2. from console: new PersistantStorage( '__history_panel' ).set( 'debugging', true ) https://bitbucket.org/galaxy/galaxy-central/commits/342617fcad86/ Changeset: 342617fcad86 User: carlfeberhard Date: 2013-05-21 21:20:03 Summary: merge next-stable Affected #: 3 files diff -r cbc1808b261b662f8dc9682b31f86c9b6c429bbd -r 342617fcad86f6948dabaf136fe0c04a83212aad lib/galaxy/util/debugging.py --- a/lib/galaxy/util/debugging.py +++ b/lib/galaxy/util/debugging.py @@ -1,6 +1,7 @@ import inspect import pprint +import time import logging log = logging.getLogger( __name__ ) @@ -27,3 +28,39 @@ stack_list.append( line_format.format( **caller_data ) ) return '\n'.join( stack_list ) + + +class SimpleProfiler( object ): + """ + Simple profiler that captures the duration between calls to `report` + and stores the results in a list. + """ + REPORT_FORMAT = '%20f: %s' + + def __init__( self, log=None ): + self.log = log + self.start_time = 0 + self.reports = [] + + def start( self, msg=None ): + msg = msg or 'Start' + self.start_time = time.time() + report = self.REPORT_FORMAT %( self.start_time, msg ) + self.reports.append( report ) + if self.log: + self.log( report ) + + def report( self, msg ): + if not self.start_time: + self.start() + duration = time.time() - self.start_time + report = self.REPORT_FORMAT %( duration, msg ) + self.reports.append( report ) + if self.log: + self.log( report ) + + def get_reports( self ): + return self.reports + + def __str__( self ): + return '\n'.join( self.reports ) diff -r cbc1808b261b662f8dc9682b31f86c9b6c429bbd -r 342617fcad86f6948dabaf136fe0c04a83212aad lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -2,7 +2,6 @@ Contains the main interface in the Universe class """ import cgi -import logging import os import urllib @@ -15,6 +14,9 @@ from galaxy.web.base.controller import BaseUIController, UsesHistoryDatasetAssociationMixin, UsesHistoryMixin from galaxy.util.json import to_json_string +from galaxy.util.debugging import SimpleProfiler + +import logging log = logging.getLogger( __name__ ) class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, UsesAnnotations ): @@ -176,6 +178,73 @@ history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ), show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status ) + @web.expose + def profile_history( self, trans, as_xml=False, show_deleted=None, show_hidden=None, hda_id=None, **kwd ): + """ + Same as above but adds SimpleProfiler to get some + profiling times for the operations done. + """ + if as_xml: + return self.history_as_xml( trans, + show_deleted=string_as_bool( show_deleted ), show_hidden=string_as_bool( show_hidden ) ) + + # get all datasets server-side, client-side will get flags and render appropriately + show_deleted = string_as_bool_or_none( show_deleted ) + show_purged = show_deleted + show_hidden = string_as_bool_or_none( show_hidden ) + params = Params( kwd ) + message = params.get( 'message', '' ) + #TODO: ugh... + message = message if message != 'None' else '' + status = params.get( 'status', 'done' ) + + if trans.app.config.require_login and not trans.user: + return trans.fill_template( '/no_access.mako', message = 'Please log in to access Galaxy histories.' ) + + def err_msg( where=None ): + where = where if where else 'getting the history data from the server' + err_msg = ( 'An error occurred %s. ' + + 'Please contact a Galaxy administrator if the problem persists.' ) %( where ) + return err_msg, 'error' + + profiler = SimpleProfiler() + profiler.start() + + history_dictionary = {} + hda_dictionaries = [] + try: + history = trans.get_history( create=True ) + profiler.report( 'trans.get_history' ) + hdas = self.get_history_datasets( trans, history, + show_deleted=True, show_hidden=True, show_purged=True ) + profiler.report( 'get_history_datasets' ) + + for hda in hdas: + try: + hda_dictionaries.append( self.get_hda_dict( trans, hda ) ) + profiler.report( '\t hda -> dictionary (%s)' %( hda.name ) ) + + except Exception, exc: + # don't fail entire list if hda err's, record and move on + log.error( 'Error bootstrapping hda %d: %s', hda.id, str( exc ), exc_info=True ) + hda_dictionaries.append( self.get_hda_dict_with_error( trans, hda, str( exc ) ) ) + profiler.report( 'hdas -> dictionaries' ) + + # re-use the hdas above to get the history data... + history_dictionary = self.get_history_dict( trans, history, hda_dictionaries=hda_dictionaries ) + profiler.report( 'history -> dictionary' ) + + except Exception, exc: + user_id = str( trans.user.id ) if trans.user else '(anonymous)' + log.error( 'Error bootstrapping history for user %s: %s', user_id, str( exc ), exc_info=True ) + message, status = err_msg() + history_dictionary[ 'error' ] = message + + return trans.stream_template_mako( "root/history.mako", + history_json = to_json_string( history_dictionary ), hda_json = to_json_string( hda_dictionaries ), + show_deleted=show_deleted, show_hidden=show_hidden, hda_id=hda_id, log=log, message=message, status=status, + profiling=profiler.get_reports() ) + ## ---- Dataset display / editing ---------------------------------------- @web.expose def display( self, trans, id=None, hid=None, tofile=None, toext=".txt", **kwd ): diff -r cbc1808b261b662f8dc9682b31f86c9b6c429bbd -r 342617fcad86f6948dabaf136fe0c04a83212aad templates/webapps/galaxy/root/history.mako --- a/templates/webapps/galaxy/root/history.mako +++ b/templates/webapps/galaxy/root/history.mako @@ -213,6 +213,10 @@ galaxyPageSetUp(); // ostensibly, this is the App + %if 'profiling' in self.context.kwargs: + Galaxy.profiling = ${h.to_json_string( profiling )}.join( '\n' ); + %endif + //NOTE: for debugging on non-local instances (main/test) // 1. load history panel in own tab // 2. from console: new PersistantStorage( '__history_panel' ).set( 'debugging', true ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0