5 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3af604bcd666/ Changeset: 3af604bcd666 User: jmchilton Date: 2014-12-14 03:09:51+00:00 Summary: Remove incorrect usage of threading.Lock(). Not sure what the intention was - but the Lock is created locally in the function so nothing else would be waiting on that particular lock. At some point we may wish to revisit other usages introduced in 1bb04279216e0abf4e24b9299cccc06c10b9c0ac. Affected #: 1 file diff -r fa061192b5d9c6df7ef65edb39a846e5a9a227bf -r 3af604bcd66681f890ca86d3c7f0395e9d5e8d05 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1688,14 +1688,10 @@ self.help = tool_source.root.find( 'help' ) if self.repository_id and self.help.text.find( '.. image:: ' ) >= 0: # Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy. - lock = threading.Lock() - lock.acquire( True ) try: self.help.text = suc.set_image_paths( self.app, self.repository_id, self.help.text ) except Exception, e: log.exception( "Exception in parse_help, so images may not be properly displayed:\n%s" % str( e ) ) - finally: - lock.release() help_pages = self.help.findall( "page" ) help_header = self.help.text try: https://bitbucket.org/galaxy/galaxy-central/commits/3d7d15601e49/ Changeset: 3d7d15601e49 User: jmchilton Date: 2014-12-14 03:09:51+00:00 Summary: Improvements to loading help text from tools. Do not convert rst to a mako template until needed, this is a costly operation and has the potential to speed update Galaxy start time. Move logic related to parsing of simple help text blocks out of tool and into the new tool parser interface and add implementation for YAML-based tools as well as unit tests for both. More advanced, multi-page tool help is still possible, workflows with the old tool form, but is only available to XML-based tools. Affected #: 5 files diff -r 3af604bcd66681f890ca86d3c7f0395e9d5e8d05 -r 3d7d15601e493d1d7ed6a10814f6ca2b36a611fb lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -101,6 +101,8 @@ </when></conditional>""" +HELP_UNINITIALIZED = threading.Lock() + class ToolNotFoundException( Exception ): pass @@ -1680,43 +1682,9 @@ This implementation supports multiple pages. """ # TODO: Allow raw HTML or an external link. - self.help = None - self.help_by_page = list() - help_header = "" - help_footer = "" - if hasattr( tool_source, 'root' ) and tool_source.root.find( 'help' ) is not None: - self.help = tool_source.root.find( 'help' ) - if self.repository_id and self.help.text.find( '.. image:: ' ) >= 0: - # Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy. - try: - self.help.text = suc.set_image_paths( self.app, self.repository_id, self.help.text ) - except Exception, e: - log.exception( "Exception in parse_help, so images may not be properly displayed:\n%s" % str( e ) ) - help_pages = self.help.findall( "page" ) - help_header = self.help.text - try: - self.help = Template( rst_to_html(self.help.text), input_encoding='utf-8', - output_encoding='utf-8', default_filters=[ 'decode.utf8' ], - encoding_errors='replace' ) - except: - log.exception( "error in help for tool %s" % self.name ) - # Multiple help page case - if help_pages: - for help_page in help_pages: - self.help_by_page.append( help_page.text ) - help_footer = help_footer + help_page.tail - # Each page has to rendered all-together because of backreferences allowed by rst - try: - self.help_by_page = [ Template( rst_to_html( help_header + x + help_footer ), - input_encoding='utf-8', output_encoding='utf-8', - default_filters=[ 'decode.utf8' ], - encoding_errors='replace' ) - for x in self.help_by_page ] - except: - log.exception( "error in multi-page help for tool %s" % self.name ) - # Pad out help pages to match npages ... could this be done better? - while len( self.help_by_page ) < self.npages: - self.help_by_page.append( self.help ) + self.__help = HELP_UNINITIALIZED + self.__help_by_page = HELP_UNINITIALIZED + self.__help_source = tool_source def parse_outputs( self, tool_source ): """ @@ -1891,6 +1859,67 @@ self.repository_owner = tool_shed_repository.owner self.installed_changeset_revision = tool_shed_repository.installed_changeset_revision + @property + def help(self): + if self.__help is HELP_UNINITIALIZED: + self.__ensure_help() + return self.__help + + @property + def help_by_page(self): + if self.__help_by_page is HELP_UNINITIALIZED: + self.__ensure_help() + return self.__help_by_page + + def __ensure_help(self): + with HELP_UNINITIALIZED: + if self.__help is HELP_UNINITIALIZED: + self.__inititalize_help() + + def __inititalize_help(self): + tool_source = self.__help_source + self.__help = None + self.__help_by_page = [] + help_header = "" + help_footer = "" + help_text = tool_source.parse_help() + if help_text is not None: + if self.repository_id and help_text.find( '.. image:: ' ) >= 0: + # Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy. + try: + help_text = suc.set_image_paths( self.app, self.repository_id, help_text ) + except Exception, e: + log.exception( "Exception in parse_help, so images may not be properly displayed:\n%s" % str( e ) ) + try: + self.__help = Template( rst_to_html(help_text), input_encoding='utf-8', + output_encoding='utf-8', default_filters=[ 'decode.utf8' ], + encoding_errors='replace' ) + except: + log.exception( "error in help for tool %s" % self.name ) + + # Handle deprecated multi-page help text in XML case. + if hasattr(tool_source, "root"): + help_elem = tool_source.root.find("help") + help_header = help_text + help_pages = help_elem.findall( "page" ) + # Multiple help page case + if help_pages: + for help_page in help_pages: + self.__help_by_page.append( help_page.text ) + help_footer = help_footer + help_page.tail + # Each page has to rendered all-together because of backreferences allowed by rst + try: + self.__help_by_page = [ Template( rst_to_html( help_header + x + help_footer ), + input_encoding='utf-8', output_encoding='utf-8', + default_filters=[ 'decode.utf8' ], + encoding_errors='replace' ) + for x in self.__help_by_page ] + except: + log.exception( "error in multi-page help for tool %s" % self.name ) + # Pad out help pages to match npages ... could this be done better? + while len( self.__help_by_page ) < self.npages: + self.__help_by_page.append( self.__help ) + def check_workflow_compatible( self, tool_source ): """ Determine if a tool can be used in workflows. External tools and the diff -r 3af604bcd66681f890ca86d3c7f0395e9d5e8d05 -r 3d7d15601e493d1d7ed6a10814f6ca2b36a611fb lib/galaxy/tools/parser/interface.py --- a/lib/galaxy/tools/parser/interface.py +++ b/lib/galaxy/tools/parser/interface.py @@ -140,6 +140,12 @@ """ return [], [] + @abstractmethod + def parse_help(self): + """ Return RST definition of help text for tool or None if the tool + doesn't define help text. + """ + def parse_tests_to_dict(self): return {'tests': []} diff -r 3af604bcd66681f890ca86d3c7f0395e9d5e8d05 -r 3d7d15601e493d1d7ed6a10814f6ca2b36a611fb lib/galaxy/tools/parser/xml.py --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -162,6 +162,10 @@ parser = StdioParser(self.root) return parser.stdio_exit_codes, parser.stdio_regexes + def parse_help(self): + help_elem = self.root.find( 'help' ) + return help_elem.text if help_elem is not None else None + def parse_tests_to_dict(self): tests_elem = self.root.find("tests") tests = [] diff -r 3af604bcd66681f890ca86d3c7f0395e9d5e8d05 -r 3d7d15601e493d1d7ed6a10814f6ca2b36a611fb lib/galaxy/tools/parser/yaml.py --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -68,6 +68,9 @@ exit_code_lower.error_level = StdioErrorLevel.FATAL return [exit_code_lower, exit_code_high], [] + def parse_help(self): + return self.root_dict.get("help", None) + def parse_outputs(self, tool): outputs = self.root_dict.get("outputs", {}) output_defs = [] diff -r 3af604bcd66681f890ca86d3c7f0395e9d5e8d05 -r 3d7d15601e493d1d7ed6a10814f6ca2b36a611fb test/unit/tools/test_parsing.py --- a/test/unit/tools/test_parsing.py +++ b/test/unit/tools/test_parsing.py @@ -26,6 +26,7 @@ <stdio><exit_code range="1:" level="fatal" /></stdio> + <help>This is HELP TEXT1!!!</help><tests><test><param name="foo" value="5" /> @@ -75,6 +76,9 @@ type: data - name: nestsample type: text +help: +| + This is HELP TEXT2!!! tests: - inputs: foo: 5 @@ -173,6 +177,10 @@ assert exit[0].range_start == 1 assert isinf(exit[0].range_end) + def test_help(self): + help_text = self._tool_source.parse_help() + assert help_text.strip() == "This is HELP TEXT1!!!" + def test_tests(self): tests_dict = self._tool_source.parse_tests_to_dict() tests = tests_dict["tests"] @@ -278,6 +286,10 @@ assert exit[1].range_start == 1 assert isinf(exit[1].range_end) + def test_help(self): + help_text = self._tool_source.parse_help() + assert help_text.strip() == "This is HELP TEXT2!!!" + def test_inputs(self): input_pages = self._tool_source.parse_input_pages() assert input_pages.inputs_defined https://bitbucket.org/galaxy/galaxy-central/commits/942a98d1548b/ Changeset: 942a98d1548b User: jmchilton Date: 2014-12-14 03:09:51+00:00 Summary: PEP-8 fixes for lib/galaxy/tools/search/__init__.py. Affected #: 1 file diff -r 3d7d15601e493d1d7ed6a10814f6ca2b36a611fb -r 942a98d1548b9e31960ecb208ae5cb15ae32a6fa lib/galaxy/tools/search/__init__.py --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -3,11 +3,11 @@ require( "Whoosh" ) from whoosh.filedb.filestore import RamStorage -from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT -from whoosh.index import Index +from whoosh.fields import Schema, STORED, TEXT from whoosh.scoring import BM25F from whoosh.qparser import MultifieldParser -schema = Schema( id = STORED, title = TEXT, description = TEXT, help = TEXT ) +schema = Schema( id=STORED, title=TEXT, description=TEXT, help=TEXT ) + class ToolBoxSearch( object ): """ @@ -33,10 +33,12 @@ def search( self, query, return_attribute='id' ): # Change field boosts for searcher to place more weight on title, description than help. - searcher = self.index.searcher( \ - weighting=BM25F( field_B={ 'title_B' : 3, 'description_B' : 2, 'help_B' : 1 } \ - ) ) + searcher = self.index.searcher( + weighting=BM25F( + field_B={ 'title_B': 3, 'description_B': 2, 'help_B': 1 } + ) + ) # Set query to search title, description, and help. - parser = MultifieldParser( [ 'title', 'description', 'help' ], schema = schema ) + parser = MultifieldParser( [ 'title', 'description', 'help' ], schema=schema ) results = searcher.search( parser.parse( query ) ) return [ result[ return_attribute ] for result in results ] https://bitbucket.org/galaxy/galaxy-central/commits/5c44ac15c5a9/ Changeset: 5c44ac15c5a9 User: jmchilton Date: 2014-12-14 03:09:51+00:00 Summary: Bugfix and enhancement related to indexing tool help in tool search. Added a config option for disabling indexing of tool config. This combined with the previous commit, shave over a second off (slightly more than 25%) Galaxy's startup time for the default toolbox on my laptop. Enable this option for functional tests - which will likely never care about seaching this index. When I went to test this - I noticed that searching wasn't actually working for help messages - this is because tool.help is not a string but a mako template - so I switched it over to render the page and stick it in the whoosh index instead of probably some default to string object pointer or something. Affected #: 2 files diff -r 942a98d1548b9e31960ecb208ae5cb15ae32a6fa -r 5c44ac15c5a96fad6995435f88fd230ef72a93c5 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -256,6 +256,9 @@ self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) self.watch_tools = kwargs.get( 'watch_tools', False ) + # On can mildly speed up Galaxy startup time by disabling index of help, + # not needed on production systems but useful if running many functional tests. + self.index_tool_help = string_as_bool( kwargs.get( "index_tool_help", True ) ) # Location for tool dependencies. if 'tool_dependency_dir' in kwargs: self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root ) @@ -705,7 +708,8 @@ self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) # Search support for tools import galaxy.tools.search - self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) + index_help = getattr( self.config, "index_tool_help", True ) + self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox, index_help ) from galaxy.tools.deps import containers galaxy_root_dir = os.path.abspath(self.config.root) diff -r 942a98d1548b9e31960ecb208ae5cb15ae32a6fa -r 5c44ac15c5a96fad6995435f88fd230ef72a93c5 lib/galaxy/tools/search/__init__.py --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -15,20 +15,33 @@ the "whoosh" search library. """ - def __init__( self, toolbox ): + def __init__( self, toolbox, index_help=True ): """ Create a searcher for `toolbox`. """ self.toolbox = toolbox - self.build_index() + self.build_index( index_help ) - def build_index( self ): + def build_index( self, index_help ): self.storage = RamStorage() self.index = self.storage.create_index( schema ) writer = self.index.writer() ## TODO: would also be nice to search section headers. for id, tool in self.toolbox.tools_by_id.iteritems(): - writer.add_document( id=id, title=to_unicode(tool.name), description=to_unicode(tool.description), help=to_unicode(tool.help) ) + add_doc_kwds = { + "id": id, + "title": to_unicode(tool.name), + "description": to_unicode(tool.description), + "help": to_unicode(""), + } + if index_help and tool.help: + try: + add_doc_kwds['help'] = to_unicode(tool.help.render( host_url="", static_path="" )) + except Exception: + # Don't fail to build index just because a help message + # won't render. + pass + writer.add_document( **add_doc_kwds ) writer.commit() def search( self, query, return_attribute='id' ): https://bitbucket.org/galaxy/galaxy-central/commits/6db06b5b370a/ Changeset: 6db06b5b370a User: jmchilton Date: 2014-12-14 03:09:51+00:00 Summary: Fix bug with watch tool config option parsing. Affected #: 1 file diff -r 5c44ac15c5a96fad6995435f88fd230ef72a93c5 -r 6db06b5b370ae0c955c66d6e2668aaf6260fe221 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -255,7 +255,7 @@ self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) - self.watch_tools = kwargs.get( 'watch_tools', False ) + self.watch_tools = string_as_bool( kwargs.get( 'watch_tools', False ) ) # On can mildly speed up Galaxy startup time by disabling index of help, # not needed on production systems but useful if running many functional tests. self.index_tool_help = string_as_bool( kwargs.get( "index_tool_help", True ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.