August 2013 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: dan: Explicitly set TEMP dir in Local Runner, when a temp dir value is not already set.
by commits-noreply＠bitbucket.org 02 Aug '13

02 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/57c593cc70a1/ Changeset: 57c593cc70a1 User: dan Date: 2013-08-02 19:28:22 Summary: Explicitly set TEMP dir in Local Runner, when a temp dir value is not already set. Affected #: 1 file diff -r 22fb05ea1b47953070570e5622ea4e37ea87ac45 -r 57c593cc70a1de89f32e77833cbf1e329cf52c6b lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -26,11 +26,18 @@ def __init__( self, app, nworkers ): """Start the job runner """ + #create a local copy of os.environ to use as env for subprocess.Popen + self._environ = os.environ.copy() + # put lib into the PYTHONPATH for subprocesses - if 'PYTHONPATH' in os.environ: - os.environ['PYTHONPATH'] = '%s:%s' % ( os.environ['PYTHONPATH'], os.path.abspath( 'lib' ) ) + if 'PYTHONPATH' in self._environ: + self._environ['PYTHONPATH'] = '%s:%s' % ( self._environ['PYTHONPATH'], os.path.abspath( 'lib' ) ) else: - os.environ['PYTHONPATH'] = os.path.abspath( 'lib' ) + self._environ['PYTHONPATH'] = os.path.abspath( 'lib' ) + + #Set TEMP if a valid temp value is not already set + if not ( 'TMPDIR' in self._environ or 'TEMP' in self._environ or 'TMP' in self._environ ): + self._environ[ 'TEMP' ] = tempfile.gettempdir() super( LocalJobRunner, self ).__init__( app, nworkers ) self._init_worker_threads() @@ -57,7 +64,7 @@ cwd = job_wrapper.working_directory, stdout = stdout_file, stderr = stderr_file, - env = os.environ, + env = self._environ, preexec_fn = os.setpgrp ) job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid) job_wrapper.change_state( model.Job.states.RUNNING ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Explicitly set the test_install_error in the automated testing framework when a repository installs and passes or fails functional tests.
by commits-noreply＠bitbucket.org 02 Aug '13

02 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/22fb05ea1b47/ Changeset: 22fb05ea1b47 User: Dave Bouvier Date: 2013-08-02 19:13:07 Summary: Explicitly set the test_install_error in the automated testing framework when a repository installs and passes or fails functional tests. Affected #: 1 file diff -r 59243394031ab93acb5f76b2ea1ed10a6b685533 -r 22fb05ea1b47953070570e5622ea4e37ea87ac45 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -1057,6 +1057,7 @@ repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) params[ 'tools_functionally_correct' ] = True params[ 'do_not_test' ] = False + params[ 'test_install_error' ] = False register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, @@ -1108,6 +1109,7 @@ repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) params[ 'tools_functionally_correct' ] = False + params[ 'test_install_error' ] = False params[ 'do_not_test' ] = str( set_do_not_test ) register_test_result( galaxy_tool_shed_url, metadata_revision_id, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Fix the install and test framework's handling of external metadata.
by commits-noreply＠bitbucket.org 02 Aug '13

02 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/59243394031a/ Changeset: 59243394031a User: Dave Bouvier Date: 2013-08-02 16:53:41 Summary: Fix the install and test framework's handling of external metadata. Affected #: 1 file diff -r 5b9ebcc05ea7f187c2a2f528f643bfda30756e5b -r 59243394031ab93acb5f76b2ea1ed10a6b685533 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -6,6 +6,7 @@ import os, sys, shutil, tempfile, re, string, urllib, platform from time import strftime +from ConfigParser import SafeConfigParser # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() @@ -73,6 +74,40 @@ default_galaxy_test_port_max = 10999 default_galaxy_test_host = '127.0.0.1' +# should this serve static resources (scripts, images, styles, etc.) +STATIC_ENABLED = True + +def get_static_settings(): + """Returns dictionary of the settings necessary for a galaxy App + to be wrapped in the static middleware. + + This mainly consists of the filesystem locations of url-mapped + static resources. + """ + cwd = os.getcwd() + static_dir = os.path.join( cwd, 'static' ) + #TODO: these should be copied from universe_wsgi.ini + return dict( + #TODO: static_enabled needed here? + static_enabled = True, + static_cache_time = 360, + static_dir = static_dir, + static_images_dir = os.path.join( static_dir, 'images', '' ), + static_favicon_dir = os.path.join( static_dir, 'favicon.ico' ), + static_scripts_dir = os.path.join( static_dir, 'scripts', '' ), + static_style_dir = os.path.join( static_dir, 'june_2007_style', 'blue' ), + static_robots_txt = os.path.join( static_dir, 'robots.txt' ), + ) + +def get_webapp_global_conf(): + """Get the global_conf dictionary sent as the first argument to app_factory. + """ + # (was originally sent 'dict()') - nothing here for now except static settings + global_conf = dict() + if STATIC_ENABLED: + global_conf.update( get_static_settings() ) + return global_conf + # Optionally, set the environment variable GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF # to the location of a tool sheds configuration file that includes the tool shed # that repositories will be installed from. @@ -219,6 +254,36 @@ success = result.wasSuccessful() return success +def generate_config_file( input_filename, output_filename, config_items ): + ''' + Generate a config file with the configuration that has been defined for the embedded web application. + This is mostly relevant when setting metadata externally, since the script for doing that does not + have access to app.config. + ''' + cp = SafeConfigParser() + cp.read( input_filename ) + config_items_by_section = [] + for label, value in config_items: + found = False + # Attempt to determine the correct section for this configuration option. + for section in cp.sections(): + if cp.has_option( section, label ): + config_tuple = section, label, value + config_items_by_section.append( config_tuple ) + found = True + continue + # Default to app:main if no section was found. + if not found: + config_tuple = 'app:main', label, value + config_items_by_section.append( config_tuple ) + # Replace the default values with the provided configuration. + for section, label, value in config_items_by_section: + cp.remove_option( section, label ) + cp.set( section, label, str( value ) ) + fh = open( output_filename, 'w' ) + cp.write( fh ) + fh.close() + def get_api_url( base, parts=[], params=None, key=None ): if 'api' in parts and parts.index( 'api' ) != 0: parts.pop( parts.index( 'api' ) ) @@ -554,48 +619,64 @@ # Generate the migrated_tool_conf.xml file. migrated_tool_conf_xml = tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_migrated_tool_path ) file( galaxy_migrated_tool_conf_file, 'w' ).write( migrated_tool_conf_xml ) - + # Write the embedded web application's specific configuration to a temporary file. This is necessary in order for + # the external metadata script to find the right datasets. + kwargs = dict( admin_users = 'test(a)bx.psu.edu', + allow_user_creation = True, + allow_user_deletion = True, + allow_library_path_paste = True, + database_connection = database_connection, + datatype_converters_config_file = "datatype_converters_conf.xml.sample", + file_path = galaxy_file_path, + id_secret = galaxy_encode_secret, + job_queue_workers = 5, + log_destination = "stdout", + migrated_tools_config = galaxy_migrated_tool_conf_file, + new_file_path = galaxy_tempfiles, + running_functional_tests = True, + shed_tool_data_table_config = shed_tool_data_table_conf_file, + shed_tool_path = galaxy_shed_tool_path, + template_path = "templates", + tool_config_file = ','.join( [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ] ), + tool_data_path = tool_data_path, + tool_data_table_config_path = galaxy_tool_data_table_conf_file, + tool_dependency_dir = tool_dependency_dir, + tool_path = tool_path, + tool_parse_help = False, + tool_sheds_config_file = galaxy_tool_sheds_conf_file, + update_integrated_tool_panel = False, + use_heartbeat = False ) + galaxy_config_file = os.environ.get( 'GALAXY_INSTALL_TEST_INI_FILE', None ) + # If the user has passed in a path for the .ini file, do not overwrite it. + if not galaxy_config_file: + galaxy_config_file = os.path.join( galaxy_test_tmp_dir, 'install_test_tool_shed_repositories_wsgi.ini' ) + config_items = [] + for label in kwargs: + config_tuple = label, kwargs[ label ] + config_items.append( config_tuple ) + # Write a temporary file, based on universe_wsgi.ini.sample, using the configuration options defined above. + generate_config_file( 'universe_wsgi.ini.sample', galaxy_config_file, config_items ) + kwargs[ 'tool_config_file' ] = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ] + # Set the global_conf[ '__file__' ] option to the location of the temporary .ini file, which gets passed to set_metadata.sh. + kwargs[ 'global_conf' ] = get_webapp_global_conf() + kwargs[ 'global_conf' ][ '__file__' ] = galaxy_config_file # ---- Build Galaxy Application -------------------------------------------------- - global_conf = { '__file__' : 'universe_wsgi.ini.sample' } if not database_connection.startswith( 'sqlite://' ): kwargs[ 'database_engine_option_max_overflow' ] = '20' kwargs[ 'database_engine_option_pool_size' ] = '10' - app = UniverseApplication( admin_users = 'test(a)bx.psu.edu', - allow_user_creation = True, - allow_user_deletion = True, - allow_library_path_paste = True, - database_connection = database_connection, - datatype_converters_config_file = "datatype_converters_conf.xml.sample", - file_path = galaxy_file_path, - global_conf = global_conf, - id_secret = galaxy_encode_secret, - job_queue_workers = 5, - log_destination = "stdout", - migrated_tools_config = galaxy_migrated_tool_conf_file, - new_file_path = galaxy_tempfiles, - running_functional_tests=True, - shed_tool_data_table_config = shed_tool_data_table_conf_file, - shed_tool_path = galaxy_shed_tool_path, - template_path = "templates", - tool_config_file = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ], - tool_data_path = tool_data_path, - tool_data_table_config_path = galaxy_tool_data_table_conf_file, - tool_dependency_dir = tool_dependency_dir, - tool_path = tool_path, - tool_parse_help = False, - tool_sheds_config_file = galaxy_tool_sheds_conf_file, - update_integrated_tool_panel = False, - use_heartbeat = False, - **kwargs ) + kwargs[ 'config_file' ] = galaxy_config_file + app = UniverseApplication( **kwargs ) log.info( "Embedded Galaxy application started" ) # ---- Run galaxy webserver ------------------------------------------------------ server = None - webapp = buildapp.app_factory( dict( database_file=database_connection ), - use_translogger=False, - static_enabled=False, - app=app ) + global_conf = get_webapp_global_conf() + global_conf[ 'database_file' ] = database_connection + webapp = buildapp.app_factory( global_conf, + use_translogger=False, + static_enabled=STATIC_ENABLED, + app=app ) # Serve the app on a specified or random port. if galaxy_test_port is not None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 01 Aug '13

01 Aug '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/96eba6391a06/ Changeset: 96eba6391a06 Branch: next-stable User: jgoecks Date: 2013-08-02 01:38:24 Summary: Fix imports and parentheses spacing. Affected #: 1 file diff -r 208f93f6cdc00c6d866b8029b0f53990ce0bfadc -r 96eba6391a06e7e6230e06208347dfb51a64a1e1 lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -13,6 +13,7 @@ from galaxy.util.json import from_json_string from bx.interval_index_file import Indexes from bx.bbi.bigwig_file import BigWigFile +from bx.bbi.bigbed_file import BigBedFile from galaxy.util.lrucache import LRUCache from galaxy.visualization.data_providers.basic import BaseDataProvider from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and_cigar @@ -861,14 +862,14 @@ """ Returns an iterator that provides data in the region chrom:start-end """ - start, end = int(start), int(end) + start, end = int( start ), int( end ) orig_data_filename = self.original_dataset.file_name index_filename = self.converted_dataset.file_name # Attempt to open the BAM file with index bamfile = csamtools.Samfile( filename=orig_data_filename, mode='rb', index_filename=index_filename ) try: - data = bamfile.fetch(start=start, end=end, reference=chrom) + data = bamfile.fetch( start=start, end=end, reference=chrom ) except ValueError, e: # Try alternative chrom naming. chrom = _convert_between_ucsc_and_ensemble_naming( chrom ) https://bitbucket.org/galaxy/galaxy-central/commits/5b9ebcc05ea7/ Changeset: 5b9ebcc05ea7 User: jgoecks Date: 2013-08-02 01:38:49 Summary: Automated merge of next-stable branch Affected #: 1 file diff -r 318f84a819ef9d71dbfffa700d93665b596ef440 -r 5b9ebcc05ea7f187c2a2f528f643bfda30756e5b lib/galaxy/visualization/data_providers/genome.py --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -13,6 +13,7 @@ from galaxy.util.json import from_json_string from bx.interval_index_file import Indexes from bx.bbi.bigwig_file import BigWigFile +from bx.bbi.bigbed_file import BigBedFile from galaxy.util.lrucache import LRUCache from galaxy.visualization.data_providers.basic import BaseDataProvider from galaxy.visualization.data_providers.cigar import get_ref_based_read_seq_and_cigar @@ -861,14 +862,14 @@ """ Returns an iterator that provides data in the region chrom:start-end """ - start, end = int(start), int(end) + start, end = int( start ), int( end ) orig_data_filename = self.original_dataset.file_name index_filename = self.converted_dataset.file_name # Attempt to open the BAM file with index bamfile = csamtools.Samfile( filename=orig_data_filename, mode='rb', index_filename=index_filename ) try: - data = bamfile.fetch(start=start, end=end, reference=chrom) + data = bamfile.fetch( start=start, end=end, reference=chrom ) except ValueError, e: # Try alternative chrom naming. chrom = _convert_between_ucsc_and_ensemble_naming( chrom ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: API Documentation: small fixes for histories, history_contents
by commits-noreply＠bitbucket.org 01 Aug '13

01 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/318f84a819ef/ Changeset: 318f84a819ef User: carlfeberhard Date: 2013-08-01 21:35:45 Summary: API Documentation: small fixes for histories, history_contents Affected #: 2 files diff -r 8be96f5579e7f228fac8e7b70ac4bbec261b445d -r 318f84a819ef9d71dbfffa700d93665b596ef440 lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -28,9 +28,7 @@ return undeleted histories for the current user * GET /api/histories/deleted: return deleted histories for the current user - - If the user is logged-in, a full list of all histories is shown. - If not logged in, only the curernt history (if any) is shown. + .. note:: Anonymous users are allowed to get their current history :type deleted: boolean :param deleted: if True, show only deleted histories, if False, non-deleted @@ -76,6 +74,7 @@ return the deleted history with ``id`` * GET /api/histories/most_recently_used: return the most recently used history + .. note:: Anonymous users are allowed to get their current history :type id: an encoded id string :param id: the encoded id of the history to query or the string 'most_recently_used' @@ -127,8 +126,8 @@ create a new history :type payload: dict - :param payload: (optional) dictionary structure containing:: - 'name': the new history's name + :param payload: (optional) dictionary structure containing: + * name: the new history's name :rtype: dict :returns: element view of new history @@ -159,16 +158,16 @@ :type id: str :param id: the encoded id of the history to delete :type kwd: dict - :param kwd: (optional) dictionary structure containing:: - 'payload': a dictionary itself containing:: - 'purge': if True, purge the history and all of it's HDAs + :param kwd: (optional) dictionary structure containing: + + * payload: a dictionary itself containing: + * purge: if True, purge the history and all of it's HDAs :rtype: dict - :returns: an error object if an error occurred or a dictionary containing:: - - id: the encoded id of the history, - deleted: if the history was marked as deleted, - purged: if the history was purged + :returns: an error object if an error occurred or a dictionary containing: + * id: the encoded id of the history, + * deleted: if the history was marked as deleted, + * purged: if the history was purged """ history_id = id # a request body is optional here @@ -255,9 +254,9 @@ :param id: the encoded id of the history to undelete :type payload: dict :param payload: a dictionary containing any or all the - fields in :func:`galaxy.model.History.get_api_value` and/or the following:: + fields in :func:`galaxy.model.History.get_api_value` and/or the following: - 'annotation': an annotation for the history + * annotation: an annotation for the history :rtype: dict :returns: an error object if an error occurred or a dictionary containing @@ -330,4 +329,3 @@ pass #log.warn( 'unknown key: %s', str( key ) ) return validated_payload - diff -r 8be96f5579e7f228fac8e7b70ac4bbec261b445d -r 318f84a819ef9d71dbfffa700d93665b596ef440 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -18,28 +18,24 @@ index( self, trans, history_id, ids=None, **kwd ) * GET /api/histories/{history_id}/contents return a list of HDA data for the history with the given ``id`` + .. note:: Anonymous users are allowed to get their current history contents + + If Ids is not given, index returns a list of *summary* objects for + every HDA associated with the given `history_id`. + + If ids is given, index returns a *more complete* json object for each + HDA in the ids list. :type history_id: str :param history_id: encoded id string of the HDA's History :type ids: str :param ids: (optional) a comma separated list of encoded `HDA` ids - If Ids is not given, index returns a list of *summary* objects for - every HDA associated with the given `history_id`. - - .. seealso:: - :func:`_summary_hda_dict` - - If ids is given, index returns a *more complete* json object for each - HDA in the ids list. - - .. seealso:: - :func:`galaxy.web.base.controller.UsesHistoryDatasetAssociationMixin.get_hda_dict` - - .. note:: Anonymous users are allowed to get their current history contents - :rtype: list :returns: dictionaries containing summary or detailed HDA information + .. seealso:: + :func:`_summary_hda_dict` and + :func:`galaxy.web.base.controller.UsesHistoryDatasetAssociationMixin.get_hda_dict` """ rval = [] try: @@ -111,18 +107,16 @@ show( self, trans, id, history_id, **kwd ) * GET /api/histories/{history_id}/contents/{id} return detailed information about an HDA within a history + .. note:: Anonymous users are allowed to get their current history contents :type id: str :param ids: the encoded id of the HDA to return :type history_id: str :param history_id: encoded id string of the HDA's History - .. seealso:: :func:`galaxy.web.base.controller.UsesHistoryDatasetAssociationMixin.get_hda_dict` - - .. note:: Anonymous users are allowed to get their current history contents - :rtype: dict :returns: dictionary containing detailed HDA information + .. seealso:: :func:`galaxy.web.base.controller.UsesHistoryDatasetAssociationMixin.get_hda_dict` """ hda_dict = {} try: @@ -217,9 +211,9 @@ :type payload: dict :param payload: a dictionary containing any or all the fields in :func:`galaxy.model.HistoryDatasetAssociation.get_api_value` - and/or the following:: + and/or the following: - 'annotation': an annotation for the history + * annotation: an annotation for the HDA :rtype: dict :returns: an error object if an error occurred or a dictionary containing Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: API Documentation: libraries, library_contents
by commits-noreply＠bitbucket.org 01 Aug '13

01 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8be96f5579e7/ Changeset: 8be96f5579e7 User: carlfeberhard Date: 2013-08-01 21:21:46 Summary: API Documentation: libraries, library_contents Affected #: 2 files diff -r d34d6d1da813bfa6e318b0ceb563186e52d94095 -r 8be96f5579e7f228fac8e7b70ac4bbec261b445d lib/galaxy/webapps/galaxy/api/libraries.py --- a/lib/galaxy/webapps/galaxy/api/libraries.py +++ b/lib/galaxy/webapps/galaxy/api/libraries.py @@ -15,9 +15,18 @@ @web.expose_api def index( self, trans, deleted='False', **kwd ): """ - GET /api/libraries - GET /api/libraries/deleted - Displays a collection (list) of libraries. + index( self, trans, deleted='False', **kwd ) + * GET /api/libraries: + returns a list of summary data for libraries + * GET /api/libraries/deleted: + returns a list of summary data for deleted libraries + + :type deleted: boolean + :param deleted: if True, show only deleted libraries, if False, non-deleted + + :rtype: list + :returns: list of dictionaries containing library information + .. seealso:: :attr:`galaxy.model.Library.api_collection_visible_keys` """ log.debug( "LibrariesController.index: enter" ) query = trans.sa_session.query( trans.app.model.Library ) @@ -49,9 +58,20 @@ @web.expose_api def show( self, trans, id, deleted='False', **kwd ): """ - GET /api/libraries/{encoded_library_id} - GET /api/libraries/deleted/{encoded_library_id} - Displays information about a library. + show( self, trans, id, deleted='False', **kwd ) + * GET /api/libraries/{id}: + returns detailed information about a library + * GET /api/libraries/deleted/{id}: + returns detailed information about a deleted library + + :type id: an encoded id string + :param id: the encoded id of the library + :type deleted: boolean + :param deleted: if True, allow information on a deleted library + + :rtype: dictionary + :returns: detailed library information + .. seealso:: :attr:`galaxy.model.Library.api_element_visible_keys` """ log.debug( "LibraryContentsController.show: enter" ) library_id = id @@ -75,8 +95,20 @@ @web.expose_api def create( self, trans, payload, **kwd ): """ - POST /api/libraries - Creates a new library. + create( self, trans, payload, **kwd ) + * POST /api/libraries: + create a new library + .. note:: Currently, only admin users can create libraries. + + :type payload: dict + :param payload: (optional) dictionary structure containing:: + 'name': the new library's name + 'description': the new library's description + 'synopsis': the new library's synopsis + + :rtype: dict + :returns: a dictionary containing the id, name, and 'show' url + of the new library """ if not trans.user_is_admin(): raise HTTPForbidden( detail='You are not authorized to create a new library.' ) @@ -102,6 +134,19 @@ @web.expose_api def delete( self, trans, id, **kwd ): + """ + delete( self, trans, id, **kwd ) + * DELETE /api/histories/{id} + mark the library with the given ``id`` as deleted + .. note:: Currently, only admin users can delete libraries. + + :type id: str + :param id: the encoded id of the library to delete + + :rtype: dictionary + :returns: detailed library information + .. seealso:: :attr:`galaxy.model.Library.api_element_visible_keys` + """ if not trans.user_is_admin(): raise HTTPForbidden( detail='You are not authorized to delete libraries.' ) try: diff -r d34d6d1da813bfa6e318b0ceb563186e52d94095 -r 8be96f5579e7f228fac8e7b70ac4bbec261b445d lib/galaxy/webapps/galaxy/api/library_contents.py --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -19,8 +19,21 @@ # TODO: Add parameter to only get top level of datasets/subfolders. def index( self, trans, library_id, **kwd ): """ - GET /api/libraries/{encoded_library_id}/contents - Displays a collection (list) of library contents (files and folders). + index( self, trans, library_id, **kwd ) + * GET /api/libraries/{library_id}/contents: + return a list of library files and folders + + :type library_id: str + :param library_id: encoded id string of the library that contains this item + + :rtype: list + :returns: list of dictionaries of the form: + + * id: the encoded id of the library item + * name: the 'libary path' + or relationship of the library item to the root + * type: 'file' or 'folder' + * url: the url to get detailed information on the library item """ rval = [] current_user_roles = trans.get_current_user_roles() @@ -80,8 +93,20 @@ @web.expose_api def show( self, trans, id, library_id, **kwd ): """ - GET /api/libraries/{encoded_library_id}/contents/{encoded_content_id} - Displays information about a library content (file or folder). + show( self, trans, id, library_id, **kwd ) + * GET /api/libraries/{library_id}/contents/{id} + return information about library file or folder + + :type id: str + :param id: the encoded id of the library item to return + :type library_id: str + :param library_id: encoded id string of the library that contains this item + + :rtype: dict + :returns: detailed library item information + .. seealso:: + :func:`galaxy.model.LibraryDataset.get_api_value` and + :attr:`galaxy.model.LibraryFolder.api_element_visible_keys` """ class_name, content_id = self.__decode_library_content_id( trans, id ) if class_name == 'LibraryFolder': @@ -93,8 +118,29 @@ @web.expose_api def create( self, trans, library_id, payload, **kwd ): """ - POST /api/libraries/{encoded_library_id}/contents - Creates a new library content item (file or folder). + create( self, trans, library_id, payload, **kwd ) + * POST /api/libraries/{library_id}/contents: + create a new library file or folder + + To copy an HDA into a library send ``create_type`` of 'file' and + the HDA's encoded id in ``from_hda_id`` (and optionally ``ldda_message``). + + :type library_id: str + :param library_id: encoded id string of the library that contains this item + :type payload: dict + :param payload: dictionary structure containing: + + * folder_id: the parent folder of the new item + * create_type: the type of item to create ('file' or 'folder') + * from_hda_id: (optional) the id of an accessible HDA to copy into the + library + * ldda_message: (optional) the new message attribute of the LDDA created + * extended_metadata: (optional) dub-dictionary containing any extended + metadata to associate with the item + + :rtype: dict + :returns: a dictionary containing the id, name, + and 'show' url of the new item """ create_type = None if 'create_type' not in payload: @@ -195,10 +241,10 @@ def _copy_hda_to_library_folder( self, trans, from_hda_id, library_id, folder_id, ldda_message='' ): """ - Copies hda `from_hda_id` to library folder `library_folder_id` optionally - adding `ldda_message` to the new ldda's `message`. + Copies hda ``from_hda_id`` to library folder ``library_folder_id`` optionally + adding ``ldda_message`` to the new ldda's ``message``. - `library_contents.create` will branch to this if called with 'from_hda_id' + ``library_contents.create`` will branch to this if called with 'from_hda_id' in it's payload. """ log.debug( '_copy_hda_to_library_folder: %s' %( str(( from_hda_id, library_id, folder_id, ldda_message )) ) ) @@ -236,10 +282,23 @@ return rval @web.expose_api - def update( self, trans, id, library_id, payload, **kwd ): + def update( self, trans, id, library_id, payload, **kwd ): """ - PUT /api/libraries/{encoded_library_id}/contents/{encoded_content_type_and_id} - Sets relationships among items + update( self, trans, id, library_id, payload, **kwd ) + * PUT /api/libraries/{library_id}/contents/{id} + create a ImplicitlyConvertedDatasetAssociation + .. seealso:: :class:`galaxy.model.ImplicitlyConvertedDatasetAssociation` + + :type id: str + :param id: the encoded id of the library item to return + :type library_id: str + :param library_id: encoded id string of the library that contains this item + :type payload: dict + :param payload: dictionary structure containing:: + 'converted_dataset_id': + + :rtype: None + :returns: None """ if 'converted_dataset_id' in payload: converted_id = payload.pop( 'converted_dataset_id' ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Add a sanity check to the twobit sniffer.
by commits-noreply＠bitbucket.org 01 Aug '13

01 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d34d6d1da813/ Changeset: d34d6d1da813 User: Dave Bouvier Date: 2013-08-01 21:03:37 Summary: Add a sanity check to the twobit sniffer. Affected #: 1 file diff -r 143f7f55098ae936481e5fcb1be85a3e5b5f159b -r d34d6d1da813bfa6e318b0ceb563186e52d94095 lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -475,6 +475,9 @@ def sniff(self, filename): try: + # All twobit files start with a 16-byte header. If the file is smaller than 16 bytes, it's obviously not a valid twobit file. + if os.path.getsize(filename) < 16: + return False input = file(filename) magic = struct.unpack(">L", input.read(TWOBIT_MAGIC_SIZE))[0] if magic == TWOBIT_MAGIC_NUMBER or magic == TWOBIT_MAGIC_NUMBER_SWAP: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jgoecks: Cuffdiff wrapper: add option to output read group datasets.
by commits-noreply＠bitbucket.org 01 Aug '13

01 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/143f7f55098a/ Changeset: 143f7f55098a User: jgoecks Date: 2013-08-01 20:51:06 Summary: Cuffdiff wrapper: add option to output read group datasets. Affected #: 1 file diff -r 6e36021502522213aca87e0b49f0bf0384eddfa1 -r 143f7f55098ae936481e5fcb1be85a3e5b5f159b tools/ngs_rna/cuffdiff_wrapper.xml --- a/tools/ngs_rna/cuffdiff_wrapper.xml +++ b/tools/ngs_rna/cuffdiff_wrapper.xml @@ -96,8 +96,13 @@ <when value="No"></when></conditional> + <param name="include_read_group_files" type="select" label="Include Read Group Datasets" help="Read group datasets provide information on replicates."> + <option value="No" selected="true">No</option> + <option value="Yes">Yes</option> + </param> + <conditional name="additional"> - <param name="sAdditional" type="select" label="Set Additional Parameters? (not recommended)"> + <param name="sAdditional" type="select" label="Set Additional Parameters? (not recommended for paired-end reads)"><option value="No">No</option><option value="Yes">Yes</option></param> @@ -114,6 +119,21 @@ </stdio><outputs> +  + <data format="tabular" name="isoforms_read_group" label="${tool.name} on ${on_string}: isoforms read group tracking" from_work_dir="isoforms.read_group_tracking" > + <filter>(params['include_read_group_files'] == 'Yes'</filter> + </data> + <data format="tabular" name="genes_read_group" label="${tool.name} on ${on_string}: genes read group tracking" from_work_dir="genes.read_group_tracking" > + <filter>(params['include_read_group_files'] == 'Yes'</filter> + </data> + <data format="tabular" name="cds_read_group" label="${tool.name} on ${on_string}: CDs read group tracking" from_work_dir="cds.read_group_tracking" > + <filter>(params['include_read_group_files'] == 'Yes'</filter> + </data> + <data format="tabular" name="tss_groups_read_group" label="${tool.name} on ${on_string}: TSS groups read group tracking" from_work_dir="tss_groups.read_group_tracking" > + <filter>(params['include_read_group_files'] == 'Yes'</filter> + </data> + + <data format="tabular" name="splicing_diff" label="${tool.name} on ${on_string}: splicing differential expression testing" from_work_dir="splicing.diff" /><data format="tabular" name="promoters_diff" label="${tool.name} on ${on_string}: promoters differential expression testing" from_work_dir="promoters.diff" /><data format="tabular" name="cds_diff" label="${tool.name} on ${on_string}: CDS overloading diffential expression testing" from_work_dir="cds.diff" /> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: Dave Bouvier: Implement baseline support for defining a tool dependency that downloads a precompiled binary for the target platform.
by commits-noreply＠bitbucket.org 01 Aug '13

01 Aug '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6e3602150252/ Changeset: 6e3602150252 User: Dave Bouvier Date: 2013-08-01 17:05:34 Summary: Implement baseline support for defining a tool dependency that downloads a precompiled binary for the target platform. Affected #: 4 files diff -r fe5867b89ec01bfde18591df8809526d85fb821e -r 6e36021502522213aca87e0b49f0bf0384eddfa1 lib/tool_shed/galaxy_install/tool_dependencies/common_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -8,6 +8,7 @@ import zipfile import tool_shed.util.shed_util_common as suc from galaxy.datatypes import checkers +from urllib2 import HTTPError log = logging.getLogger( __name__ ) @@ -70,6 +71,23 @@ __shellquote(env_shell_file_path)) return cmd +def download_binary_from_url( url, work_dir, install_dir ): + ''' + Download a pre-compiled binary from the specified URL. If the downloaded file is an archive, + extract it into install_dir and delete the archive. + ''' + downloaded_filename = os.path.split( url )[ -1 ] + try: + dir = url_download( work_dir, downloaded_filename, url, extract=True ) + downloaded_filepath = os.path.join( work_dir, downloaded_filename ) + if is_compressed( downloaded_filepath ): + os.remove( downloaded_filepath ) + move_directory_files( current_dir=work_dir, + source_dir=dir, + destination_dir=install_dir ) + return True + except HTTPError: + return False def extract_tar( file_name, file_path ): if isgzip( file_name ) or isbz2( file_name ): @@ -190,6 +208,12 @@ def iszip( file_path ): return checkers.check_zip( file_path ) +def is_compressed( file_path ): + if isjar( file_path ): + return False + else: + return iszip( file_path ) or isgzip( file_path ) or istar( file_path ) or isbz2( file_path ) + def make_directory( full_path ): if not os.path.exists( full_path ): os.makedirs( full_path ) diff -r fe5867b89ec01bfde18591df8809526d85fb821e -r 6e36021502522213aca87e0b49f0bf0384eddfa1 lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -165,12 +165,29 @@ actions = actions_dict.get( 'actions', None ) filtered_actions = [] env_shell_file_paths = [] + # Default to false so that the install process will default to compiling. + binary_found = False if actions: with make_tmp_dir() as work_dir: with lcd( work_dir ): # The first action in the list of actions will be the one that defines the installation process. There # are currently only two supported processes; download_by_url and clone via a "shell_command" action type. action_type, action_dict = actions[ 0 ] + if action_type == 'download_binary': + # Eliminate the download_binary action so remaining actions can be processed correctly. + filtered_actions = actions[ 1: ] + url = action_dict[ 'url' ] + # Attempt to download a binary from the specified URL. + log.debug( 'Attempting to download from %s', url ) + binary_found = common_util.download_binary_from_url( url, work_dir, install_dir ) + if binary_found: + # If the attempt succeeded, set the action_type to binary_found, in order to skip any download_by_url or shell_command actions. + actions = filtered_actions + action_type = 'binary_found' + else: + # No binary exists, or there was an error downloading the binary from the generated URL. Proceed with the remaining actions. + del actions[ 0 ] + action_type, action_dict = actions[ 0 ] if action_type == 'download_by_url': # Eliminate the download_by_url action so remaining actions can be processed correctly. filtered_actions = actions[ 1: ] @@ -220,6 +237,9 @@ current_dir = os.path.abspath( os.path.join( work_dir, dir ) ) with lcd( current_dir ): action_type, action_dict = action_tup + # If a binary was found, we only need to process environment variables, file permissions, and any other binary downloads. + if binary_found and action_type not in [ 'set_environment', 'chmod', 'download_binary' ]: + continue if action_type == 'make_directory': common_util.make_directory( full_path=action_dict[ 'full_path' ] ) elif action_type == 'move_directory_files': @@ -348,6 +368,18 @@ dir = target_directory.replace( os.path.realpath( work_dir ), '' ).lstrip( '/' ) else: log.error( 'Invalid or nonexistent directory %s specified, ignoring change_directory action.', target_directory ) + elif action_type == 'chmod': + for target_file, mode in action_dict[ 'change_modes' ]: + if os.path.exists( target_file ): + os.chmod( target_file, mode ) + elif action_type == 'download_binary': + url = action_dict[ 'url' ] + binary_found = common_util.download_binary_from_url( url, work_dir, install_dir ) + if binary_found: + log.debug( 'Successfully downloaded binary from %s', url ) + else: + log.error( 'Unable to download binary from %s', url ) + def log_results( command, fabric_AttributeString, file_path ): """ diff -r fe5867b89ec01bfde18591df8809526d85fb821e -r 6e36021502522213aca87e0b49f0bf0384eddfa1 lib/tool_shed/galaxy_install/tool_dependencies/install_util.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -1,6 +1,7 @@ import logging import os import sys +import stat import subprocess import tempfile from string import Template @@ -379,7 +380,22 @@ for action_elem in actions_elem.findall( 'action' ): action_dict = {} action_type = action_elem.get( 'type', 'shell_command' ) - if action_type == 'shell_command': + if action_type == 'download_binary': + platform_info_dict = tool_dependency_util.get_platform_info_dict() + platform_info_dict[ 'name' ] = tool_dependency.name + platform_info_dict[ 'version' ] = tool_dependency.version + url_template_elems = action_elem.findall( 'url_template' ) + # Check if there are multiple url_template elements, each with attrib entries for a specific platform. + if len( url_template_elems ) > 1: + # <base_url os="darwin" extract="false">http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit</base_url> + # This method returns the url_elem that best matches the current platform as received from os.uname(). + # Currently checked attributes are os and architecture. + # These correspond to the values sysname and processor from the Python documentation for os.uname(). + url_template_elem = tool_dependency_util.get_download_url_for_platform( url_template_elems, platform_info_dict ) + else: + url_template_elem = url_template_elems[ 0 ] + action_dict[ 'url' ] = Template( url_template_elem.text ).safe_substitute( platform_info_dict ) + elif action_type == 'shell_command': # <action type="shell_command">make</action> action_elem_text = evaluate_template( action_elem.text ) if action_elem_text: @@ -492,6 +508,27 @@ # lxml==2.3.0</action> ## Manually specify contents of requirements.txt file to create dynamically. action_dict[ 'requirements' ] = evaluate_template( action_elem.text or 'requirements.txt' ) + elif action_type == 'chmod': + # Change the read, write, and execute bits on a file. + file_elems = action_elem.findall( 'file' ) + chmod_actions = [] + # A unix octal mode is the sum of the following values: + # Owner: + # 400 Read 200 Write 100 Execute + # Group: + # 040 Read 020 Write 010 Execute + # World: + # 004 Read 002 Write 001 Execute + for file_elem in file_elems: + # So by the above table, owner read/write/execute and group read permission would be 740. + # Python's os.chmod uses base 10 modes, convert received unix-style octal modes to base 10. + received_mode = int( file_elem.get( 'mode', 600 ), base=8 ) + # For added security, ensure that the setuid and setgid bits are not set. + mode = received_mode & ~( stat.S_ISUID | stat.S_ISGID ) + file = evaluate_template( file_elem.text ) + chmod_tuple = ( file, mode ) + chmod_actions.append( chmod_tuple ) + action_dict[ 'change_modes' ] = chmod_actions else: log.debug( "Unsupported action type '%s'. Not proceeding." % str( action_type ) ) raise Exception( "Unsupported action type '%s' in tool dependency definition." % str( action_type ) ) diff -r fe5867b89ec01bfde18591df8809526d85fb821e -r 6e36021502522213aca87e0b49f0bf0384eddfa1 lib/tool_shed/util/tool_dependency_util.py --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -39,6 +39,37 @@ tool_dependencies[ dependency_key ] = requirements_dict return tool_dependencies +def get_download_url_for_platform( url_templates, platform_info_dict ): + ''' + Compare the dict returned by get_platform_info() with the values specified in the base_url element. Return + true if and only if all defined attributes match the corresponding dict entries. If an entry is not + defined in the base_url element, it is assumed to be irrelevant at this stage. For example, + <base_url os="darwin">http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit</base_url> + where the OS must be 'darwin', but the architecture is filled in later using string.Template. + ''' + os_ok = False + architecture_ok = False + for url_template in url_templates: + os_name = url_template.get( 'os', None ) + architecture = url_template.get( 'architecture', None ) + if os_name: + if os_name.lower() == platform_info_dict[ 'os' ]: + os_ok = True + else: + os_ok = False + else: + os_ok = True + if architecture: + if architecture.lower() == platform_info_dict[ 'architecture' ]: + architecture_ok = True + else: + architecture_ok = False + else: + architecture_ok = True + if os_ok and architecture_ok: + return url_template + return None + def create_or_update_tool_dependency( app, tool_shed_repository, name, version, type, status, set_status=True ): # Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled repository is being reinstalled. sa_session = app.model.context.current @@ -204,6 +235,14 @@ missing_tool_dependencies = None return tool_dependencies, missing_tool_dependencies +def get_platform_info_dict(): + '''Return a dict with information about the current platform.''' + platform_dict = {} + sysname, nodename, release, version, machine = os.uname() + platform_dict[ 'os' ] = sysname.lower() + platform_dict[ 'architecture' ] = machine.lower() + return platform_dict + def get_tool_dependency( trans, id ): """Get a tool_dependency from the database via id""" return trans.sa_session.query( trans.model.ToolDependency ).get( trans.security.decode_id( id ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0