[galaxy-commits] commit/galaxy-central: 12 new changesets

12 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/cfd4ff7e3972/ Changeset: cfd4ff7e3972 Branch: release_15.03 User: guerler Date: 2015-04-23 20:05:00+00:00 Summary: Fix base url tool parameter Affected #: 1 file diff -r d7a9896af111d6f86735eb4f2295eee67f7e3527 -r cfd4ff7e397264336ccafd0c6086c91bd2d3a533 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -689,7 +689,7 @@ return form_builder.HiddenField( self.name, self.get_value( trans ) ) def get_initial_value( self, trans, context, history=None ): - return self.value + return self.get_value( trans ) def get_label( self ): # BaseURLToolParameters are ultimately "hidden" parameters https://bitbucket.org/galaxy/galaxy-central/commits/a5400c7f61d8/ Changeset: a5400c7f61d8 Branch: release_15.03 User: natefoo Date: 2015-04-27 19:56:30+00:00 Summary: Make the `enable_tool_shed_check` option perform as advertised in the sample config file. Also disables checks at startup. Affected #: 2 files diff -r cfd4ff7e397264336ccafd0c6086c91bd2d3a533 -r a5400c7f61d8ccae54e8f72502b4117c9ce82076 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -121,6 +121,7 @@ self.manage_dependency_relationships = string_as_bool( kwargs.get( 'manage_dependency_relationships', False ) ) self.running_functional_tests = string_as_bool( kwargs.get( 'running_functional_tests', False ) ) self.hours_between_check = kwargs.get( 'hours_between_check', 12 ) + self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) ) if isinstance( self.hours_between_check, basestring ): self.hours_between_check = float( self.hours_between_check ) try: diff -r cfd4ff7e397264336ccafd0c6086c91bd2d3a533 -r a5400c7f61d8ccae54e8f72502b4117c9ce82076 lib/tool_shed/galaxy_install/update_repository_manager.py --- a/lib/tool_shed/galaxy_install/update_repository_manager.py +++ b/lib/tool_shed/galaxy_install/update_repository_manager.py @@ -16,12 +16,13 @@ self.app = app self.context = self.app.install_model.context # Ideally only one Galaxy server process should be able to check for repository updates. - self.running = True - self.sleeper = Sleeper() - self.restarter = threading.Thread( target=self.__restarter ) - self.restarter.daemon = True - self.restarter.start() - self.seconds_to_sleep = int( app.config.hours_between_check * 3600 ) + if self.app.config.enable_tool_shed_check: + self.running = True + self.sleeper = Sleeper() + self.restarter = threading.Thread( target=self.__restarter ) + self.restarter.daemon = True + self.restarter.start() + self.seconds_to_sleep = int( app.config.hours_between_check * 3600 ) def get_update_to_changeset_revision_and_ctx_rev( self, repository ): """Return the changeset revision hash to which the repository can be updated.""" @@ -94,8 +95,9 @@ log.info( 'Update repository manager restarter shutting down...' ) def shutdown( self ): - self.running = False - self.sleeper.wake() + if self.app.config.enable_tool_shed_check: + self.running = False + self.sleeper.wake() def update_repository_record( self, repository, updated_metadata_dict, updated_changeset_revision, updated_ctx_rev ): """ https://bitbucket.org/galaxy/galaxy-central/commits/41d7ef6d0d64/ Changeset: 41d7ef6d0d64 Branch: release_15.03 User: natefoo Date: 2015-04-28 16:43:18+00:00 Summary: Fix job stopping under some conditions: 1. track_jobs_in_database is False 2. Jobs being stopped were in the local runner 3. Jobs were deleted with the deleted state and not deleted_new Affected #: 2 files diff -r a5400c7f61d8ccae54e8f72502b4117c9ce82076 -r 41d7ef6d0d6488feb987c573a363b0468b16d6b3 lib/galaxy/jobs/handler.py --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -680,7 +680,10 @@ except Empty: pass for job, error_msg in jobs_to_check: - if job.state != job.states.DELETED_NEW and job.finished: + if ( job.state not in ( job.states.DELETED_NEW, + job.states.DELETED ) + and job.finished ): + # terminated before it got here log.debug('Job %s already finished, not deleting or stopping', job.id) continue final_state = job.states.DELETED @@ -695,7 +698,8 @@ self.dispatcher.stop( job ) def put( self, job_id, error_msg=None ): - self.queue.put( ( job_id, error_msg ) ) + if not self.app.config.track_jobs_in_database: + self.queue.put( ( job_id, error_msg ) ) def shutdown( self ): """Attempts to gracefully shut down the worker thread""" diff -r a5400c7f61d8ccae54e8f72502b4117c9ce82076 -r 41d7ef6d0d6488feb987c573a363b0468b16d6b3 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -134,9 +134,11 @@ def stop_job( self, job ): #if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished job_ext_output_metadata = job.get_external_output_metadata() - if job_ext_output_metadata: + try: pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them - else: + assert pid not in [ None, '' ] + except: + # metadata internal or job not complete yet pid = job.get_job_runner_external_id() if pid in [ None, '' ]: log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.get_id() ) https://bitbucket.org/galaxy/galaxy-central/commits/aceedb22b312/ Changeset: aceedb22b312 Branch: release_15.03 User: natefoo Date: 2015-04-28 20:26:29+00:00 Summary: Don't use a bare except Affected #: 1 file diff -r 41d7ef6d0d6488feb987c573a363b0468b16d6b3 -r aceedb22b31220f9a388935d8ac2cdd5b0dee406 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -137,7 +137,7 @@ try: pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them assert pid not in [ None, '' ] - except: + except Exception: # metadata internal or job not complete yet pid = job.get_job_runner_external_id() if pid in [ None, '' ]: https://bitbucket.org/galaxy/galaxy-central/commits/832a82ca1937/ Changeset: 832a82ca1937 Branch: release_15.03 User: natefoo Date: 2015-05-01 16:48:04+00:00 Summary: Ensure JobWrapper.galaxy_lib_dir is set, necessary for locally setting metadata externally with Pulsar, which (sometimes?) does not call the wrapper's prepare() method. Affected #: 1 file diff -r aceedb22b31220f9a388935d8ac2cdd5b0dee406 -r 832a82ca19379381894d82c7e2683d2506ec2a2c lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -720,7 +720,7 @@ # Tool versioning variables self.write_version_cmd = None self.version_string = "" - self.galaxy_lib_dir = None + self.__galaxy_lib_dir = None # With job outputs in the working directory, we need the working # directory to be set before prepare is run, or else premature deletion # and job recovery fail. @@ -769,6 +769,12 @@ def commands_in_new_shell(self): return self.app.config.commands_in_new_shell + @property + def galaxy_lib_dir(self): + if self.__galaxy_lib_dir is None: + self.__galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root + return self.__galaxy_lib_dir + # legacy naming get_job_runner = get_job_runner_url @@ -830,8 +836,8 @@ self.sa_session.flush() self.command_line, self.extra_filenames = tool_evaluator.build() - # FIXME: for now, tools get Galaxy's lib dir in their path - self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root + # Ensure galaxy_lib_dir is set in case there are any later chdirs + self.galaxy_lib_dir # Shell fragment to inject dependencies self.dependency_shell_commands = self.tool.build_dependency_shell_commands() # We need command_line persisted to the db in order for Galaxy to re-queue the job @@ -1631,9 +1637,8 @@ self.command_line, self.extra_filenames = tool_evaluator.build() - # FIXME: for now, tools get Galaxy's lib dir in their path - if self.command_line and self.command_line.startswith( 'python' ): - self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root + # Ensure galaxy_lib_dir is set in case there are any later chdirs + self.galaxy_lib_dir # Shell fragment to inject dependencies self.dependency_shell_commands = self.tool.build_dependency_shell_commands() # We need command_line persisted to the db in order for Galaxy to re-queue the job https://bitbucket.org/galaxy/galaxy-central/commits/21fed755993a/ Changeset: 21fed755993a Branch: release_15.03 User: BjoernGruening Date: 2015-05-02 01:54:53+00:00 Summary: Bind the IPython IE to a fixed Docker Image Affected #: 1 file diff -r 832a82ca19379381894d82c7e2683d2506ec2a2c -r 21fed755993a6f6f6e834b0e787785c2c3edd7f1 config/plugins/interactive_environments/ipython/config/ipython.ini.sample --- a/config/plugins/interactive_environments/ipython/config/ipython.ini.sample +++ b/config/plugins/interactive_environments/ipython/config/ipython.ini.sample @@ -8,7 +8,7 @@ [docker] command = docker -image = bgruening/docker-ipython-notebook +image = bgruening/docker-ipython-notebook:0.2 # URL to access the Galaxy API with from the spawn Docker containter, if empty # this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the https://bitbucket.org/galaxy/galaxy-central/commits/73c4d3c6689c/ Changeset: 73c4d3c6689c Branch: release_15.03 User: mvdbeek Date: 2015-05-01 00:39:23+00:00 Summary: Import safe_dumps from galaxy_utils to properly set metadata for interval files. Affected #: 1 file diff -r 21fed755993a6f6f6e834b0e787785c2c3edd7f1 -r 73c4d3c6689ce8ba96a6c8c9f7a693d1a32dbe32 lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -24,6 +24,7 @@ from galaxy.util import stringify_dictionary_keys from galaxy.util import string_as_bool from galaxy.util import in_directory +from galaxy.util.json import safe_dumps from galaxy.util.odict import odict from galaxy.web import form_builder @@ -486,7 +487,7 @@ def to_safe_string( self, value ): # We do not sanitize json dicts - return json.safe_dumps( value ) + return safe_dumps( value ) class PythonObjectParameter( MetadataParameter ): https://bitbucket.org/galaxy/galaxy-central/commits/720bf46e134e/ Changeset: 720bf46e134e Branch: release_15.03 User: jmchilton Date: 2015-05-05 13:22:45+00:00 Summary: Fix job files API for case when nginx upload not being used. Fixes regressions introduced with PR #110. Affected #: 2 files diff -r 73c4d3c6689ce8ba96a6c8c9f7a693d1a32dbe32 -r 720bf46e134e16ab3a1af84a67f57cc81715dae9 lib/galaxy/jobs/runners/pulsar.py --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -342,7 +342,7 @@ encoded_job_id = self.app.security.encode_id(job_id) job_key = self.app.security.encode_id( job_id, kind="jobs_files" ) - endpoint_base = "%s/api/jobs/%s?job_key=%s" + endpoint_base = "%s/api/jobs/%s/files?job_key=%s" if self.app.config.nginx_upload_job_files_path: endpoint_base = "%s" + \ self.app.config.nginx_upload_job_files_path + \ diff -r 73c4d3c6689ce8ba96a6c8c9f7a693d1a32dbe32 -r 720bf46e134e16ab3a1af84a67f57cc81715dae9 lib/galaxy/webapps/galaxy/api/job_files.py --- a/lib/galaxy/webapps/galaxy/api/job_files.py +++ b/lib/galaxy/webapps/galaxy/api/job_files.py @@ -99,7 +99,12 @@ try: shutil.move( input_file.name, path ) finally: - input_file.close() + try: + input_file.close() + except OSError: + # Fails to close file if not using nginx upload because the + # tempfile has moved and Python wants to delete it. + pass return {"message": "ok"} def __authorize_job_access(self, trans, encoded_job_id, **kwargs): https://bitbucket.org/galaxy/galaxy-central/commits/f1c47e6554eb/ Changeset: f1c47e6554eb Branch: release_15.03 User: dannon Date: 2015-04-10 16:23:46+00:00 Summary: Check to see if toolshed install is actually bound prior to disposing (exact same functionality here, but it's less chatty this way, since it might not be bound) Affected #: 1 file diff -r 720bf46e134e16ab3a1af84a67f57cc81715dae9 -r f1c47e6554ebc5fe79e697c1959da335ca18a8a5 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -110,7 +110,9 @@ except: log.exception("Unable to dispose of pooled galaxy model database connections.") try: - galaxy.model.tool_shed_install.mapping.metadata.bind.dispose() + # This model may not actually be bound. + if galaxy.model.tool_shed_install.mapping.metadata.bind: + galaxy.model.tool_shed_install.mapping.metadata.bind.dispose() except: log.exception("Unable to dispose of pooled toolshed install model database connections.") https://bitbucket.org/galaxy/galaxy-central/commits/f6e636fe4269/ Changeset: f6e636fe4269 Branch: release_15.03 User: jmchilton Date: 2015-04-21 14:29:50+00:00 Summary: Bugfix: Resolve metadata dependencies for task runner. Also deduplicates code for next fix. Affected #: 1 file diff -r f1c47e6554ebc5fe79e697c1959da335ca18a8a5 -r f6e636fe42692e5e8d1c7aef95e4f2a7d6d11111 lib/galaxy/jobs/runners/tasks.py --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -124,19 +124,7 @@ #run the metadata setting script here #this is terminate-able when output dataset/job is deleted #so that long running set_meta()s can be canceled without having to reboot the server - if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths: - external_metadata_script = job_wrapper.setup_external_metadata( output_fnames=job_wrapper.get_output_fnames(), - set_extension=True, - kwds={ 'overwrite' : False } ) # we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior - log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) ) - external_metadata_proc = subprocess.Popen( args=external_metadata_script, - shell=True, - env=os.environ, - preexec_fn=os.setpgrp ) - job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session ) - external_metadata_proc.wait() - log.debug( 'execution of external set_meta finished for job %d' % job_wrapper.job_id ) - + self._handle_metadata_externally(job_wrapper, resolve_requirements=True ) # Finish the job try: job_wrapper.finish( stdout, stderr, job_exit_code ) https://bitbucket.org/galaxy/galaxy-central/commits/d03588c4c3ed/ Changeset: d03588c4c3ed Branch: release_15.03 User: jmchilton Date: 2015-04-21 14:36:44+00:00 Summary: Bugfix: Fix metadata python path when using _handle_metadata_externally. This is used by lwr, pulsar, and task runners and possibly by the local job runner in some configurations. Broken with PR #22 which was merged into 15.03. Affected #: 1 file diff -r f6e636fe42692e5e8d1c7aef95e4f2a7d6d11111 -r d03588c4c3ed222ba0c4f40aa813b9bf5cec65d4 lib/galaxy/jobs/runners/__init__.py --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -34,6 +34,8 @@ JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE = "Job runner parameter '%s' value '%s' could not be converted to the correct type" JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE = "Job runner parameter %s failed validation" +GALAXY_LIB_ADJUST_TEMPLATE = """GALAXY_LIB="%s"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi;""" + class RunnerParams( ParamsWithSpecs ): @@ -250,11 +252,13 @@ #this is terminate-able when output dataset/job is deleted #so that long running set_meta()s can be canceled without having to reboot the server if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths: + lib_adjust = GALAXY_LIB_ADJUST_TEMPLATE % job_wrapper.galaxy_lib_dir external_metadata_script = job_wrapper.setup_external_metadata( output_fnames=job_wrapper.get_output_fnames(), set_extension=True, tmp_dir=job_wrapper.working_directory, #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior kwds={ 'overwrite' : False } ) + external_metadata_script = "%s %s" % (lib_adjust, external_metadata_script) if resolve_requirements: dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands() if dependency_shell_commands: https://bitbucket.org/galaxy/galaxy-central/commits/a9d766325bd8/ Changeset: a9d766325bd8 Branch: release_15.03 User: guerler Date: 2015-04-22 17:21:24+00:00 Summary: Fix default values for nested conditional test parameters Affected #: 1 file diff -r d03588c4c3ed222ba0c4f40aa813b9bf5cec65d4 -r a9d766325bd8b659482b5c15f37ca7ffef323859 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2446,7 +2446,8 @@ elif input.type == 'conditional': if 'test_param' in tool_dict: test_param = tool_dict['test_param'] - test_param['value'] = jsonify(group_state.get(test_param['name'], None)) + test_param['default_value'] = jsonify(input.test_param.get_initial_value(trans, other_values)) + test_param['value'] = jsonify(group_state.get(test_param['name'], test_param['default_value'])) for i in range (len ( tool_dict['cases'] ) ): current_state = {} if i == group_state.get('__current_case__', None): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.