11 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b928679cc4ac/ Changeset: b928679cc4ac User: jmchilton Date: 2014-01-28 21:35:14 Summary: PEP-8 and style fixes for various files related to history import/export. Affected #: 3 files diff -r 0ab0eb81d9828b0527d68f404a36acc64359f5f8 -r b928679cc4acd843e8a6dcfb24c89f5a0b40d3a6 lib/galaxy/tools/imp_exp/__init__.py --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -1,4 +1,8 @@ -import os, shutil, logging, tempfile, json +import os +import shutil +import logging +import tempfile +import json from galaxy import model from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.web.framework.helpers import to_unicode @@ -8,6 +12,7 @@ log = logging.getLogger(__name__) + def load_history_imp_exp_tools( toolbox ): """ Adds tools for importing/exporting histories to archives. """ # Use same process as that used in load_external_metadata_tool; see that @@ -42,6 +47,7 @@ toolbox.tools_by_id[ history_imp_tool.id ] = history_imp_tool log.debug( "Loaded history import tool: %s", history_imp_tool.id ) + class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations ): """ Class provides support for performing jobs that import a history from @@ -144,23 +150,23 @@ metadata = dataset_attrs['metadata'] # Create dataset and HDA. - hda = model.HistoryDatasetAssociation( name = dataset_attrs['name'].encode( 'utf-8' ), - extension = dataset_attrs['extension'], - info = dataset_attrs['info'].encode( 'utf-8' ), - blurb = dataset_attrs['blurb'], - peek = dataset_attrs['peek'], - designation = dataset_attrs['designation'], - visible = dataset_attrs['visible'], - dbkey = metadata['dbkey'], - metadata = metadata, - history = new_history, - create_dataset = True, - sa_session = self.sa_session ) + hda = model.HistoryDatasetAssociation( name=dataset_attrs['name'].encode( 'utf-8' ), + extension=dataset_attrs['extension'], + info=dataset_attrs['info'].encode( 'utf-8' ), + blurb=dataset_attrs['blurb'], + peek=dataset_attrs['peek'], + designation=dataset_attrs['designation'], + visible=dataset_attrs['visible'], + dbkey=metadata['dbkey'], + metadata=metadata, + history=new_history, + create_dataset=True, + sa_session=self.sa_session ) hda.state = hda.states.OK self.sa_session.add( hda ) self.sa_session.flush() - new_history.add_dataset( hda, genome_build = None ) - hda.hid = dataset_attrs['hid'] # Overwrite default hid set when HDA added to history. + new_history.add_dataset( hda, genome_build=None ) + hda.hid = dataset_attrs['hid'] # Overwrite default hid set when HDA added to history. # TODO: Is there a way to recover permissions? Is this needed? #permissions = trans.app.security_agent.history_get_default_permissions( new_history ) #trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions ) @@ -273,6 +279,7 @@ jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() + class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations ): """ Class provides support for performing jobs that export a history to an @@ -317,23 +324,23 @@ """ Encode an HDA, default encoding for everything else. """ if isinstance( obj, trans.app.model.HistoryDatasetAssociation ): return { - "__HistoryDatasetAssociation__" : True, - "create_time" : obj.create_time.__str__(), - "update_time" : obj.update_time.__str__(), - "hid" : obj.hid, - "name" : to_unicode( obj.name ), - "info" : to_unicode( obj.info ), - "blurb" : obj.blurb, - "peek" : obj.peek, - "extension" : obj.extension, - "metadata" : prepare_metadata( dict( obj.metadata.items() ) ), - "parent_id" : obj.parent_id, - "designation" : obj.designation, - "deleted" : obj.deleted, - "visible" : obj.visible, - "file_name" : obj.file_name, - "annotation" : to_unicode( getattr( obj, 'annotation', '' ) ), - "tags" : get_item_tag_dict( obj ), + "__HistoryDatasetAssociation__": True, + "create_time": obj.create_time.__str__(), + "update_time": obj.update_time.__str__(), + "hid": obj.hid, + "name": to_unicode( obj.name ), + "info": to_unicode( obj.info ), + "blurb": obj.blurb, + "peek": obj.peek, + "extension": obj.extension, + "metadata": prepare_metadata( dict( obj.metadata.items() ) ), + "parent_id": obj.parent_id, + "designation": obj.designation, + "deleted": obj.deleted, + "visible": obj.visible, + "file_name": obj.file_name, + "annotation": to_unicode( getattr( obj, 'annotation', '' ) ), + "tags": get_item_tag_dict( obj ), } if isinstance( obj, UnvalidatedValue ): return obj.__str__() @@ -347,15 +354,15 @@ # Write history attributes to file. history = jeha.history history_attrs = { - "create_time" : history.create_time.__str__(), - "update_time" : history.update_time.__str__(), - "name" : to_unicode( history.name ), - "hid_counter" : history.hid_counter, - "genome_build" : history.genome_build, - "annotation" : to_unicode( self.get_item_annotation_str( trans.sa_session, history.user, history ) ), - "tags" : get_item_tag_dict( history ), - "includes_hidden_datasets" : include_hidden, - "includes_deleted_datasets" : include_deleted + "create_time": history.create_time.__str__(), + "update_time": history.update_time.__str__(), + "name": to_unicode( history.name ), + "hid_counter": history.hid_counter, + "genome_build": history.genome_build, + "annotation": to_unicode( self.get_item_annotation_str( trans.sa_session, history.user, history ) ), + "tags": get_item_tag_dict( history ), + "includes_hidden_datasets": include_hidden, + "includes_deleted_datasets": include_deleted } history_attrs_filename = tempfile.NamedTemporaryFile( dir=temp_output_dir ).name history_attrs_out = open( history_attrs_filename, 'w' ) @@ -391,7 +398,7 @@ # Get the associated job, if any. If this hda was copied from another, # we need to find the job that created the origial hda job_hda = hda - while job_hda.copied_from_history_dataset_association: #should this check library datasets as well? + while job_hda.copied_from_history_dataset_association: # should this check library datasets as well? job_hda = job_hda.copied_from_history_dataset_association if not job_hda.creating_job_associations: # No viable HDA found. @@ -472,4 +479,3 @@ shutil.rmtree( temp_dir ) except Exception, e: log.debug( 'Error deleting directory containing attribute files (%s): %s' % ( temp_dir, e ) ) - diff -r 0ab0eb81d9828b0527d68f404a36acc64359f5f8 -r b928679cc4acd843e8a6dcfb24c89f5a0b40d3a6 lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -6,19 +6,25 @@ --[url|file] source type, either a URL or a file. """ -import sys, optparse, tarfile, tempfile, urllib2, math +import sys +import optparse +import tarfile +import tempfile +import urllib2 +import math # Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. MAX_SIZE = 100 * math.pow( 2, 30 ) + def url_to_file( url, dest_file ): """ Transfer a file from a remote URL to a temporary file. """ try: url_reader = urllib2.urlopen( url ) - CHUNK = 10 * 1024 # 10k + CHUNK = 10 * 1024 # 10k total = 0 fp = open( dest_file, 'wb') while True: @@ -35,6 +41,7 @@ print "Exception getting file from URL: %s" % e, sys.stderr return None + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -63,4 +70,4 @@ # Unpack archive. unpack_archive( archive_file, dest_dir ) except Exception, e: - print "Error unpacking tar/gz archive: %s" % e, sys.stderr \ No newline at end of file + print "Error unpacking tar/gz archive: %s" % e, sys.stderr diff -r 0ab0eb81d9828b0527d68f404a36acc64359f5f8 -r b928679cc4acd843e8a6dcfb24c89f5a0b40d3a6 lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -11,15 +11,17 @@ from galaxy import web from galaxy.web import _future_expose_api as expose_api from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous -from galaxy.util import string_as_bool, restore_text -from galaxy.util.sanitize_html import sanitize_html -from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesTagsMixin +from galaxy.util import string_as_bool +from galaxy.util import restore_text +from galaxy.web.base.controller import BaseAPIController +from galaxy.web.base.controller import UsesHistoryMixin +from galaxy.web.base.controller import UsesTagsMixin from galaxy.web import url_for -from galaxy.model.orm import desc import logging log = logging.getLogger( __name__ ) + class HistoriesController( BaseAPIController, UsesHistoryMixin, UsesTagsMixin ): @expose_api_anonymous @@ -46,14 +48,14 @@ histories = self.get_user_histories( trans, user=trans.user, only_deleted=deleted ) #for history in query: for history in histories: - item = history.to_dict(value_mapper={'id':trans.security.encode_id}) + item = history.to_dict(value_mapper={'id': trans.security.encode_id}) item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) rval.append( item ) elif trans.galaxy_session.current_history: #No user, this must be session authentication with an anonymous user. history = trans.galaxy_session.current_history - item = history.to_dict(value_mapper={'id':trans.security.encode_id}) + item = history.to_dict(value_mapper={'id': trans.security.encode_id}) item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) rval.append(item) @@ -257,7 +259,7 @@ log.exception( 'Histories API, delete: uncaught HTTPInternalServerError: %s, %s\n%s', history_id, str( kwd ), str( http_server_err ) ) raise - except HTTPException, http_exc: + except HTTPException: raise except Exception, exc: log.exception( 'Histories API, delete: uncaught exception: %s, %s\n%s', https://bitbucket.org/galaxy/galaxy-central/commits/f4485c8dd5f5/ Changeset: f4485c8dd5f5 Branch: job-search User: Kyle Ellrott Date: 2014-01-22 02:30:47 Summary: Adding in /api/jobs and refining the Job.to_dict method Affected #: 4 files diff -r 54defa390a91ec1db34141f97711e673218cde13 -r f4485c8dd5f5392f1cc8a1ed296fb772064f1e10 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -209,8 +209,8 @@ class Job( object, Dictifiable ): - dict_collection_visible_keys = [ 'id', 'state', 'exit_code' ] - dict_element_visible_keys = [ 'id', 'state', 'exit_code' ] + dict_collection_visible_keys = [ 'id', 'state', 'exit_code', 'update_time', 'create_time' ] + dict_element_visible_keys = [ 'id', 'state', 'exit_code', 'update_time', 'create_time' ] """ A job represents a request to run a tool given input datasets, tool @@ -416,30 +416,31 @@ dataset.info = 'Job output deleted by user before job completed' def to_dict( self, view='collection' ): rval = super( Job, self ).to_dict( view=view ) - rval['tool_name'] = self.tool_id - param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) - rval['params'] = param_dict + if view == 'element': + rval['tool_name'] = self.tool_id + param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) + rval['params'] = param_dict - input_dict = {} - for i in self.input_datasets: - if i.dataset is not None: - input_dict[i.name] = {"hda_id" : i.dataset.id} - for i in self.input_library_datasets: - if i.dataset is not None: - input_dict[i.name] = {"ldda_id" : i.dataset.id} - for k in input_dict: - if k in param_dict: - del param_dict[k] - rval['inputs'] = input_dict + input_dict = {} + for i in self.input_datasets: + if i.dataset is not None: + input_dict[i.name] = {"id" : i.dataset.id, "src" : "hda"} + for i in self.input_library_datasets: + if i.dataset is not None: + input_dict[i.name] = {"id" : i.dataset.id, "src" : "ldda"} + for k in input_dict: + if k in param_dict: + del param_dict[k] + rval['inputs'] = input_dict - output_dict = {} - for i in self.output_datasets: - if i.dataset is not None: - output_dict[i.name] = {"hda_id" : i.dataset.id} - for i in self.output_library_datasets: - if i.dataset is not None: - output_dict[i.name] = {"ldda_id" : i.dataset.id} - rval['outputs'] = output_dict + output_dict = {} + for i in self.output_datasets: + if i.dataset is not None: + output_dict[i.name] = {"id" : i.dataset.id, "src" : "hda"} + for i in self.output_library_datasets: + if i.dataset is not None: + output_dict[i.name] = {"id" : i.dataset.id, "src" : "ldda"} + rval['outputs'] = output_dict return rval diff -r 54defa390a91ec1db34141f97711e673218cde13 -r f4485c8dd5f5392f1cc8a1ed296fb772064f1e10 lib/galaxy/webapps/galaxy/api/jobs.py --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -0,0 +1,84 @@ +""" +API operations on a jobs. + +.. seealso:: :class:`galaxy.model.Jobs` +""" + +import pkg_resources +pkg_resources.require( "Paste" ) +from paste.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPInternalServerError, HTTPException + +from sqlalchemy import or_ + +from galaxy import web +from galaxy.web import _future_expose_api as expose_api +from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous +from galaxy.util import string_as_bool, restore_text +from galaxy.util.sanitize_html import sanitize_html +from galaxy.web.base.controller import BaseAPIController +from galaxy.web import url_for +from galaxy.model.orm import desc + +import logging +log = logging.getLogger( __name__ ) + +class HistoriesController( BaseAPIController ): + + @web.expose_api + def index( self, trans, **kwd ): + """ + index( trans, state=None ) + * GET /api/jobs: + return jobs for current user + + :type state: string or list + :param state: limit listing of jobs to those that match one of the included states. If none, all are returned. + Valid Galaxy job states include: + 'new', 'upload', 'waiting', 'queued', 'running', 'ok', 'error', 'paused', 'deleted', 'deleted_new' + + :rtype: list + :returns: list of dictionaries containing summary job information + """ + + state = kwd.get('state', None) + query = trans.sa_session.query(trans.app.model.Job).filter( + trans.app.model.Job.user == trans.user ) + if state is not None: + if isinstance(state, basestring): + query = query.filter( trans.app.model.Job.state == state ) + elif isinstance(state, list): + t = [] + for s in state: + t.append( trans.app.model.Job.state == s ) + query = query.filter( or_( *t ) ) + + out = [] + for job in query.order_by( + trans.app.model.Job.update_time.desc() + ).all(): + out.append( self.encode_all_ids( trans, job.to_dict('collection'), True) ) + return out + + @web.expose_api + def show( self, trans, id, **kwd ): + decoded_job_id = trans.security.decode_id(id) + query = trans.sa_session.query(trans.app.model.Job).filter( + trans.app.model.Job.user == trans.user, + trans.app.model.Job.id == decoded_job_id) + job = query.first() + if job is None: + return None + return self.encode_all_ids( trans, job.to_dict('element'), True) + + @expose_api + def create( self, trans, payload, **kwd ): + error = None + if 'tool_id' not in payload: + error = "No tool ID" + + tool_id = payload.get('tool_id') + + if error is not None: + return { "error" : error } + + diff -r 54defa390a91ec1db34141f97711e673218cde13 -r f4485c8dd5f5392f1cc8a1ed296fb772064f1e10 lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -78,7 +78,12 @@ # -- Execute tool. -- # Get tool. - tool = trans.app.toolbox.get_tool( payload[ 'tool_id' ] ) if 'tool_id' in payload else None + tool = None + if 'tool_id' in payload: + tool = trans.app.toolbox.get_tool( payload[ 'tool_id' ] ) + if 'tool_name' in payload: + #in job descriptions it is called 'tool_name' to avoid having the name 'crushed' + tool = trans.app.toolbox.get_tool( payload[ 'tool_name' ] ) if not tool: trans.response.status = 404 return { "message": { "type": "error", "text" : trans.app.model.Dataset.conversion_messages.NO_TOOL } } diff -r 54defa390a91ec1db34141f97711e673218cde13 -r f4485c8dd5f5392f1cc8a1ed296fb772064f1e10 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -228,6 +228,12 @@ path_prefix='/api/libraries/:library_id', parent_resources=dict( member_name='library', collection_name='libraries' ) ) + webapp.mapper.resource( 'job', + 'jobs', + path_prefix='/api' ) + #webapp.mapper.connect( 'job_item', '/api/jobs/:job_id', controller='jobs', action='show', conditions=dict( method=['GET'] ) ) + + _add_item_extended_metadata_controller( webapp, name_prefix="library_dataset_", path_prefix='/api/libraries/:library_id/contents/:library_content_id' ) https://bitbucket.org/galaxy/galaxy-central/commits/f78bab3d44f0/ Changeset: f78bab3d44f0 Branch: job-search User: Kyle Ellrott Date: 2014-01-22 23:09:36 Summary: Adding job search method that attempts to find previous tool runs with the same requested inputs and outputs. Affected #: 2 files diff -r f4485c8dd5f5392f1cc8a1ed296fb772064f1e10 -r f78bab3d44f00b245beebce45cac2a2287c8f4ce lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -416,8 +416,8 @@ dataset.info = 'Job output deleted by user before job completed' def to_dict( self, view='collection' ): rval = super( Job, self ).to_dict( view=view ) + rval['tool_name'] = self.tool_id if view == 'element': - rval['tool_name'] = self.tool_id param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) rval['params'] = param_dict diff -r f4485c8dd5f5392f1cc8a1ed296fb772064f1e10 -r f78bab3d44f00b245beebce45cac2a2287c8f4ce lib/galaxy/webapps/galaxy/api/jobs.py --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -8,21 +8,22 @@ pkg_resources.require( "Paste" ) from paste.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPInternalServerError, HTTPException -from sqlalchemy import or_ - +from sqlalchemy import or_, and_ +from sqlalchemy.orm import aliased +import json from galaxy import web from galaxy.web import _future_expose_api as expose_api from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous from galaxy.util import string_as_bool, restore_text from galaxy.util.sanitize_html import sanitize_html -from galaxy.web.base.controller import BaseAPIController +from galaxy.web.base.controller import BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems from galaxy.web import url_for from galaxy.model.orm import desc import logging log = logging.getLogger( __name__ ) -class HistoriesController( BaseAPIController ): +class HistoriesController( BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems ): @web.expose_api def index( self, trans, **kwd ): @@ -61,6 +62,18 @@ @web.expose_api def show( self, trans, id, **kwd ): + """ + show( trans, id ) + * GET /api/jobs/{job_id}: + return jobs for current user + + :type id: string + :param id: Specific job id + + :rtype: dictionary + :returns: dictionary containing full description of job data + """ + decoded_job_id = trans.security.decode_id(id) query = trans.sa_session.query(trans.app.model.Job).filter( trans.app.model.Job.user == trans.user, @@ -72,13 +85,100 @@ @expose_api def create( self, trans, payload, **kwd ): + """ + show( trans, payload ) + * POST /api/jobs: + return jobs for current user + + :type payload: dict + :param payload: Dictionary containing description of requested job. This is in the same format as + a request to POST /apt/tools would take to initiate a job + + :rtype: list + :returns: list of dictionaries containing summary job information of the jobs that match the requested job run + + This method is designed to scan the list of previously run jobs and find records of jobs that had + the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply + recycle the old results. + """ + error = None - if 'tool_id' not in payload: + tool_id = None + if 'tool_id' in payload: + tool_id = payload.get('tool_id') + if 'tool_name' in payload: + tool_id = payload.get('tool_name') + + tool = trans.app.toolbox.get_tool( tool_id ) + if tool is None: + error = "Requested tool not found" + if 'inputs' not in payload: + error = "No inputs defined" + if tool_id is None: error = "No tool ID" - - tool_id = payload.get('tool_id') - if error is not None: return { "error" : error } + inputs = payload['inputs'] + input_data = {} + input_param = {} + for k, v in inputs.items(): + if isinstance(v,dict): + if 'id' in v: + try: + if 'src' not in v or v['src'] == 'hda': + dataset = self.get_dataset( trans, v['id'], check_ownership=False, check_accessible=True ) + else: + dataset = self.get_library_dataset_dataset_association( trans, v['id'] ) + except Exception, e: + return { "error" : str( e ) } + if dataset is None: + return { "error" : "Dataset %s not found" % (v['id']) } + input_data[k] = dataset.dataset_id + else: + input_param[k] = json.dumps(v) + + query = trans.sa_session.query( trans.app.model.Job ).filter( + trans.app.model.Job.tool_id == tool_id, + trans.app.model.Job.user == trans.user + ).filter( + or_( + trans.app.model.Job.state == 'running', + trans.app.model.Job.state == 'queued', + trans.app.model.Job.state == 'waiting', + trans.app.model.Job.state == 'running', + trans.app.model.Job.state == 'ok', + ) + ) + + for k,v in input_param.items(): + a = aliased(trans.app.model.JobParameter) + query = query.filter( and_( + trans.app.model.Job.id == a.job_id, + a.name == k, + a.value == v + )) + + for k,v in input_data.items(): + """ + Here we are attempting to link the inputs to the underlying dataset (not the dataset association) + This way, if the calulation was done using a copied HDA (copied from the library or another history) + the search will still find the job + """ + a = aliased(trans.app.model.JobToInputDatasetAssociation) + b = aliased(trans.app.model.HistoryDatasetAssociation) + query = query.filter( and_( + trans.app.model.Job.id == a.job_id, + a.dataset_id == b.id, + b.deleted == False, + b.dataset_id == v + )) + out = [] + for job in query.all(): + """ + check to make sure none of the output files have been deleted + """ + if all(list(a.dataset.deleted == False for a in job.output_datasets)): + out.append( self.encode_all_ids( trans, job.to_dict('element'), True) ) + return out https://bitbucket.org/galaxy/galaxy-central/commits/3284b02d842a/ Changeset: 3284b02d842a Branch: job-search User: kellrott Date: 2014-01-24 21:05:03 Summary: Modified patch behavoir to match comments to pull request Affected #: 5 files diff -r f78bab3d44f00b245beebce45cac2a2287c8f4ce -r 3284b02d842a48c92bab4669982231ab0ac3ca98 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -416,7 +416,7 @@ dataset.info = 'Job output deleted by user before job completed' def to_dict( self, view='collection' ): rval = super( Job, self ).to_dict( view=view ) - rval['tool_name'] = self.tool_id + rval['tool_id'] = self.tool_id if view == 'element': param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) rval['params'] = param_dict diff -r f78bab3d44f00b245beebce45cac2a2287c8f4ce -r 3284b02d842a48c92bab4669982231ab0ac3ca98 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -161,7 +161,7 @@ if type( rval ) != dict: return rval for k, v in rval.items(): - if (k == 'id' or k.endswith( '_id' )) and v is not None: + if (k == 'id' or k.endswith( '_id' )) and v is not None and k not in ['tool_id']: try: rval[k] = trans.security.encode_id( v ) except: diff -r f78bab3d44f00b245beebce45cac2a2287c8f4ce -r 3284b02d842a48c92bab4669982231ab0ac3ca98 lib/galaxy/webapps/galaxy/api/jobs.py --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -23,7 +23,7 @@ import logging log = logging.getLogger( __name__ ) -class HistoriesController( BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems ): +class JobController( BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems ): @web.expose_api def index( self, trans, **kwd ): @@ -85,9 +85,13 @@ @expose_api def create( self, trans, payload, **kwd ): + raise NotImplementedError() + + @expose_api + def search(self, trans, payload, **kwd): """ - show( trans, payload ) - * POST /api/jobs: + search( trans, payload ) + * POST /api/jobs/search: return jobs for current user :type payload: dict @@ -106,8 +110,6 @@ tool_id = None if 'tool_id' in payload: tool_id = payload.get('tool_id') - if 'tool_name' in payload: - tool_id = payload.get('tool_name') tool = trans.app.toolbox.get_tool( tool_id ) if tool is None: @@ -142,15 +144,28 @@ query = trans.sa_session.query( trans.app.model.Job ).filter( trans.app.model.Job.tool_id == tool_id, trans.app.model.Job.user == trans.user - ).filter( - or_( - trans.app.model.Job.state == 'running', - trans.app.model.Job.state == 'queued', - trans.app.model.Job.state == 'waiting', - trans.app.model.Job.state == 'running', - trans.app.model.Job.state == 'ok', + ) + + if 'state' not in payload: + query = query.filter( + or_( + trans.app.model.Job.state == 'running', + trans.app.model.Job.state == 'queued', + trans.app.model.Job.state == 'waiting', + trans.app.model.Job.state == 'running', + trans.app.model.Job.state == 'ok', + ) ) - ) + else: + if isinstance(payload['state'], basestring): + query = query.filter( trans.app.model.Job.state == payload['state'] ) + elif isinstance(payload['state'], list): + o = [] + for s in payload['state']: + o.append( trans.app.model.Job.state == s ) + query = query.filter( + or_(*o) + ) for k,v in input_param.items(): a = aliased(trans.app.model.JobParameter) diff -r f78bab3d44f00b245beebce45cac2a2287c8f4ce -r 3284b02d842a48c92bab4669982231ab0ac3ca98 lib/galaxy/webapps/galaxy/api/tools.py --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -78,12 +78,7 @@ # -- Execute tool. -- # Get tool. - tool = None - if 'tool_id' in payload: - tool = trans.app.toolbox.get_tool( payload[ 'tool_id' ] ) - if 'tool_name' in payload: - #in job descriptions it is called 'tool_name' to avoid having the name 'crushed' - tool = trans.app.toolbox.get_tool( payload[ 'tool_name' ] ) + tool = trans.app.toolbox.get_tool( payload[ 'tool_id' ] ) if 'tool_id' in payload else None if not tool: trans.response.status = 404 return { "message": { "type": "error", "text" : trans.app.model.Dataset.conversion_messages.NO_TOOL } } diff -r f78bab3d44f00b245beebce45cac2a2287c8f4ce -r 3284b02d842a48c92bab4669982231ab0ac3ca98 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -231,7 +231,7 @@ webapp.mapper.resource( 'job', 'jobs', path_prefix='/api' ) - #webapp.mapper.connect( 'job_item', '/api/jobs/:job_id', controller='jobs', action='show', conditions=dict( method=['GET'] ) ) + webapp.mapper.connect( 'job_search', '/api/jobs/search', controller='jobs', action='search', conditions=dict( method=['POST'] ) ) _add_item_extended_metadata_controller( webapp, https://bitbucket.org/galaxy/galaxy-central/commits/558fb7b96b49/ Changeset: 558fb7b96b49 Branch: job-search User: kellrott Date: 2014-01-24 22:59:36 Summary: Adding more standard exceptions Affected #: 1 file diff -r 3284b02d842a48c92bab4669982231ab0ac3ca98 -r 558fb7b96b49bdcec0321a0c8d50afbf37ff972b lib/galaxy/webapps/galaxy/api/jobs.py --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -19,6 +19,7 @@ from galaxy.web.base.controller import BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems from galaxy.web import url_for from galaxy.model.orm import desc +from galaxy import exceptions import logging log = logging.getLogger( __name__ ) @@ -73,14 +74,16 @@ :rtype: dictionary :returns: dictionary containing full description of job data """ - - decoded_job_id = trans.security.decode_id(id) + try: + decoded_job_id = trans.security.decode_id(id) + except: + raise exceptions.ObjectAttributeInvalidException() query = trans.sa_session.query(trans.app.model.Job).filter( trans.app.model.Job.user == trans.user, trans.app.model.Job.id == decoded_job_id) job = query.first() if job is None: - return None + raise exceptions.ObjectNotFound() return self.encode_all_ids( trans, job.to_dict('element'), True) @expose_api @@ -106,20 +109,17 @@ recycle the old results. """ - error = None tool_id = None if 'tool_id' in payload: tool_id = payload.get('tool_id') + if tool_id is None: + raise exceptions.ObjectAttributeMissingException("No tool id") tool = trans.app.toolbox.get_tool( tool_id ) if tool is None: - error = "Requested tool not found" + raise exceptions.ObjectNotFound( "Requested tool not found" ) if 'inputs' not in payload: - error = "No inputs defined" - if tool_id is None: - error = "No tool ID" - if error is not None: - return { "error" : error } + raise exceptions.ObjectAttributeMissingException("No inputs defined") inputs = payload['inputs'] @@ -136,7 +136,7 @@ except Exception, e: return { "error" : str( e ) } if dataset is None: - return { "error" : "Dataset %s not found" % (v['id']) } + raise exceptions.ObjectNotFound("Dataset %s not found" % (v['id'])) input_data[k] = dataset.dataset_id else: input_param[k] = json.dumps(v) https://bitbucket.org/galaxy/galaxy-central/commits/95747eacf09c/ Changeset: 95747eacf09c Branch: job-search User: jmchilton Date: 2014-01-29 06:29:15 Summary: PEP-8 and other style adjustments for new jobs API. Affected #: 1 file diff -r 558fb7b96b49bdcec0321a0c8d50afbf37ff972b -r 95747eacf09c21335323131382de433409b86e16 lib/galaxy/webapps/galaxy/api/jobs.py --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -4,26 +4,20 @@ .. seealso:: :class:`galaxy.model.Jobs` """ -import pkg_resources -pkg_resources.require( "Paste" ) -from paste.httpexceptions import HTTPBadRequest, HTTPForbidden, HTTPInternalServerError, HTTPException - from sqlalchemy import or_, and_ from sqlalchemy.orm import aliased import json from galaxy import web from galaxy.web import _future_expose_api as expose_api -from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous -from galaxy.util import string_as_bool, restore_text -from galaxy.util.sanitize_html import sanitize_html -from galaxy.web.base.controller import BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems -from galaxy.web import url_for -from galaxy.model.orm import desc +from galaxy.web.base.controller import BaseAPIController +from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin +from galaxy.web.base.controller import UsesLibraryMixinItems from galaxy import exceptions import logging log = logging.getLogger( __name__ ) + class JobController( BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems ): @web.expose_api @@ -42,13 +36,14 @@ :returns: list of dictionaries containing summary job information """ - state = kwd.get('state', None) - query = trans.sa_session.query(trans.app.model.Job).filter( - trans.app.model.Job.user == trans.user ) + state = kwd.get( 'state', None ) + query = trans.sa_session.query( trans.app.model.Job ).filter( + trans.app.model.Job.user == trans.user + ) if state is not None: - if isinstance(state, basestring): + if isinstance( state, basestring ): query = query.filter( trans.app.model.Job.state == state ) - elif isinstance(state, list): + elif isinstance( state, list ): t = [] for s in state: t.append( trans.app.model.Job.state == s ) @@ -58,7 +53,7 @@ for job in query.order_by( trans.app.model.Job.update_time.desc() ).all(): - out.append( self.encode_all_ids( trans, job.to_dict('collection'), True) ) + out.append( self.encode_all_ids( trans, job.to_dict( 'collection' ), True ) ) return out @web.expose_api @@ -78,20 +73,21 @@ decoded_job_id = trans.security.decode_id(id) except: raise exceptions.ObjectAttributeInvalidException() - query = trans.sa_session.query(trans.app.model.Job).filter( + query = trans.sa_session.query( trans.app.model.Job ).filter( trans.app.model.Job.user == trans.user, - trans.app.model.Job.id == decoded_job_id) + trans.app.model.Job.id == decoded_job_id + ) job = query.first() if job is None: raise exceptions.ObjectNotFound() - return self.encode_all_ids( trans, job.to_dict('element'), True) + return self.encode_all_ids( trans, job.to_dict( 'element' ), True ) @expose_api def create( self, trans, payload, **kwd ): raise NotImplementedError() @expose_api - def search(self, trans, payload, **kwd): + def search( self, trans, payload, **kwd ): """ search( trans, payload ) * POST /api/jobs/search: @@ -104,96 +100,97 @@ :rtype: list :returns: list of dictionaries containing summary job information of the jobs that match the requested job run - This method is designed to scan the list of previously run jobs and find records of jobs that had - the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply + This method is designed to scan the list of previously run jobs and find records of jobs that had + the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply recycle the old results. """ tool_id = None if 'tool_id' in payload: - tool_id = payload.get('tool_id') + tool_id = payload.get( 'tool_id' ) if tool_id is None: - raise exceptions.ObjectAttributeMissingException("No tool id") + raise exceptions.ObjectAttributeMissingException( "No tool id" ) tool = trans.app.toolbox.get_tool( tool_id ) if tool is None: raise exceptions.ObjectNotFound( "Requested tool not found" ) if 'inputs' not in payload: - raise exceptions.ObjectAttributeMissingException("No inputs defined") + raise exceptions.ObjectAttributeMissingException( "No inputs defined" ) - inputs = payload['inputs'] + inputs = payload[ 'inputs' ] input_data = {} input_param = {} for k, v in inputs.items(): - if isinstance(v,dict): + if isinstance( v, dict ): if 'id' in v: try: - if 'src' not in v or v['src'] == 'hda': + if 'src' not in v or v[ 'src' ] == 'hda': dataset = self.get_dataset( trans, v['id'], check_ownership=False, check_accessible=True ) else: dataset = self.get_library_dataset_dataset_association( trans, v['id'] ) except Exception, e: return { "error" : str( e ) } if dataset is None: - raise exceptions.ObjectNotFound("Dataset %s not found" % (v['id'])) + raise exceptions.ObjectNotFound( "Dataset %s not found" % ( v[ 'id' ] ) ) input_data[k] = dataset.dataset_id else: - input_param[k] = json.dumps(v) + input_param[k] = json.dumps( v ) - query = trans.sa_session.query( trans.app.model.Job ).filter( - trans.app.model.Job.tool_id == tool_id, + query = trans.sa_session.query( trans.app.model.Job ).filter( + trans.app.model.Job.tool_id == tool_id, trans.app.model.Job.user == trans.user ) if 'state' not in payload: query = query.filter( - or_( - trans.app.model.Job.state == 'running', - trans.app.model.Job.state == 'queued', - trans.app.model.Job.state == 'waiting', + or_( trans.app.model.Job.state == 'running', - trans.app.model.Job.state == 'ok', + trans.app.model.Job.state == 'queued', + trans.app.model.Job.state == 'waiting', + trans.app.model.Job.state == 'running', + trans.app.model.Job.state == 'ok', ) ) else: - if isinstance(payload['state'], basestring): - query = query.filter( trans.app.model.Job.state == payload['state'] ) - elif isinstance(payload['state'], list): + if isinstance( payload[ 'state' ], basestring ): + query = query.filter( trans.app.model.Job.state == payload[ 'state' ] ) + elif isinstance( payload[ 'state' ], list ): o = [] - for s in payload['state']: + for s in payload[ 'state' ]: o.append( trans.app.model.Job.state == s ) query = query.filter( - or_(*o) + or_( *o ) ) - for k,v in input_param.items(): - a = aliased(trans.app.model.JobParameter) + for k, v in input_param.items(): + a = aliased( trans.app.model.JobParameter ) query = query.filter( and_( trans.app.model.Job.id == a.job_id, a.name == k, a.value == v - )) + ) ) - for k,v in input_data.items(): + for k, v in input_data.items(): """ Here we are attempting to link the inputs to the underlying dataset (not the dataset association) This way, if the calulation was done using a copied HDA (copied from the library or another history) the search will still find the job """ - a = aliased(trans.app.model.JobToInputDatasetAssociation) - b = aliased(trans.app.model.HistoryDatasetAssociation) + a = aliased( trans.app.model.JobToInputDatasetAssociation ) + b = aliased( trans.app.model.HistoryDatasetAssociation ) query = query.filter( and_( trans.app.model.Job.id == a.job_id, a.dataset_id == b.id, b.deleted == False, b.dataset_id == v - )) + ) ) + out = [] for job in query.all(): """ check to make sure none of the output files have been deleted """ - if all(list(a.dataset.deleted == False for a in job.output_datasets)): - out.append( self.encode_all_ids( trans, job.to_dict('element'), True) ) + if all( list( a.dataset.deleted == False for a in job.output_datasets ) ): + out.append( self.encode_all_ids( trans, job.to_dict( 'element' ), True ) ) return out https://bitbucket.org/galaxy/galaxy-central/commits/e20e9d60eb80/ Changeset: e20e9d60eb80 Branch: job-search User: jmchilton Date: 2014-01-29 06:34:04 Summary: Tweak Jobs API to only use new style API decorator, exception handling. Affected #: 1 file diff -r 95747eacf09c21335323131382de433409b86e16 -r e20e9d60eb8017a241976912bfceb30dd316b20e lib/galaxy/webapps/galaxy/api/jobs.py --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -7,7 +7,6 @@ from sqlalchemy import or_, and_ from sqlalchemy.orm import aliased import json -from galaxy import web from galaxy.web import _future_expose_api as expose_api from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin @@ -20,7 +19,7 @@ class JobController( BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems ): - @web.expose_api + @expose_api def index( self, trans, **kwd ): """ index( trans, state=None ) @@ -56,7 +55,7 @@ out.append( self.encode_all_ids( trans, job.to_dict( 'collection' ), True ) ) return out - @web.expose_api + @expose_api def show( self, trans, id, **kwd ): """ show( trans, id ) @@ -124,13 +123,10 @@ for k, v in inputs.items(): if isinstance( v, dict ): if 'id' in v: - try: - if 'src' not in v or v[ 'src' ] == 'hda': - dataset = self.get_dataset( trans, v['id'], check_ownership=False, check_accessible=True ) - else: - dataset = self.get_library_dataset_dataset_association( trans, v['id'] ) - except Exception, e: - return { "error" : str( e ) } + if 'src' not in v or v[ 'src' ] == 'hda': + dataset = self.get_dataset( trans, v['id'], check_ownership=False, check_accessible=True ) + else: + dataset = self.get_library_dataset_dataset_association( trans, v['id'] ) if dataset is None: raise exceptions.ObjectNotFound( "Dataset %s not found" % ( v[ 'id' ] ) ) input_data[k] = dataset.dataset_id https://bitbucket.org/galaxy/galaxy-central/commits/8d408c81c6c4/ Changeset: 8d408c81c6c4 Branch: job-search User: jmchilton Date: 2014-01-29 06:34:29 Summary: Add functional tests for new jobs API. Affected #: 1 file diff -r e20e9d60eb8017a241976912bfceb30dd316b20e -r 8d408c81c6c4abafce7461cab6c5e4b677e238d5 test/functional/api/test_jobs.py --- /dev/null +++ b/test/functional/api/test_jobs.py @@ -0,0 +1,118 @@ +import json +from operator import itemgetter + +from base import api + +from .helpers import TestsDatasets + + +class JobsApiTestCase( api.ApiTestCase, TestsDatasets ): + + def test_index( self ): + # Create HDA to ensure at least one job exists... + self.__history_with_new_dataset() + jobs_response = self._get( "jobs" ) + + self._assert_status_code_is( jobs_response, 200 ) + + jobs = jobs_response.json() + assert isinstance( jobs, list ) + assert "upload1" in map( itemgetter( "tool_id" ), jobs ) + + def test_index_state_filter( self ): + # Initial number of ok jobs + original_count = len( self.__uploads_with_state( "ok" ) ) + + # Run through dataset upload to ensure num uplaods at least greater + # by 1. + self.__history_with_ok_dataset() + + # Verify number of ok jobs is actually greater. + new_count = len( self.__uploads_with_state( "ok" ) ) + assert original_count < new_count + + def test_index_multiple_states_filter( self ): + # Initial number of ok jobs + original_count = len( self.__uploads_with_state( "ok", "new" ) ) + + # Run through dataset upload to ensure num uplaods at least greater + # by 1. + self.__history_with_ok_dataset() + + # Verify number of ok jobs is actually greater. + new_count = len( self.__uploads_with_state( "new", "ok" ) ) + assert original_count < new_count, new_count + + def test_show( self ): + # Create HDA to ensure at least one job exists... + self.__history_with_new_dataset() + + jobs_response = self._get( "jobs" ) + first_job = jobs_response.json()[ 0 ] + self._assert_has_key( first_job, 'id', 'state', 'exit_code', 'update_time', 'create_time' ) + + job_id = first_job[ "id" ] + show_jobs_response = self._get( "jobs/%s" % job_id ) + self._assert_status_code_is( show_jobs_response, 200 ) + + job_details = show_jobs_response.json() + self._assert_has_key( job_details, 'id', 'state', 'exit_code', 'update_time', 'create_time' ) + + def test_search( self ): + history_id, dataset_id = self.__history_with_ok_dataset() + + inputs = json.dumps( + dict( + input1=dict( + src='hda', + id=dataset_id, + ) + ) + ) + search_payload = dict( + tool_id="cat1", + inputs=inputs, + state="ok", + ) + + empty_search_response = self._post( "jobs/search", data=search_payload ) + self._assert_status_code_is( empty_search_response, 200 ) + assert len( empty_search_response.json() ) == 0 + + self.__run_cat_tool( history_id, dataset_id ) + self._wait_for_history( history_id, assert_ok=True ) + + search_response = self._post( "jobs/search", data=search_payload ) + self._assert_status_code_is( empty_search_response, 200 ) + assert len( search_response.json() ) == 1, search_response.json() + + def __run_cat_tool( self, history_id, dataset_id ): + # Code duplication with test_jobs.py, eliminate + payload = self._run_tool_payload( + tool_id='cat1', + inputs=dict( + input1=dict( + src='hda', + id=dataset_id + ), + ), + history_id=history_id, + ) + self._post( "tools", data=payload ) + + def __uploads_with_state( self, *states ): + jobs_response = self._get( "jobs", data=dict( state=states ) ) + self._assert_status_code_is( jobs_response, 200 ) + jobs = jobs_response.json() + assert not filter( lambda j: j[ "state" ] not in states, jobs ) + return filter( lambda j: j[ "tool_id" ] == "upload1", jobs ) + + def __history_with_new_dataset( self ): + history_id = self._new_history() + dataset_id = self._new_dataset( history_id )[ "id" ] + return history_id, dataset_id + + def __history_with_ok_dataset( self ): + history_id, dataset_id = self.__history_with_new_dataset() + self._wait_for_history( history_id, assert_ok=True ) + return history_id, dataset_id https://bitbucket.org/galaxy/galaxy-central/commits/6d8959c8866c/ Changeset: 6d8959c8866c User: jmchilton Date: 2014-01-29 06:57:46 Summary: Merge job-search branch. Affected #: 6 files diff -r d3bc7ea60b05f3025de3dca2684f1bce0be2c0eb -r 6d8959c8866c699ee0e9461db2a1fcc0381bfac2 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -211,8 +211,8 @@ class Job( object, Dictifiable ): - dict_collection_visible_keys = [ 'id', 'state', 'exit_code' ] - dict_element_visible_keys = [ 'id', 'state', 'exit_code' ] + dict_collection_visible_keys = [ 'id', 'state', 'exit_code', 'update_time', 'create_time' ] + dict_element_visible_keys = [ 'id', 'state', 'exit_code', 'update_time', 'create_time' ] """ A job represents a request to run a tool given input datasets, tool @@ -418,30 +418,31 @@ dataset.info = 'Job output deleted by user before job completed' def to_dict( self, view='collection' ): rval = super( Job, self ).to_dict( view=view ) - rval['tool_name'] = self.tool_id - param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) - rval['params'] = param_dict + rval['tool_id'] = self.tool_id + if view == 'element': + param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] ) + rval['params'] = param_dict - input_dict = {} - for i in self.input_datasets: - if i.dataset is not None: - input_dict[i.name] = {"hda_id" : i.dataset.id} - for i in self.input_library_datasets: - if i.dataset is not None: - input_dict[i.name] = {"ldda_id" : i.dataset.id} - for k in input_dict: - if k in param_dict: - del param_dict[k] - rval['inputs'] = input_dict + input_dict = {} + for i in self.input_datasets: + if i.dataset is not None: + input_dict[i.name] = {"id" : i.dataset.id, "src" : "hda"} + for i in self.input_library_datasets: + if i.dataset is not None: + input_dict[i.name] = {"id" : i.dataset.id, "src" : "ldda"} + for k in input_dict: + if k in param_dict: + del param_dict[k] + rval['inputs'] = input_dict - output_dict = {} - for i in self.output_datasets: - if i.dataset is not None: - output_dict[i.name] = {"hda_id" : i.dataset.id} - for i in self.output_library_datasets: - if i.dataset is not None: - output_dict[i.name] = {"ldda_id" : i.dataset.id} - rval['outputs'] = output_dict + output_dict = {} + for i in self.output_datasets: + if i.dataset is not None: + output_dict[i.name] = {"id" : i.dataset.id, "src" : "hda"} + for i in self.output_library_datasets: + if i.dataset is not None: + output_dict[i.name] = {"id" : i.dataset.id, "src" : "ldda"} + rval['outputs'] = output_dict return rval diff -r d3bc7ea60b05f3025de3dca2684f1bce0be2c0eb -r 6d8959c8866c699ee0e9461db2a1fcc0381bfac2 lib/galaxy/web/base/controller.py --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -160,7 +160,7 @@ if type( rval ) != dict: return rval for k, v in rval.items(): - if (k == 'id' or k.endswith( '_id' )) and v is not None: + if (k == 'id' or k.endswith( '_id' )) and v is not None and k not in ['tool_id']: try: rval[k] = trans.security.encode_id( v ) except: diff -r d3bc7ea60b05f3025de3dca2684f1bce0be2c0eb -r 6d8959c8866c699ee0e9461db2a1fcc0381bfac2 lib/galaxy/webapps/galaxy/api/jobs.py --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -0,0 +1,192 @@ +""" +API operations on a jobs. + +.. seealso:: :class:`galaxy.model.Jobs` +""" + +from sqlalchemy import or_, and_ +from sqlalchemy.orm import aliased +import json +from galaxy.web import _future_expose_api as expose_api +from galaxy.web.base.controller import BaseAPIController +from galaxy.web.base.controller import UsesHistoryDatasetAssociationMixin +from galaxy.web.base.controller import UsesLibraryMixinItems +from galaxy import exceptions + +import logging +log = logging.getLogger( __name__ ) + + +class JobController( BaseAPIController, UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems ): + + @expose_api + def index( self, trans, **kwd ): + """ + index( trans, state=None ) + * GET /api/jobs: + return jobs for current user + + :type state: string or list + :param state: limit listing of jobs to those that match one of the included states. If none, all are returned. + Valid Galaxy job states include: + 'new', 'upload', 'waiting', 'queued', 'running', 'ok', 'error', 'paused', 'deleted', 'deleted_new' + + :rtype: list + :returns: list of dictionaries containing summary job information + """ + + state = kwd.get( 'state', None ) + query = trans.sa_session.query( trans.app.model.Job ).filter( + trans.app.model.Job.user == trans.user + ) + if state is not None: + if isinstance( state, basestring ): + query = query.filter( trans.app.model.Job.state == state ) + elif isinstance( state, list ): + t = [] + for s in state: + t.append( trans.app.model.Job.state == s ) + query = query.filter( or_( *t ) ) + + out = [] + for job in query.order_by( + trans.app.model.Job.update_time.desc() + ).all(): + out.append( self.encode_all_ids( trans, job.to_dict( 'collection' ), True ) ) + return out + + @expose_api + def show( self, trans, id, **kwd ): + """ + show( trans, id ) + * GET /api/jobs/{job_id}: + return jobs for current user + + :type id: string + :param id: Specific job id + + :rtype: dictionary + :returns: dictionary containing full description of job data + """ + try: + decoded_job_id = trans.security.decode_id(id) + except: + raise exceptions.ObjectAttributeInvalidException() + query = trans.sa_session.query( trans.app.model.Job ).filter( + trans.app.model.Job.user == trans.user, + trans.app.model.Job.id == decoded_job_id + ) + job = query.first() + if job is None: + raise exceptions.ObjectNotFound() + return self.encode_all_ids( trans, job.to_dict( 'element' ), True ) + + @expose_api + def create( self, trans, payload, **kwd ): + raise NotImplementedError() + + @expose_api + def search( self, trans, payload, **kwd ): + """ + search( trans, payload ) + * POST /api/jobs/search: + return jobs for current user + + :type payload: dict + :param payload: Dictionary containing description of requested job. This is in the same format as + a request to POST /apt/tools would take to initiate a job + + :rtype: list + :returns: list of dictionaries containing summary job information of the jobs that match the requested job run + + This method is designed to scan the list of previously run jobs and find records of jobs that had + the exact some input parameters and datasets. This can be used to minimize the amount of repeated work, and simply + recycle the old results. + """ + + tool_id = None + if 'tool_id' in payload: + tool_id = payload.get( 'tool_id' ) + if tool_id is None: + raise exceptions.ObjectAttributeMissingException( "No tool id" ) + + tool = trans.app.toolbox.get_tool( tool_id ) + if tool is None: + raise exceptions.ObjectNotFound( "Requested tool not found" ) + if 'inputs' not in payload: + raise exceptions.ObjectAttributeMissingException( "No inputs defined" ) + + inputs = payload[ 'inputs' ] + + input_data = {} + input_param = {} + for k, v in inputs.items(): + if isinstance( v, dict ): + if 'id' in v: + if 'src' not in v or v[ 'src' ] == 'hda': + dataset = self.get_dataset( trans, v['id'], check_ownership=False, check_accessible=True ) + else: + dataset = self.get_library_dataset_dataset_association( trans, v['id'] ) + if dataset is None: + raise exceptions.ObjectNotFound( "Dataset %s not found" % ( v[ 'id' ] ) ) + input_data[k] = dataset.dataset_id + else: + input_param[k] = json.dumps( v ) + + query = trans.sa_session.query( trans.app.model.Job ).filter( + trans.app.model.Job.tool_id == tool_id, + trans.app.model.Job.user == trans.user + ) + + if 'state' not in payload: + query = query.filter( + or_( + trans.app.model.Job.state == 'running', + trans.app.model.Job.state == 'queued', + trans.app.model.Job.state == 'waiting', + trans.app.model.Job.state == 'running', + trans.app.model.Job.state == 'ok', + ) + ) + else: + if isinstance( payload[ 'state' ], basestring ): + query = query.filter( trans.app.model.Job.state == payload[ 'state' ] ) + elif isinstance( payload[ 'state' ], list ): + o = [] + for s in payload[ 'state' ]: + o.append( trans.app.model.Job.state == s ) + query = query.filter( + or_( *o ) + ) + + for k, v in input_param.items(): + a = aliased( trans.app.model.JobParameter ) + query = query.filter( and_( + trans.app.model.Job.id == a.job_id, + a.name == k, + a.value == v + ) ) + + for k, v in input_data.items(): + """ + Here we are attempting to link the inputs to the underlying dataset (not the dataset association) + This way, if the calulation was done using a copied HDA (copied from the library or another history) + the search will still find the job + """ + a = aliased( trans.app.model.JobToInputDatasetAssociation ) + b = aliased( trans.app.model.HistoryDatasetAssociation ) + query = query.filter( and_( + trans.app.model.Job.id == a.job_id, + a.dataset_id == b.id, + b.deleted == False, + b.dataset_id == v + ) ) + + out = [] + for job in query.all(): + """ + check to make sure none of the output files have been deleted + """ + if all( list( a.dataset.deleted == False for a in job.output_datasets ) ): + out.append( self.encode_all_ids( trans, job.to_dict( 'element' ), True ) ) + return out diff -r d3bc7ea60b05f3025de3dca2684f1bce0be2c0eb -r 6d8959c8866c699ee0e9461db2a1fcc0381bfac2 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -228,6 +228,12 @@ path_prefix='/api/libraries/:library_id', parent_resources=dict( member_name='library', collection_name='libraries' ) ) + webapp.mapper.resource( 'job', + 'jobs', + path_prefix='/api' ) + webapp.mapper.connect( 'job_search', '/api/jobs/search', controller='jobs', action='search', conditions=dict( method=['POST'] ) ) + + _add_item_extended_metadata_controller( webapp, name_prefix="library_dataset_", path_prefix='/api/libraries/:library_id/contents/:library_content_id' ) diff -r d3bc7ea60b05f3025de3dca2684f1bce0be2c0eb -r 6d8959c8866c699ee0e9461db2a1fcc0381bfac2 test/functional/api/test_jobs.py --- /dev/null +++ b/test/functional/api/test_jobs.py @@ -0,0 +1,118 @@ +import json +from operator import itemgetter + +from base import api + +from .helpers import TestsDatasets + + +class JobsApiTestCase( api.ApiTestCase, TestsDatasets ): + + def test_index( self ): + # Create HDA to ensure at least one job exists... + self.__history_with_new_dataset() + jobs_response = self._get( "jobs" ) + + self._assert_status_code_is( jobs_response, 200 ) + + jobs = jobs_response.json() + assert isinstance( jobs, list ) + assert "upload1" in map( itemgetter( "tool_id" ), jobs ) + + def test_index_state_filter( self ): + # Initial number of ok jobs + original_count = len( self.__uploads_with_state( "ok" ) ) + + # Run through dataset upload to ensure num uplaods at least greater + # by 1. + self.__history_with_ok_dataset() + + # Verify number of ok jobs is actually greater. + new_count = len( self.__uploads_with_state( "ok" ) ) + assert original_count < new_count + + def test_index_multiple_states_filter( self ): + # Initial number of ok jobs + original_count = len( self.__uploads_with_state( "ok", "new" ) ) + + # Run through dataset upload to ensure num uplaods at least greater + # by 1. + self.__history_with_ok_dataset() + + # Verify number of ok jobs is actually greater. + new_count = len( self.__uploads_with_state( "new", "ok" ) ) + assert original_count < new_count, new_count + + def test_show( self ): + # Create HDA to ensure at least one job exists... + self.__history_with_new_dataset() + + jobs_response = self._get( "jobs" ) + first_job = jobs_response.json()[ 0 ] + self._assert_has_key( first_job, 'id', 'state', 'exit_code', 'update_time', 'create_time' ) + + job_id = first_job[ "id" ] + show_jobs_response = self._get( "jobs/%s" % job_id ) + self._assert_status_code_is( show_jobs_response, 200 ) + + job_details = show_jobs_response.json() + self._assert_has_key( job_details, 'id', 'state', 'exit_code', 'update_time', 'create_time' ) + + def test_search( self ): + history_id, dataset_id = self.__history_with_ok_dataset() + + inputs = json.dumps( + dict( + input1=dict( + src='hda', + id=dataset_id, + ) + ) + ) + search_payload = dict( + tool_id="cat1", + inputs=inputs, + state="ok", + ) + + empty_search_response = self._post( "jobs/search", data=search_payload ) + self._assert_status_code_is( empty_search_response, 200 ) + assert len( empty_search_response.json() ) == 0 + + self.__run_cat_tool( history_id, dataset_id ) + self._wait_for_history( history_id, assert_ok=True ) + + search_response = self._post( "jobs/search", data=search_payload ) + self._assert_status_code_is( empty_search_response, 200 ) + assert len( search_response.json() ) == 1, search_response.json() + + def __run_cat_tool( self, history_id, dataset_id ): + # Code duplication with test_jobs.py, eliminate + payload = self._run_tool_payload( + tool_id='cat1', + inputs=dict( + input1=dict( + src='hda', + id=dataset_id + ), + ), + history_id=history_id, + ) + self._post( "tools", data=payload ) + + def __uploads_with_state( self, *states ): + jobs_response = self._get( "jobs", data=dict( state=states ) ) + self._assert_status_code_is( jobs_response, 200 ) + jobs = jobs_response.json() + assert not filter( lambda j: j[ "state" ] not in states, jobs ) + return filter( lambda j: j[ "tool_id" ] == "upload1", jobs ) + + def __history_with_new_dataset( self ): + history_id = self._new_history() + dataset_id = self._new_dataset( history_id )[ "id" ] + return history_id, dataset_id + + def __history_with_ok_dataset( self ): + history_id, dataset_id = self.__history_with_new_dataset() + self._wait_for_history( history_id, assert_ok=True ) + return history_id, dataset_id https://bitbucket.org/galaxy/galaxy-central/commits/1dd912d65cfb/ Changeset: 1dd912d65cfb Branch: job-search User: jmchilton Date: 2014-01-29 06:58:04 Summary: Close branch job-search. Affected #: 0 files https://bitbucket.org/galaxy/galaxy-central/commits/60203bad003c/ Changeset: 60203bad003c User: jmchilton Date: 2014-01-29 07:05:15 Summary: Merge. Affected #: 3 files diff -r 6d8959c8866c699ee0e9461db2a1fcc0381bfac2 -r 60203bad003cc6b4bc054500248e5a69cbe9b8bc lib/galaxy/tools/imp_exp/__init__.py --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -1,4 +1,8 @@ -import os, shutil, logging, tempfile, json +import os +import shutil +import logging +import tempfile +import json from galaxy import model from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.web.framework.helpers import to_unicode @@ -8,6 +12,7 @@ log = logging.getLogger(__name__) + def load_history_imp_exp_tools( toolbox ): """ Adds tools for importing/exporting histories to archives. """ # Use same process as that used in load_external_metadata_tool; see that @@ -42,6 +47,7 @@ toolbox.tools_by_id[ history_imp_tool.id ] = history_imp_tool log.debug( "Loaded history import tool: %s", history_imp_tool.id ) + class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations ): """ Class provides support for performing jobs that import a history from @@ -144,23 +150,23 @@ metadata = dataset_attrs['metadata'] # Create dataset and HDA. - hda = model.HistoryDatasetAssociation( name = dataset_attrs['name'].encode( 'utf-8' ), - extension = dataset_attrs['extension'], - info = dataset_attrs['info'].encode( 'utf-8' ), - blurb = dataset_attrs['blurb'], - peek = dataset_attrs['peek'], - designation = dataset_attrs['designation'], - visible = dataset_attrs['visible'], - dbkey = metadata['dbkey'], - metadata = metadata, - history = new_history, - create_dataset = True, - sa_session = self.sa_session ) + hda = model.HistoryDatasetAssociation( name=dataset_attrs['name'].encode( 'utf-8' ), + extension=dataset_attrs['extension'], + info=dataset_attrs['info'].encode( 'utf-8' ), + blurb=dataset_attrs['blurb'], + peek=dataset_attrs['peek'], + designation=dataset_attrs['designation'], + visible=dataset_attrs['visible'], + dbkey=metadata['dbkey'], + metadata=metadata, + history=new_history, + create_dataset=True, + sa_session=self.sa_session ) hda.state = hda.states.OK self.sa_session.add( hda ) self.sa_session.flush() - new_history.add_dataset( hda, genome_build = None ) - hda.hid = dataset_attrs['hid'] # Overwrite default hid set when HDA added to history. + new_history.add_dataset( hda, genome_build=None ) + hda.hid = dataset_attrs['hid'] # Overwrite default hid set when HDA added to history. # TODO: Is there a way to recover permissions? Is this needed? #permissions = trans.app.security_agent.history_get_default_permissions( new_history ) #trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions ) @@ -273,6 +279,7 @@ jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() + class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations ): """ Class provides support for performing jobs that export a history to an @@ -317,23 +324,23 @@ """ Encode an HDA, default encoding for everything else. """ if isinstance( obj, trans.app.model.HistoryDatasetAssociation ): return { - "__HistoryDatasetAssociation__" : True, - "create_time" : obj.create_time.__str__(), - "update_time" : obj.update_time.__str__(), - "hid" : obj.hid, - "name" : to_unicode( obj.name ), - "info" : to_unicode( obj.info ), - "blurb" : obj.blurb, - "peek" : obj.peek, - "extension" : obj.extension, - "metadata" : prepare_metadata( dict( obj.metadata.items() ) ), - "parent_id" : obj.parent_id, - "designation" : obj.designation, - "deleted" : obj.deleted, - "visible" : obj.visible, - "file_name" : obj.file_name, - "annotation" : to_unicode( getattr( obj, 'annotation', '' ) ), - "tags" : get_item_tag_dict( obj ), + "__HistoryDatasetAssociation__": True, + "create_time": obj.create_time.__str__(), + "update_time": obj.update_time.__str__(), + "hid": obj.hid, + "name": to_unicode( obj.name ), + "info": to_unicode( obj.info ), + "blurb": obj.blurb, + "peek": obj.peek, + "extension": obj.extension, + "metadata": prepare_metadata( dict( obj.metadata.items() ) ), + "parent_id": obj.parent_id, + "designation": obj.designation, + "deleted": obj.deleted, + "visible": obj.visible, + "file_name": obj.file_name, + "annotation": to_unicode( getattr( obj, 'annotation', '' ) ), + "tags": get_item_tag_dict( obj ), } if isinstance( obj, UnvalidatedValue ): return obj.__str__() @@ -347,15 +354,15 @@ # Write history attributes to file. history = jeha.history history_attrs = { - "create_time" : history.create_time.__str__(), - "update_time" : history.update_time.__str__(), - "name" : to_unicode( history.name ), - "hid_counter" : history.hid_counter, - "genome_build" : history.genome_build, - "annotation" : to_unicode( self.get_item_annotation_str( trans.sa_session, history.user, history ) ), - "tags" : get_item_tag_dict( history ), - "includes_hidden_datasets" : include_hidden, - "includes_deleted_datasets" : include_deleted + "create_time": history.create_time.__str__(), + "update_time": history.update_time.__str__(), + "name": to_unicode( history.name ), + "hid_counter": history.hid_counter, + "genome_build": history.genome_build, + "annotation": to_unicode( self.get_item_annotation_str( trans.sa_session, history.user, history ) ), + "tags": get_item_tag_dict( history ), + "includes_hidden_datasets": include_hidden, + "includes_deleted_datasets": include_deleted } history_attrs_filename = tempfile.NamedTemporaryFile( dir=temp_output_dir ).name history_attrs_out = open( history_attrs_filename, 'w' ) @@ -391,7 +398,7 @@ # Get the associated job, if any. If this hda was copied from another, # we need to find the job that created the origial hda job_hda = hda - while job_hda.copied_from_history_dataset_association: #should this check library datasets as well? + while job_hda.copied_from_history_dataset_association: # should this check library datasets as well? job_hda = job_hda.copied_from_history_dataset_association if not job_hda.creating_job_associations: # No viable HDA found. @@ -472,4 +479,3 @@ shutil.rmtree( temp_dir ) except Exception, e: log.debug( 'Error deleting directory containing attribute files (%s): %s' % ( temp_dir, e ) ) - diff -r 6d8959c8866c699ee0e9461db2a1fcc0381bfac2 -r 60203bad003cc6b4bc054500248e5a69cbe9b8bc lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -6,19 +6,25 @@ --[url|file] source type, either a URL or a file. """ -import sys, optparse, tarfile, tempfile, urllib2, math +import sys +import optparse +import tarfile +import tempfile +import urllib2 +import math # Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. MAX_SIZE = 100 * math.pow( 2, 30 ) + def url_to_file( url, dest_file ): """ Transfer a file from a remote URL to a temporary file. """ try: url_reader = urllib2.urlopen( url ) - CHUNK = 10 * 1024 # 10k + CHUNK = 10 * 1024 # 10k total = 0 fp = open( dest_file, 'wb') while True: @@ -35,6 +41,7 @@ print "Exception getting file from URL: %s" % e, sys.stderr return None + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -63,4 +70,4 @@ # Unpack archive. unpack_archive( archive_file, dest_dir ) except Exception, e: - print "Error unpacking tar/gz archive: %s" % e, sys.stderr \ No newline at end of file + print "Error unpacking tar/gz archive: %s" % e, sys.stderr diff -r 6d8959c8866c699ee0e9461db2a1fcc0381bfac2 -r 60203bad003cc6b4bc054500248e5a69cbe9b8bc lib/galaxy/webapps/galaxy/api/histories.py --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -11,15 +11,17 @@ from galaxy import web from galaxy.web import _future_expose_api as expose_api from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous -from galaxy.util import string_as_bool, restore_text -from galaxy.util.sanitize_html import sanitize_html -from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesTagsMixin +from galaxy.util import string_as_bool +from galaxy.util import restore_text +from galaxy.web.base.controller import BaseAPIController +from galaxy.web.base.controller import UsesHistoryMixin +from galaxy.web.base.controller import UsesTagsMixin from galaxy.web import url_for -from galaxy.model.orm import desc import logging log = logging.getLogger( __name__ ) + class HistoriesController( BaseAPIController, UsesHistoryMixin, UsesTagsMixin ): @expose_api_anonymous @@ -46,14 +48,14 @@ histories = self.get_user_histories( trans, user=trans.user, only_deleted=deleted ) #for history in query: for history in histories: - item = history.to_dict(value_mapper={'id':trans.security.encode_id}) + item = history.to_dict(value_mapper={'id': trans.security.encode_id}) item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) rval.append( item ) elif trans.galaxy_session.current_history: #No user, this must be session authentication with an anonymous user. history = trans.galaxy_session.current_history - item = history.to_dict(value_mapper={'id':trans.security.encode_id}) + item = history.to_dict(value_mapper={'id': trans.security.encode_id}) item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) rval.append(item) @@ -257,7 +259,7 @@ log.exception( 'Histories API, delete: uncaught HTTPInternalServerError: %s, %s\n%s', history_id, str( kwd ), str( http_server_err ) ) raise - except HTTPException, http_exc: + except HTTPException: raise except Exception, exc: log.exception( 'Histories API, delete: uncaught exception: %s, %s\n%s', Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.