- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: natefoo: Only include the username in the job name when not running as a system user.
by Bitbucket 03 Oct '12

03 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/e4e1c621b025/ changeset: e4e1c621b025 user: natefoo date: 2012-10-03 19:43:38 summary: Only include the username in the job name when not running as a system user. affected #: 1 file diff -r f16e60e79638a9ebe18f9daedb3d82ce405a1986 -r e4e1c621b025b4f0b09846e9d4410d2630394470 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -184,7 +184,9 @@ ofile = "%s.drmout" % os.path.join(job_wrapper.working_directory, job_wrapper.get_id_tag()) efile = "%s.drmerr" % os.path.join(job_wrapper.working_directory, job_wrapper.get_id_tag()) ecfile = "%s.drmec" % os.path.join(job_wrapper.working_directory, job_wrapper.get_id_tag()) - job_name = "g%s_%s_%s" % ( job_wrapper.job_id, job_wrapper.tool.id, job_wrapper.user ) + job_name = "g%s_%s" % ( job_wrapper.job_id, job_wrapper.tool.id ) + if self.external_runJob_script is None: + job_name = "%s_%s" % ( job_name, job_wrapper.user ) job_name = ''.join( map( lambda x: x if x in ( string.letters + string.digits + '_' ) else '_', job_name ) ) jt = self.ds.createJobTemplate() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Fix reporting in the set_user_disk_usage.py script.
by Bitbucket 03 Oct '12

03 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/f16e60e79638/ changeset: f16e60e79638 user: natefoo date: 2012-10-03 19:40:54 summary: Fix reporting in the set_user_disk_usage.py script. affected #: 1 file diff -r 82ef383af0e1135d300afe67aa767f73a3a781ae -r f16e60e79638a9ebe18f9daedb3d82ce405a1986 scripts/set_user_disk_usage.py --- a/scripts/set_user_disk_usage.py +++ b/scripts/set_user_disk_usage.py @@ -90,10 +90,10 @@ if new in ( current, None ): print 'none' else: - op = '-' if new > current: - op = '+' - print '%s%s' % ( op, nice_size( new ) ) + print '+%s' % ( nice_size( new - current ) ) + else: + print '-%s' % ( nice_size( current - new ) ) if not options.dryrun and engine != 'postgres': user.set_disk_usage( new ) sa_session.add( user ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Remove BWA from Main tool_conf.
by Bitbucket 03 Oct '12

03 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/82ef383af0e1/ changeset: 82ef383af0e1 user: natefoo date: 2012-10-03 19:05:35 summary: Remove BWA from Main tool_conf. affected #: 1 file diff -r 62fbfc687692dc173a9db4c9f6b86f1691afb281 -r 82ef383af0e1135d300afe67aa767f73a3a781ae tool_conf.xml.main --- a/tool_conf.xml.main +++ b/tool_conf.xml.main @@ -257,14 +257,12 @@ <section name="NGS: Mapping" id="ngs_mapping"><label text="Illumina" id="illumina"/><tool file="sr_mapping/bowtie_wrapper.xml" /> - <tool file="sr_mapping/bwa_wrapper.xml" /><label text="Roche-454" id="roche_454"/><tool file="sr_mapping/lastz_wrapper.xml" /><tool file="metag_tools/megablast_wrapper.xml" /><tool file="metag_tools/megablast_xml_parser.xml" /><label text="AB-SOLiD" id="ab_solid"/><tool file="sr_mapping/bowtie_color_wrapper.xml" /> - <tool file="sr_mapping/bwa_color_wrapper.xml" /></section><section name="NGS: SAM Tools" id="samtools"><tool file="samtools/sam_bitwise_flag_filter.xml" /> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: natefoo: Fix passing of job name when running jobs as a system user.
by Bitbucket 03 Oct '12

03 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/62fbfc687692/ changeset: 62fbfc687692 user: natefoo date: 2012-10-03 17:51:07 summary: Fix passing of job name when running jobs as a system user. affected #: 2 files diff -r 5b6ed441a320a133a6931fab43227f40321a53b0 -r 62fbfc687692dc173a9db4c9f6b86f1691afb281 lib/galaxy/jobs/runners/drmaa.py --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -68,7 +68,7 @@ return inspect.currentframe().f_back.f_code.co_filename DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', - 'name','email','project' ] + 'jobName','email','project' ] class DRMAAJobState( object ): def __init__( self ): diff -r 5b6ed441a320a133a6931fab43227f40321a53b0 -r 62fbfc687692dc173a9db4c9f6b86f1691afb281 scripts/drmaa_external_runner.py --- a/scripts/drmaa_external_runner.py +++ b/scripts/drmaa_external_runner.py @@ -25,7 +25,7 @@ import drmaa DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', - 'name','email','project' ] + 'jobName','email','project' ] def load_job_template_from_file(jt, filename): f = open(filename,'r') Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by Bitbucket 03 Oct '12

03 Oct '12

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/74c2c49d5180/ changeset: 74c2c49d5180 user: jmchilton date: 2012-09-20 06:41:22 summary: Allow tool shed upload URLs that start with hg:// or hgs://. For URLs of this form, replace hg with http (or hgs with https) and copy external mercurial repository contents into tool shed repository essentially as if it were a tar file. The tool shed doesn't display a mercurial URL to push to until an initial upload has occurred and pasting in bitbucket download URLs (e.g. https://bitbucket.org/<user>/<repo>/get/<rev>.tar.gz) result in an extra directory being added to the top level of the tool shed repository. Hence this mechanism enables the rapid creation of a tool shed repository from an existing bitbucket repository. affected #: 1 file diff -r be1f6ebb8d6bd3864e5ca08c59f8edaa4f471004 -r 74c2c49d51808d0f3f8a4b41b385be2d38245bc3 lib/galaxy/webapps/community/controllers/upload.py --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -42,11 +42,19 @@ # receive them. One scenario occurs when the first change set is produced for the repository. # See the handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new + uploaded_directory = None if params.get( 'upload_button', False ): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None + elif url and url.startswith("hg"): + # Use mercurial clone to fetch repository, contents will then + # be copied over. + uploaded_directory = tempfile.mkdtemp() + repo_url = "http%s" % url[len("hg"):] + repo_url = repo_url.encode('ascii', 'replace') + commands.clone(get_configured_ui(), repo_url, uploaded_directory) elif url: valid_url = True try: @@ -72,29 +80,34 @@ uploaded_file_name = uploaded_file.name uploaded_file_filename = file_data.filename isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0 - if uploaded_file: + if uploaded_file or uploaded_directory: + ok = True isgzip = False isbz2 = False - if uncompress_file: - isgzip = is_gzip( uploaded_file_name ) - if not isgzip: - isbz2 = is_bz2( uploaded_file_name ) - ok = True - if isempty: - tar = None - istar = False - else: - # Determine what we have - a single file or an archive - try: - if ( isgzip or isbz2 ) and uncompress_file: - # Open for reading with transparent compression. - tar = tarfile.open( uploaded_file_name, 'r:*' ) - else: - tar = tarfile.open( uploaded_file_name ) - istar = True - except tarfile.ReadError, e: + if uploaded_file: + + if uncompress_file: + isgzip = is_gzip( uploaded_file_name ) + if not isgzip: + isbz2 = is_bz2( uploaded_file_name ) + if isempty: tar = None istar = False + else: + # Determine what we have - a single file or an archive + try: + if ( isgzip or isbz2 ) and uncompress_file: + # Open for reading with transparent compression. + tar = tarfile.open( uploaded_file_name, 'r:*' ) + else: + tar = tarfile.open( uploaded_file_name ) + istar = True + except tarfile.ReadError, e: + tar = None + istar = False + else: + # Uploaded directory + istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = self.upload_tar( trans, repository, @@ -104,6 +117,14 @@ remove_repo_files_not_in_tar, commit_message, new_repo_alert ) + elif uploaded_directory: + ok,message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = self.upload_directory( trans, + repository, + uploaded_directory, + upload_point, + remove_repo_files_not_in_tar, + commit_message, + new_repo_alert ) else: if ( isgzip or isbz2 ) and uncompress_file: uploaded_file_filename = self.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ) @@ -144,7 +165,13 @@ uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' - message = "The file '%s' has been successfully%suploaded to the repository. " % ( uploaded_file_filename, uncompress_str ) + if uploaded_directory: + source_type = "repository" + source = url + else: + source_type = "file" + source = uploaded_file_filename + message = "The %s '%s' has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str ) if istar and ( undesirable_dirs_removed or undesirable_files_removed ): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed @@ -177,19 +204,54 @@ remove_repo_files_not_in_tar=remove_repo_files_not_in_tar, message=message, status=status ) + def upload_directory( self, trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + undesirable_dirs_removed = 0 + undesirable_files_removed = 0 + + if upload_point is not None: + full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) + else: + full_path = os.path.abspath( repo_dir ) + + filenames_in_archive = [] + for root, dirs, files in os.walk( uploaded_directory ): + for uploaded_file in files: + relative_path = os.path.normpath(os.path.join(os.path.relpath(root, uploaded_directory), uploaded_file)) + ok = os.path.basename( uploaded_file ) not in undesirable_files + if ok: + for file_path_item in relative_path.split( '/' ): + if file_path_item in undesirable_dirs: + undesirable_dirs_removed += 1 + ok = False + break + else: + undesirable_files_removed += 1 + if ok: + repo_path = os.path.join(full_path, relative_path) + repo_basedir = os.path.normpath(os.path.join(repo_path, os.path.pardir)) + if not os.path.exists(repo_basedir): + os.makedirs(repo_basedir) + if os.path.exists(repo_path): + if os.path.isdir(repo_path): + shutil.rmtree(repo_path) + else: + os.remove(repo_path) + shutil.move(os.path.join(uploaded_directory, relative_path), repo_path) + filenames_in_archive.append( relative_path ) + return self.__handle_directory_changes(trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed) def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): # Upload a tar archive of files. repo_dir = repository.repo_path repo = hg.repository( get_configured_ui(), repo_dir ) - files_to_remove = [] - content_alert_str = '' undesirable_dirs_removed = 0 undesirable_files_removed = 0 ok, message = self.__check_archive( tar ) if not ok: tar.close() uploaded_file.close() - return ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed + return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed else: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) @@ -208,70 +270,76 @@ undesirable_files_removed += 1 if ok: filenames_in_archive.append( tarinfo_obj.name ) - filenames_in_archive = [ os.path.join( full_path, name ) for name in filenames_in_archive ] # Extract the uploaded tar to the load_point within the repository hierarchy. tar.extractall( path=full_path ) tar.close() uploaded_file.close() - if remove_repo_files_not_in_tar and not repository.is_new: - # We have a repository that is not new (it contains files), so discover - # those files that are in the repository, but not in the uploaded archive. - for root, dirs, files in os.walk( full_path ): - if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: - for undesirable_dir in undesirable_dirs: - if undesirable_dir in dirs: - dirs.remove( undesirable_dir ) - undesirable_dirs_removed += 1 - for undesirable_file in undesirable_files: - if undesirable_file in files: - files.remove( undesirable_file ) - undesirable_files_removed += 1 - for name in files: - full_name = os.path.join( root, name ) - if full_name not in filenames_in_archive: - files_to_remove.append( full_name ) - for repo_file in files_to_remove: - # Remove files in the repository (relative to the upload point) that are not in the uploaded archive. - try: - commands.remove( repo.ui, repo, repo_file, force=True ) - except Exception, e: - log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str( e )) - relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' ) - repo.dirstate.remove( relative_selected_file ) - repo.dirstate.write() - absolute_selected_file = os.path.abspath( selected_file ) - if os.path.isdir( absolute_selected_file ): - try: - os.rmdir( absolute_selected_file ) - except OSError, e: - # The directory is not empty - pass - elif os.path.isfile( absolute_selected_file ): - os.remove( absolute_selected_file ) - dir = os.path.split( absolute_selected_file )[0] - try: - os.rmdir( dir ) - except OSError, e: - # The directory is not empty - pass - # See if any admin users have chosen to receive email alerts when a repository is - # updated. If so, check every uploaded file to ensure content is appropriate. - check_contents = check_file_contents( trans ) - for filename_in_archive in filenames_in_archive: - # Check file content to ensure it is appropriate. - if check_contents and os.path.isfile( filename_in_archive ): - content_alert_str += self.__check_file_content( filename_in_archive ) - commands.add( repo.ui, repo, filename_in_archive ) - if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ): - # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries - # to the in-memory trans.app.tool_data_tables dictionary. - error, message = handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive ) - if error: - return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed - commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) - admin_only = len( repository.downloadable_revisions ) != 1 - handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) - return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed + return self.__handle_directory_changes(trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed) + def __handle_directory_changes( self, trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed ): + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + content_alert_str = '' + files_to_remove = [] + filenames_in_archive = [ os.path.join( full_path, name ) for name in filenames_in_archive ] + if remove_repo_files_not_in_tar and not repository.is_new: + # We have a repository that is not new (it contains files), so discover + # those files that are in the repository, but not in the uploaded archive. + for root, dirs, files in os.walk( full_path ): + if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: + for undesirable_dir in undesirable_dirs: + if undesirable_dir in dirs: + dirs.remove( undesirable_dir ) + undesirable_dirs_removed += 1 + for undesirable_file in undesirable_files: + if undesirable_file in files: + files.remove( undesirable_file ) + undesirable_files_removed += 1 + for name in files: + full_name = os.path.join( root, name ) + if full_name not in filenames_in_archive: + files_to_remove.append( full_name ) + for repo_file in files_to_remove: + # Remove files in the repository (relative to the upload point) that are not in the uploaded archive. + try: + commands.remove( repo.ui, repo, repo_file, force=True ) + except Exception, e: + log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str( e )) + relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' ) + repo.dirstate.remove( relative_selected_file ) + repo.dirstate.write() + absolute_selected_file = os.path.abspath( selected_file ) + if os.path.isdir( absolute_selected_file ): + try: + os.rmdir( absolute_selected_file ) + except OSError, e: + # The directory is not empty + pass + elif os.path.isfile( absolute_selected_file ): + os.remove( absolute_selected_file ) + dir = os.path.split( absolute_selected_file )[0] + try: + os.rmdir( dir ) + except OSError, e: + # The directory is not empty + pass + # See if any admin users have chosen to receive email alerts when a repository is + # updated. If so, check every uploaded file to ensure content is appropriate. + check_contents = check_file_contents( trans ) + for filename_in_archive in filenames_in_archive: + # Check file content to ensure it is appropriate. + if check_contents and os.path.isfile( filename_in_archive ): + content_alert_str += self.__check_file_content( filename_in_archive ) + commands.add( repo.ui, repo, filename_in_archive ) + if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ): + # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries + # to the in-memory trans.app.tool_data_tables dictionary. + error, message = handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive ) + if error: + return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed + commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) + admin_only = len( repository.downloadable_revisions ) != 1 + handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) + return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed def uncompress( self, repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ): if isgzip: self.__handle_gzip( repository, uploaded_file_name ) https://bitbucket.org/galaxy/galaxy-central/changeset/5b6ed441a320/ changeset: 5b6ed441a320 user: greg date: 2012-10-03 16:29:09 summary: Merged in jmchilton/galaxy-central-tool-shed-hg-urls (pull request #69) affected #: 1 file diff -r ae683c38bdf2aa61403dc220e91561945a855158 -r 5b6ed441a320a133a6931fab43227f40321a53b0 lib/galaxy/webapps/community/controllers/upload.py --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -42,11 +42,19 @@ # receive them. One scenario occurs when the first change set is produced for the repository. # See the handle_email_alerts() method for the definition of the scenarios. new_repo_alert = repository.is_new + uploaded_directory = None if params.get( 'upload_button', False ): if file_data == '' and url == '': message = 'No files were entered on the upload form.' status = 'error' uploaded_file = None + elif url and url.startswith("hg"): + # Use mercurial clone to fetch repository, contents will then + # be copied over. + uploaded_directory = tempfile.mkdtemp() + repo_url = "http%s" % url[len("hg"):] + repo_url = repo_url.encode('ascii', 'replace') + commands.clone(get_configured_ui(), repo_url, uploaded_directory) elif url: valid_url = True try: @@ -72,29 +80,34 @@ uploaded_file_name = uploaded_file.name uploaded_file_filename = file_data.filename isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0 - if uploaded_file: + if uploaded_file or uploaded_directory: + ok = True isgzip = False isbz2 = False - if uncompress_file: - isgzip = is_gzip( uploaded_file_name ) - if not isgzip: - isbz2 = is_bz2( uploaded_file_name ) - ok = True - if isempty: - tar = None - istar = False - else: - # Determine what we have - a single file or an archive - try: - if ( isgzip or isbz2 ) and uncompress_file: - # Open for reading with transparent compression. - tar = tarfile.open( uploaded_file_name, 'r:*' ) - else: - tar = tarfile.open( uploaded_file_name ) - istar = True - except tarfile.ReadError, e: + if uploaded_file: + + if uncompress_file: + isgzip = is_gzip( uploaded_file_name ) + if not isgzip: + isbz2 = is_bz2( uploaded_file_name ) + if isempty: tar = None istar = False + else: + # Determine what we have - a single file or an archive + try: + if ( isgzip or isbz2 ) and uncompress_file: + # Open for reading with transparent compression. + tar = tarfile.open( uploaded_file_name, 'r:*' ) + else: + tar = tarfile.open( uploaded_file_name ) + istar = True + except tarfile.ReadError, e: + tar = None + istar = False + else: + # Uploaded directory + istar = False if istar: ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = self.upload_tar( trans, repository, @@ -104,6 +117,14 @@ remove_repo_files_not_in_tar, commit_message, new_repo_alert ) + elif uploaded_directory: + ok,message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = self.upload_directory( trans, + repository, + uploaded_directory, + upload_point, + remove_repo_files_not_in_tar, + commit_message, + new_repo_alert ) else: if ( isgzip or isbz2 ) and uncompress_file: uploaded_file_filename = self.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ) @@ -144,7 +165,13 @@ uncompress_str = ' uncompressed and ' else: uncompress_str = ' ' - message = "The file '%s' has been successfully%suploaded to the repository. " % ( uploaded_file_filename, uncompress_str ) + if uploaded_directory: + source_type = "repository" + source = url + else: + source_type = "file" + source = uploaded_file_filename + message = "The %s '%s' has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str ) if istar and ( undesirable_dirs_removed or undesirable_files_removed ): items_removed = undesirable_dirs_removed + undesirable_files_removed message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed @@ -176,19 +203,54 @@ remove_repo_files_not_in_tar=remove_repo_files_not_in_tar, message=message, status=status ) + def upload_directory( self, trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + undesirable_dirs_removed = 0 + undesirable_files_removed = 0 + + if upload_point is not None: + full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) + else: + full_path = os.path.abspath( repo_dir ) + + filenames_in_archive = [] + for root, dirs, files in os.walk( uploaded_directory ): + for uploaded_file in files: + relative_path = os.path.normpath(os.path.join(os.path.relpath(root, uploaded_directory), uploaded_file)) + ok = os.path.basename( uploaded_file ) not in undesirable_files + if ok: + for file_path_item in relative_path.split( '/' ): + if file_path_item in undesirable_dirs: + undesirable_dirs_removed += 1 + ok = False + break + else: + undesirable_files_removed += 1 + if ok: + repo_path = os.path.join(full_path, relative_path) + repo_basedir = os.path.normpath(os.path.join(repo_path, os.path.pardir)) + if not os.path.exists(repo_basedir): + os.makedirs(repo_basedir) + if os.path.exists(repo_path): + if os.path.isdir(repo_path): + shutil.rmtree(repo_path) + else: + os.remove(repo_path) + shutil.move(os.path.join(uploaded_directory, relative_path), repo_path) + filenames_in_archive.append( relative_path ) + return self.__handle_directory_changes(trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed) def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ): # Upload a tar archive of files. repo_dir = repository.repo_path repo = hg.repository( get_configured_ui(), repo_dir ) - files_to_remove = [] - content_alert_str = '' undesirable_dirs_removed = 0 undesirable_files_removed = 0 ok, message = self.__check_archive( tar ) if not ok: tar.close() uploaded_file.close() - return ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed + return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed else: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) @@ -207,70 +269,76 @@ undesirable_files_removed += 1 if ok: filenames_in_archive.append( tarinfo_obj.name ) - filenames_in_archive = [ os.path.join( full_path, name ) for name in filenames_in_archive ] # Extract the uploaded tar to the load_point within the repository hierarchy. tar.extractall( path=full_path ) tar.close() uploaded_file.close() - if remove_repo_files_not_in_tar and not repository.is_new: - # We have a repository that is not new (it contains files), so discover - # those files that are in the repository, but not in the uploaded archive. - for root, dirs, files in os.walk( full_path ): - if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: - for undesirable_dir in undesirable_dirs: - if undesirable_dir in dirs: - dirs.remove( undesirable_dir ) - undesirable_dirs_removed += 1 - for undesirable_file in undesirable_files: - if undesirable_file in files: - files.remove( undesirable_file ) - undesirable_files_removed += 1 - for name in files: - full_name = os.path.join( root, name ) - if full_name not in filenames_in_archive: - files_to_remove.append( full_name ) - for repo_file in files_to_remove: - # Remove files in the repository (relative to the upload point) that are not in the uploaded archive. - try: - commands.remove( repo.ui, repo, repo_file, force=True ) - except Exception, e: - log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str( e )) - relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' ) - repo.dirstate.remove( relative_selected_file ) - repo.dirstate.write() - absolute_selected_file = os.path.abspath( selected_file ) - if os.path.isdir( absolute_selected_file ): - try: - os.rmdir( absolute_selected_file ) - except OSError, e: - # The directory is not empty - pass - elif os.path.isfile( absolute_selected_file ): - os.remove( absolute_selected_file ) - dir = os.path.split( absolute_selected_file )[0] - try: - os.rmdir( dir ) - except OSError, e: - # The directory is not empty - pass - # See if any admin users have chosen to receive email alerts when a repository is - # updated. If so, check every uploaded file to ensure content is appropriate. - check_contents = check_file_contents( trans ) - for filename_in_archive in filenames_in_archive: - # Check file content to ensure it is appropriate. - if check_contents and os.path.isfile( filename_in_archive ): - content_alert_str += self.__check_file_content( filename_in_archive ) - commands.add( repo.ui, repo, filename_in_archive ) - if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ): - # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries - # to the in-memory trans.app.tool_data_tables dictionary. - error, message = handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive ) - if error: - return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed - commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) - admin_only = len( repository.downloadable_revisions ) != 1 - handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) - return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed + return self.__handle_directory_changes(trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed) + def __handle_directory_changes( self, trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, undesirable_dirs_removed, undesirable_files_removed ): + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + content_alert_str = '' + files_to_remove = [] + filenames_in_archive = [ os.path.join( full_path, name ) for name in filenames_in_archive ] + if remove_repo_files_not_in_tar and not repository.is_new: + # We have a repository that is not new (it contains files), so discover + # those files that are in the repository, but not in the uploaded archive. + for root, dirs, files in os.walk( full_path ): + if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: + for undesirable_dir in undesirable_dirs: + if undesirable_dir in dirs: + dirs.remove( undesirable_dir ) + undesirable_dirs_removed += 1 + for undesirable_file in undesirable_files: + if undesirable_file in files: + files.remove( undesirable_file ) + undesirable_files_removed += 1 + for name in files: + full_name = os.path.join( root, name ) + if full_name not in filenames_in_archive: + files_to_remove.append( full_name ) + for repo_file in files_to_remove: + # Remove files in the repository (relative to the upload point) that are not in the uploaded archive. + try: + commands.remove( repo.ui, repo, repo_file, force=True ) + except Exception, e: + log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str( e )) + relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' ) + repo.dirstate.remove( relative_selected_file ) + repo.dirstate.write() + absolute_selected_file = os.path.abspath( selected_file ) + if os.path.isdir( absolute_selected_file ): + try: + os.rmdir( absolute_selected_file ) + except OSError, e: + # The directory is not empty + pass + elif os.path.isfile( absolute_selected_file ): + os.remove( absolute_selected_file ) + dir = os.path.split( absolute_selected_file )[0] + try: + os.rmdir( dir ) + except OSError, e: + # The directory is not empty + pass + # See if any admin users have chosen to receive email alerts when a repository is + # updated. If so, check every uploaded file to ensure content is appropriate. + check_contents = check_file_contents( trans ) + for filename_in_archive in filenames_in_archive: + # Check file content to ensure it is appropriate. + if check_contents and os.path.isfile( filename_in_archive ): + content_alert_str += self.__check_file_content( filename_in_archive ) + commands.add( repo.ui, repo, filename_in_archive ) + if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ): + # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries + # to the in-memory trans.app.tool_data_tables dictionary. + error, message = handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive ) + if error: + return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed + commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) + admin_only = len( repository.downloadable_revisions ) != 1 + handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) + return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed def uncompress( self, repository, uploaded_file_name, uploaded_file_filename, isgzip, isbz2 ): if isgzip: self.__handle_gzip( repository, uploaded_file_name ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Change the import for the base message template for the tool shed to eliminate the bus error when you point your browser to the tool shed.
by Bitbucket 03 Oct '12

03 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/ae683c38bdf2/ changeset: ae683c38bdf2 user: greg date: 2012-10-03 16:13:12 summary: Change the import for the base message template for the tool shed to eliminate the bus error when you point your browser to the tool shed. affected #: 1 file diff -r 7b3a7a3ea9afc3de218c7c5f2c52cdfa717ee5c5 -r ae683c38bdf2aa61403dc220e91561945a855158 templates/webapps/community/message.mako --- a/templates/webapps/community/message.mako +++ b/templates/webapps/community/message.mako @@ -1,1 +1,1 @@ -<%inherit file="/message.mako"/> +<%inherit file="../../message.mako"/> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: smcmanus: Cleanup
by Bitbucket 02 Oct '12

02 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/7b3a7a3ea9af/ changeset: 7b3a7a3ea9af user: smcmanus date: 2012-10-03 00:30:08 summary: Cleanup affected #: 3 files diff -r 3d07a7800f9af46e46c1f3ad3f0fe432949f3b51 -r 7b3a7a3ea9afc3de218c7c5f2c52cdfa717ee5c5 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -321,31 +321,23 @@ log.exception( '(%s) Failed to change ownership of %s, failing' % ( job.id, self.working_directory ) ) return self.fail( job.info, stdout=stdout, stderr=stderr, exit_code=tool_exit_code ) - log.debug( "############## JobWrapper.finish: %s exit code" - % ( "None" if None == tool_exit_code else str(tool_exit_code))) # if the job was deleted, don't finish it if job.state == job.states.DELETED or job.state == job.states.ERROR: - # ERROR at this point means the job was deleted by an administrator. # SM: Note that, at this point, the exit code must be saved in case # there was an error. Errors caught here could mean that the job # was deleted by an administrator (based on old comments), but it # could also mean that a job was broken up into tasks and one of - # the tasks failed. So + # the tasks failed. So include the stderr, stdout, and exit code: return self.fail( job.info, stderr=stderr, stdout=stdout, exit_code=tool_exit_code ) # Check the tool's stdout, stderr, and exit code for errors, but only # if the job has not already been marked as having an error. # The job's stdout and stderr will be set accordingly. - log.debug( "############## JobWrapper.finish: Post-check exit code: %s/%s" - % ( ( "None" if None == tool_exit_code else str(tool_exit_code) ), - ( "None" if None == job.exit_code else str(job.exit_code) ) ) ) if job.states.ERROR != job.state: if ( self.check_tool_output( stdout, stderr, tool_exit_code, job )): job.state = job.states.OK else: job.state = job.states.ERROR - log.debug( "############## JobWrapper.finish: Post-check exit code: %s" - % ( "None" if None == tool_exit_code else str(tool_exit_code))) if self.version_string_cmd: version_filename = self.get_version_string_path() @@ -471,8 +463,6 @@ # is either incorrect or has the wrong semantics. if None != tool_exit_code: job.exit_code = tool_exit_code - log.debug( "############## JobWrapper.finish: storing %s exit code" - % ( "None" if None == job.exit_code else str(job.exit_code))) # custom post process setup inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] ) out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) diff -r 3d07a7800f9af46e46c1f3ad3f0fe432949f3b51 -r 7b3a7a3ea9afc3de218c7c5f2c52cdfa717ee5c5 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -143,10 +143,7 @@ # Finish the job! try: - #job_wrapper.finish( stdout, stderr, exit_code ) - # DELETEME: This is for testing how null exit codes are handled: - log.debug( "############## Finishing job - None exit code" ) - job_wrapper.finish( stdout, stderr, None ) + job_wrapper.finish( stdout, stderr, exit_code ) except: log.exception("Job wrapper finish method failed") job_wrapper.fail("Unable to finish job", exception=True) diff -r 3d07a7800f9af46e46c1f3ad3f0fe432949f3b51 -r 7b3a7a3ea9afc3de218c7c5f2c52cdfa717ee5c5 lib/galaxy/jobs/runners/tasks.py --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -64,7 +64,7 @@ # thing as the last task to complete, which could be added later. # o if a task fails, then the job will fail and the failing task's # exit code will become the job's exit code. - job_exit_code = "" + job_exit_code = None # If we were able to get a command line, run the job. ( must be passed to tasks ) if command_line: @@ -112,17 +112,14 @@ # Deleted tasks are not included right now. # while tasks_complete is False: - log.debug( "************ Rechecking tasks" ) count_complete = 0 tasks_complete = True for tw in task_wrappers: task_state = tw.get_state() - log.debug( "***** Checking task %d: state %s" - % (tw.task_id, task_state) ) if ( model.Task.states.ERROR == task_state ): job_exit_code = tw.get_exit_code() - log.debug( "Canceling job %d: Task %s returned an error (exit code %d)" - % ( tw.job_id, tw.task_id, job_exit_code ) ) + log.debug( "Canceling job %d: Task %s returned an error" + % ( tw.job_id, tw.task_id ) ) self.cancel_job( job_wrapper, task_wrappers ) tasks_complete = True break @@ -136,8 +133,6 @@ if sleep_time < 8: sleep_time *= 2 import time - log.debug( "####################### Finished with tasks; job exit code: %d" % job_exit_code ) - job_wrapper.reclaim_ownership() # if running as the actual user, change ownership before merging. log.debug('execution finished - beginning merge: %s' % command_line) stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers) @@ -164,7 +159,6 @@ # Finish the job try: - log.debug( "$$$$$$$$$$$$$$ job_exit_code before finish: %d" % job_exit_code ) job_wrapper.finish( stdout, stderr, job_exit_code ) except: log.exception("Job wrapper finish method failed") Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: smcmanus: The task runner now chooses an exit code based on the tasks' exit codes - if the tasks are successful, then the last exit code scanned is chosen, and if a task fails then the first scanned failing task's exit code is chosen. The JobWrapper's fail method also stores stdout, stderr, and the exit code if they're available, though this will currently only applies to when a Job using the task runner fails. A null/None exit code is also now supported.
by Bitbucket 02 Oct '12

02 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/3d07a7800f9a/ changeset: 3d07a7800f9a user: smcmanus date: 2012-10-03 00:20:12 summary: The task runner now chooses an exit code based on the tasks' exit codes - if the tasks are successful, then the last exit code scanned is chosen, and if a task fails then the first scanned failing task's exit code is chosen. The JobWrapper's fail method also stores stdout, stderr, and the exit code if they're available, though this will currently only applies to when a Job using the task runner fails. A null/None exit code is also now supported. affected #: 4 files diff -r 6d45fd1d830ee85edcdc07ba93831c316102e81a -r 3d07a7800f9af46e46c1f3ad3f0fe432949f3b51 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -204,7 +204,7 @@ self.version_string_cmd = self.tool.version_string_cmd return extra_filenames - def fail( self, message, exception=False ): + def fail( self, message, exception=False, stdout="", stderr="", exit_code=None ): """ Indicate job failure by setting state and message on all output datasets. @@ -249,6 +249,20 @@ job.state = job.states.ERROR job.command_line = self.command_line job.info = message + # TODO: Put setting the stdout, stderr, and exit code in one place + # (not duplicated with the finish method). + if ( len( stdout ) > 32768 ): + stdout = stdout[:32768] + log.info( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id ) + job.stdout = stdout + if ( len( stderr ) > 32768 ): + stderr = stderr[:32768] + log.info( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id ) + job.stderr = stderr + # Let the exit code be Null if one is not provided: + if ( exit_code != None ): + job.exit_code = exit_code + self.sa_session.add( job ) self.sa_session.flush() #Perform email action even on failure. @@ -290,7 +304,7 @@ self.sa_session.add( job ) self.sa_session.flush() - def finish( self, stdout, stderr, tool_exit_code=0 ): + def finish( self, stdout, stderr, tool_exit_code=None ): """ Called to indicate that the associated command has been run. Updates the output datasets based on stderr and stdout from the command, and @@ -300,25 +314,38 @@ self.sa_session.expunge_all() job = self.get_job() + # TODO: After failing here, consider returning from the function. try: self.reclaim_ownership() except: - self.fail( job.info ) log.exception( '(%s) Failed to change ownership of %s, failing' % ( job.id, self.working_directory ) ) + return self.fail( job.info, stdout=stdout, stderr=stderr, exit_code=tool_exit_code ) + log.debug( "############## JobWrapper.finish: %s exit code" + % ( "None" if None == tool_exit_code else str(tool_exit_code))) # if the job was deleted, don't finish it if job.state == job.states.DELETED or job.state == job.states.ERROR: - #ERROR at this point means the job was deleted by an administrator. - return self.fail( job.info ) + # ERROR at this point means the job was deleted by an administrator. + # SM: Note that, at this point, the exit code must be saved in case + # there was an error. Errors caught here could mean that the job + # was deleted by an administrator (based on old comments), but it + # could also mean that a job was broken up into tasks and one of + # the tasks failed. So + return self.fail( job.info, stderr=stderr, stdout=stdout, exit_code=tool_exit_code ) # Check the tool's stdout, stderr, and exit code for errors, but only # if the job has not already been marked as having an error. # The job's stdout and stderr will be set accordingly. + log.debug( "############## JobWrapper.finish: Post-check exit code: %s/%s" + % ( ( "None" if None == tool_exit_code else str(tool_exit_code) ), + ( "None" if None == job.exit_code else str(job.exit_code) ) ) ) if job.states.ERROR != job.state: if ( self.check_tool_output( stdout, stderr, tool_exit_code, job )): job.state = job.states.OK else: job.state = job.states.ERROR + log.debug( "############## JobWrapper.finish: Post-check exit code: %s" + % ( "None" if None == tool_exit_code else str(tool_exit_code))) if self.version_string_cmd: version_filename = self.get_version_string_path() @@ -358,7 +385,6 @@ # TODO: The context['stderr'] holds stderr's contents. An error # only really occurs if the job also has an error. So check the # job's state: - #if context['stderr']: if job.states.ERROR == job.state: dataset.blurb = "error" elif dataset.has_data(): @@ -435,12 +461,18 @@ self.sa_session.flush() # Save stdout and stderr if len( job.stdout ) > 32768: - log.error( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id ) + log.info( "stdout for job %d is greater than 32K, only first part will be logged to database" % job.id ) job.stdout = job.stdout[:32768] if len( job.stderr ) > 32768: - log.error( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id ) + log.info( "stderr for job %d is greater than 32K, only first part will be logged to database" % job.id ) job.stderr = job.stderr[:32768] - job.exit_code = tool_exit_code + # The exit code will be null if there is no exit code to be set. + # This is so that we don't assign an exit code, such as 0, that + # is either incorrect or has the wrong semantics. + if None != tool_exit_code: + job.exit_code = tool_exit_code + log.debug( "############## JobWrapper.finish: storing %s exit code" + % ( "None" if None == job.exit_code else str(job.exit_code))) # custom post process setup inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] ) out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] ) @@ -513,26 +545,27 @@ # that range, then apply the error level and add a message. # If we've reached a fatal error rule, then stop. max_error_level = galaxy.tools.StdioErrorLevel.NO_ERROR - for stdio_exit_code in self.tool.stdio_exit_codes: - if ( tool_exit_code >= stdio_exit_code.range_start and - tool_exit_code <= stdio_exit_code.range_end ): - # Tack on a generic description of the code - # plus a specific code description. For example, - # this might prepend "Job 42: Warning: Out of Memory\n". - code_desc = stdio_exit_code.desc - if ( None == code_desc ): - code_desc = "" - tool_msg = ( "%s: Exit code %d: %s" % ( - galaxy.tools.StdioErrorLevel.desc( stdio_exit_code.error_level ), - tool_exit_code, - code_desc ) ) - log.info( "Job %s: %s" % (job.get_id_tag(), tool_msg) ) - stderr = tool_msg + "\n" + stderr - max_error_level = max( max_error_level, - stdio_exit_code.error_level ) - if ( max_error_level >= - galaxy.tools.StdioErrorLevel.FATAL ): - break + if tool_exit_code != None: + for stdio_exit_code in self.tool.stdio_exit_codes: + if ( tool_exit_code >= stdio_exit_code.range_start and + tool_exit_code <= stdio_exit_code.range_end ): + # Tack on a generic description of the code + # plus a specific code description. For example, + # this might prepend "Job 42: Warning (Out of Memory)\n". + code_desc = stdio_exit_code.desc + if ( None == code_desc ): + code_desc = "" + tool_msg = ( "%s: Exit code %d (%s)" % ( + galaxy.tools.StdioErrorLevel.desc( stdio_exit_code.error_level ), + tool_exit_code, + code_desc ) ) + log.info( "Job %s: %s" % (job.get_id_tag(), tool_msg) ) + stderr = tool_msg + "\n" + stderr + max_error_level = max( max_error_level, + stdio_exit_code.error_level ) + if ( max_error_level >= + galaxy.tools.StdioErrorLevel.FATAL ): + break if max_error_level < galaxy.tools.StdioErrorLevel.FATAL: # We'll examine every regex. Each regex specifies whether @@ -1013,6 +1046,11 @@ self.sa_session.refresh( task ) return task.state + def get_exit_code( self ): + task = self.get_task() + self.sa_session.refresh( task ) + return task.exit_code + def set_runner( self, runner_url, external_id ): task = self.get_task() self.sa_session.refresh( task ) @@ -1022,7 +1060,7 @@ self.sa_session.add( task ) self.sa_session.flush() - def finish( self, stdout, stderr, tool_exit_code=0 ): + def finish( self, stdout, stderr, tool_exit_code=None ): # DBTODO integrate previous finish logic. # Simple finish for tasks. Just set the flag OK. """ @@ -1032,7 +1070,8 @@ """ # This may have ended too soon log.debug( 'task %s for job %d ended; exit code: %d' - % (self.task_id, self.job_id, tool_exit_code) ) + % (self.task_id, self.job_id, + tool_exit_code if tool_exit_code != None else -256 ) ) # default post job setup_external_metadata self.sa_session.expunge_all() task = self.get_task() diff -r 6d45fd1d830ee85edcdc07ba93831c316102e81a -r 3d07a7800f9af46e46c1f3ad3f0fe432949f3b51 lib/galaxy/jobs/runners/local.py --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -111,11 +111,8 @@ if sleep_time < 8: # So we don't stat every second sleep_time *= 2 - # Reap the process and get the exit code. The exit code should - # only be None if the process isn't finished, but check anyway. - exit_code = proc.wait() # reap - if None == exit_code: - exit_code = 0 + # Reap the process and get the exit code. + exit_code = proc.wait() stdout_file.seek( 0 ) stderr_file.seek( 0 ) stdout = stdout_file.read( 32768 ) @@ -146,7 +143,10 @@ # Finish the job! try: - job_wrapper.finish( stdout, stderr, exit_code ) + #job_wrapper.finish( stdout, stderr, exit_code ) + # DELETEME: This is for testing how null exit codes are handled: + log.debug( "############## Finishing job - None exit code" ) + job_wrapper.finish( stdout, stderr, None ) except: log.exception("Job wrapper finish method failed") job_wrapper.fail("Unable to finish job", exception=True) diff -r 6d45fd1d830ee85edcdc07ba93831c316102e81a -r 3d07a7800f9af46e46c1f3ad3f0fe432949f3b51 lib/galaxy/jobs/runners/tasks.py --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -56,6 +56,16 @@ job_wrapper.fail( "failure preparing job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return + + # This is the job's exit code, which will depend on the tasks' + # exit code. The overall job's exit code will be one of two values: + # o if the job is successful, then the last task scanned will be + # used to determine the exit code. Note that this is not the same + # thing as the last task to complete, which could be added later. + # o if a task fails, then the job will fail and the failing task's + # exit code will become the job's exit code. + job_exit_code = "" + # If we were able to get a command line, run the job. ( must be passed to tasks ) if command_line: try: @@ -69,7 +79,9 @@ job_wrapper.fail("Job Splitting Failed, no match for '%s'" % job_wrapper.tool.parallelism) return tasks = splitter.do_split(job_wrapper) - # Not an option for now. Task objects don't *do* anything useful yet, but we'll want them tracked outside this thread to do anything. + # Not an option for now. Task objects don't *do* anything + # useful yet, but we'll want them tracked outside this thread + # to do anything. # if track_tasks_in_database: task_wrappers = [] for task in tasks: @@ -100,25 +112,31 @@ # Deleted tasks are not included right now. # while tasks_complete is False: + log.debug( "************ Rechecking tasks" ) count_complete = 0 tasks_complete = True for tw in task_wrappers: task_state = tw.get_state() + log.debug( "***** Checking task %d: state %s" + % (tw.task_id, task_state) ) if ( model.Task.states.ERROR == task_state ): - log.debug( "Canceling job %d: Task %d returned an error" - % ( tw.job_id, tw.task_id ) ) + job_exit_code = tw.get_exit_code() + log.debug( "Canceling job %d: Task %s returned an error (exit code %d)" + % ( tw.job_id, tw.task_id, job_exit_code ) ) self.cancel_job( job_wrapper, task_wrappers ) tasks_complete = True break elif not task_state in completed_states: tasks_complete = False else: + job_exit_code = tw.get_exit_code() count_complete = count_complete + 1 if tasks_complete is False: sleep( sleep_time ) if sleep_time < 8: sleep_time *= 2 import time + log.debug( "####################### Finished with tasks; job exit code: %d" % job_exit_code ) job_wrapper.reclaim_ownership() # if running as the actual user, change ownership before merging. log.debug('execution finished - beginning merge: %s' % command_line) @@ -146,7 +164,8 @@ # Finish the job try: - job_wrapper.finish( stdout, stderr ) + log.debug( "$$$$$$$$$$$$$$ job_exit_code before finish: %d" % job_exit_code ) + job_wrapper.finish( stdout, stderr, job_exit_code ) except: log.exception("Job wrapper finish method failed") job_wrapper.fail("Unable to finish job", exception=True) diff -r 6d45fd1d830ee85edcdc07ba93831c316102e81a -r 3d07a7800f9af46e46c1f3ad3f0fe432949f3b51 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -134,7 +134,7 @@ self.post_job_actions = [] self.imported = False self.handler = None - self.exit_code = 0 + self.exit_code = None # TODO: Add accessors for members defined in SQL Alchemy for the Job table and # for the mapper defined to the Job table. @@ -317,10 +317,9 @@ self.task_runner_name = None self.task_runner_external_id = None self.job = job - # SM: Using default empty strings avoids None exceptions later on. self.stdout = "" self.stderr = "" - self.exit_code = 0 + self.exit_code = None self.prepare_input_files_cmd = prepare_files_cmd def get_param_values( self, app ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: scatterplot: controls to client-side template
by Bitbucket 02 Oct '12

02 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/6d45fd1d830e/ changeset: 6d45fd1d830e user: carlfeberhard date: 2012-10-02 23:29:06 summary: scatterplot: controls to client-side template affected #: 4 files diff -r f66f88b2f743e3b3733e00756f7655003bf1880f -r 6d45fd1d830ee85edcdc07ba93831c316102e81a static/scripts/templates/compiled/template-visualization-scatterplotControlForm.js --- /dev/null +++ b/static/scripts/templates/compiled/template-visualization-scatterplotControlForm.js @@ -0,0 +1,51 @@ +(function() { + var template = Handlebars.template, templates = Handlebars.templates = Handlebars.templates || {}; +templates['template-visualization-scatterplotControlForm'] = template(function (Handlebars,depth0,helpers,partials,data) { + helpers = helpers || Handlebars.helpers; + var buffer = "", stack1, foundHelper, functionType="function", escapeExpression=this.escapeExpression, self=this; + +function program1(depth0,data) { + + var buffer = "", stack1, foundHelper; + buffer += "\n <option value=\""; + foundHelper = helpers.index; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{}}); } + else { stack1 = depth0.index; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + buffer += escapeExpression(stack1) + "\">"; + foundHelper = helpers.name; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{}}); } + else { stack1 = depth0.name; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + buffer += escapeExpression(stack1) + "</option>\n "; + return buffer;} + +function program3(depth0,data) { + + var buffer = "", stack1, foundHelper; + buffer += "\n <option value=\""; + foundHelper = helpers.index; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{}}); } + else { stack1 = depth0.index; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + buffer += escapeExpression(stack1) + "\">"; + foundHelper = helpers.name; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{}}); } + else { stack1 = depth0.name; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + buffer += escapeExpression(stack1) + "</option>\n "; + return buffer;} + + buffer += "\n<div id=\"loading-indicator\" style=\"display: none;\">\n <img class=\"loading-img\" src=\""; + foundHelper = helpers.loadingIndicatorImagePath; + if (foundHelper) { stack1 = foundHelper.call(depth0, {hash:{}}); } + else { stack1 = depth0.loadingIndicatorImagePath; stack1 = typeof stack1 === functionType ? stack1() : stack1; } + buffer += escapeExpression(stack1) + "\" />\n <span class=\"loading-message\"></span>\n</div>\n\n"; + buffer += "\n<div id=\"chart-settings\">\n\n "; + buffer += "\n <div id=\"x-column-input\">\n <label for=\"\">Data column for X: </label>\n <select name=\"x-column\">\n "; + stack1 = depth0.availableColumns; + stack1 = helpers.each.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(1, program1, data)}); + if(stack1 || stack1 === 0) { buffer += stack1; } + buffer += "\n </select>\n </div>\n <div id=\"y-column-input\">\n <label for=\"\">Data column for Y: </label>\n <select name=\"y-column\">\n "; + stack1 = depth0.availableColumns; + stack1 = helpers.each.call(depth0, stack1, {hash:{},inverse:self.noop,fn:self.program(3, program3, data)}); + if(stack1 || stack1 === 0) { buffer += stack1; } + buffer += "\n </select>\n </div>\n \n <input id=\"render-button\" type=\"button\" value=\"Draw\" />\n <div class=\"clear\"></div>\n</div>"; + return buffer;}); +})(); \ No newline at end of file diff -r f66f88b2f743e3b3733e00756f7655003bf1880f -r 6d45fd1d830ee85edcdc07ba93831c316102e81a static/scripts/templates/visualization-templates.html --- /dev/null +++ b/static/scripts/templates/visualization-templates.html @@ -0,0 +1,33 @@ +<script type="text/template" class="template-visualization" id="template-visualization-scatterplotControlForm"> + +{{! loading indicator - initially hidden }} +<div id="loading-indicator" style="display: none;"> + <img class="loading-img" src="{{loadingIndicatorImagePath}}" /> + <span class="loading-message"></span> +</div> + +{{! main controls }} +<div id="chart-settings"> + + {{! column selector containers }} + <div id="x-column-input"> + <label for="">Data column for X: </label> + <select name="x-column"> + {{#each availableColumns}} + <option value="{{index}}">{{name}}</option> + {{/each}} + </select> + </div> + <div id="y-column-input"> + <label for="">Data column for Y: </label> + <select name="y-column"> + {{#each availableColumns}} + <option value="{{index}}">{{name}}</option> + {{/each}} + </select> + </div> + + <input id="render-button" type="button" value="Draw" /> + <div class="clear"></div> +</div> +</script> diff -r f66f88b2f743e3b3733e00756f7655003bf1880f -r 6d45fd1d830ee85edcdc07ba93831c316102e81a static/scripts/viz/scatterplot.js --- a/static/scripts/viz/scatterplot.js +++ b/static/scripts/viz/scatterplot.js @@ -1,7 +1,8 @@ define([ "../libs/underscore", "../libs/d3", - "../mvc/base-mvc" + "../mvc/base-mvc", + "../templates/compiled/template-visualization-scatterplotControlForm" ], function(){ /* ============================================================================= @@ -389,7 +390,7 @@ * Scatterplot control UI as a backbone view * */ -var ScatterplotView = BaseView.extend( LoggableMixin ).extend({ +var ScatterplotControlForm = BaseView.extend( LoggableMixin ).extend({ //logger : console, tagName : 'form', className : 'scatterplot-settings-form', @@ -418,48 +419,31 @@ }, render : function(){ - //TODO: to template var view = this, - html = '', - columnHtml = ''; + html = ''; // build column select controls for each x, y (based on name if available) - // ugh...hafta preprocess - this.dataset.metadata_column_types = this.dataset.metadata_column_types.split( ', ' ); - _.each( this.dataset.metadata_column_types, function( type, index ){ + var formData = { + loadingIndicatorImagePath : this.loadingIndicatorImagePath, + config : this.chartConfig, + availableColumns : [] + }; + _.each( this.dataset.metadata_column_types.split( ', ' ), function( type, index ){ // use only numeric columns if( type === 'int' || type === 'float' ){ + //TODO: using 0-based indeces var name = 'column ' + index; // label with the name if available if( view.dataset.metadata_column_names ){ name = view.dataset.metadata_column_names[ index ]; } - columnHtml += '<option value="' + index + '">' + name + '</option>'; + formData.availableColumns.push({ index: index, name: name }); } }); - - // loading indicator - initially hidden - html += '<div id="loading-indicator" style="display: none;">'; - html += '<img class="loading-img" src=' + this.loadingIndicatorImagePath + ' />'; - html += '<span class="loading-message"></span>'; - html += '</div>'; - - // column selector containers - html += '<div id="x-column-input">'; - html += '<label for="">Data column for X: </label><select name="x-column">' + columnHtml + '</select>'; - html += '</div>'; - - html += '<div id="y-column-input">'; - html += '<label for="">Data column for Y: </label><select name="y-column">' + columnHtml + '</select>'; - html += '</div>'; - - html += '<input id="render-button" type="button" value="Draw" />'; - html += '<div class="clear"></div>'; - //TODO: other vals: max_vals, start_val, pagination + html = ScatterplotControlForm.templates.form( formData ); this.$el.append( html ); - this.$el.find( '#render-button' ); return this; }, @@ -509,13 +493,15 @@ url : url, dataType : 'json', success : function( response ){ + // save the endpoint (number of next line, fileptr) for this object //TODO: server sends back an endpoint, cache for next pagination request + view.endpoint = response.endpoint; + view.showLoadingIndicator( 'Rendering...' ); - // save the endpoint (number of next line, fileptr) for this object - view.endpoint = response.endpoint; view.plot.render( response.data, response.meta ); view.hideLoadingIndicator(); }, + error : function( xhr, status, error ){ view.hideLoadingIndicator(); alert( 'ERROR:' + status + '\n' + error ); @@ -523,9 +509,14 @@ }); } }); +ScatterplotControlForm.templates = CompiledTemplateLoader.getTemplates({ + 'visualization-templates.html' : { + form : 'template-visualization-scatterplotControlForm' + } +}); //============================================================================== return { - //TwoVarScatterplot : TwoVarScatterplot, - ScatterplotView : ScatterplotView + TwoVarScatterplot : TwoVarScatterplot, + ScatterplotControlForm : ScatterplotControlForm };}); \ No newline at end of file diff -r f66f88b2f743e3b3733e00756f7655003bf1880f -r 6d45fd1d830ee85edcdc07ba93831c316102e81a templates/visualization/scatterplot.mako --- a/templates/visualization/scatterplot.mako +++ b/templates/visualization/scatterplot.mako @@ -4,18 +4,19 @@ ${parent.stylesheets()} <style type="text/css"> +/*TODO: use/move into base.less*/ +* { margin: 0px; padding: 0px; } +#chart-header { + padding : 8px; + background-color: #ebd9b2; +} .title { - margin: 0px; - padding: 8px; - background-color: #ebd9b2; - border: 2px solid #ebd9b2; } -.subtitle { - margin: 0px; - padding: 0px 8px 8px 16px; - background-color: #ebd9b2; +#chart-header .subtitle { + margin: -4px 0px 0px 4px; + padding : 0; color: white; font-size: small; } @@ -28,7 +29,7 @@ padding-top: 1em; } -#chart-settings-form > * { +#chart-settings > * { margin: 8px; } @@ -96,7 +97,7 @@ //?? hmmmm //kwargs = ${h.to_json_string( kwargs )}; - var settingsForm = new scatterplot.ScatterplotView({ + var settingsForm = new scatterplot.ScatterplotControlForm({ dataset : hda, el : $( '#chart-settings-form' ), apiDatasetsURL : apiDatasetsURL, @@ -112,8 +113,10 @@ </%def><%def name="body()"> - <h2 class="title">Scatterplot of '${hda['name']}'</h2> - <p class="subtitle">${hda['misc_info']}</p> + <div id="chart-header"> + <h2 class="title">Scatterplot of '${hda['name']}'</h2> + <p class="subtitle">${hda['misc_info']}</p> + </div><div id="chart-holder"></div><div id="chart-settings-form"></div></%def> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: basic.py, ColumnDataProvider: altered return format to be array of column arrays (opposed to array or rows); compute min, max of each column on server; scatterplot: altered to use that format;
by Bitbucket 02 Oct '12

02 Oct '12

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/f66f88b2f743/ changeset: f66f88b2f743 user: carlfeberhard date: 2012-10-02 22:37:45 summary: basic.py, ColumnDataProvider: altered return format to be array of column arrays (opposed to array or rows); compute min, max of each column on server; scatterplot: altered to use that format; affected #: 4 files diff -r 369a941393909434e0efb5c84dd831e5ab096095 -r f66f88b2f743e3b3733e00756f7655003bf1880f lib/galaxy/visualization/data_providers/basic.py --- a/lib/galaxy/visualization/data_providers/basic.py +++ b/lib/galaxy/visualization/data_providers/basic.py @@ -92,22 +92,26 @@ and self.original_dataset.metadata.comment_lines ): start_val = int( self.original_dataset.metadata.comment_lines ) + 1 - response = {} - response[ 'data' ] = data = [] - #TODO bail if columns None, not parsable, not within meta.columns # columns is an array of ints for now (should handle column names later) columns = from_json_string( columns ) - assert( all([ column < self.original_dataset.metadata.columns for column in columns ]) ),( - "column index (%d) must be less" % ( column ) + for column in columns: + assert( ( column < self.original_dataset.metadata.columns ) + and ( column >= 0 ) ),( + "column index (%d) must be positive and less" % ( column ) + " than the number of columns: %d" % ( self.original_dataset.metadata.columns ) ) - - #print columns, start_val, max_vals, skip_comments, kwargs - # alter meta by column_selectors (if any) + # set up the response, column lists + response = {} + response[ 'data' ] = data = [ [] for column in columns ] + response[ 'meta' ] = meta = [ { 'min': None, 'max': None } for column in columns ] + + column_types = [ self.original_dataset.metadata.column_types[ column ] for column in columns ] + + # function for casting by column_types def cast_val( val, type ): - """ Cast value based on type. """ + """ Cast value based on type. Return None if can't be cast """ if type == 'int': try: val = int( val ) except: return None @@ -119,21 +123,30 @@ f = open( self.original_dataset.file_name ) #TODO: add f.seek if given fptr in kwargs for count, line in enumerate( f ): + + # check line v. desired start, end if count < start_val: continue - if ( count - start_val ) >= max_vals: break fields = line.split() fields_len = len( fields ) - #TODO: this will return the wrong number of columns for abberrant lines - line_data = [ cast_val( fields[c], self.original_dataset.metadata.column_types[c] ) - for c in columns if ( c < fields_len ) ] - data.append( line_data ) + #NOTE: this will return None/null for abberrant column values (including bad indeces) + for index, column in enumerate( columns ): + column_val = None + if column < fields_len: + column_val = cast_val( fields[ column ], column_types[ index ] ) + if column_val != None: + if( meta[ index ][ 'min' ] == None + or column_val < meta[ index ][ 'min' ] ): + meta[ index ][ 'min' ] = column_val + if( meta[ index ][ 'max' ] == None + or column_val > meta[ index ][ 'max' ] ): + meta[ index ][ 'max' ] = column_val + data[ index ].append( column_val ) response[ 'endpoint' ] = dict( last_line=( count - 1 ), file_ptr=f.tell() ) f.close() return response - diff -r 369a941393909434e0efb5c84dd831e5ab096095 -r f66f88b2f743e3b3733e00756f7655003bf1880f lib/galaxy/webapps/galaxy/api/datasets.py --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -56,7 +56,7 @@ rval = dataset.get_api_value() except Exception, e: - rval = "Error in dataset API at listing contents" + rval = "Error in dataset API at listing contents: " + str( e ) log.error( rval + ": %s" % str(e), exc_info=True ) trans.response.status = 500 return rval diff -r 369a941393909434e0efb5c84dd831e5ab096095 -r f66f88b2f743e3b3733e00756f7655003bf1880f static/scripts/packed/viz/scatterplot.js --- a/static/scripts/packed/viz/scatterplot.js +++ b/static/scripts/packed/viz/scatterplot.js @@ -1,1 +1,1 @@ -define(["../libs/underscore","../libs/d3","../mvc/base-mvc"],function(){function b(f){var i=this,d=10,h=7,g=10,e=8,c=5;this.log=function(){if(this.debugging&&console&&console.debug){var j=Array.prototype.slice.call(arguments);j.unshift(this.toString());console.debug.apply(null,j)}};this.log("new TwoVarScatterplot:",f);this.defaults={id:"TwoVarScatterplot",containerSelector:"body",maxDataPoints:30000,bubbleRadius:4,entryAnimDuration:500,xNumTicks:10,yNumTicks:10,xAxisLabelBumpY:40,yAxisLabelBumpX:-35,width:500,height:500,marginTop:50,marginRight:50,marginBottom:50,marginLeft:50,xMin:null,xMax:null,yMin:null,yMax:null,xLabel:"X",yLabel:"Y"};this.config=_.extend({},this.defaults,f);this.updateConfig=function(j){_.extend(this.config,j)};this.toString=function(){return this.config.id};this.translateStr=function(j,k){return"translate("+j+","+k+")"};this.rotateStr=function(k,j,l){return"rotate("+k+","+j+","+l+")"};this.svg=d3.select(this.config.containerSelector).append("svg:svg").attr("class","chart").style("display","none");this.content=this.svg.append("svg:g").attr("class","content");this.xAxis=this.content.append("g").attr("class","axis").attr("id","x-axis");this.xAxisLabel=this.xAxis.append("text").attr("class","axis-label").attr("id","x-axis-label");this.yAxis=this.content.append("g").attr("class","axis").attr("id","y-axis");this.yAxisLabel=this.yAxis.append("text").attr("class","axis-label").attr("id","y-axis-label");this.log("built svg:",d3.selectAll("svg"));this.adjustChartDimensions=function(m,k,j,l){m=m||0;k=k||0;j=j||0;l=l||0;this.svg.attr("width",this.config.width+(this.config.marginRight+k)+(this.config.marginLeft+l)).attr("height",this.config.height+(this.config.marginTop+m)+(this.config.marginBottom+j)).style("display","block");this.content=this.svg.select("g.content").attr("transform",this.translateStr(this.config.marginLeft+l,this.config.marginTop+m))};this.preprocessData=function(j){return j.slice(0,this.config.maxDataPoints)};this.setUpDomains=function(j,k){this.xMin=this.config.xMin||d3.min(j);this.xMax=this.config.xMax||d3.max(j);this.yMin=this.config.yMin||d3.min(k);this.yMax=this.config.yMax||d3.max(k)};this.setUpScales=function(){this.xScale=d3.scale.linear().domain([this.xMin,this.xMax]).range([0,this.config.width]),this.yScale=d3.scale.linear().domain([this.yMin,this.yMax]).range([this.config.height,0])};this.setUpXAxis=function(){this.xAxisFn=d3.svg.axis().scale(this.xScale).ticks(this.config.xNumTicks).orient("bottom");this.xAxis.attr("transform",this.translateStr(0,this.config.height)).call(this.xAxisFn);this.xLongestLabel=d3.max(_.map([this.xMin,this.xMax],function(j){return(String(j)).length}));if(this.xLongestLabel>=c){this.xAxis.selectAll("g").filter(":nth-child(odd)").style("display","none")}this.xAxisLabel.attr("x",this.config.width/2).attr("y",this.config.xAxisLabelBumpY).attr("text-anchor","middle").text(this.config.xLabel)};this.setUpYAxis=function(){this.yAxisFn=d3.svg.axis().scale(this.yScale).ticks(this.config.yNumTicks).orient("left");this.yAxis.call(this.yAxisFn);this.log("yAxis:",this.yAxis);var j=this.yAxis.selectAll("text").filter(function(n,m){return m!==0});this.yLongestLabel=d3.max(j[0].map(function(n,m){return(d3.select(n).text()).length}))||0;var k=d+(this.yLongestLabel*h)+e+g;this.config.yAxisLabelBumpX=-(k-g);if(this.config.marginLeft<k){var l=(k)-this.config.marginLeft;l=(l<0)?(0):(l);this.log("adjusting:",l);this.adjustChartDimensions(0,0,0,l)}this.yAxisLabel.attr("x",this.config.yAxisLabelBumpX).attr("y",this.config.height/2).attr("text-anchor","middle").attr("transform",this.rotateStr(-90,this.config.yAxisLabelBumpX,this.config.height/2)).text(this.config.yLabel)};this.renderGrid=function(){this.vGridLines=this.content.selectAll("line.v-grid-line").data(this.xScale.ticks(this.xAxisFn.ticks()[0]));this.vGridLines.enter().append("svg:line").classed("grid-line v-grid-line",true);this.vGridLines.attr("x1",this.xScale).attr("y1",0).attr("x2",this.xScale).attr("y2",this.config.height);this.vGridLines.exit().remove();this.hGridLines=this.content.selectAll("line.h-grid-line").data(this.yScale.ticks(this.yAxisFn.ticks()[0]));this.hGridLines.enter().append("svg:line").classed("grid-line h-grid-line",true);this.hGridLines.attr("x1",0).attr("y1",this.yScale).attr("x2",this.config.width).attr("y2",this.yScale);this.hGridLines.exit().remove()};this.glyphEnterState=function(j){};this.glyphFinalState=function(j){};this.glyphExitState=function(j){};this.renderDatapoints=function(j,m){var l=function(o,n){return i.xScale(j[n])};var k=function(o,n){return i.yScale(m[n])};this.datapoints=this.content.selectAll(".glyph").data(j);this.datapoints.enter().append("svg:circle").attr("class","glyph").attr("cx",l).attr("cy",0).attr("r",0);this.datapoints.transition().duration(this.config.entryAnimDuration).attr("cx",l).attr("cy",k).attr("r",this.config.bubbleRadius);this.datapoints.exit().transition().duration(this.config.entryAnimDuration).attr("cy",this.config.height).attr("r",0).style("fill-opacity",0).remove()};this.render=function(j,k){this.log("renderScatterplot",j.length,k.length,this.config);j=this.preprocessData(j);k=this.preprocessData(k);this.setUpDomains(j,k);this.setUpScales();this.adjustChartDimensions();this.setUpXAxis();this.setUpYAxis();this.renderGrid();this.renderDatapoints(j,k)}}var a=BaseView.extend(LoggableMixin).extend({tagName:"form",className:"scatterplot-settings-form",loadingIndicatorImagePath:(galaxy_paths.get("image_path")+"/loading_large_white_bg.gif"),events:{"click #render-button":"renderScatterplot"},initialize:function(c){if(!c||!c.dataset){throw ("ScatterplotView requires a dataset")}else{this.dataset=c.dataset}this.apiDatasetsURL=c.apiDatasetsURL;this.chartConfig=c.chartConfig||{};this.log("this.chartConfig:",this.chartConfig);this.plot=new b(this.chartConfig)},render:function(){var c=this,e="",d="";this.dataset.metadata_column_types=this.dataset.metadata_column_types.split(", ");_.each(this.dataset.metadata_column_types,function(h,g){if(h==="int"||h==="float"){var f="column "+g;if(c.dataset.metadata_column_names){f=c.dataset.metadata_column_names[g]}d+='<option value="'+g+'">'+f+"</option>"}});e+='<div id="loading-indicator" style="display: none;">';e+='<img class="loading-img" src='+this.loadingIndicatorImagePath+" />";e+='<span class="loading-message"></span>';e+="</div>";e+='<div id="x-column-input">';e+='<label for="">Data column for X: </label><select name="x-column">'+d+"</select>";e+="</div>";e+='<div id="y-column-input">';e+='<label for="">Data column for Y: </label><select name="y-column">'+d+"</select>";e+="</div>";e+='<input id="render-button" type="button" value="Draw" />';e+='<div class="clear"></div>';this.$el.append(e);this.$el.find("#render-button");return this},showLoadingIndicator:function(c){c=c||"";this.$el.find("div#loading-indicator").children(".loading-message").text(c);this.$el.find("div#loading-indicator").show("fast")},hideLoadingIndicator:function(){this.$el.find("div#loading-indicator").hide("fast")},renderScatterplot:function(){var d=this,e=this.apiDatasetsURL+"/"+this.dataset.id+"?data_type=raw_data&",i=this.$el.find('[name="x-column"]'),j=i.val(),g=i.children('[value="'+j+'"]').text(),h=this.$el.find('[name="y-column"]'),f=h.val(),c=h.children('[value="'+f+'"]').text();this.log(g,c);this.chartConfig.xLabel=g;this.chartConfig.yLabel=c;d.plot.updateConfig(this.chartConfig);e+=jQuery.param({columns:"["+[j,f]+"]"});this.log("url:",e);this.showLoadingIndicator("Fetching data...");jQuery.ajax({url:e,dataType:"json",success:function(k){d.showLoadingIndicator("Rendering...");d.endpoint=k.endpoint;d.plot.render(_.map(k.data,function(l){return l[0]}),_.map(k.data,function(l){return l[1]}));d.hideLoadingIndicator()},error:function(m,k,l){d.hideLoadingIndicator();alert("ERROR:"+k+"\n"+l)}})}});return{ScatterplotView:a}}); \ No newline at end of file +define(["../libs/underscore","../libs/d3","../mvc/base-mvc"],function(){function b(f){var i=this,d=10,h=7,g=10,e=8,c=5;this.log=function(){if(this.debugging&&console&&console.debug){var j=Array.prototype.slice.call(arguments);j.unshift(this.toString());console.debug.apply(null,j)}};this.log("new TwoVarScatterplot:",f);this.defaults={id:"TwoVarScatterplot",containerSelector:"body",maxDataPoints:30000,bubbleRadius:4,entryAnimDuration:500,xNumTicks:10,yNumTicks:10,xAxisLabelBumpY:40,yAxisLabelBumpX:-35,width:500,height:500,marginTop:50,marginRight:50,marginBottom:50,marginLeft:50,xMin:null,xMax:null,yMin:null,yMax:null,xLabel:"X",yLabel:"Y"};this.config=_.extend({},this.defaults,f);this.updateConfig=function(j){_.extend(this.config,j)};this.toString=function(){return this.config.id};this.translateStr=function(j,k){return"translate("+j+","+k+")"};this.rotateStr=function(k,j,l){return"rotate("+k+","+j+","+l+")"};this.svg=d3.select(this.config.containerSelector).append("svg:svg").attr("class","chart").style("display","none");this.content=this.svg.append("svg:g").attr("class","content");this.xAxis=this.content.append("g").attr("class","axis").attr("id","x-axis");this.xAxisLabel=this.xAxis.append("text").attr("class","axis-label").attr("id","x-axis-label");this.yAxis=this.content.append("g").attr("class","axis").attr("id","y-axis");this.yAxisLabel=this.yAxis.append("text").attr("class","axis-label").attr("id","y-axis-label");this.log("built svg:",d3.selectAll("svg"));this.adjustChartDimensions=function(m,k,j,l){m=m||0;k=k||0;j=j||0;l=l||0;this.svg.attr("width",this.config.width+(this.config.marginRight+k)+(this.config.marginLeft+l)).attr("height",this.config.height+(this.config.marginTop+m)+(this.config.marginBottom+j)).style("display","block");this.content=this.svg.select("g.content").attr("transform",this.translateStr(this.config.marginLeft+l,this.config.marginTop+m))};this.preprocessData=function(j){return(j.length>this.config.maxDataPoints)?(j.slice(0,this.config.maxDataPoints)):(j)};this.setUpDomains=function(j,l,k){this.log("setUpDomains");this.xMin=this.config.xMin||(k)?(k[0].min):(d3.min(j));this.xMax=this.config.xMax||(k)?(k[0].max):(d3.max(j));this.yMin=this.config.yMin||(k)?(k[1].min):(d3.min(l));this.yMax=this.config.yMax||(k)?(k[1].max):(d3.max(l))};this.setUpScales=function(){this.xScale=d3.scale.linear().domain([this.xMin,this.xMax]).range([0,this.config.width]),this.yScale=d3.scale.linear().domain([this.yMin,this.yMax]).range([this.config.height,0])};this.setUpXAxis=function(){this.xAxisFn=d3.svg.axis().scale(this.xScale).ticks(this.config.xNumTicks).orient("bottom");this.xAxis.attr("transform",this.translateStr(0,this.config.height)).call(this.xAxisFn);this.xLongestLabel=d3.max(_.map([this.xMin,this.xMax],function(j){return(String(j)).length}));if(this.xLongestLabel>=c){this.xAxis.selectAll("g").filter(":nth-child(odd)").style("display","none")}this.xAxisLabel.attr("x",this.config.width/2).attr("y",this.config.xAxisLabelBumpY).attr("text-anchor","middle").text(this.config.xLabel)};this.setUpYAxis=function(){this.yAxisFn=d3.svg.axis().scale(this.yScale).ticks(this.config.yNumTicks).orient("left");this.yAxis.call(this.yAxisFn);this.log("yAxis:",this.yAxis);var j=this.yAxis.selectAll("text").filter(function(n,m){return m!==0});this.yLongestLabel=d3.max(j[0].map(function(n,m){return(d3.select(n).text()).length}))||0;var k=d+(this.yLongestLabel*h)+e+g;this.config.yAxisLabelBumpX=-(k-g);if(this.config.marginLeft<k){var l=(k)-this.config.marginLeft;l=(l<0)?(0):(l);this.log("adjusting:",l);this.adjustChartDimensions(0,0,0,l)}this.yAxisLabel.attr("x",this.config.yAxisLabelBumpX).attr("y",this.config.height/2).attr("text-anchor","middle").attr("transform",this.rotateStr(-90,this.config.yAxisLabelBumpX,this.config.height/2)).text(this.config.yLabel)};this.renderGrid=function(){this.vGridLines=this.content.selectAll("line.v-grid-line").data(this.xScale.ticks(this.xAxisFn.ticks()[0]));this.vGridLines.enter().append("svg:line").classed("grid-line v-grid-line",true);this.vGridLines.attr("x1",this.xScale).attr("y1",0).attr("x2",this.xScale).attr("y2",this.config.height);this.vGridLines.exit().remove();this.hGridLines=this.content.selectAll("line.h-grid-line").data(this.yScale.ticks(this.yAxisFn.ticks()[0]));this.hGridLines.enter().append("svg:line").classed("grid-line h-grid-line",true);this.hGridLines.attr("x1",0).attr("y1",this.yScale).attr("x2",this.config.width).attr("y2",this.yScale);this.hGridLines.exit().remove()};this.glyphEnterState=function(j){};this.glyphFinalState=function(j){};this.glyphExitState=function(j){};this.renderDatapoints=function(j,m){var l=function(o,n){return i.xScale(j[n])};var k=function(o,n){return i.yScale(m[n])};this.datapoints=this.content.selectAll(".glyph").data(j);this.datapoints.enter().append("svg:circle").attr("class","glyph").attr("cx",l).attr("cy",0).attr("r",0);this.datapoints.transition().duration(this.config.entryAnimDuration).attr("cx",l).attr("cy",k).attr("r",this.config.bubbleRadius);this.datapoints.exit().transition().duration(this.config.entryAnimDuration).attr("cy",this.config.height).attr("r",0).style("fill-opacity",0).remove()};this.render=function(k,l){var j=k[0],m=k[1];this.log("renderScatterplot",j.length,m.length,this.config);j=this.preprocessData(j);m=this.preprocessData(m);this.setUpDomains(j,m,l);this.log("xMin, xMax, yMin, yMax:",this.xMin,this.xMax,this.yMin,this.yMax);this.setUpScales();this.adjustChartDimensions();this.setUpXAxis();this.setUpYAxis();this.renderGrid();this.renderDatapoints(j,m)}}var a=BaseView.extend(LoggableMixin).extend({tagName:"form",className:"scatterplot-settings-form",loadingIndicatorImagePath:(galaxy_paths.get("image_path")+"/loading_large_white_bg.gif"),events:{"click #render-button":"renderScatterplot"},initialize:function(c){if(!c||!c.dataset){throw ("ScatterplotView requires a dataset")}else{this.dataset=c.dataset}this.apiDatasetsURL=c.apiDatasetsURL;this.chartConfig=c.chartConfig||{};this.log("this.chartConfig:",this.chartConfig);this.plot=new b(this.chartConfig)},render:function(){var c=this,e="",d="";this.dataset.metadata_column_types=this.dataset.metadata_column_types.split(", ");_.each(this.dataset.metadata_column_types,function(h,g){if(h==="int"||h==="float"){var f="column "+g;if(c.dataset.metadata_column_names){f=c.dataset.metadata_column_names[g]}d+='<option value="'+g+'">'+f+"</option>"}});e+='<div id="loading-indicator" style="display: none;">';e+='<img class="loading-img" src='+this.loadingIndicatorImagePath+" />";e+='<span class="loading-message"></span>';e+="</div>";e+='<div id="x-column-input">';e+='<label for="">Data column for X: </label><select name="x-column">'+d+"</select>";e+="</div>";e+='<div id="y-column-input">';e+='<label for="">Data column for Y: </label><select name="y-column">'+d+"</select>";e+="</div>";e+='<input id="render-button" type="button" value="Draw" />';e+='<div class="clear"></div>';this.$el.append(e);this.$el.find("#render-button");return this},showLoadingIndicator:function(c){c=c||"";this.$el.find("div#loading-indicator").children(".loading-message").text(c);this.$el.find("div#loading-indicator").show("fast")},hideLoadingIndicator:function(){this.$el.find("div#loading-indicator").hide("fast")},renderScatterplot:function(){var d=this,e=this.apiDatasetsURL+"/"+this.dataset.id+"?data_type=raw_data&",i=this.$el.find('[name="x-column"]'),j=i.val(),g=i.children('[value="'+j+'"]').text(),h=this.$el.find('[name="y-column"]'),f=h.val(),c=h.children('[value="'+f+'"]').text();this.log(g,c);this.chartConfig.xLabel=g;this.chartConfig.yLabel=c;d.plot.updateConfig(this.chartConfig);e+=jQuery.param({columns:"["+[j,f]+"]"});this.log("url:",e);this.showLoadingIndicator("Fetching data...");jQuery.ajax({url:e,dataType:"json",success:function(k){d.showLoadingIndicator("Rendering...");d.endpoint=k.endpoint;d.plot.render(k.data,k.meta);d.hideLoadingIndicator()},error:function(m,k,l){d.hideLoadingIndicator();alert("ERROR:"+k+"\n"+l)}})}});return{ScatterplotView:a}}); \ No newline at end of file diff -r 369a941393909434e0efb5c84dd831e5ab096095 -r f66f88b2f743e3b3733e00756f7655003bf1880f static/scripts/viz/scatterplot.js --- a/static/scripts/viz/scatterplot.js +++ b/static/scripts/viz/scatterplot.js @@ -158,14 +158,16 @@ // ........................................................ data and scales this.preprocessData = function( data ){ // set a cap on the data, limit to first n points - return data.slice( 0, this.config.maxDataPoints ); + return ( data.length > this.config.maxDataPoints )? ( data.slice( 0, this.config.maxDataPoints ) ): ( data ); }; - this.setUpDomains = function( xCol, yCol ){ - this.xMin = this.config.xMin || d3.min( xCol ); - this.xMax = this.config.xMax || d3.max( xCol ); - this.yMin = this.config.yMin || d3.min( yCol ); - this.yMax = this.config.yMax || d3.max( yCol ); + this.setUpDomains = function( xCol, yCol, meta ){ + this.log( 'setUpDomains' ); + // configuration takes priority, otherwise meta (from the server) if passed, last-resort: compute it here + this.xMin = this.config.xMin || ( meta )?( meta[0].min ):( d3.min( xCol ) ); + this.xMax = this.config.xMax || ( meta )?( meta[0].max ):( d3.max( xCol ) ); + this.yMin = this.config.yMin || ( meta )?( meta[1].min ):( d3.min( yCol ) ); + this.yMax = this.config.yMax || ( meta )?( meta[1].max ):( d3.max( yCol ) ); }; this.setUpScales = function(){ @@ -352,8 +354,12 @@ //this.log( this.datapoints, 'glyphs rendered' ); }; - this.render = function( xCol, yCol ){ + this.render = function( columnData, meta ){ //pre: columns passed are numeric + //pre: at least two columns are passed + //assume: first column is x, second column is y, any remaining aren't used + var xCol = columnData[0], + yCol = columnData[1]; this.log( 'renderScatterplot', xCol.length, yCol.length, this.config ); //pre: xCol.len == yCol.len @@ -363,8 +369,8 @@ //this.log( 'xCol len', xCol.length, 'yCol len', yCol.length ); //TODO: compute min, max on server. - this.setUpDomains( xCol, yCol ); - //this.log( 'xMin, xMax, yMin, yMax:', this.xMin, this.xMax, this.yMin, this.yMax ); + this.setUpDomains( xCol, yCol, meta ); + this.log( 'xMin, xMax, yMin, yMax:', this.xMin, this.xMax, this.yMin, this.yMax ); this.setUpScales(); this.adjustChartDimensions(); @@ -505,12 +511,9 @@ success : function( response ){ //TODO: server sends back an endpoint, cache for next pagination request view.showLoadingIndicator( 'Rendering...' ); + // save the endpoint (number of next line, fileptr) for this object view.endpoint = response.endpoint; - view.plot.render( - // pull apart first two regardless of number of columns - _.map( response.data, function( columns ){ return columns[0]; } ), - _.map( response.data, function( columns ){ return columns[1]; } ) - ); + view.plot.render( response.data, response.meta ); view.hideLoadingIndicator(); }, error : function( xhr, status, error ){ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0