2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1a3dc9d098e0/ Changeset: 1a3dc9d098e0 Branch: next-stable User: natefoo Date: 2013-05-20 21:54:08 Summary: Open next-stable branch for upcoming June 3, 2013 release. Affected #: 607 files diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b .hgignore --- a/.hgignore +++ b/.hgignore @@ -20,6 +20,7 @@ database/pbs database/tmp database/*.sqlite +database/openid_consumer_cache # Python bytecode *.pyc @@ -35,6 +36,11 @@ tool_shed_webapp.pid hgweb.config* +# Reports Runtime Files +reports_webapp.lock +reports_webapp.log +reports_webapp.pid + # Config files universe_wsgi.ini reports_wsgi.ini @@ -54,7 +60,7 @@ job_conf.xml data_manager_conf.xml shed_data_manager_conf.xml - +visualizations_conf.xml static/welcome.html.* static/welcome.html @@ -75,6 +81,7 @@ # Test output run_functional_tests.html +test/tool_shed/tmp/* # Project files *.kpf diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b .hgtags --- a/.hgtags +++ b/.hgtags @@ -1,1 +1,4 @@ a4113cc1cb5eaa68091c9a73375f00555b66dd11 release_2013.01.13 +1c717491139269651bb59687563da9410b84c65d release_2013.02.08 +75f09617abaadbc8cc732bb8ee519decaeb56ea7 release_2013.04.01 +2cc8d10988e03257dc7b97f8bb332c7df745d1dd security_2013.04.08 diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b buildbot_setup.sh --- a/buildbot_setup.sh +++ b/buildbot_setup.sh @@ -93,26 +93,49 @@ JARS="/galaxy/software/jars" -for link in $LINKS; do - echo "Linking $link" - rm -f tool-data/`basename $link` - ln -sf $link tool-data -done - -if [ -d "$HYPHY" ]; then - echo "Linking $HYPHY" - rm -f tool-data/HYPHY - ln -sf $HYPHY tool-data/HYPHY +if [ ! $1 ]; then + type="standard" +elif [ $1 == "-ec2" ]; then + type="external-ec2" +else + type="unknown" fi -if [ -d "$JARS" ]; then - echo "Linking $JARS" - rm -f tool-data/shared/jars - ln -sf $JARS tool-data/shared/jars -fi +case $type in + external*) + echo "Running standalone buildbot setup..." + for sample in tool-data/*.sample; do + basename=${sample%.sample} + if [ ! -f $basename ]; then + echo "Copying $sample to $basename" + cp "$sample" "$basename" + fi + done + ;; + *) + echo "Running standard buildbot setup..." + for link in $LINKS; do + echo "Linking $link" + rm -f tool-data/`basename $link` + ln -sf $link tool-data + done + + if [ -d "$HYPHY" ]; then + echo "Linking $HYPHY" + rm -f tool-data/HYPHY + ln -sf $HYPHY tool-data/HYPHY + fi + + if [ -d "$JARS" ]; then + echo "Linking $JARS" + rm -f tool-data/shared/jars + ln -sf $JARS tool-data/shared/jars + fi + ;; +esac for sample in $SAMPLES; do - file=`echo $sample | sed -e 's/\.sample$//'` + file=${sample%.sample} echo "Copying $sample to $file" cp $sample $file done diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -3,16 +3,13 @@ <registration converters_path="lib/galaxy/datatypes/converters" display_path="display_applications"><datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true"/><datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false"/> + <datatype extension="asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" display_in_upload="true" /> + <datatype extension="asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" display_in_upload="true" /><datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true"/><datatype extension="fli" type="galaxy.datatypes.tabular:FeatureLocationIndex" display_in_upload="false"/><datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream" display_in_upload="true"><converter file="bam_to_bai.xml" target_datatype="bai"/> - <converter file="bam_to_summary_tree_converter.xml" target_datatype="summary_tree"/> - <!-- - Caution: (a) this converter requires bedtools to be installed and (b) it is very memory intensive and - is not recommended for most laptops/desktops. - <converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/> - --> + <converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/><display file="ucsc/bam.xml" /><display file="ensembl/ensembl_bam.xml" /><display file="igv/bam.xml" /> @@ -20,10 +17,9 @@ </datatype><datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true"><converter file="bed_to_gff_converter.xml" target_datatype="gff"/> - <converter file="interval_to_coverage.xml" target_datatype="coverage"/><converter file="bed_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="bed_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - <converter file="bed_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/><converter file="bed_to_fli_converter.xml" target_datatype="fli"/><!-- <display file="ucsc/interval_as_bed.xml" /> --><display file="igb/bed.xml" /> @@ -49,7 +45,7 @@ <datatype extension="chrint" type="galaxy.datatypes.interval:ChromatinInteractions" display_in_upload="True"><converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - <converter file="interval_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/></datatype><!-- MSI added Datatypes --><datatype extension="csv" type="galaxy.datatypes.tabular:Tabular" subclass="True" display_in_upload="true" /><!-- FIXME: csv is 'tabular'ized data, but not 'tab-delimited'; the class used here is intended for 'tab-delimited' --> @@ -91,7 +87,7 @@ <datatype extension="gff" type="galaxy.datatypes.interval:Gff" display_in_upload="true"><converter file="gff_to_bed_converter.xml" target_datatype="bed"/><converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/> - <converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/><converter file="gff_to_fli_converter.xml" target_datatype="fli"/><display file="ensembl/ensembl_gff.xml" inherit="True"/><!-- <display file="gbrowse/gbrowse_gff.xml" inherit="True" /> --> @@ -101,7 +97,7 @@ <datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/><datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"><converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/> - <converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/></datatype><datatype extension="toolshed.gz" type="galaxy.datatypes.binary:Binary" mimetype="multipart/x-gzip" subclass="True" /><datatype extension="h5" type="galaxy.datatypes.binary:Binary" mimetype="application/octet-stream" subclass="True" /> @@ -113,7 +109,7 @@ <converter file="interval_to_bed12_converter.xml" target_datatype="bed12"/><converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - <converter file="interval_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="interval_to_bigwig_converter.xml" target_datatype="bigwig"/><!-- <display file="ucsc/interval_as_bed.xml" inherit="True" /> --><display file="ensembl/ensembl_interval_as_bed.xml" inherit="True"/><display file="gbrowse/gbrowse_interval_as_bed.xml" inherit="True"/> @@ -154,10 +150,13 @@ <datatype extension="encodepeak" type="galaxy.datatypes.interval:ENCODEPeak" display_in_upload="True"><converter file="encodepeak_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/><converter file="encodepeak_to_bgzip_converter.xml" target_datatype="bgzip"/> - <converter file="encodepeak_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/></datatype><datatype extension="pdf" type="galaxy.datatypes.images:Pdf" mimetype="application/pdf"/> - <datatype extension="pileup" type="galaxy.datatypes.tabular:Pileup" display_in_upload="true" /> + <datatype extension="pileup" type="galaxy.datatypes.tabular:Pileup" display_in_upload="true"> + <converter file="interval_to_bgzip_converter.xml" target_datatype="bgzip"/> + <converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> + </datatype><datatype extension="png" type="galaxy.datatypes.images:Png" mimetype="image/png"/><datatype extension="qual" type="galaxy.datatypes.qualityscore:QualityScore" /><datatype extension="qualsolexa" type="galaxy.datatypes.qualityscore:QualityScoreSolexa" display_in_upload="true"/> @@ -167,7 +166,7 @@ <datatype extension="Roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/><datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true"><converter file="sam_to_bam.xml" target_datatype="bam"/> - <converter file="sam_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="sam_to_bigwig_converter.xml" target_datatype="bigwig"/></datatype><datatype extension="scf" type="galaxy.datatypes.binary:Scf" mimetype="application/octet-stream" display_in_upload="true"/><datatype extension="Sequences" type="galaxy.datatypes.assembly:Sequences" display_in_upload="false"/> @@ -185,7 +184,7 @@ <converter file="vcf_to_bgzip_converter.xml" target_datatype="bgzip"/><converter file="vcf_to_vcf_bgzip_converter.xml" target_datatype="vcf_bgzip"/><converter file="vcf_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/> - <converter file="vcf_to_summary_tree_converter.xml" target_datatype="summary_tree"/> + <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/><display file="ucsc/vcf.xml" /><display file="igv/vcf.xml" /><display file="rviewer/vcf.xml" inherit="True"/> @@ -198,7 +197,6 @@ <!-- <display file="gbrowse/gbrowse_wig.xml" /> --><display file="igb/wig.xml" /></datatype> - <datatype extension="summary_tree" type="galaxy.datatypes.binary:Binary" subclass="True" /><datatype extension="interval_index" type="galaxy.datatypes.binary:Binary" subclass="True" /><datatype extension="tabix" type="galaxy.datatypes.binary:Binary" subclass="True" /><datatype extension="bgzip" type="galaxy.datatypes.binary:Binary" subclass="True" /> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.api.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.api.rst @@ -0,0 +1,19 @@ +api Package +=========== + +:mod:`repositories` Module +-------------------------- + +.. automodule:: galaxy.webapps.tool_shed.api.repositories + :members: + :undoc-members: + :show-inheritance: + +:mod:`repository_revisions` Module +---------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.api.repository_revisions + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.controllers.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.controllers.rst @@ -0,0 +1,59 @@ +controllers Package +=================== + +:mod:`controllers` Package +-------------------------- + +.. automodule:: galaxy.webapps.tool_shed.controllers + :members: + :undoc-members: + :show-inheritance: + +:mod:`admin` Module +------------------- + +.. automodule:: galaxy.webapps.tool_shed.controllers.admin + :members: + :undoc-members: + :show-inheritance: + +:mod:`hg` Module +---------------- + +.. automodule:: galaxy.webapps.tool_shed.controllers.hg + :members: + :undoc-members: + :show-inheritance: + +:mod:`repository` Module +------------------------ + +.. automodule:: galaxy.webapps.tool_shed.controllers.repository + :members: + :undoc-members: + :show-inheritance: + +:mod:`repository_review` Module +------------------------------- + +.. automodule:: galaxy.webapps.tool_shed.controllers.repository_review + :members: + :undoc-members: + :show-inheritance: + +:mod:`upload` Module +-------------------- + +.. automodule:: galaxy.webapps.tool_shed.controllers.upload + :members: + :undoc-members: + :show-inheritance: + +:mod:`user` Module +------------------ + +.. automodule:: galaxy.webapps.tool_shed.controllers.user + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.framework.middleware.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.framework.middleware.rst @@ -0,0 +1,27 @@ +middleware Package +================== + +:mod:`middleware` Package +------------------------- + +.. automodule:: galaxy.webapps.tool_shed.framework.middleware + :members: + :undoc-members: + :show-inheritance: + +:mod:`hg` Module +---------------- + +.. automodule:: galaxy.webapps.tool_shed.framework.middleware.hg + :members: + :undoc-members: + :show-inheritance: + +:mod:`remoteuser` Module +------------------------ + +.. automodule:: galaxy.webapps.tool_shed.framework.middleware.remoteuser + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.framework.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.framework.rst @@ -0,0 +1,18 @@ +framework Package +================= + +:mod:`framework` Package +------------------------ + +.. automodule:: galaxy.webapps.tool_shed.framework + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + galaxy.webapps.tool_shed.framework.middleware + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.model.migrate.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.model.migrate.rst @@ -0,0 +1,11 @@ +migrate Package +=============== + +:mod:`check` Module +------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.migrate.check + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.model.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.model.rst @@ -0,0 +1,26 @@ +model Package +============= + +:mod:`model` Package +-------------------- + +.. automodule:: galaxy.webapps.tool_shed.model + :members: + :undoc-members: + :show-inheritance: + +:mod:`mapping` Module +--------------------- + +.. automodule:: galaxy.webapps.tool_shed.model.mapping + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + galaxy.webapps.tool_shed.model.migrate + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.rst @@ -0,0 +1,47 @@ +tool_shed Package +================= + +:mod:`tool_shed` Package +------------------------ + +.. automodule:: galaxy.webapps.tool_shed + :members: + :undoc-members: + :show-inheritance: + +:mod:`app` Module +----------------- + +.. automodule:: galaxy.webapps.tool_shed.app + :members: + :undoc-members: + :show-inheritance: + +:mod:`buildapp` Module +---------------------- + +.. automodule:: galaxy.webapps.tool_shed.buildapp + :members: + :undoc-members: + :show-inheritance: + +:mod:`config` Module +-------------------- + +.. automodule:: galaxy.webapps.tool_shed.config + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + galaxy.webapps.tool_shed.api + galaxy.webapps.tool_shed.controllers + galaxy.webapps.tool_shed.framework + galaxy.webapps.tool_shed.model + galaxy.webapps.tool_shed.security + galaxy.webapps.tool_shed.util + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.security.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.security.rst @@ -0,0 +1,11 @@ +security Package +================ + +:mod:`security` Package +----------------------- + +.. automodule:: galaxy.webapps.tool_shed.security + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/galaxy.webapps.tool_shed.util.rst --- /dev/null +++ b/doc/source/lib/galaxy.webapps.tool_shed.util.rst @@ -0,0 +1,35 @@ +util Package +============ + +:mod:`common_util` Module +------------------------- + +.. automodule:: galaxy.webapps.tool_shed.util.common_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`container_util` Module +---------------------------- + +.. automodule:: galaxy.webapps.tool_shed.util.container_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`hgweb_config` Module +-------------------------- + +.. automodule:: galaxy.webapps.tool_shed.util.hgweb_config + :members: + :undoc-members: + :show-inheritance: + +:mod:`shed_statistics` Module +----------------------------- + +.. automodule:: galaxy.webapps.tool_shed.util.shed_statistics + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/tool_shed.galaxy_install.grids.rst --- /dev/null +++ b/doc/source/lib/tool_shed.galaxy_install.grids.rst @@ -0,0 +1,11 @@ +grids Package +============= + +:mod:`admin_toolshed_grids` Module +---------------------------------- + +.. automodule:: tool_shed.galaxy_install.grids.admin_toolshed_grids + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/tool_shed.galaxy_install.migrate.rst --- /dev/null +++ b/doc/source/lib/tool_shed.galaxy_install.migrate.rst @@ -0,0 +1,19 @@ +migrate Package +=============== + +:mod:`check` Module +------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.check + :members: + :undoc-members: + :show-inheritance: + +:mod:`common` Module +-------------------- + +.. automodule:: tool_shed.galaxy_install.migrate.common + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/tool_shed.galaxy_install.rst --- /dev/null +++ b/doc/source/lib/tool_shed.galaxy_install.rst @@ -0,0 +1,44 @@ +galaxy_install Package +====================== + +:mod:`galaxy_install` Package +----------------------------- + +.. automodule:: tool_shed.galaxy_install + :members: + :undoc-members: + :show-inheritance: + +:mod:`install_manager` Module +----------------------------- + +.. automodule:: tool_shed.galaxy_install.install_manager + :members: + :undoc-members: + :show-inheritance: + +:mod:`repository_util` Module +----------------------------- + +.. automodule:: tool_shed.galaxy_install.repository_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`update_manager` Module +---------------------------- + +.. automodule:: tool_shed.galaxy_install.update_manager + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + tool_shed.galaxy_install.grids + tool_shed.galaxy_install.migrate + tool_shed.galaxy_install.tool_dependencies + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/tool_shed.galaxy_install.tool_dependencies.rst --- /dev/null +++ b/doc/source/lib/tool_shed.galaxy_install.tool_dependencies.rst @@ -0,0 +1,27 @@ +tool_dependencies Package +========================= + +:mod:`common_util` Module +------------------------- + +.. automodule:: tool_shed.galaxy_install.tool_dependencies.common_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`fabric_util` Module +------------------------- + +.. automodule:: tool_shed.galaxy_install.tool_dependencies.fabric_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`install_util` Module +-------------------------- + +.. automodule:: tool_shed.galaxy_install.tool_dependencies.install_util + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/tool_shed.grids.rst --- /dev/null +++ b/doc/source/lib/tool_shed.grids.rst @@ -0,0 +1,35 @@ +grids Package +============= + +:mod:`admin_grids` Module +------------------------- + +.. automodule:: tool_shed.grids.admin_grids + :members: + :undoc-members: + :show-inheritance: + +:mod:`repository_grids` Module +------------------------------ + +.. automodule:: tool_shed.grids.repository_grids + :members: + :undoc-members: + :show-inheritance: + +:mod:`repository_review_grids` Module +------------------------------------- + +.. automodule:: tool_shed.grids.repository_review_grids + :members: + :undoc-members: + :show-inheritance: + +:mod:`util` Module +------------------ + +.. automodule:: tool_shed.grids.util + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/tool_shed.rst --- /dev/null +++ b/doc/source/lib/tool_shed.rst @@ -0,0 +1,20 @@ +tool_shed Package +================= + +:mod:`tool_shed_registry` Module +-------------------------------- + +.. automodule:: tool_shed.tool_shed_registry + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + tool_shed.galaxy_install + tool_shed.grids + tool_shed.util + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b doc/source/lib/tool_shed.util.rst --- /dev/null +++ b/doc/source/lib/tool_shed.util.rst @@ -0,0 +1,107 @@ +util Package +============ + +:mod:`common_install_util` Module +--------------------------------- + +.. automodule:: tool_shed.util.common_install_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`common_util` Module +------------------------- + +.. automodule:: tool_shed.util.common_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`data_manager_util` Module +------------------------------- + +.. automodule:: tool_shed.util.data_manager_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`datatype_util` Module +--------------------------- + +.. automodule:: tool_shed.util.datatype_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`encoding_util` Module +--------------------------- + +.. automodule:: tool_shed.util.encoding_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`metadata_util` Module +--------------------------- + +.. automodule:: tool_shed.util.metadata_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`readme_util` Module +------------------------- + +.. automodule:: tool_shed.util.readme_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`repository_dependency_util` Module +---------------------------------------- + +.. automodule:: tool_shed.util.repository_dependency_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`review_util` Module +------------------------- + +.. automodule:: tool_shed.util.review_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`shed_util_common` Module +------------------------------ + +.. automodule:: tool_shed.util.shed_util_common + :members: + :undoc-members: + :show-inheritance: + +:mod:`tool_dependency_util` Module +---------------------------------- + +.. automodule:: tool_shed.util.tool_dependency_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`tool_util` Module +----------------------- + +.. automodule:: tool_shed.util.tool_util + :members: + :undoc-members: + :show-inheritance: + +:mod:`workflow_util` Module +--------------------------- + +.. automodule:: tool_shed.util.workflow_util + :members: + :undoc-members: + :show-inheritance: + diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -14,7 +14,6 @@ [eggs:platform] bx_python = 0.7.1 Cheetah = 2.2.2 -ctypes = 1.0.2 DRMAA_python = 0.2 MarkupSafe = 0.12 mercurial = 2.2.3 @@ -29,6 +28,7 @@ simplejson = 2.1.1 threadframe = 0.2 guppy = 0.1.8 +SQLAlchemy = 0.7.9 ; msgpack_python = 0.2.4 [eggs:noplatform] @@ -46,17 +46,17 @@ nose = 0.11.1 NoseHTML = 0.4.1 NoseTestDiff = 0.1 +Parsley = 1.1 Paste = 1.7.5.1 PasteDeploy = 1.5.0 pexpect = 2.4 python_openid = 2.2.5 python_daemon = 1.5.5 Routes = 1.12.3 -SQLAlchemy = 0.5.6 -sqlalchemy_migrate = 0.5.4 +sqlalchemy_migrate = 0.7.2 ssh = 1.7.14 SVGFig = 1.1.6 -Tempita = 0.1 +Tempita = 0.5.1 twill = 0.9 WebError = 0.8a WebHelpers = 0.2 @@ -75,7 +75,6 @@ MySQL_python = _5.1.41_static bx_python = _7b95ff194725 GeneTrack = _dev_48da9e998f0caf01c5be731e926f4b0481f658f0 -SQLAlchemy = _dev_r6498 pysam = _kanwei_b10f6e722e9a ; dependency source urls, necessary for scrambling. for an explanation, see diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b job_conf.xml.sample_advanced --- a/job_conf.xml.sample_advanced +++ b/job_conf.xml.sample_advanced @@ -6,8 +6,8 @@ --><plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner"/><plugin id="pbs" type="runner" load="galaxy.jobs.runners.pbs:PBSJobRunner" workers="2"/> - <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAARunner"/> - <plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr.LwrJobRunner" /><!-- https://lwr.readthedocs.org --> + <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"/> + <plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr:LwrJobRunner" /><!-- https://lwr.readthedocs.org --><plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" /><plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" /></plugins> @@ -40,7 +40,6 @@ </destination><destination id="dynamic" runner="dynamic"><!-- A destination that represents a method in the dynamic runner. --> - <param id="type">python</param><param id="function">foo</param></destination><destination id="secure_lwr" runner="lwr"> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/fpconst.py --- a/lib/fpconst.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Utilities for handling IEEE 754 floating point special values - -This python module implements constants and functions for working with -IEEE754 double-precision special values. It provides constants for -Not-a-Number (NaN), Positive Infinity (PosInf), and Negative Infinity -(NegInf), as well as functions to test for these values. - -The code is implemented in pure python by taking advantage of the -'struct' standard module. Care has been taken to generate proper -results on both big-endian and little-endian machines. Some efficiency -could be gained by translating the core routines into C. - -See <http://babbage.cs.qc.edu/courses/cs341/IEEE-754references.html> -for reference material on the IEEE 754 floating point standard. - -Further information on this package is available at -<http://www.analytics.washington.edu/statcomp/projects/rzope/fpconst/>. - -Author: Gregory R. Warnes <gregory_r_warnes@groton.pfizer.com> -Date:: 2003-04-08 -Copyright: (c) 2003, Pfizer, Inc. -""" - -__version__ = "0.7.0" -ident = "$Id: fpconst.py,v 1.12 2004/05/22 04:38:17 warnes Exp $" - -import struct, operator - -# check endianess -_big_endian = struct.pack('i',1)[0] != '\x01' - -# and define appropriate constants -if(_big_endian): - NaN = struct.unpack('d', '\x7F\xF8\x00\x00\x00\x00\x00\x00')[0] - PosInf = struct.unpack('d', '\x7F\xF0\x00\x00\x00\x00\x00\x00')[0] - NegInf = -PosInf -else: - NaN = struct.unpack('d', '\x00\x00\x00\x00\x00\x00\xf8\xff')[0] - PosInf = struct.unpack('d', '\x00\x00\x00\x00\x00\x00\xf0\x7f')[0] - NegInf = -PosInf - -def _double_as_bytes(dval): - "Use struct.unpack to decode a double precision float into eight bytes" - tmp = list(struct.unpack('8B',struct.pack('d', dval))) - if not _big_endian: - tmp.reverse() - return tmp - -## -## Functions to extract components of the IEEE 754 floating point format -## - -def _sign(dval): - "Extract the sign bit from a double-precision floating point value" - bb = _double_as_bytes(dval) - return bb[0] >> 7 & 0x01 - -def _exponent(dval): - """Extract the exponentent bits from a double-precision floating - point value. - - Note that for normalized values, the exponent bits have an offset - of 1023. As a consequence, the actual exponentent is obtained - by subtracting 1023 from the value returned by this function - """ - bb = _double_as_bytes(dval) - return (bb[0] << 4 | bb[1] >> 4) & 0x7ff - -def _mantissa(dval): - """Extract the _mantissa bits from a double-precision floating - point value.""" - - bb = _double_as_bytes(dval) - mantissa = bb[1] & 0x0f << 48 - mantissa += bb[2] << 40 - mantissa += bb[3] << 32 - mantissa += bb[4] - return mantissa - -def _zero_mantissa(dval): - """Determine whether the mantissa bits of the given double are all - zero.""" - bb = _double_as_bytes(dval) - return ((bb[1] & 0x0f) | reduce(operator.or_, bb[2:])) == 0 - -## -## Functions to test for IEEE 754 special values -## - -def isNaN(value): - "Determine if the argument is a IEEE 754 NaN (Not a Number) value." - return (_exponent(value)==0x7ff and not _zero_mantissa(value)) - -def isInf(value): - """Determine if the argument is an infinite IEEE 754 value (positive - or negative inifinity)""" - return (_exponent(value)==0x7ff and _zero_mantissa(value)) - -def isFinite(value): - """Determine if the argument is an finite IEEE 754 value (i.e., is - not NaN, positive or negative inifinity)""" - return (_exponent(value)!=0x7ff) - -def isPosInf(value): - "Determine if the argument is a IEEE 754 positive infinity value" - return (_sign(value)==0 and _exponent(value)==0x7ff and \ - _zero_mantissa(value)) - -def isNegInf(value): - "Determine if the argument is a IEEE 754 negative infinity value" - return (_sign(value)==1 and _exponent(value)==0x7ff and \ - _zero_mantissa(value)) - -## -## Functions to test public functions. -## - -def test_isNaN(): - assert( not isNaN(PosInf) ) - assert( not isNaN(NegInf) ) - assert( isNaN(NaN ) ) - assert( not isNaN( 1.0) ) - assert( not isNaN( -1.0) ) - -def test_isInf(): - assert( isInf(PosInf) ) - assert( isInf(NegInf) ) - assert( not isInf(NaN ) ) - assert( not isInf( 1.0) ) - assert( not isInf( -1.0) ) - -def test_isFinite(): - assert( not isFinite(PosInf) ) - assert( not isFinite(NegInf) ) - assert( not isFinite(NaN ) ) - assert( isFinite( 1.0) ) - assert( isFinite( -1.0) ) - -def test_isPosInf(): - assert( isPosInf(PosInf) ) - assert( not isPosInf(NegInf) ) - assert( not isPosInf(NaN ) ) - assert( not isPosInf( 1.0) ) - assert( not isPosInf( -1.0) ) - -def test_isNegInf(): - assert( not isNegInf(PosInf) ) - assert( isNegInf(NegInf) ) - assert( not isNegInf(NaN ) ) - assert( not isNegInf( 1.0) ) - assert( not isNegInf( -1.0) ) - -# overall test -def test(): - test_isNaN() - test_isInf() - test_isFinite() - test_isPosInf() - test_isNegInf() - -if __name__ == "__main__": - test() - diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/__init__.py --- a/lib/galaxy/__init__.py +++ b/lib/galaxy/__init__.py @@ -95,10 +95,15 @@ pkg_resources.Distribution._insert_on = pkg_resources.Distribution.insert_on pkg_resources.Distribution.insert_on = _insert_on -# patch to add the NullHandler class to logging -if sys.version_info[:2] < ( 2, 7 ): - import logging +# compat: BadZipFile introduced in Python 2.7 +import zipfile +if not hasattr( zipfile, 'BadZipFile' ): + zipfile.BadZipFile = zipfile.error + +# compat: patch to add the NullHandler class to logging +import logging +if not hasattr( logging, 'NullHandler' ): class NullHandler( logging.Handler ): def emit( self, record ): pass - logging.NullHandler = NullHandler + logging.NullHandler = NullHandler \ No newline at end of file diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -15,6 +15,7 @@ from galaxy.tags.tag_handler import GalaxyTagHandler from galaxy.visualization.genomes import Genomes from galaxy.visualization.data_providers.registry import DataProviderRegistry +from galaxy.visualization.registry import VisualizationsRegistry from galaxy.tools.imp_exp import load_history_imp_exp_tools from galaxy.tools.genome_index import load_genome_index_tools from galaxy.sample_tracking import external_service_types @@ -120,6 +121,9 @@ load_history_imp_exp_tools( self.toolbox ) # Load genome indexer tool. load_genome_index_tools( self.toolbox ) + # visualizations registry: associates resources with visualizations, controls how to render + self.visualizations_registry = ( VisualizationsRegistry( self.config.root, self.config.visualizations_conf_path ) + if self.config.visualizations_conf_path else None ) # Load security policy. self.security_agent = self.model.security_agent self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions ) diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -86,7 +86,6 @@ self.galaxy_data_manager_data_path = kwargs.get( 'galaxy_data_manager_data_path', self.tool_data_path ) self.tool_secret = kwargs.get( "tool_secret", "" ) self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" ) - self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) ) self.retry_metadata_internally = string_as_bool( kwargs.get( "retry_metadata_internally", "True" ) ) self.use_remote_user = string_as_bool( kwargs.get( "use_remote_user", "False" ) ) self.remote_user_maildomain = kwargs.get( "remote_user_maildomain", None ) @@ -155,6 +154,10 @@ self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225" ).lower().split(",") self.brand = kwargs.get( 'brand', None ) + # Configuration for the message box directly below the masthead. + self.message_box_visible = kwargs.get( 'message_box_visible', False ) + self.message_box_content = kwargs.get( 'message_box_content', None ) + self.message_box_class = kwargs.get( 'message_box_class', 'info' ) self.support_url = kwargs.get( 'support_url', 'http://wiki.g2.bx.psu.edu/Support' ) self.wiki_url = kwargs.get( 'wiki_url', 'http://g2.trac.bx.psu.edu/' ) self.blog_url = kwargs.get( 'blog_url', None ) @@ -166,6 +169,7 @@ self.enable_whoosh_library_search = string_as_bool( kwargs.get( 'enable_whoosh_library_search', False ) ) self.whoosh_index_dir = resolve_path( kwargs.get( "whoosh_index_dir", "database/whoosh_indexes" ), self.root ) self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) + self.ftp_upload_dir_identifier = kwargs.get( 'ftp_upload_dir_identifier', 'email' ) # attribute on user - email, username, id, etc... self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) @@ -271,6 +275,8 @@ self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) ) self.fluent_host = kwargs.get( 'fluent_host', 'localhost' ) self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) ) + # visualizations registry config path + self.visualizations_conf_path = kwargs.get( 'visualizations_conf_path', None ) @property def sentry_dsn_public( self ): diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/assembly.py --- a/lib/galaxy/datatypes/assembly.py +++ b/lib/galaxy/datatypes/assembly.py @@ -5,14 +5,14 @@ """ import data +import logging +import os +import re +import sys from galaxy.datatypes import sequence -import logging, os, sys, time, tempfile, shutil, string, glob, re -import galaxy.model -from galaxy.datatypes import metadata +from galaxy.datatypes.images import Html from galaxy.datatypes.metadata import MetadataElement -from galaxy import util -from galaxy.datatypes.images import Html -from sniff import * + log = logging.getLogger(__name__) @@ -174,7 +174,6 @@ gen_msg = '' try: efp = dataset.extra_files_path - flist = os.listdir(efp) log_path = os.path.join(efp,'Log') f = open(log_path,'r') log_content = f.read(1000) @@ -223,5 +222,5 @@ self.regenerate_primary_file(dataset) if __name__ == '__main__': - import doctest, sys + import doctest doctest.testmod(sys.modules[__name__]) diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -2,18 +2,26 @@ Binary classes """ -import data, logging, binascii +import binascii +import data +import gzip +import logging +import os +import shutil +import struct +import subprocess +import tempfile +import zipfile + +from urllib import urlencode, quote_plus +from galaxy import eggs +eggs.require( "bx-python" ) + +from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE + from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes import metadata from galaxy.datatypes.sniff import * -from galaxy import eggs -import pkg_resources -pkg_resources.require( "bx-python" ) -from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE -from urllib import urlencode, quote_plus -import zipfile, gzip -import os, subprocess, tempfile -import struct log = logging.getLogger(__name__) @@ -85,9 +93,18 @@ Binary.register_unsniffable_binary_ext("ab1") +class GenericAsn1Binary( Binary ): + """Class for generic ASN.1 binary format""" + file_ext = "asn1-binary" + +Binary.register_unsniffable_binary_ext("asn1-binary") + class Bam( Binary ): """Class describing a BAM binary file""" file_ext = "bam" + track_type = "ReadTrack" + data_sources = { "data": "bai", "index": "bigwig" } + MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, no_value=None, visible=False, optional=True ) def _get_samtools_version( self ): @@ -238,9 +255,7 @@ return dataset.peek except: return "Binary bam alignments file (%s)" % ( data.nice_size( dataset.get_size() ) ) - def get_track_type( self ): - return "ReadTrack", { "data": "bai", "index": [ "bigwig", "summary_tree" ] } - + Binary.register_sniffable_binary_format("bam", "bam", Bam) class H5( Binary ): @@ -318,6 +333,9 @@ The supplemental info in the paper has the binary details: http://bioinformatics.oxfordjournals.org/cgi/content/abstract/btq351v1 """ + track_type = "LineTrack" + data_sources = { "data_standalone": "bigwig" } + def __init__( self, **kwd ): Binary.__init__( self, **kwd ) self._magic = 0x888FFC26 @@ -342,19 +360,18 @@ return dataset.peek except: return "Binary UCSC %s file (%s)" % ( self._name, data.nice_size( dataset.get_size() ) ) - def get_track_type( self ): - return "LineTrack", {"data_standalone": "bigwig"} - + Binary.register_sniffable_binary_format("bigwig", "bigwig", BigWig) class BigBed(BigWig): """BigBed support from UCSC.""" + + data_sources = { "data_standalone": "bigbed" } + def __init__( self, **kwd ): Binary.__init__( self, **kwd ) self._magic = 0x8789F2EB self._name = "BigBed" - def get_track_type( self ): - return "LineTrack", {"data_standalone": "bigbed"} Binary.register_sniffable_binary_format("bigbed", "bigbed", BigBed) diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/checkers.py --- a/lib/galaxy/datatypes/checkers.py +++ b/lib/galaxy/datatypes/checkers.py @@ -58,7 +58,7 @@ for chars in temp: for char in chars: chars_read += 1 - if ord( char ) > 128: + if util.is_binary( char ): is_binary = True break if chars_read > 100: diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/chrominfo.py --- a/lib/galaxy/datatypes/chrominfo.py +++ b/lib/galaxy/datatypes/chrominfo.py @@ -1,7 +1,3 @@ -import data -from galaxy import util -from galaxy.datatypes.sniff import * -from galaxy.web import url_for from tabular import Tabular from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml --- a/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml +++ b/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml @@ -1,7 +1,14 @@ <tool id="CONVERTER_bam_to_bigwig_0" name="Convert BAM to BigWig" version="1.0.0" hidden="true"><!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --><command> - bedtools genomecov -bg -split -ibam $input -g $chromInfo | wigToBigWig stdin $chromInfo $output + bedtools genomecov -bg -split -ibam $input -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output </command><inputs><param format="bam" name="input" type="data" label="Choose BAM file"/> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/bam_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_bam_to_summary_tree_0" name="Convert BAM to Summary Tree" version="1.0.0" hidden="true"> - <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> - <command interpreter="python"> - sam_or_bam_to_summary_tree_converter.py --bam $input1 $input1.metadata.bam_index $output1 - </command> - <inputs> - <param format="bam" name="input1" type="data" label="Choose BAM file"/> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml @@ -0,0 +1,25 @@ +<tool id="CONVERTER_bed_gff_or_vcf_to_bigwig_0" name="Convert BED, GFF, or VCF to BigWig" version="1.0.0" hidden="true"> + <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> + <command> + ## Remove comments and sort by chromosome. + grep -v '^#' $input | sort -k1,1 | + + ## Generate coverage bedgraph. + bedtools genomecov -bg -split -i stdin -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output + </command> + <inputs> + <param format="bed,gff,vcf" name="input" type="data" label="Choose input file"/> + </inputs> + <outputs> + <data format="bigwig" name="output"/> + </outputs> + <help> + </help> +</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/bed_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/bed_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_bed_to_summary_tree_0" name="Convert BED to Summary Tree" version="1.0.0" hidden="true"> -<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> - <command interpreter="python">interval_to_summary_tree_converter.py $input1 $output1</command> - <inputs> - <page> - <param format="bed" name="input1" type="data" label="Choose BED file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/encodepeak_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/encodepeak_to_summary_tree_converter.xml +++ /dev/null @@ -1,20 +0,0 @@ -<tool id="CONVERTER_encodepeak_to_summary_tree_0" name="Convert ENCODEPeak to Summary Tree" version="1.0.0" hidden="true"> -<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> - <command interpreter="python">interval_to_summary_tree_converter.py - -c ${input1.metadata.chromCol} - -s ${input1.metadata.startCol} - -e ${input1.metadata.endCol} - $input1 $output1 - </command> - - <inputs> - <page> - <param format="ENCODEPeak" name="input1" type="data" label="Choose ENCODEPeak file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/gff_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/gff_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_gff_to_summary_tree_0" name="Convert GFF to Summary Tree" version="1.0.0" hidden="true"> -<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> - <command interpreter="python">interval_to_summary_tree_converter.py $input1 $output1 --gff</command> - <inputs> - <page> - <param format="gff" name="input1" type="data" label="Choose GFF file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml @@ -0,0 +1,33 @@ +<tool id="CONVERTER_interval_to_bigwig_0" name="Convert Genomic Intervals To Coverage"> + <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> + <!-- Used on the metadata edit page. --> + <command> + + ## Remove comments and sort by chromosome. + grep -v '^#' $input1 | sort -k${input1.metadata.chromCol},${input1.metadata.chromCol} | + + ## Create simple BED by cutting chrom, start, and end columns. + awk -v OFS=' ' '{print $${input1.metadata.chromCol},$${input1.metadata.startCol},$${input1.metadata.endCol} }' | + + ## Generate coverage bedgraph. + bedtools genomecov -bg -split -i stdin -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output + + </command> + <inputs> + <page> + <param format="interval" name="input1" type="data" label="Choose intervals"/> + </page> + </inputs> + <outputs> + <data format="bigwig" name="output"/> + </outputs> + <help> + </help> +</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.py --- a/lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -""" -Convert from interval file to summary tree file. Default input file format is BED (0-based, half-open intervals). - -usage: %prog <options> in_file out_file - -c, --chr-col: chromosome column, default=1 - -s, --start-col: start column, default=2 - -e, --end-col: end column, default=3 - -t, --strand-col: strand column, default=6 - -G, --gff: input is GFF format, meaning start and end coordinates are 1-based, closed interval -""" -from __future__ import division - -import sys, fileinput, optparse -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -from galaxy.visualization.tracks.summary import * -from bx.intervals.io import * -from galaxy.datatypes.util.gff_util import * - -def main(): - # Read options, args. - parser = optparse.OptionParser() - parser.add_option( '-c', '--chr-col', type='int', dest='chrom_col', default=1 ) - parser.add_option( '-s', '--start-col', type='int', dest='start_col', default=2 ) - parser.add_option( '-e', '--end-col', type='int', dest='end_col', default=3 ) - parser.add_option( '-t', '--strand-col', type='int', dest='strand_col', default=6 ) - parser.add_option( '-G', '--gff', dest="gff_format", action="store_true" ) - (options, args) = parser.parse_args() - input_fname, output_fname = args - - # Convert column indices to 0-based. - options.chrom_col -= 1 - options.start_col -= 1 - options.end_col -= 1 - options.strand_col -= 1 - - # Do conversion. - if options.gff_format: - reader_wrapper_class = GFFReaderWrapper - chr_col, start_col, end_col, strand_col = ( 0, 3, 4, 6 ) - else: - reader_wrapper_class = NiceReaderWrapper - chr_col, start_col, end_col, strand_col = ( options.chrom_col, options.start_col, options.end_col, options.strand_col ) - reader_wrapper = reader_wrapper_class( fileinput.FileInput( input_fname ), - chrom_col=chr_col, - start_col=start_col, - end_col=end_col, - strand_col=strand_col, - fix_strand=True ) - st = SummaryTree() - for feature in list( reader_wrapper ): - if isinstance( feature, GenomicInterval ): - # Tree expects BED coordinates. - if type( feature ) is GFFFeature: - convert_gff_coords_to_bed( feature ) - st.insert_range( feature.chrom, long( feature.start ), long( feature.end ) ) - - st.write( output_fname ) - -if __name__ == "__main__": - main() diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/interval_to_summary_tree_converter.xml +++ /dev/null @@ -1,20 +0,0 @@ -<tool id="CONVERTER_interval_to_summary_tree_0" name="Convert Interval to Summary Tree" version="1.0.0" hidden="true"> -<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> - <command interpreter="python">interval_to_summary_tree_converter.py - -c ${input1.metadata.chromCol} - -s ${input1.metadata.startCol} - -e ${input1.metadata.endCol} - $input1 $output1 - </command> - - <inputs> - <page> - <param format="interval" name="input1" type="data" label="Choose Interval file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py --- /dev/null +++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +""" +Convert from pileup file to interval index file. + +usage: %prog <options> in_file out_file +""" + +from __future__ import division + +import sys, fileinput, optparse +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +from galaxy.visualization.tracks.summary import * +from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed +from bx.interval_index_file import Indexes + +def main(): + + # Read options, args. + parser = optparse.OptionParser() + (options, args) = parser.parse_args() + input_fname, output_fname = args + + # Do conversion. + index = Indexes() + offset = 0 + for line in open( input_fname, "r" ): + chrom, start = line.split()[ 0:2 ] + # Pileup format is 1-based. + start = int( start ) - 1 + index.add( chrom, start, start + 1, offset ) + offset += len( line ) + + index.write( open(output_fname, "w") ) + +if __name__ == "__main__": + main() + \ No newline at end of file diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml @@ -0,0 +1,15 @@ +<tool id="CONVERTER_pileup_to_interval_index_0" name="Convert Pileup to Interval Index" version="1.0.0" hidden="true"> +<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> + <command interpreter="python">pileup_to_interval_index_converter.py $input $output + </command> + <inputs> + <page> + <param format="pileup" name="input" type="data" label="Choose Pileup file"/> + </page> + </inputs> + <outputs> + <data format="interval_index" name="output"/> + </outputs> + <help> + </help> +</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/sam_or_bam_to_summary_tree_converter.py --- a/lib/galaxy/datatypes/converters/sam_or_bam_to_summary_tree_converter.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python - -from __future__ import division - -import sys, os, optparse -sys.stderr = open(os.devnull, 'w') # suppress stderr as cython produces warning on some systems: - # csamtools.so:6: RuntimeWarning: __builtin__.file size changed - -from galaxy import eggs -import pkg_resources - -if sys.version_info[:2] == (2, 4): - pkg_resources.require( "ctypes" ) -pkg_resources.require( "pysam" ) - -from pysam import csamtools -from galaxy.visualization.tracks.summary import * - -def main(): - parser = optparse.OptionParser() - parser.add_option( '-S', '--sam', action="store_true", dest="is_sam" ) - parser.add_option( '-B', '--bam', action="store_true", dest="is_bam" ) - options, args = parser.parse_args() - - if options.is_bam: - input_fname = args[0] - index_fname = args[1] - out_fname = args[2] - samfile = csamtools.Samfile( filename=input_fname, mode='rb', index_filename=index_fname ) - elif options.is_sam: - input_fname = args[0] - out_fname = args[1] - samfile = csamtools.Samfile( filename=input_fname, mode='r' ) - - st = SummaryTree() - for read in samfile.fetch(): - st.insert_range( samfile.getrname( read.rname ), read.pos, read.pos + read.rlen ) - - st.write(out_fname) - -if __name__ == "__main__": - main() diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml --- /dev/null +++ b/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml @@ -0,0 +1,20 @@ +<tool id="CONVERTER_sam_to_bigwig_0" name="Convert SAM to BigWig" version="1.0.0" hidden="true"> + <command> + samtools view -bh $input | bedtools genomecov -bg -split -ibam stdin -g $chromInfo + + ## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this + ## should only be used on systems with large RAM. + ## | wigToBigWig stdin $chromInfo $output + + ## This can be used anywhere. + > temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output + </command> + <inputs> + <param format="bam" name="input" type="data" label="Choose BAM file"/> + </inputs> + <outputs> + <data format="bigwig" name="output"/> + </outputs> + <help> + </help> +</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/sam_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/sam_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_sam_to_summary_tree_0" name="Convert SAM to Summary Tree" version="1.0.0" hidden="true"> -<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> - <command interpreter="python">sam_or_bam_to_summary_tree_converter.py --sam $input1 $output1</command> - <inputs> - <page> - <param format="sam" name="input1" type="data" label="Choose sam file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.py --- a/lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -""" -Convert from VCF file to summary tree file. - -usage: %prog in_file out_file -""" -from __future__ import division - -import optparse -import galaxy_utils.sequence.vcf -from galaxy.visualization.tracks.summary import SummaryTree - -def main(): - # Read options, args. - parser = optparse.OptionParser() - (options, args) = parser.parse_args() - in_file, out_file = args - - # Do conversion. - st = SummaryTree() - for line in list( galaxy_utils.sequence.vcf.Reader( open( in_file ) ) ): - # VCF format provides a chrom and 1-based position for each variant. - # SummaryTree expects 0-based coordinates. - st.insert_range( line.chrom, long( line.pos-1 ), long( line.pos ) ) - - st.write(out_file) - -if __name__ == "__main__": - main() \ No newline at end of file diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.xml --- a/lib/galaxy/datatypes/converters/vcf_to_summary_tree_converter.xml +++ /dev/null @@ -1,14 +0,0 @@ -<tool id="CONVERTER_vcf_to_summary_tree_0" name="Convert VCF to Summary Tree" version="1.0.0" hidden="true"> - <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> --> - <command interpreter="python">vcf_to_summary_tree_converter.py $input1 $output1</command> - <inputs> - <page> - <param format="vcf" name="input1" type="data" label="Choose VCF file"/> - </page> - </inputs> - <outputs> - <data format="summary_tree" name="output1"/> - </outputs> - <help> - </help> -</tool> diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/coverage.py --- a/lib/galaxy/datatypes/coverage.py +++ b/lib/galaxy/datatypes/coverage.py @@ -2,21 +2,14 @@ Coverage datatypes """ -import pkg_resources -pkg_resources.require( "bx-python" ) -import logging, os, sys, time, tempfile, shutil -import data -from galaxy import util -from galaxy.datatypes.sniff import * -from galaxy.web import url_for -from cgi import escape -import urllib -from bx.intervals.io import * +import logging +import math + +from galaxy import eggs from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.tabular import Tabular -import math log = logging.getLogger(__name__) @@ -34,7 +27,7 @@ Assumes we have a numpy file. """ # Maybe if we import here people will still be able to use Galaxy when numpy kills it - pkg_resources.require("numpy>=1.2.1") + eggs.require("numpy>=1.2.1") #from numpy.lib import format import numpy diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/data.py --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -2,6 +2,7 @@ import metadata import mimetypes import os +import shutil import sys import tempfile import zipfile @@ -17,12 +18,6 @@ eggs.require( "Paste" ) import paste - -if sys.version_info[:2] < ( 2, 6 ): - zipfile.BadZipFile = zipfile.error -if sys.version_info[:2] < ( 2, 5 ): - zipfile.LargeZipFile = zipfile.error - log = logging.getLogger(__name__) tmpd = tempfile.mkdtemp() @@ -103,6 +98,12 @@ #A per datatype setting (inherited): max file size (in bytes) for setting optional metadata _max_optional_metadata_filesize = None + # Trackster track type. + track_type = None + + # Data sources. + data_sources = {} + def __init__(self, **kwd): """Initialize the datatype""" object.__init__(self, **kwd) @@ -343,6 +344,7 @@ max_peek_size = 1000000 # 1 MB if isinstance(data.datatype, datatypes.images.Html): max_peek_size = 10000000 # 10 MB for html + preview = util.string_as_bool( preview ) if not preview or isinstance(data.datatype, datatypes.images.Image) or os.stat( data.file_name ).st_size < max_peek_size: if trans.app.config.sanitize_all_html and trans.response.get_content_type() == "text/html": # Sanitize anytime we respond with plain text/html content. @@ -544,21 +546,21 @@ return False - def merge( split_files, output_file): """ - TODO: Do we need to merge gzip files using gzjoin? cat seems to work, - but might be brittle. Need to revisit this. + Merge files with copy.copyfileobj() will not hit the + max argument limitation of cat. gz and bz2 files are also working. """ if not split_files: raise ValueError('Asked to merge zero files as %s' % output_file) elif len(split_files) == 1: - cmd = 'mv -f %s %s' % ( split_files[0], output_file ) + shutil.copyfileobj(open(split_files[0], 'rb'), open(output_file, 'wb')) else: - cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file ) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + fdst = open(output_file, 'wb') + for fsrc in split_files: + shutil.copyfileobj(open(fsrc, 'rb'), fdst) + fdst.close() + merge = staticmethod(merge) def get_visualizations( self, dataset ): @@ -566,7 +568,7 @@ Returns a list of visualizations for datatype. """ - if hasattr( self, 'get_track_type' ): + if self.track_type: return [ 'trackster', 'circster' ] return [] @@ -739,6 +741,10 @@ f.close() split = classmethod(split) +class GenericAsn1( Text ): + """Class for generic ASN.1 text format""" + file_ext = 'asn1' + class LineCount( Text ): """ Dataset contains a single line with a single integer that denotes the @@ -859,7 +865,7 @@ text = "%s file" % file_type else: try: - text = unicode( '\n'.join( lines ), 'utf-8' ) + text = util.unicodify( '\n'.join( lines ) ) except UnicodeDecodeError: text = "binary/unknown file" return text diff -r 98055c5c428ec019eb9738db8bb6f4331c163377 -r 1a3dc9d098e053592b6a066c8344b00c12ea905b lib/galaxy/datatypes/display_applications/application.py --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -12,7 +12,7 @@ log = logging.getLogger( __name__ ) #Any basic functions that we want to provide as a basic part of parameter dict should be added to this dict -BASE_PARAMS = { 'qp': quote_plus, 'url_for':url_for } #url_for has route memory... +BASE_PARAMS = { 'qp': quote_plus, 'url_for':url_for } class DisplayApplicationLink( object ): @classmethod @@ -40,7 +40,7 @@ self.name = None def get_display_url( self, data, trans ): dataset_hash, user_hash = encode_dataset_user( trans, data, None ) - return url_for( controller='/dataset', + return url_for( controller='dataset', action="display_application", dataset_id=dataset_hash, user_id=user_hash, This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/b1850aa88b03/ Changeset: b1850aa88b03 Branch: next-stable User: natefoo Date: 2013-05-20 21:55:46 Summary: Merge stable branch to next-stable. Affected #: 1 file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.