Hi,
I'm using bioblend to populate a library in galaxy. it's work well (bam and vcf).
But I have issues with some gff3.gz files.
For few of them (over 40 files) I see errors in the log.
If I upload that file (that fails with bioblend) it works using the upload tool on the
interface. It's auto-detected as "tabular". I can then change datatype to
gff3.gz. It works (with a little issue, the file is described as: 1 line, 41,085 comments.
That's not true, there is not comments in my file).
If I upload that file using the upload tool on the interface, it will fail if I set the
datatype to gff3.gz during the upload process. I see then the same type of error (as I see
with my bioblend script).
I need some ideas, do you have tips ? Maybe some clues to follow...
thank you.
error type is :
Traceback (most recent call last):
File "lib/galaxy/jobs/runners/__init__.py", line 510, in
_finish_or_resubmit_job
job_wrapper.finish(tool_stdout, tool_stderr, exit_code,
check_output_detected_state=check_output_detected_state, job_stdout=job_stdout,
job_stderr=job_stderr)
File "lib/galaxy/jobs/__init__.py", line 1428, in finish
self.sa_session.flush()
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/orm/scoping.py",
line 162, in do
return getattr(self.registry(), name)(*args, **kwargs)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/orm/session.py",
line 2446, in flush
self._flush(objects)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/orm/session.py",
line 2584, in _flush
transaction.rollback(_capture_exception=True)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py",
line 67, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/orm/session.py",
line 2544, in _flush
flush_context.execute()
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 416, in execute
rec.execute(self)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py",
line 583, in execute
uow,
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py",
line 245, in save_obj
insert,
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py",
line 1116, in _emit_insert_statements
statement, params
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py",
line 980, in execute
return meth(self, multiparams, params)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/sql/elements.py",
line 273, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py",
line 1099, in _execute_clauseelement
distilled_params,
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py",
line 1174, in _execute_context
e, util.text_type(statement), parameters, None, None
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py",
line 1458, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/util/compat.py",
line 296, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py",
line 1171, in _execute_context
context = constructor(dialect, self, conn, *args)
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/engine/default.py",
line 719, in _init_compiled
for key in compiled_params
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/engine/default.py",
line 719, in <genexpr>
for key in compiled_params
File
"/softs/bioinfo/galaxy-dev/.venv/lib/python2.7/site-packages/sqlalchemy/sql/type_api.py",
line 1189, in process
return impl_processor(process_param(value, dialect))
File "lib/galaxy/model/custom_types.py", line 319, in process_bind_param
value = json_encoder.encode(value).encode()
File "/softs/add-ons/Python-2.7.12/lib/python2.7/json/encoder.py", line 209,
in encode
chunks = list(chunks)
File "/softs/add-ons/Python-2.7.12/lib/python2.7/json/encoder.py", line 434,
in _iterencode
for chunk in _iterencode_dict(o, _current_indent_level):
File "/softs/add-ons/Python-2.7.12/lib/python2.7/json/encoder.py", line 408,
in _iterencode_dict
for chunk in chunks:
File "/softs/add-ons/Python-2.7.12/lib/python2.7/json/encoder.py", line 387,
in _iterencode_dict
yield _encoder(key)
StatementError: (exceptions.UnicodeDecodeError) 'utf8' codec can't decode byte
0xa5 in position 9: invalid start byte [SQL: u'INSERT INTO
history_dataset_association_history (history_dataset_association_id, update_time, version,
name, extension, metadata, extended_metadata_id) VALUES
(%(history_dataset_association_id)s, %(update_time)s, %(version)s, %(name)s,
%(extension)s, %(_metadata)s, %(extended_metadata_id)s) RETURNING
history_dataset_association_history.id'] [parameters: [{'update_time':
datetime.datetime(2019, 8, 26, 7, 53, 29, 457330),
'history_dataset_association_id': 3, 'extension': u'gff3.gz',
'extended_metadata_id': None, '_metadata': {'attribute_types':
{'\xc4\xa5v\x10\x04>\xd5\x81U\xa5\\\xc7\xa2\xb3t_d\x07': 'str'},
'comment_lines': 2528, u'dbkey': u'?', 'delimiter':
'\t', 'columns': 9, 'column_types': ['str', 'str',
'str', 'int', 'int', 'float', 'str',
'str', 'str'], 'attributes': 1, 'data_lines': 38558},
'version': 1, 'name': u'Final_ZmCML247v1a.collinearity.gff.gz'}]]
Fred