Hello,
I am getting an error when I run the following command:
ukbb.write('file_path', overwrite=True)
The error is posted below. When I ran the same code yesterday with the same set up I didn’t get an error, but I am now working with a larger dataset. Thanks!
ERROR:root:Exception while sending command.
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1159, in send_command
raise Py4JNetworkError("Answer from Java side is empty")
py4j.protocol.Py4JNetworkError: Answer from Java side is empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 985, in send_command
response = connection.send_command(command)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1164, in send_command
"Error while receiving", e, proto.ERROR_ON_RECEIVE)
py4j.protocol.Py4JNetworkError: Error while receiving
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:35125)
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-fbd959e44a2b>", line 8, in <module>
metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py", line 2426, in count
return Env.backend().execute(count_ir)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 16, in deco
return f(*args, **kwargs)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 336, in get_return_value
format(target_id, ".", name))
py4j.protocol.Py4JError: An error occurred while calling o164.executeJSON
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2044, in showtraceback
stb = value._render_traceback_()
AttributeError: 'Py4JError' object has no attribute '_render_traceback_'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 929, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1067, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:35125)
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-fbd959e44a2b>", line 8, in <module>
metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py", line 2426, in count
return Env.backend().execute(count_ir)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 16, in deco
return f(*args, **kwargs)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 336, in get_return_value
format(target_id, ".", name))
py4j.protocol.Py4JError: An error occurred while calling o164.executeJSON
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2044, in showtraceback
stb = value._render_traceback_()
AttributeError: 'Py4JError' object has no attribute '_render_traceback_'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 929, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1067, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:35125)
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-fbd959e44a2b>", line 8, in <module>
metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py", line 2426, in count
return Env.backend().execute(count_ir)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 16, in deco
return f(*args, **kwargs)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 336, in get_return_value
format(target_id, ".", name))
py4j.protocol.Py4JError: An error occurred while calling o164.executeJSON
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2044, in showtraceback
stb = value._render_traceback_()
AttributeError: 'Py4JError' object has no attribute '_render_traceback_'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 929, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1067, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:35125)
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-fbd959e44a2b>", line 8, in <module>
metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py", line 2426, in count
return Env.backend().execute(count_ir)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 16, in deco
return f(*args, **kwargs)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 336, in get_return_value
format(target_id, ".", name))
py4j.protocol.Py4JError: An error occurred while calling o164.executeJSON
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2044, in showtraceback
stb = value._render_traceback_()
AttributeError: 'Py4JError' object has no attribute '_render_traceback_'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 929, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1067, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:35125)
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-fbd959e44a2b>", line 8, in <module>
metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py", line 2426, in count
return Env.backend().execute(count_ir)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 16, in deco
return f(*args, **kwargs)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 336, in get_return_value
format(target_id, ".", name))
py4j.protocol.Py4JError: An error occurred while calling o164.executeJSON
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2044, in showtraceback
stb = value._render_traceback_()
AttributeError: 'Py4JError' object has no attribute '_render_traceback_'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 929, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1067, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:35125)
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-fbd959e44a2b>", line 8, in <module>
metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py", line 2426, in count
return Env.backend().execute(count_ir)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 16, in deco
return f(*args, **kwargs)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 336, in get_return_value
format(target_id, ".", name))
py4j.protocol.Py4JError: An error occurred while calling o164.executeJSON
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2044, in showtraceback
stb = value._render_traceback_()
AttributeError: 'Py4JError' object has no attribute '_render_traceback_'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 929, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1067, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:35125)
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-fbd959e44a2b>", line 8, in <module>
metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py", line 2426, in count
return Env.backend().execute(count_ir)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 16, in deco
return f(*args, **kwargs)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 336, in get_return_value
format(target_id, ".", name))
py4j.protocol.Py4JError: An error occurred while calling o164.executeJSON
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2044, in showtraceback
stb = value._render_traceback_()
AttributeError: 'Py4JError' object has no attribute '_render_traceback_'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 929, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1067, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:35125)
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-fbd959e44a2b>", line 8, in <module>
metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py", line 2426, in count
return Env.backend().execute(count_ir)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
answer, self.gateway_client, self.target_id, self.name)
File "/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py", line 16, in deco
return f(*args, **kwargs)
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py", line 336, in get_return_value
format(target_id, ".", name))
py4j.protocol.Py4JError: An error occurred while calling o164.executeJSON
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/conda/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2044, in showtraceback
stb = value._render_traceback_()
AttributeError: 'Py4JError' object has no attribute '_render_traceback_'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 929, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1067, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
---------------------------------------------------------------------------
Py4JError Traceback (most recent call last)
<ipython-input-7-fbd959e44a2b> in <module>
6 while(True):
7 if not path.exists("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt"):
----> 8 metadata.to_csv("gs://ukb-gt/ukbQC_"+str(ukbb.count()[0])+"_x_"+str(ukbb.count()[1])+"_metadata"+str(imeta)+".txt", index=False)
9 break
10
/opt/conda/miniconda3/lib/python3.6/site-packages/hail/matrixtable.py in count(self)
2424 """
2425 count_ir = ir.MatrixCount(self._mir)
-> 2426 return Env.backend().execute(count_ir)
2427
2428 @typecheck_method(output=str,
/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py in execute(self, ir, timed)
72 # print(self._hail_package.expr.ir.Pretty.apply(jir, True, -1))
73 try:
---> 74 result = json.loads(self._jhc.backend().executeJSON(jir))
75 value = ir.typ._from_json(result['value'])
76 timings = result['timings']
/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py in __call__(self, *args)
1255 answer = self.gateway_client.send_command(command)
1256 return_value = get_return_value(
-> 1257 answer, self.gateway_client, self.target_id, self.name)
1258
1259 for temp_arg in temp_args:
/opt/conda/miniconda3/lib/python3.6/site-packages/hail/backend/py4j_backend.py in deco(*args, **kwargs)
14 import pyspark
15 try:
---> 16 return f(*args, **kwargs)
17 except py4j.protocol.Py4JJavaError as e:
18 s = e.java_exception.toString()
/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
334 raise Py4JError(
335 "An error occurred while calling {0}{1}{2}".
--> 336 format(target_id, ".", name))
337 else:
338 type = answer[1]
Py4JError: An error occurred while calling o164.executeJSON
'''