--------------------------------------------------------------------------- FatalError Traceback (most recent call last) in 3 4 # pp.to_csv("~/Documents/Code/ukb_core_cats/test_r.tsv.gz", sep='\t', compression='gzip', index=False) ----> 5 rt.export("~/Documents/Code/ukb_core_cats/test_r.tsv.bgz") in export(self, output, types_file, header, parallel, delimiter) ~/opt/anaconda3/lib/python3.8/site-packages/hail/typecheck/check.py in wrapper(__original_func, *args, **kwargs) 575 def wrapper(__original_func, *args, **kwargs): 576 args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method) --> 577 return __original_func(*args_, **kwargs_) 578 579 return wrapper ~/opt/anaconda3/lib/python3.8/site-packages/hail/table.py in export(self, output, types_file, header, parallel, delimiter) 1096 """ 1097 parallel = ir.ExportType.default(parallel) -> 1098 Env.backend().execute( 1099 ir.TableWrite(self._tir, ir.TableTextWriter(output, types_file, header, parallel, delimiter))) 1100 ~/opt/anaconda3/lib/python3.8/site-packages/hail/backend/service_backend.py in execute(self, ir, timed) 448 449 def execute(self, ir: BaseIR, timed: bool = False): --> 450 return async_to_blocking(self._async_execute(ir, timed=timed)) 451 452 async def _async_execute(self, ir: BaseIR, timed: bool = False): ~/opt/anaconda3/lib/python3.8/site-packages/hailtop/utils/utils.py in async_to_blocking(coro) 149 150 def async_to_blocking(coro: Awaitable[T]) -> T: --> 151 return asyncio.get_event_loop().run_until_complete(coro) 152 153 ~/opt/anaconda3/lib/python3.8/site-packages/nest_asyncio.py in run_until_complete(self, future) 79 raise RuntimeError( 80 'Event loop stopped before Future completed.') ---> 81 return f.result() 82 83 def _run_once(self): ~/opt/anaconda3/lib/python3.8/asyncio/futures.py in result(self) 176 self.__log_traceback = False 177 if self._exception is not None: --> 178 raise self._exception 179 return self._result 180 ~/opt/anaconda3/lib/python3.8/asyncio/tasks.py in __step(***failed resolving arguments***) 278 # We use the `send` method directly, because coroutines 279 # don't have `__iter__` and `__next__` methods. --> 280 result = coro.send(None) 281 else: 282 result = coro.throw(exc) ~/opt/anaconda3/lib/python3.8/site-packages/hail/backend/service_backend.py in _async_execute(self, ir, timed) 463 await write_str(infile, '{"name":"StreamBufferSpec"}') 464 --> 465 _, resp, timings = await self._rpc('execute(...)', inputs, ir=ir) 466 typ: HailType = ir.typ 467 if typ == tvoid: ~/opt/anaconda3/lib/python3.8/site-packages/hail/backend/service_backend.py in _rpc(self, name, inputs, ir) 444 'cause': {'id': batch_id, 'batch_status': b2_status, 'failed_jobs': failed_jobs}} 445 log.error(yaml.dump(message)) --> 446 raise FatalError(orjson.dumps(message).decode('utf-8')) 447 raise FatalError(f'batch id was {b.id}\n' + short_message + '\n' + expanded_message) 448 FatalError: {"id":2077740,"service_backend_debug_info":{"jar_spec":"GitRevision(d77cdf0157c9c1f37657f0da86747b61ae25de88)","billing_project":"rwalters-trial","batch_attributes":{},"user_local_reference_cache_dir":"/Users/raymondwalters/.config/hail/cache/references/0.2.93-d77cdf0157c9","remote_tmpdir":"gs://rwalters-hail-tmp/","flags":{"use_new_shuffle":"1"},"driver_cores":"1","driver_memory":"standard"},"short_message":"HailBatchFailure: 2077741","expanded_message":"is.hail.backend.service.HailBatchFailure: 2077741\n\tat is.hail.backend.service.ServiceBackend.parallelizeAndComputeWithIndex(ServiceBackend.scala:192)\n\tat is.hail.backend.BackendUtils.collectDArray(BackendUtils.scala:28)\n\tat __C10486Compiled.__m10559split_CollectDistributedArray(Emit.scala)\n\tat __C10486Compiled.__m10558begin_group_0(Emit.scala)\n\tat __C10486Compiled.apply(Emit.scala)\n\tat is.hail.backend.service.ServiceBackend.execute(ServiceBackend.scala:302)\n\tat is.hail.backend.service.ServiceBackend.execute(ServiceBackend.scala:329)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2.$anonfun$executeOneCommand$12(ServiceBackend.scala:602)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2.withIRFunctionsReadFromInput(ServiceBackend.scala:727)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2.$anonfun$executeOneCommand$11(ServiceBackend.scala:600)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2.$anonfun$executeOneCommand$4(ServiceBackend.scala:552)\n\tat is.hail.backend.ExecuteContext$.$anonfun$scoped$3(ExecuteContext.scala:69)\n\tat is.hail.utils.package$.using(package.scala:638)\n\tat is.hail.backend.ExecuteContext$.$anonfun$scoped$2(ExecuteContext.scala:69)\n\tat is.hail.utils.package$.using(package.scala:638)\n\tat is.hail.annotations.RegionPool$.scoped(RegionPool.scala:17)\n\tat is.hail.backend.ExecuteContext$.scoped(ExecuteContext.scala:58)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2.$anonfun$executeOneCommand$1(ServiceBackend.scala:550)\n\tat is.hail.utils.ExecutionTimer$.time(ExecutionTimer.scala:52)\n\tat is.hail.utils.ExecutionTimer$.logTime(ExecutionTimer.scala:59)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2.withExecuteContext$1(ServiceBackend.scala:535)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2.executeOneCommand(ServiceBackend.scala:597)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2$.$anonfun$main$7(ServiceBackend.scala:433)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2$.$anonfun$main$7$adapted(ServiceBackend.scala:432)\n\tat is.hail.utils.package$.using(package.scala:638)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2$.$anonfun$main$6(ServiceBackend.scala:432)\n\tat scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\n\tat is.hail.services.package$.retryTransientErrors(package.scala:77)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2$.$anonfun$main$5(ServiceBackend.scala:432)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2$.$anonfun$main$5$adapted(ServiceBackend.scala:430)\n\tat is.hail.utils.package$.using(package.scala:638)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2$.$anonfun$main$4(ServiceBackend.scala:430)\n\tat scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\n\tat is.hail.services.package$.retryTransientErrors(package.scala:77)\n\tat is.hail.backend.service.ServiceBackendSocketAPI2$.main(ServiceBackend.scala:430)\n\tat is.hail.backend.service.Main$.main(Main.scala:33)\n\tat is.hail.backend.service.Main.main(Main.scala)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat is.hail.JVMEntryway$1.run(JVMEntryway.java:105)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)","cause":{"id":"2077741","batch_status":{"id":2077741,"user":"rwalters","billing_project":"rwalters-trial","token":"fk_NYnDaiPkqwqIKmJZ9DU7wTX9N0Fm9rjrLJzF4BIE=","state":"failure","complete":true,"closed":true,"n_jobs":22925,"n_completed":22925,"n_succeeded":22923,"n_failed":2,"n_cancelled":0,"time_created":"2022-04-20T02:35:15Z","time_closed":"2022-04-20T02:35:19Z","time_completed":"2022-04-20T02:50:57Z","duration":"15 minutes","attributes":{"name":"execute(...)_0"},"msec_mcpu":0,"cost":2.7086310726973526},"failed_jobs":[{"partial_status":{"batch_id":2077741,"job_id":5110,"name":null,"user":"rwalters","billing_project":"rwalters-trial","state":"Failed","exit_code":null,"duration":1864,"msec_mcpu":1864000,"cost":0.000011357708951251048},"status":{"batch_id":2077741,"job_id":5110,"name":null,"user":"rwalters","billing_project":"rwalters-trial","state":"Failed","exit_code":null,"duration":1864,"msec_mcpu":1864000,"cost":0.000011357708951251048,"status":{"version":5,"worker":"batch-worker-default-standard-kkviu","batch_id":2077741,"job_id":5110,"attempt_id":"55fGEX","user":"rwalters","state":"failed","format_version":6,"resources":[{"name":"compute/n1-preemptible/1","quantity":1000},{"name":"memory/n1-preemptible/1","quantity":3840},{"name":"disk/pd-ssd/1","quantity":640},{"name":"disk/local-ssd/1","quantity":24000},{"name":"ip-fee/1024/1","quantity":64},{"name":"service-fee/1","quantity":1000}],"error":"Traceback (most recent call last):\n File \"/usr/local/lib/python3.7/dist-packages/batch/worker/worker.py\", line 1856, in run\n await self.jvm.execute(local_jar_location, self.scratch, self.log_file, self.jar_url, self.argv)\n File \"/usr/local/lib/python3.7/dist-packages/batch/worker/worker.py\", line 2275, in execute\n raise JVMUserError(exception)\nJVMUserError: java.util.concurrent.ExecutionException: java.lang.RuntimeException: java.lang.reflect.InvocationTargetException\n\tat java.util.concurrent.FutureTask.report(FutureTask.java:122)\n\tat java.util.concurrent.FutureTask.get(FutureTask.java:192)\n\tat is.hail.JVMEntryway.retrieveException(JVMEntryway.java:224)\n\tat is.hail.JVMEntryway.finishFutures(JVMEntryway.java:186)\n\tat is.hail.JVMEntryway.main(JVMEntryway.java:156)\nCaused by: java.lang.RuntimeException: java.lang.reflect.InvocationTargetException\n\tat is.hail.JVMEntryway$1.run(JVMEntryway.java:107)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat is.hail.JVMEntryway$1.run(JVMEntryway.java:105)\n\t... 7 more\nCaused by: is.hail.relocated.com.google.cloud.storage.StorageException: Connection reset\n\tat is.hail.relocated.com.google.cloud.storage.StorageException.translate(StorageException.java:97)\n\tat is.hail.relocated.com.google.cloud.storage.spi.v1.HttpStorageRpc.read(HttpStorageRpc.java:704)\n\tat is.hail.relocated.com.google.cloud.storage.BlobReadChannel$1.call(BlobReadChannel.java:127)\n\tat is.hail.relocated.com.google.cloud.storage.BlobReadChannel$1.call(BlobReadChannel.java:124)\n\tat com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105)\n\tat is.hail.relocated.com.google.cloud.RetryHelper.run(RetryHelper.java:76)\n\tat is.hail.relocated.com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50)\n\tat is.hail.relocated.com.google.cloud.storage.BlobReadChannel.read(BlobReadChannel.java:123)\n\tat is.hail.io.fs.GoogleStorageFS$$anon$1.fill(GoogleStorageFS.scala:158)\n\tat is.hail.io.fs.GoogleStorageFS$$anon$1.read(GoogleStorageFS.scala:188)\n\tat java.io.DataInputStream.read(DataInputStream.java:149)\n\tat is.hail.utils.richUtils.RichInputStream$.readRepeatedly$extension0(RichInputStream.scala:21)\n\tat is.hail.utils.richUtils.RichInputStream$.readFully$extension1(RichInputStream.scala:12)\n\tat is.hail.io.StreamBlockInputBuffer.readBlock(InputBuffers.scala:550)\n\tat is.hail.io.LZ4InputBlockBuffer.readBlock(InputBuffers.scala:584)\n\tat is.hail.io.BlockingInputBuffer.readBlock(InputBuffers.scala:382)\n\tat is.hail.io.BlockingInputBuffer.ensure(InputBuffers.scala:388)\n\tat is.hail.io.BlockingInputBuffer.skipDouble(InputBuffers.scala:499)\n\tat is.hail.io.LEB128InputBuffer.skipDouble(InputBuffers.scala:270)\n\tat __C10862collect_distributed_array.__m10889SKIP_o_float64(Unknown Source)\n\tat __C10862collect_distributed_array.__m10896INPLACE_DECODE_o_struct_of_o_float64ANDo_float64ANDo_boolANDo_int32ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64END_TO_o_struct_of_o_boolANDo_float64END(Unknown Source)\n\tat __C10862collect_distributed_array.__m10895INPLACE_DECODE_o_array_of_o_struct_of_o_float64ANDo_float64ANDo_boolANDo_int32ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64END_TO_o_array_of_o_struct_of_o_boolANDo_float64END(Unknown Source)\n\tat __C10862collect_distributed_array.__m10894DECODE_r_struct_of_o_array_of_o_struct_of_o_float64ANDo_float64ANDo_boolANDo_int32ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64ENDEND_TO_SBaseStructPointer(Unknown Source)\n\tat __C10862collect_distributed_array.apply_region9_28(Unknown Source)\n\tat __C10862collect_distributed_array.apply(Unknown Source)\n\tat __C10862collect_distributed_array.apply(Unknown Source)\n\tat is.hail.backend.BackendUtils.$anonfun$collectDArray$2(BackendUtils.scala:31)\n\tat is.hail.utils.package$.using(package.scala:638)\n\tat is.hail.annotations.RegionPool.scopedRegion(RegionPool.scala:162)\n\tat is.hail.backend.BackendUtils.$anonfun$collectDArray$1(BackendUtils.scala:30)\n\tat is.hail.backend.service.Worker$.main(Worker.scala:141)\n\tat is.hail.backend.service.Main$.main(Main.scala:32)\n\tat is.hail.backend.service.Main.main(Main.scala)\n\t... 12 more\nCaused by: javax.net.ssl.SSLException: Connection reset\n\tat sun.security.ssl.Alert.createSSLException(Alert.java:127)\n\tat sun.security.ssl.TransportContext.fatal(TransportContext.java:324)\n\tat sun.security.ssl.TransportContext.fatal(TransportContext.java:267)\n\tat sun.security.ssl.TransportContext.fatal(TransportContext.java:262)\n\tat sun.security.ssl.SSLTransport.decode(SSLTransport.java:138)\n\tat sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1395)\n\tat sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1363)\n\tat sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73)\n\tat sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:957)\n\tat java.io.BufferedInputStream.fill(BufferedInputStream.java:246)\n\tat java.io.BufferedInputStream.read1(BufferedInputStream.java:286)\n\tat java.io.BufferedInputStream.read(BufferedInputStream.java:345)\n\tat sun.net.www.MeteredStream.read(MeteredStream.java:134)\n\tat java.io.FilterInputStream.read(FilterInputStream.java:133)\n\tat sun.net.www.protocol.http.HttpURLConnection$HttpInputStream.read(HttpURLConnection.java:3451)\n\tat com.google.api.client.http.javanet.NetHttpResponse$SizeValidatingInputStream.read(NetHttpResponse.java:164)\n\tat java.io.FilterInputStream.read(FilterInputStream.java:107)\n\tat com.google.api.client.util.ByteStreams.copy(ByteStreams.java:49)\n\tat com.google.api.client.util.IOUtils.copy(IOUtils.java:87)\n\tat com.google.api.client.util.IOUtils.copy(IOUtils.java:59)\n\tat com.google.api.client.http.HttpResponse.download(HttpResponse.java:410)\n\tat is.hail.relocated.com.google.cloud.storage.spi.v1.HttpStorageRpc.read(HttpStorageRpc.java:699)\n\t... 43 more\n\tSuppressed: java.net.SocketException: Broken pipe (Write failed)\n\t\tat java.net.SocketOutputStream.socketWrite0(Native Method)\n\t\tat java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)\n\t\tat java.net.SocketOutputStream.write(SocketOutputStream.java:155)\n\t\tat sun.security.ssl.SSLSocketOutputRecord.encodeAlert(SSLSocketOutputRecord.java:81)\n\t\tat sun.security.ssl.TransportContext.fatal(TransportContext.java:355)\n\t\t... 63 more\nCaused by: java.net.SocketException: Connection reset\n\tat java.net.SocketInputStream.read(SocketInputStream.java:210)\n\tat java.net.SocketInputStream.read(SocketInputStream.java:141)\n\tat sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:464)\n\tat sun.security.ssl.SSLSocketInputRecord.decodeInputRecord(SSLSocketInputRecord.java:237)\n\tat sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:190)\n\tat sun.security.ssl.SSLTransport.decode(SSLTransport.java:109)\n\t... 60 more","start_time":1650422361982,"end_time":1650422363846,"container_statuses":{"main":{"name":"main","state":"failed","timing":{"connecting_to_jvm":{"start_time":1650422361982,"finish_time":1650422361982,"duration":0},"downloading_jar":{"start_time":1650422362004,"finish_time":1650422362004,"duration":0},"running":{"start_time":1650422362005},"uploading_log":{"start_time":1650422363718,"finish_time":1650422363835,"duration":117}}}},"jvm":"JVM-3"},"spec":{"job_id":5110,"process":{"jar_spec":{"type":"jar_url","value":"gs://hail-query/jars/d77cdf0157c9c1f37657f0da86747b61ae25de88.jar"},"command":["worker","gs://rwalters-hail-tmp/parallelizeAndComputeWithIndex/fk_NYnDaiPkqwqIKmJZ9DU7wTX9N0Fm9rjrLJzF4BIE=","5109","22925"],"type":"jvm"},"mount_tokens":true,"resources":{"preemptible":true,"req_cpu":"1","req_memory":"standard","req_storage":"0Gi","cores_mcpu":1000,"memory_bytes":4026531840,"storage_gib":0},"secrets":[{"namespace":"default","name":"rwalters-i04mj-gsa-key","mount_path":"/gsa-key","mount_in_copy":true},{"namespace":"default","name":"rwalters-i04mj-tokens","mount_path":"/user-tokens","mount_in_copy":false},{"namespace":"default","name":"worker-deploy-config","mount_path":"/deploy-config","mount_in_copy":false},{"namespace":"default","name":"ssl-config-batch-user-code","mount_path":"/ssl-config","mount_in_copy":false}],"env":[{"name":"GOOGLE_APPLICATION_CREDENTIALS","value":"/gsa-key/key.json"}]}},"log":"2022-04-20 02:39:22 Tokens: INFO: tokens found for namespaces {default}\n2022-04-20 02:39:22 tls: INFO: ssl config file found at /batch/3aa22c2561af48db8f97d091666020b6/secrets/ssl-config/ssl-config.json\n2022-04-20 02:39:22 Worker$: INFO: is.hail.backend.service.Worker d77cdf0157c9c1f37657f0da86747b61ae25de88\n2022-04-20 02:39:22 Worker$: INFO: running job 5109/22925 at root gs://rwalters-hail-tmp/parallelizeAndComputeWithIndex/fk_NYnDaiPkqwqIKmJZ9DU7wTX9N0Fm9rjrLJzF4BIE= with scratch directory '/batch/3aa22c2561af48db8f97d091666020b6'\n2022-04-20 02:39:22 GoogleStorageFS$: INFO: Initializing google storage client from service account key\n2022-04-20 02:39:22 WorkerTimer$: INFO: readInputs took 142.663198 ms.\n2022-04-20 02:39:22 root: INFO: RegionPool: initialized for thread 8: pool-1-thread-1\n2022-04-20 02:39:22 root: INFO: RegionPool: REPORT_THRESHOLD: 320.0K allocated (192.0K blocks / 128.0K chunks), regions.size = 3, 0 current java objects, thread 8: pool-1-thread-1"},{"partial_status":{"batch_id":2077741,"job_id":5328,"name":null,"user":"rwalters","billing_project":"rwalters-trial","state":"Failed","exit_code":null,"duration":1924,"msec_mcpu":1924000,"cost":0.000011723300441098184},"status":{"batch_id":2077741,"job_id":5328,"name":null,"user":"rwalters","billing_project":"rwalters-trial","state":"Failed","exit_code":null,"duration":1924,"msec_mcpu":1924000,"cost":0.000011723300441098184,"status":{"version":5,"worker":"batch-worker-default-standard-kkviu","batch_id":2077741,"job_id":5328,"attempt_id":"Gl2kdZ","user":"rwalters","state":"failed","format_version":6,"resources":[{"name":"compute/n1-preemptible/1","quantity":1000},{"name":"memory/n1-preemptible/1","quantity":3840},{"name":"disk/pd-ssd/1","quantity":640},{"name":"disk/local-ssd/1","quantity":24000},{"name":"ip-fee/1024/1","quantity":64},{"name":"service-fee/1","quantity":1000}],"error":"Traceback (most recent call last):\n File \"/usr/local/lib/python3.7/dist-packages/batch/worker/worker.py\", line 1856, in run\n await self.jvm.execute(local_jar_location, self.scratch, self.log_file, self.jar_url, self.argv)\n File \"/usr/local/lib/python3.7/dist-packages/batch/worker/worker.py\", line 2275, in execute\n raise JVMUserError(exception)\nJVMUserError: java.util.concurrent.ExecutionException: java.lang.RuntimeException: java.lang.reflect.InvocationTargetException\n\tat java.util.concurrent.FutureTask.report(FutureTask.java:122)\n\tat java.util.concurrent.FutureTask.get(FutureTask.java:192)\n\tat is.hail.JVMEntryway.retrieveException(JVMEntryway.java:224)\n\tat is.hail.JVMEntryway.finishFutures(JVMEntryway.java:186)\n\tat is.hail.JVMEntryway.main(JVMEntryway.java:156)\nCaused by: java.lang.RuntimeException: java.lang.reflect.InvocationTargetException\n\tat is.hail.JVMEntryway$1.run(JVMEntryway.java:107)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: java.lang.reflect.InvocationTargetException\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat is.hail.JVMEntryway$1.run(JVMEntryway.java:105)\n\t... 7 more\nCaused by: is.hail.relocated.com.google.cloud.storage.StorageException: Connection reset\n\tat is.hail.relocated.com.google.cloud.storage.StorageException.translate(StorageException.java:97)\n\tat is.hail.relocated.com.google.cloud.storage.spi.v1.HttpStorageRpc.read(HttpStorageRpc.java:704)\n\tat is.hail.relocated.com.google.cloud.storage.BlobReadChannel$1.call(BlobReadChannel.java:127)\n\tat is.hail.relocated.com.google.cloud.storage.BlobReadChannel$1.call(BlobReadChannel.java:124)\n\tat com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:105)\n\tat is.hail.relocated.com.google.cloud.RetryHelper.run(RetryHelper.java:76)\n\tat is.hail.relocated.com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:50)\n\tat is.hail.relocated.com.google.cloud.storage.BlobReadChannel.read(BlobReadChannel.java:123)\n\tat is.hail.io.fs.GoogleStorageFS$$anon$1.fill(GoogleStorageFS.scala:158)\n\tat is.hail.io.fs.GoogleStorageFS$$anon$1.read(GoogleStorageFS.scala:188)\n\tat java.io.DataInputStream.read(DataInputStream.java:149)\n\tat is.hail.utils.richUtils.RichInputStream$.readRepeatedly$extension0(RichInputStream.scala:21)\n\tat is.hail.utils.richUtils.RichInputStream$.readFully$extension1(RichInputStream.scala:12)\n\tat is.hail.io.StreamBlockInputBuffer.readBlock(InputBuffers.scala:550)\n\tat is.hail.io.LZ4InputBlockBuffer.readBlock(InputBuffers.scala:584)\n\tat is.hail.io.BlockingInputBuffer.readBlock(InputBuffers.scala:382)\n\tat is.hail.io.BlockingInputBuffer.ensure(InputBuffers.scala:388)\n\tat is.hail.io.BlockingInputBuffer.skipDouble(InputBuffers.scala:499)\n\tat is.hail.io.LEB128InputBuffer.skipDouble(InputBuffers.scala:270)\n\tat __C10862collect_distributed_array.__m10889SKIP_o_float64(Unknown Source)\n\tat __C10862collect_distributed_array.__m10896INPLACE_DECODE_o_struct_of_o_float64ANDo_float64ANDo_boolANDo_int32ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64END_TO_o_struct_of_o_boolANDo_float64END(Unknown Source)\n\tat __C10862collect_distributed_array.__m10895INPLACE_DECODE_o_array_of_o_struct_of_o_float64ANDo_float64ANDo_boolANDo_int32ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64END_TO_o_array_of_o_struct_of_o_boolANDo_float64END(Unknown Source)\n\tat __C10862collect_distributed_array.__m10894DECODE_r_struct_of_o_array_of_o_struct_of_o_float64ANDo_float64ANDo_boolANDo_int32ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64ANDo_float64ENDEND_TO_SBaseStructPointer(Unknown Source)\n\tat __C10862collect_distributed_array.apply_region9_28(Unknown Source)\n\tat __C10862collect_distributed_array.apply(Unknown Source)\n\tat __C10862collect_distributed_array.apply(Unknown Source)\n\tat is.hail.backend.BackendUtils.$anonfun$collectDArray$2(BackendUtils.scala:31)\n\tat is.hail.utils.package$.using(package.scala:638)\n\tat is.hail.annotations.RegionPool.scopedRegion(RegionPool.scala:162)\n\tat is.hail.backend.BackendUtils.$anonfun$collectDArray$1(BackendUtils.scala:30)\n\tat is.hail.backend.service.Worker$.main(Worker.scala:141)\n\tat is.hail.backend.service.Main$.main(Main.scala:32)\n\tat is.hail.backend.service.Main.main(Main.scala)\n\t... 12 more\nCaused by: javax.net.ssl.SSLException: Connection reset\n\tat sun.security.ssl.Alert.createSSLException(Alert.java:127)\n\tat sun.security.ssl.TransportContext.fatal(TransportContext.java:324)\n\tat sun.security.ssl.TransportContext.fatal(TransportContext.java:267)\n\tat sun.security.ssl.TransportContext.fatal(TransportContext.java:262)\n\tat sun.security.ssl.SSLTransport.decode(SSLTransport.java:138)\n\tat sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1395)\n\tat sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1363)\n\tat sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73)\n\tat sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:957)\n\tat java.io.BufferedInputStream.fill(BufferedInputStream.java:246)\n\tat java.io.BufferedInputStream.read1(BufferedInputStream.java:286)\n\tat java.io.BufferedInputStream.read(BufferedInputStream.java:345)\n\tat sun.net.www.MeteredStream.read(MeteredStream.java:134)\n\tat java.io.FilterInputStream.read(FilterInputStream.java:133)\n\tat sun.net.www.protocol.http.HttpURLConnection$HttpInputStream.read(HttpURLConnection.java:3451)\n\tat com.google.api.client.http.javanet.NetHttpResponse$SizeValidatingInputStream.read(NetHttpResponse.java:164)\n\tat java.io.FilterInputStream.read(FilterInputStream.java:107)\n\tat com.google.api.client.util.ByteStreams.copy(ByteStreams.java:49)\n\tat com.google.api.client.util.IOUtils.copy(IOUtils.java:87)\n\tat com.google.api.client.util.IOUtils.copy(IOUtils.java:59)\n\tat com.google.api.client.http.HttpResponse.download(HttpResponse.java:410)\n\tat is.hail.relocated.com.google.cloud.storage.spi.v1.HttpStorageRpc.read(HttpStorageRpc.java:699)\n\t... 43 more\n\tSuppressed: java.net.SocketException: Broken pipe (Write failed)\n\t\tat java.net.SocketOutputStream.socketWrite0(Native Method)\n\t\tat java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)\n\t\tat java.net.SocketOutputStream.write(SocketOutputStream.java:155)\n\t\tat sun.security.ssl.SSLSocketOutputRecord.encodeAlert(SSLSocketOutputRecord.java:81)\n\t\tat sun.security.ssl.TransportContext.fatal(TransportContext.java:355)\n\t\t... 63 more\nCaused by: java.net.SocketException: Connection reset\n\tat java.net.SocketInputStream.read(SocketInputStream.java:210)\n\tat java.net.SocketInputStream.read(SocketInputStream.java:141)\n\tat sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:464)\n\tat sun.security.ssl.SSLSocketInputRecord.decodeInputRecord(SSLSocketInputRecord.java:237)\n\tat sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:190)\n\tat sun.security.ssl.SSLTransport.decode(SSLTransport.java:109)\n\t... 60 more","start_time":1650422364988,"end_time":1650422366912,"container_statuses":{"main":{"name":"main","state":"failed","timing":{"connecting_to_jvm":{"start_time":1650422364988,"finish_time":1650422364988,"duration":0},"downloading_jar":{"start_time":1650422365008,"finish_time":1650422365009,"duration":1},"running":{"start_time":1650422365009},"uploading_log":{"start_time":1650422366836,"finish_time":1650422366901,"duration":65}}}},"jvm":"JVM-0"},"spec":{"job_id":5328,"process":{"jar_spec":{"type":"jar_url","value":"gs://hail-query/jars/d77cdf0157c9c1f37657f0da86747b61ae25de88.jar"},"command":["worker","gs://rwalters-hail-tmp/parallelizeAndComputeWithIndex/fk_NYnDaiPkqwqIKmJZ9DU7wTX9N0Fm9rjrLJzF4BIE=","5327","22925"],"type":"jvm"},"mount_tokens":true,"resources":{"preemptible":true,"req_cpu":"1","req_memory":"standard","req_storage":"0Gi","cores_mcpu":1000,"memory_bytes":4026531840,"storage_gib":0},"secrets":[{"namespace":"default","name":"rwalters-i04mj-gsa-key","mount_path":"/gsa-key","mount_in_copy":true},{"namespace":"default","name":"rwalters-i04mj-tokens","mount_path":"/user-tokens","mount_in_copy":false},{"namespace":"default","name":"worker-deploy-config","mount_path":"/deploy-config","mount_in_copy":false},{"namespace":"default","name":"ssl-config-batch-user-code","mount_path":"/ssl-config","mount_in_copy":false}],"env":[{"name":"GOOGLE_APPLICATION_CREDENTIALS","value":"/gsa-key/key.json"}]}},"log":"2022-04-20 02:39:25 Tokens: INFO: tokens found for namespaces {default}\n2022-04-20 02:39:25 tls: INFO: ssl config file found at /batch/f49e90d2d54842cba3650f3713919025/secrets/ssl-config/ssl-config.json\n2022-04-20 02:39:25 Worker$: INFO: is.hail.backend.service.Worker d77cdf0157c9c1f37657f0da86747b61ae25de88\n2022-04-20 02:39:25 Worker$: INFO: running job 5327/22925 at root gs://rwalters-hail-tmp/parallelizeAndComputeWithIndex/fk_NYnDaiPkqwqIKmJZ9DU7wTX9N0Fm9rjrLJzF4BIE= with scratch directory '/batch/f49e90d2d54842cba3650f3713919025'\n2022-04-20 02:39:25 GoogleStorageFS$: INFO: Initializing google storage client from service account key\n2022-04-20 02:39:25 WorkerTimer$: INFO: readInputs took 136.951090 ms.\n2022-04-20 02:39:25 root: INFO: RegionPool: initialized for thread 8: pool-1-thread-1\n2022-04-20 02:39:25 root: INFO: RegionPool: REPORT_THRESHOLD: 320.0K allocated (192.0K blocks / 128.0K chunks), regions.size = 3, 0 current java objects, thread 8: pool-1-thread-1"}]}}