thank you so much for this! I got this stack trace again:
Traceback (most recent call last):
File "/tmp/4a4566b10b97448eb948461c601340cd/prepare_data_release.py", line 839, in <module>
main(args)
File "/tmp/4a4566b10b97448eb948461c601340cd/prepare_data_release.py", line 808, in main
contig_mt.rows().show()
File "</opt/conda/default/lib/python3.6/site-packages/decorator.py:decorator-gen-918>", line 2, in show
File "/opt/conda/default/lib/python3.6/site-packages/hail/typecheck/check.py", line 585, in wrapper
return __original_func(*args_, **kwargs_)
File "/opt/conda/default/lib/python3.6/site-packages/hail/table.py", line 1450, in show
handler(self._show(n, width, truncate, types))
File "/opt/conda/default/lib/python3.6/site-packages/IPython/core/display.py", line 282, in display
print(*objs)
File "/opt/conda/default/lib/python3.6/site-packages/hail/table.py", line 1241, in __str__
return self._ascii_str()
File "/opt/conda/default/lib/python3.6/site-packages/hail/table.py", line 1268, in _ascii_str
rows, has_more, dtype = self.data()
File "/opt/conda/default/lib/python3.6/site-packages/hail/table.py", line 1251, in data
rows, has_more = t._take_n(self.n)
File "/opt/conda/default/lib/python3.6/site-packages/hail/table.py", line 1372, in _take_n
rows = self.take(n + 1)
File "</opt/conda/default/lib/python3.6/site-packages/decorator.py:decorator-gen-930>", line 2, in take
File "/opt/conda/default/lib/python3.6/site-packages/hail/typecheck/check.py", line 585, in wrapper
return __original_func(*args_, **kwargs_)
File "/opt/conda/default/lib/python3.6/site-packages/hail/table.py", line 2064, in take
return self.head(n).collect(_localize)
File "</opt/conda/default/lib/python3.6/site-packages/decorator.py:decorator-gen-924>", line 2, in collect
File "/opt/conda/default/lib/python3.6/site-packages/hail/typecheck/check.py", line 585, in wrapper
return __original_func(*args_, **kwargs_)
File "/opt/conda/default/lib/python3.6/site-packages/hail/table.py", line 1863, in collect
return Env.backend().execute(e._ir)
File "/opt/conda/default/lib/python3.6/site-packages/hail/backend/backend.py", line 109, in execute
result = json.loads(Env.hc()._jhc.backend().executeJSON(self._to_java_ir(ir)))
File "/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py", line 1257, in __call__
File "/opt/conda/default/lib/python3.6/site-packages/hail/utils/java.py", line 225, in deco
'Error summary: %s' % (deepest, full, hail.__version__, deepest)) from None
hail.utils.java.FatalError: ArrayIndexOutOfBoundsException: 1
Java stack trace:
java.lang.ArrayIndexOutOfBoundsException: 1
at org.apache.spark.sql.catalyst.expressions.GenericRow.get(rows.scala:174)
at is.hail.annotations.RegionValueBuilder.addRow(RegionValueBuilder.scala:299)
at is.hail.annotations.RegionValueBuilder.addAnnotation(RegionValueBuilder.scala:542)
at is.hail.annotations.RegionValueBuilder.addAnnotation(RegionValueBuilder.scala:580)
at is.hail.annotations.RegionValueBuilder.addAnnotation(RegionValueBuilder.scala:531)
at is.hail.expr.ir.EmitFunctionBuilder$$anonfun$encodeLiterals$1$$anonfun$apply$1.apply(EmitFunctionBuilder.scala:249)
at is.hail.expr.ir.EmitFunctionBuilder$$anonfun$encodeLiterals$1$$anonfun$apply$1.apply(EmitFunctionBuilder.scala:249)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186)
at is.hail.expr.ir.EmitFunctionBuilder$$anonfun$encodeLiterals$1.apply(EmitFunctionBuilder.scala:249)
at is.hail.expr.ir.EmitFunctionBuilder$$anonfun$encodeLiterals$1.apply(EmitFunctionBuilder.scala:245)
at is.hail.utils.package$.using(package.scala:596)
at is.hail.annotations.Region$.scoped(Region.scala:18)
at is.hail.expr.ir.EmitFunctionBuilder.encodeLiterals(EmitFunctionBuilder.scala:245)
at is.hail.expr.ir.EmitFunctionBuilder.resultWithIndex(EmitFunctionBuilder.scala:610)
at is.hail.expr.ir.Compile$.apply(Compile.scala:55)
at is.hail.expr.ir.Compile$.apply(Compile.scala:80)
at is.hail.expr.ir.Compile$.apply(Compile.scala:121)
at is.hail.expr.ir.Compile$.apply(Compile.scala:130)
at is.hail.expr.ir.TableFilter.execute(TableIR.scala:447)
at is.hail.expr.ir.TableKeyByAndAggregate.execute(TableIR.scala:1359)
at is.hail.expr.ir.TableExplode.execute(TableIR.scala:1205)
at is.hail.expr.ir.TableMapRows.execute(TableIR.scala:995)
at is.hail.expr.ir.Interpret$.apply(Interpret.scala:722)
at is.hail.expr.ir.Interpret$.alreadyLowered(Interpret.scala:67)
at is.hail.expr.ir.InterpretNonCompilable$.interpretAndCoerce$1(InterpretNonCompilable.scala:16)
at is.hail.expr.ir.InterpretNonCompilable$.is$hail$expr$ir$InterpretNonCompilable$$rewrite$1(InterpretNonCompilable.scala:53)
at is.hail.expr.ir.InterpretNonCompilable$$anonfun$1.apply(InterpretNonCompilable.scala:25)
at is.hail.expr.ir.InterpretNonCompilable$$anonfun$1.apply(InterpretNonCompilable.scala:25)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:35)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractTraversable.map(Traversable.scala:104)
at is.hail.expr.ir.InterpretNonCompilable$.rewriteChildren$1(InterpretNonCompilable.scala:25)
at is.hail.expr.ir.InterpretNonCompilable$.is$hail$expr$ir$InterpretNonCompilable$$rewrite$1(InterpretNonCompilable.scala:54)
at is.hail.expr.ir.InterpretNonCompilable$.apply(InterpretNonCompilable.scala:58)
at is.hail.expr.ir.lowering.InterpretNonCompilablePass$.transform(LoweringPass.scala:45)
at is.hail.expr.ir.lowering.LoweringPass$class.apply(LoweringPass.scala:12)
at is.hail.expr.ir.lowering.InterpretNonCompilablePass$.apply(LoweringPass.scala:40)
at is.hail.expr.ir.lowering.LoweringPipeline$$anonfun$apply$1.apply(LoweringPipeline.scala:19)
at is.hail.expr.ir.lowering.LoweringPipeline$$anonfun$apply$1.apply(LoweringPipeline.scala:17)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:35)
at is.hail.expr.ir.lowering.LoweringPipeline.apply(LoweringPipeline.scala:17)
at is.hail.expr.ir.CompileAndEvaluate$.apply(CompileAndEvaluate.scala:14)
at is.hail.backend.Backend$$anonfun$execute$1.apply(Backend.scala:57)
at is.hail.backend.Backend$$anonfun$execute$1.apply(Backend.scala:57)
at is.hail.utils.package$.using(package.scala:596)
at is.hail.expr.ir.ExecuteContext$$anonfun$scoped$1.apply(ExecuteContext.scala:10)
at is.hail.expr.ir.ExecuteContext$$anonfun$scoped$1.apply(ExecuteContext.scala:9)
at is.hail.utils.package$.using(package.scala:596)
at is.hail.annotations.Region$.scoped(Region.scala:18)
at is.hail.expr.ir.ExecuteContext$.scoped(ExecuteContext.scala:9)
at is.hail.backend.Backend.execute(Backend.scala:57)
at is.hail.backend.Backend.executeJSON(Backend.scala:63)
at sun.reflect.GeneratedMethodAccessor44.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Thread.java:748)
Hail version: 0.2.26-2706ad7eee6a
Error summary: ArrayIndexOutOfBoundsException: 1
ERROR: (gcloud.dataproc.jobs.submit.pyspark) Job [4a4566b10b97448eb948461c601340cd] failed with error:
Google Cloud Dataproc Agent reports job failure. If logs are available, they can be found at 'https://console.cloud.google.com/dataproc/jobs/4a4566b10b97448eb948461c601340cd?project=maclab-ukbb®ion=us-central1' and in 'gs://dataproc-1aca38e4-67fe-4b64-b451-258ef1aea4d1-us-central1/google-cloud-dataproc-metainfo/9dc3e497-f7d8-4930-ac01-2a417920aca5/jobs/4a4566b10b97448eb948461c601340cd/driveroutput'.
Traceback (most recent call last):
File "/Users/kchao/.conda/envs/hail-test/bin/hailctl", line 8, in <module>
sys.exit(main())
File "/Users/kchao/.conda/envs/hail-test/lib/python3.8/site-packages/hailtop/hailctl/__main__.py", line 94, in main
cli.main(args)
File "/Users/kchao/.conda/envs/hail-test/lib/python3.8/site-packages/hailtop/hailctl/dataproc/cli.py", line 107, in main
jmp[args.module].main(args, pass_through_args)
File "/Users/kchao/.conda/envs/hail-test/lib/python3.8/site-packages/hailtop/hailctl/dataproc/submit.py", line 75, in main
check_call(cmd)
File "/Users/kchao/.conda/envs/hail-test/lib/python3.8/subprocess.py", line 364, in check_call
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['gcloud', 'dataproc', 'jobs', 'submit', 'pyspark', 'prepare_data_release.py', '--cluster=kc', '--files=', '--py-files=/var/folders/8_/hllv8xr94p3539n63wzrm0c11_2fgd/T/pyscripts_k1ygihg5.zip', '--properties=', '--', '-f', '4', '--prepare_release_vcf']' returned non-zero exit status 1.