Hi,
I am running hail and spark on local linux cluster and using the Pan-UKBB data which comes in hail tables. I am heap errors repeatedly I tried throwing more memory at it increasing the available memory etc. I am running on executors distributed over couple of nodes. Compiled hail locally with make. Code and error below. Any help would be great. I updated to the latest version and still not getting same error.
from pyspark import SparkConf, SparkContext
spark_master='spark://nc041:7077'
hail_jars = "/home/mambaforge/lib/python3.9/site-packages/hail/backend/hail-all-spark.jar"
conf = SparkConf().setAll([
('spark.master', str(spark_master)),
('spark.app.name', 'Hail'),
('spark.jars', str(hail_jars)),
('spark.driver.extraClassPath', str(hail_jars)),
('spark.executor.extraClassPath', './hail-all-spark.jar'),
('spark.serializer', 'org.apache.spark.serializer.KryoSerializer'),
('spark.kryo.registrator', 'is.hail.kryo.HailKryoRegistrator'),
('spark.executor.memory', '20g'),
('spark.executor.memoryOverhead','10g')])
sc = SparkContext(conf=conf)
# from pyspark import SparkConf, SparkContext
import hail as hl
from hail.linalg import BlockMatrix
hl.plot.output_notebook()
from ukbb_pan_ancestry import *
# conf = SparkConf().setAll([('spark.master', str(spark_master)),
# ('spark.executor.memory', '8g')])
# sc = SparkContext(conf=conf)
hl.init(sc=sc)
import pandas as pd
import numpy as np
import math
from os import path
import os
ht_idx = hl.read_table(pandb+'UKBB.EUR.ldadj.variant.ht')
ht_score = hl.read_table(pandb+'UKBB.EUR.ldscore.ht')
score_pd = ht_score.to_pandas()
score_pd[['a0','a1']] = pd.DataFrame(score_pd.alleles.tolist(), index = score_pd.index)
score_pd = score_pd.rename(columns={'locus.contig':'chr', 'locus.position':'pos'})
score_pd = score_pd[['chr', 'pos', 'a0', 'a1', 'rsid', 'varid', 'AF', 'ld_score']]
Exception in thread "refresh progress" (9106 + 150) / 18254]java.lang.OutOfMemoryError: Java heap space
Exception in thread "SparkUI-47" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-2" Exception in thread "spark-listener-group-appStatus" java.lang.OutOfMemoryError: Java heap space
at com.codahale.metrics.Timer.time(Timer.java:160)
at org.apache.spark.scheduler.AsyncEventQueue.$anonfun$dispatch$1(AsyncEventQueue.scala:103)
at org.apache.spark.scheduler.AsyncEventQueue$$Lambda$787/0x0000000100752440.apply$mcJ$sp(Unknown Source)
at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
at org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$dispatch(AsyncEventQueue.scala:100)
at org.apache.spark.scheduler.AsyncEventQueue$$anon$2.$anonfun$run$1(AsyncEventQueue.scala:96)
at org.apache.spark.scheduler.AsyncEventQueue$$anon$2$$Lambda$786/0x0000000100751440.apply$mcV$sp(Unknown Source)
at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1381)
at org.apache.spark.scheduler.AsyncEventQueue$$anon$2.run(AsyncEventQueue.scala:96)
java.lang.OutOfMemoryError: Java heap space
Exception in thread "Spark Context Cleaner" java.lang.OutOfMemoryError: Java heap space
Exception in thread "refresh progress" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-1" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-3" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-6" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dag-scheduler-event-loop" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-0" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-5" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
---------------------------------------------------------------------------
FatalError Traceback (most recent call last)
Input In [3], in <cell line: 2>()
1 ## Import scores table
----> 2 score_pd = ht_score.to_pandas()
3 score_pd[['a0','a1']] = pd.DataFrame(score_pd.alleles.tolist(), index = score_pd.index)
4 score_pd = score_pd.rename(columns={'locus.contig':'chr', 'locus.position':'pos'})
File ~/mambaforge/lib/python3.9/site-packages/decorator.py:232, in decorate.<locals>.fun(*args, **kw)
230 if not kwsyntax:
231 args, kw = fix(args, kw, sig)
--> 232 return caller(func, *(extras + args), **kw)
File ~/mambaforge/lib/python3.9/site-packages/hail/typecheck/check.py:577, in _make_dec.<locals>.wrapper(__original_func, *args, **kwargs)
574 @decorator
575 def wrapper(__original_func, *args, **kwargs):
576 args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 577 return __original_func(*args_, **kwargs_)
File ~/mambaforge/lib/python3.9/site-packages/hail/table.py:3340, in Table.to_pandas(self, flatten)
3338 dtypes_struct = table.row.dtype
3339 collect_dict = {key: hl.agg.collect(value) for key, value in table.row.items()}
-> 3340 column_struct_array = table.aggregate(hl.struct(**collect_dict))
3341 columns = list(column_struct_array.keys())
3342 data_dict = {}
File ~/mambaforge/lib/python3.9/site-packages/decorator.py:232, in decorate.<locals>.fun(*args, **kw)
230 if not kwsyntax:
231 args, kw = fix(args, kw, sig)
--> 232 return caller(func, *(extras + args), **kw)
File ~/mambaforge/lib/python3.9/site-packages/hail/typecheck/check.py:577, in _make_dec.<locals>.wrapper(__original_func, *args, **kwargs)
574 @decorator
575 def wrapper(__original_func, *args, **kwargs):
576 args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 577 return __original_func(*args_, **kwargs_)
File ~/mambaforge/lib/python3.9/site-packages/hail/table.py:1231, in Table.aggregate(self, expr, _localize)
1228 agg_ir = ir.TableAggregate(base._tir, expr._ir)
1230 if _localize:
-> 1231 return Env.backend().execute(hl.ir.MakeTuple([agg_ir]))[0]
1233 return construct_expr(ir.LiftMeOut(agg_ir), expr.dtype)
File ~/mambaforge/lib/python3.9/site-packages/hail/backend/py4j_backend.py:104, in Py4JBackend.execute(self, ir, timed)
102 return (value, timings) if timed else value
103 except FatalError as e:
--> 104 self._handle_fatal_error_from_backend(e, ir)
File ~/mambaforge/lib/python3.9/site-packages/hail/backend/backend.py:181, in Backend._handle_fatal_error_from_backend(self, err, ir)
179 error_sources = ir.base_search(lambda x: x._error_id == err._error_id)
180 if len(error_sources) == 0:
--> 181 raise err
183 better_stack_trace = error_sources[0]._stack_trace
184 error_message = str(err)
File ~/mambaforge/lib/python3.9/site-packages/hail/backend/py4j_backend.py:98, in Py4JBackend.execute(self, ir, timed)
96 # print(self._hail_package.expr.ir.Pretty.apply(jir, True, -1))
97 try:
---> 98 result_tuple = self._jbackend.executeEncode(jir, stream_codec)
99 (result, timings) = (result_tuple._1(), result_tuple._2())
100 value = ir.typ._from_encoding(result)
File ~/mambaforge/lib/python3.9/site-packages/py4j/java_gateway.py:1304, in JavaMember.__call__(self, *args)
1298 command = proto.CALL_COMMAND_NAME +\
1299 self.command_header +\
1300 args_command +\
1301 proto.END_COMMAND_PART
1303 answer = self.gateway_client.send_command(command)
-> 1304 return_value = get_return_value(
1305 answer, self.gateway_client, self.target_id, self.name)
1307 for temp_arg in temp_args:
1308 temp_arg._detach()
File ~/mambaforge/lib/python3.9/site-packages/hail/backend/py4j_backend.py:31, in handle_java_exception.<locals>.deco(*args, **kwargs)
29 tpl = Env.jutils().handleForPython(e.java_exception)
30 deepest, full, error_id = tpl._1(), tpl._2(), tpl._3()
---> 31 raise fatal_error_from_java_error_triplet(deepest, full, error_id) from None
32 except pyspark.sql.utils.CapturedException as e:
33 raise FatalError('%s\n\nJava stack trace:\n%s\n'
34 'Hail version: %s\n'
35 'Error summary: %s' % (e.desc, e.stackTrace, hail.__version__, e.desc)) from None
FatalError: SparkException: Job 0 cancelled as part of cancellation of all jobs
Java stack trace:
org.apache.spark.SparkException: Job 0 cancelled as part of cancellation of all jobs
deleted some info here whicih was all from shceduler
at scala.runtime.java8.JFunction1$mcVI$sp.apply(JFunction1$mcVI$sp.java:23)
at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
at org.apache.spark.scheduler.DAGScheduler.cancelRunningIndependentStages(DAGScheduler.scala:2266)
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2299)
at org.apache.spark.scheduler.DAGScheduler.handleJobCancellation(DAGScheduler.scala:2199)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$doCancelAllJobs$2(DAGScheduler.scala:1017)
at scala.runtime.java8.JFunction1$mcVI$sp.apply(JFunction1$mcVI$sp.java:23)
at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
at org.apache.spark.scheduler.DAGScheduler.doCancelAllJobs(DAGScheduler.scala:1016)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onError(DAGScheduler.scala:2499)
at org.apache.spark.util.EventLoop.post(EventLoop.scala:107)
at org.apache.spark.scheduler.DAGScheduler.taskEnded(DAGScheduler.scala:280)
at org.apache.spark.scheduler.TaskSetManager.handleSuccessfulTask(TaskSetManager.scala:783)
at org.apache.spark.scheduler.TaskSchedulerImpl.handleSuccessfulTask(TaskSchedulerImpl.scala:824)
at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:119)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1996)
at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:902)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2217)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2236)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2261)
at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
at org.apache.spark.rdd.RDD.collect(RDD.scala:1029)
at is.hail.backend.spark.SparkBackend.parallelizeAndComputeWithIndex(SparkBackend.scala:321)
at is.hail.backend.BackendUtils.collectDArray(BackendUtils.scala:37)
at __C3Compiled.__m4split_Let_region126_128(Emit.scala)
at __C3Compiled.__m4split_Let(Emit.scala)
at __C3Compiled.apply(Emit.scala)
at is.hail.expr.ir.CompileAndEvaluate$.$anonfun$_apply$6(CompileAndEvaluate.scala:68)
at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:68)
at is.hail.expr.ir.CompileAndEvaluate$.evalToIR(CompileAndEvaluate.scala:30)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.evaluate$1(LowerOrInterpretNonCompilable.scala:30)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.rewrite$1(LowerOrInterpretNonCompilable.scala:67)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.rewrite$1(LowerOrInterpretNonCompilable.scala:53)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.apply(LowerOrInterpretNonCompilable.scala:72)
at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.transform(LoweringPass.scala:69)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$3(LoweringPass.scala:16)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$1(LoweringPass.scala:16)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
at is.hail.expr.ir.lowering.LoweringPass.apply(LoweringPass.scala:14)
at is.hail.expr.ir.lowering.LoweringPass.apply$(LoweringPass.scala:13)
at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.apply(LoweringPass.scala:64)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1(LoweringPipeline.scala:15)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1$adapted(LoweringPipeline.scala:13)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
at is.hail.expr.ir.lowering.LoweringPipeline.apply(LoweringPipeline.scala:13)
at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:47)
at is.hail.backend.spark.SparkBackend._execute(SparkBackend.scala:416)
at is.hail.backend.spark.SparkBackend.$anonfun$executeEncode$2(SparkBackend.scala:452)
at is.hail.backend.ExecuteContext$.$anonfun$scoped$3(ExecuteContext.scala:70)
at is.hail.utils.package$.using(package.scala:640)
at is.hail.backend.ExecuteContext$.$anonfun$scoped$2(ExecuteContext.scala:70)
at is.hail.utils.package$.using(package.scala:640)
at is.hail.annotations.RegionPool$.scoped(RegionPool.scala:17)
at is.hail.backend.ExecuteContext$.scoped(ExecuteContext.scala:59)
at is.hail.backend.spark.SparkBackend.withExecuteContext(SparkBackend.scala:310)
at is.hail.backend.spark.SparkBackend.$anonfun$executeEncode$1(SparkBackend.scala:449)
at is.hail.utils.ExecutionTimer$.time(ExecutionTimer.scala:52)
at is.hail.backend.spark.SparkBackend.executeEncode(SparkBackend.scala:448)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.base/java.lang.Thread.run(Thread.java:829)
Hail version: 0.2.97-b7f85db2968d
Error summary: SparkException: Job 0 cancelled as part of cancellation of all jobs