Getting java heap error tried a bunch of things with the executor and memory settings

Hi,

I am running hail and spark on local linux cluster and using the Pan-UKBB data which comes in hail tables. I am heap errors repeatedly I tried throwing more memory at it increasing the available memory etc. I am running on executors distributed over couple of nodes. Compiled hail locally with make. Code and error below. Any help would be great. I updated to the latest version and still not getting same error.

from pyspark import SparkConf, SparkContext
spark_master='spark://nc041:7077'
hail_jars = "/home/mambaforge/lib/python3.9/site-packages/hail/backend/hail-all-spark.jar"
conf = SparkConf().setAll([
    ('spark.master', str(spark_master)),
    ('spark.app.name', 'Hail'),
    ('spark.jars', str(hail_jars)),
    ('spark.driver.extraClassPath', str(hail_jars)),
    ('spark.executor.extraClassPath', './hail-all-spark.jar'),
    ('spark.serializer', 'org.apache.spark.serializer.KryoSerializer'),
    ('spark.kryo.registrator', 'is.hail.kryo.HailKryoRegistrator'),
    ('spark.executor.memory', '20g'),
    ('spark.executor.memoryOverhead','10g')])
sc = SparkContext(conf=conf)
# from pyspark import SparkConf, SparkContext
import hail as hl
from hail.linalg import BlockMatrix
hl.plot.output_notebook()
from ukbb_pan_ancestry import *

# conf = SparkConf().setAll([('spark.master', str(spark_master)),
#                            ('spark.executor.memory', '8g')])
# sc = SparkContext(conf=conf)

hl.init(sc=sc)

import pandas as pd
import numpy as np
import math
from os import path
import os


ht_idx = hl.read_table(pandb+'UKBB.EUR.ldadj.variant.ht')
ht_score = hl.read_table(pandb+'UKBB.EUR.ldscore.ht')

score_pd = ht_score.to_pandas()
score_pd[['a0','a1']] = pd.DataFrame(score_pd.alleles.tolist(), index = score_pd.index)
score_pd = score_pd.rename(columns={'locus.contig':'chr', 'locus.position':'pos'})
score_pd = score_pd[['chr', 'pos', 'a0', 'a1', 'rsid', 'varid', 'AF', 'ld_score']]

Exception in thread "refresh progress"                     (9106 + 150) / 18254]java.lang.OutOfMemoryError: Java heap space
Exception in thread "SparkUI-47" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-2" Exception in thread "spark-listener-group-appStatus" java.lang.OutOfMemoryError: Java heap space
        at com.codahale.metrics.Timer.time(Timer.java:160)
        at org.apache.spark.scheduler.AsyncEventQueue.$anonfun$dispatch$1(AsyncEventQueue.scala:103)
        at org.apache.spark.scheduler.AsyncEventQueue$$Lambda$787/0x0000000100752440.apply$mcJ$sp(Unknown Source)
        at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
        at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
        at org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$dispatch(AsyncEventQueue.scala:100)
        at org.apache.spark.scheduler.AsyncEventQueue$$anon$2.$anonfun$run$1(AsyncEventQueue.scala:96)
        at org.apache.spark.scheduler.AsyncEventQueue$$anon$2$$Lambda$786/0x0000000100751440.apply$mcV$sp(Unknown Source)
        at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1381)
        at org.apache.spark.scheduler.AsyncEventQueue$$anon$2.run(AsyncEventQueue.scala:96)
java.lang.OutOfMemoryError: Java heap space
Exception in thread "Spark Context Cleaner" java.lang.OutOfMemoryError: Java heap space
Exception in thread "refresh progress" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-1" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-3" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-6" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dag-scheduler-event-loop" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-0" java.lang.OutOfMemoryError: Java heap space
Exception in thread "task-result-getter-5" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
Exception in thread "dispatcher-CoarseGrainedScheduler" java.lang.OutOfMemoryError: Java heap space
---------------------------------------------------------------------------
FatalError                                Traceback (most recent call last)
Input In [3], in <cell line: 2>()
      1 ## Import scores  table
----> 2 score_pd = ht_score.to_pandas()
      3 score_pd[['a0','a1']] = pd.DataFrame(score_pd.alleles.tolist(), index = score_pd.index)
      4 score_pd = score_pd.rename(columns={'locus.contig':'chr', 'locus.position':'pos'})

File ~/mambaforge/lib/python3.9/site-packages/decorator.py:232, in decorate.<locals>.fun(*args, **kw)
    230 if not kwsyntax:
    231     args, kw = fix(args, kw, sig)
--> 232 return caller(func, *(extras + args), **kw)

File ~/mambaforge/lib/python3.9/site-packages/hail/typecheck/check.py:577, in _make_dec.<locals>.wrapper(__original_func, *args, **kwargs)
    574 @decorator
    575 def wrapper(__original_func, *args, **kwargs):
    576     args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 577     return __original_func(*args_, **kwargs_)

File ~/mambaforge/lib/python3.9/site-packages/hail/table.py:3340, in Table.to_pandas(self, flatten)
   3338 dtypes_struct = table.row.dtype
   3339 collect_dict = {key: hl.agg.collect(value) for key, value in table.row.items()}
-> 3340 column_struct_array = table.aggregate(hl.struct(**collect_dict))
   3341 columns = list(column_struct_array.keys())
   3342 data_dict = {}

File ~/mambaforge/lib/python3.9/site-packages/decorator.py:232, in decorate.<locals>.fun(*args, **kw)
    230 if not kwsyntax:
    231     args, kw = fix(args, kw, sig)
--> 232 return caller(func, *(extras + args), **kw)

File ~/mambaforge/lib/python3.9/site-packages/hail/typecheck/check.py:577, in _make_dec.<locals>.wrapper(__original_func, *args, **kwargs)
    574 @decorator
    575 def wrapper(__original_func, *args, **kwargs):
    576     args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 577     return __original_func(*args_, **kwargs_)

File ~/mambaforge/lib/python3.9/site-packages/hail/table.py:1231, in Table.aggregate(self, expr, _localize)
   1228 agg_ir = ir.TableAggregate(base._tir, expr._ir)
   1230 if _localize:
-> 1231     return Env.backend().execute(hl.ir.MakeTuple([agg_ir]))[0]
   1233 return construct_expr(ir.LiftMeOut(agg_ir), expr.dtype)

File ~/mambaforge/lib/python3.9/site-packages/hail/backend/py4j_backend.py:104, in Py4JBackend.execute(self, ir, timed)
    102     return (value, timings) if timed else value
    103 except FatalError as e:
--> 104     self._handle_fatal_error_from_backend(e, ir)

File ~/mambaforge/lib/python3.9/site-packages/hail/backend/backend.py:181, in Backend._handle_fatal_error_from_backend(self, err, ir)
    179 error_sources = ir.base_search(lambda x: x._error_id == err._error_id)
    180 if len(error_sources) == 0:
--> 181     raise err
    183 better_stack_trace = error_sources[0]._stack_trace
    184 error_message = str(err)

File ~/mambaforge/lib/python3.9/site-packages/hail/backend/py4j_backend.py:98, in Py4JBackend.execute(self, ir, timed)
     96 # print(self._hail_package.expr.ir.Pretty.apply(jir, True, -1))
     97 try:
---> 98     result_tuple = self._jbackend.executeEncode(jir, stream_codec)
     99     (result, timings) = (result_tuple._1(), result_tuple._2())
    100     value = ir.typ._from_encoding(result)

File ~/mambaforge/lib/python3.9/site-packages/py4j/java_gateway.py:1304, in JavaMember.__call__(self, *args)
   1298 command = proto.CALL_COMMAND_NAME +\
   1299     self.command_header +\
   1300     args_command +\
   1301     proto.END_COMMAND_PART
   1303 answer = self.gateway_client.send_command(command)
-> 1304 return_value = get_return_value(
   1305     answer, self.gateway_client, self.target_id, self.name)
   1307 for temp_arg in temp_args:
   1308     temp_arg._detach()

File ~/mambaforge/lib/python3.9/site-packages/hail/backend/py4j_backend.py:31, in handle_java_exception.<locals>.deco(*args, **kwargs)
     29     tpl = Env.jutils().handleForPython(e.java_exception)
     30     deepest, full, error_id = tpl._1(), tpl._2(), tpl._3()
---> 31     raise fatal_error_from_java_error_triplet(deepest, full, error_id) from None
     32 except pyspark.sql.utils.CapturedException as e:
     33     raise FatalError('%s\n\nJava stack trace:\n%s\n'
     34                      'Hail version: %s\n'
     35                      'Error summary: %s' % (e.desc, e.stackTrace, hail.__version__, e.desc)) from None

FatalError: SparkException: Job 0 cancelled as part of cancellation of all jobs

Java stack trace:
org.apache.spark.SparkException: Job 0 cancelled as part of cancellation of all jobs

deleted some info here whicih was all from shceduler

        at scala.runtime.java8.JFunction1$mcVI$sp.apply(JFunction1$mcVI$sp.java:23)
        at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
        at org.apache.spark.scheduler.DAGScheduler.cancelRunningIndependentStages(DAGScheduler.scala:2266)
        at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2299)
        at org.apache.spark.scheduler.DAGScheduler.handleJobCancellation(DAGScheduler.scala:2199)
        at org.apache.spark.scheduler.DAGScheduler.$anonfun$doCancelAllJobs$2(DAGScheduler.scala:1017)
        at scala.runtime.java8.JFunction1$mcVI$sp.apply(JFunction1$mcVI$sp.java:23)
        at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
        at org.apache.spark.scheduler.DAGScheduler.doCancelAllJobs(DAGScheduler.scala:1016)
        at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onError(DAGScheduler.scala:2499)
        at org.apache.spark.util.EventLoop.post(EventLoop.scala:107)
        at org.apache.spark.scheduler.DAGScheduler.taskEnded(DAGScheduler.scala:280)
        at org.apache.spark.scheduler.TaskSetManager.handleSuccessfulTask(TaskSetManager.scala:783)
        at org.apache.spark.scheduler.TaskSchedulerImpl.handleSuccessfulTask(TaskSchedulerImpl.scala:824)
        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(TaskResultGetter.scala:119)
        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1996)
        at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(TaskResultGetter.scala:63)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base/java.lang.Thread.run(Thread.java:829)
        at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:902)
        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2196)
        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2217)
        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2236)
        at org.apache.spark.SparkContext.runJob(SparkContext.scala:2261)
        at org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1030)
        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
        at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
        at org.apache.spark.rdd.RDD.collect(RDD.scala:1029)
        at is.hail.backend.spark.SparkBackend.parallelizeAndComputeWithIndex(SparkBackend.scala:321)
        at is.hail.backend.BackendUtils.collectDArray(BackendUtils.scala:37)
        at __C3Compiled.__m4split_Let_region126_128(Emit.scala)
        at __C3Compiled.__m4split_Let(Emit.scala)
        at __C3Compiled.apply(Emit.scala)
        at is.hail.expr.ir.CompileAndEvaluate$.$anonfun$_apply$6(CompileAndEvaluate.scala:68)
        at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
        at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
        at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:68)
        at is.hail.expr.ir.CompileAndEvaluate$.evalToIR(CompileAndEvaluate.scala:30)
        at is.hail.expr.ir.LowerOrInterpretNonCompilable$.evaluate$1(LowerOrInterpretNonCompilable.scala:30)
        at is.hail.expr.ir.LowerOrInterpretNonCompilable$.rewrite$1(LowerOrInterpretNonCompilable.scala:67)
        at is.hail.expr.ir.LowerOrInterpretNonCompilable$.rewrite$1(LowerOrInterpretNonCompilable.scala:53)
        at is.hail.expr.ir.LowerOrInterpretNonCompilable$.apply(LowerOrInterpretNonCompilable.scala:72)
        at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.transform(LoweringPass.scala:69)
        at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$3(LoweringPass.scala:16)
        at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
        at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$1(LoweringPass.scala:16)
        at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
        at is.hail.expr.ir.lowering.LoweringPass.apply(LoweringPass.scala:14)
        at is.hail.expr.ir.lowering.LoweringPass.apply$(LoweringPass.scala:13)
        at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.apply(LoweringPass.scala:64)
        at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1(LoweringPipeline.scala:15)
        at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1$adapted(LoweringPipeline.scala:13)
        at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
        at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
        at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
        at is.hail.expr.ir.lowering.LoweringPipeline.apply(LoweringPipeline.scala:13)
        at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:47)
        at is.hail.backend.spark.SparkBackend._execute(SparkBackend.scala:416)
        at is.hail.backend.spark.SparkBackend.$anonfun$executeEncode$2(SparkBackend.scala:452)
        at is.hail.backend.ExecuteContext$.$anonfun$scoped$3(ExecuteContext.scala:70)
        at is.hail.utils.package$.using(package.scala:640)
        at is.hail.backend.ExecuteContext$.$anonfun$scoped$2(ExecuteContext.scala:70)
        at is.hail.utils.package$.using(package.scala:640)
        at is.hail.annotations.RegionPool$.scoped(RegionPool.scala:17)
        at is.hail.backend.ExecuteContext$.scoped(ExecuteContext.scala:59)
        at is.hail.backend.spark.SparkBackend.withExecuteContext(SparkBackend.scala:310)
        at is.hail.backend.spark.SparkBackend.$anonfun$executeEncode$1(SparkBackend.scala:449)
        at is.hail.utils.ExecutionTimer$.time(ExecutionTimer.scala:52)
        at is.hail.backend.spark.SparkBackend.executeEncode(SparkBackend.scala:448)
        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
        at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
        at py4j.Gateway.invoke(Gateway.java:282)
        at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
        at py4j.commands.CallCommand.execute(CallCommand.java:79)
        at py4j.GatewayConnection.run(GatewayConnection.java:238)
        at java.base/java.lang.Thread.run(Thread.java:829)



Hail version: 0.2.97-b7f85db2968d
Error summary: SparkException: Job 0 cancelled as part of cancellation of all jobs