", line 2, in write
File "/opt/conda/default/lib/python3.8/site-packages/hail/typecheck/check.py", line 577, in wrapper
return __original_func(*args_, **kwargs_)
File "/opt/conda/default/lib/python3.8/site-packages/hail/table.py", line 1271, in write
Env.backend().execute(ir.TableWrite(self._tir, ir.TableNativeWriter(output, overwrite, stage_locally, _codec_spec)))
File "/opt/conda/default/lib/python3.8/site-packages/hail/backend/py4j_backend.py", line 98, in execute
raise e
File "/opt/conda/default/lib/python3.8/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1304, in __call__
File "/opt/conda/default/lib/python3.8/site-packages/hail/backend/py4j_backend.py", line 30, in deco
raise FatalError('%s\n\nJava stack trace:\n%s\n'
hail.utils.java.FatalError: SparkException: Job aborted due to stage failure: ResultStage 13 (runJob at ContextRDD.scala:238) has failed the maximum allowable number of times: 4. Most recent failure reason: org.apache.spark.shuffle.FetchFailedException at org.apache.spark.storage.ShuffleBlockFetcherIterator.throwFetchFailedException(ShuffleBlockFetcherIterator.scala:770) at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:685) at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:70) at org.apache.spark.util.CompletionIterator.next(CompletionIterator.scala:29) at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:31) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37) at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:200) at org.apache.spark.shuffle.BlockStoreShuffleReader.read(BlockStoreShuffleReader.scala:128) at org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:106) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.ZippedPartitionsRDD2.compute(ZippedPartitionsRDD.scala:89) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.nio.channels.ClosedChannelException at org.apache.spark.network.client.StreamInterceptor.channelInactive(StreamInterceptor.java:62) at org.apache.spark.network.util.TransportFrameDecoder.channelInactive(TransportFrameDecoder.java:223) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:497) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more
Java stack trace:
org.apache.spark.SparkException: Job aborted due to stage failure: ResultStage 13 (runJob at ContextRDD.scala:238) has failed the maximum allowable number of times: 4. Most recent failure reason: org.apache.spark.shuffle.FetchFailedException at org.apache.spark.storage.ShuffleBlockFetcherIterator.throwFetchFailedException(ShuffleBlockFetcherIterator.scala:770) at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:685) at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:70) at org.apache.spark.util.CompletionIterator.next(CompletionIterator.scala:29) at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:31) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37) at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:200) at org.apache.spark.shuffle.BlockStoreShuffleReader.read(BlockStoreShuffleReader.scala:128) at org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:106) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.ZippedPartitionsRDD2.compute(ZippedPartitionsRDD.scala:89) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.nio.channels.ClosedChannelException at org.apache.spark.network.client.StreamInterceptor.channelInactive(StreamInterceptor.java:62) at org.apache.spark.network.util.TransportFrameDecoder.channelInactive(TransportFrameDecoder.java:223) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:497) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more
at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2254)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2203)
at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2202)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2202)
at org.apache.spark.scheduler.DAGScheduler.handleTaskCompletion(DAGScheduler.scala:1763)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2438)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2383)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2372)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2255)
at is.hail.sparkextras.ContextRDD.crunJobWithIndex(ContextRDD.scala:238)
at is.hail.rvd.RVD$.getKeyInfo(RVD.scala:1264)
at is.hail.rvd.RVD$.makeCoercer(RVD.scala:1339)
at is.hail.rvd.RVD$.coerce(RVD.scala:1295)
at is.hail.rvd.RVD.changeKey(RVD.scala:176)
at is.hail.rvd.RVD.changeKey(RVD.scala:169)
at is.hail.rvd.RVD.enforceKey(RVD.scala:161)
at is.hail.expr.ir.TableKeyBy.execute(TableIR.scala:1263)
at is.hail.expr.ir.TableMapRows.execute(TableIR.scala:1903)
at is.hail.expr.ir.Interpret$.run(Interpret.scala:790)
at is.hail.expr.ir.Interpret$.alreadyLowered(Interpret.scala:56)
at is.hail.expr.ir.InterpretNonCompilable$.interpretAndCoerce$1(InterpretNonCompilable.scala:16)
at is.hail.expr.ir.InterpretNonCompilable$.rewrite$1(InterpretNonCompilable.scala:53)
at is.hail.expr.ir.InterpretNonCompilable$.apply(InterpretNonCompilable.scala:58)
at is.hail.expr.ir.lowering.InterpretNonCompilablePass$.transform(LoweringPass.scala:67)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$3(LoweringPass.scala:15)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$1(LoweringPass.scala:15)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
at is.hail.expr.ir.lowering.LoweringPass.apply(LoweringPass.scala:13)
at is.hail.expr.ir.lowering.LoweringPass.apply$(LoweringPass.scala:12)
at is.hail.expr.ir.lowering.InterpretNonCompilablePass$.apply(LoweringPass.scala:62)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1(LoweringPipeline.scala:14)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1$adapted(LoweringPipeline.scala:12)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
at is.hail.expr.ir.lowering.LoweringPipeline.apply(LoweringPipeline.scala:12)
at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:29)
at is.hail.backend.spark.SparkBackend._execute(SparkBackend.scala:381)
at is.hail.backend.spark.SparkBackend.$anonfun$execute$1(SparkBackend.scala:365)
at is.hail.expr.ir.ExecuteContext$.$anonfun$scoped$3(ExecuteContext.scala:47)
at is.hail.utils.package$.using(package.scala:627)
at is.hail.expr.ir.ExecuteContext$.$anonfun$scoped$2(ExecuteContext.scala:47)
at is.hail.utils.package$.using(package.scala:627)
at is.hail.annotations.RegionPool$.scoped(RegionPool.scala:17)
at is.hail.expr.ir.ExecuteContext$.scoped(ExecuteContext.scala:46)
at is.hail.backend.spark.SparkBackend.withExecuteContext(SparkBackend.scala:275)
at is.hail.backend.spark.SparkBackend.execute(SparkBackend.scala:362)
at is.hail.backend.spark.SparkBackend.$anonfun$executeJSON$1(SparkBackend.scala:406)
at is.hail.utils.ExecutionTimer$.time(ExecutionTimer.scala:52)
at is.hail.backend.spark.SparkBackend.executeJSON(SparkBackend.scala:404)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Thread.java:748)
Hail version: 0.2.74-0c3a74d12093
Error summary: SparkException: Job aborted due to stage failure: ResultStage 13 (runJob at ContextRDD.scala:238) has failed the maximum allowable number of times: 4. Most recent failure reason: org.apache.spark.shuffle.FetchFailedException at org.apache.spark.storage.ShuffleBlockFetcherIterator.throwFetchFailedException(ShuffleBlockFetcherIterator.scala:770) at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:685) at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:70) at org.apache.spark.util.CompletionIterator.next(CompletionIterator.scala:29) at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:31) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37) at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:200) at org.apache.spark.shuffle.BlockStoreShuffleReader.read(BlockStoreShuffleReader.scala:128) at org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:106) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.ZippedPartitionsRDD2.compute(ZippedPartitionsRDD.scala:89) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.nio.channels.ClosedChannelException at org.apache.spark.network.client.StreamInterceptor.channelInactive(StreamInterceptor.java:62) at org.apache.spark.network.util.TransportFrameDecoder.channelInactive(TransportFrameDecoder.java:223) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:497) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more
ERROR: (gcloud.dataproc.jobs.wait) Job [a623cf71902d466daac2aa5f4ea0c68a] failed with error:
Google Cloud Dataproc Agent reports job failure. If logs are available, they can be found at:
https://console.cloud.google.com/dataproc/jobs/a623cf71902d466daac2aa5f4ea0c68a?project=broad-mpg-gnomad®ion=us-central1
gcloud dataproc jobs wait 'a623cf71902d466daac2aa5f4ea0c68a' --region 'us-central1' --project 'broad-mpg-gnomad'
https://console.cloud.google.com/storage/browser/dataproc-faa46220-ec08-4f5b-92bd-9722e1963047-us-central1/google-cloud-dataproc-metainfo/6128cd07-3075-4284-9f6a-88ace61011e9/jobs/a623cf71902d466daac2aa5f4ea0c68a/
gs://dataproc-faa46220-ec08-4f5b-92bd-9722e1963047-us-central1/google-cloud-dataproc-metainfo/6128cd07-3075-4284-9f6a-88ace61011e9/jobs/a623cf71902d466daac2aa5f4ea0c68a/driveroutput
```
I have the log as well, but it looks like it’s a bit too big to upload here.
This last job (with the error above) ran for 1 hour, 33 minutes on 30 non-preemptible highmem-8s (with 200GB disk space) before failing. I also tried running this job on 50 non-preemptible highmem-8s with 100GB disk space a couple times before that and saw the same error (all errors started with ` ResultStage 13 (runJob at ContextRDD.scala:238) has failed the maximum allowable number of times: 4`).
Should I use more workers? Or add more disk space, or something else? I’d appreciate any tips – I’m hoping to generate these results before ASHG. Also, that repo is currently private, but I’m happy to share with anyone who doesn’t have read access!
---
### Author:  [@konradjk](https://discuss.hail.is/u/konradjk)
#### Post date: [September 1, 2021, 1:59pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/2 "2021-09-01T13:59:41Z")
Likely executors are still dying, and the 2 usual culprits are memory and disk (in that order). Knowing your pipeline, disk would be unlikely but you never know. You could look at executor logs to get an idea, or ssh into a machine and watch its usage as the job is happening. You can also try the horrible trick of “doubling your memory” (by halving your number of cores), which I think for -8’s was: `--properties 'spark.executor.memory=15g,spark.yarn.executor.memoryOverhead=25g'`
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 1, 2021, 4:05pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/3 "2021-09-01T16:05:18Z")
thanks for the suggestions Konrad! I added this to my cluster of 30 highmem-8s (pulled out of my notes from when I last ran a densify): `--properties 'spark:spark.executor.memoryOverhead=14g,spark:spark.executor.memory=12g'`. I also monitored the workers using the spark-ui and `free -h` on a few workers. It doesn’t seem like the worker nodes used disk space (the “Disk Used” section showed zeroes every time I checked), and none of the workers used more than 13GB of memory.
the job crashed again after 1 hr 26 min at the same step – I can try using the properties you sent instead, do you think that would help?
---
### Author:  [@johnc1231](https://discuss.hail.is/u/johnc1231)
#### Post date: [September 10, 2021, 1:58pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/4 "2021-09-10T13:58:06Z")
You can email us the log if it’s too big to post. I am currently pretty confused about what’s going on.
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 10, 2021, 2:06pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/5 "2021-09-10T14:06:12Z")
ah sorry about that – just sent the logs I have to [hail-team@broadinstitute.org](mailto:hail-team@broadinstitute.org)
---
### Author:  [@tpoterba](https://discuss.hail.is/u/tpoterba)
#### Post date: [September 10, 2021, 2:50pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/6 "2021-09-10T14:50:13Z")
There’s an issue here:
[https://github.com/broadinstitute/regional\_missense\_constraint/blob/simul\_fix/rmc/pipeline/regional\_constraint.py#L383-L390](https://github.com/broadinstitute/regional_missense_constraint/blob/simul_fix/rmc/pipeline/regional_constraint.py#L383-L390)
```python
context_ht = context_ht.annotate(
start_pos=transcript_ht[context_ht.transcript].start_pos,
end_pos=transcript_ht[context_ht.transcript].end_pos,
transcript_size=(
transcript_ht[context_ht.transcript].end_pos
- transcript_ht[context_ht.transcript].start_pos
)
+ 1,
)
```
This is doing four shuffle joins – one for each time that `transcript_ht` appears, I believe. You could reimplement this as:
```python
context_ht = context_ht.annotate(tx = transcript_ht[context_ht.transcript])
context_ht = context_ht.annotate(start_pos=context_ht.start_pos, ...)
```
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 10, 2021, 3:13pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/7 "2021-09-10T15:13:55Z")
ooh good catch. I’ll try that, thank you!!
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 10, 2021, 5:10pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/8 "2021-09-10T17:10:02Z")
ah that doesn’t seem to have fixed it – will send the newest log
---
### Author:  [@danking](https://discuss.hail.is/u/danking)
#### Post date: [September 10, 2021, 6:19pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/9 "2021-09-10T18:19:42Z")
Hey @ch-kr!
We’re still digging into this. Could you share which arguments you supplied to the gnomad scripts? For example, is `args.pre_process_data` enabled?
---
### Author:  [@danking](https://discuss.hail.is/u/danking)
#### Post date: [September 10, 2021, 6:23pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/10 "2021-09-10T18:23:07Z")
Also, do you know if the `{temp_path}/transcript.ht` is successfully written? This is around line 380. of regional\_constraint.py.
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 10, 2021, 6:27pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/11 "2021-09-10T18:27:24Z")
thank you for looking into this!!
this is the command I used:
```auto
hailctl dataproc submit kc pipeline/regional_constraint.py --search-for-simul-breaks --slack-channel "@kc (she/her)" --chisq-threshold 6.6
```
re: transcript HT: yup! I actually wrote the HT in June, here’s the schema:
```auto
----------------------------------------
File Type: Table
Partitions: 57735
Rows: 57735
Empty partitions: 0
Min(rows/partition): 1
Max(rows/partition): 1
Median(rows/partition): 1
Mean(rows/partition): 1
StdDev(rows/partition): 0
----------------------------------------
Global fields:
----------------------------------------
Row fields:
'transcript': str
'end_pos': int32
'start_pos': int32
----------------------------------------
Key: ['transcript']
----------------------------------------
```
---
### Author:  [@danking](https://discuss.hail.is/u/danking)
#### Post date: [September 10, 2021, 7:21pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/12 "2021-09-10T19:21:16Z")
Great. And I assume you’re stuck trying to write the checkpoint file: `{temp_path}/simul_breaks.ht`, yeah?
EDIT:
Seems like you’d actually be stuck earlier at: `f"{temp_path}/simul_break_{window_size}_temp.ht"`
---
### Author:  [@danking](https://discuss.hail.is/u/danking)
#### Post date: [September 10, 2021, 7:28pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/13 "2021-09-10T19:28:59Z")
Actually, @ch-kr , do you have the output log from this job? I’m trying to narrow down which part of the Python code corresponds to your issue. It’d also be helpful to know which temp tables were created and which were not, if you still have that information.
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 10, 2021, 7:43pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/14 "2021-09-10T19:43:44Z")
yup! I’m running it now (actually sorry it just crashed), and it seems to be failing here: [https://github.com/broadinstitute/regional\_missense\_constraint/blob/simul\_fix/rmc/utils/constraint.py#L1378](https://github.com/broadinstitute/regional_missense_constraint/blob/simul_fix/rmc/utils/constraint.py#L1378) [output.txt](https://discuss.hail.is/uploads/short-url/fdqbYEJtzgmOTtGYpCphubqJdcA.txt) (58.1 KB)
Cluster: [Google Cloud Platform](https://console.cloud.google.com/dataproc/clusters/kc/jobs?region=us-central1&project=broad-mpg-gnomad)
Job page: [Google Cloud Platform](https://console.cloud.google.com/dataproc/jobs/70781f611e214b7f9cf269cff9615de6/monitoring?region=us-central1&project=broad-mpg-gnomad)
---
### Author:  [@danking](https://discuss.hail.is/u/danking)
#### Post date: [September 10, 2021, 8:36pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/15 "2021-09-10T20:36:29Z")
OK, just to confirm, that output.txt seems to suggest you fail on this [checkpoint](https://github.com/broadinstitute/regional_missense_constraint/blob/simul_fix/rmc/utils/constraint.py#L1378-L1380). Does that agree with your understanding? And also just confirming: the code you’re running is the same as the code pushed to GitHub in the branch `simul_fix`?
Here’s a theory, in [constraint.py lines 637-638](https://github.com/broadinstitute/regional_missense_constraint/blob/d63670c41769ab88a7e79395e4556b1fe12cc020/rmc/utils/constraint.py#L637-L638)
```auto
group_ht = ht.group_by(search_field).aggregate(max_chisq=hl.agg.max(ht.chisq))
ht = ht.annotate(max_chisq=group_ht[ht.transcript].max_chisq)
```
Hail implements this as _three_ shuffles. The first shuffle computes the max chisq statistic per-transcript group. The second shuffle builds a Table which maps from transcript to (locus, alleles). The third shuffle sends the per-transcript table back to a per-locus-&-alleles table.
Can you try adding
```auto
group_ht = group_ht.checkpoint(...)
```
after line 637 (the `group_by` line)?
I think that will alleviate some of the pressure on the Spark workers during these shuffles by splitting them into two separate Spark runs.
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 10, 2021, 8:40pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/16 "2021-09-10T20:40:51Z")
I wanted to send a quick update that I just tried running this code again (dropping more annotations before checkpointing – I had a `drop` here [https://github.com/broadinstitute/regional\_missense\_constraint/blob/simul\_fix/rmc/utils/constraint.py#L1373](https://github.com/broadinstitute/regional_missense_constraint/blob/simul_fix/rmc/utils/constraint.py#L1373) but changed it to the `select` that is currently on that branch of the repo), and I finally managed to checkpoint the table!
thanks also for the new suggestion! I’ll take a closer look (just noticed it while typing this reply) and also try that out. also, for your first two questions: yes and yes (code was failing at that checkpoint, and I’m running the code on the `simul_fix` branch)
---
### Author:  [@danking](https://discuss.hail.is/u/danking)
#### Post date: [September 10, 2021, 8:56pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/17 "2021-09-10T20:56:48Z")
Awesome! That was another thing I noticed in search\_for\_break. It looks like you create a number of annotations that you use to calculate the chisq value. Then you group\_by().aggregate() which destroys those annotations _on `group_ht`_, but those annotations still exist on `ht`. It might be valuable to, in general, drop those annotations from `ht` before the annotate on line 638.
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 10, 2021, 9:00pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/18 "2021-09-10T21:00:43Z")
great point about the annotations – I’ll take another look through the code to clean it up a bit more.
thank you for all of the help digging into this issue!! I really appreciate the tips about which parts of the code trigger shuffles and could benefit from checkpoints.
---
### Author:  [@ch-kr](https://discuss.hail.is/u/ch-kr)
#### Post date: [September 13, 2021, 4:58pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/19 "2021-09-13T16:58:16Z")
hi again, and thanks for the tips from before! I’m running into a new issue trying to run the same pieces of code: I’m trying to run this function ([https://github.com/broadinstitute/regional\_missense\_constraint/blob/simul\_fix/rmc/utils/constraint.py#L1229](https://github.com/broadinstitute/regional_missense_constraint/blob/simul_fix/rmc/utils/constraint.py#L1229)), but it seems to be crashing at this checkpoint ([https://github.com/broadinstitute/regional\_missense\_constraint/blob/simul\_fix/rmc/utils/constraint.py#L1384](https://github.com/broadinstitute/regional_missense_constraint/blob/simul_fix/rmc/utils/constraint.py#L1384)).
Output:
```auto
Welcome to
____ <>__
/ /_/ / ____ / /
/ __ / _ `/ / /
/_/ /_/\_,_/_/_/ version 0.2.75-52791e9868e4
LOGGING: writing to /RMC.log
INFO (regional_missense_constraint 356): Searching for two simultaneous breaks in transcripts that didn't have a single significant break...
INFO (regional_missense_constraint 376): Getting start and end positions and total size for each transcript...
INFO (regional_missense_constraint_generic 247): Total number of bases in the exome: 54426835
INFO (regional_missense_constraint_generic 248): Total number of missense variants in gnomAD exomes: 5257859
INFO (regional_missense_constraint_generic 251): Getting average bases between missense variants and returning...
INFO (regional_missense_constraint 422): Minimum window size (window size needed to observe 10 missense variants on average): 100
INFO (regional_missense_constraint 428): Searching for transcripts with simultaneous breaks...
INFO (constraint_utils 1273): Getting smallest window end and post-window positions...
INFO (constraint_utils 1274): Also getting maximum simultaneous break size (0.900000 * largest transcript size)...
INFO (constraint_utils 1009): Annotating each transcript with max window size...
2021-09-13 16:46:33 Hail: INFO: Ordering unsorted dataset with network shuffle0]
INFO (constraint_utils 1012): Maximum window size: 2083174===>(6909 + 2) / 6910]
INFO (constraint_utils 1059): HT count: 22750897
INFO (constraint_utils 1290): Checking all possible window sizes... + 2) / 8043]
----------------------------------------
Global fields:
None
----------------------------------------
Row fields:
'locus': locus
'transcript': str
'mu_snp': float64
'total_exp': float64
'_mu_scan': dict
'total_mu': float64
'cumulative_obs': dict
'observed': int32
'cumulative_exp': float64
'total_obs': int64
'reverse': struct {
obs: int64,
exp: float64
}
'overall_oe': float64
'post_window_pos': int32
'window_end': int32
----------------------------------------
Key: ['locus', 'transcript']
----------------------------------------
INFO (constraint_utils 1382): Window size: 101
INFO (constraint_utils 1383): Checkpointing first temp ht...
INFO (regional_missense_constraint 687): Copying hail log to logging bucket...
2021-09-13 16:52:58 Hail: INFO: copying log to 'gs://regional_missense_constraint/logs/RMC.log'...
Traceback (most recent call last):
File "/tmp/19569c6f88d54662afe2c56ab0ee51fb/regional_constraint.py", line 807, in
main(args)
File "/tmp/19569c6f88d54662afe2c56ab0ee51fb/regional_constraint.py", line 429, in main
context_ht = search_two_break_windows(
File "/tmp/19569c6f88d54662afe2c56ab0ee51fb/pyscripts_1j7_3mzj.zip/rmc/utils/constraint.py", line 1384, in search_two_break_windows
File "", line 2, in checkpoint
File "/opt/conda/default/lib/python3.8/site-packages/hail/typecheck/check.py", line 577, in wrapper
return __original_func(*args_, **kwargs_)
File "/opt/conda/default/lib/python3.8/site-packages/hail/table.py", line 1238, in checkpoint
self.write(output=output, overwrite=overwrite, stage_locally=stage_locally, _codec_spec=_codec_spec)
File "", line 2, in write
File "/opt/conda/default/lib/python3.8/site-packages/hail/typecheck/check.py", line 577, in wrapper
return __original_func(*args_, **kwargs_)
File "/opt/conda/default/lib/python3.8/site-packages/hail/table.py", line 1271, in write
Env.backend().execute(ir.TableWrite(self._tir, ir.TableNativeWriter(output, overwrite, stage_locally, _codec_spec)))
File "/opt/conda/default/lib/python3.8/site-packages/hail/backend/py4j_backend.py", line 98, in execute
raise e
File "/opt/conda/default/lib/python3.8/site-packages/hail/backend/py4j_backend.py", line 74, in execute
result = json.loads(self._jhc.backend().executeJSON(jir))
File "/usr/lib/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1304, in __call__
File "/opt/conda/default/lib/python3.8/site-packages/hail/backend/py4j_backend.py", line 30, in deco
raise FatalError('%s\n\nJava stack trace:\n%s\n'
hail.utils.java.FatalError: MethodTooLargeException: Method too large: __C514Compiled.__ m516split_Let_region6_8052 (L __C9213__ m516split_LetSpills;)V
Java stack trace:
is.hail.relocated.org.objectweb.asm.MethodTooLargeException: Method too large: __C514Compiled.__ m516split_Let_region6_8052 (L __C9213__ m516split_LetSpills;)V
at is.hail.relocated.org.objectweb.asm.MethodWriter.computeMethodInfoSize(MethodWriter.java:2087)
at is.hail.relocated.org.objectweb.asm.ClassWriter.toByteArray(ClassWriter.java:489)
at is.hail.lir.Emit$.apply(Emit.scala:217)
at is.hail.lir.Classx.$anonfun$asBytes$4(X.scala:110)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:461)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)
at scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
at scala.collection.TraversableOnce.to(TraversableOnce.scala:366)
at scala.collection.TraversableOnce.to$(TraversableOnce.scala:364)
at scala.collection.AbstractIterator.to(Iterator.scala:1431)
at scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:358)
at scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:358)
at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1431)
at scala.collection.TraversableOnce.toArray(TraversableOnce.scala:345)
at scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:339)
at scala.collection.AbstractIterator.toArray(Iterator.scala:1431)
at is.hail.lir.Classx.asBytes(X.scala:123)
at is.hail.asm4s.ClassBuilder.classBytes(ClassBuilder.scala:351)
at is.hail.asm4s.ModuleBuilder.$anonfun$classesBytes$1(ClassBuilder.scala:152)
at is.hail.asm4s.ModuleBuilder.$anonfun$classesBytes$1$adapted(ClassBuilder.scala:152)
at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:62)
at scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:53)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:105)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:49)
at scala.collection.TraversableOnce.to(TraversableOnce.scala:366)
at scala.collection.TraversableOnce.to$(TraversableOnce.scala:364)
at scala.collection.AbstractIterator.to(Iterator.scala:1431)
at scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:358)
at scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:358)
at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1431)
at scala.collection.TraversableOnce.toArray(TraversableOnce.scala:345)
at scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:339)
at scala.collection.AbstractIterator.toArray(Iterator.scala:1431)
at is.hail.asm4s.ModuleBuilder.classesBytes(ClassBuilder.scala:153)
at is.hail.expr.ir.EmitClassBuilder.resultWithIndex(EmitClassBuilder.scala:660)
at is.hail.expr.ir.WrappedEmitClassBuilder.resultWithIndex(EmitClassBuilder.scala:154)
at is.hail.expr.ir.WrappedEmitClassBuilder.resultWithIndex$(EmitClassBuilder.scala:154)
at is.hail.expr.ir.EmitFunctionBuilder.resultWithIndex(EmitClassBuilder.scala:1052)
at is.hail.expr.ir.Compile$.apply(Compile.scala:78)
at is.hail.expr.ir.CompileAndEvaluate$.$anonfun$_apply$1(CompileAndEvaluate.scala:49)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:49)
at is.hail.expr.ir.CompileAndEvaluate$.evalToIR(CompileAndEvaluate.scala:29)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.evaluate$1(LowerOrInterpretNonCompilable.scala:29)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.rewrite$1(LowerOrInterpretNonCompilable.scala:66)
at is.hail.expr.ir.LowerOrInterpretNonCompilable$.apply(LowerOrInterpretNonCompilable.scala:71)
at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.transform(LoweringPass.scala:68)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$3(LoweringPass.scala:15)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$1(LoweringPass.scala:15)
at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
at is.hail.expr.ir.lowering.LoweringPass.apply(LoweringPass.scala:13)
at is.hail.expr.ir.lowering.LoweringPass.apply$(LoweringPass.scala:12)
at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.apply(LoweringPass.scala:63)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1(LoweringPipeline.scala:14)
at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1$adapted(LoweringPipeline.scala:12)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
at is.hail.expr.ir.lowering.LoweringPipeline.apply(LoweringPipeline.scala:12)
at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:46)
at is.hail.backend.spark.SparkBackend._execute(SparkBackend.scala:381)
at is.hail.backend.spark.SparkBackend.$anonfun$execute$1(SparkBackend.scala:365)
at is.hail.expr.ir.ExecuteContext$.$anonfun$scoped$3(ExecuteContext.scala:47)
at is.hail.utils.package$.using(package.scala:638)
at is.hail.expr.ir.ExecuteContext$.$anonfun$scoped$2(ExecuteContext.scala:47)
at is.hail.utils.package$.using(package.scala:638)
at is.hail.annotations.RegionPool$.scoped(RegionPool.scala:17)
at is.hail.expr.ir.ExecuteContext$.scoped(ExecuteContext.scala:46)
at is.hail.backend.spark.SparkBackend.withExecuteContext(SparkBackend.scala:275)
at is.hail.backend.spark.SparkBackend.execute(SparkBackend.scala:362)
at is.hail.backend.spark.SparkBackend.$anonfun$executeJSON$1(SparkBackend.scala:406)
at is.hail.utils.ExecutionTimer$.time(ExecutionTimer.scala:52)
at is.hail.backend.spark.SparkBackend.executeJSON(SparkBackend.scala:404)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Thread.java:748)
Hail version: 0.2.75-52791e9868e4
Error summary: MethodTooLargeException: Method too large: __C514Compiled.__ m516split_Let_region6_8052 (L __C9213__ m516split_LetSpills;)V
ERROR: (gcloud.dataproc.jobs.submit.pyspark) Job [19569c6f88d54662afe2c56ab0ee51fb] failed with error:
Google Cloud Dataproc Agent reports job failure. If logs are available, they can be found at:
https://console.cloud.google.com/dataproc/jobs/19569c6f88d54662afe2c56ab0ee51fb?project=broad-mpg-gnomad®ion=us-central1
gcloud dataproc jobs wait '19569c6f88d54662afe2c56ab0ee51fb' --region 'us-central1' --project 'broad-mpg-gnomad'
https://console.cloud.google.com/storage/browser/dataproc-faa46220-ec08-4f5b-92bd-9722e1963047-us-central1/google-cloud-dataproc-metainfo/ca21d0ac-2117-4224-a272-a36d7774a17a/jobs/19569c6f88d54662afe2c56ab0ee51fb/
gs://dataproc-faa46220-ec08-4f5b-92bd-9722e1963047-us-central1/google-cloud-dataproc-metainfo/ca21d0ac-2117-4224-a272-a36d7774a17a/jobs/19569c6f88d54662afe2c56ab0ee51fb/driveroutput
Traceback (most recent call last):
File "/Users/kchao/anaconda3/envs/hail/bin/hailctl", line 8, in
sys.exit(main())
File "/Users/kchao/anaconda3/envs/hail/lib/python3.7/site-packages/hailtop/hailctl/ __main__.py", line 100, in main
cli.main(args)
File "/Users/kchao/anaconda3/envs/hail/lib/python3.7/site-packages/hailtop/hailctl/dataproc/cli.py", line 122, in main
jmp[args.module].main(args, pass_through_args)
File "/Users/kchao/anaconda3/envs/hail/lib/python3.7/site-packages/hailtop/hailctl/dataproc/submit.py", line 78, in main
gcloud.run(cmd)
File "/Users/kchao/anaconda3/envs/hail/lib/python3.7/site-packages/hailtop/hailctl/dataproc/gcloud.py", line 9, in run
return subprocess.check_call(["gcloud"] + command)
File "/Users/kchao/anaconda3/envs/hail/lib/python3.7/subprocess.py", line 328, in check_call
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['gcloud', 'dataproc', 'jobs', 'submit', 'pyspark', 'pipeline/regional_constraint.py', '--cluster=kc', '--files=', '--py-files=/var/folders/xq/8jnhrt2s2h58ts2v0br5g8gm0000gp/T/pyscripts_1j7_3mzj.zip', '--properties=', '--', '--search-for-simul-breaks', '--slack-channel', '@kc (she/her)', '--chisq-threshold', '6.6']' returned non-zero exit status 1.
```
I’ll send the log via email. I don’t think there are any other fields I can drop from that table – is there something else I could try to bypass this new error?
---
### Author:  [@danking](https://discuss.hail.is/u/danking)
#### Post date: [September 13, 2021, 5:58pm UTC](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222/20 "2021-09-13T17:58:40Z")
Does this pipeline include the `part_min_and_max` code we spoke about in the other thread?
[Next page](https://discuss.hail.is/t/resultstage-has-failed-the-maximum-allowable-number-of-times/2222.md?page=2)