Overwrite on an existing matrix table

Hail Query & hailctl
1

I am using the command below to write the results on a file name that I previously generated, however I have set overwrite parameter to True as command below still I am getting error in below.
Any help to resolve is highly appreciated.

Command:

mt_vep.write(output="dnax://" + dbid_out + "/" + filepre + ".mt", overwrite=True)

Error:

FatalError                                Traceback (most recent call last)
Cell In[10], line 16
     13 # VEP annotation
     14 mt_vep = hl.methods.vep(mt2, config="file:///mnt/project/Work/vep.json")
---> 16 mt_vep.write("dnax://" + dbid_out + "/" + filepre + ".mt", overwrite=True)
     17 print("end Time =", datetime.now())
     18 continue

File <decorator-gen-1266>:2, in write(self, output, overwrite, stage_locally, _codec_spec, _partitions)

File /opt/conda/lib/python3.9/site-packages/hail/typecheck/check.py:584, in _make_dec.<locals>.wrapper(__original_func, *args, **kwargs)
    581 @decorator
    582 def wrapper(__original_func, *args, **kwargs):
    583     args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 584     return __original_func(*args_, **kwargs_)

File /opt/conda/lib/python3.9/site-packages/hail/matrixtable.py:2739, in MatrixTable.write(self, output, overwrite, stage_locally, _codec_spec, _partitions)
   2736     _partitions_type = None
   2738 writer = ir.MatrixNativeWriter(output, overwrite, stage_locally, _codec_spec, _partitions, _partitions_type)
-> 2739 Env.backend().execute(ir.MatrixWrite(self._mir, writer))

File /opt/conda/lib/python3.9/site-packages/hail/backend/py4j_backend.py:82, in Py4JBackend.execute(self, ir, timed)
     80     return (value, timings) if timed else value
     81 except FatalError as e:
---> 82     raise e.maybe_user_error(ir) from None

File /opt/conda/lib/python3.9/site-packages/hail/backend/py4j_backend.py:76, in Py4JBackend.execute(self, ir, timed)
     74 # print(self._hail_package.expr.ir.Pretty.apply(jir, True, -1))
     75 try:
---> 76     result_tuple = self._jbackend.executeEncode(jir, stream_codec, timed)
     77     (result, timings) = (result_tuple._1(), result_tuple._2())
     78     value = ir.typ._from_encoding(result)

File /cluster/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/java_gateway.py:1321, in JavaMember.__call__(self, *args)
   1315 command = proto.CALL_COMMAND_NAME +\
   1316     self.command_header +\
   1317     args_command +\
   1318     proto.END_COMMAND_PART
   1320 answer = self.gateway_client.send_command(command)
-> 1321 return_value = get_return_value(
   1322     answer, self.gateway_client, self.target_id, self.name)
   1324 for temp_arg in temp_args:
   1325     temp_arg._detach()

File /opt/conda/lib/python3.9/site-packages/hail/backend/py4j_backend.py:35, in handle_java_exception.<locals>.deco(*args, **kwargs)
     33     tpl = Env.jutils().handleForPython(e.java_exception)
     34     deepest, full, error_id = tpl._1(), tpl._2(), tpl._3()
---> 35     raise fatal_error_from_java_error_triplet(deepest, full, error_id) from None
     36 except pyspark.sql.utils.CapturedException as e:
     37     raise FatalError('%s\n\nJava stack trace:\n%s\n'
     38                      'Hail version: %s\n'
     39                      'Error summary: %s' % (e.desc, e.stackTrace, hail.__version__, e.desc)) from None

FatalError: HailException: file already exists: dnax://database-GXKvB5QJK2zZ4Jq80gggX48v/ukb23157_c10_b0_v1.mt/rows

Java stack trace:
is.hail.utils.HailException: file already exists: dnax://database-GXKvB5QJK2zZ4Jq80gggX48v/ukb23157_c10_b0_v1.mt/rows
	at __C2328Compiled.__m2332split_WriteMetadata(Emit.scala)
	at __C2328Compiled.apply(Emit.scala)
	at is.hail.expr.ir.CompileAndEvaluate$.$anonfun$_apply$4(CompileAndEvaluate.scala:61)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
	at is.hail.expr.ir.CompileAndEvaluate$.$anonfun$_apply$2(CompileAndEvaluate.scala:61)
	at is.hail.expr.ir.CompileAndEvaluate$.$anonfun$_apply$2$adapted(CompileAndEvaluate.scala:59)
	at is.hail.backend.ExecuteContext.$anonfun$scopedExecution$1(ExecuteContext.scala:140)
	at is.hail.utils.package$.using(package.scala:635)
	at is.hail.backend.ExecuteContext.scopedExecution(ExecuteContext.scala:140)
	at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:59)
	at is.hail.expr.ir.CompileAndEvaluate$.evalToIR(CompileAndEvaluate.scala:33)
	at is.hail.expr.ir.LowerOrInterpretNonCompilable$.evaluate$1(LowerOrInterpretNonCompilable.scala:30)
	at is.hail.expr.ir.LowerOrInterpretNonCompilable$.rewrite$1(LowerOrInterpretNonCompilable.scala:67)
	at is.hail.expr.ir.LowerOrInterpretNonCompilable$.apply(LowerOrInterpretNonCompilable.scala:72)
	at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.transform(LoweringPass.scala:67)
	at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$3(LoweringPass.scala:16)
	at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
	at is.hail.expr.ir.lowering.LoweringPass.$anonfun$apply$1(LoweringPass.scala:16)
	at is.hail.utils.ExecutionTimer.time(ExecutionTimer.scala:81)
	at is.hail.expr.ir.lowering.LoweringPass.apply(LoweringPass.scala:14)
	at is.hail.expr.ir.lowering.LoweringPass.apply$(LoweringPass.scala:13)
	at is.hail.expr.ir.lowering.LowerOrInterpretNonCompilablePass$.apply(LoweringPass.scala:62)
	at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1(LoweringPipeline.scala:22)
	at is.hail.expr.ir.lowering.LoweringPipeline.$anonfun$apply$1$adapted(LoweringPipeline.scala:20)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at is.hail.expr.ir.lowering.LoweringPipeline.apply(LoweringPipeline.scala:20)
	at is.hail.expr.ir.CompileAndEvaluate$._apply(CompileAndEvaluate.scala:50)
	at is.hail.backend.spark.SparkBackend._execute(SparkBackend.scala:463)
	at is.hail.backend.spark.SparkBackend.$anonfun$executeEncode$2(SparkBackend.scala:499)
	at is.hail.backend.ExecuteContext$.$anonfun$scoped$3(ExecuteContext.scala:75)
	at is.hail.utils.package$.using(package.scala:635)
	at is.hail.backend.ExecuteContext$.$anonfun$scoped$2(ExecuteContext.scala:75)
	at is.hail.utils.package$.using(package.scala:635)
	at is.hail.annotations.RegionPool$.scoped(RegionPool.scala:17)
	at is.hail.backend.ExecuteContext$.scoped(ExecuteContext.scala:63)
	at is.hail.backend.spark.SparkBackend.withExecuteContext(SparkBackend.scala:351)
	at is.hail.backend.spark.SparkBackend.$anonfun$executeEncode$1(SparkBackend.scala:496)
	at is.hail.utils.ExecutionTimer$.time(ExecutionTimer.scala:52)
	at is.hail.backend.spark.SparkBackend.executeEncode(SparkBackend.scala:495)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
	at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
	at java.lang.Thread.run(Thread.java:750)



Hail version: 0.2.116-cd64e0876c94
Error summary: HailException: file already exists: dnax://database-GXKvB5QJK2zZ4Jq80gggX48v/ukb23157_c10_b0_v1.mt/rows
2

Hey @asppagh54 ,

This is indeed fishy. Can you share the Hail log file that was generated? It should live on the same machine on which the Python script was executing.

This looks suspiciously like delete(..., recursive=true) is not implemented correctly by the DNAX Hadoop FileSystem.

Can you try something simple like:

hl.utils.range_table(1).write(f'dnax://{dbid_out}/test.ht')
hl.utils.range_table(1).write(f'dnax://{dbid_out}/test.ht', overwrite=True)

I know this works (it is used extensively in our tests). If it fails in DNANexus, then it seems quite likely this Java code will also fail to recursively delete the directory:

getFileSystem("dnax://...").delete(
    new hadoop.fs.Path("dnax://..."),
    true
)

That’s the same code we use to recursively delete a directory before overwriting.