Hail Tutorial Error

andie_d · March 31, 2024, 7:35pm

I am trying to follow the general hail tutorial using my Mac OS (M2 Chip). I followed the download instructions and downloaded Java 8 with Azul. I continue to get the follow error when I simply try to initialize Hail: hl.init()

Error:
Py4JJavaError Traceback (most recent call last)
Cell In[3], line 14
11 from pprint import pprint
13 # Initialize Hail
—> 14 hl.init()

File :2, in init(sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, default_reference, idempotent, global_seed, spark_conf, skip_logging_configuration, local_tmpdir, _optimizer_iterations, backend, driver_cores, driver_memory, worker_cores, worker_memory, gcs_requester_pays_configuration, regions, gcs_bucket_allow_list)

File ~/mambaforge/lib/python3.10/site-packages/hail/typecheck/check.py:584, in _make_dec..wrapper(__original_func, *args, **kwargs)
581 @decorator
582 def wrapper(original_func: Callable[…, T], *args, **kwargs) → T:
583 args, kwargs = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
→ 584 return original_func(*args, **kwargs)

File ~/mambaforge/lib/python3.10/site-packages/hail/context.py:386, in init(sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, default_reference, idempotent, global_seed, spark_conf, skip_logging_configuration, local_tmpdir, _optimizer_iterations, backend, driver_cores, driver_memory, worker_cores, worker_memory, gcs_requester_pays_configuration, regions, gcs_bucket_allow_list)
366 return hail_event_loop().run_until_complete(
367 init_batch(
368 log=log,
(…)
383 )
384 )
385 if backend == ‘spark’:
→ 386 return init_spark(
387 sc=sc,
388 app_name=app_name,
389 master=master,
390 local=local,
391 min_block_size=min_block_size,
392 branching_factor=branching_factor,
393 spark_conf=spark_conf,
394 _optimizer_iterations=_optimizer_iterations,
395 log=log,
396 quiet=quiet,
397 append=append,
398 tmp_dir=tmp_dir,
399 local_tmpdir=local_tmpdir,
400 default_reference=default_reference,
401 global_seed=global_seed,
402 skip_logging_configuration=skip_logging_configuration,
403 gcs_requester_pays_configuration=gcs_requester_pays_configuration,
404 )
405 if backend == ‘local’:
406 return init_local(
407 log=log,
408 quiet=quiet,
(…)
414 gcs_requester_pays_configuration=gcs_requester_pays_configuration,
415 )

File :2, in init_spark(sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, default_reference, idempotent, global_seed, spark_conf, skip_logging_configuration, local_tmpdir, _optimizer_iterations, gcs_requester_pays_configuration)

File ~/mambaforge/lib/python3.10/site-packages/hail/typecheck/check.py:584, in _make_dec..wrapper(__original_func, *args, **kwargs)
581 @decorator
582 def wrapper(original_func: Callable[…, T], *args, **kwargs) → T:
583 args, kwargs = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
→ 584 return original_func(*args, **kwargs)

File ~/mambaforge/lib/python3.10/site-packages/hail/context.py:476, in init_spark(sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, default_reference, idempotent, global_seed, spark_conf, skip_logging_configuration, local_tmpdir, _optimizer_iterations, gcs_requester_pays_configuration)
467 app_name = app_name or ‘Hail’
468 (
469 gcs_requester_pays_project,
470 gcs_requester_pays_buckets,
(…)
474 )
475 )
→ 476 backend = SparkBackend(
477 idempotent,
478 sc,
479 spark_conf,
480 app_name,
481 master,
482 local,
483 log,
484 quiet,
485 append,
486 min_block_size,
487 branching_factor,
488 tmpdir,
489 local_tmpdir,
490 skip_logging_configuration,
491 optimizer_iterations,
492 gcs_requester_pays_project=gcs_requester_pays_project,
493 gcs_requester_pays_buckets=gcs_requester_pays_buckets,
494 )
495 if not backend.fs.exists(tmpdir):
496 backend.fs.mkdir(tmpdir)

File ~/mambaforge/lib/python3.10/site-packages/hail/backend/spark_backend.py:126, in SparkBackend.init(self, idempotent, sc, spark_conf, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmpdir, local_tmpdir, skip_logging_configuration, optimizer_iterations, gcs_requester_pays_project, gcs_requester_pays_buckets)
124 jhc = hail_package.HailContext.getOrCreate(jbackend, branching_factor, optimizer_iterations)
125 else:
→ 126 jbackend = hail_package.backend.spark.SparkBackend.apply(
127 jsc,
128 app_name,
129 master,
130 local,
131 log,
132 True,
133 append,
134 skip_logging_configuration,
135 min_block_size,
136 tmpdir,
137 local_tmpdir,
138 gcs_requester_pays_project,
139 gcs_requester_pays_buckets,
140 )
141 jhc = hail_package.HailContext.apply(jbackend, branching_factor, optimizer_iterations)
143 self._jsc = jbackend.sc()

File ~/mambaforge/lib/python3.10/site-packages/py4j/java_gateway.py:1321, in JavaMember.call(self, *args)
1315 command = proto.CALL_COMMAND_NAME +
1316 self.command_header +
1317 args_command +
1318 proto.END_COMMAND_PART
1320 answer = self.gateway_client.send_command(command)
→ 1321 return_value = get_return_value(
1322 answer, self.gateway_client, self.target_id, self.name)
1324 for temp_arg in temp_args:
1325 temp_arg._detach()

File ~/mambaforge/lib/python3.10/site-packages/py4j/protocol.py:326, in get_return_value(answer, gateway_client, target_id, name)
324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
325 if answer[1] == REFERENCE_TYPE:
→ 326 raise Py4JJavaError(
327 “An error occurred while calling {0}{1}{2}.\n”.
328 format(target_id, “.”, name), value)
329 else:
330 raise Py4JError(
331 “An error occurred while calling {0}{1}{2}. Trace:\n{3}\n”.
332 format(target_id, “.”, name, value))

Py4JJavaError: An error occurred while calling z:is.hail.backend.spark.SparkBackend.apply.
: java.lang.NoClassDefFoundError: Could not initialize class org.apache.spark.unsafe.array.ByteArrayMethods
at org.apache.spark.memory.MemoryManager.defaultPageSizeBytes$lzycompute(MemoryManager.scala:264)
at org.apache.spark.memory.MemoryManager.defaultPageSizeBytes(MemoryManager.scala:254)
at org.apache.spark.memory.MemoryManager.$anonfun$pageSizeBytes$1(MemoryManager.scala:273)
at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.memory.MemoryManager.(MemoryManager.scala:273)
at org.apache.spark.memory.UnifiedMemoryManager.(UnifiedMemoryManager.scala:58)
at org.apache.spark.memory.UnifiedMemoryManager$.apply(UnifiedMemoryManager.scala:207)
at org.apache.spark.SparkEnv$.create(SparkEnv.scala:320)
at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194)
at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:279)
at org.apache.spark.SparkContext.(SparkContext.scala:464)
at is.hail.backend.spark.SparkBackend$.configureAndCreateSparkContext(SparkBackend.scala:156)
at is.hail.backend.spark.SparkBackend$.apply(SparkBackend.scala:262)
at is.hail.backend.spark.SparkBackend.apply(SparkBackend.scala)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.ExceptionInInitializerError: Exception java.lang.ExceptionInInitializerError [in thread “Thread-3”]
at org.apache.spark.unsafe.array.ByteArrayMethods.(ByteArrayMethods.java:56)
… 25 more

hl.plot.output_notebook()
BokehJS 3.4.0 successfully loaded.

Download public 1000 Genomes data

hl.utils.get_1kg(‘data/’)
Initializing Hail with default parameters…

Py4JJavaError Traceback (most recent call last)
Cell In[5], line 2
1 # Download public 1000 Genomes data
----> 2 hl.utils.get_1kg(‘data/’)

File ~/mambaforge/lib/python3.10/site-packages/hail/utils/tutorial.py:59, in get_1kg(output_dir, overwrite)
44 def get_1kg(output_dir, overwrite: bool = False):
45 “”“Download subset of the 1000 Genomes <http://www.internationalgenome.org/>__
46 dataset and sample annotations.
47
(…)
57 If True, overwrite any existing files/directories at output_dir.
58 “””
—> 59 fs = Env.fs()
61 if not _dir_exists(fs, output_dir):
62 fs.mkdir(output_dir)

File ~/mambaforge/lib/python3.10/site-packages/hail/utils/java.py:115, in Env.fs()
113 @staticmethod
114 def fs():
→ 115 return Env.backend().fs

File ~/mambaforge/lib/python3.10/site-packages/hail/utils/java.py:91, in Env.backend()
89 @staticmethod
90 def backend() → ‘hail.backend.Backend’:
—> 91 return Env.hc()._backend

File ~/mambaforge/lib/python3.10/site-packages/hail/utils/java.py:64, in Env.hc()
61 sys.stderr.flush()
62 from …context import init
—> 64 init()
66 assert Env._hc is not None
67 return Env._hc