Hi,
So I am trying to start using hail on a cluster, and apparently everything that I need should be already there, but when I try to check if hails is working I get the following error:
Initializing Spark and Hail with default parameters...
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
<ipython-input-2-91f3646439ee> in <module>()
----> 1 mt = hl.balding_nichols_model(3,100,100)
2 mt._force_count_rows()
<decorator-gen-1530> in balding_nichols_model(n_populations, n_samples, n_variants, n_partitions, pop_dist, fst, af_dist, reference_genome, mixture)
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/typecheck/check.py in wrapper(__original_func, *args, **kwargs)
582 @decorator
583 def wrapper(__original_func, *args, **kwargs):
--> 584 args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
585 return __original_func(*args_, **kwargs_)
586
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/typecheck/check.py in check_all(f, args, kwargs, checks, is_method)
510 if i < len(args):
511 arg = args[i]
--> 512 args_.append(checker.check(arg, name, arg_name))
513 # passed as keyword
514 else:
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/typecheck/check.py in check(self, x, caller, param)
54 for tc in self.checkers:
55 try:
---> 56 return tc.check(x, caller, param)
57 except TypecheckFailure:
58 pass
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/typecheck/check.py in check(self, x, caller, param)
301 for tc, f in self.fs:
302 try:
--> 303 return f(tc.check(x, caller, param))
304 except TypecheckFailure:
305 pass
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/genetics/reference_genome.py in <lambda>(x)
8
9 rg_type = lazy()
---> 10 reference_genome_type = oneof(transformed((str, lambda x: hl.get_reference(x))), rg_type)
11
12
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/context.py in get_reference(name)
384 :class:`.ReferenceGenome`
385 """
--> 386 Env.hc()
387 if name == 'default':
388 return default_reference()
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/utils/java.py in hc()
65 import sys
66 sys.stderr.write("Initializing Spark and Hail with default parameters...\n")
---> 67 init()
68 assert Env._hc is not None
69 return Env._hc
<decorator-gen-1242> in init(sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, default_reference, idempotent, global_seed, _optimizer_iterations, _backend)
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/typecheck/check.py in wrapper(__original_func, *args, **kwargs)
583 def wrapper(__original_func, *args, **kwargs):
584 args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 585 return __original_func(*args_, **kwargs_)
586
587 return wrapper
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/context.py in init(sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, default_reference, idempotent, global_seed, _optimizer_iterations, _backend)
277 min_block_size, branching_factor, tmp_dir,
278 default_reference, idempotent, global_seed,
--> 279 _optimizer_iterations,_backend)
280
281
<decorator-gen-1240> in __init__(self, sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, default_reference, idempotent, global_seed, optimizer_iterations, _backend)
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/typecheck/check.py in wrapper(__original_func, *args, **kwargs)
583 def wrapper(__original_func, *args, **kwargs):
584 args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
--> 585 return __original_func(*args_, **kwargs_)
586
587 return wrapper
/net/software/local/python/3.6.5/lib/python3.6/site-packages/hail/context.py in __init__(self, sc, app_name, master, local, log, quiet, append, min_block_size, branching_factor, tmp_dir, default_reference, idempotent, global_seed, optimizer_iterations, _backend)
58 conf.set('spark.executor.extraClassPath', './hail-all-spark.jar')
59 if sc is None:
---> 60 SparkContext._ensure_initialized(conf=conf)
61 else:
62 import warnings
/net/software/local/python/3.6.5/lib/python3.6/site-packages/pyspark/context.py in _ensure_initialized(cls, instance, gateway, conf)
314 with SparkContext._lock:
315 if not SparkContext._gateway:
--> 316 SparkContext._gateway = gateway or launch_gateway(conf)
317 SparkContext._jvm = SparkContext._gateway.jvm
318
/net/software/local/python/3.6.5/lib/python3.6/site-packages/pyspark/java_gateway.py in launch_gateway(conf)
44 :return: a JVM gateway
45 """
---> 46 return _launch_gateway(conf)
47
48
/net/software/local/python/3.6.5/lib/python3.6/site-packages/pyspark/java_gateway.py in _launch_gateway(conf, insecure)
106
107 if not os.path.isfile(conn_info_file):
--> 108 raise Exception("Java gateway process exited before sending its port number")
109
110 with open(conn_info_file, "rb") as info:
Exception: Java gateway process exited before sending its port number
Could you point me to the problem here? I have some assistance for the cluster, but I don’t really understand what am I missing here for hail to run. Thank you so much!