Is there a way to run hail with docker image on a spark cluster? I’m using compute Canada but the docker file that I used does not enable multiprocessing features.
[zhupy@gra797 zhupy]$ singularity shell -B /home -B /project -B /scratch -B /localscratch hail.sif
Singularity> python
Python 3.6.9 (default, Nov 23 2019, 07:02:27)
[GCC 6.3.0 20170516] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import hail as hl
>>> mt = hl.import_plink("/scratch/zhupy/ukb_imp_chr1_v3_pruned_trimmed.bed",
... "/scratch/zhupy/ukb_imp_chr1_v3_pruned_trimmed.bim",
... "/scratch/zhupy/ukb_imp_chr1_v3_pruned_trimmed.fam")
Initializing Hail with default parameters...
Traceback (most recent call last):
File "<stdin>", line 3, in <module>
File "<decorator-gen-1417>", line 2, in import_plink
File "/usr/local/lib/python3.6/site-packages/hail/typecheck/check.py", line 613, in wrapper
args_, kwargs_ = check_all(__original_func, args, kwargs, checkers, is_method=is_method)
File "/usr/local/lib/python3.6/site-packages/hail/typecheck/check.py", line 541, in check_all
args_.append(checker.check(arg, name, arg_name))
File "/usr/local/lib/python3.6/site-packages/hail/typecheck/check.py", line 82, in check
return tc.check(x, caller, param)
File "/usr/local/lib/python3.6/site-packages/hail/typecheck/check.py", line 328, in check
return f(tc.check(x, caller, param))
File "/usr/local/lib/python3.6/site-packages/hail/genetics/reference_genome.py", line 10, in <lambda>
reference_genome_type = oneof(transformed((str, lambda x: hl.get_reference(x))), rg_type)
File "/usr/local/lib/python3.6/site-packages/hail/context.py", line 416, in get_reference
Env.hc()
File "/usr/local/lib/python3.6/site-packages/hail/utils/java.py", line 55, in hc
init()
File "<decorator-gen-1759>", line 2, in init
File "/usr/local/lib/python3.6/site-packages/hail/typecheck/check.py", line 614, in wrapper
return __original_func(*args_, **kwargs_)
File "/usr/local/lib/python3.6/site-packages/hail/context.py", line 231, in init
skip_logging_configuration, optimizer_iterations)
File "/usr/local/lib/python3.6/site-packages/hail/backend/spark_backend.py", line 143, in __init__
pyspark.SparkContext._ensure_initialized(conf=conf)
File "/usr/local/lib/python3.6/site-packages/pyspark/context.py", line 316, in _ensure_initialized
SparkContext._gateway = gateway or launch_gateway(conf)
File "/usr/local/lib/python3.6/site-packages/pyspark/java_gateway.py", line 46, in launch_gateway
return _launch_gateway(conf)
File "/usr/local/lib/python3.6/site-packages/pyspark/java_gateway.py", line 98, in _launch_gateway
proc = Popen(command, stdin=PIPE, preexec_fn=preexec_func, env=env)
File "/usr/local/lib/python3.6/subprocess.py", line 729, in __init__
restore_signals, start_new_session)
File "/usr/local/lib/python3.6/subprocess.py", line 1364, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: '/cvmfs/soft.computecanada.ca/easybuild/software/2017/Core/spark/2.4.4/./bin/spark-submit': '/cvmfs/soft.computecanada.ca/easybuild/software/2017/Core/spark/2.4.4/./bin/spark-submit'
On compute Canada, only spark 2.4.4 is installed but the build is for 2.4.5.