Hail 0.2.27 not running on Cloudera spark 2.4.0

Hail Query & hailctl
1

Tried running hail 0.2.27 (compiled using gcc/7.2.0,python 3.2.6 and spark 2.4.0 cloudera1)

Following is the error

[root@hpcsrcn04 hail]# pyspark --jars $HAIL_HOME/hail-all-spark.jar --conf spark.driver.extraClassPath=$HAIL_HOME/hail-all-spark.jar --conf spark.executor.extraClassPath=./hail-all-spark.jar --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf spark.kryo.registrator=is.hail.kryo.HailKryoRegistrator
WARNING: User-defined SPARK_HOME (/opt/cloudera/parcels/SPARK2-2.4.0.cloudera1-1.cdh5.13.3.p0.1007356/lib/spark2) overrides detected (/opt/cloudera/parcels/SPARK2/lib/spark2).
WARNING: Running pyspark from user-defined location.
Python 3.6.2 (default, Aug 3 2017, 09:09:39)
[GCC 4.9.1] on linux
Type “help”, “copyright”, “credits” or “license” for more information.
Setting default log level to “WARN”.
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Welcome to
____ __
/ / ___ / /
\ / _ / _ `/ __/ '/
/ / ._/_,// //_\ version 2.4.0.cloudera1
//

Using Python version 3.6.2 (default, Aug 3 2017 09:09:39)
SparkSession available as ‘spark’.

import hail as hl
Traceback (most recent call last):
File “”, line 1, in
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/init.py”, line 28, in
from .context import init, stop, spark_context, default_reference,
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/context.py”, line 10, in
from hail.backend import Backend, ServiceBackend, SparkBackend
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/backend/init.py”, line 1, in
from .backend import *
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/backend/backend.py”, line 11, in
from hail.table import Table
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/table.py”, line 2, in
import pandas
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/init.py”, line 55, in
from pandas.core.api import (
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/api.py”, line 24, in
from pandas.core.groupby import Grouper, NamedAgg
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/groupby/init.py”, line 1, in
from pandas.core.groupby.generic import ( # noqa: F401
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/groupby/generic.py”, line 44, in
from pandas.core.frame import DataFrame
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/frame.py”, line 88, in
from pandas.core.generic import NDFrame, _shared_docs
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/generic.py”, line 70, in
from pandas.io.formats.format import DataFrameFormatter, format_percentiles
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/io/formats/format.py”, line 48, in
from pandas.io.common import _expand_user, _stringify_path
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/io/common.py”, line 3, in
import bz2
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/bz2.py”, line 23, in
from bz2 import BZ2Compressor, BZ2Decompressor
ImportError: cannot import name ‘BZ2Compressor’

2

try just python -c "import pandas"?

3

[root@hpcsrcn04 hail]# python -c “import pandas”
Traceback (most recent call last):
File “”, line 1, in
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/init.py”, line 55, in
from pandas.core.api import (
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/api.py”, line 24, in
from pandas.core.groupby import Grouper, NamedAgg
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/groupby/init.py”, line 1, in
from pandas.core.groupby.generic import ( # noqa: F401
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/groupby/generic.py”, line 44, in
from pandas.core.frame import DataFrame
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/frame.py”, line 88, in
from pandas.core.generic import NDFrame, _shared_docs
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/generic.py”, line 70, in
from pandas.io.formats.format import DataFrameFormatter, format_percentiles
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/io/formats/format.py”, line 48, in
from pandas.io.common import _expand_user, _stringify_path
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/io/common.py”, line 3, in
import bz2
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/bz2.py”, line 23, in
from bz2 import BZ2Compressor, BZ2Decompressor
ImportError: cannot import name ‘BZ2Compressor’

4

seems like you have a broken pandas installation, try reinstalling?

5

Ok Trying it…

6

I reinstalled pandas again but getting the same error

[root@hpcsrcn04 ~]# pyspark --jars $HAIL_HOME/hail-all-spark.jar --conf spark.driver.extraClassPath=$HAIL_HOME/hail-all-spark.jar --conf spark.executor.extraClassPath=./hail-all-spark.jar --conf spark.serializer=org.apache.spark.serializer.KryoSerializer --conf spark.kryo.registrator=is.hail.kryo.HailKryoRegistrator
WARNING: User-defined SPARK_HOME (/opt/cloudera/parcels/SPARK2-2.4.0.cloudera1-1.cdh5.13.3.p0.1007356/lib/spark2) overrides detected (/opt/cloudera/parcels/SPARK2/lib/spark2).

WARNING: Running pyspark from user-defined location.
Python 3.6.2 (default, Aug 3 2017, 09:09:39)
[GCC 4.9.1] on linux
Type “help”, “copyright”, “credits” or “license” for more information.
Setting default log level to “WARN”.
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Welcome to
____ __
/ / ___ / /
\ / _ / _ `/ __/ '/
/ / ._/_,// //_\ version 2.4.0.cloudera1
//

Using Python version 3.6.2 (default, Aug 3 2017 09:09:39)
SparkSession available as ‘spark’.

import hail as hl
Traceback (most recent call last):
File “”, line 1, in
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/init.py”, line 28, in
from .context import init, stop, spark_context, default_reference,
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/context.py”, line 10, in
from hail.backend import Backend, ServiceBackend, SparkBackend
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/backend/init.py”, line 1, in
from .backend import *
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/backend/backend.py”, line 11, in
from hail.table import Table
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/table.py”, line 2, in
import pandas
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/init.py”, line 55, in
from pandas.core.api import (
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/api.py”, line 24, in
from pandas.core.groupby import Grouper, NamedAgg
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/groupby/init.py”, line 1, in
from pandas.core.groupby.generic import ( # noqa: F401
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/groupby/generic.py”, line 44, in
from pandas.core.frame import DataFrame
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/frame.py”, line 88, in
from pandas.core.generic import NDFrame, _shared_docs
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/generic.py”, line 70, in
from pandas.io.formats.format import DataFrameFormatter, format_percentiles
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/io/formats/format.py”, line 48, in
from pandas.io.common import _expand_user, _stringify_path
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/io/common.py”, line 3, in
import bz2
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/bz2.py”, line 23, in
from bz2 import BZ2Compressor, BZ2Decompressor
ImportError: cannot import name ‘BZ2Compressor’

import hail as hl

Traceback (most recent call last):
File “”, line 1, in
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/init.py”, line 28, in
from .context import init, stop, spark_context, default_reference,
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/context.py”, line 10, in
from hail.backend import Backend, ServiceBackend, SparkBackend
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/backend/init.py”, line 1, in
from .backend import *
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/backend/backend.py”, line 11, in
from hail.table import Table
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/hail/table.py”, line 2, in
import pandas
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/init.py”, line 55, in
from pandas.core.api import (
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/api.py”, line 24, in
from pandas.core.groupby import Grouper, NamedAgg
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/groupby/init.py”, line 1, in
from pandas.core.groupby.generic import ( # noqa: F401
File “/gpfs/software/tools/python-3.6.2/lib/python3.6/site-packages/pandas/core/groupby/generic.py”, line 41, in
import pandas.core.algorithms as algorithms
AttributeError: module ‘pandas’ has no attribute ‘core’

7

if you can’t run python -c "import pandas" then this isn’t a Hail problem. Sounds like you’re in Python dependency hell and might want a fresh Python installation.

8

OK will try with a fresh install

9

Hello @tpoterba it worked like a charm on python 3.6.6 and also needed pyspark==2.4 version to run