Hive on Spark

Konstantin Alpashkin
Contents

Hive on Spark provides Hive with the ability to utilize Apache Spark as its execution engine. By default, Hive is configured to work via Tez.

Configuration

The easiest way to turn on the Spark execution engine is to use ADCM.

Use the Hive Configuration page to change the hive.execution.engine parameter value. Also, you can change this parameter in the hive-site.xml file located in the default Hive configuration directory /etc/hive/conf/. The parameter hive.execution.engine is shown below.

<property>
    <name>hive.execution.engine</name>
    <value>spark</value>
</property>

See the Spark on Hive page to read more about Spark configuration.

There is a Hive configuration file in the home Spark directory (/etc/spark/conf/hive-site.xml). You can change or add any parameter to this file if you want your Spark application to work with Hive in any custom way.

Default configuration file
<?xml version="1.0"?>
<configuration>
        <property>
                <name>hive.cbo.enable</name>
                <value>True</value>
        </property>
        <property>
                <name>hive.compute.query.using.stats</name>
                <value>False</value>
        </property>
        <property>
                <name>hive.execution.engine</name>
                <value>spark</value>
        </property>
        <property>
                <name>hive.log.explain.output</name>
                <value>True</value>
        </property>
        <property>
                <name>hive.metastore.event.db.notification.api.auth</name>
                <value>False</value>
        </property>
        <property>
                <name>hive.metastore.kerberos.keytab.file</name>
                <value>/etc/security/keytabs/hive.service.keytab</value>
        </property>
        <property>
                <name>hive.metastore.kerberos.principal</name>
                <value>hive/_HOST@EXAMPLE.COM</value>
        </property>
        <property>
                <name>hive.metastore.sasl.enabled</name>
                <value>False</value>
        </property>
        <property>
                <name>hive.metastore.uris</name>
                <value>thrift://aai-adh.ru-central1.internal:9083</value>
        </property>
        <property>
               	<name>hive.metastore.warehouse.dir</name>
                <value>/apps/hive/warehouse</value>
        </property>
        <property>
               	<name>hive.server2.authentication</name>
                <value>NONE</value>
        </property>
        <property>
               	<name>hive.server2.authentication.kerberos.keytab</name>
                <value>/etc/security/keytabs/hive.service.keytab</value>
        </property>
        <property>
               	<name>hive.server2.authentication.kerberos.principal</name>
               	<value>hive/_HOST@EXAMPLE.COM</value>
        </property>
        <property>
               	<name>hive.server2.authentication.spnego.keytab</name>
               	<value>/etc/security/keytabs/HTTP.service.keytab</value>
        </property>
        <property>
               	<name>hive.server2.authentication.spnego.principal</name>
               	<value>HTTP/_HOST@EXAMPLE.COM</value>
        </property>
        <property>
                <name>hive.server2.enable.doAs</name>
                <value>False</value>
        </property>
        <property>
                <name>hive.stats.fetch.column.stats</name>
                <value>False</value>
        </property>
        <property>
                <name>hive.support.concurrency</name>
                <value>False</value>
        </property>
        <property>
                <name>hive.tez.container.size</name>
                <value>1024</value>
        </property>
        <property>
                <name>hive.txn.manager</name>
                <value>org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager</value>
        </property>
        <property>
                <name>javax.jdo.option.ConnectionDriverName</name>
                <value>com.mysql.jdbc.Driver</value>
        </property>
        <property>
                <name>javax.jdo.option.ConnectionPassword</name>
                <value>admin</value>
        </property>
        <property>
                <name>javax.jdo.option.ConnectionURL</name>
                <value>jdbc:mysql://aai-adh3.ru-central1.internal:3306/hive</value>
        </property>
        <property>
                <name>javax.jdo.option.ConnectionUserName</name>
                <value>APP</value>
        </property>
</configuration>
Found a mistake? Seleсt text and press Ctrl+Enter to report it