diff --git a/docker/playground/.env b/docker/playground/.env new file mode 100644 index 000000000..94a207c21 --- /dev/null +++ b/docker/playground/.env @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +AWS_JAVA_SDK_VERSION=1.12.239 +HADOOP_VERSION=3.3.1 +HIVE_VERSION=2.3.9 +ICEBERG_VERSION=0.14.1 +KYUUBI_VERSION=1.6.0-incubating +KYUUBI_HADOOP_VERSION=3.3.4 +POSTGRES_VERSION=12 +POSTGRES_JDBC_VERSION=42.3.4 +SCALA_BINARY_VERSION=2.12 +SPARK_VERSION=3.3.0 +SPARK_BINARY_VERSION=3.3 +SPARK_HADOOP_VERSION=3.3.2 +ZOOKEEPER_VERSION=3.6.3 diff --git a/docker/playground/README.md b/docker/playground/README.md new file mode 100644 index 000000000..818e8c864 --- /dev/null +++ b/docker/playground/README.md @@ -0,0 +1,41 @@ +Playground +=== + +## For Users + +### Setup + +1. Install [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/); +2. Go to `docker/playground`, and use `docker compose up` to start compose services in the foreground, or use `docker compose up -d` to run compose services as daemon; + +### Play + +1. Connect using `beeline` + +`docker exec -it kyuubi /opt/kyuubi/bin/beeline -u 'jdbc:hive2://0.0.0.0:10009/'`; + +2. Connect using DBeaver + +Add a Kyuubi datasource with + +- connection url `jdbc:hive2://0.0.0.0:10009/` +- username: `anonymous` +- password: `` + +### Access Service + +- MinIO: http://localhost:9001 +- PostgreSQL localhost:5432 (username: postgres, password: postgres) +- Spark UI: http://localhost:4040 (available after Spark application launching by Kyuubi, port may be 4041, 4042... if you launch more than one Spark applications) + +### Shutdown + +1. Stop compose services by pressing `CTRL+C` if they are running on the foreground, or by `docker compose down` if they are running as daemon; +2. Remove the stopped containers `docker compose rm`; + +## For Maintainers + +### Build + +1. Build images `docker/playground/build-image.sh`; +2. Optional to use `buildx` to build and publish cross-platform images `BUILDX=1 docker/playground/build-image.sh`; diff --git a/docker/playground/build-image.sh b/docker/playground/build-image.sh new file mode 100755 index 000000000..24ab8f840 --- /dev/null +++ b/docker/playground/build-image.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set Mirror: +# APACHE_MIRROR=mirrors.cloud.tencent.com/apache +# MAVEN_MIRROR=mirrors.cloud.tencent.com/maven + +set -e + + +APACHE_MIRROR=${APACHE_MIRROR:-https://dlcdn.apache.org} +MAVEN_MIRROR=${MAVEN_MIRROR:-https://repo1.maven.org/maven2} +BUILD_CMD="docker build" + +if [ $BUILDX ]; then + echo "Using buildx to build cross-platform images" + BUILD_CMD="docker buildx build --platform=linux/amd64,linux/arm64 --push" +fi + +SELF_DIR="$(cd "$(dirname "$0")"; pwd)" + +source "${SELF_DIR}/.env" + +${BUILD_CMD} \ + --build-arg APACHE_MIRROR=${APACHE_MIRROR} \ + --build-arg MAVEN_MIRROR=${MAVEN_MIRROR} \ + --build-arg KYUUBI_VERSION=${KYUUBI_VERSION} \ + --file "${SELF_DIR}/image/kyuubi-playground-base.Dockerfile" \ + --tag nekyuubi/kyuubi-playground-base:${KYUUBI_VERSION} \ + "${SELF_DIR}/image" $@ + +${BUILD_CMD} \ + --build-arg APACHE_MIRROR=${APACHE_MIRROR} \ + --build-arg MAVEN_MIRROR=${MAVEN_MIRROR} \ + --build-arg KYUUBI_VERSION=${KYUUBI_VERSION} \ + --build-arg AWS_JAVA_SDK_VERSION=${AWS_JAVA_SDK_VERSION} \ + --build-arg HADOOP_VERSION=${HADOOP_VERSION} \ + --file "${SELF_DIR}/image/kyuubi-playground-hadoop.Dockerfile" \ + --tag nekyuubi/kyuubi-playground-hadoop:${KYUUBI_VERSION} \ + "${SELF_DIR}/image" $@ + +${BUILD_CMD} \ + --build-arg APACHE_MIRROR=${APACHE_MIRROR} \ + --build-arg MAVEN_MIRROR=${MAVEN_MIRROR} \ + --build-arg KYUUBI_VERSION=${KYUUBI_VERSION} \ + --build-arg HIVE_VERSION=${HIVE_VERSION} \ + --file "${SELF_DIR}/image/kyuubi-playground-metastore.Dockerfile" \ + --tag nekyuubi/kyuubi-playground-metastore:${KYUUBI_VERSION} \ + "${SELF_DIR}/image" $@ + +${BUILD_CMD} \ + --build-arg APACHE_MIRROR=${APACHE_MIRROR} \ + --build-arg MAVEN_MIRROR=${MAVEN_MIRROR} \ + --build-arg KYUUBI_VERSION=${KYUUBI_VERSION} \ + --build-arg AWS_JAVA_SDK_VERSION=${AWS_JAVA_SDK_VERSION} \ + --build-arg CLICKHOUSE_JDBC_VERSION=${CLICKHOUSE_JDBC_VERSION} \ + --build-arg SPARK_HADOOP_VERSION=${SPARK_HADOOP_VERSION} \ + --build-arg ICEBERG_VERSION=${ICEBERG_VERSION} \ + --build-arg POSTGRES_JDBC_VERSION=${POSTGRES_JDBC_VERSION} \ + --build-arg SCALA_BINARY_VERSION=${SCALA_BINARY_VERSION} \ + --build-arg SPARK_VERSION=${SPARK_VERSION} \ + --build-arg SPARK_BINARY_VERSION=${SPARK_BINARY_VERSION} \ + --file "${SELF_DIR}/image/kyuubi-playground-spark.Dockerfile" \ + --tag nekyuubi/kyuubi-playground-spark:${KYUUBI_VERSION} \ + "${SELF_DIR}/image" $@ + +${BUILD_CMD} \ + --build-arg APACHE_MIRROR=${APACHE_MIRROR} \ + --build-arg MAVEN_MIRROR=${MAVEN_MIRROR} \ + --build-arg KYUUBI_VERSION=${KYUUBI_VERSION} \ + --build-arg AWS_JAVA_SDK_VERSION=${AWS_JAVA_SDK_VERSION} \ + --build-arg KYUUBI_HADOOP_VERSION=${KYUUBI_HADOOP_VERSION} \ + --file "${SELF_DIR}/image/kyuubi-playground-kyuubi.Dockerfile" \ + --tag nekyuubi/kyuubi-playground-kyuubi:${KYUUBI_VERSION} \ + "${SELF_DIR}/image" $@ diff --git a/docker/playground/compose.yml b/docker/playground/compose.yml new file mode 100644 index 000000000..cb965ff7f --- /dev/null +++ b/docker/playground/compose.yml @@ -0,0 +1,79 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +services: + minio: + image: alekcander/bitnami-minio-multiarch:RELEASE.2022-05-26T05-48-41Z + environment: + MINIO_ROOT_USER: minio + MINIO_ROOT_PASSWORD: minio_minio + MINIO_DEFAULT_BUCKETS: spark-bucket,iceberg-bucket + container_name: minio + hostname: minio + ports: + - 9000 + - 9001:9001 + + postgres: + image: postgres:${POSTGRES_VERSION} + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: metastore + container_name: postgres + hostname: postgres + ports: + - 5432:5432 + volumes: + - ./image/hive-schema-2.3.0.postgres.sql:/docker-entrypoint-initdb.d/hive-schema-2.3.0.postgres.sql + - ./image/hive-txn-schema-2.3.0.postgres.sql:/docker-entrypoint-initdb.d/hive-txn-schema-2.3.0.postgres.sql + + zookeeper: + image: zookeeper:${ZOOKEEPER_VERSION} + ports: + - 2181 + container_name: zookeeper + hostname: zookeeper + + metastore: + image: nekyuubi/kyuubi-playground-metastore:${KYUUBI_VERSION} + container_name: metastore + hostname: metastore + ports: + - 9083 + volumes: + - ./image/core-site.xml:/etc/hadoop/conf/core-site.xml + - ./image/hive-site.xml:/etc/hive/conf/hive-site.xml + depends_on: + - postgres + + kyuubi: + image: nekyuubi/kyuubi-playground-kyuubi:${KYUUBI_VERSION} + container_name: kyuubi + hostname: kyuubi + ports: + - 4040-4050:4040-4050 + - 10009:10009 + volumes: + - ./image/core-site.xml:/etc/hadoop/conf/core-site.xml + - ./image/hive-site.xml:/etc/hive/conf/hive-site.xml + - ./image/spark-defaults.conf:/etc/spark/conf/spark-defaults.conf + - ./image/kyuubi-defaults.conf:/etc/kyuubi/conf/kyuubi-defaults.conf + depends_on: + - metastore + - minio + - zookeeper diff --git a/docker/playground/image/core-site.xml b/docker/playground/image/core-site.xml new file mode 100644 index 000000000..017749386 --- /dev/null +++ b/docker/playground/image/core-site.xml @@ -0,0 +1,48 @@ + + + + + + fs.s3a.access.key + minio + + + fs.s3a.secret.key + minio_minio + + + fs.s3a.connection.ssl.enabled + false + + + fs.s3a.path.style.access + true + + + fs.s3a.endpoint + http://minio:9000 + + + fs.s3a.committer.name + magic + + + fs.defaultFS + s3a://spark-bucket/ + + diff --git a/docker/playground/image/hive-schema-2.3.0.postgres.sql b/docker/playground/image/hive-schema-2.3.0.postgres.sql new file mode 100644 index 000000000..3a0c76888 --- /dev/null +++ b/docker/playground/image/hive-schema-2.3.0.postgres.sql @@ -0,0 +1,1492 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- PostgreSQL database dump +-- + +SET statement_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = off; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET escape_string_warning = off; + +SET search_path = public, pg_catalog; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: BUCKETING_COLS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "BUCKETING_COLS" ( + "SD_ID" bigint NOT NULL, + "BUCKET_COL_NAME" character varying(256) DEFAULT NULL::character varying, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: CDS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "CDS" ( + "CD_ID" bigint NOT NULL +); + + +-- +-- Name: COLUMNS_V2; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "COLUMNS_V2" ( + "CD_ID" bigint NOT NULL, + "COMMENT" character varying(4000), + "COLUMN_NAME" character varying(767) NOT NULL, + "TYPE_NAME" text, + "INTEGER_IDX" integer NOT NULL +); + + +-- +-- Name: DATABASE_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "DATABASE_PARAMS" ( + "DB_ID" bigint NOT NULL, + "PARAM_KEY" character varying(180) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: DBS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "DBS" ( + "DB_ID" bigint NOT NULL, + "DESC" character varying(4000) DEFAULT NULL::character varying, + "DB_LOCATION_URI" character varying(4000) NOT NULL, + "NAME" character varying(128) DEFAULT NULL::character varying, + "OWNER_NAME" character varying(128) DEFAULT NULL::character varying, + "OWNER_TYPE" character varying(10) DEFAULT NULL::character varying +); + + +-- +-- Name: DB_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "DB_PRIVS" ( + "DB_GRANT_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "DB_ID" bigint, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "DB_PRIV" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: GLOBAL_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "GLOBAL_PRIVS" ( + "USER_GRANT_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "USER_PRIV" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: IDXS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "IDXS" ( + "INDEX_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "DEFERRED_REBUILD" boolean NOT NULL, + "INDEX_HANDLER_CLASS" character varying(4000) DEFAULT NULL::character varying, + "INDEX_NAME" character varying(128) DEFAULT NULL::character varying, + "INDEX_TBL_ID" bigint, + "LAST_ACCESS_TIME" bigint NOT NULL, + "ORIG_TBL_ID" bigint, + "SD_ID" bigint +); + + +-- +-- Name: INDEX_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "INDEX_PARAMS" ( + "INDEX_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: NUCLEUS_TABLES; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "NUCLEUS_TABLES" ( + "CLASS_NAME" character varying(128) NOT NULL, + "TABLE_NAME" character varying(128) NOT NULL, + "TYPE" character varying(4) NOT NULL, + "OWNER" character varying(2) NOT NULL, + "VERSION" character varying(20) NOT NULL, + "INTERFACE_NAME" character varying(255) DEFAULT NULL::character varying +); + + +-- +-- Name: PARTITIONS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITIONS" ( + "PART_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "LAST_ACCESS_TIME" bigint NOT NULL, + "PART_NAME" character varying(767) DEFAULT NULL::character varying, + "SD_ID" bigint, + "TBL_ID" bigint +); + + +-- +-- Name: PARTITION_EVENTS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITION_EVENTS" ( + "PART_NAME_ID" bigint NOT NULL, + "DB_NAME" character varying(128), + "EVENT_TIME" bigint NOT NULL, + "EVENT_TYPE" integer NOT NULL, + "PARTITION_NAME" character varying(767), + "TBL_NAME" character varying(256) +); + + +-- +-- Name: PARTITION_KEYS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITION_KEYS" ( + "TBL_ID" bigint NOT NULL, + "PKEY_COMMENT" character varying(4000) DEFAULT NULL::character varying, + "PKEY_NAME" character varying(128) NOT NULL, + "PKEY_TYPE" character varying(767) NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: PARTITION_KEY_VALS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITION_KEY_VALS" ( + "PART_ID" bigint NOT NULL, + "PART_KEY_VAL" character varying(256) DEFAULT NULL::character varying, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: PARTITION_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PARTITION_PARAMS" ( + "PART_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: PART_COL_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PART_COL_PRIVS" ( + "PART_COLUMN_GRANT_ID" bigint NOT NULL, + "COLUMN_NAME" character varying(767) DEFAULT NULL::character varying, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_ID" bigint, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_COL_PRIV" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: PART_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PART_PRIVS" ( + "PART_GRANT_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_ID" bigint, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_PRIV" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: ROLES; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "ROLES" ( + "ROLE_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "OWNER_NAME" character varying(128) DEFAULT NULL::character varying, + "ROLE_NAME" character varying(128) DEFAULT NULL::character varying +); + + +-- +-- Name: ROLE_MAP; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "ROLE_MAP" ( + "ROLE_GRANT_ID" bigint NOT NULL, + "ADD_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "ROLE_ID" bigint +); + + +-- +-- Name: SDS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SDS" ( + "SD_ID" bigint NOT NULL, + "INPUT_FORMAT" character varying(4000) DEFAULT NULL::character varying, + "IS_COMPRESSED" boolean NOT NULL, + "LOCATION" character varying(4000) DEFAULT NULL::character varying, + "NUM_BUCKETS" bigint NOT NULL, + "OUTPUT_FORMAT" character varying(4000) DEFAULT NULL::character varying, + "SERDE_ID" bigint, + "CD_ID" bigint, + "IS_STOREDASSUBDIRECTORIES" boolean NOT NULL +); + + +-- +-- Name: SD_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SD_PARAMS" ( + "SD_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" text DEFAULT NULL +); + + +-- +-- Name: SEQUENCE_TABLE; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SEQUENCE_TABLE" ( + "SEQUENCE_NAME" character varying(255) NOT NULL, + "NEXT_VAL" bigint NOT NULL +); + + +-- +-- Name: SERDES; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SERDES" ( + "SERDE_ID" bigint NOT NULL, + "NAME" character varying(128) DEFAULT NULL::character varying, + "SLIB" character varying(4000) DEFAULT NULL::character varying +); + + +-- +-- Name: SERDE_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SERDE_PARAMS" ( + "SERDE_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" text DEFAULT NULL +); + + +-- +-- Name: SORT_COLS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "SORT_COLS" ( + "SD_ID" bigint NOT NULL, + "COLUMN_NAME" character varying(767) DEFAULT NULL::character varying, + "ORDER" bigint NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: TABLE_PARAMS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TABLE_PARAMS" ( + "TBL_ID" bigint NOT NULL, + "PARAM_KEY" character varying(256) NOT NULL, + "PARAM_VALUE" text DEFAULT NULL +); + + +-- +-- Name: TBLS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TBLS" ( + "TBL_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "DB_ID" bigint, + "LAST_ACCESS_TIME" bigint NOT NULL, + "OWNER" character varying(767) DEFAULT NULL::character varying, + "RETENTION" bigint NOT NULL, + "SD_ID" bigint, + "TBL_NAME" character varying(256) DEFAULT NULL::character varying, + "TBL_TYPE" character varying(128) DEFAULT NULL::character varying, + "VIEW_EXPANDED_TEXT" text, + "VIEW_ORIGINAL_TEXT" text, + "IS_REWRITE_ENABLED" boolean NOT NULL +); + + +-- +-- Name: TBL_COL_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TBL_COL_PRIVS" ( + "TBL_COLUMN_GRANT_ID" bigint NOT NULL, + "COLUMN_NAME" character varying(767) DEFAULT NULL::character varying, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "TBL_COL_PRIV" character varying(128) DEFAULT NULL::character varying, + "TBL_ID" bigint +); + + +-- +-- Name: TBL_PRIVS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TBL_PRIVS" ( + "TBL_GRANT_ID" bigint NOT NULL, + "CREATE_TIME" bigint NOT NULL, + "GRANT_OPTION" smallint NOT NULL, + "GRANTOR" character varying(128) DEFAULT NULL::character varying, + "GRANTOR_TYPE" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_NAME" character varying(128) DEFAULT NULL::character varying, + "PRINCIPAL_TYPE" character varying(128) DEFAULT NULL::character varying, + "TBL_PRIV" character varying(128) DEFAULT NULL::character varying, + "TBL_ID" bigint +); + + +-- +-- Name: TYPES; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TYPES" ( + "TYPES_ID" bigint NOT NULL, + "TYPE_NAME" character varying(128) DEFAULT NULL::character varying, + "TYPE1" character varying(767) DEFAULT NULL::character varying, + "TYPE2" character varying(767) DEFAULT NULL::character varying +); + + +-- +-- Name: TYPE_FIELDS; Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "TYPE_FIELDS" ( + "TYPE_NAME" bigint NOT NULL, + "COMMENT" character varying(256) DEFAULT NULL::character varying, + "FIELD_NAME" character varying(128) NOT NULL, + "FIELD_TYPE" character varying(767) NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + +CREATE TABLE "SKEWED_STRING_LIST" ( + "STRING_LIST_ID" bigint NOT NULL +); + +CREATE TABLE "SKEWED_STRING_LIST_VALUES" ( + "STRING_LIST_ID" bigint NOT NULL, + "STRING_LIST_VALUE" character varying(256) DEFAULT NULL::character varying, + "INTEGER_IDX" bigint NOT NULL +); + +CREATE TABLE "SKEWED_COL_NAMES" ( + "SD_ID" bigint NOT NULL, + "SKEWED_COL_NAME" character varying(256) DEFAULT NULL::character varying, + "INTEGER_IDX" bigint NOT NULL +); + +CREATE TABLE "SKEWED_COL_VALUE_LOC_MAP" ( + "SD_ID" bigint NOT NULL, + "STRING_LIST_ID_KID" bigint NOT NULL, + "LOCATION" character varying(4000) DEFAULT NULL::character varying +); + +CREATE TABLE "SKEWED_VALUES" ( + "SD_ID_OID" bigint NOT NULL, + "STRING_LIST_ID_EID" bigint NOT NULL, + "INTEGER_IDX" bigint NOT NULL +); + + +-- +-- Name: TAB_COL_STATS Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "MASTER_KEYS" +( + "KEY_ID" SERIAL, + "MASTER_KEY" varchar(767) NULL, + PRIMARY KEY ("KEY_ID") +); + +CREATE TABLE "DELEGATION_TOKENS" +( + "TOKEN_IDENT" varchar(767) NOT NULL, + "TOKEN" varchar(767) NULL, + PRIMARY KEY ("TOKEN_IDENT") +); + +CREATE TABLE "TAB_COL_STATS" ( + "CS_ID" bigint NOT NULL, + "DB_NAME" character varying(128) DEFAULT NULL::character varying, + "TABLE_NAME" character varying(256) DEFAULT NULL::character varying, + "COLUMN_NAME" character varying(767) DEFAULT NULL::character varying, + "COLUMN_TYPE" character varying(128) DEFAULT NULL::character varying, + "TBL_ID" bigint NOT NULL, + "LONG_LOW_VALUE" bigint, + "LONG_HIGH_VALUE" bigint, + "DOUBLE_LOW_VALUE" double precision, + "DOUBLE_HIGH_VALUE" double precision, + "BIG_DECIMAL_LOW_VALUE" character varying(4000) DEFAULT NULL::character varying, + "BIG_DECIMAL_HIGH_VALUE" character varying(4000) DEFAULT NULL::character varying, + "NUM_NULLS" bigint NOT NULL, + "NUM_DISTINCTS" bigint, + "AVG_COL_LEN" double precision, + "MAX_COL_LEN" bigint, + "NUM_TRUES" bigint, + "NUM_FALSES" bigint, + "LAST_ANALYZED" bigint NOT NULL +); + +-- +-- Table structure for VERSION +-- +CREATE TABLE "VERSION" ( + "VER_ID" bigint, + "SCHEMA_VERSION" character varying(127) NOT NULL, + "VERSION_COMMENT" character varying(255) NOT NULL +); + +-- +-- Name: PART_COL_STATS Type: TABLE; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE TABLE "PART_COL_STATS" ( + "CS_ID" bigint NOT NULL, + "DB_NAME" character varying(128) DEFAULT NULL::character varying, + "TABLE_NAME" character varying(256) DEFAULT NULL::character varying, + "PARTITION_NAME" character varying(767) DEFAULT NULL::character varying, + "COLUMN_NAME" character varying(767) DEFAULT NULL::character varying, + "COLUMN_TYPE" character varying(128) DEFAULT NULL::character varying, + "PART_ID" bigint NOT NULL, + "LONG_LOW_VALUE" bigint, + "LONG_HIGH_VALUE" bigint, + "DOUBLE_LOW_VALUE" double precision, + "DOUBLE_HIGH_VALUE" double precision, + "BIG_DECIMAL_LOW_VALUE" character varying(4000) DEFAULT NULL::character varying, + "BIG_DECIMAL_HIGH_VALUE" character varying(4000) DEFAULT NULL::character varying, + "NUM_NULLS" bigint NOT NULL, + "NUM_DISTINCTS" bigint, + "AVG_COL_LEN" double precision, + "MAX_COL_LEN" bigint, + "NUM_TRUES" bigint, + "NUM_FALSES" bigint, + "LAST_ANALYZED" bigint NOT NULL +); + +-- +-- Table structure for FUNCS +-- +CREATE TABLE "FUNCS" ( + "FUNC_ID" BIGINT NOT NULL, + "CLASS_NAME" VARCHAR(4000), + "CREATE_TIME" INTEGER NOT NULL, + "DB_ID" BIGINT, + "FUNC_NAME" VARCHAR(128), + "FUNC_TYPE" INTEGER NOT NULL, + "OWNER_NAME" VARCHAR(128), + "OWNER_TYPE" VARCHAR(10), + PRIMARY KEY ("FUNC_ID") +); + +-- +-- Table structure for FUNC_RU +-- +CREATE TABLE "FUNC_RU" ( + "FUNC_ID" BIGINT NOT NULL, + "RESOURCE_TYPE" INTEGER NOT NULL, + "RESOURCE_URI" VARCHAR(4000), + "INTEGER_IDX" INTEGER NOT NULL, + PRIMARY KEY ("FUNC_ID", "INTEGER_IDX") +); + +CREATE TABLE "NOTIFICATION_LOG" +( + "NL_ID" BIGINT NOT NULL, + "EVENT_ID" BIGINT NOT NULL, + "EVENT_TIME" INTEGER NOT NULL, + "EVENT_TYPE" VARCHAR(32) NOT NULL, + "DB_NAME" VARCHAR(128), + "TBL_NAME" VARCHAR(256), + "MESSAGE" text, + "MESSAGE_FORMAT" VARCHAR(16), + PRIMARY KEY ("NL_ID") +); + +CREATE TABLE "NOTIFICATION_SEQUENCE" +( + "NNI_ID" BIGINT NOT NULL, + "NEXT_EVENT_ID" BIGINT NOT NULL, + PRIMARY KEY ("NNI_ID") +); + +CREATE TABLE "KEY_CONSTRAINTS" +( + "CHILD_CD_ID" BIGINT, + "CHILD_INTEGER_IDX" BIGINT, + "CHILD_TBL_ID" BIGINT, + "PARENT_CD_ID" BIGINT NOT NULL, + "PARENT_INTEGER_IDX" BIGINT NOT NULL, + "PARENT_TBL_ID" BIGINT NOT NULL, + "POSITION" BIGINT NOT NULL, + "CONSTRAINT_NAME" VARCHAR(400) NOT NULL, + "CONSTRAINT_TYPE" SMALLINT NOT NULL, + "UPDATE_RULE" SMALLINT, + "DELETE_RULE" SMALLINT, + "ENABLE_VALIDATE_RELY" SMALLINT NOT NULL, + PRIMARY KEY ("CONSTRAINT_NAME", "POSITION") +) ; + +CREATE INDEX "CONSTRAINTS_PARENT_TBLID_INDEX" ON "KEY_CONSTRAINTS" USING BTREE ("PARENT_TBL_ID"); + +-- +-- Name: BUCKETING_COLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "BUCKETING_COLS" + ADD CONSTRAINT "BUCKETING_COLS_pkey" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + + +-- +-- Name: CDS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "CDS" + ADD CONSTRAINT "CDS_pkey" PRIMARY KEY ("CD_ID"); + + +-- +-- Name: COLUMNS_V2_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "COLUMNS_V2" + ADD CONSTRAINT "COLUMNS_V2_pkey" PRIMARY KEY ("CD_ID", "COLUMN_NAME"); + + +-- +-- Name: DATABASE_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DATABASE_PARAMS" + ADD CONSTRAINT "DATABASE_PARAMS_pkey" PRIMARY KEY ("DB_ID", "PARAM_KEY"); + + +-- +-- Name: DBPRIVILEGEINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DB_PRIVS" + ADD CONSTRAINT "DBPRIVILEGEINDEX" UNIQUE ("DB_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "DB_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: DBS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DBS" + ADD CONSTRAINT "DBS_pkey" PRIMARY KEY ("DB_ID"); + + +-- +-- Name: DB_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DB_PRIVS" + ADD CONSTRAINT "DB_PRIVS_pkey" PRIMARY KEY ("DB_GRANT_ID"); + + +-- +-- Name: GLOBALPRIVILEGEINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "GLOBAL_PRIVS" + ADD CONSTRAINT "GLOBALPRIVILEGEINDEX" UNIQUE ("PRINCIPAL_NAME", "PRINCIPAL_TYPE", "USER_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: GLOBAL_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "GLOBAL_PRIVS" + ADD CONSTRAINT "GLOBAL_PRIVS_pkey" PRIMARY KEY ("USER_GRANT_ID"); + + +-- +-- Name: IDXS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "IDXS_pkey" PRIMARY KEY ("INDEX_ID"); + + +-- +-- Name: INDEX_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "INDEX_PARAMS" + ADD CONSTRAINT "INDEX_PARAMS_pkey" PRIMARY KEY ("INDEX_ID", "PARAM_KEY"); + + +-- +-- Name: NUCLEUS_TABLES_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "NUCLEUS_TABLES" + ADD CONSTRAINT "NUCLEUS_TABLES_pkey" PRIMARY KEY ("CLASS_NAME"); + + +-- +-- Name: PARTITIONS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITIONS" + ADD CONSTRAINT "PARTITIONS_pkey" PRIMARY KEY ("PART_ID"); + + +-- +-- Name: PARTITION_EVENTS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITION_EVENTS" + ADD CONSTRAINT "PARTITION_EVENTS_pkey" PRIMARY KEY ("PART_NAME_ID"); + + +-- +-- Name: PARTITION_KEYS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITION_KEYS" + ADD CONSTRAINT "PARTITION_KEYS_pkey" PRIMARY KEY ("TBL_ID", "PKEY_NAME"); + + +-- +-- Name: PARTITION_KEY_VALS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITION_KEY_VALS" + ADD CONSTRAINT "PARTITION_KEY_VALS_pkey" PRIMARY KEY ("PART_ID", "INTEGER_IDX"); + + +-- +-- Name: PARTITION_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITION_PARAMS" + ADD CONSTRAINT "PARTITION_PARAMS_pkey" PRIMARY KEY ("PART_ID", "PARAM_KEY"); + + +-- +-- Name: PART_COL_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PART_COL_PRIVS" + ADD CONSTRAINT "PART_COL_PRIVS_pkey" PRIMARY KEY ("PART_COLUMN_GRANT_ID"); + + +-- +-- Name: PART_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PART_PRIVS" + ADD CONSTRAINT "PART_PRIVS_pkey" PRIMARY KEY ("PART_GRANT_ID"); + + +-- +-- Name: ROLEENTITYINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "ROLES" + ADD CONSTRAINT "ROLEENTITYINDEX" UNIQUE ("ROLE_NAME"); + + +-- +-- Name: ROLES_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "ROLES" + ADD CONSTRAINT "ROLES_pkey" PRIMARY KEY ("ROLE_ID"); + + +-- +-- Name: ROLE_MAP_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "ROLE_MAP" + ADD CONSTRAINT "ROLE_MAP_pkey" PRIMARY KEY ("ROLE_GRANT_ID"); + + +-- +-- Name: SDS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SDS" + ADD CONSTRAINT "SDS_pkey" PRIMARY KEY ("SD_ID"); + + +-- +-- Name: SD_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SD_PARAMS" + ADD CONSTRAINT "SD_PARAMS_pkey" PRIMARY KEY ("SD_ID", "PARAM_KEY"); + + +-- +-- Name: SEQUENCE_TABLE_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SEQUENCE_TABLE" + ADD CONSTRAINT "SEQUENCE_TABLE_pkey" PRIMARY KEY ("SEQUENCE_NAME"); + + +-- +-- Name: SERDES_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SERDES" + ADD CONSTRAINT "SERDES_pkey" PRIMARY KEY ("SERDE_ID"); + + +-- +-- Name: SERDE_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SERDE_PARAMS" + ADD CONSTRAINT "SERDE_PARAMS_pkey" PRIMARY KEY ("SERDE_ID", "PARAM_KEY"); + + +-- +-- Name: SORT_COLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "SORT_COLS" + ADD CONSTRAINT "SORT_COLS_pkey" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + + +-- +-- Name: TABLE_PARAMS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TABLE_PARAMS" + ADD CONSTRAINT "TABLE_PARAMS_pkey" PRIMARY KEY ("TBL_ID", "PARAM_KEY"); + + +-- +-- Name: TBLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TBLS" + ADD CONSTRAINT "TBLS_pkey" PRIMARY KEY ("TBL_ID"); + + +-- +-- Name: TBL_COL_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TBL_COL_PRIVS" + ADD CONSTRAINT "TBL_COL_PRIVS_pkey" PRIMARY KEY ("TBL_COLUMN_GRANT_ID"); + + +-- +-- Name: TBL_PRIVS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TBL_PRIVS" + ADD CONSTRAINT "TBL_PRIVS_pkey" PRIMARY KEY ("TBL_GRANT_ID"); + + +-- +-- Name: TYPES_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TYPES" + ADD CONSTRAINT "TYPES_pkey" PRIMARY KEY ("TYPES_ID"); + + +-- +-- Name: TYPE_FIELDS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TYPE_FIELDS" + ADD CONSTRAINT "TYPE_FIELDS_pkey" PRIMARY KEY ("TYPE_NAME", "FIELD_NAME"); + +ALTER TABLE ONLY "SKEWED_STRING_LIST" + ADD CONSTRAINT "SKEWED_STRING_LIST_pkey" PRIMARY KEY ("STRING_LIST_ID"); + +ALTER TABLE ONLY "SKEWED_STRING_LIST_VALUES" + ADD CONSTRAINT "SKEWED_STRING_LIST_VALUES_pkey" PRIMARY KEY ("STRING_LIST_ID", "INTEGER_IDX"); + + +ALTER TABLE ONLY "SKEWED_COL_NAMES" + ADD CONSTRAINT "SKEWED_COL_NAMES_pkey" PRIMARY KEY ("SD_ID", "INTEGER_IDX"); + +ALTER TABLE ONLY "SKEWED_COL_VALUE_LOC_MAP" + ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_pkey" PRIMARY KEY ("SD_ID", "STRING_LIST_ID_KID"); + +ALTER TABLE ONLY "SKEWED_VALUES" + ADD CONSTRAINT "SKEWED_VALUES_pkey" PRIMARY KEY ("SD_ID_OID", "INTEGER_IDX"); + +-- +-- Name: TAB_COL_STATS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- +ALTER TABLE ONLY "TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_pkey" PRIMARY KEY("CS_ID"); + +-- +-- Name: PART_COL_STATS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- +ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_pkey" PRIMARY KEY("CS_ID"); + +-- +-- Name: UNIQUEINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "UNIQUEINDEX" UNIQUE ("INDEX_NAME", "ORIG_TBL_ID"); + + +-- +-- Name: UNIQUEPARTITION; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "PARTITIONS" + ADD CONSTRAINT "UNIQUEPARTITION" UNIQUE ("PART_NAME", "TBL_ID"); + + +-- +-- Name: UNIQUETABLE; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TBLS" + ADD CONSTRAINT "UNIQUETABLE" UNIQUE ("TBL_NAME", "DB_ID"); + + +-- +-- Name: UNIQUE_DATABASE; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "DBS" + ADD CONSTRAINT "UNIQUE_DATABASE" UNIQUE ("NAME"); + + +-- +-- Name: UNIQUE_TYPE; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "TYPES" + ADD CONSTRAINT "UNIQUE_TYPE" UNIQUE ("TYPE_NAME"); + + +-- +-- Name: USERROLEMAPINDEX; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: +-- + +ALTER TABLE ONLY "ROLE_MAP" + ADD CONSTRAINT "USERROLEMAPINDEX" UNIQUE ("PRINCIPAL_NAME", "ROLE_ID", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: BUCKETING_COLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "BUCKETING_COLS_N49" ON "BUCKETING_COLS" USING btree ("SD_ID"); + + +-- +-- Name: DATABASE_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "DATABASE_PARAMS_N49" ON "DATABASE_PARAMS" USING btree ("DB_ID"); + + +-- +-- Name: DB_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "DB_PRIVS_N49" ON "DB_PRIVS" USING btree ("DB_ID"); + + +-- +-- Name: IDXS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "IDXS_N49" ON "IDXS" USING btree ("ORIG_TBL_ID"); + + +-- +-- Name: IDXS_N50; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "IDXS_N50" ON "IDXS" USING btree ("INDEX_TBL_ID"); + + +-- +-- Name: IDXS_N51; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "IDXS_N51" ON "IDXS" USING btree ("SD_ID"); + + +-- +-- Name: INDEX_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "INDEX_PARAMS_N49" ON "INDEX_PARAMS" USING btree ("INDEX_ID"); + + +-- +-- Name: PARTITIONCOLUMNPRIVILEGEINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITIONCOLUMNPRIVILEGEINDEX" ON "PART_COL_PRIVS" USING btree ("PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: PARTITIONEVENTINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITIONEVENTINDEX" ON "PARTITION_EVENTS" USING btree ("PARTITION_NAME"); + + +-- +-- Name: PARTITIONS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITIONS_N49" ON "PARTITIONS" USING btree ("TBL_ID"); + + +-- +-- Name: PARTITIONS_N50; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITIONS_N50" ON "PARTITIONS" USING btree ("SD_ID"); + + +-- +-- Name: PARTITION_KEYS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITION_KEYS_N49" ON "PARTITION_KEYS" USING btree ("TBL_ID"); + + +-- +-- Name: PARTITION_KEY_VALS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITION_KEY_VALS_N49" ON "PARTITION_KEY_VALS" USING btree ("PART_ID"); + + +-- +-- Name: PARTITION_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTITION_PARAMS_N49" ON "PARTITION_PARAMS" USING btree ("PART_ID"); + + +-- +-- Name: PARTPRIVILEGEINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PARTPRIVILEGEINDEX" ON "PART_PRIVS" USING btree ("PART_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: PART_COL_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PART_COL_PRIVS_N49" ON "PART_COL_PRIVS" USING btree ("PART_ID"); + + +-- +-- Name: PART_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PART_PRIVS_N49" ON "PART_PRIVS" USING btree ("PART_ID"); + + +-- +-- Name: PCS_STATS_IDX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PCS_STATS_IDX" ON "PART_COL_STATS" USING btree ("DB_NAME","TABLE_NAME","COLUMN_NAME","PARTITION_NAME"); + + +-- +-- Name: ROLE_MAP_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "ROLE_MAP_N49" ON "ROLE_MAP" USING btree ("ROLE_ID"); + + +-- +-- Name: SDS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "SDS_N49" ON "SDS" USING btree ("SERDE_ID"); + + +-- +-- Name: SD_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "SD_PARAMS_N49" ON "SD_PARAMS" USING btree ("SD_ID"); + + +-- +-- Name: SERDE_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "SERDE_PARAMS_N49" ON "SERDE_PARAMS" USING btree ("SERDE_ID"); + + +-- +-- Name: SORT_COLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "SORT_COLS_N49" ON "SORT_COLS" USING btree ("SD_ID"); + + +-- +-- Name: TABLECOLUMNPRIVILEGEINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TABLECOLUMNPRIVILEGEINDEX" ON "TBL_COL_PRIVS" USING btree ("TBL_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: TABLEPRIVILEGEINDEX; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TABLEPRIVILEGEINDEX" ON "TBL_PRIVS" USING btree ("TBL_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + + +-- +-- Name: TABLE_PARAMS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TABLE_PARAMS_N49" ON "TABLE_PARAMS" USING btree ("TBL_ID"); + + +-- +-- Name: TBLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TBLS_N49" ON "TBLS" USING btree ("DB_ID"); + + +-- +-- Name: TBLS_N50; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TBLS_N50" ON "TBLS" USING btree ("SD_ID"); + + +-- +-- Name: TBL_COL_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TBL_COL_PRIVS_N49" ON "TBL_COL_PRIVS" USING btree ("TBL_ID"); + + +-- +-- Name: TBL_PRIVS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TBL_PRIVS_N49" ON "TBL_PRIVS" USING btree ("TBL_ID"); + + +-- +-- Name: TYPE_FIELDS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TYPE_FIELDS_N49" ON "TYPE_FIELDS" USING btree ("TYPE_NAME"); + +-- +-- Name: TAB_COL_STATS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "TAB_COL_STATS_N49" ON "TAB_COL_STATS" USING btree ("TBL_ID"); + +-- +-- Name: PART_COL_STATS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "PART_COL_STATS_N49" ON "PART_COL_STATS" USING btree ("PART_ID"); + +-- +-- Name: UNIQUEFUNCTION; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE UNIQUE INDEX "UNIQUEFUNCTION" ON "FUNCS" ("FUNC_NAME", "DB_ID"); + +-- +-- Name: FUNCS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "FUNCS_N49" ON "FUNCS" ("DB_ID"); + +-- +-- Name: FUNC_RU_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: +-- + +CREATE INDEX "FUNC_RU_N49" ON "FUNC_RU" ("FUNC_ID"); + + +ALTER TABLE ONLY "SKEWED_STRING_LIST_VALUES" + ADD CONSTRAINT "SKEWED_STRING_LIST_VALUES_fkey" FOREIGN KEY ("STRING_LIST_ID") REFERENCES "SKEWED_STRING_LIST"("STRING_LIST_ID") DEFERRABLE; + + +ALTER TABLE ONLY "SKEWED_COL_NAMES" + ADD CONSTRAINT "SKEWED_COL_NAMES_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +ALTER TABLE ONLY "SKEWED_COL_VALUE_LOC_MAP" + ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_fkey1" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + +ALTER TABLE ONLY "SKEWED_COL_VALUE_LOC_MAP" + ADD CONSTRAINT "SKEWED_COL_VALUE_LOC_MAP_fkey2" FOREIGN KEY ("STRING_LIST_ID_KID") REFERENCES "SKEWED_STRING_LIST"("STRING_LIST_ID") DEFERRABLE; + +ALTER TABLE ONLY "SKEWED_VALUES" + ADD CONSTRAINT "SKEWED_VALUES_fkey1" FOREIGN KEY ("STRING_LIST_ID_EID") REFERENCES "SKEWED_STRING_LIST"("STRING_LIST_ID") DEFERRABLE; + +ALTER TABLE ONLY "SKEWED_VALUES" + ADD CONSTRAINT "SKEWED_VALUES_fkey2" FOREIGN KEY ("SD_ID_OID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: BUCKETING_COLS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "BUCKETING_COLS" + ADD CONSTRAINT "BUCKETING_COLS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: COLUMNS_V2_CD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "COLUMNS_V2" + ADD CONSTRAINT "COLUMNS_V2_CD_ID_fkey" FOREIGN KEY ("CD_ID") REFERENCES "CDS"("CD_ID") DEFERRABLE; + + +-- +-- Name: DATABASE_PARAMS_DB_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "DATABASE_PARAMS" + ADD CONSTRAINT "DATABASE_PARAMS_DB_ID_fkey" FOREIGN KEY ("DB_ID") REFERENCES "DBS"("DB_ID") DEFERRABLE; + + +-- +-- Name: DB_PRIVS_DB_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "DB_PRIVS" + ADD CONSTRAINT "DB_PRIVS_DB_ID_fkey" FOREIGN KEY ("DB_ID") REFERENCES "DBS"("DB_ID") DEFERRABLE; + + +-- +-- Name: IDXS_INDEX_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "IDXS_INDEX_TBL_ID_fkey" FOREIGN KEY ("INDEX_TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: IDXS_ORIG_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "IDXS_ORIG_TBL_ID_fkey" FOREIGN KEY ("ORIG_TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: IDXS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "IDXS" + ADD CONSTRAINT "IDXS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: INDEX_PARAMS_INDEX_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "INDEX_PARAMS" + ADD CONSTRAINT "INDEX_PARAMS_INDEX_ID_fkey" FOREIGN KEY ("INDEX_ID") REFERENCES "IDXS"("INDEX_ID") DEFERRABLE; + + +-- +-- Name: PARTITIONS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITIONS" + ADD CONSTRAINT "PARTITIONS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: PARTITIONS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITIONS" + ADD CONSTRAINT "PARTITIONS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: PARTITION_KEYS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITION_KEYS" + ADD CONSTRAINT "PARTITION_KEYS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: PARTITION_KEY_VALS_PART_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITION_KEY_VALS" + ADD CONSTRAINT "PARTITION_KEY_VALS_PART_ID_fkey" FOREIGN KEY ("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +-- +-- Name: PARTITION_PARAMS_PART_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PARTITION_PARAMS" + ADD CONSTRAINT "PARTITION_PARAMS_PART_ID_fkey" FOREIGN KEY ("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +-- +-- Name: PART_COL_PRIVS_PART_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PART_COL_PRIVS" + ADD CONSTRAINT "PART_COL_PRIVS_PART_ID_fkey" FOREIGN KEY ("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +-- +-- Name: PART_PRIVS_PART_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "PART_PRIVS" + ADD CONSTRAINT "PART_PRIVS_PART_ID_fkey" FOREIGN KEY ("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +-- +-- Name: ROLE_MAP_ROLE_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "ROLE_MAP" + ADD CONSTRAINT "ROLE_MAP_ROLE_ID_fkey" FOREIGN KEY ("ROLE_ID") REFERENCES "ROLES"("ROLE_ID") DEFERRABLE; + + +-- +-- Name: SDS_CD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SDS" + ADD CONSTRAINT "SDS_CD_ID_fkey" FOREIGN KEY ("CD_ID") REFERENCES "CDS"("CD_ID") DEFERRABLE; + + +-- +-- Name: SDS_SERDE_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SDS" + ADD CONSTRAINT "SDS_SERDE_ID_fkey" FOREIGN KEY ("SERDE_ID") REFERENCES "SERDES"("SERDE_ID") DEFERRABLE; + + +-- +-- Name: SD_PARAMS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SD_PARAMS" + ADD CONSTRAINT "SD_PARAMS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: SERDE_PARAMS_SERDE_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SERDE_PARAMS" + ADD CONSTRAINT "SERDE_PARAMS_SERDE_ID_fkey" FOREIGN KEY ("SERDE_ID") REFERENCES "SERDES"("SERDE_ID") DEFERRABLE; + + +-- +-- Name: SORT_COLS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "SORT_COLS" + ADD CONSTRAINT "SORT_COLS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: TABLE_PARAMS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TABLE_PARAMS" + ADD CONSTRAINT "TABLE_PARAMS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: TBLS_DB_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TBLS" + ADD CONSTRAINT "TBLS_DB_ID_fkey" FOREIGN KEY ("DB_ID") REFERENCES "DBS"("DB_ID") DEFERRABLE; + + +-- +-- Name: TBLS_SD_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TBLS" + ADD CONSTRAINT "TBLS_SD_ID_fkey" FOREIGN KEY ("SD_ID") REFERENCES "SDS"("SD_ID") DEFERRABLE; + + +-- +-- Name: TBL_COL_PRIVS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TBL_COL_PRIVS" + ADD CONSTRAINT "TBL_COL_PRIVS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: TBL_PRIVS_TBL_ID_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TBL_PRIVS" + ADD CONSTRAINT "TBL_PRIVS_TBL_ID_fkey" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: TYPE_FIELDS_TYPE_NAME_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- + +ALTER TABLE ONLY "TYPE_FIELDS" + ADD CONSTRAINT "TYPE_FIELDS_TYPE_NAME_fkey" FOREIGN KEY ("TYPE_NAME") REFERENCES "TYPES"("TYPES_ID") DEFERRABLE; + +-- +-- Name: TAB_COL_STATS_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- +ALTER TABLE ONLY "TAB_COL_STATS" ADD CONSTRAINT "TAB_COL_STATS_fkey" FOREIGN KEY("TBL_ID") REFERENCES "TBLS"("TBL_ID") DEFERRABLE; + + +-- +-- Name: PART_COL_STATS_fkey; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +-- +ALTER TABLE ONLY "PART_COL_STATS" ADD CONSTRAINT "PART_COL_STATS_fkey" FOREIGN KEY("PART_ID") REFERENCES "PARTITIONS"("PART_ID") DEFERRABLE; + + +ALTER TABLE ONLY "VERSION" ADD CONSTRAINT "VERSION_pkey" PRIMARY KEY ("VER_ID"); + +-- Name: FUNCS_FK1; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +ALTER TABLE ONLY "FUNCS" + ADD CONSTRAINT "FUNCS_FK1" FOREIGN KEY ("DB_ID") REFERENCES "DBS" ("DB_ID") DEFERRABLE; + +-- Name: FUNC_RU_FK1; Type: FK CONSTRAINT; Schema: public; Owner: hiveuser +ALTER TABLE ONLY "FUNC_RU" + ADD CONSTRAINT "FUNC_RU_FK1" FOREIGN KEY ("FUNC_ID") REFERENCES "FUNCS" ("FUNC_ID") DEFERRABLE; + +-- +-- Name: public; Type: ACL; Schema: -; Owner: hiveuser +-- + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT ALL ON SCHEMA public TO PUBLIC; + +-- +-- PostgreSQL database dump complete +-- + +------------------------------ +-- Transaction and lock tables +------------------------------ +-- \i hive-txn-schema-2.3.0.postgres.sql; + +-- ----------------------------------------------------------------- +-- Record schema version. Should be the last step in the init script +-- ----------------------------------------------------------------- +INSERT INTO "VERSION" ("VER_ID", "SCHEMA_VERSION", "VERSION_COMMENT") VALUES (1, '2.3.0', 'Hive release version 2.3.0'); diff --git a/docker/playground/image/hive-site.xml b/docker/playground/image/hive-site.xml new file mode 100644 index 000000000..475af8345 --- /dev/null +++ b/docker/playground/image/hive-site.xml @@ -0,0 +1,45 @@ + + + + + + javax.jdo.option.ConnectionURL + jdbc:postgresql://postgres:5432/metastore + + + javax.jdo.option.ConnectionDriverName + org.postgresql.Driver + + + javax.jdo.option.ConnectionUserName + postgres + + + javax.jdo.option.ConnectionPassword + postgres + + + hive.metastore.warehouse.dir + s3a://spark-bucket/warehouse + location of default database for the warehouse + + + hive.metastore.uris + thrift://metastore:9083 + + diff --git a/docker/playground/image/hive-txn-schema-2.3.0.postgres.sql b/docker/playground/image/hive-txn-schema-2.3.0.postgres.sql new file mode 100644 index 000000000..1fa99aff5 --- /dev/null +++ b/docker/playground/image/hive-txn-schema-2.3.0.postgres.sql @@ -0,0 +1,133 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +-- Tables for transaction management +-- + +CREATE TABLE TXNS ( + TXN_ID bigint PRIMARY KEY, + TXN_STATE char(1) NOT NULL, + TXN_STARTED bigint NOT NULL, + TXN_LAST_HEARTBEAT bigint NOT NULL, + TXN_USER varchar(128) NOT NULL, + TXN_HOST varchar(128) NOT NULL, + TXN_AGENT_INFO varchar(128), + TXN_META_INFO varchar(128), + TXN_HEARTBEAT_COUNT integer +); + +CREATE TABLE TXN_COMPONENTS ( + TC_TXNID bigint REFERENCES TXNS (TXN_ID), + TC_DATABASE varchar(128) NOT NULL, + TC_TABLE varchar(128), + TC_PARTITION varchar(767) DEFAULT NULL, + TC_OPERATION_TYPE char(1) NOT NULL +); + +CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS USING hash (TC_TXNID); + +CREATE TABLE COMPLETED_TXN_COMPONENTS ( + CTC_TXNID bigint, + CTC_DATABASE varchar(128) NOT NULL, + CTC_TABLE varchar(256), + CTC_PARTITION varchar(767) +); + +CREATE TABLE NEXT_TXN_ID ( + NTXN_NEXT bigint NOT NULL +); +INSERT INTO NEXT_TXN_ID VALUES(1); + +CREATE TABLE HIVE_LOCKS ( + HL_LOCK_EXT_ID bigint NOT NULL, + HL_LOCK_INT_ID bigint NOT NULL, + HL_TXNID bigint, + HL_DB varchar(128) NOT NULL, + HL_TABLE varchar(128), + HL_PARTITION varchar(767) DEFAULT NULL, + HL_LOCK_STATE char(1) NOT NULL, + HL_LOCK_TYPE char(1) NOT NULL, + HL_LAST_HEARTBEAT bigint NOT NULL, + HL_ACQUIRED_AT bigint, + HL_USER varchar(128) NOT NULL, + HL_HOST varchar(128) NOT NULL, + HL_HEARTBEAT_COUNT integer, + HL_AGENT_INFO varchar(128), + HL_BLOCKEDBY_EXT_ID bigint, + HL_BLOCKEDBY_INT_ID bigint, + PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) +); + +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS USING hash (HL_TXNID); + +CREATE TABLE NEXT_LOCK_ID ( + NL_NEXT bigint NOT NULL +); +INSERT INTO NEXT_LOCK_ID VALUES(1); + +CREATE TABLE COMPACTION_QUEUE ( + CQ_ID bigint PRIMARY KEY, + CQ_DATABASE varchar(128) NOT NULL, + CQ_TABLE varchar(128) NOT NULL, + CQ_PARTITION varchar(767), + CQ_STATE char(1) NOT NULL, + CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), + CQ_WORKER_ID varchar(128), + CQ_START bigint, + CQ_RUN_AS varchar(128), + CQ_HIGHEST_TXN_ID bigint, + CQ_META_INFO bytea, + CQ_HADOOP_JOB_ID varchar(32) +); + +CREATE TABLE NEXT_COMPACTION_QUEUE_ID ( + NCQ_NEXT bigint NOT NULL +); +INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1); + +CREATE TABLE COMPLETED_COMPACTIONS ( + CC_ID bigint PRIMARY KEY, + CC_DATABASE varchar(128) NOT NULL, + CC_TABLE varchar(128) NOT NULL, + CC_PARTITION varchar(767), + CC_STATE char(1) NOT NULL, + CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), + CC_WORKER_ID varchar(128), + CC_START bigint, + CC_END bigint, + CC_RUN_AS varchar(128), + CC_HIGHEST_TXN_ID bigint, + CC_META_INFO bytea, + CC_HADOOP_JOB_ID varchar(32) +); + +CREATE TABLE AUX_TABLE ( + MT_KEY1 varchar(128) NOT NULL, + MT_KEY2 bigint NOT NULL, + MT_COMMENT varchar(255), + PRIMARY KEY(MT_KEY1, MT_KEY2) +); + +CREATE TABLE WRITE_SET ( + WS_DATABASE varchar(128) NOT NULL, + WS_TABLE varchar(128) NOT NULL, + WS_PARTITION varchar(767), + WS_TXNID bigint NOT NULL, + WS_COMMIT_ID bigint NOT NULL, + WS_OPERATION_TYPE char(1) NOT NULL +); diff --git a/docker/playground/image/kyuubi-defaults.conf b/docker/playground/image/kyuubi-defaults.conf new file mode 100644 index 000000000..b45841cd6 --- /dev/null +++ b/docker/playground/image/kyuubi-defaults.conf @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## Kyuubi Configurations + +kyuubi.authentication=NONE +kyuubi.frontend.bind.host=0.0.0.0 +kyuubi.frontend.bind.port=10009 +kyuubi.ha.zookeeper.quorum=zookeeper:2181 +kyuubi.session.engine.idle.timeout=PT5M +kyuubi.operation.incremental.collect=true +kyuubi.operation.progress.enabled=true + +kyuubi.engine.session.initialize.sql \ + show namespaces in tpcds; \ + show namespaces in tpch; \ + show namespaces in postgres; diff --git a/docker/playground/image/kyuubi-playground-base.Dockerfile b/docker/playground/image/kyuubi-playground-base.Dockerfile new file mode 100644 index 000000000..6ee4ed405 --- /dev/null +++ b/docker/playground/image/kyuubi-playground-base.Dockerfile @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM eclipse-temurin:8-focal + +RUN set -x && \ + ln -snf /usr/bin/bash /usr/bin/sh && \ + apt-get update -q && \ + apt-get install -yq retry busybox && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir /opt/busybox && \ + busybox --install /opt/busybox + +ENV PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/busybox diff --git a/docker/playground/image/kyuubi-playground-hadoop.Dockerfile b/docker/playground/image/kyuubi-playground-hadoop.Dockerfile new file mode 100644 index 000000000..588ff8d80 --- /dev/null +++ b/docker/playground/image/kyuubi-playground-hadoop.Dockerfile @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG KYUUBI_VERSION + +FROM nekyuubi/kyuubi-playground-base:${KYUUBI_VERSION} + +ARG AWS_JAVA_SDK_VERSION +ARG HADOOP_VERSION + +ARG APACHE_MIRROR +ARG MAVEN_MIRROR + +ENV HADOOP_HOME=/opt/hadoop +ENV HADOOP_CONF_DIR=/etc/hadoop/conf + +RUN set -x && \ + if [ $(uname -m) = "aarch64" ]; then HADOOP_TAR_NAME=hadoop-${HADOOP_VERSION}-aarch64; else HADOOP_TAR_NAME=hadoop-${HADOOP_VERSION}; fi && \ + wget -q ${APACHE_MIRROR}/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_TAR_NAME}.tar.gz && \ + tar -xzf ${HADOOP_TAR_NAME}.tar.gz -C /opt && \ + ln -s /opt/hadoop-${HADOOP_VERSION} ${HADOOP_HOME} && \ + rm ${HADOOP_TAR_NAME}.tar.gz && \ + HADOOP_CLOUD_STORAGE_JAR_NAME=hadoop-cloud-storage && \ + wget -q ${MAVEN_MIRROR}/org/apache/hadoop/${HADOOP_CLOUD_STORAGE_JAR_NAME}/${HADOOP_VERSION}/${HADOOP_CLOUD_STORAGE_JAR_NAME}-${HADOOP_VERSION}.jar -P ${HADOOP_HOME}/share/hadoop/hdfs/lib && \ + HADOOP_AWS_JAR_NAME=hadoop-aws && \ + wget -q ${MAVEN_MIRROR}/org/apache/hadoop/${HADOOP_AWS_JAR_NAME}/${HADOOP_VERSION}/${HADOOP_AWS_JAR_NAME}-${HADOOP_VERSION}.jar -P ${HADOOP_HOME}/share/hadoop/hdfs/lib && \ + AWS_JAVA_SDK_BUNDLE_JAR_NAME=aws-java-sdk-bundle && \ + wget -q ${MAVEN_MIRROR}/com/amazonaws/${AWS_JAVA_SDK_BUNDLE_JAR_NAME}/${AWS_JAVA_SDK_VERSION}/${AWS_JAVA_SDK_BUNDLE_JAR_NAME}-${AWS_JAVA_SDK_VERSION}.jar -P ${HADOOP_HOME}/share/hadoop/hdfs/lib diff --git a/docker/playground/image/kyuubi-playground-kyuubi.Dockerfile b/docker/playground/image/kyuubi-playground-kyuubi.Dockerfile new file mode 100644 index 000000000..3873544b9 --- /dev/null +++ b/docker/playground/image/kyuubi-playground-kyuubi.Dockerfile @@ -0,0 +1,40 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG KYUUBI_VERSION + +FROM nekyuubi/kyuubi-playground-spark:${KYUUBI_VERSION} + +ARG AWS_JAVA_SDK_VERSION +ARG KYUUBI_VERSION +ARG KYUUBI_HADOOP_VERSION + +ARG APACHE_MIRROR +ARG MAVEN_MIRROR + +ENV KYUUBI_HOME=/opt/kyuubi +ENV KYUUBI_CONF_DIR=/etc/kyuubi/conf + +RUN set -x && \ + wget -q ${APACHE_MIRROR}/incubator/kyuubi/kyuubi-${KYUUBI_VERSION}/apache-kyuubi-${KYUUBI_VERSION}-bin.tgz && \ + tar -xzf apache-kyuubi-${KYUUBI_VERSION}-bin.tgz -C /opt && \ + ln -s /opt/apache-kyuubi-${KYUUBI_VERSION}-bin ${KYUUBI_HOME} && \ + rm apache-kyuubi-${KYUUBI_VERSION}-bin.tgz && \ + HADOOP_CLOUD_STORAGE_JAR_NAME=hadoop-cloud-storage && \ + wget -q ${MAVEN_MIRROR}/org/apache/hadoop/${HADOOP_CLOUD_STORAGE_JAR_NAME}/${KYUUBI_HADOOP_VERSION}/${HADOOP_CLOUD_STORAGE_JAR_NAME}-${KYUUBI_HADOOP_VERSION}.jar -P ${KYUUBI_HOME}/jars && \ + HADOOP_AWS_JAR_NAME=hadoop-aws && \ + wget -q ${MAVEN_MIRROR}/org/apache/hadoop/${HADOOP_AWS_JAR_NAME}/${KYUUBI_HADOOP_VERSION}/${HADOOP_AWS_JAR_NAME}-${KYUUBI_HADOOP_VERSION}.jar -P ${KYUUBI_HOME}/jars && \ + AWS_JAVA_SDK_BUNDLE_JAR_NAME=aws-java-sdk-bundle && \ + wget -q ${MAVEN_MIRROR}/com/amazonaws/${AWS_JAVA_SDK_BUNDLE_JAR_NAME}/${AWS_JAVA_SDK_VERSION}/${AWS_JAVA_SDK_BUNDLE_JAR_NAME}-${AWS_JAVA_SDK_VERSION}.jar -P ${KYUUBI_HOME}/jars && \ + useradd anonymous + +ENTRYPOINT ["/opt/kyuubi/bin/kyuubi", "run"] diff --git a/docker/playground/image/kyuubi-playground-metastore.Dockerfile b/docker/playground/image/kyuubi-playground-metastore.Dockerfile new file mode 100644 index 000000000..36ce31cda --- /dev/null +++ b/docker/playground/image/kyuubi-playground-metastore.Dockerfile @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG KYUUBI_VERSION + +FROM nekyuubi/kyuubi-playground-hadoop:${KYUUBI_VERSION} + +ARG HIVE_VERSION + +ARG APACHE_MIRROR + +ENV HIVE_HOME=/opt/hive +ENV HIVE_CONF_DIR=/etc/hive/conf + +RUN set -x && \ + wget -q ${APACHE_MIRROR}/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz && \ + tar -xzf apache-hive-${HIVE_VERSION}-bin.tar.gz -C /opt && \ + ln -s /opt/apache-hive-${HIVE_VERSION}-bin ${HIVE_HOME} && \ + rm apache-hive-${HIVE_VERSION}-bin.tar.gz + +ENTRYPOINT ["/opt/hive/bin/hive", "--service", "metastore"] diff --git a/docker/playground/image/kyuubi-playground-spark.Dockerfile b/docker/playground/image/kyuubi-playground-spark.Dockerfile new file mode 100644 index 000000000..4dd9df66c --- /dev/null +++ b/docker/playground/image/kyuubi-playground-spark.Dockerfile @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG KYUUBI_VERSION + +FROM nekyuubi/kyuubi-playground-base:${KYUUBI_VERSION} + +ARG AWS_JAVA_SDK_VERSION +ARG ICEBERG_VERSION +ARG KYUUBI_VERSION +ARG SPARK_HADOOP_VERSION +ARG POSTGRES_JDBC_VERSION +ARG SCALA_BINARY_VERSION +ARG SPARK_VERSION +ARG SPARK_BINARY_VERSION + +ARG APACHE_MIRROR +ARG MAVEN_MIRROR + +ENV SPARK_HOME=/opt/spark +ENV HADOOP_CONF_DIR=/etc/hadoop/conf +ENV HIVE_CONF_DIR=/etc/hive/conf +ENV SPARK_CONF_DIR=/etc/spark/conf + +RUN set -x && \ + wget -q ${APACHE_MIRROR}/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz && \ + tar -xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz -C /opt && \ + ln -s /opt/spark-${SPARK_VERSION}-bin-hadoop3 ${SPARK_HOME} && \ + rm spark-${SPARK_VERSION}-bin-hadoop3.tgz && \ + ICEBERG_SPARK_JAR_NAME=iceberg-spark-runtime-${SPARK_BINARY_VERSION}_${SCALA_BINARY_VERSION} && \ + wget -q ${MAVEN_MIRROR}/org/apache/iceberg/${ICEBERG_SPARK_JAR_NAME}/${ICEBERG_VERSION}/${ICEBERG_SPARK_JAR_NAME}-${ICEBERG_VERSION}.jar -P ${SPARK_HOME}/jars && \ + SPARK_HADOOP_CLOUD_JAR_NAME=spark-hadoop-cloud_${SCALA_BINARY_VERSION} && \ + wget -q ${MAVEN_MIRROR}/org/apache/spark/${SPARK_HADOOP_CLOUD_JAR_NAME}/${SPARK_VERSION}/${SPARK_HADOOP_CLOUD_JAR_NAME}-${SPARK_VERSION}.jar -P ${SPARK_HOME}/jars && \ + HADOOP_CLOUD_STORAGE_JAR_NAME=hadoop-cloud-storage && \ + wget -q ${MAVEN_MIRROR}/org/apache/hadoop/${HADOOP_CLOUD_STORAGE_JAR_NAME}/${SPARK_HADOOP_VERSION}/${HADOOP_CLOUD_STORAGE_JAR_NAME}-${SPARK_HADOOP_VERSION}.jar -P ${SPARK_HOME}/jars && \ + HADOOP_AWS_JAR_NAME=hadoop-aws && \ + wget -q ${MAVEN_MIRROR}/org/apache/hadoop/${HADOOP_AWS_JAR_NAME}/${SPARK_HADOOP_VERSION}/${HADOOP_AWS_JAR_NAME}-${SPARK_HADOOP_VERSION}.jar -P ${SPARK_HOME}/jars && \ + AWS_JAVA_SDK_BUNDLE_JAR_NAME=aws-java-sdk-bundle && \ + wget -q ${MAVEN_MIRROR}/com/amazonaws/${AWS_JAVA_SDK_BUNDLE_JAR_NAME}/${AWS_JAVA_SDK_VERSION}/${AWS_JAVA_SDK_BUNDLE_JAR_NAME}-${AWS_JAVA_SDK_VERSION}.jar -P ${SPARK_HOME}/jars && \ + POSTGRES_JDBC_JAR_NAME=postgresql && \ + wget -q ${MAVEN_MIRROR}/org/postgresql/${POSTGRES_JDBC_JAR_NAME}/${POSTGRES_JDBC_VERSION}/${POSTGRES_JDBC_JAR_NAME}-${POSTGRES_JDBC_VERSION}.jar -P ${SPARK_HOME}/jars && \ + TPCDS_CONNECTOR_JAR_NAME=kyuubi-spark-connector-tpcds_${SCALA_BINARY_VERSION} && \ + wget -q ${MAVEN_MIRROR}/org/apache/kyuubi/${TPCDS_CONNECTOR_JAR_NAME}/${KYUUBI_VERSION}/${TPCDS_CONNECTOR_JAR_NAME}-${KYUUBI_VERSION}.jar -P ${SPARK_HOME}/jars && \ + TPCH_CONNECTOR_JAR_NAME=kyuubi-spark-connector-tpch_${SCALA_BINARY_VERSION} && \ + wget -q ${MAVEN_MIRROR}/org/apache/kyuubi/${TPCH_CONNECTOR_JAR_NAME}/${KYUUBI_VERSION}/${TPCH_CONNECTOR_JAR_NAME}-${KYUUBI_VERSION}.jar -P ${SPARK_HOME}/jars diff --git a/docker/playground/image/spark-defaults.conf b/docker/playground/image/spark-defaults.conf new file mode 100644 index 000000000..7891c83db --- /dev/null +++ b/docker/playground/image/spark-defaults.conf @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default system properties included when running spark-submit. +# This is useful for setting default environmental settings. + +spark.driver.host=0.0.0.0 +spark.ui.port=4040 +spark.sql.shuffle.partitions=16 +spark.sql.warehouse.dir=s3a://spark-bucket/warehouse +spark.sql.defaultCatalog=spark_catalog +spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions +spark.hadoop.fs.s3a.bucket.spark-bucket.committer.magic.enabled=true +spark.hadoop.fs.s3a.bucket.iceberg-bucket.committer.magic.enabled=true + +spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog +spark.sql.catalog.spark_catalog.type=hive + +spark.sql.catalog.tpcds=org.apache.kyuubi.spark.connector.tpcds.TPCDSCatalog +spark.sql.catalog.tpch=org.apache.kyuubi.spark.connector.tpch.TPCHCatalog + +spark.sql.catalog.postgres=org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog +spark.sql.catalog.postgres.url=jdbc:postgresql://postgres:5432/metastore +spark.sql.catalog.postgres.driver=org.postgresql.Driver +spark.sql.catalog.postgres.user=postgres +spark.sql.catalog.postgres.password=postgres + +spark.sql.catalog.iceberg=org.apache.iceberg.spark.SparkCatalog +spark.sql.catalog.iceberg.type=hadoop +spark.sql.catalog.iceberg.warehouse=s3a://iceberg-bucket/iceberg-warehouse