[KYUUBI #887] [GA] Daily publish snapshot
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request Closes #887 from pan3793/publish-snapshot. Closes #887 5b2044a1 [Cheng Pan] Pin ubuntu-20.04 in GA 98945a12 [Cheng Pan] Remove Kyuubi local cache before deploy f29cb6e4 [Cheng Pan] [GA] Daily publish snapshot Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
This commit is contained in:
Cheng Pan
parent
c6bee7c1b8
commit
962d721842
8
.github/workflows/master.yml
vendored
8
.github/workflows/master.yml
vendored
@ -13,7 +13,7 @@ on:
|
||||
jobs:
|
||||
rat:
|
||||
name: Check License
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-java@v1
|
||||
@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@ -102,7 +102,7 @@ jobs:
|
||||
|
||||
tpcds:
|
||||
name: TPC-DS Tests
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
env:
|
||||
SPARK_LOCAL_IP: localhost
|
||||
steps:
|
||||
@ -141,7 +141,7 @@ jobs:
|
||||
./build/mvn test -Pspark-3.1 -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds -Dmaven.plugin.scalatest.exclude.tags=''
|
||||
minikube-it:
|
||||
name: Minikube Integration Test
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
2
.github/workflows/nightly.yml
vendored
2
.github/workflows/nightly.yml
vendored
@ -7,7 +7,7 @@ on:
|
||||
jobs:
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
profiles:
|
||||
|
||||
39
.github/workflows/publish-snapshot.yml
vendored
Normal file
39
.github/workflows/publish-snapshot.yml
vendored
Normal file
@ -0,0 +1,39 @@
|
||||
name: Publish Snapshot
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
|
||||
jobs:
|
||||
publish-snapshot:
|
||||
if: ${{ startsWith(github.repository, 'apache/') }}
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
branch:
|
||||
- master
|
||||
- branch-1.3
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@master
|
||||
with:
|
||||
ref: ${{ matrix.branch }}
|
||||
- name: Cache Maven local repository
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.m2/repository
|
||||
key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
snapshot-maven-
|
||||
- name: Install Java 8
|
||||
uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: 8
|
||||
- name: Publish snapshot
|
||||
env:
|
||||
ASF_USERNAME: ${{ secrets.NEXUS_USER }}
|
||||
ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
|
||||
run: |
|
||||
rm -rf ~/.m2/repository/org/apache/kyuubi
|
||||
./build/mvn clean deploy -DskipTests -Pspark-provided -s ./build/release/asf-settings.xml
|
||||
34
build/release/asf-settings.xml
Normal file
34
build/release/asf-settings.xml
Normal file
@ -0,0 +1,34 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
|
||||
<servers>
|
||||
<server>
|
||||
<id>apache.snapshots.https</id>
|
||||
<username>${env.ASF_USERNAME}</username>
|
||||
<password>${env.ASF_PASSWORD}</password>
|
||||
</server>
|
||||
<server>
|
||||
<id>apache.releases.https</id>
|
||||
<username>${env.ASF_USERNAME}</username>
|
||||
<password>${env.ASF_PASSWORD}</password>
|
||||
</server>
|
||||
</servers>
|
||||
</settings>
|
||||
@ -21,13 +21,18 @@ import org.apache.spark.SparkConf
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.slf4j.LoggerFactory
|
||||
|
||||
// scalastyle:off line.size.limit
|
||||
/**
|
||||
* Usage:
|
||||
* If you want to generate 10GB data with new database `tpcds_sf10`, you can run the cmd:
|
||||
* `$SPARK_HOME/bin/spark-submit --conf spark.sql.tpcds.scale.factor=10 --conf spark.sql.tpcds.database=tpcds_sf10 --class org.apache.kyuubi.tpcds.DataGenerator kyuubi-tpcds-*.jar`
|
||||
* <p>
|
||||
* Run following command to generate 10GB data with new database `tpcds_sf10`.
|
||||
* {{{
|
||||
* `$SPARK_HOME/bin/spark-submit \
|
||||
* --conf spark.sql.tpcds.scale.factor=10 \
|
||||
* --conf spark.sql.tpcds.database=tpcds_sf10 \
|
||||
* --class org.apache.kyuubi.tpcds.DataGenerator \
|
||||
* kyuubi-tpcds-*.jar`
|
||||
* }}}
|
||||
*/
|
||||
// scalastyle:on line.size.limit
|
||||
object DataGenerator {
|
||||
private val logger =
|
||||
LoggerFactory.getLogger(this.getClass.getSimpleName.stripSuffix("$"))
|
||||
|
||||
@ -107,4 +107,15 @@
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
219
pom.xml
219
pom.xml
@ -110,34 +110,38 @@
|
||||
<hadoop.version>3.2.2</hadoop.version>
|
||||
<hadoop.binary.version>3.2</hadoop.binary.version>
|
||||
<hive.version>2.3.7</hive.version>
|
||||
<iceberg.name>iceberg-spark3-runtime</iceberg.name>
|
||||
<iceberg.version>0.11.1</iceberg.version>
|
||||
<jackson.version>2.11.4</jackson.version>
|
||||
<jakarta.servlet-api.version>4.0.4</jakarta.servlet-api.version>
|
||||
<jaxb.version>2.2.11</jaxb.version>
|
||||
<javax-activation.version>1.1.1</javax-activation.version>
|
||||
<jetty.version>9.4.41.v20210516</jetty.version>
|
||||
<kubernetes-client.version>5.5.0</kubernetes-client.version>
|
||||
<ldapsdk.version>5.1.4</ldapsdk.version>
|
||||
<prometheus.version>0.10.0</prometheus.version>
|
||||
<scalatest.version>3.2.9</scalatest.version>
|
||||
<scopt.version>4.0.1</scopt.version>
|
||||
<slf4j.version>1.7.30</slf4j.version>
|
||||
<spark.version>3.1.2</spark.version>
|
||||
<spark.archive.name>spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz</spark.archive.name>
|
||||
<spark.archive.mirror>https://archive.apache.org/dist/spark/spark-${spark.version}</spark.archive.mirror>
|
||||
<spark.archive.download.skip>false</spark.archive.download.skip>
|
||||
|
||||
<slf4j.version>1.7.30</slf4j.version>
|
||||
<zookeeper.version>3.4.14</zookeeper.version>
|
||||
|
||||
<kubernetes-client.version>5.5.0</kubernetes-client.version>
|
||||
|
||||
<scalatest.version>3.2.9</scalatest.version>
|
||||
<iceberg.name>iceberg-spark3-runtime</iceberg.name>
|
||||
<iceberg.version>0.11.1</iceberg.version>
|
||||
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<jars.target.dir>${project.build.directory}/scala-${scala.binary.version}/jars</jars.target.dir>
|
||||
|
||||
<maven.plugin.antrun.version>3.0.0</maven.plugin.antrun.version>
|
||||
<maven.plugin.assembly.version>3.2.0</maven.plugin.assembly.version>
|
||||
<maven.plugin.build.helper.version>3.2.0</maven.plugin.build.helper.version>
|
||||
<maven.plugin.clean.version>3.1.0</maven.plugin.clean.version>
|
||||
<maven.plugin.compiler.version>3.8.1</maven.plugin.compiler.version>
|
||||
<maven.plugin.dependency.version>3.1.1</maven.plugin.dependency.version>
|
||||
<maven.plugin.deploy.version>3.0.0-M1</maven.plugin.deploy.version>
|
||||
<maven.plugin.download.version>1.6.0</maven.plugin.download.version>
|
||||
<maven.plugin.gpg.version>3.0.1</maven.plugin.gpg.version>
|
||||
<maven.plugin.javadoc.version>3.3.0</maven.plugin.javadoc.version>
|
||||
<!-- DO NOT bump 4.4.0, see https://github.com/apache/incubator-kyuubi/pull/441 -->
|
||||
<!-- DO NOT bump 4.5.3, see https://github.com/apache/incubator-kyuubi/issues/708 -->
|
||||
<maven.plugin.scala.version>4.3.0</maven.plugin.scala.version>
|
||||
@ -146,14 +150,11 @@
|
||||
<maven.plugin.scalatest.exclude.tags>org.apache.kyuubi.tags.ExtendedSQLTest,org.apache.kyuubi.tags.IcebergTest</maven.plugin.scalatest.exclude.tags>
|
||||
<maven.plugin.scalatest.include.tags></maven.plugin.scalatest.include.tags>
|
||||
<maven.plugin.jacoco.version>0.8.6</maven.plugin.jacoco.version>
|
||||
<maven.plugin.shade.version>3.2.4</maven.plugin.shade.version>
|
||||
<maven.plugin.jar.version>3.2.0</maven.plugin.jar.version>
|
||||
<maven.plugin.assembly.version>3.2.0</maven.plugin.assembly.version>
|
||||
<maven.plugin.dependency.version>3.1.1</maven.plugin.dependency.version>
|
||||
<maven.plugin.clean.version>3.1.0</maven.plugin.clean.version>
|
||||
<maven.plugin.scalastyle.version>1.0.0</maven.plugin.scalastyle.version>
|
||||
<maven.plugin.download.version>1.6.0</maven.plugin.download.version>
|
||||
<maven.plugin.shade.version>3.2.4</maven.plugin.shade.version>
|
||||
<maven.plugin.silencer.version>1.7.5</maven.plugin.silencer.version>
|
||||
<maven.plugin.source.version>3.2.1</maven.plugin.source.version>
|
||||
<maven.plugin.rat.version>0.13</maven.plugin.rat.version>
|
||||
|
||||
<!-- Needed for consistent times -->
|
||||
@ -1097,6 +1098,20 @@
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<version>${maven.plugin.antrun.version}</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-tmp-dir</id>
|
||||
<phase>generate-test-resources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<mkdir dir="${project.build.directory}/tmp"/>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
@ -1184,6 +1199,13 @@
|
||||
<goal>testCompile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>attach-scaladocs</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>doc-jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<scalaVersion>${scala.version}</scalaVersion>
|
||||
@ -1319,6 +1341,28 @@
|
||||
<includeScope>runtime</includeScope>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>generate-test-classpath</id>
|
||||
<phase>test-compile</phase>
|
||||
<goals>
|
||||
<goal>build-classpath</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<includeScope>test</includeScope>
|
||||
<outputProperty>test_classpath</outputProperty>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>copy-module-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<includeScope>runtime</includeScope>
|
||||
<outputDirectory>${jars.target.dir}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
@ -1329,15 +1373,51 @@
|
||||
<configuration>
|
||||
<filesets>
|
||||
<fileset>
|
||||
<directory>embedded_zookeeper</directory>
|
||||
<directory>${project.basedir}</directory>
|
||||
<includes>
|
||||
<include>**/*.log</include>
|
||||
</includes>
|
||||
</fileset>
|
||||
<fileset>
|
||||
<directory>metastore_db</directory>
|
||||
<directory>${project.basedir}/embedded_zookeeper</directory>
|
||||
</fileset>
|
||||
<fileset>
|
||||
<directory>${project.basedir}/metastore_db</directory>
|
||||
</fileset>
|
||||
<fileset>
|
||||
<directory>${project.basedir}/metrics</directory>
|
||||
</fileset>
|
||||
<fileset>
|
||||
<directory>${project.basedir}/spark-warehouse</directory>
|
||||
</fileset>
|
||||
</filesets>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<version>${maven.plugin.deploy.version}</version>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-source-plugin</artifactId>
|
||||
<version>${maven.plugin.source.version}</version>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>${maven.plugin.javadoc.version}</version>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-gpg-plugin</artifactId>
|
||||
<version>${maven.plugin.gpg.version}</version>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.scalastyle</groupId>
|
||||
<artifactId>scalastyle-maven-plugin</artifactId>
|
||||
@ -1402,6 +1482,7 @@
|
||||
<exclude>**/licenses-binary/LICENSE*</exclude>
|
||||
<exclude>**/dependency-reduced-pom.xml</exclude>
|
||||
<exclude>**/scalastyle-output.xml</exclude>
|
||||
<exclude>NOTICE*</exclude>
|
||||
<exclude>docs/**</exclude>
|
||||
<exclude>build/apache-maven-*/**</exclude>
|
||||
<exclude>build/scala-*/**</exclude>
|
||||
@ -1416,34 +1497,10 @@
|
||||
</pluginManagement>
|
||||
|
||||
<plugins>
|
||||
<!-- This plugin dumps the test classpath into a file -->
|
||||
<!-- dump test classpath into a folder -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>generate-test-classpath</id>
|
||||
<phase>test-compile</phase>
|
||||
<goals>
|
||||
<goal>build-classpath</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<includeScope>test</includeScope>
|
||||
<outputProperty>test_classpath</outputProperty>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>copy-module-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<includeScope>runtime</includeScope>
|
||||
<outputDirectory>${jars.target.dir}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
@ -1481,20 +1538,6 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-tmp-dir</id>
|
||||
<phase>generate-test-resources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<mkdir dir="${project.build.directory}/tmp"/>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
@ -1606,5 +1649,73 @@
|
||||
<module>kubernetes/integration-tests</module>
|
||||
</modules>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>apache-release</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<!-- Prevent the source-release-assembly execution defined in the Apache parent POM
|
||||
from running so we can control it ourselves -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>source-release-assembly</id>
|
||||
<phase>none</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-deploy-plugin</artifactId>
|
||||
<inherited>true</inherited>
|
||||
<configuration>
|
||||
<updateReleaseInfo>true</updateReleaseInfo>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-source-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-sources</id>
|
||||
<goals>
|
||||
<goal>jar-no-fork</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<configuration>
|
||||
<quiet>true</quiet>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<goals>
|
||||
<goal>jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-gpg-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>sign-release-artifacts</id>
|
||||
<phase>verify</phase>
|
||||
<goals>
|
||||
<goal>sign</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user