[CELEBORN-1346] Add build changes and test resources for ssl support

### What changes were proposed in this pull request?

Build changes and test resources for enabling SSL support.
Please see #2416 for the consolidate PR with all the changes for reference.

Note: I closed the older PR #2413 and reopened this one give the repo changes.

### Why are the changes needed?

Build dependency updates and addition of test resources for use with tests.
The specific tests leveraging these will be added in subsequent jiras linked off of CELEBORN-1343
Splitting it up into multiple PR's to reduce the review load.

### Does this PR introduce _any_ user-facing change?

io.netty:netty-tcnative-boringssl-static is an additional dependency.
org.bouncycastle:* are test dependencies which should have no user facing changes.

### How was this patch tested?
The overall PR #2411 passes all tests, this is specifically pulling out the dependency changes and resources.

Closes #2417 from mridulm/build-and-test-for-tls.

Lead-authored-by: Mridul Muralidharan <mridul@gmail.com>
Co-authored-by: Mridul Muralidharan <mridulatgmail.com>
Signed-off-by: Cheng Pan <chengpan@apache.org>

.gitattributes

@@ -31,3 +31,6 @@ assets/img/* export-ignore
 *.scala text eol=lf
 *.xml text eol=lf
 *.py text eol=lf
+common/src/test/resources/ssl/generate_certs.sh text
+common/src/test/resources/ssl/* -text
+worker/src/test/resources/ssl/* -text

common/pom.xml

@@ -147,6 +147,17 @@
       <artifactId>log4j-1.2-api</artifactId>
       <scope>test</scope>
     </dependency>
+    <!-- for SSL support -->
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcprov-jdk18on</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcpkix-jdk18on</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

common/src/test/resources/ssl/generate_certs.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# A simple utility to (re-)generate the files within resources/ssl
+# These generated certificates are used for a variety of test scenarios for SSL.
+# The utility ends up generating two certificates - which are saved into two different keystores
+# The certificates generated are signed by two different CA cert's (also generated here).
+# There are two truststores generated - the first truststore has both CA certs as part of it
+# Hence this trust can be used to validate both client certificates.
+# The second trust store has NO CA certs in it - and so when used will fail both the certificates.
+# Requires: "openssl" (typically the openssl package) and java "keytool" in the PATH
+
+function gen_certs() {
+
+  openssl genrsa -out ca.key 2048
+  openssl req -x509 -new -days 9000 -key ca.key -out ca.crt -subj "/C=US/ST=YourState/L=YourCity/O=YourOrganization/CN=MyCACert"
+  openssl genrsa -out server.key 2048
+  openssl req -new -key server.key -out server.csr -subj "/C=US/ST=YourState/L=YourCity/O=YourOrganization/CN=MyServer"
+  openssl x509 -req -CA ca.crt -CAkey ca.key -CAcreateserial -in server.csr -out server.crt -days 8000
+  openssl pkcs12 -export -in server.crt -inkey server.key -out keystore.p12 -name servercert -password pass:password
+  keytool -importkeystore -destkeystore server.jks -srckeystore keystore.p12 -srcstoretype PKCS12 -deststoretype pkcs12 -srcstorepass password -deststorepass password -noprompt
+
+  keytool -import -trustcacerts -alias CACert -file ca.crt -keystore truststore.jks -storepass password -noprompt
+
+  rm ca.srl keystore.p12 server.csr ca.key server.key server.crt
+}
+
+
+mkdir for_default
+cd for_default
+gen_certs
+cd ..
+mkdir for_secondary
+cd for_secondary
+gen_certs
+cd ..
+
+
+cp ./for_default/truststore.jks ./for_default/server.jks .
+cp ./for_secondary/server.jks ./server_another.jks
+
+
+keytool -import -trustcacerts -alias 'CACertAnother' -file for_secondary/ca.crt -keystore ./truststore.jks -storepass password -noprompt
+
+# Copy the secondary trust store and remove the ca to generate truststore-without-ca.jks
+cp ./for_secondary/truststore.jks ./truststore-without-ca.jks
+keytool -delete -alias 'CACert' -keystore ./truststore-without-ca.jks -storepass password -noprompt
+
+rm -rf for_default for_secondary

pom.xml

@@ -88,6 +88,7 @@
     <mockito.version>4.11.0</mockito.version>
     <mockito-scalatest.version>1.17.14</mockito-scalatest.version>
     <netty.version>4.1.107.Final</netty.version>
+    <bouncycastle.version>1.77</bouncycastle.version>
     <protobuf.version>3.21.7</protobuf.version>
     <ratis.version>2.5.1</ratis.version>
     <scalatest.version>3.2.16</scalatest.version>
@@ -522,6 +523,19 @@
         <artifactId>snappy-java</artifactId>
         <version>${snappy.version}</version>
       </dependency>
+      <!-- for SSL support -->
+      <dependency>
+        <groupId>org.bouncycastle</groupId>
+        <artifactId>bcprov-jdk18on</artifactId>
+        <version>${bouncycastle.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.bouncycastle</groupId>
+        <artifactId>bcpkix-jdk18on</artifactId>
+        <version>${bouncycastle.version}</version>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 

project/CelebornBuild.scala

@@ -68,6 +68,9 @@ object Dependencies {
   val hikaricpVersion = "4.0.3"
   val h2Version = "2.2.224"
 
+  // For SSL support
+  val bouncycastleVersion = "1.77"
+
   // Versions for proto
   val protocVersion = "3.21.7"
   val protoVersion = "3.21.7"
@@ -140,6 +143,10 @@ object Dependencies {
   val scalatestMockito = "org.mockito" %% "mockito-scala-scalatest" % scalatestMockitoVersion
   val scalatest = "org.scalatest" %% "scalatest" % scalatestVersion
   val h2 = "com.h2database" % "h2" % h2Version
+
+  // SSL support
+  val bouncycastleBcprovJdk18on = "org.bouncycastle" % "bcprov-jdk18on" % bouncycastleVersion % "test"
+  val bouncycastleBcpkixJdk18on = "org.bouncycastle" % "bcpkix-jdk18on" % bouncycastleVersion % "test"
 }
 
 object CelebornCommonSettings {
@@ -394,7 +401,10 @@ object CelebornCommon {
         Dependencies.jacksonDatabind,
         Dependencies.jacksonAnnotations,
         Dependencies.log4jSlf4jImpl % "test",
-        Dependencies.log4j12Api % "test"
+        Dependencies.log4j12Api % "test",
+        // SSL support
+        Dependencies.bouncycastleBcprovJdk18on,
+        Dependencies.bouncycastleBcpkixJdk18on
       ) ++ commonUnitTestDependencies,
 
       Compile / sourceGenerators += Def.task {