if [[ -n "$(ss -Hltn "sport = :${NODE_PORT}")" ]]; then
NODE_LISTENING="true"
break
fi
sleep 1
done
if [[ "${NODE_LISTENING}" == "true" ]]; then
echo "[+] Node is listening on ${NODE_PORT}/tcp."
else
echo "[WARN] Node is not listening on ${NODE_PORT}/tcp yet." >&2
echo " This is expected during the initial database load. Monitor with:" >&2
echo " ss -ltn 'sport = :${NODE_PORT}'" >&2
echo " sudo tail -f ${WRAPPER_LOG}" >&2
fi
# ----------------------------------------------------
# 19b. Legacy Installation Removal
# ----------------------------------------------------
#
# Runs only after the new node is verified running, so a failed
# provisioning run leaves legacy installations (and their keystores) intact.
# Legacy directories and units are root-owned, so removal uses sudo.
#
if [[ "${LEGACY_UNIT_FOUND}" == "true" ]]; then
echo -e "\n[*] Removing legacy systemd unit ${LEGACY_SERVICE_NAME}..."
sudo systemctl disable "${LEGACY_SERVICE_NAME}" 2>/dev/null || true
sudo rm -f "${LEGACY_SERVICE_FILE}"
sudo systemctl daemon-reload
sudo systemctl reset-failed "${LEGACY_SERVICE_NAME}" 2>/dev/null || true
echo "[+] Legacy systemd unit removed."
fi
for LEGACY_DIR in "${LEGACY_FOUND_DIRS[@]}"; do
echo -e "\n[*] Removing legacy installation at ${LEGACY_DIR}..."
sudo rm -rf "${LEGACY_DIR}"
#
# Remove the parent directory only if nothing else lives there.
#
LEGACY_PARENT_DIR="$(dirname "${LEGACY_DIR}")"
sudo rmdir "${LEGACY_PARENT_DIR}" 2>/dev/null || \
echo " - ${LEGACY_PARENT_DIR} not empty; left in place."
#
# Remove the persistent SELinux rules registered for the legacy paths.
#
if command -v semanage >/dev/null 2>&1 || sudo test -x /usr/sbin/semanage; then
sudo semanage fcontext -d "${LEGACY_DIR}(/.*)?" 2>/dev/null || true
if [[ ! -d "${LEGACY_PARENT_DIR}" ]]; then
sudo semanage fcontext -d "${LEGACY_PARENT_DIR}" 2>/dev/null || true
fi
echo " - Legacy SELinux file-context rules removed."
fi
echo "[+] Legacy installation removed: ${LEGACY_DIR}"
done
# ----------------------------------------------------
# 20. Final Service Status
# ----------------------------------------------------
echo -e "\n[*] Final systemd status..."
systemctl status "${SERVICE_NAME}" --no-pager -l || true
# ----------------------------------------------------
# 21. Completion Summary
# ----------------------------------------------------
echo
echo "=================================================="
echo " SERVICENOW APPLICATION NODE PROVISIONING COMPLETE"
echo "=================================================="
echo
echo " Node: ${NODE_ID}"
echo " Node Port: ${NODE_PORT}/tcp (listening: ${NODE_LISTENING})"
echo " Application Directory: ${APP_DIR}"
echo " Provisioned By: $(id -un) (sudo for system changes)"
echo " Application User: ${APP_USER}"
echo " Application Group: ${APP_GROUP}"
echo " Java Home: ${JAVA_HOME}"
echo " Database: ${DB_HOST}:${DB_PORT}/${DB_NAME}"
echo " Database User: ${DB_USER}"
echo " KMF Keystore: ${KMF_KEYSTORE}"
echo " KMF Alias: ${KMF_ALIAS}"
echo " KMF Backup: ${KEYS_DIR}"
if [[ "${KMF_PASSWORD_CHANGED}" == "true" ]]; then
echo " KMF Password: differs from KMF_PASSWORD; copy it from ${KMF_PROPERTIES}"
fi
echo " HTTPS Keystore: ${SSL_KEYSTORE}"
echo " DB mTLS Keystore: ${DB_TLS_KEYSTORE}"
echo " DB mTLS Truststore: ${DB_TLS_TRUSTSTORE}"
echo " JVM Overrides: 51-memory, 92-access, 93-legacy-jdk, 98-general"
echo " Service: ${SERVICE_NAME}"
echo " Wrapper PID: ${WRAPPER_PID}"
echo " JVM PID: ${JAVA_PID}"
for LEGACY_DIR in "${LEGACY_FOUND_DIRS[@]}"; do
echo " Legacy Installation: ${LEGACY_DIR} (removed)"
done
if [[ -n "${LEGACY_SOURCE_DIR}" ]]; then
echo " Keystores Migrated: from ${LEGACY_SOURCE_DIR}"
fi
echo
echo "=================================================="
Category: Uncategorized
-
The Script Updated (untested)
-
Do I Need To Modify App Provision Script?
I reviewed Anthony’s
provision.sh. We do not need to rewrite the application provisioning script. We need to add a separate database mTLS configuration workflow and modify a few existing sections.The important distinction is that this script currently manages three different security functions, and only one needs to be added.
Function Existing script Change needed KMF database encryption Section 7, BCFKS keystore Leave alone Application HTTPS certificate Sections 10–12, BCFKS keystore Leave alone Application-to-MariaDB TLS Section 6, JDBC connection Add mutual TLS 1. Add a separate database client certificate
The application must prove its identity to MariaDB.
It needs three files:
/glide/keys/snowapp_16000/db-mtls/ ├── root-ca.crt ├── client-cert.pem └── client-key.pemThese are separate from the existing
keystorekmf.bcfksandservicenow.keystore.Why: KMF encrypts application data. The HTTPS keystore identifies the application to browsers. Neither is automatically the client certificate used by JDBC to authenticate to MariaDB.
2. Generate the client CSR on the application server
Run as
servicenow:mkdir -p /glide/keys/snowapp_16000/db-mtls chmod 700 /glide/keys/snowapp_16000/db-mtls cd /glide/keys/snowapp_16000/db-mtls openssl genpkey \ -algorithm RSA \ -pkeyopt rsa_keygen_bits:3072 \ -out client-key.pem chmod 600 client-key.pem openssl req \ -new \ -sha256 \ -key client-key.pem \ -out client.csr \ -subj "/O=OVHcloud Lab/CN=servicenow-mtls-client"Send
client.csrto the CA team.The CA team returns
client-cert.pemandroot-ca.crt.Why: MariaDB needs a certificate identifying the application client, signed by a CA that MariaDB trusts.
3. Convert the certificate into Java keystore format
Anthony’s script uses Java 17, and the JDBC connection runs inside the JVM.
We should use a dedicated PKCS#12 client keystore and truststore, leaving the existing BCFKS files untouched.
After the CA returns the certificates:
cd /glide/keys/snowapp_16000/db-mtls openssl pkcs12 -export \ -inkey client-key.pem \ -in client-cert.pem \ -certfile root-ca.crt \ -name mariadb-client \ -out client-keystore.p12Create the truststore(before):
/glide/java/bin/keytool \ -importcert \ -noprompt \ -alias mariadb-root-ca \ -file root-ca.crt \ -keystore db-truststore.p12 \ -storetype PKCS12After version. updated:
/glide/java/bin/keytool \ -importcert \ -noprompt \ -alias mariadb-root-ca \ -file root-ca.crt \ -keystore db-truststore.bcfks \ -storetype BCFKS \ -providername BCFIPS \ -providerclass org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \ -providerpath /glide/nodes/snowapp_16000/dist/lib/jsw/bc-fips-2.0.0.jarBoth commands will request passwords.
chmod 600 client-keystore.p12 db-truststore.p12Why: The client keystore holds the certificate and private key presented to MariaDB. The truststore holds the CA certificate used to verify MariaDB’s identity.
4. Change Section 6 — Database Configuration
Currently, Anthony’s script writes:
glide.db.url=jdbc:mysql://51.81.89.82:3306/We need to change the JDBC connection to require TLS, verify the database server, and present the client certificate.
For MySQL Connector/J, the connection URL would use settings such as:
glide.db.url=jdbc:mysql://DB_HOST:3306/?sslMode=VERIFY_IDENTITY&trustCertificateKeyStoreUrl=file:/glide/keys/snowapp_16000/db-mtls/db-truststore.p12&trustCertificateKeyStoreType=PKCS12&clientCertificateKeyStoreUrl=file:/glide/keys/snowapp_16000/db-mtls/client-keystore.p12&clientCertificateKeyStoreType=PKCS12Keystore and truststore passwords must also be supplied securely through supported Connector/J properties.
Two critical details:
- The current database address is an IP address. MariaDB’s server certificate must contain that IP in its SAN, or the connection must use a DNS hostname covered by the certificate.
- We must identify the actual JDBC driver JAR and version before finalizing the connection properties. MySQL Connector/J and MariaDB Connector/J use different TLS configuration options.
5. Change Section 6b — Database Login Check
This section creates a temporary Java program to test the database connection.
Currently, it uses:
DriverManager.getConnection( System.getenv("DBCHECK_URL"), System.getenv("DBCHECK_USER"), System.getenv("DBCHECK_PASS") )The test must use the same mTLS configuration as the real application.
Otherwise, the script could report a successful database login even though the actual ServiceNow connection cannot complete mutual TLS.
We should also verify the negotiated connection is encrypted and that MariaDB accepted the client certificate.
6. Change the TLS error diagnostics
Section
diagnose_wrapper_log()currently tells Anthony:The MariaDB server is provisioned without TLS. Verify require_secure_transport=OFF.That is no longer appropriate.
The updated diagnostics should distinguish certificate trust failures, hostname mismatches, missing client certificates, and database account authentication failures.
7. Preserve KMF and HTTPS behavior
This is especially important because Anthony is already troubleshooting KMF.
We should not change:
keystorekmf.bcfks glide.kmf.keystore.properties servicenow.keystore 02-https.propertiesWe should also ensure
--reinstallpreserves the new database mTLS certificate material.Changes required, summarized
Script section Action Configuration variables Add database mTLS paths and settings New enrollment section Generate client key/CSR, or reuse existing issued certificate New certificate installation section Validate certificate, create PKCS#12 stores Section 6 Configure JDBC for mTLS Section 6b Test database login using mTLS Diagnostics Replace outdated non-TLS troubleshooting Reinstallation Preserve mTLS certificate material Sections 7–12 Preserve existing KMF and HTTPS functionality One thing we need to establish before writing the final patch: Which JDBC driver is installed?
On the application server, run:
find /glide/nodes/snowapp_16000/dist/lib \ -type f \ \( -iname '*mysql*.jar' -o -iname '*mariadb*.jar' \)That determines the exact JDBC properties we need. Once identified, we can modify Anthony’s script without disturbing his existing KMF configuration.
Also, the uploaded script contains plaintext database and keystore credentials. Those should be rotated and moved out of the provisioning source before the script is shared or committed to a repository.