From 82834f32c57bacc8688a4a51532d003f52a11b85 Mon Sep 17 00:00:00 2001 From: sanket-shelar Date: Wed, 25 Mar 2026 17:36:32 +0530 Subject: [PATCH] RANGER-5205: Liquibase based Zero-Downtime-Upgrade framework for Ranger --- distro/pom.xml | 2 + .../src/main/assembly/liquibase-upgrade.xml | 77 ++++ .../crypto/key/RangerKeyStoreProvider.java | 5 +- .../kms/db/db.changelog-master-finalize.xml | 29 ++ .../kms/db/db.changelog-master-upgrade.xml | 30 ++ .../create_trigger_kmsencoded_postgres.sql | 42 ++ .../kms/db/test_3.x/cleanup_kms_column.xml | 41 ++ .../kms/db/test_3.x/cleanup_kms_trigger.xml | 33 ++ .../kms/db/test_3.x/db_javapatches.xml | 27 ++ .../kms/db/test_3.x/kms_rename_col_zdu.xml | 50 +++ .../db/test_3.x/master-finalize-test_3.x.xml | 29 ++ .../db/test_3.x/master-upgrade-test_3.x.xml | 29 ++ .../changelogs/kms/db/test_3.x/sleep.xml | 37 ++ .../db/db.changelog-master-finalize.xml | 29 ++ .../ranger/db/db.changelog-master-upgrade.xml | 30 ++ .../ranger/db/test_3.x/add_xgroup_column.xml | 33 ++ .../db/test_3.x/db_javapatches_ozone.xml | 27 ++ .../db/test_3.x/master-finalize-test_3.x.xml | 26 ++ .../db/test_3.x/master-upgrade-test_3.x.xml | 29 ++ .../changelogs/ranger/db/test_3.x/sleep.xml | 34 ++ liquibase-database-upgrade/pom.xml | 245 ++++++++++++ liquibase-database-upgrade/scripts/README.md | 66 ++++ .../scripts/kms_liquibase_utils.sh | 93 +++++ .../scripts/upgrade_kms_db.sh | 72 ++++ .../scripts/upgrade_ranger_db.sh | 72 ++++ .../ArgsParserLiquibaseDBUtilsMain.java | 48 +++ .../ArgsParserLiquibaseUpdateDriverMain.java | 47 +++ .../apache/ranger/db/upgrade/Constants.java | 36 ++ .../ranger/db/upgrade/ICommandDriver.java | 32 ++ .../ranger/db/upgrade/IConfigProvider.java | 31 ++ .../ranger/db/upgrade/KmsConfigProvider.java | 55 +++ .../db/upgrade/LiquibaseCommandDriver.java | 94 +++++ .../upgrade/LiquibaseCommandResultsUtil.java | 57 +++ .../db/upgrade/LiquibaseDBUtilsMain.java | 109 +++++ .../upgrade/LiquibasePropertiesFactory.java | 70 ++++ .../db/upgrade/LiquibaseUpdateDriverMain.java | 170 ++++++++ .../db/upgrade/RangerConfigProvider.java | 74 ++++ .../ranger/db/upgrade/SpringContext.java | 88 +++++ .../StatusCheckScheduledExecutorService.java | 84 ++++ .../org/apache/ranger/db/upgrade/Utils.java | 31 ++ .../upgrade/patches/kms/UpdateKMSAttrs.java | 115 ++++++ ...sePatchForOzoneServiceDefUpdateJ10041.java | 371 ++++++++++++++++++ .../src/main/resources/kms_log4j.properties | 39 ++ .../main/resources/ranger_log4j.properties | 39 ++ .../upgrade/DummyConfigProviderForTest.java | 66 ++++ .../ranger/db/upgrade/FileDBForTest.java | 185 +++++++++ ...iquibaseCommandDriverFileBasedForTest.java | 95 +++++ .../TestLiquibaseUpdateDriverMain.java | 141 +++++++ .../resources/finalizeChangelogForTest.txt | 3 + .../liquibaseTestApplicationContext.xml | 31 ++ .../src/test/resources/log4j2-test.xml | 29 ++ .../test/resources/masterChangelogForTest.txt | 3 + pom.xml | 5 + security-admin/pom.xml | 3 + 54 files changed, 3337 insertions(+), 1 deletion(-) create mode 100644 distro/src/main/assembly/liquibase-upgrade.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-finalize.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-upgrade.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/postgres/create_trigger_kmsencoded_postgres.sql create mode 100644 liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_column.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_trigger.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/test_3.x/db_javapatches.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/test_3.x/kms_rename_col_zdu.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/test_3.x/master-finalize-test_3.x.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/test_3.x/master-upgrade-test_3.x.xml create mode 100644 liquibase-database-upgrade/changelogs/kms/db/test_3.x/sleep.xml create mode 100644 liquibase-database-upgrade/changelogs/ranger/db/db.changelog-master-finalize.xml create mode 100644 liquibase-database-upgrade/changelogs/ranger/db/db.changelog-master-upgrade.xml create mode 100644 liquibase-database-upgrade/changelogs/ranger/db/test_3.x/add_xgroup_column.xml create mode 100644 liquibase-database-upgrade/changelogs/ranger/db/test_3.x/db_javapatches_ozone.xml create mode 100644 liquibase-database-upgrade/changelogs/ranger/db/test_3.x/master-finalize-test_3.x.xml create mode 100644 liquibase-database-upgrade/changelogs/ranger/db/test_3.x/master-upgrade-test_3.x.xml create mode 100644 liquibase-database-upgrade/changelogs/ranger/db/test_3.x/sleep.xml create mode 100644 liquibase-database-upgrade/pom.xml create mode 100644 liquibase-database-upgrade/scripts/README.md create mode 100644 liquibase-database-upgrade/scripts/kms_liquibase_utils.sh create mode 100644 liquibase-database-upgrade/scripts/upgrade_kms_db.sh create mode 100644 liquibase-database-upgrade/scripts/upgrade_ranger_db.sh create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ArgsParserLiquibaseDBUtilsMain.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ArgsParserLiquibaseUpdateDriverMain.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/Constants.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ICommandDriver.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/IConfigProvider.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/KmsConfigProvider.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseCommandDriver.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseCommandResultsUtil.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseDBUtilsMain.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibasePropertiesFactory.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseUpdateDriverMain.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/RangerConfigProvider.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/SpringContext.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/StatusCheckScheduledExecutorService.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/Utils.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/patches/kms/UpdateKMSAttrs.java create mode 100644 liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/patches/ranger/SampleLiquibasePatchForOzoneServiceDefUpdateJ10041.java create mode 100644 liquibase-database-upgrade/src/main/resources/kms_log4j.properties create mode 100644 liquibase-database-upgrade/src/main/resources/ranger_log4j.properties create mode 100644 liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/DummyConfigProviderForTest.java create mode 100644 liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/FileDBForTest.java create mode 100644 liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/LiquibaseCommandDriverFileBasedForTest.java create mode 100644 liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/TestLiquibaseUpdateDriverMain.java create mode 100644 liquibase-database-upgrade/src/test/resources/finalizeChangelogForTest.txt create mode 100644 liquibase-database-upgrade/src/test/resources/liquibaseTestApplicationContext.xml create mode 100644 liquibase-database-upgrade/src/test/resources/log4j2-test.xml create mode 100644 liquibase-database-upgrade/src/test/resources/masterChangelogForTest.txt mode change 100755 => 100644 pom.xml diff --git a/distro/pom.xml b/distro/pom.xml index cb3c09a8c0..b685347291 100644 --- a/distro/pom.xml +++ b/distro/pom.xml @@ -502,6 +502,7 @@ src/main/assembly/plugin-presto.xml src/main/assembly/plugin-trino.xml src/main/assembly/sample-client.xml + src/main/assembly/liquibase-upgrade.xml @@ -1090,6 +1091,7 @@ src/main/assembly/plugin-presto.xml src/main/assembly/plugin-trino.xml src/main/assembly/sample-client.xml + src/main/assembly/liquibase-upgrade.xml diff --git a/distro/src/main/assembly/liquibase-upgrade.xml b/distro/src/main/assembly/liquibase-upgrade.xml new file mode 100644 index 0000000000..33c6214a5c --- /dev/null +++ b/distro/src/main/assembly/liquibase-upgrade.xml @@ -0,0 +1,77 @@ + + + + liquibase-upgrade + + tar.gz + + ${project.parent.name}-${project.version}-liquibase-upgrade + true + + + true + + org.apache.ranger:ranger-liquibase-upgrade:jar:${project.version} + + + lib + true + false + 755 + 644 + + org.liquibase:liquibase-core:jar:${liquibase.version} + com.beust:jcommander:jar:${jcommander.version} + org.slf4j:slf4j-api:jar:${slf4j.version} + + org.slf4j:jul-to-slf4j:jar:${slf4j.version} + + org.springframework:spring-context:jar:${springframework.version} + org.springframework:spring-beans:jar:${springframework.version} + org.springframework:spring-core:jar:${springframework.version} + org.springframework:spring-aop:jar:${springframework.version} + org.springframework:spring-expression:jar:${springframework.version} + + + + + + + + ${project.parent.basedir}/liquibase-database-upgrade/target/scripts + + + *.sh + + + + ${project.parent.basedir}/liquibase-database-upgrade/target/changelogs + lib + + ** + + + + ${project.parent.basedir}/liquibase-database-upgrade/target/classes + sample_resources + + *.properties + + + + \ No newline at end of file diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java index ecf58654a3..a9b34dcb31 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -68,7 +68,7 @@ public class RangerKeyStoreProvider extends KeyProvider { private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; private static final String MK_CREDENTIAL_ALIAS = "ranger.ks.masterkey.credential.alias"; private static final String DB_CREDENTIAL_ALIAS = "ranger.ks.jpa.jdbc.credential.alias"; - private static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password"; + public static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password"; private static final String HSM_ENABLED = "ranger.ks.hsm.enabled"; private static final String HSM_PARTITION_PASSWORD_ALIAS = "ranger.ks.hsm.partition.password.alias"; private static final String HSM_PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; @@ -81,6 +81,9 @@ public class RangerKeyStoreProvider extends KeyProvider { private static final String AZURE_CLIENT_SECRET_ALIAS = "ranger.kms.azure.client.secret.alias"; private static final String AZURE_CLIENT_SECRET = "ranger.kms.azure.client.secret"; private static final String AWS_KMS_ENABLED = "ranger.kms.awskms.enabled"; + public static final String DB_URL = "ranger.ks.jpa.jdbc.url"; + public static final String DB_USER = "ranger.ks.jpa.jdbc.user"; + public static final String DB_DRIVER = "ranger.ks.jpa.jdbc.driver"; private static final String AWS_CLIENT_SECRETKEY_ALIAS = RangerAWSKMSProvider.AWS_CLIENT_SECRETKEY + ".alias"; private static final String AWS_CLIENT_SECRETKEY = RangerAWSKMSProvider.AWS_CLIENT_SECRETKEY; private static final String TENCENT_KMS_ENABLED = "ranger.kms.tencentkms.enabled"; diff --git a/liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-finalize.xml b/liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-finalize.xml new file mode 100644 index 0000000000..87001c4672 --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-finalize.xml @@ -0,0 +1,29 @@ + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-upgrade.xml b/liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-upgrade.xml new file mode 100644 index 0000000000..eadcf1990e --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/db.changelog-master-upgrade.xml @@ -0,0 +1,30 @@ + + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/kms/db/postgres/create_trigger_kmsencoded_postgres.sql b/liquibase-database-upgrade/changelogs/kms/db/postgres/create_trigger_kmsencoded_postgres.sql new file mode 100644 index 0000000000..370ff3c1be --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/postgres/create_trigger_kmsencoded_postgres.sql @@ -0,0 +1,42 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +--liquibase formatted sql + +-- Create a trigger function +CREATE OR REPLACE FUNCTION copy_data_trigger_function() +RETURNS TRIGGER AS +' +BEGIN + -- If the legacy column has data but the new column does not, sync forward + IF NEW.kms_encoded IS NOT NULL AND NEW.kms_encoded_value IS NULL THEN + NEW.kms_encoded_value := NEW.kms_encoded; + + -- If the new column has data but the legacy column does not, sync backward + ELSIF NEW.kms_encoded_value IS NOT NULL AND NEW.kms_encoded IS NULL THEN + NEW.kms_encoded := NEW.kms_encoded_value; + + END IF; + + RETURN NEW; +END; +' +LANGUAGE plpgsql; + +-- Create a trigger +CREATE TRIGGER copy_data_trigger +BEFORE INSERT OR UPDATE ON ranger_keystore +FOR EACH ROW +EXECUTE PROCEDURE copy_data_trigger_function(); diff --git a/liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_column.xml b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_column.xml new file mode 100644 index 0000000000..e3cac8ad1a --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_column.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_trigger.xml b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_trigger.xml new file mode 100644 index 0000000000..3675dd16a1 --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/cleanup_kms_trigger.xml @@ -0,0 +1,33 @@ + + + + + + + DROP TRIGGER IF EXISTS copy_data_trigger ON ranger_keystore; + + empty + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/kms/db/test_3.x/db_javapatches.xml b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/db_javapatches.xml new file mode 100644 index 0000000000..5d14052eda --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/db_javapatches.xml @@ -0,0 +1,27 @@ + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/kms/db/test_3.x/kms_rename_col_zdu.xml b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/kms_rename_col_zdu.xml new file mode 100644 index 0000000000..c6c1ef98e1 --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/kms_rename_col_zdu.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + update ranger_keystore set kms_encoded_value = kms_encoded; + + empty + + + + + + + + DROP TRIGGER IF EXISTS copy_data_trigger ON ranger_keystore; + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/kms/db/test_3.x/master-finalize-test_3.x.xml b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/master-finalize-test_3.x.xml new file mode 100644 index 0000000000..9605a986cb --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/master-finalize-test_3.x.xml @@ -0,0 +1,29 @@ + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/kms/db/test_3.x/master-upgrade-test_3.x.xml b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/master-upgrade-test_3.x.xml new file mode 100644 index 0000000000..83f233102b --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/master-upgrade-test_3.x.xml @@ -0,0 +1,29 @@ + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/kms/db/test_3.x/sleep.xml b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/sleep.xml new file mode 100644 index 0000000000..98af0c905f --- /dev/null +++ b/liquibase-database-upgrade/changelogs/kms/db/test_3.x/sleep.xml @@ -0,0 +1,37 @@ + + + + + + select pg_sleep(60); + + + select SLEEP(60); + + empty + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/ranger/db/db.changelog-master-finalize.xml b/liquibase-database-upgrade/changelogs/ranger/db/db.changelog-master-finalize.xml new file mode 100644 index 0000000000..833b19e573 --- /dev/null +++ b/liquibase-database-upgrade/changelogs/ranger/db/db.changelog-master-finalize.xml @@ -0,0 +1,29 @@ + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/ranger/db/db.changelog-master-upgrade.xml b/liquibase-database-upgrade/changelogs/ranger/db/db.changelog-master-upgrade.xml new file mode 100644 index 0000000000..de256c7db9 --- /dev/null +++ b/liquibase-database-upgrade/changelogs/ranger/db/db.changelog-master-upgrade.xml @@ -0,0 +1,30 @@ + + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/add_xgroup_column.xml b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/add_xgroup_column.xml new file mode 100644 index 0000000000..1c168b9eaa --- /dev/null +++ b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/add_xgroup_column.xml @@ -0,0 +1,33 @@ + + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/db_javapatches_ozone.xml b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/db_javapatches_ozone.xml new file mode 100644 index 0000000000..d0c13d7a2f --- /dev/null +++ b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/db_javapatches_ozone.xml @@ -0,0 +1,27 @@ + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/master-finalize-test_3.x.xml b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/master-finalize-test_3.x.xml new file mode 100644 index 0000000000..622e931313 --- /dev/null +++ b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/master-finalize-test_3.x.xml @@ -0,0 +1,26 @@ + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/master-upgrade-test_3.x.xml b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/master-upgrade-test_3.x.xml new file mode 100644 index 0000000000..ab30ede327 --- /dev/null +++ b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/master-upgrade-test_3.x.xml @@ -0,0 +1,29 @@ + + + + + + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/sleep.xml b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/sleep.xml new file mode 100644 index 0000000000..2847a88cd2 --- /dev/null +++ b/liquibase-database-upgrade/changelogs/ranger/db/test_3.x/sleep.xml @@ -0,0 +1,34 @@ + + + + + + select pg_sleep(60); + + empty + + + \ No newline at end of file diff --git a/liquibase-database-upgrade/pom.xml b/liquibase-database-upgrade/pom.xml new file mode 100644 index 0000000000..3a8036e953 --- /dev/null +++ b/liquibase-database-upgrade/pom.xml @@ -0,0 +1,245 @@ + + + + 4.0.0 + + + org.apache.ranger + ranger + 3.0.0-SNAPSHOT + + ranger-liquibase-upgrade + jar + Ranger Liquibase Upgrade + Ranger Liquibase Database Upgrade + + + true + false + 8 + 8 + UTF-8 + + + + + com.beust + jcommander + ${jcommander.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j2.version} + + + org.apache.ranger + embeddedwebserver + 3.0.0-SNAPSHOT + + + org.apache.ranger + ranger-kms + 3.0.0-SNAPSHOT + + + org.apache.ranger + security-admin-web + 3.0.0-SNAPSHOT + classes + compile + + + org.junit.jupiter + junit-jupiter-api + ${junit.jupiter.version} + test + + + org.junit.vintage + junit-vintage-engine + ${junit.jupiter.version} + test + + + org.liquibase + liquibase-core + ${liquibase.version} + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.mockito + mockito-inline + ${mockito.version} + test + + + org.mockito + mockito-junit-jupiter + ${mockito.version} + test + + + org.slf4j + jul-to-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.springframework + spring-aop + ${springframework.version} + + + org.springframework + spring-beans + ${springframework.version} + + + org.springframework + spring-context + ${springframework.version} + + + org.springframework + spring-core + ${springframework.version} + + + org.springframework + spring-expression + ${springframework.version} + + + + + + + + com.github.ekryd.sortpom + sortpom-maven-plugin + 3.0.1 + + groupId + groupId + false + + + + + verify + + + validate + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + default-testCompile + + testCompile + + test-compile + + + **/org/apache/ranger/db/upgrade/patches/** + **/KmsConfigProvider.java + **/RangerConfigProvider.java + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + exclude-from-test + + analyze-only + + test-compile + + false + org.apache.ranger:security-admin-web:jar:classes:3.0.0-SNAPSHOT + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-changelogs + + copy-resources + + validate + + target/changelogs + + + ${basedir}/changelogs + + + + + + copy-scripts + + copy-resources + + validate + + target/scripts + + + ${basedir}/scripts + + *.sh + + + + + + + + + + diff --git a/liquibase-database-upgrade/scripts/README.md b/liquibase-database-upgrade/scripts/README.md new file mode 100644 index 0000000000..2a90215350 --- /dev/null +++ b/liquibase-database-upgrade/scripts/README.md @@ -0,0 +1,66 @@ + +# KMS +## Sample exports and commands +``` +export JAVA_HOME=/usr/java/jdk1.8.0 +export RANGER_KMS_HOME=/var/lib/ranger-kms +#get process folder for kms: ps -ef | grep ranger-kms +export RANGER_KMS_CONF=/var/run/process/1546336381-ranger_kms-RANGER_KMS_SERVER/conf +export SQL_CONNECTOR_JAR=/usr/share/java/postgresql-connector-java.jar +export LIQUI_SEARCH_PATH=/root/ranger-3.0.0-SNAPSHOT-liquibase-upgrade/lib +#get HADOOP_CREDSTORE_PASSWORD: cat /var/run/process/1546336381-ranger_kms-RANGER_KMS_SERVER/proc.json | grep HADOOP +export HADOOP_CREDSTORE_PASSWORD=eo7im023ga7kv6ttbaj7y7a41 +export LOG4J_PROPERTIES_FILE_PATH=/root/ranger-3.0.0-SNAPSHOT-liquibase-upgrade/sample_resources/kms_log4j.properties +===== +bash upgrade_kms_db.sh kms update 3.x +===== +bash upgrade_kms_db.sh kms finalize 3.x +===== +bash upgrade_kms_db.sh kms rollback 3.x +===== +This prints 2 variables OP_STATUS and IS_UPGRADE_COMPLETE +bash kms_liquibase_utils.sh kms isUpgradeComplete 7.3.2 +===== +This prints 2 variables OP_STATUS and IS_FINALIZE_COMPLETE +bash kms_liquibase_utils.sh kms isFinalizeComplete 3.x +===== + +``` + +# Ranger +## Sample exports and commands +``` +#FOR RANGER + +export JAVA_HOME_RANGER=/usr/java/jdk1.8.0 +export RANGER_HOME=/var/lib/ranger-admin +export RANGER_CONF=/var/process/1546336029-ranger-RANGER_ADMIN/conf +export SQL_CONNECTOR_JAR_RANGER=/usr/share/java/postgresql-connector-java.jar +export LIQUI_SEARCH_PATH_RANGER=/root/ranger-3.0.0-SNAPSHOT-liquibase-upgrade +#export HADOOP_CREDSTORE_PASSWORD= +export HADOOP_CREDSTORE_PASSWORD=8ezax7470bysu6ng0a3qghfzd +export LOG4J_PROPERTIES_FILE_PATH_RANGER=/root/ranger-3.0.0-SNAPSHOT-liquibase-upgrade/sample_resources/ranger_log4j.properties + +===== +bash upgrade_ranger_db.sh ranger update 3.x +===== +bash upgrade_ranger_db.sh ranger finalize 3.x +===== +bash upgrade_ranger_db.sh ranger rollback 3.x +===== +``` diff --git a/liquibase-database-upgrade/scripts/kms_liquibase_utils.sh b/liquibase-database-upgrade/scripts/kms_liquibase_utils.sh new file mode 100644 index 0000000000..02893710cb --- /dev/null +++ b/liquibase-database-upgrade/scripts/kms_liquibase_utils.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- + +if [ -z "${JAVA_HOME}" ]; then + echo "PLEASE EXPORT VARIABLE JAVA_HOME" +exit; +else + echo "JAVA_HOME : "$JAVA_HOME +fi + +if [ -z "${RANGER_KMS_HOME}" ]; then + echo "PLEASE EXPORT VARIABLE RANGER_KMS_HOME" +exit; +else + echo "RANGER_KMS_HOME : "$RANGER_KMS_HOME +fi + +if [ -z "${RANGER_KMS_CONF}" ]; then + echo "PLEASE EXPORT VARIABLE RANGER_KMS_CONF" +exit; +else + echo "RANGER_KMS_CONF : "$RANGER_KMS_CONF +fi + +if [ -z "${SQL_CONNECTOR_JAR}" ]; then + echo "PLEASE EXPORT VARIABLE SQL_CONNECTOR_JAR" +exit; +else + echo "SQL_CONNECTOR_JAR : "$SQL_CONNECTOR_JAR +fi + +if [ -z "${HADOOP_CREDSTORE_PASSWORD}" ]; then + echo "PLEASE EXPORT VARIABLE HADOOP_CREDSTORE_PASSWORD" +exit; +else + echo "HADOOP_CREDSTORE_PASSWORD : "$HADOOP_CREDSTORE_PASSWORD +fi + + +if [ -z "${LIQUI_SEARCH_PATH}" ]; then + echo "PLEASE EXPORT VARIABLE LIQUI_SEARCH_PATH" +exit; +else + echo "LIQUI_SEARCH_PATH : "$LIQUI_SEARCH_PATH +fi + + +if [ -z "${LOG4J_PROPERTIES_FILE_PATH}" ]; then + echo "PLEASE EXPORT VARIABLE LOG4J_PROPERTIES_FILE_PATH" +exit; +else + echo "LOG4J_PROPERTIES_FILE_PATH : "$LOG4J_PROPERTIES_FILE_PATH +fi + +set -x + +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_CONF}:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${SQL_CONNECTOR_JAR}:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF:${RANGER_KMS_CONF}/*:${LIQUI_SEARCH_PATH}/*:${LIQUI_SEARCH_PATH}" + +output=$(${JAVA_HOME}/bin/java -Dlog4j.configurationFile=file:${LOG4J_PROPERTIES_FILE_PATH} -cp "${cp}" org.apache.ranger.db.upgrade.LiquibaseDBUtilsMain -serviceName ${1} -op ${2} -tag ${3}) + +json_output=$(echo "$output" | tail -n 1) + +OP_STATUS=$(echo "$json_output" | jq -r '.op_status // ""') +IS_FINALIZE_COMPLETE=$(echo "$json_output" | jq -r '.isFinalizeComplete // ""') +IS_UPGRADE_COMPLETE=$(echo "$json_output" | jq -r '.isUpgradeComplete // ""') + +set +x + +if [ -n "$OP_STATUS" ]; then + echo "OP_STATUS=$OP_STATUS" +fi + +if [ -n "$IS_UPGRADE_COMPLETE" ]; then + echo "IS_UPGRADE_COMPLETE=$IS_UPGRADE_COMPLETE" +fi + +if [ -n "$IS_FINALIZE_COMPLETE" ]; then + echo "IS_FINALIZE_COMPLETE=$IS_FINALIZE_COMPLETE" +fi diff --git a/liquibase-database-upgrade/scripts/upgrade_kms_db.sh b/liquibase-database-upgrade/scripts/upgrade_kms_db.sh new file mode 100644 index 0000000000..16fff98277 --- /dev/null +++ b/liquibase-database-upgrade/scripts/upgrade_kms_db.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +set -x + +if [ -z "${JAVA_HOME}" ]; then + echo "PLEASE EXPORT VARIABLE JAVA_HOME" +exit; +else + echo "JAVA_HOME : "$JAVA_HOME +fi + +if [ -z "${RANGER_KMS_HOME}" ]; then + echo "PLEASE EXPORT VARIABLE RANGER_KMS_HOME" +exit; +else + echo "RANGER_KMS_HOME : "$RANGER_KMS_HOME +fi + +if [ -z "${RANGER_KMS_CONF}" ]; then + echo "PLEASE EXPORT VARIABLE RANGER_KMS_CONF" +exit; +else + echo "RANGER_KMS_CONF : "$RANGER_KMS_CONF +fi + +if [ -z "${SQL_CONNECTOR_JAR}" ]; then + echo "PLEASE EXPORT VARIABLE SQL_CONNECTOR_JAR" +exit; +else + echo "SQL_CONNECTOR_JAR : "$SQL_CONNECTOR_JAR +fi + +if [ -z "${HADOOP_CREDSTORE_PASSWORD}" ]; then + echo "PLEASE EXPORT VARIABLE HADOOP_CREDSTORE_PASSWORD" +exit; +else + echo "HADOOP_CREDSTORE_PASSWORD : "$HADOOP_CREDSTORE_PASSWORD +fi + + +if [ -z "${LIQUI_SEARCH_PATH}" ]; then + echo "PLEASE EXPORT VARIABLE LIQUI_SEARCH_PATH" +exit; +else + echo "LIQUI_SEARCH_PATH : "$LIQUI_SEARCH_PATH +fi + + +if [ -z "${LOG4J_PROPERTIES_FILE_PATH}" ]; then + echo "PLEASE EXPORT VARIABLE LOG4J_PROPERTIES_FILE_PATH" +exit; +else + echo "LOG4J_PROPERTIES_FILE_PATH : "$LOG4J_PROPERTIES_FILE_PATH +fi + +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_CONF}:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${SQL_CONNECTOR_JAR}:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF:${RANGER_KMS_CONF}/*:${LIQUI_SEARCH_PATH}/*:${LIQUI_SEARCH_PATH}" + +${JAVA_HOME}/bin/java -Dlog4j.configurationFile=file:${LOG4J_PROPERTIES_FILE_PATH} -cp "${cp}" org.apache.ranger.db.upgrade.LiquibaseUpdateDriverMain -serviceName ${1} -op ${2} -tag ${3} \ No newline at end of file diff --git a/liquibase-database-upgrade/scripts/upgrade_ranger_db.sh b/liquibase-database-upgrade/scripts/upgrade_ranger_db.sh new file mode 100644 index 0000000000..44ea693045 --- /dev/null +++ b/liquibase-database-upgrade/scripts/upgrade_ranger_db.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +set -x + +if [ -z "${JAVA_HOME_RANGER}" ]; then + echo "PLEASE EXPORT VARIABLE JAVA_HOME_RANGER" +exit; +else + echo "JAVA_HOME_RANGER : "$JAVA_HOME_RANGER +fi + +if [ -z "${RANGER_HOME}" ]; then + echo "PLEASE EXPORT VARIABLE RANGER_HOME" +exit; +else + echo "RANGER_HOME : "$RANGER_HOME +fi + +if [ -z "${RANGER_CONF}" ]; then + echo "PLEASE EXPORT VARIABLE RANGER_CONF" +exit; +else + echo "RANGER_CONF : "$RANGER_CONF +fi + +if [ -z "${SQL_CONNECTOR_JAR_RANGER}" ]; then + echo "PLEASE EXPORT VARIABLE SQL_CONNECTOR_JAR_RANGER" +exit; +else + echo "SQL_CONNECTOR_JAR_RANGER : "$SQL_CONNECTOR_JAR_RANGER +fi + +if [ -z "${HADOOP_CREDSTORE_PASSWORD}" ]; then + echo "PLEASE EXPORT VARIABLE HADOOP_CREDSTORE_PASSWORD" +exit; +else + echo "HADOOP_CREDSTORE_PASSWORD : "$HADOOP_CREDSTORE_PASSWORD +fi + + +if [ -z "${LIQUI_SEARCH_PATH_RANGER}" ]; then + echo "PLEASE EXPORT VARIABLE LIQUI_SEARCH_PATH_RANGER" +exit; +else + echo "LIQUI_SEARCH_PATH_RANGER : "$LIQUI_SEARCH_PATH_RANGER +fi + + +if [ -z "${LOG4J_PROPERTIES_FILE_PATH_RANGER}" ]; then + echo "PLEASE EXPORT VARIABLE LOG4J_PROPERTIES_FILE_PATH_RANGER" +exit; +else + echo "LOG4J_PROPERTIES_FILE_PATH_RANGER : "$LOG4J_PROPERTIES_FILE_PATH_RANGER +fi + +cp="${RANGER_HOME}/cred/lib/*:${RANGER_CONF}:${RANGER_HOME}/ews/webapp/WEB-INF/classes/:${RANGER_HOME}/ews/webapp/WEB-INF/classes/META-INF:${RANGER_HOME}/ews/webapp/WEB-INF/lib/*:${SQL_CONNECTOR_JAR_RANGER}:${RANGER_HOME}/conf:${RANGER_HOME}/ews/lib/*:${RANGER_HOME}/ews/webapp/libs/*:${RANGER_HOME}/ews/webapp/META-INF/:${RANGER_HOME}/ews/ranger_jaas/*:${RANGER_CONF}/*:${LIQUI_SEARCH_PATH_RANGER}/*:${LIQUI_SEARCH_PATH_RANGER}/lib/*:${LIQUI_SEARCH_PATH_RANGER}/lib/:${RANGER_HOME}/ews/webapp/WEB-INF/:${RANGER_HOME}/ews/webapp/WEB-INF/classes/lib/*:${RANGER_HOME}/ews/webapp/WEB-INF/conf" + +${JAVA_HOME_RANGER}/bin/java -DapplicationContext.files=applicationContext.xml,security-applicationContext.xml,asynctask-applicationContext.xml -Dlog4j.configurationFile=file:${LOG4J_PROPERTIES_FILE_PATH_RANGER} -cp "${cp}" org.apache.ranger.db.upgrade.LiquibaseUpdateDriverMain -serviceName ${1} -op ${2} -tag ${3} \ No newline at end of file diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ArgsParserLiquibaseDBUtilsMain.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ArgsParserLiquibaseDBUtilsMain.java new file mode 100644 index 0000000000..5ab6d1b874 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ArgsParserLiquibaseDBUtilsMain.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.db.upgrade; + +import com.beust.jcommander.IParameterValidator; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; + +import java.util.HashSet; +import java.util.Set; + +public class ArgsParserLiquibaseDBUtilsMain { + @Parameter(names = "-serviceName", description = "The name of the service for which db upgrade is required)") + public String serviceName; + @Parameter(names = "-tag", description = "The current version of the cluster (user for driving upgrades/rollbacks/other commands)") + public String tagName; + @Parameter(names = "-op", description = "The operation to do on the cluster (e.g. isUpgradeComplete/isFinalizeComplete)", validateWith = ArgsParserLiquibaseDBUtilsMain.SupportedOps.class) + public String opName; + + public static class SupportedOps implements IParameterValidator { + @Override + public void validate(String name, String value) throws ParameterException { + Set validOps = new HashSet() {{ + add("isUpgradeComplete"); + add("isFinalizeComplete"); + add("releaseLock"); + }}; + if (!validOps.contains(value)) { + throw new ParameterException(value + " is not a supported operation." + " Valid operations are[" + validOps + "]"); + } + } + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ArgsParserLiquibaseUpdateDriverMain.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ArgsParserLiquibaseUpdateDriverMain.java new file mode 100644 index 0000000000..fd879954a6 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ArgsParserLiquibaseUpdateDriverMain.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import com.beust.jcommander.IParameterValidator; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; + +import java.util.HashSet; +import java.util.Set; + +class ArgsParserLiquibaseUpdateDriverMain { + @Parameter(names = "-serviceName", description = "The name of the service for which db upgrade is required)") + public String serviceName; + @Parameter(names = "-tag", description = "The current version of the cluster (user for driving upgrades/rollbacks/other commands)") + public String tagName; + @Parameter(names = "-op", description = "The operation to do on the cluster (e.g. update/rollback/finalize)", validateWith = SupportedOps.class) + public String opName; + + public static class SupportedOps implements IParameterValidator { + @Override + public void validate(String name, String value) throws ParameterException { + Set validOps = new HashSet() {{ + add("update"); + add("rollback"); + add("finalize"); + }}; + if (!validOps.contains(value)) { + throw new ParameterException(value + " is not a supported operation"); + } + } + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/Constants.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/Constants.java new file mode 100644 index 0000000000..fc8f423cbd --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/Constants.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +public class Constants { + private Constants() { + } + + //Constants for KMS + public static final String KMS_UPGRADE_CHANGELOG_RELATIVE_PATH = "kms/db/db.changelog-master-upgrade.xml"; + public static final String KMS_FINALIZE_CHANGELOG_RELATIVE_PATH = "kms/db/db.changelog-master-finalize.xml"; + + //Constants for Ranger + public static final String RANGER_UPGRADE_CHANGELOG_RELATIVE_PATH = "ranger/db/db.changelog-master-upgrade.xml"; + public static final String RANGER_FINALIZE_CHANGELOG_RELATIVE_PATH = "ranger/db/db.changelog-master-finalize.xml"; + + public static final String TAG_POST_UPGRADE_POSTFIX = "_postupdate"; + public static final String TAG_FINALIZE_POSTFIX = "_finalized"; + + public static final int RETURN_CODE_SUCCESS = 1; + public static final int RETURN_CODE_FAILURE = 0; +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ICommandDriver.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ICommandDriver.java new file mode 100644 index 0000000000..c3a512d327 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/ICommandDriver.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import liquibase.command.CommandResults; +import liquibase.exception.CommandExecutionException; + +public interface ICommandDriver { + CommandResults tag(String serviceName, String tagName) throws CommandExecutionException; + + CommandResults rollback(String serviceName, String tagName, String changelogFilename) throws CommandExecutionException; + + CommandResults tagExists(String serviceName, String tagName) throws CommandExecutionException; + + CommandResults update(String serviceName, String changelogFilename) throws CommandExecutionException; + + CommandResults status(String serviceName, String changelogFilename) throws CommandExecutionException; +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/IConfigProvider.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/IConfigProvider.java new file mode 100644 index 0000000000..55f669bfe0 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/IConfigProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +public interface IConfigProvider { + String getUrl(); + + String getUsername(); + + String getPassword(); + + String getDriver(); + + String getMasterChangelogRelativePath(); + + String getFinalizeChangelogRelativePath(); +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/KmsConfigProvider.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/KmsConfigProvider.java new file mode 100644 index 0000000000..4ac5ce4a9b --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/KmsConfigProvider.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import org.apache.hadoop.crypto.key.RangerKeyStoreProvider; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Component("kms") +@Lazy +public class KmsConfigProvider implements IConfigProvider { + @Override + public String getUrl() { + return RangerKeyStoreProvider.getDBKSConf().get(RangerKeyStoreProvider.DB_URL); + } + + @Override + public String getUsername() { + return RangerKeyStoreProvider.getDBKSConf().get(RangerKeyStoreProvider.DB_USER); + } + + @Override + public String getPassword() { + return RangerKeyStoreProvider.getDBKSConf().get(RangerKeyStoreProvider.DB_PASSWORD); + } + + @Override + public String getDriver() { + return RangerKeyStoreProvider.getDBKSConf().get(RangerKeyStoreProvider.DB_DRIVER); + } + + @Override + public String getMasterChangelogRelativePath() { + return Constants.KMS_UPGRADE_CHANGELOG_RELATIVE_PATH; + } + + @Override + public String getFinalizeChangelogRelativePath() { + return Constants.KMS_FINALIZE_CHANGELOG_RELATIVE_PATH; + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseCommandDriver.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseCommandDriver.java new file mode 100644 index 0000000000..25d96054a9 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseCommandDriver.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import liquibase.command.CommandResults; +import liquibase.command.CommandScope; +import liquibase.exception.CommandExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.HashMap; + +@Component +public class LiquibaseCommandDriver implements ICommandDriver { + private static final Logger LOG = LoggerFactory.getLogger(LiquibaseCommandDriver.class); + @Autowired + LiquibasePropertiesFactory factory; + + @Override + public CommandResults tag(String serviceName, String tagName) throws CommandExecutionException { + HashMap tagParams = new HashMap<>(); + tagParams.put("tag", tagName); + return executeLiquibaseCommand(serviceName, "tag", tagParams); + } + + @Override + public CommandResults rollback(String serviceName, String tagName, String changelogFilename) + throws CommandExecutionException { + HashMap rollbackParams = new HashMap<>(); + rollbackParams.put("tag", tagName); + rollbackParams.put("changelogFile", changelogFilename); + return executeLiquibaseCommand(serviceName, "rollback", rollbackParams); + } + + @Override + public CommandResults tagExists(String serviceName, String tagName) throws CommandExecutionException { + HashMap tagParams = new HashMap<>(); + tagParams.put("tag", tagName); + return executeLiquibaseCommand(serviceName, "tagExists", tagParams); + } + + @Override + public CommandResults update(String serviceName, String changelogFilename) throws CommandExecutionException { + HashMap updateParams = new HashMap<>(); + updateParams.put("changelogFile", changelogFilename); + return executeLiquibaseCommand(serviceName, "update", updateParams); + } + + @Override + public CommandResults status(String serviceName, String changelogFilename) throws CommandExecutionException { + HashMap statusParams = new HashMap<>(); + statusParams.put("changelogFile", changelogFilename); + return executeLiquibaseCommand(serviceName, "status", statusParams); + } + + private CommandResults executeLiquibaseCommand( + String serviceName, + String commandName, + HashMap commandSpecificArguments + ) throws CommandExecutionException { + if (factory == null) { + LOG.error("LiquibasePropertiesFactory is null, liquibase command cannot execute"); + } + String url = factory.getUrl(serviceName); + String username = factory.getUsername(serviceName); + String password = factory.getPassword(serviceName); + String driver = factory.getDriver(serviceName); + CommandScope commandScope = new CommandScope(commandName); + commandScope.addArgumentValue("url", url); + commandScope.addArgumentValue("username", username); + commandScope.addArgumentValue("password", password); + commandScope.addArgumentValue("driver", driver); + for (String argumentName : commandSpecificArguments.keySet()) { + commandScope.addArgumentValue(argumentName, commandSpecificArguments.get(argumentName)); + } + return commandScope.execute(); + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseCommandResultsUtil.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseCommandResultsUtil.java new file mode 100644 index 0000000000..402a26bfb5 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseCommandResultsUtil.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import liquibase.command.CommandResults; +import liquibase.report.OperationInfo; +import liquibase.report.UpdateReportParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LiquibaseCommandResultsUtil { + private static final Logger LOG = LoggerFactory.getLogger(LiquibaseCommandResultsUtil.class); + + private LiquibaseCommandResultsUtil() { + } + + public static boolean doesTagExist(CommandResults tagExistsResult) { + boolean res = (Boolean) tagExistsResult.getResult("tagExistsResult"); + LOG.info("doesTagExist({})={}", tagExistsResult, res); + return res; + } + + public static boolean isUpdateSuccessful(CommandResults updateResult) { + /* + TODO: revisit this + */ + boolean operationInfoResult = false; + Object operationInfoObj = updateResult.getResult("operationInfo"); + + if (operationInfoObj instanceof OperationInfo) { + operationInfoResult = "success".equals(((OperationInfo) operationInfoObj).getOperationOutcome()); + } + + boolean updateReportResult = false; + Object updateReportObj = updateResult.getResult("updateReport"); + + if (updateReportObj instanceof UpdateReportParameters) { + updateReportResult = Boolean.TRUE.equals(((UpdateReportParameters) updateReportObj).getSuccess()); + } + + return operationInfoResult || updateReportResult; + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseDBUtilsMain.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseDBUtilsMain.java new file mode 100644 index 0000000000..d433da9004 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseDBUtilsMain.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.db.upgrade; + +import com.beust.jcommander.JCommander; +import com.google.gson.Gson; +import liquibase.Scope; +import liquibase.command.CommandResults; +import liquibase.resource.ClassLoaderResourceAccessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.bridge.SLF4JBridgeHandler; + +import java.util.HashMap; + +public class LiquibaseDBUtilsMain { + private static final Logger LOG = LoggerFactory.getLogger(LiquibaseDBUtilsMain.class); + + private LiquibaseDBUtilsMain() { + } + + public static void main(String[] argv) { + ArgsParserLiquibaseDBUtilsMain argsParserLiquibaseDBUtilsMain = new ArgsParserLiquibaseDBUtilsMain(); + + JCommander.newBuilder() + .addObject(argsParserLiquibaseDBUtilsMain) + .build() + .parse(argv); + + String op = argsParserLiquibaseDBUtilsMain.opName; + String tagName = argsParserLiquibaseDBUtilsMain.tagName; + String serviceName = argsParserLiquibaseDBUtilsMain.serviceName; + ICommandDriver commandDriver = SpringContext.getBean(ICommandDriver.class); + HashMap response = new HashMap<>(); + response.put("op_status", Constants.RETURN_CODE_FAILURE); + try { + Scope.child(Scope.Attr.resourceAccessor, new ClassLoaderResourceAccessor(), () -> { + switch (op) { + case "isUpgradeComplete": + final String tagNamePostUpdate = Utils.getPostUpdateTagName(tagName); + CommandResults upgradeCompleteTagExistsResult = commandDriver.tagExists(serviceName, tagNamePostUpdate); + if (LiquibaseCommandResultsUtil.doesTagExist(upgradeCompleteTagExistsResult)) { + LOG.info("isUpgradeComplete={}", Constants.RETURN_CODE_SUCCESS); + response.put("op_status", Constants.RETURN_CODE_SUCCESS); + response.put("isUpgradeComplete", Constants.RETURN_CODE_SUCCESS); + } else { + LOG.info("isUpgradeComplete={}", Constants.RETURN_CODE_FAILURE); + response.put("op_status", Constants.RETURN_CODE_SUCCESS); + response.put("isUpgradeComplete", Constants.RETURN_CODE_FAILURE); + } + break; + case "isFinalizeComplete": + final String tagNameForFinalizeComplete = Utils.getFinalizeTagName(tagName); + CommandResults finalizeCompleteTagExistsResult = commandDriver.tagExists(serviceName, tagNameForFinalizeComplete); + if (LiquibaseCommandResultsUtil.doesTagExist(finalizeCompleteTagExistsResult)) { + LOG.info("isFinalizeComplete={}", Constants.RETURN_CODE_SUCCESS); + response.put("op_status", Constants.RETURN_CODE_SUCCESS); + response.put("isFinalizeComplete", Constants.RETURN_CODE_SUCCESS); + } else { + LOG.info("isFinalizeComplete={}", Constants.RETURN_CODE_FAILURE); + response.put("op_status", Constants.RETURN_CODE_SUCCESS); + response.put("isFinalizeComplete", Constants.RETURN_CODE_FAILURE); + } + break; + /* + TODO: Implement releaseLock here. Workaround: execute command manually or use sql to change value in databasechangeloglock table manually + Required in some cases when liquibase does not automatically release lock incase of unclean exit of the process + case "releaseLock": + throw new NotImplementedException("releaseLock not yet implemented"); + */ + default: + throw new UnsupportedOperationException("op=" + op + " is not valid"); + } + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // this sysout should be the last printed line (assumption only for the bash script consuming the returned result, otherwise not required) + System.out.println(new Gson().toJson(response)); + } + + static { + // Liquibase logs to java.util.logging + // Below statements ensures that the jul-to-slf4j binding works properly + // Remove existing handlers attached to j.u.l root logger + SLF4JBridgeHandler.removeHandlersForRootLogger(); + // Bridge j.u.l logging to SLF4J + SLF4JBridgeHandler.install(); + + //initialize spring beans + SpringContext.init(); + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibasePropertiesFactory.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibasePropertiesFactory.java new file mode 100644 index 0000000000..75bd0c3f7d --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibasePropertiesFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +import java.util.Map; + +@Component +public class LiquibasePropertiesFactory { + private static final Logger LOG = LoggerFactory.getLogger(LiquibasePropertiesFactory.class); + + Map configProviders; + + @Autowired + public LiquibasePropertiesFactory(@Lazy Map configProviders) { + this.configProviders = configProviders; + } + + private IConfigProvider getProvider(String serviceName) { + IConfigProvider provider = configProviders.get(serviceName); + if (provider == null) { + LOG.error("ConfigProvider for service {} is null", serviceName); + throw new IllegalArgumentException("ConfigProvider for service " + serviceName + " is null"); + } + return provider; + } + + public String getUrl(String serviceName) { + return getProvider(serviceName).getUrl(); + } + + public String getUsername(String serviceName) { + return getProvider(serviceName).getUsername(); + } + + public String getPassword(String serviceName) { + return getProvider(serviceName).getPassword(); + } + + public String getDriver(String serviceName) { + return getProvider(serviceName).getDriver(); + } + + public String getMasterChangelog(String serviceName) { + return getProvider(serviceName).getMasterChangelogRelativePath(); + } + + public String getFinalizeChangelog(String serviceName) { + return getProvider(serviceName).getFinalizeChangelogRelativePath(); + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseUpdateDriverMain.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseUpdateDriverMain.java new file mode 100644 index 0000000000..e96247129f --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/LiquibaseUpdateDriverMain.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import com.beust.jcommander.JCommander; +import liquibase.Scope; +import liquibase.command.CommandResults; +import liquibase.exception.CommandExecutionException; +import liquibase.resource.ClassLoaderResourceAccessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.bridge.SLF4JBridgeHandler; + +public class LiquibaseUpdateDriverMain { + private static final Logger LOG = LoggerFactory.getLogger(LiquibaseUpdateDriverMain.class); + + private LiquibaseUpdateDriverMain() { + } + + public static void main(String[] argv) throws Exception { + ArgsParserLiquibaseUpdateDriverMain argsParserLiquibaseUpdateDriverMain = new ArgsParserLiquibaseUpdateDriverMain(); + + JCommander.newBuilder() + .addObject(argsParserLiquibaseUpdateDriverMain) + .build() + .parse(argv); + + String serviceName = argsParserLiquibaseUpdateDriverMain.serviceName; + + LiquibasePropertiesFactory factory = SpringContext.getBean(LiquibasePropertiesFactory.class); + if (factory == null) { + throw new RuntimeException("Could not initialize LiquibasePropertiesFactory. Exiting."); + } + final String masterChangelogFilenameUpgrade = factory.getMasterChangelog(serviceName); + final String masterChangelogFilenameFinalize = factory.getFinalizeChangelog(serviceName); + final String tagName = argsParserLiquibaseUpdateDriverMain.tagName; + final String tagNamePostUpdate = Utils.getPostUpdateTagName(tagName); + final String tagNameForFinalizeComplete = Utils.getFinalizeTagName(tagName); + LOG.info("tagName={}, tagNamePostUpdate={}, tagNameForFinalizeComplete={}", tagName, tagNamePostUpdate, tagNameForFinalizeComplete); + LOG.info("masterChangelogFilenameUpgrade={}, masterChangelogFilenameFinalize={}", masterChangelogFilenameUpgrade, masterChangelogFilenameFinalize); + StatusCheckScheduledExecutorService updateStatusCheckService = SpringContext.getBean( + StatusCheckScheduledExecutorService.class); + if (updateStatusCheckService == null) { + LOG.error("Could not initialize updateStatusCheckService but continuing execution"); + } + ICommandDriver commandDriver = SpringContext.getBean(ICommandDriver.class); + if (commandDriver == null) { + throw new RuntimeException("Could not initialize ICommandDriver bean. Exiting."); + } + LOG.info("Main_commandDriverClass={}", commandDriver.getClass()); + try { + Scope.child(Scope.Attr.resourceAccessor, new ClassLoaderResourceAccessor(), () -> { + switch (argsParserLiquibaseUpdateDriverMain.opName) { + case "rollback": + CommandResults result = commandDriver.tagExists(serviceName, tagNameForFinalizeComplete); + LOG.info("finalize tag check result (for rollback): {} ", result.getResults()); + LOG.info("finalize tag check result (for rollback): {} ", result.getResult("tagExistsResult").equals("false")); + LOG.info("finalize tag check result (for rollback): {} ", result.getResult("tagExistsResult").getClass()); + if (LiquibaseCommandResultsUtil.doesTagExist(result)) { + throw new Exception("ATTEMPTING TO ROLLBACK AFTER FINALIZE WAS COMPLETED. ABORTING."); + } else { + LOG.info("Starting Rollback"); + result = commandDriver.rollback(serviceName, tagName, masterChangelogFilenameUpgrade); + LOG.info("rollback result: {}", result.getResults()); + } + break; + case "update": + if (updateStatusCheckService != null) { + updateStatusCheckService.startStatusChecks(serviceName, masterChangelogFilenameUpgrade); + } + result = commandDriver.tagExists(serviceName, tagNameForFinalizeComplete); + if (LiquibaseCommandResultsUtil.doesTagExist(result)) { + throw new Exception("ATTEMPTING TO ROLLBACK AFTER FINALIZE WAS COMPLETED. ABORTING."); + } else { + LOG.info("Starting Schema Upgrade"); + result = commandDriver.tag(serviceName, tagName); + LOG.info("tagging result: {}", result.getResults()); + result = commandDriver.update(serviceName, masterChangelogFilenameUpgrade); + LOG.info("update result: {}", result.getResults()); + if (!LiquibaseCommandResultsUtil.isUpdateSuccessful(result)) { + throw new CommandExecutionException( + "Issue occurred during execution of update command"); + } else { + result = commandDriver.tag(serviceName, tagNamePostUpdate); + LOG.info("tagging result post update: {}", result.getResults()); + LOG.info("Successfully upgraded {} database", serviceName); + } + } + break; + case "finalize": + result = commandDriver.tagExists(serviceName, tagNamePostUpdate); + LOG.info("tag check result (for finalize): {}", result.getResult("tagExistsResult")); + LOG.info("tag check result (for finalize): {}", result.getResult("tagExistsResult")); + + Boolean tagExists = LiquibaseCommandResultsUtil.doesTagExist(result); + LOG.info("tagExists={}", tagExists); + if (tagExists) { + LOG.info("Finalizing Schema Upgrade. Do not try rollback after this."); + result = commandDriver.update(serviceName, masterChangelogFilenameFinalize); + if (LiquibaseCommandResultsUtil.isUpdateSuccessful(result)) { + result = commandDriver.tag(serviceName, tagNameForFinalizeComplete); + LOG.info("tagging result after finalize: {}", result.getResults()); + LOG.info("Database upgrade finalized to: {}", tagNameForFinalizeComplete); + } else { + throw new CommandExecutionException("Issue occurred during execution of finalize command"); + } + } else { + throw new Exception("ATTEMPTING TO FINALIZE BEFORE UPDATE COMPLETED. ABORTING."); + } + break; + default: + throw new UnsupportedOperationException(argsParserLiquibaseUpdateDriverMain.opName + " not supported"); + } + }); + } catch (CommandExecutionException e) { + switch (argsParserLiquibaseUpdateDriverMain.opName) { + case "update": + LOG.info(e.getMessage()); + LOG.info("Starting automatic rollback since exception occurred during upgrade"); + CommandResults result; + try { + result = commandDriver.rollback(serviceName, tagName, masterChangelogFilenameUpgrade); + } catch (CommandExecutionException ex) { + LOG.info("Failed to auto rollback to the previous state. Manually restore using the database dump/backup as soon as possible"); + throw new CommandExecutionException(ex); + } + LOG.info("Auto rollback result: {}", result.getResults()); + break; + case "rollback": + LOG.info("Exception Occurred during rollback"); + break; + case "finalize": + LOG.info("Exception Occurred during finalize"); + break; + } + throw new CommandExecutionException(e); + } finally { + if (updateStatusCheckService != null) { + updateStatusCheckService.stopStatusChecks(); + } + LOG.info("=============================END OF UPGRADE EXECUTION============================="); + } + } + + static { + // Liquibase logs to java.util.logging + // Below statements ensures that the jul-to-slf4j binding works properly + // Remove existing handlers attached to j.u.l root logger + SLF4JBridgeHandler.removeHandlersForRootLogger(); + // Bridge j.u.l logging to SLF4J + SLF4JBridgeHandler.install(); + + //initialize spring beans + SpringContext.init(); + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/RangerConfigProvider.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/RangerConfigProvider.java new file mode 100644 index 0000000000..a087c02314 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/RangerConfigProvider.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.credentialapi.CredentialReader; +import org.apache.ranger.server.tomcat.EmbeddedServer; +import org.apache.ranger.server.tomcat.EmbeddedServerUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Component("ranger") +@Lazy +public class RangerConfigProvider implements IConfigProvider { + private static final Logger LOG = LoggerFactory.getLogger(RangerConfigProvider.class); + + @Override + public String getUrl() { + return EmbeddedServerUtil.getConfig("ranger.jpa.jdbc.url"); + } + + @Override + public String getUsername() { + return EmbeddedServerUtil.getConfig("ranger.jpa.jdbc.user"); + } + + @Override + public String getPassword() { + LOG.info("==> getPassword()"); + String providerPath = EmbeddedServerUtil.getConfig("ranger.credential.provider.path"); + String keyAlias = EmbeddedServerUtil.getConfig("ranger.jpa.jdbc.credential.alias", "keyStoreCredentialAlias"); + String storeType = EmbeddedServerUtil.getConfig("ranger.keystore.file.type", EmbeddedServer.RANGER_KEYSTORE_FILE_TYPE_DEFAULT); + String keystorePass = null; + if (providerPath != null && keyAlias != null) { + keystorePass = CredentialReader.getDecryptedString(providerPath.trim(), keyAlias.trim(), storeType.trim()); + if (StringUtils.isBlank(keystorePass) || "none".equalsIgnoreCase(keystorePass.trim())) { + keystorePass = EmbeddedServerUtil.getConfig("ranger.service.https.attrib.keystore.pass"); + } + } + LOG.info("<== getPassword()"); + return keystorePass; + } + + @Override + public String getDriver() { + return EmbeddedServerUtil.getConfig("ranger.jpa.jdbc.driver"); + } + + @Override + public String getMasterChangelogRelativePath() { + return Constants.RANGER_UPGRADE_CHANGELOG_RELATIVE_PATH; + } + + @Override + public String getFinalizeChangelogRelativePath() { + return Constants.RANGER_FINALIZE_CHANGELOG_RELATIVE_PATH; + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/SpringContext.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/SpringContext.java new file mode 100644 index 0000000000..33c956b913 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/SpringContext.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +public class SpringContext { + private static final Logger logger = LoggerFactory.getLogger(SpringContext.class); + private static ApplicationContext context; + + private SpringContext() { + } + + public static void init() { + if (context == null) { + logger.info("context is null, re-initializing from ApplicationContext files"); + String contextFilesString = System.getProperty("applicationContext.files"); + if (contextFilesString != null) { + logger.info("ApplicationContext Files={}", contextFilesString); + String[] contextFiles = contextFilesString.split(","); + context = new ClassPathXmlApplicationContext(contextFiles); + } else { + logger.info("No applicationContext files found in system property params. Set -DapplicationContext.files=file1,file2,file3 in the java command"); + logger.info("Creating application context from annotations only"); + context = new AnnotationConfigApplicationContext("org.apache.ranger"); + } + } else { + logger.info("context not null"); + } + if (logger.isDebugEnabled()) { + if (context != null) { + String[] beanNames = context.getBeanDefinitionNames(); + logger.debug("Beans initialized by Spring from classpath:"); + for (String beanName : beanNames) { + logger.debug(beanName); + } + } else { + logger.debug("context is null"); + } + } + } + + public static T getBean(Class beanClass) { + if (context == null) { + logger.info("SpringContext.init()"); + init(); + } + if (context != null) { + logger.info("SpringContext.context not null"); + return context.getBean(beanClass); + } else { + logger.info("SpringContext.context is null"); + return null; + } + } + + public static T getBean(String beanName) { + if (context == null) { + logger.info("SpringContext.init()"); + init(); + } + if (context != null) { + logger.info("SpringContext.context not null"); + return (T) context.getBean(beanName); + } else { + logger.info("SpringContext.context is null"); + return null; + } + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/StatusCheckScheduledExecutorService.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/StatusCheckScheduledExecutorService.java new file mode 100644 index 0000000000..7e387ec524 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/StatusCheckScheduledExecutorService.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import liquibase.command.CommandResults; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; + +import static java.util.concurrent.TimeUnit.SECONDS; + +@Component +@Scope("prototype") +class StatusCheckScheduledExecutorService { + private static final Logger LOG = LoggerFactory.getLogger(StatusCheckScheduledExecutorService.class); + private static final int SLEEP_DURATION = 30; + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + @Autowired + ICommandDriver commandDriver; + private String changelogFilename; + private ScheduledFuture statusHandler; + + public void startStatusChecks(String serviceName, String changelogFilename) { + LOG.info("StatusCheckScheduledExecutorService commandDriverClass={} \n", commandDriver.getClass()); + this.changelogFilename = changelogFilename; + stopStatusChecks(); + final Runnable liquibaseStatusCommand = () -> { + LOG.info("Executing status check command"); + try { + CommandResults statusCheckResult = commandDriver.status(serviceName, changelogFilename); + LOG.info("statusCheckResult={}", statusCheckResult.getResults()); + } catch (Exception e) { + stopStatusChecks(); + throw new RuntimeException(e); + } + }; + LOG.info("Starting status check scheduler"); + statusHandler = scheduler.scheduleWithFixedDelay(liquibaseStatusCommand, 0, SLEEP_DURATION, SECONDS); + } + + public void stopStatusChecks() { + if (statusHandler == null) { + LOG.info("statusHandler is null, stop not required for statusChecks"); + return; + } + LOG.info("Attempting to cancel statusChecks"); + try { + scheduler.shutdown(); + statusHandler.cancel(true); + } catch (Exception e) { + LOG.info("Exception while trying to shutdown/cancel status check schedule"); + } + if (statusHandler.isCancelled()) { + LOG.info("Canceled the Status Check service for {}", changelogFilename); + } else { + try { + LOG.info("Could not cancel the Status Check service for {}", changelogFilename); + Thread.sleep(3000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/Utils.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/Utils.java new file mode 100644 index 0000000000..f17f8b71bd --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/Utils.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.db.upgrade; + +public class Utils { + private Utils() { + } + + public static String getPostUpdateTagName(String tagName) { + return tagName + Constants.TAG_POST_UPGRADE_POSTFIX; + } + + public static String getFinalizeTagName(String tagName) { + return tagName + Constants.TAG_FINALIZE_POSTFIX; + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/patches/kms/UpdateKMSAttrs.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/patches/kms/UpdateKMSAttrs.java new file mode 100644 index 0000000000..02e91e5509 --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/patches/kms/UpdateKMSAttrs.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade.patches.kms; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import liquibase.change.custom.CustomTaskChange; +import liquibase.change.custom.CustomTaskRollback; +import liquibase.database.Database; +import liquibase.exception.CustomChangeException; +import liquibase.exception.SetupException; +import liquibase.exception.ValidationErrors; +import liquibase.resource.ResourceAccessor; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.RangerKMSDB; +import org.apache.hadoop.crypto.key.RangerKeyStoreProvider; +import org.apache.ranger.entity.XXRangerKeyStore; +import org.apache.ranger.kms.dao.DaoManager; +import org.apache.ranger.kms.dao.RangerKMSDao; + +import java.util.List; +import java.util.Map; + +@SuppressWarnings("PMD.JUnit4TestShouldUseBeforeAnnotation") +public class UpdateKMSAttrs implements CustomTaskChange, CustomTaskRollback { + @Override + public void execute(Database database) throws CustomChangeException { + try { + Gson gson = new GsonBuilder().create(); + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); + DaoManager daoManager = rangerKMSDB.getDaoManager(); + RangerKMSDao kmsDao = daoManager.getRangerKMSDao(); + List rangerKeyStoreList = kmsDao.getAllKeys(); + for (XXRangerKeyStore xxRangerKeyStore : rangerKeyStoreList) { + String kmsAttrsStr = xxRangerKeyStore.getAttributes(); + Map kmsAttrs = gson.fromJson(kmsAttrsStr, Map.class); + if (MapUtils.isNotEmpty(kmsAttrs)) { + String kmsAclName = kmsAttrs.get("key.acl.name"); + if (StringUtils.isNotEmpty(kmsAclName)) { + kmsAttrs.put("key.acl.new.name", kmsAclName); + //kmsAttrs.remove("key.acl.name"); -- For backward compatibility + kmsAttrsStr = gson.toJson(kmsAttrs); + xxRangerKeyStore.setAttributes(kmsAttrsStr); + kmsDao.update(xxRangerKeyStore); + } + } + } + } catch (Exception e) { + throw new CustomChangeException(e); + } + } + + @Override + public void rollback(Database database) throws CustomChangeException { + try { + Gson gson = new GsonBuilder().create(); + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); + DaoManager daoManager = rangerKMSDB.getDaoManager(); + RangerKMSDao kmsDao = daoManager.getRangerKMSDao(); + List rangerKeyStoreList = kmsDao.getAllKeys(); + for (XXRangerKeyStore xxRangerKeyStore : rangerKeyStoreList) { + String kmsAttrsStr = xxRangerKeyStore.getAttributes(); + Map kmsAttrs = gson.fromJson(kmsAttrsStr, Map.class); + if (MapUtils.isNotEmpty(kmsAttrs)) { + String kmsAclName = kmsAttrs.get("key.acl.new.name"); + if (StringUtils.isNotEmpty(kmsAclName)) { + kmsAttrs.remove("key.acl.new.name"); + kmsAttrsStr = gson.toJson(kmsAttrs); + xxRangerKeyStore.setAttributes(kmsAttrsStr); + kmsDao.update(xxRangerKeyStore); + } + } + } + } catch (Exception e) { + throw new CustomChangeException(e); + } + } + + @Override + public String getConfirmationMessage() { + return null; + } + + @Override + public void setUp() throws SetupException { + } + + @Override + public void setFileOpener(ResourceAccessor resourceAccessor) { + } + + @Override + public ValidationErrors validate(Database database) { + return null; + } +} diff --git a/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/patches/ranger/SampleLiquibasePatchForOzoneServiceDefUpdateJ10041.java b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/patches/ranger/SampleLiquibasePatchForOzoneServiceDefUpdateJ10041.java new file mode 100644 index 0000000000..f1ae77457f --- /dev/null +++ b/liquibase-database-upgrade/src/main/java/org/apache/ranger/db/upgrade/patches/ranger/SampleLiquibasePatchForOzoneServiceDefUpdateJ10041.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.db.upgrade.patches.ranger; + +import liquibase.change.custom.CustomTaskChange; +import liquibase.change.custom.CustomTaskRollback; +import liquibase.database.Database; +import liquibase.exception.CustomChangeException; +import liquibase.exception.RollbackImpossibleException; +import liquibase.exception.SetupException; +import liquibase.exception.ValidationErrors; +import liquibase.resource.ResourceAccessor; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.common.JSONUtil; +import org.apache.ranger.common.RangerValidatorFactory; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.db.upgrade.SpringContext; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; +import org.apache.ranger.plugin.model.validation.RangerValidator; +import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; +import org.apache.ranger.plugin.util.SearchFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +// Sample for demonstration of how java patch PatchForOzoneServiceDefUpdate_J10041 can be ported into liquibase. +// Instead of @Autowired, SpringContext.getBean(...) is used to initialize the spring components required in this class +// Since @Transactional annotation cannot be used in this class, TransactionTemplate is used to start a transaction which is required to write into the database +// rollback() method is only partially implemented here for demonstration purposes only and does not fully rollback changes in the execute() method +@SuppressWarnings("PMD.JUnit4TestShouldUseBeforeAnnotation") +public class SampleLiquibasePatchForOzoneServiceDefUpdateJ10041 + implements CustomTaskChange, CustomTaskRollback { + private static final Logger logger = LoggerFactory.getLogger( + SampleLiquibasePatchForOzoneServiceDefUpdateJ10041.class); + private static final List OZONE_CONFIGS = new ArrayList<>( + Arrays.asList("dfs.datanode.kerberos.principal", "dfs.namenode.kerberos.principal", "dfs.secondary.namenode.kerberos.principal", "commonNameForCertificate")); + private static final String OZONE_RESOURCE_VOLUME = "volume"; + private static final String OZONE_RESOURCE_KEY = "key"; + private static final String ACCESS_TYPE_READ_ACL = "read_acl"; + private static final String ACCESS_TYPE_WRITE_ACL = "write_acl"; + + //@Autowired + RangerDaoManager daoMgr = SpringContext.getBean(RangerDaoManager.class); + ServiceDBStore svcDBStore = SpringContext.getBean(ServiceDBStore.class); + JSONUtil jsonUtil = SpringContext.getBean(JSONUtil.class); + PlatformTransactionManager txManager = SpringContext.getBean("transactionManager"); + RangerValidatorFactory validatorFactory = SpringContext.getBean(RangerValidatorFactory.class); + + @Override + public void execute(Database database) throws CustomChangeException { + boolean isServiceDefUpdateSuccessful; + try { + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + + isServiceDefUpdateSuccessful = Boolean.TRUE.equals(txTemplate.execute(status -> { + try { + return updateOzoneServiceDef(); + } catch (Exception e) { + throw new RuntimeException(e); + } + })); + } catch (Exception e) { + throw new CustomChangeException(e); + } + if (!isServiceDefUpdateSuccessful) { + throw new CustomChangeException("Updating ozone service def was unsuccessful"); + } + } + + @Override + public String getConfirmationMessage() { + return null; + } + + @Override + public void setUp() throws SetupException { + } + + @Override + public void setFileOpener(ResourceAccessor resourceAccessor) { + } + + @Override + public ValidationErrors validate(Database database) { + return null; + } + + @Override + public void rollback(Database database) + throws CustomChangeException, RollbackImpossibleException { + boolean isServiceDefUpdateSuccessful; + try { + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + isServiceDefUpdateSuccessful = Boolean.TRUE.equals(txTemplate.execute(status -> { + try { + return rollbackOzoneServiceDef(); + } catch (Exception e) { + throw new RuntimeException(e); + } + })); + } catch (Exception e) { + throw new CustomChangeException(e); + } + if (!isServiceDefUpdateSuccessful) { + throw new CustomChangeException("Rollback service def was unsuccessful"); + } + } + + protected Map jsonStringToMap(String jsonStr) { + Map ret = null; + if (!StringUtils.isEmpty(jsonStr)) { + try { + ret = jsonUtil.jsonToMap(jsonStr); + } catch (Exception ex) { + // fallback to earlier format: "name1=value1;name2=value2" + for (String optionString : jsonStr.split(";")) { + if (StringUtils.isEmpty(optionString)) { + continue; + } + String[] nvArr = optionString.split("="); + String name = (nvArr != null && nvArr.length > 0) ? nvArr[0].trim() : null; + String value = (nvArr != null && nvArr.length > 1) ? nvArr[1].trim() : null; + if (StringUtils.isEmpty(name)) { + continue; + } + if (ret == null) { + ret = new HashMap(); + } + ret.put(name, value); + } + } + } + return ret; + } + + private boolean rollbackOzoneServiceDef() throws Exception { + RangerServiceDef dbOzoneServiceDef = svcDBStore.getServiceDefByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME); + RangerServiceDef embeddedOzoneServiceDef = EmbeddedServiceDefsUtil.instance().getEmbeddedServiceDef(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME); + List embeddedOzoneAccessTypes = embeddedOzoneServiceDef.getAccessTypes(); + logger.info("Before deletion embeddedOzoneAccessTypes=" + embeddedOzoneAccessTypes); + logger.info("Before deletion dbOzoneServiceDef.accessTypes=" + dbOzoneServiceDef.getAccessTypes().toString()); + embeddedOzoneAccessTypes.removeIf(item -> item.getItemId() == 101L); + logger.info("After deletion embeddedOzoneAccessTypes=" + embeddedOzoneAccessTypes); + boolean ret = false; + if (!embeddedOzoneAccessTypes.toString().equalsIgnoreCase(dbOzoneServiceDef.getAccessTypes().toString())) { + logger.info("Trying to write to DB new access types after deleting access type added suring update"); + dbOzoneServiceDef.setAccessTypes(embeddedOzoneAccessTypes); + ret = true; + } else { + logger.info("Deletion of access type with id 101 not possible as not present. Rollback failed"); + } + return ret; + } + + private boolean updateOzoneServiceDef() throws Exception { + RangerServiceDef ret; + RangerServiceDef embeddedOzoneServiceDef; + RangerServiceDef dbOzoneServiceDef; + List embeddedOzoneConfigDefs; + List embeddedOzoneResourceDefs; + List embeddedOzoneAccessTypes; + XXServiceDef xXServiceDefObj; + + embeddedOzoneServiceDef = EmbeddedServiceDefsUtil.instance().getEmbeddedServiceDef(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME); + logger.info("EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME" + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME); + if (embeddedOzoneServiceDef != null) { + if (daoMgr == null) { + logger.error("daoMgr is null"); + } else if (daoMgr.getXXServiceDef() == null) { + logger.error("daoMgr.getXXServiceDef() is null"); + } else if (daoMgr.getXXServiceDef().findByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME) == null) { + logger.error("daoMgr.getXXServiceDef().findByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME) is null"); + } else { + logger.info("Nothing was null, all good"); + } + xXServiceDefObj = daoMgr.getXXServiceDef().findByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME); + Map serviceDefOptionsPreUpdate; + String jsonPreUpdate; + + if (xXServiceDefObj != null) { + jsonPreUpdate = xXServiceDefObj.getDefOptions(); + serviceDefOptionsPreUpdate = jsonStringToMap(jsonPreUpdate); + } else { + logger.error("Ozone service-definition does not exist in the Ranger DAO. No patching is needed!!"); + return true; + } + dbOzoneServiceDef = svcDBStore.getServiceDefByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME); + + if (dbOzoneServiceDef != null) { + // Remove old Ozone configs + embeddedOzoneConfigDefs = embeddedOzoneServiceDef.getConfigs(); + if (checkNotConfigPresent(embeddedOzoneConfigDefs)) { + dbOzoneServiceDef.setConfigs(embeddedOzoneConfigDefs); + } + + // Update volume resource with recursive flag false and key resource with recursive flag true + embeddedOzoneResourceDefs = embeddedOzoneServiceDef.getResources(); + if (checkVolKeyResUpdate(embeddedOzoneResourceDefs)) { + dbOzoneServiceDef.setResources(embeddedOzoneResourceDefs); + } + + // Add new access types + embeddedOzoneAccessTypes = embeddedOzoneServiceDef.getAccessTypes(); + embeddedOzoneAccessTypes.add(new RangerServiceDef.RangerAccessTypeDef(101L, "liquibase_java_patch", "liquibase_java_patch", "liquibase_java_patch", new ArrayList<>())); + + if (embeddedOzoneAccessTypes != null) { + if (checkAccessTypesPresent(embeddedOzoneAccessTypes)) { + if (!embeddedOzoneAccessTypes.toString().equalsIgnoreCase(dbOzoneServiceDef.getAccessTypes().toString())) { + dbOzoneServiceDef.setAccessTypes(embeddedOzoneAccessTypes); + } + } + } + } else { + logger.error("Ozone service-definition does not exist in the db store."); + return false; + } + RangerServiceDefValidator validator = validatorFactory.getServiceDefValidator(svcDBStore); + validator.validate(dbOzoneServiceDef, RangerValidator.Action.UPDATE); + + ret = svcDBStore.updateServiceDef(dbOzoneServiceDef); + if (ret == null) { + throw new RuntimeException("Error while updating " + EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME + " service-def"); + } + xXServiceDefObj = daoMgr.getXXServiceDef().findByName(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_OZONE_NAME); + if (xXServiceDefObj != null) { + String jsonStrPostUpdate = xXServiceDefObj.getDefOptions(); + Map serviceDefOptionsPostUpdate = jsonStringToMap(jsonStrPostUpdate); + if (serviceDefOptionsPostUpdate != null && serviceDefOptionsPostUpdate.containsKey(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES)) { + if (serviceDefOptionsPreUpdate == null || !serviceDefOptionsPreUpdate.containsKey(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES)) { + String preUpdateValue = serviceDefOptionsPreUpdate == null ? null : serviceDefOptionsPreUpdate.get(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES); + if (preUpdateValue == null) { + serviceDefOptionsPostUpdate.remove(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES); + } else { + serviceDefOptionsPostUpdate.put(RangerServiceDef.OPTION_ENABLE_DENY_AND_EXCEPTIONS_IN_POLICIES, preUpdateValue); + } + xXServiceDefObj.setDefOptions(mapToJsonString(serviceDefOptionsPostUpdate)); + daoMgr.getXXServiceDef().update(xXServiceDefObj); + } + } + } else { + logger.error("Ozone service-definition does not exist in the Ranger DAO."); + return false; + } + List dbServices = daoMgr.getXXService().findByServiceDefId(embeddedOzoneServiceDef.getId()); + if (CollectionUtils.isNotEmpty(dbServices)) { + for (XXService dbService : dbServices) { + SearchFilter filter = new SearchFilter(); + filter.setParam(SearchFilter.SERVICE_NAME, dbService.getName()); + updateExisitngOzonePolicies(svcDBStore.getServicePolicies(dbService.getId(), filter)); + } + } + } else { + logger.error("The embedded Ozone service-definition does not exist."); + return false; + } + return true; + } + + private boolean checkNotConfigPresent(List configDefs) { + boolean ret = false; + List configNames = new ArrayList<>(); + for (RangerServiceDef.RangerServiceConfigDef configDef : configDefs) { + configNames.add(configDef.getName()); + } + for (String delConfig : OZONE_CONFIGS) { + if (!configNames.contains(delConfig)) { + ret = true; + break; + } + } + return ret; + } + + private boolean checkVolKeyResUpdate(List embeddedOzoneResDefs) { + boolean ret = false; + for (RangerServiceDef.RangerResourceDef resDef : embeddedOzoneResDefs) { + if ((resDef.getName().equals(OZONE_RESOURCE_VOLUME) && (!resDef.getRecursiveSupported() || resDef.getExcludesSupported())) || + (resDef.getName().equals(OZONE_RESOURCE_KEY) && resDef.getRecursiveSupported())) { + ret = true; + break; + } + } + return ret; + } + + private boolean checkAccessTypesPresent(List embeddedOzoneAccessTypes) { + boolean ret = false; + for (RangerServiceDef.RangerAccessTypeDef accessDef : embeddedOzoneAccessTypes) { + if (ACCESS_TYPE_READ_ACL.equals(accessDef.getName()) || ACCESS_TYPE_WRITE_ACL.equals(accessDef.getName())) { + ret = true; + break; + } + } + return ret; + } + + private void updateExisitngOzonePolicies(List policies) throws Exception { + if (CollectionUtils.isNotEmpty(policies)) { + for (RangerPolicy policy : policies) { + List policyItems = policy.getPolicyItems(); + if (CollectionUtils.isNotEmpty(policyItems)) { + for (RangerPolicy.RangerPolicyItem policyItem : policyItems) { + List policyItemAccesses = policyItem.getAccesses(); + // Add new access types + policyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("read_acl")); + policyItemAccesses.add(new RangerPolicy.RangerPolicyItemAccess("write_acl")); + policyItem.setAccesses(policyItemAccesses); + } + } + Map policyResources = policy.getResources(); + if (MapUtils.isNotEmpty(policyResources)) { + if (policyResources.containsKey(OZONE_RESOURCE_VOLUME)) { + // Set recursive flag as false for volume resource + policyResources.get(OZONE_RESOURCE_VOLUME).setIsRecursive(false); + // Set exclude support flag as true for volume resource + policyResources.get(OZONE_RESOURCE_VOLUME).setIsExcludes(false); + } + if (policyResources.containsKey(OZONE_RESOURCE_KEY)) { + // Set is recursive flag as true for volume resource + policyResources.get(OZONE_RESOURCE_KEY).setIsRecursive(true); + } + } + svcDBStore.updatePolicy(policy); + } + } + } + + private String mapToJsonString(Map map) { + String ret = null; + if (map != null) { + try { + ret = jsonUtil.readMapToString(map); + } catch (Exception ex) { + logger.warn("mapToJsonString() failed to convert map: " + map, ex); + } + } + return ret; + } +} diff --git a/liquibase-database-upgrade/src/main/resources/kms_log4j.properties b/liquibase-database-upgrade/src/main/resources/kms_log4j.properties new file mode 100644 index 0000000000..c3c535ba96 --- /dev/null +++ b/liquibase-database-upgrade/src/main/resources/kms_log4j.properties @@ -0,0 +1,39 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# +# Define the File File Appender +appender.file.type=File +appender.file.name=FileAppender +appender.file.fileName=/var/log/ranger/kms/ranger-liquibase-database-upgrade-${date:yyyy-MM-dd_HH-mm-ss}.log +appender.file.layout.type=PatternLayout +appender.file.layout.pattern=%d %d{Z} [%t] %-5p (%F:%L) - %m%n +# Define the Console Appender +appender.console.type=Console +appender.console.name=ConsoleAppender +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%d %d{Z} [%t] %-5p (%F:%L) - %m%n +# Define Loggers +loggers=console, file +logger.console.name=ConsoleLogger +logger.console.level=info +logger.console.additivity=false +logger.console.appenderRef.console.ref=ConsoleAppender +logger.file.name=FileLogger +logger.file.level=info +logger.file.additivity=false +logger.file.appenderRef.file.ref=FileAppender +# Define Root Logger +rootLogger.level=info +rootLogger.appenderRefs=console, file +rootLogger.appenderRef.console.ref=ConsoleAppender +rootLogger.appenderRef.file.ref=FileAppender diff --git a/liquibase-database-upgrade/src/main/resources/ranger_log4j.properties b/liquibase-database-upgrade/src/main/resources/ranger_log4j.properties new file mode 100644 index 0000000000..0d274ad177 --- /dev/null +++ b/liquibase-database-upgrade/src/main/resources/ranger_log4j.properties @@ -0,0 +1,39 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# +# Define the File File Appender +appender.file.type=File +appender.file.name=FileAppender +appender.file.fileName=/var/log/ranger/admin/ranger-liquibase-database-upgrade-${date:yyyy-MM-dd_HH-mm-ss}.log +appender.file.layout.type=PatternLayout +appender.file.layout.pattern=%d %d{Z} [%t] %-5p (%F:%L) - %m%n +# Define the Console Appender +appender.console.type=Console +appender.console.name=ConsoleAppender +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=%d %d{Z} [%t] %-5p (%F:%L) - %m%n +# Define Loggers +loggers=console, file +logger.console.name=ConsoleLogger +logger.console.level=info +logger.console.additivity=false +logger.console.appenderRef.console.ref=ConsoleAppender +logger.file.name=FileLogger +logger.file.level=info +logger.file.additivity=false +logger.file.appenderRef.file.ref=FileAppender +# Define Root Logger +rootLogger.level=info +rootLogger.appenderRefs=console, file +rootLogger.appenderRef.console.ref=ConsoleAppender +rootLogger.appenderRef.file.ref=FileAppender diff --git a/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/DummyConfigProviderForTest.java b/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/DummyConfigProviderForTest.java new file mode 100644 index 0000000000..cf8298c3ff --- /dev/null +++ b/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/DummyConfigProviderForTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +import java.net.URL; + +@Component("test") +@Lazy +public class DummyConfigProviderForTest implements IConfigProvider { + public static String getResourceFilePath(String resourceRelativePath) { + // Get the resource URL + ClassLoader classLoader = DummyConfigProviderForTest.class.getClassLoader(); + URL resourceURL = classLoader.getResource(resourceRelativePath); + if (resourceURL == null) { + throw new IllegalArgumentException("Resource not found: " + resourceRelativePath); + } + return resourceURL.getFile(); + } + + @Override + public String getUrl() { + return "url"; + } + + @Override + public String getUsername() { + return "user"; + } + + @Override + public String getPassword() { + return "pwd"; + } + + @Override + public String getDriver() { + return "driver"; + } + + @Override + public String getMasterChangelogRelativePath() { + return getResourceFilePath("masterChangelogForTest.txt"); + } + + @Override + public String getFinalizeChangelogRelativePath() { + return getResourceFilePath("finalizeChangelogForTest.txt"); + } +} diff --git a/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/FileDBForTest.java b/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/FileDBForTest.java new file mode 100644 index 0000000000..db719e6c83 --- /dev/null +++ b/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/FileDBForTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class FileDBForTest { + private FileDBForTest() { + } + + public static void deleteFileIfExists(String filename) { + File file = new File(filename); + if (file.exists()) { + if (file.delete()) { + System.out.println("File deleted successfully: " + filename); + } else { + System.out.println("Failed to delete the file: " + filename); + } + } else { + System.out.println("File does not exist: " + filename); + } + } + + public static boolean tagExists(String filePath, String tagName) { + createFileIfNotExists(filePath); + try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { + String line; + while ((line = reader.readLine()) != null) { + String[] changeLogRow = line.split(","); + if (changeLogRow[changeLogRow.length - 1].equals(tagName)) { + return true; // String found + } + } + } catch (IOException e) { + System.err.println("Error reading file: " + e.getMessage()); + } + return false; // String not found + } + + public static void tag(String filePath, String tagName) { + createFileIfNotExists(filePath); + if (isFileEmpty(filePath)) { + List lines = new ArrayList<>(); + lines.add(",,," + tagName); + writeLines(filePath, lines, true); + } else { + try { + readAndModifyLastLine(filePath, tagName); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + public static List readLines(String filePath) { + createFileIfNotExists(filePath); + List lines = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { + String line; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return lines; + } + + public static void writeLines(String filePath, List lines, boolean doAppend) { + createFileIfNotExists(filePath); + try (FileWriter writer = new FileWriter(filePath, doAppend)) { // 'true' enables appending + for (String line : lines) { + writer.write(line); + writer.write("\n"); + } + System.out.println("File=" + filePath + " Lines written to file= " + lines); + } catch (IOException e) { + System.err.println("Error appending to file: " + e.getMessage()); + } + } + + public static void removeLinesAfterTag(String filePath, String tagName) throws Exception { + createFileIfNotExists(filePath); + int lastLineWithTag = -1; + List lines = readLines(filePath); + for (int i = 0; i < lines.size(); i++) { + String[] changeLogRow = lines.get(i).split(","); + if (changeLogRow[changeLogRow.length - 1].equals(tagName)) { + lastLineWithTag = i; + } + } + System.out.println("lastLineWithTag " + tagName + " = " + lastLineWithTag); + if (lastLineWithTag == -1) { + throw new Exception("Couldn't find Tag " + tagName + " in file, can't remove lines"); + } + lines = lines.subList(0, lastLineWithTag + 1); + + writeLines(filePath, lines, false); + } + + private static boolean isFileEmpty(String filePath) { + File file = new File(filePath); + // Check if file exists and its size is 0 + return file.exists() && file.isFile() && file.length() == 0; + } + + private static void createFileIfNotExists(String filePath) { + File file = new File(filePath); + try { + // Check if the file already exists + if (file.exists()) { + System.out.println("File already exists: " + file.getAbsolutePath()); + } else { + // Create the file if it does not exist + if (file.createNewFile()) { + System.out.println("File created successfully: " + file.getAbsolutePath()); + } else { + System.out.println("Failed to create the file."); + } + } + } catch (IOException e) { + // Handle exceptions + System.out.println("An error occurred while creating the file."); + e.printStackTrace(); + } + } + + private static void readAndModifyLastLine(String filePath, String tagName) throws Exception { + File file = new File(filePath); + if (!file.exists() || !file.isFile()) { + throw new FileNotFoundException("File does not exist or is not valid: " + filePath); + } + + // Step 1: Read all lines into a list + List lines = readLines(filePath); + + if (lines.isEmpty()) { + System.out.println("The file is empty. Nothing to modify."); + return; + } + + // Step 2: Modify the last line + int lastIndex = lines.size() - 1; + String lastLine = lines.get(lastIndex); + String[] lastLineComponents = lastLine.split(","); + if (lastLineComponents.length == 4) { + //replace existing tag + lastLineComponents[3] = tagName; + lastLine = String.join(",", lastLineComponents); + } else if (lastLineComponents.length == 3) { + //add new tag to last line + lastLine += ("," + tagName); + } else { + throw new Exception("Unhandled scenario for test case. Modify the logic if changes made to sample changelog file"); + } + + lines.set(lastIndex, lastLine); + + // Step 3: Write the modified content back to the file + writeLines(filePath, lines, false); + + System.out.println("Successfully modified the last line."); + } +} diff --git a/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/LiquibaseCommandDriverFileBasedForTest.java b/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/LiquibaseCommandDriverFileBasedForTest.java new file mode 100644 index 0000000000..477a7d2273 --- /dev/null +++ b/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/LiquibaseCommandDriverFileBasedForTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.db.upgrade; + +import liquibase.command.CommandResults; +import liquibase.command.CommandScope; +import liquibase.exception.CommandExecutionException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; + +@Component +public class LiquibaseCommandDriverFileBasedForTest implements ICommandDriver { + @Value("${test_db_file.name:test.txt}") + private String filename; + + @Override + public CommandResults tag(String serviceName, String tagName) throws CommandExecutionException { + FileDBForTest.tag(filename, tagName); + return getCommandResultsObject(new TreeMap<>(), new CommandScope("tag")); + } + + @Override + public CommandResults rollback(String serviceName, String tagName, String changelogFilename) throws CommandExecutionException { + try { + FileDBForTest.removeLinesAfterTag(filename, tagName); + } catch (Exception e) { + throw new RuntimeException(e); + } + return getCommandResultsObject(new TreeMap<>(), new CommandScope("rollback")); + } + + @Override + public CommandResults tagExists(String serviceName, String tagName) throws CommandExecutionException { + SortedMap result = new TreeMap<>(); + if (FileDBForTest.tagExists(filename, tagName)) { + result.put("tagExistsResult", true); + } else { + result.put("tagExistsResult", false); + } + return getCommandResultsObject(result, new CommandScope("tagExists")); + } + + @Override + public CommandResults update(String serviceName, String changelogFilename) throws CommandExecutionException { + List lines = FileDBForTest.readLines(changelogFilename); + System.out.println("lines=" + lines); + FileDBForTest.writeLines(filename, lines, true); + return getCommandResultsObject(new TreeMap<>(), new CommandScope("update")); + } + + @Override + public CommandResults status(String serviceName, String changelogFilename) throws CommandExecutionException { + SortedMap result = new TreeMap<>(); + int numRows = FileDBForTest.readLines(filename).size(); + result.put("numRows", numRows); + return getCommandResultsObject(result, new CommandScope("status")); + } + + private CommandResults getCommandResultsObject(SortedMap resultValues, CommandScope commandScope) { + Class clazz = CommandResults.class; + Constructor constructor = null; + try { + constructor = clazz.getDeclaredConstructor(SortedMap.class, CommandScope.class); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + constructor.setAccessible(true); + try { + return (CommandResults) constructor.newInstance(resultValues, commandScope); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } +} diff --git a/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/TestLiquibaseUpdateDriverMain.java b/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/TestLiquibaseUpdateDriverMain.java new file mode 100644 index 0000000000..6ce33c6fd6 --- /dev/null +++ b/liquibase-database-upgrade/src/test/java/org/apache/ranger/db/upgrade/TestLiquibaseUpdateDriverMain.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ranger.db.upgrade; + +import liquibase.command.CommandResults; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import static org.mockito.Mockito.mockStatic; + +@SuppressWarnings("PMD") +class TestLiquibaseUpdateDriverMain { + static String testFile = "test.txt"; + + @BeforeAll + static void setup() { + FileDBForTest.deleteFileIfExists(testFile); + } + + @AfterAll + static void cleanup() { + FileDBForTest.deleteFileIfExists(testFile); + } + + @Test + void testUpgradeSucceed() { + System.out.println("===== Test Update===="); + + FileDBForTest.deleteFileIfExists(testFile); + try { + try (MockedStatic liquibaseCommandResultsUtilMock = mockStatic(LiquibaseCommandResultsUtil.class)) { + liquibaseCommandResultsUtilMock.when(() -> LiquibaseCommandResultsUtil.isUpdateSuccessful(Mockito.any(CommandResults.class))).thenReturn(true); + System.setProperty("applicationContext.files", "liquibaseTestApplicationContext.xml"); + System.setProperty("test_db_file.name", testFile); + LiquibaseUpdateDriverMain.main(new String[] {"-serviceName", "test", "-tag", "test_3.x", "-op", "update"}); + ICommandDriver driver = SpringContext.getBean(ICommandDriver.class); + IConfigProvider configProvider = SpringContext.getBean("test"); + Assertions.assertNotNull(configProvider); + Assertions.assertNotNull(driver); + + CommandResults result = driver.status("test", configProvider.getMasterChangelogRelativePath()); + int numRows = (int) result.getResults().get("numRows"); + Assertions.assertEquals(4, numRows); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + System.out.println("===== Test Update Complete===="); + } + + @Test + void testRollback() { + System.out.println("===== Test Rollback===="); + FileDBForTest.deleteFileIfExists(testFile); + try { + try (MockedStatic liquibaseCommandResultsUtilMock = mockStatic(LiquibaseCommandResultsUtil.class)) { + liquibaseCommandResultsUtilMock.when(() -> LiquibaseCommandResultsUtil.isUpdateSuccessful(Mockito.any(CommandResults.class))).thenReturn(true); + liquibaseCommandResultsUtilMock.when(() -> LiquibaseCommandResultsUtil.doesTagExist(Mockito.any(CommandResults.class))).thenCallRealMethod(); + System.setProperty("applicationContext.files", "liquibaseTestApplicationContext.xml"); + System.setProperty("test_db_file.name", testFile); + LiquibaseUpdateDriverMain.main(new String[] {"-serviceName", "test", "-tag", "test_3.x", "-op", "update"}); + LiquibaseUpdateDriverMain.main(new String[] {"-serviceName", "test", "-tag", "test_3.x", "-op", "rollback"}); + ICommandDriver driver = SpringContext.getBean(ICommandDriver.class); + IConfigProvider configProvider = SpringContext.getBean("test"); + CommandResults result = driver.status("test", configProvider.getMasterChangelogRelativePath()); + int numRows = (int) result.getResults().get("numRows"); + Assertions.assertEquals(1, numRows); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + System.out.println("===== Test Rollback Complete===="); + } + + @Test + void testFinalize() { + System.out.println("===== Test Finalize ======"); + FileDBForTest.deleteFileIfExists(testFile); + try { + try (MockedStatic liquibaseCommandResultsUtilMock = mockStatic(LiquibaseCommandResultsUtil.class)) { + liquibaseCommandResultsUtilMock.when(() -> LiquibaseCommandResultsUtil.isUpdateSuccessful(Mockito.any(CommandResults.class))).thenReturn(true); + liquibaseCommandResultsUtilMock.when(() -> LiquibaseCommandResultsUtil.doesTagExist(Mockito.any(CommandResults.class))).thenCallRealMethod(); + System.setProperty("applicationContext.files", "liquibaseTestApplicationContext.xml"); + System.setProperty("test_db_file.name", testFile); + LiquibaseUpdateDriverMain.main(new String[] {"-serviceName", "test", "-tag", "test_3.x", "-op", "update"}); + LiquibaseUpdateDriverMain.main(new String[] {"-serviceName", "test", "-tag", "test_3.x", "-op", "finalize"}); + ICommandDriver driver = SpringContext.getBean(ICommandDriver.class); + IConfigProvider configProvider = SpringContext.getBean("test"); + + Assertions.assertNotNull(configProvider, "ConfigProvider should not be null after initialization"); + Assertions.assertNotNull(driver, "CommandDriver should not be null after initialization"); + + CommandResults result = driver.status("test", configProvider.getMasterChangelogRelativePath()); + int numRows = (int) result.getResults().get("numRows"); + Assertions.assertEquals(7, numRows); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + System.out.println("===== Test Finalize complete ====="); + } + + @Test + void testFinalizeAfterRollbackShouldFail() { + System.out.println("===== Test Finalize after Rollback Should Fail======"); + FileDBForTest.deleteFileIfExists(testFile); + try { + try (MockedStatic liquibaseCommandResultsUtilMock = mockStatic(LiquibaseCommandResultsUtil.class)) { + liquibaseCommandResultsUtilMock.when(() -> LiquibaseCommandResultsUtil.isUpdateSuccessful(Mockito.any(CommandResults.class))).thenReturn(true); + liquibaseCommandResultsUtilMock.when(() -> LiquibaseCommandResultsUtil.doesTagExist(Mockito.any(CommandResults.class))).thenCallRealMethod(); + System.setProperty("applicationContext.files", "liquibaseTestApplicationContext.xml"); + System.setProperty("test_db_file.name", testFile); + LiquibaseUpdateDriverMain.main(new String[] {"-serviceName", "test", "-tag", "test_3.x", "-op", "update"}); + LiquibaseUpdateDriverMain.main(new String[] {"-serviceName", "test", "-tag", "test_3.x", "-op", "rollback"}); + LiquibaseUpdateDriverMain.main(new String[] {"-serviceName", "test", "-tag", "test_3.x", "-op", "finalize"}); + Assertions.fail("Failed: Code flow should not have reached here. Finalize not allowed after rollback"); + } + } catch (Exception e) { + // expected + } + System.out.println("===== Test Finalize after Rollback Should Fail Complete======"); + } +} diff --git a/liquibase-database-upgrade/src/test/resources/finalizeChangelogForTest.txt b/liquibase-database-upgrade/src/test/resources/finalizeChangelogForTest.txt new file mode 100644 index 0000000000..d691b6ba02 --- /dev/null +++ b/liquibase-database-upgrade/src/test/resources/finalizeChangelogForTest.txt @@ -0,0 +1,3 @@ +id4,author1,changeset4.xml +id5,author2,changeset5.xml +id6,author3,changeset6.xml \ No newline at end of file diff --git a/liquibase-database-upgrade/src/test/resources/liquibaseTestApplicationContext.xml b/liquibase-database-upgrade/src/test/resources/liquibaseTestApplicationContext.xml new file mode 100644 index 0000000000..58f15d4d6d --- /dev/null +++ b/liquibase-database-upgrade/src/test/resources/liquibaseTestApplicationContext.xml @@ -0,0 +1,31 @@ + + + + + + + + + + diff --git a/liquibase-database-upgrade/src/test/resources/log4j2-test.xml b/liquibase-database-upgrade/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..b310aced64 --- /dev/null +++ b/liquibase-database-upgrade/src/test/resources/log4j2-test.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + diff --git a/liquibase-database-upgrade/src/test/resources/masterChangelogForTest.txt b/liquibase-database-upgrade/src/test/resources/masterChangelogForTest.txt new file mode 100644 index 0000000000..6f10470d9e --- /dev/null +++ b/liquibase-database-upgrade/src/test/resources/masterChangelogForTest.txt @@ -0,0 +1,3 @@ +id1,author1,changeset1.xml +id2,author2,changeset2.xml +id3,author3,changeset3.xml \ No newline at end of file diff --git a/pom.xml b/pom.xml old mode 100755 new mode 100644 index f15d440bb8..62f70c3efc --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,9 @@ pom ranger Security for Enforcing Enterprise Policies + + liquibase-database-upgrade + 4.0.3 0.27 @@ -137,6 +140,7 @@ 2.0.1.Final 2.3.3 2.3.1 + 1.82 3.3 1.19.4 2.35 @@ -162,6 +166,7 @@ 4.0.4 1.10 0.14.0 + 4.29.0 ${project.basedir}/../lib/local 2.17.2 1.3.14 diff --git a/security-admin/pom.xml b/security-admin/pom.xml index b317202f5e..8b7a099343 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -875,6 +875,9 @@ org.apache.maven.plugins maven-war-plugin 3.4.0 + + true + prepare