From 415aac1ce0c3e847f3e40fa64ef7ed00e783bb92 Mon Sep 17 00:00:00 2001 From: Milan Perera Date: Wed, 28 Feb 2018 17:44:45 +0530 Subject: [PATCH] Fix issues in data archival feature There were few issues regarding the data retreiving SQL query and archived config operations --- .../core/archival/ArchivalServiceImpl.java | 6 ++ .../mgt/core/archival/dao/ArchivalDAO.java | 2 + .../archival/dao/impl/ArchivalDAOImpl.java | 57 ++++++++++++++++++- .../archival/ArchivalTaskConfiguration.java | 4 +- .../dbscripts/cdm/archival/mysql.sql | 8 +++ 5 files changed, 73 insertions(+), 4 deletions(-) diff --git a/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/ArchivalServiceImpl.java b/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/ArchivalServiceImpl.java index 7528d2270c9..b8ebd30a0e5 100644 --- a/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/ArchivalServiceImpl.java +++ b/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/ArchivalServiceImpl.java @@ -111,6 +111,12 @@ public class ArchivalServiceImpl implements ArchivalService { } archivalDAO.moveProfileOperations(); + //Purge the config operation table, DM_CONFIG_OPERATION + if (log.isDebugEnabled()) { + log.debug("## Purging config operations"); + } + archivalDAO.moveConfigOperations(); + //Purge the enrolment mappings table, DM_ENROLMENT_OP_MAPPING if (log.isDebugEnabled()) { log.debug("## Purging enrolment mappings"); diff --git a/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/dao/ArchivalDAO.java b/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/dao/ArchivalDAO.java index a19437919c4..2c7063741d1 100644 --- a/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/dao/ArchivalDAO.java +++ b/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/dao/ArchivalDAO.java @@ -41,6 +41,8 @@ public interface ArchivalDAO { void moveProfileOperations() throws ArchivalDAOException; + void moveConfigOperations() throws ArchivalDAOException; + void moveEnrolmentMappings() throws ArchivalDAOException; void moveOperations() throws ArchivalDAOException; diff --git a/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/dao/impl/ArchivalDAOImpl.java b/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/dao/impl/ArchivalDAOImpl.java index 6635d0800f2..73f44bacd6a 100644 --- a/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/dao/impl/ArchivalDAOImpl.java +++ b/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/archival/dao/impl/ArchivalDAOImpl.java @@ -56,8 +56,8 @@ public class ArchivalDAOImpl implements ArchivalDAO { try { Connection conn = ArchivalSourceDAOFactory.getConnection(); String sql = "SELECT DISTINCT OPERATION_ID FROM DM_ENROLMENT_OP_MAPPING " + - "WHERE CREATED_TIMESTAMP BETWEEN DATE_SUB(NOW(), INTERVAL " + - this.retentionPeriod + " DAY) AND NOW()"; + "WHERE CREATED_TIMESTAMP BETWEEN DATE(TIMESTAMPADD(DAY, " + + this.retentionPeriod + ", NOW())) AND NOW()"; stmt = this.createMemoryEfficientStatement(conn); rs = stmt.executeQuery(sql); while (rs.next()) { @@ -83,7 +83,8 @@ public class ArchivalDAOImpl implements ArchivalDAO { Connection conn = ArchivalSourceDAOFactory.getConnection(); String sql = "SELECT DISTINCT OPERATION_ID " + " FROM DM_ENROLMENT_OP_MAPPING WHERE STATUS IN('PENDING', 'IN_PROGRESS') " + - " AND CREATED_TIMESTAMP BETWEEN DATE_SUB(NOW(), INTERVAL " + this.retentionPeriod +" DAY) AND NOW()"; + " AND CREATED_TIMESTAMP BETWEEN DATE(TIMESTAMPADD(DAY, " + this.retentionPeriod +", NOW())) " + + "AND NOW()"; stmt = this.createMemoryEfficientStatement(conn); rs = stmt.executeQuery(sql); while (rs.next()) { @@ -337,6 +338,56 @@ public class ArchivalDAOImpl implements ArchivalDAO { } } + @Override + public void moveConfigOperations() throws ArchivalDAOException { + Statement stmt = null; + PreparedStatement stmt2 = null; + Statement stmt3 = null; + ResultSet rs = null; + try { + Connection conn = ArchivalSourceDAOFactory.getConnection(); + String sql = "SELECT * FROM DM_CONFIG_OPERATION WHERE OPERATION_ID IN " + + "(SELECT ID FROM DM_ARCHIVED_OPERATIONS)"; + stmt = this.createMemoryEfficientStatement(conn); + rs = stmt.executeQuery(sql); + + Connection conn2 = ArchivalDestinationDAOFactory.getConnection(); + + sql = "INSERT INTO DM_CONFIG_OPERATION_ARCH VALUES(?, ?, ?, ?)"; + stmt2 = conn2.prepareStatement(sql); + + int count = 0; + while (rs.next()) { + stmt2.setInt(1, rs.getInt("OPERATION_ID")); + stmt2.setBytes(2, rs.getBytes("OPERATION_CONFIG")); + stmt2.setInt(3, rs.getInt("ENABLED")); + stmt2.setTimestamp(4,this.currentTimestamp ); + stmt2.addBatch(); + + if (++count % batchSize == 0) { + stmt2.executeBatch(); + } + } + stmt2.executeBatch(); + if (log.isDebugEnabled()) { + log.debug(count + " [CONFIG_OPERATION] Records copied to the archival table. Starting deletion"); + } + sql = "DELETE FROM DM_CONFIG_OPERATION" + + " WHERE OPERATION_ID IN (SELECT ID FROM DM_ARCHIVED_OPERATIONS)"; + stmt3 = conn.createStatement(); + int affected = stmt3.executeUpdate(sql); + if (log.isDebugEnabled()) { + log.debug(affected + " Rows deleted"); + } + } catch (SQLException e) { + throw new ArchivalDAOException("Error occurred while moving config operations", e); + } finally { + ArchivalDAOUtil.cleanupResources(stmt, rs); + ArchivalDAOUtil.cleanupResources(stmt2); + ArchivalDAOUtil.cleanupResources(stmt3); + } + } + @Override public void moveEnrolmentMappings() throws ArchivalDAOException { Statement stmt = null; diff --git a/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/config/archival/ArchivalTaskConfiguration.java b/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/config/archival/ArchivalTaskConfiguration.java index 32e4aa178be..609172beb5e 100644 --- a/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/config/archival/ArchivalTaskConfiguration.java +++ b/components/device-mgt/org.wso2.carbon.device.mgt.core/src/main/java/org/wso2/carbon/device/mgt/core/config/archival/ArchivalTaskConfiguration.java @@ -29,6 +29,7 @@ public class ArchivalTaskConfiguration { private int retentionPeriod; private int batchSize; private PurgingTaskConfiguration purgingTaskConfiguration; + private final int MULTIPLIER = -1; @XmlElement(name = "Enabled", required = true) public boolean isEnabled() { @@ -59,7 +60,8 @@ public class ArchivalTaskConfiguration { @XmlElement(name = "RetentionPeriod", required = true) public int getRetentionPeriod() { - return retentionPeriod; + // multiply by -1 to get the diff + return retentionPeriod * MULTIPLIER; } public void setRetentionPeriod(int retentionPeriod) { diff --git a/features/device-mgt/org.wso2.carbon.device.mgt.basics.feature/src/main/resources/dbscripts/cdm/archival/mysql.sql b/features/device-mgt/org.wso2.carbon.device.mgt.basics.feature/src/main/resources/dbscripts/cdm/archival/mysql.sql index 56dc7a7755e..630cb495dc5 100644 --- a/features/device-mgt/org.wso2.carbon.device.mgt.basics.feature/src/main/resources/dbscripts/cdm/archival/mysql.sql +++ b/features/device-mgt/org.wso2.carbon.device.mgt.basics.feature/src/main/resources/dbscripts/cdm/archival/mysql.sql @@ -52,6 +52,14 @@ CREATE TABLE IF NOT EXISTS DM_COMMAND_OPERATION_ARCH ( PRIMARY KEY (OPERATION_ID) )ENGINE = InnoDB; +CREATE TABLE IF NOT EXISTS DM_CONFIG_OPERATION_ARCH ( + OPERATION_ID INTEGER NOT NULL, + OPERATION_CONFIG BLOB DEFAULT NULL, + ENABLED BOOLEAN NOT NULL DEFAULT FALSE, + ARCHIVED_AT TIMESTAMP DEFAULT NOW(), + PRIMARY KEY (OPERATION_ID) +)ENGINE = InnoDB; + CREATE TABLE IF NOT EXISTS DM_PROFILE_OPERATION_ARCH ( OPERATION_ID INTEGER NOT NULL, ENABLED INTEGER NOT NULL DEFAULT 0,