diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java index 19b487193834..e7bf18b4fd8f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java @@ -1041,25 +1041,23 @@ public void testInvalidMaxPointNumber() { @Test public void testStorageGroupWithHyphenInName() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.setFetchSize(5); statement.execute("CREATE DATABASE root.group_with_hyphen"); - } catch (SQLException e) { + } catch (final SQLException e) { fail(); } - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) { while (resultSet.next()) { - StringBuilder builder = new StringBuilder(); - builder.append(resultSet.getString(1)); - Assert.assertEquals(builder.toString(), "root.group_with_hyphen"); + Assert.assertEquals("root.group_with_hyphen", resultSet.getString(1)); + Assert.assertEquals("TREE", resultSet.getString(12)); } } - } catch (SQLException e) { + } catch (final SQLException e) { fail(); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java index 34b8af428a38..1b4f100bb44c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java @@ -47,7 +47,7 @@ public class IoTDBMultiIDsWithAttributesTableIT { "CREATE DATABASE db", "USE db", "CREATE TABLE table0 (device string id, level string id, attr1 string attribute, attr2 string attribute, num int32 measurement, bigNum int64 measurement, " - + "floatNum double measurement, str TEXT measurement, bool BOOLEAN measurement, date DATE measurement, blob BLOB measurement, ts TIMESTAMP measurement, stringV STRING measurement, doubleNum DOUBLE measurement)", + + "floatNum FLOAT measurement, str TEXT measurement, bool BOOLEAN measurement, date DATE measurement, blob BLOB measurement, ts TIMESTAMP measurement, stringV STRING measurement, doubleNum DOUBLE measurement)", "insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool) values('d1', 'l1', 'c', 'd', 0,3,2947483648,231.2121,'coconut',FALSE)", "insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool,blob,ts,doubleNum) values('d1', 'l2', 'y', 'z', 20,2,2147483648,434.12,'pineapple',TRUE,X'108DCD62',2024-09-24T06:15:35.000+00:00,6666.8)", "insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool) values('d1', 'l3', 't', 'a', 40,1,2247483648,12.123,'apricot',TRUE)", @@ -1251,7 +1251,44 @@ public void modeTest() { "select mode(device),mode(level),mode(attr1),mode(attr2),mode(date),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum) from table0 where device='d2' and level='l1'"; retArray = new String[] { - "d2,l1,d,c,null,false,null,2024-08-01T06:15:35.000Z,test-string3,null,", + "d2,l1,d,c,null,false,null,null,null,null,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(1); + sql = + "select mode(stringv) from table0 where device='d2' and level='l1' and stringv is not null"; + retArray = + new String[] { + "test-string3,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // no push-down, test GroupedAccumulator + expectedHeader = buildHeaders(16); + sql = + "select mode(time),mode(device),mode(level),mode(attr1),mode(attr2),mode(num),mode(bignum),mode(floatnum),mode(date),mode(str),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum),count(num+1) from table0 where device='d2' and level='l4' and time=80 group by device, level"; + retArray = + new String[] { + "1970-01-01T00:00:00.080Z,d2,l4,null,null,9,2147483646,43.12,null,apple,false,null,2024-09-20T06:15:35.000Z,test-string2,6666.7,1,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(11); + sql = + "select mode(device),mode(level),mode(attr1),mode(attr2),mode(date),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum),count(num+1) from table0 where device='d2' and level='l1' group by device, level"; + retArray = + new String[] { + "d2,l1,d,c,null,false,null,null,null,null,3,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(2); + sql = + "select mode(stringv),count(num+1) from table0 where device='d2' and level='l1' and stringv is not null group by device, level"; + retArray = + new String[] { + "test-string3,1,", }; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } @@ -1269,6 +1306,43 @@ public void varianceTest() { "16.0,10.7,16.0,4.0,3.3,4.0,50.0,33.3,50.0,7.1,5.8,7.1,null,0.0,null,null,0.0,null,", }; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(19); + sql = + "select \n" + + "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n" + + "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n" + + "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1), count(num+1) from table0 where device='d2' and level='l4' group by device, level"; + retArray = + new String[] { + "16.0,10.7,16.0,4.0,3.3,4.0,50.0,33.3,50.0,7.1,5.8,7.1,null,0.0,null,null,0.0,null,3,", + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "select \n" + + "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n" + + "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n" + + "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1), count(num+1) from table0 group by device"; + retArray = + new String[] { + "20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.1,0.1,0.1,0.4,0.2,0.4,15,", + "20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.0,0.0,0.0,0.2,0.2,0.2,15," + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = buildHeaders(18); + sql = + "select \n" + + "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n" + + "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n" + + "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1) from table0 group by device"; + retArray = + new String[] { + "20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.1,0.1,0.1,0.4,0.2,0.4,", + "20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.0,0.0,0.0,0.2,0.2,0.2," + }; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } // ================================================================== diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java index d24d50fc4312..db0530bf41f9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java @@ -32,6 +32,7 @@ import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.apache.iotdb.relational.it.db.it.IoTDBMultiIDsWithAttributesTableIT.buildHeaders; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) @@ -3608,4 +3609,20 @@ public void specialCasesTest() { retArray, DATABASE_NAME); } + + @Test + public void modeTest() { + // AggTableScan + Agg mixed test + String[] expectedHeader = buildHeaders(11); + String[] retArray = + new String[] { + "A,null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + "A,null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,", + }; + tableResultSetEqualTest( + "select mode(type), mode(s1),mode(s2),mode(s3),mode(s4),mode(s5),mode(s6),mode(s7),mode(s8),mode(s9),mode(s10) from table1 group by city", + expectedHeader, + retArray, + DATABASE_NAME); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index d15528268720..1c8683f6c325 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -40,6 +40,7 @@ import java.util.Collections; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showDBColumnHeaders; +import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showDBDetailsColumnHeaders; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -83,6 +84,7 @@ public void testManageDatabase() { int[] schemaReplicaFactors = new int[] {1}; int[] dataReplicaFactors = new int[] {1}; int[] timePartitionInterval = new int[] {604800000}; + String[] model = new String[] {"TABLE"}; // show try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) { @@ -102,6 +104,26 @@ public void testManageDatabase() { assertEquals(databaseNames.length, cnt); } + // show + try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) { + int cnt = 0; + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showDBDetailsColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showDBDetailsColumnHeaders.size(); i++) { + assertEquals( + showDBDetailsColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + while (resultSet.next()) { + assertEquals(databaseNames[cnt], resultSet.getString(1)); + assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(2)); + assertEquals(dataReplicaFactors[cnt], resultSet.getInt(3)); + assertEquals(timePartitionInterval[cnt], resultSet.getLong(4)); + assertEquals(model[cnt], resultSet.getString(5)); + cnt++; + } + assertEquals(databaseNames.length, cnt); + } + // use statement.execute("use test"); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 70175e3ec359..bdd93f9844b4 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -86,6 +86,7 @@ public enum TSStatusCode { MEASUREMENT_ALREADY_EXISTS_IN_TEMPLATE(527), TYPE_NOT_FOUND(528), DATABASE_CONFLICT(529), + DATABASE_MODEL(530), TABLE_NOT_EXISTS(550), TABLE_ALREADY_EXISTS(551), diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 93e05ed09926..356cd20ed6e6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -726,8 +726,8 @@ public TSStatus setDatabase(final DatabaseSchemaPlan databaseSchemaPlan) { } @Override - public TSStatus alterDatabase(DatabaseSchemaPlan databaseSchemaPlan) { - TSStatus status = confirmLeader(); + public TSStatus alterDatabase(final DatabaseSchemaPlan databaseSchemaPlan) { + final TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return clusterSchemaManager.alterDatabase(databaseSchemaPlan, false); } else { @@ -736,19 +736,19 @@ public TSStatus alterDatabase(DatabaseSchemaPlan databaseSchemaPlan) { } @Override - public synchronized TSStatus deleteDatabases(TDeleteDatabasesReq tDeleteReq) { - TSStatus status = confirmLeader(); + public synchronized TSStatus deleteDatabases(final TDeleteDatabasesReq tDeleteReq) { + final TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - List deletedPaths = tDeleteReq.getPrefixPathList(); + final List deletedPaths = tDeleteReq.getPrefixPathList(); // remove wild - Map deleteDatabaseSchemaMap = + final Map deleteDatabaseSchemaMap = getClusterSchemaManager().getMatchedDatabaseSchemasByName(deletedPaths); if (deleteDatabaseSchemaMap.isEmpty()) { return RpcUtils.getStatus( TSStatusCode.PATH_NOT_EXIST.getStatusCode(), String.format("Path %s does not exist", Arrays.toString(deletedPaths.toArray()))); } - ArrayList parsedDeleteDatabases = + final ArrayList parsedDeleteDatabases = new ArrayList<>(deleteDatabaseSchemaMap.values()); return procedureManager.deleteDatabases( parsedDeleteDatabases, @@ -1834,14 +1834,15 @@ public TShowConfigNodesResp showConfigNodes() { } @Override - public TShowDatabaseResp showDatabase(TGetDatabaseReq req) { - TSStatus status = confirmLeader(); + public TShowDatabaseResp showDatabase(final TGetDatabaseReq req) { + final TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - PathPatternTree scope = + final PathPatternTree scope = req.getScopePatternTree() == null ? SchemaConstant.ALL_MATCH_SCOPE : PathPatternTree.deserialize(ByteBuffer.wrap(req.getScopePatternTree())); - GetDatabasePlan getDatabasePlan = new GetDatabasePlan(req.getDatabasePathPattern(), scope); + final GetDatabasePlan getDatabasePlan = + new GetDatabasePlan(req.getDatabasePathPattern(), scope); return getClusterSchemaManager().showDatabase(getDatabasePlan); } else { return new TShowDatabaseResp().setStatus(status); @@ -1943,17 +1944,18 @@ public TGetPathsSetTemplatesResp getPathsSetTemplate(TGetPathsSetTemplatesReq re } @Override - public TSStatus deactivateSchemaTemplate(TDeactivateSchemaTemplateReq req) { - TSStatus status = confirmLeader(); + public TSStatus deactivateSchemaTemplate(final TDeactivateSchemaTemplateReq req) { + final TSStatus status = confirmLeader(); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return status; } - PathPatternTree patternTree = + final PathPatternTree patternTree = PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree())); - List patternList = patternTree.getAllPathPatterns(); - TemplateSetInfoResp templateSetInfoResp = clusterSchemaManager.getTemplateSetInfo(patternList); + final List patternList = patternTree.getAllPathPatterns(); + final TemplateSetInfoResp templateSetInfoResp = + clusterSchemaManager.getTemplateSetInfo(patternList); if (templateSetInfoResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return templateSetInfoResp.getStatus(); } @@ -1968,9 +1970,9 @@ public TSStatus deactivateSchemaTemplate(TDeactivateSchemaTemplateReq req) { } if (!req.getTemplateName().equals(ONE_LEVEL_PATH_WILDCARD)) { - Map> filteredTemplateSetInfo = new HashMap<>(); - for (Map.Entry> entry : templateSetInfo.entrySet()) { - for (Template template : entry.getValue()) { + final Map> filteredTemplateSetInfo = new HashMap<>(); + for (final Map.Entry> entry : templateSetInfo.entrySet()) { + for (final Template template : entry.getValue()) { if (template.getName().equals(req.getTemplateName())) { filteredTemplateSetInfo.put(entry.getKey(), Collections.singletonList(template)); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index 43f17bcfb6de..671675000337 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -222,7 +222,7 @@ private TSStatus executePlan(final ConfigPhysicalPlan plan) throws ConsensusExce case CreateDatabase: // Here we only reserve database name and substitute the sender's local information // with the receiver's default configurations - TDatabaseSchema schema = ((DatabaseSchemaPlan) plan).getSchema(); + final TDatabaseSchema schema = ((DatabaseSchemaPlan) plan).getSchema(); schema.setSchemaReplicationFactor( ConfigNodeDescriptor.getInstance().getConf().getSchemaReplicationFactor()); schema.setDataReplicationFactor( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index c306d8fe1ef2..c319f7eb9aa9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -336,11 +336,11 @@ public DatabaseSchemaResp getMatchedDatabaseSchema(GetDatabasePlan getDatabasePl } /** Only used in cluster tool show Databases. */ - public TShowDatabaseResp showDatabase(GetDatabasePlan getDatabasePlan) { + public TShowDatabaseResp showDatabase(final GetDatabasePlan getDatabasePlan) { DatabaseSchemaResp databaseSchemaResp; try { databaseSchemaResp = (DatabaseSchemaResp) getConsensusManager().read(getDatabasePlan); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_READ_ERROR, e); TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); res.setMessage(e.getMessage()); @@ -352,10 +352,10 @@ public TShowDatabaseResp showDatabase(GetDatabasePlan getDatabasePlan) { return new TShowDatabaseResp().setStatus(databaseSchemaResp.getStatus()); } - Map infoMap = new ConcurrentHashMap<>(); - for (TDatabaseSchema databaseSchema : databaseSchemaResp.getSchemaMap().values()) { - String database = databaseSchema.getName(); - TDatabaseInfo databaseInfo = new TDatabaseInfo(); + final Map infoMap = new ConcurrentHashMap<>(); + for (final TDatabaseSchema databaseSchema : databaseSchemaResp.getSchemaMap().values()) { + final String database = databaseSchema.getName(); + final TDatabaseInfo databaseInfo = new TDatabaseInfo(); databaseInfo.setName(database); databaseInfo.setSchemaReplicationFactor(databaseSchema.getSchemaReplicationFactor()); databaseInfo.setDataReplicationFactor(databaseSchema.getDataReplicationFactor()); @@ -369,13 +369,14 @@ public TShowDatabaseResp showDatabase(GetDatabasePlan getDatabasePlan) { getMinRegionGroupNum(database, TConsensusGroupType.DataRegion)); databaseInfo.setMaxDataRegionNum( getMaxRegionGroupNum(database, TConsensusGroupType.DataRegion)); + databaseInfo.setIsTableModel(databaseSchema.isIsTableModel()); try { databaseInfo.setSchemaRegionNum( getPartitionManager().getRegionGroupCount(database, TConsensusGroupType.SchemaRegion)); databaseInfo.setDataRegionNum( getPartitionManager().getRegionGroupCount(database, TConsensusGroupType.DataRegion)); - } catch (DatabaseNotExistsException e) { + } catch (final DatabaseNotExistsException e) { // Skip pre-deleted Database LOGGER.warn( "The Database: {} doesn't exist. Maybe it has been pre-deleted.", diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index fe48dd507977..a6ee20ea4d7f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -991,9 +991,9 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept } } - public void processLoadSnapshot(File snapshotDir) throws TException, IOException { + public void processLoadSnapshot(final File snapshotDir) throws TException, IOException { - File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME); + final File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME); if (!snapshotFile.exists() || !snapshotFile.isFile()) { LOGGER.error( "Failed to load snapshot,snapshot file [{}] is not exist.", @@ -1001,11 +1001,11 @@ public void processLoadSnapshot(File snapshotDir) throws TException, IOException return; } - try (BufferedInputStream fileInputStream = + try (final BufferedInputStream fileInputStream = new BufferedInputStream( Files.newInputStream(snapshotFile.toPath()), PARTITION_TABLE_BUFFER_SIZE); - TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileInputStream)) { - TProtocol protocol = new TBinaryProtocol(tioStreamTransport); + final TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileInputStream)) { + final TProtocol protocol = new TBinaryProtocol(tioStreamTransport); // before restoring a snapshot, clear all old data clear(); @@ -1015,11 +1015,12 @@ public void processLoadSnapshot(File snapshotDir) throws TException, IOException // restore StorageGroupPartitionTable int length = ReadWriteIOUtils.readInt(fileInputStream); for (int i = 0; i < length; i++) { - String storageGroup = ReadWriteIOUtils.readString(fileInputStream); + final String storageGroup = ReadWriteIOUtils.readString(fileInputStream); if (storageGroup == null) { throw new IOException("Failed to load snapshot because get null StorageGroup name"); } - DatabasePartitionTable databasePartitionTable = new DatabasePartitionTable(storageGroup); + final DatabasePartitionTable databasePartitionTable = + new DatabasePartitionTable(storageGroup); databasePartitionTable.deserialize(fileInputStream, protocol); databasePartitionTables.put(storageGroup, databasePartitionTable); } @@ -1027,7 +1028,8 @@ public void processLoadSnapshot(File snapshotDir) throws TException, IOException // restore deletedRegionSet length = ReadWriteIOUtils.readInt(fileInputStream); for (int i = 0; i < length; i++) { - RegionMaintainTask task = RegionMaintainTask.Factory.create(fileInputStream, protocol); + final RegionMaintainTask task = + RegionMaintainTask.Factory.create(fileInputStream, protocol); regionMaintainTaskList.add(task); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 22991eedb2b5..ebd754b9b096 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -154,7 +154,7 @@ public ClusterSchemaInfo() throws IOException { /** * Cache DatabaseSchema. * - * @param plan DatabaseSchemaPlan + * @param plan {@link DatabaseSchemaPlan} * @return {@link TSStatusCode#SUCCESS_STATUS} if the Database is set successfully. */ public TSStatus createDatabase(final DatabaseSchemaPlan plan) { @@ -195,7 +195,7 @@ public TSStatus alterDatabase(final DatabaseSchemaPlan plan) { final TDatabaseSchema alterSchema = plan.getSchema(); final PartialPath partialPathName = new PartialPath(alterSchema.getName()); - TDatabaseSchema currentSchema = + final TDatabaseSchema currentSchema = mTree.getDatabaseNodeByDatabasePath(partialPathName).getAsMNode().getDatabaseSchema(); // TODO: Support alter other fields if (alterSchema.isSetMinSchemaRegionGroupNum()) { @@ -234,7 +234,7 @@ public TSStatus alterDatabase(final DatabaseSchemaPlan plan) { .getAsMNode() .setDatabaseSchema(currentSchema); result.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - } catch (MetadataException e) { + } catch (final MetadataException e) { LOGGER.error(ERROR_NAME, e); result.setCode(e.getErrorCode()).setMessage(e.getMessage()); } finally { @@ -455,7 +455,7 @@ public TSStatus adjustMaxRegionGroupCount(final AdjustMaxRegionGroupNumPlan plan public List getDatabaseNames() { databaseReadWriteLock.readLock().lock(); try { - return mTree.getAllDatabasePaths().stream() + return mTree.getAllDatabasePaths(false).stream() .map(PartialPath::getFullPath) .collect(Collectors.toList()); } finally { @@ -759,12 +759,12 @@ public Template getTemplate(int id) throws MetadataException { } public synchronized TemplateInfoResp checkTemplateSettable( - CheckTemplateSettablePlan checkTemplateSettablePlan) { - TemplateInfoResp resp = new TemplateInfoResp(); - PartialPath path; + final CheckTemplateSettablePlan checkTemplateSettablePlan) { + final TemplateInfoResp resp = new TemplateInfoResp(); + final PartialPath path; try { path = new PartialPath(checkTemplateSettablePlan.getPath()); - } catch (IllegalPathException e) { + } catch (final IllegalPathException e) { LOGGER.error(e.getMessage()); resp.setStatus(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); return resp; @@ -776,7 +776,7 @@ public synchronized TemplateInfoResp checkTemplateSettable( resp.setTemplateList( Collections.singletonList( templateTable.getTemplate(checkTemplateSettablePlan.getName()))); - } catch (MetadataException e) { + } catch (final MetadataException e) { LOGGER.error(e.getMessage(), e); resp.setStatus(RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); } @@ -933,13 +933,12 @@ public AllTemplateSetInfoResp getAllTemplateSetInfo() { * into specified path pattern start with template set path. The result set is organized as * specified path pattern -> template id */ - public TemplateSetInfoResp getTemplateSetInfo(GetTemplateSetInfoPlan plan) { - TemplateSetInfoResp resp = new TemplateSetInfoResp(); + public TemplateSetInfoResp getTemplateSetInfo(final GetTemplateSetInfoPlan plan) { + final TemplateSetInfoResp resp = new TemplateSetInfoResp(); try { - - Map> allTemplateSetInfo = new HashMap<>(); - for (PartialPath pattern : plan.getPatternList()) { - Map> templateSetInfo = mTree.getTemplateSetInfo(pattern); + final Map> allTemplateSetInfo = new HashMap<>(); + for (final PartialPath pattern : plan.getPatternList()) { + final Map> templateSetInfo = mTree.getTemplateSetInfo(pattern); if (templateSetInfo.isEmpty()) { continue; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index df93713239ac..f761f5f1aa14 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -36,6 +36,7 @@ import org.apache.iotdb.confignode.persistence.schema.mnode.impl.ConfigTableNode; import org.apache.iotdb.db.exception.metadata.DatabaseAlreadySetException; import org.apache.iotdb.db.exception.metadata.DatabaseConflictException; +import org.apache.iotdb.db.exception.metadata.DatabaseModelException; import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; @@ -227,13 +228,16 @@ protected void collectDatabase(IDatabaseMNode node) { * * @return a list contains all distinct databases */ - public List getAllDatabasePaths() { - List res = new ArrayList<>(); - Deque nodeStack = new ArrayDeque<>(); + public List getAllDatabasePaths(final boolean onlyTableModel) { + final List res = new ArrayList<>(); + final Deque nodeStack = new ArrayDeque<>(); nodeStack.add(root); while (!nodeStack.isEmpty()) { - IConfigMNode current = nodeStack.pop(); + final IConfigMNode current = nodeStack.pop(); if (current.isDatabase()) { + if (onlyTableModel && !current.getDatabaseSchema().isIsTableModel()) { + continue; + } res.add(current.getPartialPath()); } else { nodeStack.addAll(current.getChildren().values()); @@ -457,7 +461,7 @@ protected Void collectMNode(IConfigMNode node) { * check whether there is template on given path and the subTree has template return true, * otherwise false */ - public void checkTemplateOnPath(PartialPath path) throws MetadataException { + public void checkTemplateOnPath(final PartialPath path) throws MetadataException { String[] nodeNames = path.getNodes(); IConfigMNode cur = root; IConfigMNode child; @@ -475,25 +479,23 @@ public void checkTemplateOnPath(PartialPath path) throws MetadataException { if (cur.getSchemaTemplateId() != NON_TEMPLATE) { throw new MetadataException("Template already exists on " + cur.getFullPath()); } - if (cur.isMeasurement()) { - return; + if (cur.isDatabase() && cur.getDatabaseSchema().isIsTableModel()) { + throw new DatabaseModelException(cur.getFullPath(), true); } } checkTemplateOnSubtree(cur); } - // traverse all the descendant of the given path node - private void checkTemplateOnSubtree(IConfigMNode node) throws MetadataException { - if (node.isMeasurement()) { - return; - } + // traverse all the descendant of the given path node + private void checkTemplateOnSubtree(final IConfigMNode node) throws MetadataException { IConfigMNode child; IMNodeIterator iterator = store.getChildrenIterator(node); while (iterator.hasNext()) { child = iterator.next(); - if (child.isMeasurement()) { + // Skip table model databases + if (child.isDatabase() && child.getDatabaseSchema().isIsTableModel()) { continue; } if (child.getSchemaTemplateId() != NON_TEMPLATE) { @@ -504,13 +506,14 @@ private void checkTemplateOnSubtree(IConfigMNode node) throws MetadataException } public List getPathsSetOnTemplate( - int templateId, PathPatternTree scope, boolean filterPreUnset) throws MetadataException { - List resSet = new ArrayList<>(); - try (MNodeCollector collector = + final int templateId, final PathPatternTree scope, final boolean filterPreUnset) + throws MetadataException { + final List resSet = new ArrayList<>(); + try (final MNodeCollector collector = new MNodeCollector( root, new PartialPath(ALL_RESULT_NODES), store, false, scope) { @Override - protected boolean acceptFullMatchedNode(IConfigMNode node) { + protected boolean acceptFullMatchedNode(final IConfigMNode node) { if (super.acceptFullMatchedNode(node)) { // if node not set template, go on traversing if (node.getSchemaTemplateId() != NON_TEMPLATE) { @@ -526,23 +529,25 @@ protected boolean acceptFullMatchedNode(IConfigMNode node) { } @Override - protected Void collectMNode(IConfigMNode node) { + protected Void collectMNode(final IConfigMNode node) { resSet.add(node.getFullPath()); return null; } @Override - protected boolean shouldVisitSubtreeOfFullMatchedNode(IConfigMNode node) { + protected boolean shouldVisitSubtreeOfFullMatchedNode(final IConfigMNode node) { // descendants of the node cannot set another template, exit from this branch - return (node.getSchemaTemplateId() == NON_TEMPLATE) + return node.getSchemaTemplateId() == NON_TEMPLATE + && !(node.isDatabase() && node.getDatabaseSchema().isIsTableModel()) && super.shouldVisitSubtreeOfFullMatchedNode(node); } @Override - protected boolean shouldVisitSubtreeOfInternalMatchedNode(IConfigMNode node) { + protected boolean shouldVisitSubtreeOfInternalMatchedNode(final IConfigMNode node) { // descendants of the node cannot set another template, exit from this branch - return (node.getSchemaTemplateId() == NON_TEMPLATE) - && super.shouldVisitSubtreeOfFullMatchedNode(node); + return node.getSchemaTemplateId() == NON_TEMPLATE + && !(node.isDatabase() && node.getDatabaseSchema().isIsTableModel()) + && super.shouldVisitSubtreeOfInternalMatchedNode(node); } }) { collector.traverse(); @@ -551,25 +556,24 @@ protected boolean shouldVisitSubtreeOfInternalMatchedNode(IConfigMNode node) { } /** This method returns the templateId set on paths covered by input path pattern. */ - public Map> getTemplateSetInfo(PartialPath pathPattern) + public Map> getTemplateSetInfo(final PartialPath pathPattern) throws MetadataException { - Map> result = new HashMap<>(); - try (MNodeCollector collector = + final Map> result = new HashMap<>(); + try (final MNodeCollector collector = new MNodeCollector(root, pathPattern, store, false, ALL_MATCH_SCOPE) { @Override - protected boolean acceptFullMatchedNode(IConfigMNode node) { - return (node.getSchemaTemplateId() != NON_TEMPLATE) - || super.acceptFullMatchedNode(node); + protected boolean acceptFullMatchedNode(final IConfigMNode node) { + return node.getSchemaTemplateId() != NON_TEMPLATE || super.acceptFullMatchedNode(node); } @Override - protected boolean acceptInternalMatchedNode(IConfigMNode node) { - return (node.getSchemaTemplateId() != NON_TEMPLATE) + protected boolean acceptInternalMatchedNode(final IConfigMNode node) { + return node.getSchemaTemplateId() != NON_TEMPLATE || super.acceptInternalMatchedNode(node); } @Override - protected Void collectMNode(IConfigMNode node) { + protected Void collectMNode(final IConfigMNode node) { if (node.getSchemaTemplateId() != NON_TEMPLATE && !node.isSchemaTemplatePreUnset()) { result .computeIfAbsent(node.getSchemaTemplateId(), k -> new HashSet<>()) @@ -579,17 +583,19 @@ protected Void collectMNode(IConfigMNode node) { } @Override - protected boolean shouldVisitSubtreeOfFullMatchedNode(IConfigMNode node) { + protected boolean shouldVisitSubtreeOfFullMatchedNode(final IConfigMNode node) { // descendants of the node cannot set another template, exit from this branch - return (node.getSchemaTemplateId() == NON_TEMPLATE) + return node.getSchemaTemplateId() == NON_TEMPLATE + && !(node.isDatabase() && node.getDatabaseSchema().isIsTableModel()) && super.shouldVisitSubtreeOfFullMatchedNode(node); } @Override - protected boolean shouldVisitSubtreeOfInternalMatchedNode(IConfigMNode node) { + protected boolean shouldVisitSubtreeOfInternalMatchedNode(final IConfigMNode node) { // descendants of the node cannot set another template, exit from this branch - return (node.getSchemaTemplateId() == NON_TEMPLATE) - && super.shouldVisitSubtreeOfFullMatchedNode(node); + return node.getSchemaTemplateId() == NON_TEMPLATE + && !(node.isDatabase() && node.getDatabaseSchema().isIsTableModel()) + && super.shouldVisitSubtreeOfInternalMatchedNode(node); } }) { collector.traverse(); @@ -597,7 +603,8 @@ protected boolean shouldVisitSubtreeOfInternalMatchedNode(IConfigMNode node) { return result; } - public void setTemplate(int templateId, PartialPath templateSetPath) throws MetadataException { + public void setTemplate(final int templateId, final PartialPath templateSetPath) + throws MetadataException { getNodeWithAutoCreate(templateSetPath).setSchemaTemplateId(templateId); } @@ -637,6 +644,9 @@ private IConfigMNode getNodeSetTemplate(int templateId, PartialPath path) public void preCreateTable(final PartialPath database, final TsTable table) throws MetadataException { final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.getDatabaseSchema().isIsTableModel()) { + throw new DatabaseModelException(database.getFullPath(), false); + } final IConfigMNode node = databaseNode.getChild(table.getTableName()); if (node == null) { final ConfigTableNode tableNode = @@ -735,7 +745,7 @@ public Map getSpecificTablesUnderSpecificDatabase( } public Map> getAllUsingTables() { - return getAllDatabasePaths().stream() + return getAllDatabasePaths(true).stream() .collect( Collectors.toMap( PartialPath::getFullPath, @@ -752,7 +762,7 @@ public Map> getAllUsingTables() { public Map> getAllPreCreateTables() throws MetadataException { final Map> result = new HashMap<>(); - final List databaseList = getAllDatabasePaths(); + final List databaseList = getAllDatabasePaths(true); for (PartialPath databasePath : databaseList) { final String database = databasePath.getFullPath().substring(ROOT.length() + 1); final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(databasePath).getAsMNode(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/IConfigMNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/IConfigMNode.java index 805c270ec462..dd2b5932b003 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/IConfigMNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/IConfigMNode.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.confignode.persistence.schema.mnode; import org.apache.iotdb.commons.schema.node.IMNode; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigDatabaseMNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigDatabaseMNode.java index ea5e8b51b945..67058b68c277 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigDatabaseMNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigDatabaseMNode.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.confignode.persistence.schema.mnode.impl; import org.apache.iotdb.commons.schema.node.common.AbstractDatabaseMNode; @@ -28,13 +29,13 @@ public class ConfigDatabaseMNode extends AbstractDatabaseMNode pipeTaskInfo; @@ -298,10 +303,13 @@ protected void rollbackState(ConfigNodeProcedureEnv env, OperatePipeTaskState st switch (state) { case VALIDATE_TASK: - try { - rollbackFromValidateTask(env); - } catch (Exception e) { - LOGGER.warn("ProcedureId {}: Failed to rollback from validate task.", getProcId(), e); + if (!isRollbackFromValidateTaskSuccessful) { + try { + rollbackFromValidateTask(env); + isRollbackFromValidateTaskSuccessful = true; + } catch (Exception e) { + LOGGER.warn("ProcedureId {}: Failed to rollback from validate task.", getProcId(), e); + } } break; case CALCULATE_INFO_FOR_TASK: @@ -330,7 +338,7 @@ protected void rollbackState(ConfigNodeProcedureEnv env, OperatePipeTaskState st break; case OPERATE_ON_DATA_NODES: try { - // We have to make sure that rollbackFromOperateOnDataNodes is executed before + // We have to make sure that rollbackFromOperateOnDataNodes is executed after // rollbackFromWriteConfigNodeConsensus, because rollbackFromOperateOnDataNodes is // executed based on the consensus of config nodes that is written by // rollbackFromWriteConfigNodeConsensus diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java index 30f818164841..9180904cade1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTemplateProcedure.java @@ -83,12 +83,15 @@ public class SetTemplateProcedure private static final String CONSENSUS_WRITE_ERROR = "Failed in the write API executing the consensus layer due to: "; - public SetTemplateProcedure(boolean isGeneratedByPipe) { + public SetTemplateProcedure(final boolean isGeneratedByPipe) { super(isGeneratedByPipe); } public SetTemplateProcedure( - String queryId, String templateName, String templateSetPath, boolean isGeneratedByPipe) { + final String queryId, + final String templateName, + final String templateSetPath, + final boolean isGeneratedByPipe) { super(isGeneratedByPipe); this.queryId = queryId; this.templateName = templateName; @@ -96,7 +99,7 @@ public SetTemplateProcedure( } @Override - protected Flow executeFromState(ConfigNodeProcedureEnv env, SetTemplateState state) + protected Flow executeFromState(final ConfigNodeProcedureEnv env, final SetTemplateState state) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { long startTime = System.currentTimeMillis(); try { @@ -148,18 +151,18 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, SetTemplateState sta } } - private void validateTemplateExistence(ConfigNodeProcedureEnv env) { + private void validateTemplateExistence(final ConfigNodeProcedureEnv env) { // check whether the template can be set on given path - CheckTemplateSettablePlan checkTemplateSettablePlan = + final CheckTemplateSettablePlan checkTemplateSettablePlan = new CheckTemplateSettablePlan(templateName, templateSetPath); TemplateInfoResp resp; try { resp = (TemplateInfoResp) env.getConfigManager().getConsensusManager().read(checkTemplateSettablePlan); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); - TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + final TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); res.setMessage(e.getMessage()); resp = new TemplateInfoResp(); resp.setStatus(res); @@ -173,13 +176,13 @@ private void validateTemplateExistence(ConfigNodeProcedureEnv env) { } } - private void preSetTemplate(ConfigNodeProcedureEnv env) { - PreSetSchemaTemplatePlan preSetSchemaTemplatePlan = + private void preSetTemplate(final ConfigNodeProcedureEnv env) { + final PreSetSchemaTemplatePlan preSetSchemaTemplatePlan = new PreSetSchemaTemplatePlan(templateName, templateSetPath); TSStatus status; try { status = env.getConfigManager().getConsensusManager().write(preSetSchemaTemplatePlan); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); status.setMessage(e.getMessage()); @@ -196,26 +199,26 @@ private void preSetTemplate(ConfigNodeProcedureEnv env) { } } - private void preReleaseTemplate(ConfigNodeProcedureEnv env) { - Template template = getTemplate(env); + private void preReleaseTemplate(final ConfigNodeProcedureEnv env) { + final Template template = getTemplate(env); if (template == null) { // already setFailure return; } - TUpdateTemplateReq req = new TUpdateTemplateReq(); + final TUpdateTemplateReq req = new TUpdateTemplateReq(); req.setType(TemplateInternalRPCUpdateType.ADD_TEMPLATE_PRE_SET_INFO.toByte()); req.setTemplateInfo( TemplateInternalRPCUtil.generateAddTemplateSetInfoBytes(template, templateSetPath)); - Map dataNodeLocationMap = + final Map dataNodeLocationMap = env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); - DataNodeAsyncRequestContext clientHandler = + final DataNodeAsyncRequestContext clientHandler = new DataNodeAsyncRequestContext<>( CnToDnAsyncRequestType.UPDATE_TEMPLATE, req, dataNodeLocationMap); CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); - Map statusMap = clientHandler.getResponseMap(); - for (Map.Entry entry : statusMap.entrySet()) { + final Map statusMap = clientHandler.getResponseMap(); + for (final Map.Entry entry : statusMap.entrySet()) { if (entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn( "Failed to sync template {} pre-set info on path {} to DataNode {}", @@ -229,16 +232,16 @@ private void preReleaseTemplate(ConfigNodeProcedureEnv env) { setNextState(SetTemplateState.VALIDATE_TIMESERIES_EXISTENCE); } - private Template getTemplate(ConfigNodeProcedureEnv env) { - GetSchemaTemplatePlan getSchemaTemplatePlan = new GetSchemaTemplatePlan(templateName); + private Template getTemplate(final ConfigNodeProcedureEnv env) { + final GetSchemaTemplatePlan getSchemaTemplatePlan = new GetSchemaTemplatePlan(templateName); TemplateInfoResp templateResp; try { templateResp = (TemplateInfoResp) env.getConfigManager().getConsensusManager().read(getSchemaTemplatePlan); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); - TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + final TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); res.setMessage(e.getMessage()); templateResp = new TemplateInfoResp(); templateResp.setStatus(res); @@ -257,25 +260,25 @@ private Template getTemplate(ConfigNodeProcedureEnv env) { return templateResp.getTemplateList().get(0); } - private void validateTimeSeriesExistence(ConfigNodeProcedureEnv env) { - PathPatternTree patternTree = new PathPatternTree(); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + private void validateTimeSeriesExistence(final ConfigNodeProcedureEnv env) { + final PathPatternTree patternTree = new PathPatternTree(); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); PartialPath path = null; try { path = new PartialPath(templateSetPath); patternTree.appendPathPattern(path); patternTree.appendPathPattern(path.concatAsMeasurementPath(MULTI_LEVEL_PATH_WILDCARD)); patternTree.serialize(dataOutputStream); - } catch (IllegalPathException | IOException ignored) { + } catch (final IllegalPathException | IOException ignored) { } - ByteBuffer patternTreeBytes = ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); + final ByteBuffer patternTreeBytes = ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); - Map relatedSchemaRegionGroup = + final Map relatedSchemaRegionGroup = env.getConfigManager().getRelatedSchemaRegionGroup(patternTree); - List respList = new ArrayList<>(); - DataNodeRegionTaskExecutor + final List respList = new ArrayList<>(); + final DataNodeRegionTaskExecutor regionTask = new DataNodeRegionTaskExecutor< TCheckTimeSeriesExistenceReq, TCheckTimeSeriesExistenceResp>( @@ -288,17 +291,17 @@ private void validateTimeSeriesExistence(ConfigNodeProcedureEnv env) { @Override protected List processResponseOfOneDataNode( - TDataNodeLocation dataNodeLocation, - List consensusGroupIdList, - TCheckTimeSeriesExistenceResp response) { + final TDataNodeLocation dataNodeLocation, + final List consensusGroupIdList, + final TCheckTimeSeriesExistenceResp response) { respList.add(response); - List failedRegionList = new ArrayList<>(); + final List failedRegionList = new ArrayList<>(); if (response.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return failedRegionList; } if (response.getStatus().getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { - List subStatus = response.getStatus().getSubStatus(); + final List subStatus = response.getStatus().getSubStatus(); for (int i = 0; i < subStatus.size(); i++) { if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && subStatus.get(i).getCode() @@ -314,7 +317,8 @@ protected List processResponseOfOneDataNode( @Override protected void onAllReplicasetFailure( - TConsensusGroupId consensusGroupId, Set dataNodeLocationSet) { + final TConsensusGroupId consensusGroupId, + final Set dataNodeLocationSet) { setFailure( new ProcedureException( new MetadataException( @@ -333,7 +337,7 @@ protected void onAllReplicasetFailure( return; } - for (TCheckTimeSeriesExistenceResp resp : respList) { + for (final TCheckTimeSeriesExistenceResp resp : respList) { if (resp.isExists()) { setFailure(new ProcedureException(new TemplateIncompatibleException(templateName, path))); } @@ -341,8 +345,8 @@ protected void onAllReplicasetFailure( setNextState(SetTemplateState.COMMIT_SET); } - private void commitSetTemplate(ConfigNodeProcedureEnv env) { - CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan = + private void commitSetTemplate(final ConfigNodeProcedureEnv env) { + final CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan = new CommitSetSchemaTemplatePlan(templateName, templateSetPath); TSStatus status; try { @@ -353,7 +357,7 @@ private void commitSetTemplate(ConfigNodeProcedureEnv env) { isGeneratedByPipe ? new PipeEnrichedPlan(commitSetSchemaTemplatePlan) : commitSetSchemaTemplatePlan); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); status.setMessage(e.getMessage()); @@ -370,26 +374,26 @@ private void commitSetTemplate(ConfigNodeProcedureEnv env) { } } - private void commitReleaseTemplate(ConfigNodeProcedureEnv env) { - Template template = getTemplate(env); + private void commitReleaseTemplate(final ConfigNodeProcedureEnv env) { + final Template template = getTemplate(env); if (template == null) { // already setFailure return; } - TUpdateTemplateReq req = new TUpdateTemplateReq(); + final TUpdateTemplateReq req = new TUpdateTemplateReq(); req.setType(TemplateInternalRPCUpdateType.COMMIT_TEMPLATE_SET_INFO.toByte()); req.setTemplateInfo( TemplateInternalRPCUtil.generateAddTemplateSetInfoBytes(template, templateSetPath)); - Map dataNodeLocationMap = + final Map dataNodeLocationMap = env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); - DataNodeAsyncRequestContext clientHandler = + final DataNodeAsyncRequestContext clientHandler = new DataNodeAsyncRequestContext<>( CnToDnAsyncRequestType.UPDATE_TEMPLATE, req, dataNodeLocationMap); CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); - Map statusMap = clientHandler.getResponseMap(); - for (Map.Entry entry : statusMap.entrySet()) { + final Map statusMap = clientHandler.getResponseMap(); + for (final Map.Entry entry : statusMap.entrySet()) { if (entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn( "Failed to sync template {} commit-set info on path {} to DataNode {}", @@ -413,14 +417,14 @@ private void submitTemplateMaintainTask(TDataNodeLocation dataNodeLocation) { } @Override - protected boolean isRollbackSupported(SetTemplateState setTemplateState) { + protected boolean isRollbackSupported(final SetTemplateState setTemplateState) { return true; } @Override - protected void rollbackState(ConfigNodeProcedureEnv env, SetTemplateState state) + protected void rollbackState(final ConfigNodeProcedureEnv env, final SetTemplateState state) throws IOException, InterruptedException, ProcedureException { - long startTime = System.currentTimeMillis(); + final long startTime = System.currentTimeMillis(); try { switch (state) { case PRE_SET: @@ -451,13 +455,13 @@ protected void rollbackState(ConfigNodeProcedureEnv env, SetTemplateState state) } } - private void rollbackPreSet(ConfigNodeProcedureEnv env) { - PreSetSchemaTemplatePlan preSetSchemaTemplatePlan = + private void rollbackPreSet(final ConfigNodeProcedureEnv env) { + final PreSetSchemaTemplatePlan preSetSchemaTemplatePlan = new PreSetSchemaTemplatePlan(templateName, templateSetPath, true); TSStatus status; try { status = env.getConfigManager().getConsensusManager().write(preSetSchemaTemplatePlan); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); status.setMessage(e.getMessage()); @@ -472,31 +476,31 @@ private void rollbackPreSet(ConfigNodeProcedureEnv env) { } } - private void rollbackPreRelease(ConfigNodeProcedureEnv env) { + private void rollbackPreRelease(final ConfigNodeProcedureEnv env) { Template template = getTemplate(env); if (template == null) { // already setFailure return; } - Map dataNodeLocationMap = + final Map dataNodeLocationMap = env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); - TUpdateTemplateReq invalidateTemplateSetInfoReq = new TUpdateTemplateReq(); + final TUpdateTemplateReq invalidateTemplateSetInfoReq = new TUpdateTemplateReq(); invalidateTemplateSetInfoReq.setType( TemplateInternalRPCUpdateType.INVALIDATE_TEMPLATE_SET_INFO.toByte()); invalidateTemplateSetInfoReq.setTemplateInfo( TemplateInternalRPCUtil.generateInvalidateTemplateSetInfoBytes( template.getId(), templateSetPath)); - DataNodeAsyncRequestContext clientHandler = + final DataNodeAsyncRequestContext clientHandler = new DataNodeAsyncRequestContext<>( CnToDnAsyncRequestType.UPDATE_TEMPLATE, invalidateTemplateSetInfoReq, dataNodeLocationMap); CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); - Map statusMap = clientHandler.getResponseMap(); - for (Map.Entry entry : statusMap.entrySet()) { + final Map statusMap = clientHandler.getResponseMap(); + for (final Map.Entry entry : statusMap.entrySet()) { // all dataNodes must clear the related template cache if (entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.error( @@ -510,13 +514,13 @@ private void rollbackPreRelease(ConfigNodeProcedureEnv env) { } } - private void rollbackCommitSet(ConfigNodeProcedureEnv env) { - CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan = + private void rollbackCommitSet(final ConfigNodeProcedureEnv env) { + final CommitSetSchemaTemplatePlan commitSetSchemaTemplatePlan = new CommitSetSchemaTemplatePlan(templateName, templateSetPath, true); TSStatus status; try { status = env.getConfigManager().getConsensusManager().write(commitSetSchemaTemplatePlan); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); status.setMessage(e.getMessage()); @@ -532,12 +536,12 @@ private void rollbackCommitSet(ConfigNodeProcedureEnv env) { } @Override - protected SetTemplateState getState(int stateId) { + protected SetTemplateState getState(final int stateId) { return SetTemplateState.values()[stateId]; } @Override - protected int getStateId(SetTemplateState state) { + protected int getStateId(final SetTemplateState state) { return state.ordinal(); } @@ -559,7 +563,7 @@ public String getTemplateSetPath() { } @Override - public void serialize(DataOutputStream stream) throws IOException { + public void serialize(final DataOutputStream stream) throws IOException { stream.writeShort( isGeneratedByPipe ? ProcedureType.PIPE_ENRICHED_SET_TEMPLATE_PROCEDURE.getTypeCode() @@ -571,7 +575,7 @@ public void serialize(DataOutputStream stream) throws IOException { } @Override - public void deserialize(ByteBuffer byteBuffer) { + public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); queryId = ReadWriteIOUtils.readString(byteBuffer); templateName = ReadWriteIOUtils.readString(byteBuffer); @@ -579,10 +583,14 @@ public void deserialize(ByteBuffer byteBuffer) { } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SetTemplateProcedure that = (SetTemplateProcedure) o; + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final SetTemplateProcedure that = (SetTemplateProcedure) o; return Objects.equals(getProcId(), that.getProcId()) && Objects.equals(getCurrentState(), that.getCurrentState()) && Objects.equals(getCycles(), that.getCycles()) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/AbstractOperateSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/AbstractOperateSubscriptionProcedure.java index ddd74824d131..7cb68d5b27a7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/AbstractOperateSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/AbstractOperateSubscriptionProcedure.java @@ -63,6 +63,10 @@ public abstract class AbstractOperateSubscriptionProcedure // Pure in-memory object, not involved in snapshot serialization and deserialization. // TODO: consider serializing this variable later protected boolean isRollbackFromOperateOnDataNodesSuccessful = false; + // Only used in rollback to avoid executing rollbackFromValidate multiple times + // Pure in-memory object, not involved in snapshot serialization and deserialization. + // TODO: consider serializing this variable later + protected boolean isRollbackFromValidateSuccessful = false; protected AtomicReference subscriptionInfo; @@ -255,15 +259,18 @@ protected void rollbackState(ConfigNodeProcedureEnv env, OperateSubscriptionStat switch (state) { case VALIDATE: - try { - rollbackFromValidate(env); - } catch (Exception e) { - LOGGER.warn( - "ProcedureId {}: Failed to rollback from state [{}], because {}", - getProcId(), - state, - e.getMessage(), - e); + if (!isRollbackFromValidateSuccessful) { + try { + rollbackFromValidate(env); + isRollbackFromValidateSuccessful = true; + } catch (Exception e) { + LOGGER.warn( + "ProcedureId {}: Failed to rollback from state [{}], because {}", + getProcId(), + state, + e.getMessage(), + e); + } } break; case OPERATE_ON_CONFIG_NODES: diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeTest.java index 5b8dc0b89c9a..bed27e6db837 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTreeTest.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.confignode.persistence.schema; import org.apache.iotdb.commons.exception.IllegalPathException; @@ -71,7 +72,7 @@ public void testSetStorageGroupExceptionMessage() { root.setStorageGroup(new PartialPath("root.edge1.access")); root.setStorageGroup(new PartialPath("root.edge1")); fail("Expected exception"); - } catch (MetadataException e) { + } catch (final MetadataException e) { assertEquals( "some children of root.edge1 have already been created as database", e.getMessage()); } @@ -79,20 +80,20 @@ public void testSetStorageGroupExceptionMessage() { root.setStorageGroup(new PartialPath("root.edge2")); root.setStorageGroup(new PartialPath("root.edge2.access")); fail("Expected exception"); - } catch (MetadataException e) { + } catch (final MetadataException e) { assertEquals("root.edge2 has already been created as database", e.getMessage()); } try { root.setStorageGroup(new PartialPath("root.edge1.access")); fail("Expected exception"); - } catch (MetadataException e) { + } catch (final MetadataException e) { assertEquals("root.edge1.access has already been created as database", e.getMessage()); } } @Test public void testAddAndPathExist() throws MetadataException { - String path1 = "root"; + final String path1 = "root"; root.setStorageGroup(new PartialPath("root.laptop")); assertTrue(root.isDatabaseAlreadySet(new PartialPath(path1))); assertTrue(root.isDatabaseAlreadySet(new PartialPath("root.laptop"))); @@ -105,25 +106,25 @@ public void testSetStorageGroup() throws IllegalPathException { root.setStorageGroup(new PartialPath("root.laptop.d1")); assertTrue(root.isDatabaseAlreadySet(new PartialPath("root.laptop.d1"))); assertTrue(root.isDatabaseAlreadySet(new PartialPath("root.laptop.d1.s1"))); - } catch (MetadataException e) { + } catch (final MetadataException e) { e.printStackTrace(); fail(e.getMessage()); } try { root.setStorageGroup(new PartialPath("root.laptop.d2")); - } catch (MetadataException e) { + } catch (final MetadataException e) { fail(e.getMessage()); } try { root.setStorageGroup(new PartialPath("root.laptop")); - } catch (MetadataException e) { + } catch (final MetadataException e) { Assert.assertEquals( "some children of root.laptop have already been created as database", e.getMessage()); } try { root.deleteDatabase(new PartialPath("root.laptop.d1")); - } catch (MetadataException e) { + } catch (final MetadataException e) { e.printStackTrace(); fail(e.getMessage()); } @@ -138,7 +139,7 @@ public void testGetAllFileNamesByPath() { root.setStorageGroup(new PartialPath("root.laptop.d1")); root.setStorageGroup(new PartialPath("root.laptop.d2")); - List list = new ArrayList<>(); + final List list = new ArrayList<>(); list.add(new PartialPath("root.laptop.d1")); assertEquals(list, root.getBelongedDatabases(new PartialPath("root.laptop.d1.s1"))); @@ -147,7 +148,7 @@ public void testGetAllFileNamesByPath() { list.add(new PartialPath("root.laptop.d2")); assertEquals(list, root.getBelongedDatabases(new PartialPath("root.laptop.**"))); assertEquals(list, root.getBelongedDatabases(new PartialPath("root.**"))); - } catch (MetadataException e) { + } catch (final MetadataException e) { e.printStackTrace(); fail(e.getMessage()); } @@ -175,7 +176,7 @@ public void testCheckStorageExistOfPath() { assertTrue(root.getBelongedDatabases(new PartialPath("root.vehicle1.device2")).isEmpty()); assertTrue(root.getBelongedDatabases(new PartialPath("root.vehicle1.device3")).isEmpty()); assertFalse(root.getBelongedDatabases(new PartialPath("root.vehicle1.device0")).isEmpty()); - } catch (MetadataException e) { + } catch (final MetadataException e) { e.printStackTrace(); fail(e.getMessage()); } @@ -185,7 +186,7 @@ public void testCheckStorageExistOfPath() { public void testIllegalStorageGroup() { try { root.setStorageGroup(new PartialPath("root.\"sg.ln\"")); - } catch (MetadataException e) { + } catch (final MetadataException e) { Assert.assertEquals("root.\"sg.ln\" is not a legal path", e.getMessage()); } } @@ -251,7 +252,7 @@ public void testGetNodeListInLevel() throws MetadataException { @Test public void testSerialization() throws Exception { - PartialPath[] pathList = + final PartialPath[] pathList = new PartialPath[] { new PartialPath("root.sg"), new PartialPath("root.a.sg"), @@ -260,7 +261,7 @@ public void testSerialization() throws Exception { }; for (int i = 0; i < pathList.length; i++) { root.setStorageGroup(pathList[i]); - IDatabaseMNode storageGroupMNode = + final IDatabaseMNode storageGroupMNode = root.getDatabaseNodeByDatabasePath(pathList[i]); storageGroupMNode.getAsMNode().getDatabaseSchema().setDataReplicationFactor(i); storageGroupMNode.getAsMNode().getDatabaseSchema().setSchemaReplicationFactor(i); @@ -268,15 +269,15 @@ public void testSerialization() throws Exception { root.getNodeWithAutoCreate(pathList[i].concatNode("a")).setSchemaTemplateId(i); } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); root.serialize(outputStream); - ConfigMTree newTree = new ConfigMTree(); - ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); + final ConfigMTree newTree = new ConfigMTree(); + final ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); newTree.deserialize(inputStream); for (int i = 0; i < pathList.length; i++) { - TDatabaseSchema storageGroupSchema = + final TDatabaseSchema storageGroupSchema = newTree.getDatabaseNodeByDatabasePath(pathList[i]).getAsMNode().getDatabaseSchema(); Assert.assertEquals(i, storageGroupSchema.getSchemaReplicationFactor()); Assert.assertEquals(i, storageGroupSchema.getDataReplicationFactor()); @@ -322,6 +323,7 @@ public void testTableSerialization() throws Exception { storageGroupMNode.getAsMNode().getDatabaseSchema().setDataReplicationFactor(i); storageGroupMNode.getAsMNode().getDatabaseSchema().setSchemaReplicationFactor(i); storageGroupMNode.getAsMNode().getDatabaseSchema().setTimePartitionInterval(i); + storageGroupMNode.getAsMNode().getDatabaseSchema().setIsTableModel(true); final String tableName = "table" + i; final TsTable table = new TsTable(tableName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DatabaseConflictException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DatabaseConflictException.java index a6e4ed952c46..d253c7984acf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DatabaseConflictException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DatabaseConflictException.java @@ -28,7 +28,7 @@ public class DatabaseConflictException extends MetadataException { private final String storageGroupPath; - public DatabaseConflictException(final String path, boolean isChild) { + public DatabaseConflictException(final String path, final boolean isChild) { super(getMessage(path, isChild), TSStatusCode.DATABASE_CONFLICT.getStatusCode()); this.isChild = isChild; storageGroupPath = path; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DatabaseModelException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DatabaseModelException.java new file mode 100644 index 000000000000..95c0d45c53ef --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DatabaseModelException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.exception.metadata; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.rpc.TSStatusCode; + +public class DatabaseModelException extends MetadataException { + public DatabaseModelException(final String path, final boolean isTableModel) { + super( + "The database " + path + " is a " + (isTableModel ? "table" : "tree") + " model database.", + TSStatusCode.DATABASE_MODEL.getStatusCode()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index f55da4be67bc..5e5143e5f06d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -86,6 +86,15 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_SNAPSHOT_KEY; + public class PipeDataNodeTaskAgent extends PipeTaskAgent { private static final Logger LOGGER = LoggerFactory.getLogger(PipeDataNodeTaskAgent.class); @@ -426,23 +435,12 @@ protected void collectPipeMetaListInternal( || pipeTaskMap.entrySet().stream() .filter(entry -> dataRegionIds.contains(entry.getKey())) .allMatch(entry -> ((PipeDataNodeTask) entry.getValue()).isCompleted()); - final String extractorModeValue = - pipeMeta - .getStaticMeta() - .getExtractorParameters() - .getStringOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_MODE_KEY, - PipeExtractorConstant.SOURCE_MODE_KEY), - PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE); + final boolean includeDataAndNeedDrop = DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair( pipeMeta.getStaticMeta().getExtractorParameters()) .getLeft() - && (extractorModeValue.equalsIgnoreCase( - PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE) - || extractorModeValue.equalsIgnoreCase( - PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE)); + && isSnapshotMode(pipeMeta.getStaticMeta().getExtractorParameters()); final boolean isCompleted = isAllDataRegionCompleted && includeDataAndNeedDrop; final Pair remainingEventAndTime = @@ -641,6 +639,24 @@ public boolean hasPipeReleaseRegionRelatedResource(final int consensusGroupId) { } } + private boolean isSnapshotMode(final PipeParameters parameters) { + final boolean isSnapshotMode; + if (parameters.hasAnyAttributes(EXTRACTOR_MODE_SNAPSHOT_KEY, SOURCE_MODE_SNAPSHOT_KEY)) { + isSnapshotMode = + parameters.getBooleanOrDefault( + Arrays.asList(EXTRACTOR_MODE_SNAPSHOT_KEY, SOURCE_MODE_SNAPSHOT_KEY), + EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE); + } else { + final String extractorModeValue = + parameters.getStringOrDefault( + Arrays.asList(EXTRACTOR_MODE_KEY, SOURCE_MODE_KEY), EXTRACTOR_MODE_DEFAULT_VALUE); + isSnapshotMode = + extractorModeValue.equalsIgnoreCase(EXTRACTOR_MODE_SNAPSHOT_VALUE) + || extractorModeValue.equalsIgnoreCase(EXTRACTOR_MODE_QUERY_VALUE); + } + return isSnapshotMode; + } + ///////////////////////// Pipe Consensus ///////////////////////// public ProgressIndex getPipeTaskProgressIndex(final String pipeName, final int consensusGroupId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java index 23f65311303b..84256b38e19d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionExtractor.java @@ -25,7 +25,6 @@ import org.apache.iotdb.commons.consensus.index.impl.TimeWindowStateProgressIndex; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; -import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskExtractorRuntimeEnvironment; import org.apache.iotdb.commons.pipe.datastructure.PersistentResource; @@ -81,6 +80,12 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_PATH_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_LOOSE_RANGE_TIME_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_START_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_STRICT_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODE_STRICT_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_MODS_DEFAULT_VALUE; @@ -93,6 +98,8 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_END_TIME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_LOOSE_RANGE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_START_TIME_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_SNAPSHOT_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODE_STRICT_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODS_ENABLE_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.SOURCE_MODS_KEY; @@ -374,15 +381,19 @@ public void customize( listeningOptionPair.getRight()); } - final String extractorModeValue = - parameters.getStringOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_MODE_KEY, PipeExtractorConstant.SOURCE_MODE_KEY), - PipeExtractorConstant.EXTRACTOR_MODE_DEFAULT_VALUE); - shouldTerminatePipeOnAllHistoricalEventsConsumed = - extractorModeValue.equalsIgnoreCase(PipeExtractorConstant.EXTRACTOR_MODE_QUERY_VALUE) - || extractorModeValue.equalsIgnoreCase( - PipeExtractorConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE); + if (parameters.hasAnyAttributes(EXTRACTOR_MODE_SNAPSHOT_KEY, SOURCE_MODE_SNAPSHOT_KEY)) { + shouldTerminatePipeOnAllHistoricalEventsConsumed = + parameters.getBooleanOrDefault( + Arrays.asList(EXTRACTOR_MODE_SNAPSHOT_KEY, SOURCE_MODE_SNAPSHOT_KEY), + EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE); + } else { + final String extractorModeValue = + parameters.getStringOrDefault( + Arrays.asList(EXTRACTOR_MODE_KEY, SOURCE_MODE_KEY), EXTRACTOR_MODE_DEFAULT_VALUE); + shouldTerminatePipeOnAllHistoricalEventsConsumed = + extractorModeValue.equalsIgnoreCase(EXTRACTOR_MODE_SNAPSHOT_VALUE) + || extractorModeValue.equalsIgnoreCase(EXTRACTOR_MODE_QUERY_VALUE); + } if (LOGGER.isInfoEnabled()) { LOGGER.info( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 8487af0dcc94..bf643cfbd5e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -622,7 +622,7 @@ public TSchemaPartitionTableResp getOrCreateSchemaPartitionTable(TSchemaPartitio @Override public TSchemaPartitionTableResp getOrCreateSchemaPartitionTableWithSlots( - Map> dbSlotMap) throws TException { + final Map> dbSlotMap) throws TException { return executeRemoteCallWithRetry( () -> client.getOrCreateSchemaPartitionTableWithSlots(dbSlotMap), resp -> !updateConfigNodeLeader(resp.status)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java index 32bf02aca52f..d2fa82f9feb3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java @@ -64,6 +64,7 @@ private ColumnHeaderConstant() { public static final String DATA_REGION_GROUP_NUM = "DataRegionGroupNum"; public static final String MIN_DATA_REGION_GROUP_NUM = "MinDataRegionGroupNum"; public static final String MAX_DATA_REGION_GROUP_NUM = "MaxDataRegionGroupNum"; + public static final String MODEL = "Model"; public static final String CHILD_PATHS = "ChildPaths"; public static final String NODE_TYPES = "NodeTypes"; public static final String CHILD_NODES = "ChildNodes"; @@ -281,7 +282,8 @@ private ColumnHeaderConstant() { new ColumnHeader(MAX_SCHEMA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(DATA_REGION_GROUP_NUM, TSDataType.INT32), new ColumnHeader(MIN_DATA_REGION_GROUP_NUM, TSDataType.INT32), - new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32)); + new ColumnHeader(MAX_DATA_REGION_GROUP_NUM, TSDataType.INT32), + new ColumnHeader(MODEL, TSDataType.TEXT)); public static final List showChildPathsColumnHeaders = ImmutableList.of( @@ -544,6 +546,14 @@ private ColumnHeaderConstant() { new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32), new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64)); + public static final List showDBDetailsColumnHeaders = + ImmutableList.of( + new ColumnHeader(DATABASE, TSDataType.TEXT), + new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32), + new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32), + new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64), + new ColumnHeader(MODEL, TSDataType.TEXT)); + public static final List describeTableColumnHeaders = ImmutableList.of( new ColumnHeader(COLUMN_NAME, TSDataType.TEXT), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java index 9dbbe22b05e7..3b85268bc131 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java @@ -57,7 +57,7 @@ public static DatasetHeader getShowDevicesWithSgHeader() { return new DatasetHeader(ColumnHeaderConstant.showDevicesWithSgColumnHeaders, true); } - public static DatasetHeader getShowStorageGroupHeader(boolean isDetailed) { + public static DatasetHeader getShowStorageGroupHeader(final boolean isDetailed) { return isDetailed ? new DatasetHeader(ColumnHeaderConstant.showStorageGroupsDetailColumnHeaders, true) : new DatasetHeader(ColumnHeaderConstant.showStorageGroupsColumnHeaders, true); @@ -221,6 +221,10 @@ public static DatasetHeader getShowDBHeader() { return new DatasetHeader(ColumnHeaderConstant.showDBColumnHeaders, true); } + public static DatasetHeader getShowDBDetailsHeader() { + return new DatasetHeader(ColumnHeaderConstant.showDBDetailsColumnHeaders, true); + } + public static DatasetHeader getDescribeTableHeader() { return new DatasetHeader(ColumnHeaderConstant.describeTableColumnHeaders, true); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java index 5003883eb45a..f47c8121e76a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@ -33,7 +33,9 @@ import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.GroupedMaxByAccumulator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.GroupedMinAccumulator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.GroupedMinByAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.GroupedModeAccumulator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.GroupedSumAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.GroupedVarianceAccumulator; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; @@ -135,6 +137,22 @@ private static GroupedAccumulator createBuiltinGroupedAccumulator( return new GroupedMaxByAccumulator(inputDataTypes.get(0), inputDataTypes.get(1)); case MIN_BY: return new GroupedMinByAccumulator(inputDataTypes.get(0), inputDataTypes.get(1)); + case MODE: + return new GroupedModeAccumulator(inputDataTypes.get(0)); + case STDDEV: + case STDDEV_SAMP: + return new GroupedVarianceAccumulator( + inputDataTypes.get(0), VarianceAccumulator.VarianceType.STDDEV_SAMP); + case STDDEV_POP: + return new GroupedVarianceAccumulator( + inputDataTypes.get(0), VarianceAccumulator.VarianceType.STDDEV_POP); + case VARIANCE: + case VAR_SAMP: + return new GroupedVarianceAccumulator( + inputDataTypes.get(0), VarianceAccumulator.VarianceType.VAR_SAMP); + case VAR_POP: + return new GroupedVarianceAccumulator( + inputDataTypes.get(0), VarianceAccumulator.VarianceType.VAR_POP); default: throw new IllegalArgumentException("Invalid Aggregation function: " + aggregationType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableModeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableModeAccumulator.java index 3f90db0a1bdc..1ad01095fac9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableModeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableModeAccumulator.java @@ -31,10 +31,11 @@ import java.util.HashMap; import java.util.Map; -import java.util.Optional; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.UNSUPPORTED_TYPE_MESSAGE; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeBinaryValue; +import static org.apache.tsfile.utils.BytesUtils.bytesToBool; +import static org.apache.tsfile.utils.BytesUtils.bytesToLongFromOffset; public class TableModeAccumulator implements TableAccumulator { @@ -51,6 +52,8 @@ public class TableModeAccumulator implements TableAccumulator { private Map doubleCountMap; private Map binaryCountMap; + private long nullCount; + public TableModeAccumulator(TSDataType seriesDataType) { this.seriesDataType = seriesDataType; switch (seriesDataType) { @@ -147,11 +150,14 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { if (booleanCountMap.isEmpty()) { columnBuilder.appendNull(); } else { - Optional maxKey = - booleanCountMap.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey); - maxKey.ifPresent(columnBuilder::writeBoolean); + // must be present + Map.Entry maxEntry = + booleanCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); + if (maxEntry.getValue() < nullCount) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeBoolean(maxEntry.getKey()); + } } break; case INT32: @@ -159,22 +165,26 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { if (intCountMap.isEmpty()) { columnBuilder.appendNull(); } else { - Optional maxKey = - intCountMap.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey); - maxKey.ifPresent(columnBuilder::writeInt); + Map.Entry maxEntry = + intCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); + if (maxEntry.getValue() < nullCount) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeInt(maxEntry.getKey()); + } } break; case FLOAT: if (floatCountMap.isEmpty()) { columnBuilder.appendNull(); } else { - Optional maxKey = - floatCountMap.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey); - maxKey.ifPresent(columnBuilder::writeFloat); + Map.Entry maxEntry = + floatCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); + if (maxEntry.getValue() < nullCount) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeFloat(maxEntry.getKey()); + } } break; case INT64: @@ -182,22 +192,26 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { if (longCountMap.isEmpty()) { columnBuilder.appendNull(); } else { - Optional maxKey = - longCountMap.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey); - maxKey.ifPresent(columnBuilder::writeLong); + Map.Entry maxEntry = + longCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); + if (maxEntry.getValue() < nullCount) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeLong(maxEntry.getKey()); + } } break; case DOUBLE: if (doubleCountMap.isEmpty()) { columnBuilder.appendNull(); } else { - Optional maxKey = - doubleCountMap.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey); - maxKey.ifPresent(columnBuilder::writeDouble); + Map.Entry maxEntry = + doubleCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); + if (maxEntry.getValue() < nullCount) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeDouble(maxEntry.getKey()); + } } break; case TEXT: @@ -206,11 +220,13 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { if (binaryCountMap.isEmpty()) { columnBuilder.appendNull(); } else { - Optional maxKey = - binaryCountMap.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey); - maxKey.ifPresent(columnBuilder::writeBinary); + Map.Entry maxEntry = + binaryCountMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); + if (maxEntry.getValue() < nullCount) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeBinary(maxEntry.getKey()); + } } break; default: @@ -249,15 +265,22 @@ public void reset() { if (binaryCountMap != null) { binaryCountMap.clear(); } + nullCount = 0; } + // haveNull | nullCount (optional) | countMap private byte[] serializeCountMap() { byte[] bytes; - int offset = 0; + int offset = 1 + (nullCount == 0 ? 0 : Long.BYTES); + ; switch (seriesDataType) { case BOOLEAN: - bytes = new byte[4 + (1 + 8) * booleanCountMap.size()]; + bytes = new byte[offset + 4 + (1 + 8) * booleanCountMap.size()]; + BytesUtils.boolToBytes(nullCount != 0, bytes, 0); + if (nullCount != 0) { + BytesUtils.longToBytes(nullCount, bytes, 1); + } BytesUtils.intToBytes(booleanCountMap.size(), bytes, offset); offset += 4; for (Map.Entry entry : booleanCountMap.entrySet()) { @@ -269,7 +292,11 @@ private byte[] serializeCountMap() { break; case INT32: case DATE: - bytes = new byte[4 + (4 + 8) * intCountMap.size()]; + bytes = new byte[offset + 4 + (4 + 8) * intCountMap.size()]; + BytesUtils.boolToBytes(nullCount != 0, bytes, 0); + if (nullCount != 0) { + BytesUtils.longToBytes(nullCount, bytes, 1); + } BytesUtils.intToBytes(intCountMap.size(), bytes, offset); offset += 4; for (Map.Entry entry : intCountMap.entrySet()) { @@ -280,7 +307,11 @@ private byte[] serializeCountMap() { } break; case FLOAT: - bytes = new byte[4 + (4 + 8) * floatCountMap.size()]; + bytes = new byte[offset + 4 + (4 + 8) * floatCountMap.size()]; + BytesUtils.boolToBytes(nullCount != 0, bytes, 0); + if (nullCount != 0) { + BytesUtils.longToBytes(nullCount, bytes, 1); + } BytesUtils.intToBytes(floatCountMap.size(), bytes, offset); offset += 4; for (Map.Entry entry : floatCountMap.entrySet()) { @@ -292,7 +323,11 @@ private byte[] serializeCountMap() { break; case INT64: case TIMESTAMP: - bytes = new byte[4 + (8 + 8) * longCountMap.size()]; + bytes = new byte[offset + 4 + (8 + 8) * longCountMap.size()]; + BytesUtils.boolToBytes(nullCount != 0, bytes, 0); + if (nullCount != 0) { + BytesUtils.longToBytes(nullCount, bytes, 1); + } BytesUtils.intToBytes(longCountMap.size(), bytes, offset); offset += 4; for (Map.Entry entry : longCountMap.entrySet()) { @@ -303,7 +338,11 @@ private byte[] serializeCountMap() { } break; case DOUBLE: - bytes = new byte[4 + (8 + 8) * doubleCountMap.size()]; + bytes = new byte[offset + 4 + (8 + 8) * doubleCountMap.size()]; + BytesUtils.boolToBytes(nullCount != 0, bytes, 0); + if (nullCount != 0) { + BytesUtils.longToBytes(nullCount, bytes, 1); + } BytesUtils.intToBytes(doubleCountMap.size(), bytes, offset); offset += 4; for (Map.Entry entry : doubleCountMap.entrySet()) { @@ -318,11 +357,16 @@ private byte[] serializeCountMap() { case BLOB: bytes = new byte - [4 + [offset + + 4 + (8 + 4) * binaryCountMap.size() + binaryCountMap.keySet().stream() .mapToInt(key -> key.getValues().length) .sum()]; + BytesUtils.boolToBytes(nullCount != 0, bytes, 0); + if (nullCount != 0) { + BytesUtils.longToBytes(nullCount, bytes, 1); + } BytesUtils.intToBytes(binaryCountMap.size(), bytes, offset); offset += 4; for (Map.Entry entry : binaryCountMap.entrySet()) { @@ -343,6 +387,11 @@ private byte[] serializeCountMap() { private void deserializeAndMergeCountMap(byte[] bytes) { int offset = 0; + if (bytesToBool(bytes, 0)) { + nullCount += bytesToLongFromOffset(bytes, Long.BYTES, 1); + offset += Long.BYTES; + } + offset++; int size = BytesUtils.bytesToInt(bytes, offset); offset += 4; @@ -378,7 +427,7 @@ private void deserializeAndMergeCountMap(byte[] bytes) { case INT64: case TIMESTAMP: for (int i = 0; i < size; i++) { - long key = BytesUtils.bytesToLong(bytes, offset); + long key = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); offset += 8; long count = BytesUtils.bytesToLongFromOffset(bytes, 8, offset); offset += 8; @@ -420,6 +469,8 @@ private void addBooleanInput(Column column) { if (booleanCountMap.size() > MAP_SIZE_THRESHOLD) { checkMapSize(booleanCountMap.size()); } + } else { + nullCount++; } } } @@ -429,6 +480,8 @@ private void addIntInput(Column column) { if (!column.isNull(i)) { intCountMap.compute(column.getInt(i), (k, v) -> v == null ? 1 : v + 1); checkMapSize(intCountMap.size()); + } else { + nullCount++; } } } @@ -438,6 +491,8 @@ private void addFloatInput(Column column) { if (!column.isNull(i)) { floatCountMap.compute(column.getFloat(i), (k, v) -> v == null ? 1 : v + 1); checkMapSize(floatCountMap.size()); + } else { + nullCount++; } } } @@ -447,6 +502,8 @@ private void addLongInput(Column column) { if (!column.isNull(i)) { longCountMap.compute(column.getLong(i), (k, v) -> v == null ? 1 : v + 1); checkMapSize(longCountMap.size()); + } else { + nullCount++; } } } @@ -456,6 +513,8 @@ private void addDoubleInput(Column column) { if (!column.isNull(i)) { doubleCountMap.compute(column.getDouble(i), (k, v) -> v == null ? 1 : v + 1); checkMapSize(doubleCountMap.size()); + } else { + nullCount++; } } } @@ -465,6 +524,8 @@ private void addBinaryInput(Column column) { if (!column.isNull(i)) { binaryCountMap.compute(column.getBinary(i), (k, v) -> v == null ? 1 : v + 1); checkMapSize(binaryCountMap.size()); + } else { + nullCount++; } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java index 99252b9606d3..37f2f5d912dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java @@ -73,7 +73,7 @@ public static void serializeValue( public static void serializeBinaryValue(Binary binary, byte[] valueBytes, int offset) { BytesUtils.intToBytes(binary.getValues().length, valueBytes, offset); - offset += 4; + offset += Integer.BYTES; System.arraycopy(binary.getValues(), 0, valueBytes, offset, binary.getValues().length); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java new file mode 100644 index 000000000000..f3a5a3167605 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java @@ -0,0 +1,452 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.array.MapBigArray; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.UNSUPPORTED_TYPE_MESSAGE; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Utils.serializeBinaryValue; +import static org.apache.tsfile.utils.BytesUtils.bytesToBool; +import static org.apache.tsfile.utils.BytesUtils.bytesToLongFromOffset; +import static org.apache.tsfile.utils.TsPrimitiveType.getByType; + +public class GroupedModeAccumulator implements GroupedAccumulator { + + private final int MAP_SIZE_THRESHOLD = + IoTDBDescriptor.getInstance().getConfig().getModeMapSizeThreshold(); + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedModeAccumulator.class); + private final TSDataType seriesDataType; + + private final MapBigArray countMaps = new MapBigArray(); + + private final LongBigArray nullCounts = new LongBigArray(); + + public GroupedModeAccumulator(TSDataType seriesDataType) { + this.seriesDataType = seriesDataType; + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + countMaps.sizeOf() + nullCounts.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + countMaps.ensureCapacity(groupCount); + nullCounts.ensureCapacity(groupCount); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments) { + switch (seriesDataType) { + case BOOLEAN: + addBooleanInput(groupIds, arguments[0]); + break; + case INT32: + case DATE: + addIntInput(groupIds, arguments[0]); + break; + case FLOAT: + addFloatInput(groupIds, arguments[0]); + break; + case INT64: + case TIMESTAMP: + addLongInput(groupIds, arguments[0]); + break; + case DOUBLE: + addDoubleInput(groupIds, arguments[0]); + break; + case TEXT: + case STRING: + case BLOB: + addBinaryInput(groupIds, arguments[0]); + break; + default: + throw new UnsupportedOperationException( + String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + } + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + for (int i = 0; i < argument.getPositionCount(); i++) { + if (argument.isNull(i)) { + continue; + } + + byte[] bytes = argument.getBinary(i).getValues(); + deserializeAndMergeCountMap(groupIds[i], bytes); + } + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { + columnBuilder.writeBinary(new Binary(serializeCountMap(groupId))); + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { + HashMap countMap = countMaps.get(groupId); + if (countMap.isEmpty()) { + columnBuilder.appendNull(); + return; + } + // must be present + Map.Entry maxEntry = + countMap.entrySet().stream().max(Map.Entry.comparingByValue()).get(); + if (maxEntry.getValue() < nullCounts.get(groupId)) { + columnBuilder.appendNull(); + return; + } + + switch (seriesDataType) { + case BOOLEAN: + columnBuilder.writeBoolean(maxEntry.getKey().getBoolean()); + break; + case INT32: + case DATE: + columnBuilder.writeInt(maxEntry.getKey().getInt()); + break; + case FLOAT: + columnBuilder.writeFloat(maxEntry.getKey().getFloat()); + break; + case INT64: + case TIMESTAMP: + columnBuilder.writeLong(maxEntry.getKey().getLong()); + break; + case DOUBLE: + columnBuilder.writeDouble(maxEntry.getKey().getDouble()); + break; + case TEXT: + case STRING: + case BLOB: + columnBuilder.writeBinary(maxEntry.getKey().getBinary()); + break; + default: + throw new UnsupportedOperationException( + String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + } + } + + @Override + public void prepareFinal() {} + + // haveNull | nullCount (optional) | countMap + private byte[] serializeCountMap(int groupId) { + byte[] bytes; + int offset = 1 + (nullCounts.get(groupId) == 0 ? 0 : Long.BYTES); + HashMap countMap = countMaps.get(groupId); + + switch (seriesDataType) { + case BOOLEAN: + bytes = new byte[offset + Integer.BYTES + (1 + Long.BYTES) * countMap.size()]; + BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); + if (nullCounts.get(groupId) != 0) { + BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); + } + BytesUtils.intToBytes(countMap.size(), bytes, offset); + offset += 4; + for (Map.Entry entry : countMap.entrySet()) { + BytesUtils.boolToBytes(entry.getKey().getBoolean(), bytes, offset); + offset += 1; + BytesUtils.longToBytes(entry.getValue(), bytes, offset); + offset += Long.BYTES; + } + break; + case INT32: + case DATE: + bytes = new byte[offset + Integer.BYTES + (Integer.BYTES + Long.BYTES) * countMap.size()]; + BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); + if (nullCounts.get(groupId) != 0) { + BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); + } + BytesUtils.intToBytes(countMap.size(), bytes, offset); + offset += Integer.BYTES; + for (Map.Entry entry : countMap.entrySet()) { + BytesUtils.intToBytes(entry.getKey().getInt(), bytes, offset); + offset += Integer.BYTES; + BytesUtils.longToBytes(entry.getValue(), bytes, offset); + offset += Long.BYTES; + } + break; + case FLOAT: + bytes = new byte[offset + Integer.BYTES + (Float.BYTES + Long.BYTES) * countMap.size()]; + BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); + if (nullCounts.get(groupId) != 0) { + BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); + } + BytesUtils.intToBytes(countMap.size(), bytes, offset); + offset += Integer.BYTES; + for (Map.Entry entry : countMap.entrySet()) { + BytesUtils.floatToBytes(entry.getKey().getFloat(), bytes, offset); + offset += Float.BYTES; + BytesUtils.longToBytes(entry.getValue(), bytes, offset); + offset += Long.BYTES; + } + break; + case INT64: + case TIMESTAMP: + bytes = new byte[offset + Integer.BYTES + (Long.BYTES + Long.BYTES) * countMap.size()]; + BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); + if (nullCounts.get(groupId) != 0) { + BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); + } + BytesUtils.intToBytes(countMap.size(), bytes, offset); + offset += Integer.BYTES; + for (Map.Entry entry : countMap.entrySet()) { + BytesUtils.longToBytes(entry.getKey().getLong(), bytes, offset); + offset += Long.BYTES; + BytesUtils.longToBytes(entry.getValue(), bytes, offset); + offset += Long.BYTES; + } + break; + case DOUBLE: + bytes = new byte[offset + Integer.BYTES + (Double.BYTES + Long.BYTES) * countMap.size()]; + BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); + if (nullCounts.get(groupId) != 0) { + BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); + } + BytesUtils.intToBytes(countMap.size(), bytes, offset); + offset += Integer.BYTES; + for (Map.Entry entry : countMap.entrySet()) { + BytesUtils.doubleToBytes(entry.getKey().getDouble(), bytes, offset); + offset += Double.BYTES; + BytesUtils.longToBytes(entry.getValue(), bytes, offset); + offset += Long.BYTES; + } + break; + case TEXT: + case STRING: + case BLOB: + bytes = + new byte + [offset + + Integer.BYTES + + (Integer.BYTES + Long.BYTES) * countMap.size() + + countMap.keySet().stream() + .mapToInt(key -> key.getBinary().getValues().length) + .sum()]; + BytesUtils.boolToBytes(nullCounts.get(groupId) != 0, bytes, 0); + if (nullCounts.get(groupId) != 0) { + BytesUtils.longToBytes(nullCounts.get(groupId), bytes, 1); + } + BytesUtils.intToBytes(countMap.size(), bytes, offset); + offset += Integer.BYTES; + for (Map.Entry entry : countMap.entrySet()) { + Binary binary = entry.getKey().getBinary(); + serializeBinaryValue(binary, bytes, offset); + offset += (Integer.BYTES + binary.getLength()); + BytesUtils.longToBytes(entry.getValue(), bytes, offset); + offset += Long.BYTES; + } + break; + default: + throw new UnsupportedOperationException( + String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + } + + return bytes; + } + + private void deserializeAndMergeCountMap(int groupId, byte[] bytes) { + int offset = 0; + if (bytesToBool(bytes, 0)) { + nullCounts.add(groupId, bytesToLongFromOffset(bytes, Long.BYTES, 1)); + offset += Long.BYTES; + } + offset++; + int size = BytesUtils.bytesToInt(bytes, offset); + offset += Integer.BYTES; + + HashMap countMap = countMaps.get(groupId); + + switch (seriesDataType) { + case BOOLEAN: + for (int i = 0; i < size; i++) { + TsPrimitiveType key = new TsPrimitiveType.TsBoolean(bytesToBool(bytes, offset)); + offset += 1; + long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + offset += Long.BYTES; + countMap.compute(key, (k, v) -> v == null ? count : v + count); + } + break; + case INT32: + case DATE: + for (int i = 0; i < size; i++) { + TsPrimitiveType key = new TsPrimitiveType.TsInt(BytesUtils.bytesToInt(bytes, offset)); + offset += Integer.BYTES; + long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + offset += Long.BYTES; + countMap.compute(key, (k, v) -> v == null ? count : v + count); + } + break; + case FLOAT: + for (int i = 0; i < size; i++) { + TsPrimitiveType key = new TsPrimitiveType.TsFloat(BytesUtils.bytesToFloat(bytes, offset)); + offset += Float.BYTES; + long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + offset += Long.BYTES; + countMap.compute(key, (k, v) -> v == null ? count : v + count); + } + break; + case INT64: + case TIMESTAMP: + for (int i = 0; i < size; i++) { + TsPrimitiveType key = + new TsPrimitiveType.TsLong( + BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset)); + offset += Long.BYTES; + long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + offset += Long.BYTES; + countMap.compute(key, (k, v) -> v == null ? count : v + count); + } + break; + case DOUBLE: + for (int i = 0; i < size; i++) { + TsPrimitiveType key = + new TsPrimitiveType.TsDouble(BytesUtils.bytesToDouble(bytes, offset)); + offset += Double.BYTES; + long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + offset += Long.BYTES; + countMap.compute(key, (k, v) -> v == null ? count : v + count); + } + break; + case TEXT: + case STRING: + case BLOB: + for (int i = 0; i < size; i++) { + int length = BytesUtils.bytesToInt(bytes, offset); + offset += Integer.BYTES; + TsPrimitiveType key = + new TsPrimitiveType.TsBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); + offset += length; + long count = BytesUtils.bytesToLongFromOffset(bytes, Long.BYTES, offset); + offset += Long.BYTES; + countMap.compute(key, (k, v) -> v == null ? count : v + count); + } + break; + default: + throw new UnsupportedOperationException( + String.format(UNSUPPORTED_TYPE_MESSAGE, seriesDataType)); + } + } + + private void addBooleanInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (!column.isNull(i)) { + HashMap countMap = countMaps.get(groupIds[i]); + countMap.compute( + getByType(seriesDataType, column.getBoolean(i)), (k, v) -> v == null ? 1 : v + 1); + checkMapSize(countMap.size()); + + } else { + nullCounts.increment(groupIds[i]); + } + } + } + + private void addIntInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (!column.isNull(i)) { + HashMap countMap = countMaps.get(groupIds[i]); + countMap.compute( + getByType(seriesDataType, column.getInt(i)), (k, v) -> v == null ? 1 : v + 1); + checkMapSize(countMap.size()); + } else { + nullCounts.increment(groupIds[i]); + } + } + } + + private void addFloatInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (!column.isNull(i)) { + HashMap countMap = countMaps.get(groupIds[i]); + countMap.compute( + getByType(seriesDataType, column.getFloat(i)), (k, v) -> v == null ? 1 : v + 1); + checkMapSize(countMap.size()); + } else { + nullCounts.increment(groupIds[i]); + } + } + } + + private void addLongInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (!column.isNull(i)) { + HashMap countMap = countMaps.get(groupIds[i]); + countMap.compute( + getByType(seriesDataType, column.getLong(i)), (k, v) -> v == null ? 1 : v + 1); + checkMapSize(countMap.size()); + } else { + nullCounts.increment(groupIds[i]); + } + } + } + + private void addDoubleInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (!column.isNull(i)) { + HashMap countMap = countMaps.get(groupIds[i]); + countMap.compute( + getByType(seriesDataType, column.getDouble(i)), (k, v) -> v == null ? 1 : v + 1); + checkMapSize(countMap.size()); + } else { + nullCounts.increment(groupIds[i]); + } + } + } + + private void addBinaryInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (!column.isNull(i)) { + HashMap countMap = countMaps.get(groupIds[i]); + countMap.compute( + getByType(seriesDataType, column.getBinary(i)), (k, v) -> v == null ? 1 : v + 1); + checkMapSize(countMap.size()); + } else { + nullCounts.increment(groupIds[i]); + } + } + } + + private void checkMapSize(int size) { + if (size > MAP_SIZE_THRESHOLD) { + throw new RuntimeException( + String.format( + "distinct values has exceeded the threshold %s when calculate Mode in one group", + MAP_SIZE_THRESHOLD)); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java new file mode 100644 index 000000000000..c80406e93c47 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped; + +import org.apache.iotdb.db.queryengine.execution.aggregation.VarianceAccumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.array.DoubleBigArray; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.array.LongBigArray; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +public class GroupedVarianceAccumulator implements GroupedAccumulator { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(GroupedVarianceAccumulator.class); + private final TSDataType seriesDataType; + private final VarianceAccumulator.VarianceType varianceType; + + private final LongBigArray counts = new LongBigArray(); + private final DoubleBigArray means = new DoubleBigArray(); + private final DoubleBigArray m2s = new DoubleBigArray(); + + public GroupedVarianceAccumulator( + TSDataType seriesDataType, VarianceAccumulator.VarianceType varianceType) { + this.seriesDataType = seriesDataType; + this.varianceType = varianceType; + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE + counts.sizeOf() + means.sizeOf() + m2s.sizeOf(); + } + + @Override + public void setGroupCount(long groupCount) { + counts.ensureCapacity(groupCount); + means.ensureCapacity(groupCount); + m2s.ensureCapacity(groupCount); + } + + @Override + public void addInput(int[] groupIds, Column[] arguments) { + switch (seriesDataType) { + case INT32: + addIntInput(groupIds, arguments[0]); + return; + case INT64: + addLongInput(groupIds, arguments[0]); + return; + case FLOAT: + addFloatInput(groupIds, arguments[0]); + return; + case DOUBLE: + addDoubleInput(groupIds, arguments[0]); + return; + case TEXT: + case BLOB: + case BOOLEAN: + case DATE: + case STRING: + case TIMESTAMP: + default: + throw new UnSupportedDataTypeException( + String.format("Unsupported data type in aggregation variance : %s", seriesDataType)); + } + } + + @Override + public void addIntermediate(int[] groupIds, Column argument) { + for (int i = 0; i < argument.getPositionCount(); i++) { + if (argument.isNull(i)) { + continue; + } + + byte[] bytes = argument.getBinary(i).getValues(); + long intermediateCount = BytesUtils.bytesToLong(bytes, Long.BYTES); + double intermediateMean = BytesUtils.bytesToDouble(bytes, Long.BYTES); + double intermediateM2 = BytesUtils.bytesToDouble(bytes, (Long.BYTES + Double.BYTES)); + + long newCount = counts.get(groupIds[i]) + intermediateCount; + double newMean = + ((intermediateCount * intermediateMean) + + (counts.get(groupIds[i]) * means.get(groupIds[i]))) + / newCount; + double delta = intermediateMean - means.get(groupIds[i]); + + m2s.add( + groupIds[i], + intermediateM2 + delta * delta * intermediateCount * counts.get(groupIds[i]) / newCount); + counts.set(groupIds[i], newCount); + means.set(groupIds[i], newMean); + } + } + + @Override + public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { + if (counts.get(groupId) == 0) { + columnBuilder.appendNull(); + } else { + byte[] bytes = new byte[24]; + BytesUtils.longToBytes(counts.get(groupId), bytes, 0); + BytesUtils.doubleToBytes(means.get(groupId), bytes, Long.BYTES); + BytesUtils.doubleToBytes(m2s.get(groupId), bytes, Long.BYTES + Double.BYTES); + columnBuilder.writeBinary(new Binary(bytes)); + } + } + + @Override + public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { + switch (varianceType) { + case STDDEV_POP: + if (counts.get(groupId) == 0) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeDouble(Math.sqrt(m2s.get(groupId) / counts.get(groupId))); + } + break; + case STDDEV_SAMP: + if (counts.get(groupId) < 2) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeDouble(Math.sqrt(m2s.get(groupId) / (counts.get(groupId) - 1))); + } + break; + case VAR_POP: + if (counts.get(groupId) == 0) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeDouble(m2s.get(groupId) / counts.get(groupId)); + } + break; + case VAR_SAMP: + if (counts.get(groupId) < 2) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeDouble(m2s.get(groupId) / (counts.get(groupId) - 1)); + } + break; + default: + throw new EnumConstantNotPresentException( + VarianceAccumulator.VarianceType.class, varianceType.name()); + } + } + + @Override + public void prepareFinal() {} + + private void addIntInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (column.isNull(i)) { + continue; + } + + int value = column.getInt(i); + counts.increment(groupIds[i]); + double delta = value - means.get(groupIds[i]); + means.add(groupIds[i], delta / counts.get(groupIds[i])); + m2s.add(groupIds[i], delta * (value - means.get(groupIds[i]))); + } + } + + private void addLongInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (column.isNull(i)) { + continue; + } + + long value = column.getLong(i); + counts.increment(groupIds[i]); + double delta = value - means.get(groupIds[i]); + means.add(groupIds[i], delta / counts.get(groupIds[i])); + m2s.add(groupIds[i], delta * (value - means.get(groupIds[i]))); + } + } + + private void addFloatInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (column.isNull(i)) { + continue; + } + + float value = column.getFloat(i); + counts.increment(groupIds[i]); + double delta = value - means.get(groupIds[i]); + means.add(groupIds[i], delta / counts.get(groupIds[i])); + m2s.add(groupIds[i], delta * (value - means.get(groupIds[i]))); + } + } + + private void addDoubleInput(int[] groupIds, Column column) { + for (int i = 0; i < column.getPositionCount(); i++) { + if (column.isNull(i)) { + continue; + } + + double value = column.getDouble(i); + counts.increment(groupIds[i]); + double delta = value - means.get(groupIds[i]); + means.add(groupIds[i], delta / counts.get(groupIds[i])); + m2s.add(groupIds[i], delta * (value - means.get(groupIds[i]))); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/array/MapBigArray.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/array/MapBigArray.java new file mode 100644 index 000000000000..ca674dbf2e64 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/array/MapBigArray.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.array; + +import org.apache.tsfile.utils.TsPrimitiveType; + +import java.util.HashMap; + +import static org.apache.tsfile.utils.RamUsageEstimator.shallowSizeOfInstance; +import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfObject; + +public final class MapBigArray { + private static final long INSTANCE_SIZE = shallowSizeOfInstance(MapBigArray.class); + private final ObjectBigArray> array; + private long sizeOfMaps; + + public MapBigArray() { + array = new ObjectBigArray<>(); + } + + public MapBigArray(HashMap slice) { + array = new ObjectBigArray<>(slice); + } + + /** Returns the size of this big array in bytes. */ + public long sizeOf() { + return INSTANCE_SIZE + array.sizeOf() + sizeOfMaps; + } + + /** + * Returns the element of this big array at specified index. + * + * @param index a position in this big array. + * @return the element of this big array at the specified position. + */ + public HashMap get(long index) { + HashMap result = array.get(index); + if (result == null) { + result = new HashMap<>(); + array.set(index, result); + } + return result; + } + + /** + * Sets the element of this big array at specified index. + * + * @param index a position in this big array. + */ + public void set(long index, HashMap value) { + updateRetainedSize(index, value); + array.set(index, value); + } + + /** + * Ensures this big array is at least the specified length. If the array is smaller, segments are + * added until the array is larger then the specified length. + */ + public void ensureCapacity(long length) { + array.ensureCapacity(length); + } + + private void updateRetainedSize(long index, HashMap value) { + HashMap currentValue = array.get(index); + if (currentValue != null) { + sizeOfMaps -= sizeOfObject(currentValue); + } + if (value != null) { + sizeOfMaps += sizeOfObject(value); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java index 58e7ed374f21..104fd4b9f838 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java @@ -46,6 +46,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.DatabaseModelException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; @@ -99,16 +100,16 @@ private ClusterPartitionFetcher() { } @Override - public SchemaPartition getSchemaPartition(PathPatternTree patternTree) { - try (ConfigNodeClient client = + public SchemaPartition getSchemaPartition(final PathPatternTree patternTree) { + try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { patternTree.constructTree(); - List deviceIDs = patternTree.getAllDevicePatterns(); - Map> storageGroupToDeviceMap = + final List deviceIDs = patternTree.getAllDevicePatterns(); + final Map> storageGroupToDeviceMap = partitionCache.getDatabaseToDevice(deviceIDs, true, false, null); SchemaPartition schemaPartition = partitionCache.getSchemaPartition(storageGroupToDeviceMap); if (null == schemaPartition) { - TSchemaPartitionTableResp schemaPartitionTableResp = + final TSchemaPartitionTableResp schemaPartitionTableResp = client.getSchemaPartitionTable(constructSchemaPartitionReq(patternTree)); if (schemaPartitionTableResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -123,23 +124,24 @@ public SchemaPartition getSchemaPartition(PathPatternTree patternTree) { } } return schemaPartition; - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException | DatabaseModelException e) { throw new StatementAnalyzeException( "An error occurred when executing getSchemaPartition():" + e.getMessage()); } } @Override - public SchemaPartition getOrCreateSchemaPartition(PathPatternTree patternTree, String userName) { - try (ConfigNodeClient client = + public SchemaPartition getOrCreateSchemaPartition( + final PathPatternTree patternTree, final String userName) { + try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { patternTree.constructTree(); - List deviceIDs = patternTree.getAllDevicePatterns(); - Map> storageGroupToDeviceMap = + final List deviceIDs = patternTree.getAllDevicePatterns(); + final Map> storageGroupToDeviceMap = partitionCache.getDatabaseToDevice(deviceIDs, true, true, userName); SchemaPartition schemaPartition = partitionCache.getSchemaPartition(storageGroupToDeviceMap); if (null == schemaPartition) { - TSchemaPartitionTableResp schemaPartitionTableResp = + final TSchemaPartitionTableResp schemaPartitionTableResp = client.getOrCreateSchemaPartitionTable(constructSchemaPartitionReq(patternTree)); if (schemaPartitionTableResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -154,7 +156,7 @@ public SchemaPartition getOrCreateSchemaPartition(PathPatternTree patternTree, S } } return schemaPartition; - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException | DatabaseModelException e) { throw new StatementAnalyzeException( "An error occurred when executing getOrCreateSchemaPartition():" + e.getMessage()); } @@ -162,16 +164,16 @@ public SchemaPartition getOrCreateSchemaPartition(PathPatternTree patternTree, S @Override public SchemaNodeManagementPartition getSchemaNodeManagementPartitionWithLevel( - PathPatternTree patternTree, PathPatternTree scope, Integer level) { + final PathPatternTree patternTree, final PathPatternTree scope, final Integer level) { try (ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { patternTree.constructTree(); - TSchemaNodeManagementResp schemaNodeManagementResp = + final TSchemaNodeManagementResp schemaNodeManagementResp = client.getSchemaNodeManagementPartition( constructSchemaNodeManagementPartitionReq(patternTree, scope, level)); return parseSchemaNodeManagementPartitionResp(schemaNodeManagementResp); - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException e) { throw new StatementAnalyzeException( "An error occurred when executing getSchemaNodeManagementPartition():" + e.getMessage()); } @@ -179,12 +181,12 @@ public SchemaNodeManagementPartition getSchemaNodeManagementPartitionWithLevel( @Override public DataPartition getDataPartition( - Map> sgNameToQueryParamsMap) { + final Map> sgNameToQueryParamsMap) { DataPartition dataPartition = partitionCache.getDataPartition(sgNameToQueryParamsMap); if (null == dataPartition) { try (ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TDataPartitionTableResp dataPartitionTableResp = + final TDataPartitionTableResp dataPartitionTableResp = client.getDataPartitionTable(constructDataPartitionReqForQuery(sgNameToQueryParamsMap)); if (dataPartitionTableResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -195,7 +197,7 @@ public DataPartition getDataPartition( "An error occurred when executing getDataPartition():" + dataPartitionTableResp.getStatus().getMessage()); } - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException e) { throw new StatementAnalyzeException( "An error occurred when executing getDataPartition():" + e.getMessage()); } @@ -205,13 +207,13 @@ public DataPartition getDataPartition( @Override public DataPartition getDataPartitionWithUnclosedTimeRange( - Map> sgNameToQueryParamsMap) { + final Map> sgNameToQueryParamsMap) { // In this method, we must fetch from config node because it contains -oo or +oo // and there is no need to update cache because since we will never fetch it from cache, the // update operation will be only time waste - try (ConfigNodeClient client = + try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TDataPartitionTableResp dataPartitionTableResp = + final TDataPartitionTableResp dataPartitionTableResp = client.getDataPartitionTable(constructDataPartitionReqForQuery(sgNameToQueryParamsMap)); if (dataPartitionTableResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -221,7 +223,7 @@ public DataPartition getDataPartitionWithUnclosedTimeRange( "An error occurred when executing getDataPartition():" + dataPartitionTableResp.getStatus().getMessage()); } - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException e) { throw new StatementAnalyzeException( "An error occurred when executing getDataPartition():" + e.getMessage()); } @@ -229,13 +231,13 @@ public DataPartition getDataPartitionWithUnclosedTimeRange( @Override public DataPartition getOrCreateDataPartition( - Map> sgNameToQueryParamsMap) { + final Map> sgNameToQueryParamsMap) { DataPartition dataPartition = partitionCache.getDataPartition(sgNameToQueryParamsMap); if (null == dataPartition) { // Do not use data partition cache - try (ConfigNodeClient client = + try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TDataPartitionTableResp dataPartitionTableResp = + final TDataPartitionTableResp dataPartitionTableResp = client.getOrCreateDataPartitionTable(constructDataPartitionReq(sgNameToQueryParamsMap)); if (dataPartitionTableResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -246,7 +248,7 @@ public DataPartition getOrCreateDataPartition( "An error occurred when executing getOrCreateDataPartition():" + dataPartitionTableResp.getStatus().getMessage()); } - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException e) { throw new StatementAnalyzeException( "An error occurred when executing getOrCreateDataPartition():" + e.getMessage()); } @@ -256,18 +258,19 @@ public DataPartition getOrCreateDataPartition( @Override public DataPartition getOrCreateDataPartition( - List dataPartitionQueryParams, String userName) { - - Map> splitDataPartitionQueryParams = - splitDataPartitionQueryParam( - dataPartitionQueryParams, config.isAutoCreateSchemaEnabled(), userName); - DataPartition dataPartition = partitionCache.getDataPartition(splitDataPartitionQueryParams); + final List dataPartitionQueryParams, final String userName) { + DataPartition dataPartition; + try (final ConfigNodeClient client = + configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + final Map> splitDataPartitionQueryParams = + splitDataPartitionQueryParam( + dataPartitionQueryParams, config.isAutoCreateSchemaEnabled(), userName); + dataPartition = partitionCache.getDataPartition(splitDataPartitionQueryParams); - if (null == dataPartition) { - try (ConfigNodeClient client = - configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TDataPartitionReq req = constructDataPartitionReq(splitDataPartitionQueryParams); - TDataPartitionTableResp dataPartitionTableResp = client.getOrCreateDataPartitionTable(req); + if (null == dataPartition) { + final TDataPartitionReq req = constructDataPartitionReq(splitDataPartitionQueryParams); + final TDataPartitionTableResp dataPartitionTableResp = + client.getOrCreateDataPartitionTable(req); if (dataPartitionTableResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -279,16 +282,16 @@ public DataPartition getOrCreateDataPartition( dataPartitionTableResp.getStatus().getMessage(), dataPartitionTableResp.getStatus().getCode())); } - } catch (ClientManagerException | TException e) { - throw new StatementAnalyzeException( - "An error occurred when executing getOrCreateDataPartition():" + e.getMessage()); } + } catch (final ClientManagerException | TException | DatabaseModelException e) { + throw new StatementAnalyzeException( + "An error occurred when executing getOrCreateDataPartition():" + e.getMessage()); } return dataPartition; } @Override - public boolean updateRegionCache(TRegionRouteReq req) { + public boolean updateRegionCache(final TRegionRouteReq req) { return partitionCache.updateGroupIdToReplicaSetMap(req.getTimestamp(), req.getRegionRouteMap()); } @@ -299,29 +302,33 @@ public void invalidAllCache() { @Override public SchemaPartition getOrCreateSchemaPartition( - String database, List deviceIDs, String userName) { + final String database, final List deviceIDs, final String userName) { return getOrCreateSchemaPartition(database, deviceIDs, true, userName); } @Override - public SchemaPartition getSchemaPartition(String database, List deviceIDs) { + public SchemaPartition getSchemaPartition( + final String database, final List deviceIDs) { return getOrCreateSchemaPartition(database, deviceIDs, false, null); } private SchemaPartition getOrCreateSchemaPartition( - String database, List deviceIDs, boolean isAutoCreate, String userName) { - try (ConfigNodeClient client = + final String database, + final List deviceIDs, + final boolean isAutoCreate, + final String userName) { + try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { partitionCache.checkAndAutoCreateDatabase(database, isAutoCreate, userName); SchemaPartition schemaPartition = partitionCache.getSchemaPartition(Collections.singletonMap(database, deviceIDs)); if (null == schemaPartition) { - List partitionSlots = + final List partitionSlots = deviceIDs.stream() .map(partitionExecutor::getSeriesPartitionSlot) .distinct() .collect(Collectors.toList()); - TSchemaPartitionTableResp schemaPartitionTableResp = + final TSchemaPartitionTableResp schemaPartitionTableResp = isAutoCreate ? client.getOrCreateSchemaPartitionTableWithSlots( Collections.singletonMap(database, partitionSlots)) @@ -340,7 +347,7 @@ private SchemaPartition getOrCreateSchemaPartition( } } return schemaPartition; - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException | DatabaseModelException e) { throw new StatementAnalyzeException( "An error occurred when executing getSchemaPartition():" + e.getMessage()); } @@ -361,17 +368,18 @@ public SchemaPartition getSchemaPartition(final String database) { /** split data partition query param by database */ private Map> splitDataPartitionQueryParam( - List dataPartitionQueryParams, - boolean isAutoCreate, - String userName) { - List deviceIDs = new ArrayList<>(); - for (DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) { + final List dataPartitionQueryParams, + final boolean isAutoCreate, + final String userName) + throws DatabaseModelException { + final List deviceIDs = new ArrayList<>(); + for (final DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) { deviceIDs.add(dataPartitionQueryParam.getDeviceID()); } Map deviceToDatabase = null; - Map> result = new HashMap<>(); - for (DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) { - IDeviceID deviceID = dataPartitionQueryParam.getDeviceID(); + final Map> result = new HashMap<>(); + for (final DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) { + final IDeviceID deviceID = dataPartitionQueryParam.getDeviceID(); String database = null; if (dataPartitionQueryParam.getDatabaseName() == null) { if (deviceToDatabase == null) { @@ -391,7 +399,7 @@ private Map> splitDataPartitionQueryParam( return result; } - private TSchemaPartitionReq constructSchemaPartitionReq(PathPatternTree patternTree) { + private TSchemaPartitionReq constructSchemaPartitionReq(final PathPatternTree patternTree) { try { return new TSchemaPartitionReq(patternTree.serialize()); } catch (IOException e) { @@ -400,9 +408,9 @@ private TSchemaPartitionReq constructSchemaPartitionReq(PathPatternTree patternT } private TSchemaNodeManagementReq constructSchemaNodeManagementPartitionReq( - PathPatternTree patternTree, PathPatternTree scope, Integer level) { + final PathPatternTree patternTree, final PathPatternTree scope, final Integer level) { try { - TSchemaNodeManagementReq schemaNodeManagementReq = + final TSchemaNodeManagementReq schemaNodeManagementReq = new TSchemaNodeManagementReq(patternTree.serialize()); schemaNodeManagementReq.setScopePatternTree(scope.serialize()); if (null == level) { @@ -422,28 +430,29 @@ private static class ComplexTimeSlotList { boolean needLeftAll; boolean needRightAll; - private ComplexTimeSlotList(boolean needLeftAll, boolean needRightAll) { + private ComplexTimeSlotList(final boolean needLeftAll, final boolean needRightAll) { timeSlotList = new HashSet<>(); this.needLeftAll = needLeftAll; this.needRightAll = needRightAll; } - private void putTimeSlot(List slotList) { + private void putTimeSlot(final List slotList) { timeSlotList.addAll(slotList); } } private TDataPartitionReq constructDataPartitionReq( - Map> sgNameToQueryParamsMap) { - Map> partitionSlotsMap = new HashMap<>(); - for (Map.Entry> entry : + final Map> sgNameToQueryParamsMap) { + final Map> partitionSlotsMap = new HashMap<>(); + for (final Map.Entry> entry : sgNameToQueryParamsMap.entrySet()) { // for each sg - Map deviceToTimePartitionMap = new HashMap<>(); + final Map deviceToTimePartitionMap = new HashMap<>(); - Map seriesSlotTimePartitionMap = new HashMap<>(); + final Map seriesSlotTimePartitionMap = + new HashMap<>(); - for (DataPartitionQueryParam queryParam : entry.getValue()) { + for (final DataPartitionQueryParam queryParam : entry.getValue()) { seriesSlotTimePartitionMap .computeIfAbsent( partitionExecutor.getSeriesPartitionSlot(queryParam.getDeviceID()), @@ -466,15 +475,15 @@ private TDataPartitionReq constructDataPartitionReq( /** For query, DataPartitionQueryParam is shared by each device */ private TDataPartitionReq constructDataPartitionReqForQuery( - Map> sgNameToQueryParamsMap) { - Map> partitionSlotsMap = new HashMap<>(); + final Map> sgNameToQueryParamsMap) { + final Map> partitionSlotsMap = new HashMap<>(); TTimeSlotList sharedTTimeSlotList = null; - for (Map.Entry> entry : + for (final Map.Entry> entry : sgNameToQueryParamsMap.entrySet()) { // for each sg - Map deviceToTimePartitionMap = new HashMap<>(); + final Map deviceToTimePartitionMap = new HashMap<>(); - for (DataPartitionQueryParam queryParam : entry.getValue()) { + for (final DataPartitionQueryParam queryParam : entry.getValue()) { if (sharedTTimeSlotList == null) { sharedTTimeSlotList = new TTimeSlotList( @@ -492,13 +501,14 @@ private TDataPartitionReq constructDataPartitionReqForQuery( } private SchemaPartition parseSchemaPartitionTableResp( - TSchemaPartitionTableResp schemaPartitionTableResp) { - Map> regionReplicaMap = new HashMap<>(); - for (Map.Entry> entry1 : + final TSchemaPartitionTableResp schemaPartitionTableResp) { + final Map> regionReplicaMap = + new HashMap<>(); + for (final Map.Entry> entry1 : schemaPartitionTableResp.getSchemaPartitionTable().entrySet()) { - Map result1 = + final Map result1 = regionReplicaMap.computeIfAbsent(entry1.getKey(), k -> new HashMap<>()); - for (Map.Entry entry2 : + for (final Map.Entry entry2 : entry1.getValue().entrySet()) { TSeriesPartitionSlot seriesPartitionSlot = entry2.getKey(); TConsensusGroupId consensusGroupId = entry2.getValue(); @@ -513,7 +523,7 @@ private SchemaPartition parseSchemaPartitionTableResp( } private SchemaNodeManagementPartition parseSchemaNodeManagementPartitionResp( - TSchemaNodeManagementResp schemaNodeManagementResp) { + final TSchemaNodeManagementResp schemaNodeManagementResp) { return new SchemaNodeManagementPartition( schemaNodeManagementResp.getSchemaRegionMap(), IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionExecutorClass(), @@ -521,22 +531,23 @@ private SchemaNodeManagementPartition parseSchemaNodeManagementPartitionResp( schemaNodeManagementResp.getMatchedNode()); } - private DataPartition parseDataPartitionResp(TDataPartitionTableResp dataPartitionTableResp) { - Map>>> + private DataPartition parseDataPartitionResp( + final TDataPartitionTableResp dataPartitionTableResp) { + final Map>>> regionReplicaSet = new HashMap<>(); - for (Map.Entry< + for (final Map.Entry< String, Map>>> entry1 : dataPartitionTableResp.getDataPartitionTable().entrySet()) { - Map>> result1 = + final Map>> result1 = regionReplicaSet.computeIfAbsent(entry1.getKey(), k -> new HashMap<>()); - for (Map.Entry>> + for (final Map.Entry>> entry2 : entry1.getValue().entrySet()) { - Map> result2 = + final Map> result2 = result1.computeIfAbsent(entry2.getKey(), k -> new HashMap<>()); - for (Map.Entry> entry3 : + for (final Map.Entry> entry3 : entry2.getValue().entrySet()) { - List regionReplicaSets = new LinkedList<>(); - for (TConsensusGroupId consensusGroupId : entry3.getValue()) { + final List regionReplicaSets = new LinkedList<>(); + for (final TConsensusGroupId consensusGroupId : entry3.getValue()) { regionReplicaSets.add(partitionCache.getRegionReplicaSet(consensusGroupId)); } result2.put(entry3.getKey(), regionReplicaSets); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 714d6b18082c..e5d11d74f785 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -48,6 +48,7 @@ import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.DatabaseModelException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; @@ -74,6 +75,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; public class PartitionCache { @@ -88,7 +90,7 @@ public class PartitionCache { private final SeriesPartitionExecutor partitionExecutor; /** the cache of database */ - private final Set databaseCache = new HashSet<>(); + private final Map databaseCache = new HashMap<>(); /** database -> schemaPartitionTable */ private final Cache schemaPartitionCache; @@ -136,11 +138,15 @@ public PartitionCache() { * @param userName the userName */ public Map> getDatabaseToDevice( - List deviceIDs, boolean tryToFetch, boolean isAutoCreate, String userName) { - DatabaseCacheResult> result = + final List deviceIDs, + final boolean tryToFetch, + final boolean isAutoCreate, + final String userName) + throws DatabaseModelException { + final DatabaseCacheResult> result = new DatabaseCacheResult>() { @Override - public void put(IDeviceID device, String databaseName) { + public void put(final IDeviceID device, final String databaseName) { map.computeIfAbsent(databaseName, k -> new ArrayList<>()).add(device); } }; @@ -157,11 +163,15 @@ public void put(IDeviceID device, String databaseName) { * @param userName the userName */ public Map getDeviceToDatabase( - List deviceIDs, boolean tryToFetch, boolean isAutoCreate, String userName) { - DatabaseCacheResult result = + final List deviceIDs, + final boolean tryToFetch, + final boolean isAutoCreate, + final String userName) + throws DatabaseModelException { + final DatabaseCacheResult result = new DatabaseCacheResult() { @Override - public void put(IDeviceID device, String databaseName) { + public void put(final IDeviceID device, final String databaseName) { map.put(device, databaseName); } }; @@ -173,12 +183,16 @@ public void put(IDeviceID device, String databaseName) { * get database of device * * @param deviceID the path of device - * @return database name, return null if cache miss + * @return database name, return {@code null} if cache miss */ - private String getDatabaseName(IDeviceID deviceID) { - for (String databaseName : databaseCache) { - if (PathUtils.isStartWith(deviceID, databaseName)) { - return databaseName; + private String getDatabaseName(final IDeviceID deviceID) throws DatabaseModelException { + for (final Map.Entry entry : databaseCache.entrySet()) { + final String database = entry.getKey(); + if (PathUtils.isStartWith(deviceID, database)) { + if (Boolean.TRUE.equals(entry.getValue())) { + throw new DatabaseModelException(database, true); + } + return entry.getKey(); } } return null; @@ -188,12 +202,18 @@ private String getDatabaseName(IDeviceID deviceID) { * judge whether this database is existed * * @param database name - * @return true of false + * @return {@code true} if this database exists */ - private boolean containsDatabase(String database) { + private boolean containsDatabase(final String database) throws DatabaseModelException { try { databaseCacheLock.readLock().lock(); - return databaseCache.contains(database); + if (databaseCache.containsKey(database)) { + if (Boolean.FALSE.equals(databaseCache.get(database))) { + throw new DatabaseModelException(PathUtils.unQualifyDatabaseName(database), false); + } + return true; + } + return false; } finally { databaseCacheLock.readLock().unlock(); } @@ -206,21 +226,25 @@ private boolean containsDatabase(String database) { * @param deviceIDs the devices that need to hit */ private void fetchDatabaseAndUpdateCache( - DatabaseCacheResult result, List deviceIDs) - throws ClientManagerException, TException { + final DatabaseCacheResult result, final List deviceIDs) + throws ClientManagerException, TException, DatabaseModelException { databaseCacheLock.writeLock().lock(); - try (ConfigNodeClient client = + try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { result.reset(); getDatabaseMap(result, deviceIDs, true); if (!result.isSuccess()) { - TGetDatabaseReq req = new TGetDatabaseReq(ROOT_PATH, SchemaConstant.ALL_MATCH_SCOPE_BINARY); - TDatabaseSchemaResp databaseSchemaResp = client.getMatchedDatabaseSchemas(req); + final TGetDatabaseReq req = + new TGetDatabaseReq(ROOT_PATH, SchemaConstant.ALL_MATCH_SCOPE_BINARY); + final TDatabaseSchemaResp databaseSchemaResp = client.getMatchedDatabaseSchemas(req); if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - Set databaseNames = databaseSchemaResp.getDatabaseSchemaMap().keySet(); // update all database into cache - updateDatabaseCache(databaseNames); + updateDatabaseCache( + databaseSchemaResp.getDatabaseSchemaMap().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, entry -> entry.getValue().isIsTableModel()))); getDatabaseMap(result, deviceIDs, true); } } @@ -230,17 +254,20 @@ private void fetchDatabaseAndUpdateCache( } /** get all database from configNode and update database cache. */ - private void fetchDatabaseAndUpdateCache(String database) - throws ClientManagerException, TException { + private void fetchDatabaseAndUpdateCache() throws ClientManagerException, TException { databaseCacheLock.writeLock().lock(); - try (ConfigNodeClient client = + try (final ConfigNodeClient client = configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TGetDatabaseReq req = new TGetDatabaseReq(ROOT_PATH, SchemaConstant.ALL_MATCH_SCOPE_BINARY); - TDatabaseSchemaResp databaseSchemaResp = client.getMatchedDatabaseSchemas(req); + final TGetDatabaseReq req = + new TGetDatabaseReq(ROOT_PATH, SchemaConstant.ALL_MATCH_SCOPE_BINARY); + final TDatabaseSchemaResp databaseSchemaResp = client.getMatchedDatabaseSchemas(req); if (databaseSchemaResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - Set databaseNames = databaseSchemaResp.getDatabaseSchemaMap().keySet(); // update all database into cache - updateDatabaseCache(databaseNames); + updateDatabaseCache( + databaseSchemaResp.getDatabaseSchemaMap().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, entry -> entry.getValue().isIsTableModel()))); } } finally { databaseCacheLock.writeLock().unlock(); @@ -274,7 +301,7 @@ private void createDatabaseAndUpdateCache( if (PathUtils.isStartWith(deviceID, SchemaConstant.SYSTEM_DATABASE)) { databaseNamesNeedCreated.add(SchemaConstant.SYSTEM_DATABASE); } else { - PartialPath databaseNameNeedCreated = + final PartialPath databaseNameNeedCreated = MetaUtils.getDatabasePathByLevel( new PartialPath(deviceID), config.getDefaultStorageGroupLevel()); databaseNamesNeedCreated.add(databaseNameNeedCreated.getFullPath()); @@ -282,7 +309,7 @@ private void createDatabaseAndUpdateCache( } // Try to create databases one by one until done or one database fail - final Set successFullyCreatedDatabase = new HashSet<>(); + final Map successFullyCreatedDatabase = new HashMap<>(); for (final String databaseName : databaseNamesNeedCreated) { final long startTime = System.nanoTime(); try { @@ -302,18 +329,11 @@ private void createDatabaseAndUpdateCache( } final TDatabaseSchema databaseSchema = new TDatabaseSchema(); databaseSchema.setName(databaseName); + databaseSchema.setIsTableModel(false); final TSStatus tsStatus = client.setDatabase(databaseSchema); - if (SchemaConstant.SYSTEM_DATABASE.equals(databaseName)) { - databaseSchema.setSchemaReplicationFactor(1); - databaseSchema.setDataReplicationFactor(1); - databaseSchema.setMinSchemaRegionGroupNum(1); - databaseSchema.setMaxSchemaRegionGroupNum(1); - databaseSchema.setMaxDataRegionGroupNum(1); - databaseSchema.setMaxDataRegionGroupNum(1); - } if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() || TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() == tsStatus.getCode()) { - successFullyCreatedDatabase.add(databaseName); + successFullyCreatedDatabase.put(databaseName, false); // In tree model, if the user creates a conflict database concurrently, for instance, // the database created by user is root.db.ss.a, the auto-creation failed database is // root.db, we wait till "getOrCreatePartition" to judge if the time series (like @@ -330,7 +350,7 @@ private void createDatabaseAndUpdateCache( } } // Try to update database cache when all databases have already been created - updateDatabaseCache(databaseNamesNeedCreated); + updateDatabaseCache(successFullyCreatedDatabase); getDatabaseMap(result, deviceIDs, false); } } finally { @@ -367,11 +387,12 @@ private void createDatabaseAndUpdateCache(final String database, final String us } final TDatabaseSchema databaseSchema = new TDatabaseSchema(); databaseSchema.setName(database); + databaseSchema.setIsTableModel(true); final TSStatus tsStatus = client.setDatabase(databaseSchema); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() || TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode() == tsStatus.getCode()) { // Try to update cache by databases successfully created - updateDatabaseCache(Collections.singleton(database)); + updateDatabaseCache(Collections.singletonMap(database, true)); } else { logger.warn( "[{} Cache] failed to create database {}", CacheMetrics.DATABASE_CACHE_NAME, database); @@ -387,17 +408,21 @@ private void createDatabaseAndUpdateCache(final String database, final String us * * @param result contains result(boolean), failed devices and the map * @param deviceIDs the devices that need to hit - * @param failFast if true, return when failed. if false, return when all devices hit + * @param failFast if {@code true}, return when failed. if {@code false}, return when all devices + * hit */ private void getDatabaseMap( - DatabaseCacheResult result, List deviceIDs, boolean failFast) { + final DatabaseCacheResult result, + final List deviceIDs, + final boolean failFast) + throws DatabaseModelException { try { databaseCacheLock.readLock().lock(); // reset result before try result.reset(); boolean status = true; - for (IDeviceID devicePath : deviceIDs) { - String databaseName = getDatabaseName(devicePath); + for (final IDeviceID devicePath : deviceIDs) { + final String databaseName = getDatabaseName(devicePath); if (null == databaseName) { logger.debug( "[{} Cache] miss when search device {}", @@ -435,15 +460,16 @@ private void getDatabaseMap( * @param userName */ private void getDatabaseCacheResult( - DatabaseCacheResult result, - List deviceIDs, - boolean tryToFetch, - boolean isAutoCreate, - String userName) { + final DatabaseCacheResult result, + final List deviceIDs, + final boolean tryToFetch, + final boolean isAutoCreate, + final String userName) + throws DatabaseModelException { if (!isAutoCreate) { // TODO: avoid IDeviceID contains "*" // miss when deviceId contains * - for (IDeviceID deviceID : deviceIDs) { + for (final IDeviceID deviceID : deviceIDs) { for (int i = 0; i < deviceID.segmentNum(); i++) { if (((String) deviceID.segment(i)).contains("*")) { return; @@ -464,25 +490,27 @@ private void getDatabaseCacheResult( throw new StatementAnalyzeException("Failed to get database Map"); } } - } catch (TException | MetadataException | ClientManagerException e) { + } catch (final TException | MetadataException | ClientManagerException e) { throw new StatementAnalyzeException( "An error occurred when executing getDeviceToDatabase():" + e.getMessage()); } } } - public void checkAndAutoCreateDatabase(String database, boolean isAutoCreate, String userName) { + public void checkAndAutoCreateDatabase( + final String database, final boolean isAutoCreate, final String userName) + throws DatabaseModelException { boolean isExisted = containsDatabase(database); if (!isExisted) { try { // try to fetch database from config node when miss - fetchDatabaseAndUpdateCache(database); + fetchDatabaseAndUpdateCache(); isExisted = containsDatabase(database); if (!isExisted && isAutoCreate) { // try to auto create database of failed device createDatabaseAndUpdateCache(database, userName); } - } catch (TException | ClientManagerException e) { + } catch (final TException | ClientManagerException e) { throw new StatementAnalyzeException( "An error occurred when executing getDeviceToDatabase():" + e.getMessage()); } @@ -494,10 +522,10 @@ public void checkAndAutoCreateDatabase(String database, boolean isAutoCreate, St * * @param databaseNames the database names that need to update */ - public void updateDatabaseCache(Set databaseNames) { + public void updateDatabaseCache(final Map databaseNames) { databaseCacheLock.writeLock().lock(); try { - databaseCache.addAll(databaseNames); + databaseCache.putAll(databaseNames); } finally { databaseCacheLock.writeLock().unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 1f42d8ae9a10..59918c252659 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -157,6 +157,7 @@ protected IConfigTask visitCreateDB(final CreateDB node, final MPPQueryContext c context.setQueryType(QueryType.WRITE); final TDatabaseSchema schema = new TDatabaseSchema(); + schema.setIsTableModel(true); final String dbName = node.getDbName(); validateDatabaseName(dbName); @@ -391,7 +392,7 @@ protected IConfigTask visitSetProperties( node.ifExists()); } - public static void validateDatabaseName(String dbName) throws SemanticException { + public static void validateDatabaseName(final String dbName) throws SemanticException { // Check database length here // We need to calculate the database name without "root." if (dbName.contains(PATH_SEPARATOR) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index a34c820930ac..66617ec3ef0b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -60,6 +60,7 @@ import org.apache.iotdb.commons.udf.service.UDFClassLoader; import org.apache.iotdb.commons.udf.service.UDFExecutableManager; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; @@ -129,6 +130,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.exception.metadata.DatabaseModelException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.exception.metadata.table.TableNotExistsException; @@ -414,7 +416,7 @@ public SettableFuture showDatabase( final TShowDatabaseResp resp = client.showDatabase(req); // build TSBlock showDatabaseStatement.buildTSBlock(resp.getDatabaseInfoMap(), future); - } catch (final IOException | ClientManagerException | TException | IllegalPathException e) { + } catch (final IOException | ClientManagerException | TException e) { future.setException(e); } return future; @@ -2930,18 +2932,19 @@ public TPipeTransferResp handleTransferConfigPlan(String clientId, TPipeTransfer } @Override - public SettableFuture showDatabases(ShowDB showDB) { - SettableFuture future = SettableFuture.create(); + public SettableFuture showDatabases(final ShowDB showDB) { + final SettableFuture future = SettableFuture.create(); // Construct request using statement - List databasePathPattern = Arrays.asList(ALL_RESULT_NODES); - try (ConfigNodeClient client = + final List databasePathPattern = Arrays.asList(ALL_RESULT_NODES); + try (final ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { // Send request to some API server - TGetDatabaseReq req = new TGetDatabaseReq(databasePathPattern, ALL_MATCH_SCOPE.serialize()); - TShowDatabaseResp resp = client.showDatabase(req); + final TGetDatabaseReq req = + new TGetDatabaseReq(databasePathPattern, ALL_MATCH_SCOPE.serialize()); + final TShowDatabaseResp resp = client.showDatabase(req); // build TSBlock - ShowDBTask.buildTSBlock(resp.getDatabaseInfoMap(), future); - } catch (IOException | ClientManagerException | TException e) { + ShowDBTask.buildTSBlock(resp.getDatabaseInfoMap(), future, showDB.isDetails()); + } catch (final IOException | ClientManagerException | TException e) { future.setException(e); } return future; @@ -2957,16 +2960,23 @@ public SettableFuture showCluster(ShowCluster showCluster) { } @Override - public SettableFuture useDatabase(Use useDB, IClientSession clientSession) { - SettableFuture future = SettableFuture.create(); + public SettableFuture useDatabase( + final Use useDB, final IClientSession clientSession) { + final SettableFuture future = SettableFuture.create(); // Construct request using statement - List databasePathPattern = Arrays.asList(ROOT, useDB.getDatabaseId().getValue()); - try (ConfigNodeClient client = + final List databasePathPattern = Arrays.asList(ROOT, useDB.getDatabaseId().getValue()); + try (final ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { // Send request to some API server - TGetDatabaseReq req = new TGetDatabaseReq(databasePathPattern, ALL_MATCH_SCOPE.serialize()); - TShowDatabaseResp resp = client.showDatabase(req); + final TGetDatabaseReq req = + new TGetDatabaseReq(databasePathPattern, ALL_MATCH_SCOPE.serialize()); + final TShowDatabaseResp resp = client.showDatabase(req); if (!resp.getDatabaseInfoMap().isEmpty()) { + if (!resp.getDatabaseInfoMap() + .get(PathUtils.qualifyDatabaseName(useDB.getDatabaseId().getValue())) + .isIsTableModel()) { + throw new DatabaseModelException(useDB.getDatabaseId().getValue(), false); + } clientSession.setDatabaseName(useDB.getDatabaseId().getValue()); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); } else { @@ -2975,7 +2985,7 @@ public SettableFuture useDatabase(Use useDB, IClientSession cl String.format("Unknown database %s", useDB.getDatabaseId().getValue()), TSStatusCode.DATABASE_NOT_EXIST.getStatusCode())); } - } catch (IOException | ClientManagerException | TException e) { + } catch (final IOException | ClientManagerException | TException | DatabaseModelException e) { future.setException(e); } return future; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java index 48eb140f0941..ab9dee095aaa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DatabaseSchemaTask.java @@ -50,8 +50,8 @@ public ListenableFuture execute(IConfigTaskExecutor configTask /** Construct DatabaseSchema according to statement */ public static TDatabaseSchema constructDatabaseSchema( - DatabaseSchemaStatement databaseSchemaStatement) { - TDatabaseSchema databaseSchema = new TDatabaseSchema(); + final DatabaseSchemaStatement databaseSchemaStatement) { + final TDatabaseSchema databaseSchema = new TDatabaseSchema(); databaseSchema.setName(databaseSchemaStatement.getDatabasePath().getFullPath()); if (databaseSchemaStatement.getTtl() != null) { databaseSchema.setTTL(databaseSchemaStatement.getTtl()); @@ -72,6 +72,7 @@ public static TDatabaseSchema constructDatabaseSchema( if (databaseSchemaStatement.getDataRegionGroupNum() != null) { databaseSchema.setMinDataRegionGroupNum(databaseSchemaStatement.getDataRegionGroupNum()); } + databaseSchema.setIsTableModel(false); return databaseSchema; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDatabaseTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDatabaseTask.java index fcf3f9d13dc5..34330c28c43f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDatabaseTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDatabaseTask.java @@ -30,12 +30,12 @@ public class ShowDatabaseTask implements IConfigTask { private final ShowDatabaseStatement showDatabaseStatement; - public ShowDatabaseTask(ShowDatabaseStatement showDatabaseStatement) { + public ShowDatabaseTask(final ShowDatabaseStatement showDatabaseStatement) { this.showDatabaseStatement = showDatabaseStatement; } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.showDatabase(showDatabaseStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java index 0a72f18a3d2f..64580a97528f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDBTask.java @@ -45,38 +45,80 @@ public class ShowDBTask implements IConfigTask { private final ShowDB node; - public ShowDBTask(ShowDB node) { + public ShowDBTask(final ShowDB node) { this.node = node; } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.showDatabases(node); } public static void buildTSBlock( - Map storageGroupInfoMap, SettableFuture future) { + final Map storageGroupInfoMap, + final SettableFuture future, + final boolean isDetails) { + if (isDetails) { + buildTSBlockForDetails(storageGroupInfoMap, future); + } else { + buildTSBlockForNonDetails(storageGroupInfoMap, future); + } + } - List outputDataTypes = + private static void buildTSBlockForNonDetails( + final Map storageGroupInfoMap, + final SettableFuture future) { + final List outputDataTypes = ColumnHeaderConstant.showDBColumnHeaders.stream() .map(ColumnHeader::getColumnType) .collect(Collectors.toList()); - TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); - for (Map.Entry entry : storageGroupInfoMap.entrySet()) { - String dbName = entry.getKey().substring(5); - TDatabaseInfo storageGroupInfo = entry.getValue(); + final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + for (final Map.Entry entry : storageGroupInfoMap.entrySet()) { + final String dbName = entry.getKey().substring(5); + final TDatabaseInfo storageGroupInfo = entry.getValue(); + builder.getTimeColumnBuilder().writeLong(0L); + builder.getColumnBuilder(0).writeBinary(new Binary(dbName, TSFileConfig.STRING_CHARSET)); + + builder.getColumnBuilder(1).writeInt(storageGroupInfo.getSchemaReplicationFactor()); + builder.getColumnBuilder(2).writeInt(storageGroupInfo.getDataReplicationFactor()); + builder.getColumnBuilder(3).writeLong(storageGroupInfo.getTimePartitionInterval()); + builder.declarePosition(); + } + + final DatasetHeader datasetHeader = DatasetHeaderFactory.getShowDBHeader(); + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); + } + + private static void buildTSBlockForDetails( + final Map storageGroupInfoMap, + final SettableFuture future) { + final List outputDataTypes = + ColumnHeaderConstant.showDBDetailsColumnHeaders.stream() + .map(ColumnHeader::getColumnType) + .collect(Collectors.toList()); + + final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + for (final Map.Entry entry : storageGroupInfoMap.entrySet()) { + final String dbName = entry.getKey().substring(5); + final TDatabaseInfo storageGroupInfo = entry.getValue(); builder.getTimeColumnBuilder().writeLong(0L); builder.getColumnBuilder(0).writeBinary(new Binary(dbName, TSFileConfig.STRING_CHARSET)); builder.getColumnBuilder(1).writeInt(storageGroupInfo.getSchemaReplicationFactor()); builder.getColumnBuilder(2).writeInt(storageGroupInfo.getDataReplicationFactor()); builder.getColumnBuilder(3).writeLong(storageGroupInfo.getTimePartitionInterval()); + builder + .getColumnBuilder(4) + .writeBinary( + new Binary( + storageGroupInfo.isIsTableModel() ? "TABLE" : "TREE", + TSFileConfig.STRING_CHARSET)); builder.declarePosition(); } - DatasetHeader datasetHeader = DatasetHeaderFactory.getShowDBHeader(); + final DatasetHeader datasetHeader = DatasetHeaderFactory.getShowDBDetailsHeader(); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 1d02608b317b..56d6868a24b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -129,6 +129,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.LeafColumnTransformer; @@ -145,7 +146,6 @@ import org.apache.tsfile.read.common.type.BinaryType; import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.BooleanType; -import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Binary; @@ -1368,26 +1368,21 @@ private TableAggregator buildAggregator( TypeProvider typeProvider, boolean scanAscending) { List argumentChannels = new ArrayList<>(); - List argumentTypes = new ArrayList<>(); for (Expression argument : aggregation.getArguments()) { Symbol argumentSymbol = Symbol.from(argument); argumentChannels.add(childLayout.get(argumentSymbol)); - - // get argument types - Type type = typeProvider.getTableModelType(argumentSymbol); - if (type instanceof RowType) { - type.getTypeParameters().forEach(subType -> argumentTypes.add(getTSDataType(subType))); - } else { - argumentTypes.add(getTSDataType(type)); - } } String functionName = aggregation.getResolvedFunction().getSignature().getName(); + List originalArgumentTypes = + aggregation.getResolvedFunction().getSignature().getArgumentTypes().stream() + .map(InternalTypeManager::getTSDataType) + .collect(Collectors.toList()); TableAccumulator accumulator = createAccumulator( functionName, getAggregationTypeByFuncName(functionName), - argumentTypes, + originalArgumentTypes, aggregation.getArguments(), Collections.emptyMap(), scanAscending); @@ -1454,26 +1449,21 @@ private GroupedAggregator buildGroupByAggregator( AggregationNode.Step step, TypeProvider typeProvider) { List argumentChannels = new ArrayList<>(); - List argumentTypes = new ArrayList<>(); for (Expression argument : aggregation.getArguments()) { Symbol argumentSymbol = Symbol.from(argument); argumentChannels.add(childLayout.get(argumentSymbol)); - - // get argument types - Type type = typeProvider.getTableModelType(argumentSymbol); - if (type instanceof RowType) { - type.getTypeParameters().forEach(subType -> argumentTypes.add(getTSDataType(subType))); - } else { - argumentTypes.add(getTSDataType(type)); - } } String functionName = aggregation.getResolvedFunction().getSignature().getName(); + List originalArgumentTypes = + aggregation.getResolvedFunction().getSignature().getArgumentTypes().stream() + .map(InternalTypeManager::getTSDataType) + .collect(Collectors.toList()); GroupedAccumulator accumulator = createGroupedAccumulator( functionName, getAggregationTypeByFuncName(functionName), - argumentTypes, + originalArgumentTypes, Collections.emptyList(), Collections.emptyMap(), true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableBuiltinAggregationFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableBuiltinAggregationFunction.java index 202ba8678903..b070d78b9772 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableBuiltinAggregationFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableBuiltinAggregationFunction.java @@ -21,10 +21,11 @@ import org.apache.iotdb.common.rpc.thrift.TAggregationType; -import com.google.common.collect.ImmutableList; +import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -75,18 +76,34 @@ public static Set getNativeFunctionNames() { return NATIVE_FUNCTION_NAMES; } - public static List getIntermediateTypes(String name, List originalArgumentTypes) { - if (COUNT.functionName.equalsIgnoreCase(name)) { - return ImmutableList.of(INT64); - } else if (SUM.functionName.equalsIgnoreCase(name)) { - return ImmutableList.of(DOUBLE); - } else if (AVG.functionName.equalsIgnoreCase(name)) { - return ImmutableList.of(DOUBLE, INT64); - } else if (LAST.functionName.equalsIgnoreCase(name)) { - return ImmutableList.of(originalArgumentTypes.get(0), INT64); - } else { - // TODO(beyyes) consider other aggregations which changed the result type - return ImmutableList.copyOf(originalArgumentTypes); + public static Type getIntermediateType(String name, List originalArgumentTypes) { + final String functionName = name.toLowerCase(); + switch (functionName) { + case "count": + return INT64; + case "sum": + return DOUBLE; + case "avg": + case "first": + case "first_by": + case "last": + case "last_by": + case "mode": + case "max_by": + case "min_by": + case "stddev": + case "stddev_pop": + case "stddev_samp": + case "variance": + case "var_pop": + case "var_samp": + return RowType.anonymous(Collections.emptyList()); + case "extreme": + case "max": + case "min": + return originalArgumentTypes.get(0); + default: + throw new IllegalArgumentException("Invalid Aggregation function: " + name); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index d9e7663d19b9..caf93f905739 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -192,13 +192,13 @@ public Optional validateTableHeaderSchema( return Optional.of(new TableSchema(tableSchema.getTableName(), resultColumnList)); } - private void autoCreateTable(String database, TableSchema tableSchema) { - TsTable tsTable = new TsTable(tableSchema.getTableName()); + private void autoCreateTable(final String database, final TableSchema tableSchema) { + final TsTable tsTable = new TsTable(tableSchema.getTableName()); addColumnSchema(tableSchema.getColumns(), tsTable); - CreateTableTask createTableTask = new CreateTableTask(tsTable, database, true); + final CreateTableTask createTableTask = new CreateTableTask(tsTable, database, true); try { - ListenableFuture future = createTableTask.execute(configTaskExecutor); - ConfigTaskResult result = future.get(); + final ListenableFuture future = createTableTask.execute(configTaskExecutor); + final ConfigTaskResult result = future.get(); if (result.getStatusCode().getStatusCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new RuntimeException( new IoTDBException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java index cdd8f01ab12c..9a2c9cb55e86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java @@ -28,12 +28,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; import com.google.common.collect.ImmutableList; -import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Pair; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Optional; @@ -58,14 +56,10 @@ public static Pair split( for (Map.Entry entry : node.getAggregations().entrySet()) { AggregationNode.Aggregation originalAggregation = entry.getValue(); ResolvedFunction resolvedFunction = originalAggregation.getResolvedFunction(); - List intermediateTypes = - TableBuiltinAggregationFunction.getIntermediateTypes( + Type intermediateType = + TableBuiltinAggregationFunction.getIntermediateType( resolvedFunction.getSignature().getName(), resolvedFunction.getSignature().getArgumentTypes()); - Type intermediateType = - intermediateTypes.size() == 1 - ? intermediateTypes.get(0) - : RowType.anonymous(intermediateTypes); Symbol intermediateSymbol = symbolAllocator.newSymbol(resolvedFunction.getSignature().getName(), intermediateType); // TODO put symbol and its type to TypeProvide or later process: add all map contents of @@ -128,14 +122,10 @@ public static Pair split( for (Map.Entry entry : node.getAggregations().entrySet()) { AggregationNode.Aggregation originalAggregation = entry.getValue(); ResolvedFunction resolvedFunction = originalAggregation.getResolvedFunction(); - List intermediateTypes = - TableBuiltinAggregationFunction.getIntermediateTypes( + Type intermediateType = + TableBuiltinAggregationFunction.getIntermediateType( resolvedFunction.getSignature().getName(), resolvedFunction.getSignature().getArgumentTypes()); - Type intermediateType = - intermediateTypes.size() == 1 - ? intermediateTypes.get(0) - : RowType.anonymous(intermediateTypes); Symbol intermediateSymbol = symbolAllocator.newSymbol(resolvedFunction.getSignature().getName(), intermediateType); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowDB.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowDB.java index 8b53b5641f34..b8dd93e35627 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowDB.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowDB.java @@ -26,16 +26,20 @@ import static java.util.Objects.requireNonNull; public class ShowDB extends Statement { - public ShowDB() { - super(null); - } - public ShowDB(NodeLocation location) { + private final boolean isDetails; + + public ShowDB(final NodeLocation location, final boolean isDetails) { super(requireNonNull(location, "location is null")); + this.isDetails = isDetails; + } + + public boolean isDetails() { + return isDetails; } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitShowDB(this, context); } @@ -50,7 +54,7 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 42adb9e49ea9..282ef4554dc4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -257,8 +257,9 @@ private static Identifier lowerIdentifier(Identifier identifier) { } @Override - public Node visitShowDatabasesStatement(RelationalSqlParser.ShowDatabasesStatementContext ctx) { - return new ShowDB(getLocation(ctx)); + public Node visitShowDatabasesStatement( + final RelationalSqlParser.ShowDatabasesStatementContext ctx) { + return new ShowDB(getLocation(ctx), Objects.nonNull(ctx.DETAILS())); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java index 356336bf5ad7..4a50bb0e1fc0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; -import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; @@ -74,10 +73,10 @@ public void setDetailed(boolean detailed) { } public void buildTSBlock( - Map storageGroupInfoMap, SettableFuture future) - throws IllegalPathException { + final Map storageGroupInfoMap, + final SettableFuture future) { - List outputDataTypes = + final List outputDataTypes = isDetailed ? ColumnHeaderConstant.showStorageGroupsDetailColumnHeaders.stream() .map(ColumnHeader::getColumnType) @@ -86,10 +85,10 @@ public void buildTSBlock( .map(ColumnHeader::getColumnType) .collect(Collectors.toList()); - TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); - for (Map.Entry entry : storageGroupInfoMap.entrySet()) { - String storageGroup = entry.getKey(); - TDatabaseInfo storageGroupInfo = entry.getValue(); + final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + for (final Map.Entry entry : storageGroupInfoMap.entrySet()) { + final String storageGroup = entry.getKey(); + final TDatabaseInfo storageGroupInfo = entry.getValue(); builder.getTimeColumnBuilder().writeLong(0L); builder @@ -106,11 +105,17 @@ public void buildTSBlock( builder.getColumnBuilder(8).writeInt(storageGroupInfo.getDataRegionNum()); builder.getColumnBuilder(9).writeInt(storageGroupInfo.getMinDataRegionNum()); builder.getColumnBuilder(10).writeInt(storageGroupInfo.getMaxDataRegionNum()); + builder + .getColumnBuilder(11) + .writeBinary( + new Binary( + storageGroupInfo.isIsTableModel() ? "TABLE" : "TREE", + TSFileConfig.STRING_CHARSET)); } builder.declarePosition(); } - DatasetHeader datasetHeader = DatasetHeaderFactory.getShowStorageGroupHeader(isDetailed); + final DatasetHeader datasetHeader = DatasetHeaderFactory.getShowStorageGroupHeader(isDetailed); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 9b8ed5ddbdf1..4ada1b6b95e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -369,12 +369,14 @@ private Pair checkTableExistenceOnGivenPath( for (final Map.Entry> dbEntry : tableMap.entrySet()) { final String database = dbEntry.getKey(); if (!(path.startsWith(database, dbStartIndex) + && path.length() > dbStartIndex + database.length() && path.charAt(dbStartIndex + database.length()) == PATH_SEPARATOR)) { continue; } final int tableStartIndex = dbStartIndex + database.length() + 1; for (final String tableName : dbEntry.getValue().keySet()) { if (path.startsWith(tableName, tableStartIndex) + && path.length() > tableStartIndex + tableName.length() && path.charAt(tableStartIndex + tableName.length()) == PATH_SEPARATOR) { return new Pair<>(database, tableName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java index 2b6478f79596..e6d37e49b9c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/IoTDBInternalLocalReporter.java @@ -103,12 +103,7 @@ public IoTDBInternalLocalReporter() { && showDatabaseResp.getDatabaseInfoMapSize() == 0) { TDatabaseSchema databaseSchema = new TDatabaseSchema(); databaseSchema.setName(SchemaConstant.SYSTEM_DATABASE); - databaseSchema.setSchemaReplicationFactor(1); - databaseSchema.setDataReplicationFactor(1); - databaseSchema.setMaxSchemaRegionGroupNum(1); - databaseSchema.setMinSchemaRegionGroupNum(1); - databaseSchema.setMaxDataRegionGroupNum(1); - databaseSchema.setMinDataRegionGroupNum(1); + databaseSchema.setIsTableModel(false); TSStatus tsStatus = client.setDatabase(databaseSchema); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { LOGGER.error("IoTDBSessionReporter checkOrCreateDatabase failed."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java index 3b30e798227b..1ba5ff680ec0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/BatchCompactionPlan.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.batch.utils; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; import org.apache.tsfile.file.metadata.ChunkMetadata; @@ -35,7 +36,7 @@ import java.util.Map; public class BatchCompactionPlan { - public static final long MAX_CACHED_TIME_CHUNKS_SIZE = 2 * 1024 * 1024; + public static long maxCachedTimeChunksSize = 2 * 1024 * 1024; private final List compactChunkPlans = new ArrayList<>(); private final Map> alignedPageModifiedStatusCache = new HashMap<>(); @@ -51,12 +52,12 @@ public Chunk getTimeChunkFromCache(TsFileSequenceReader reader, ChunkMetadata ch if (chunk == null) { chunk = reader.readMemChunk(chunkMetadata); } - chunk.getData().flip(); + chunk.getData().rewind(); return chunk; } public void addTimeChunkToCache(String file, long offset, Chunk chunk) { - if (cachedTimeChunkSize >= MAX_CACHED_TIME_CHUNKS_SIZE) { + if (cachedTimeChunkSize >= maxCachedTimeChunksSize) { return; } cachedTimeChunks.put( @@ -97,6 +98,16 @@ public boolean isEmpty() { return compactChunkPlans.isEmpty(); } + @TestOnly + public static void setMaxCachedTimeChunksSize(long size) { + maxCachedTimeChunksSize = size; + } + + @TestOnly + public static long getMaxCachedTimeChunksSize() { + return maxCachedTimeChunksSize; + } + @Override public String toString() { return compactChunkPlans.toString(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java index 43dd0959303f..4ec74c57d63e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java @@ -68,7 +68,7 @@ public abstract class AbstractCompactionEstimator { ((double) SystemInfo.getInstance().getMemorySizeForCompaction() / IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount() * IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion()) - + BatchCompactionPlan.MAX_CACHED_TIME_CHUNKS_SIZE; + + BatchCompactionPlan.maxCachedTimeChunksSize; protected abstract long calculatingMetadataMemoryCost(CompactionTaskInfo taskInfo); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java index c198c3971b45..725860eb70d2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/PartitionCacheTest.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.DatabaseModelException; import org.apache.iotdb.db.queryengine.plan.analyze.cache.partition.PartitionCache; import org.apache.tsfile.file.metadata.IDeviceID; @@ -48,6 +49,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -148,7 +151,8 @@ private static String getDeviceName(String storageGroupName, int deviceNumber) { @Before public void setUp() throws Exception { partitionCache = new PartitionCache(); - partitionCache.updateDatabaseCache(storageGroups); + partitionCache.updateDatabaseCache( + storageGroups.stream().collect(Collectors.toMap(Function.identity(), k -> false))); partitionCache.updateSchemaPartitionCache(schemaPartitionTable); partitionCache.updateDataPartitionCache(dataPartitionTable); partitionCache.updateGroupIdToReplicaSetMap(100, consensusGroupIdToRegionReplicaSet); @@ -160,7 +164,7 @@ public void tearDown() throws Exception { } @Test - public void testStorageGroupCache() { + public void testStorageGroupCache() throws DatabaseModelException { Map> storageGroupToDeviceMap; Map deviceToStorageGroupMap; // test devices in one database diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/BatchedAlignedSeriesFastInnerCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/BatchedAlignedSeriesFastInnerCompactionTest.java index c4f95addd282..1b4d693d6b97 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/BatchedAlignedSeriesFastInnerCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/BatchedAlignedSeriesFastInnerCompactionTest.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.batch.utils.BatchCompactionPlan; import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -319,6 +320,42 @@ public void testCompactionByDeserialize() throws Exception { validate(targetResource); } + @Test + public void testCompactionByDeserializeWithLargeTimeChunk() throws Exception { + long defaultMaxCachedTimeChunkSize = BatchCompactionPlan.getMaxCachedTimeChunksSize(); + try { + BatchCompactionPlan.setMaxCachedTimeChunksSize(1); + TsFileResource unseqResource1 = + generateSingleAlignedSeriesFile( + "d0", + Arrays.asList("s0", "s1", "s2"), + new TimeRange[][] { + new TimeRange[] {new TimeRange(100, 200), new TimeRange(500, 600)} + }, + TSEncoding.PLAIN, + CompressionType.LZ4, + Arrays.asList(false, false, false), + false); + unseqResources.add(unseqResource1); + + TsFileResource unseqResource2 = + generateSingleAlignedSeriesFile( + "d0", + Arrays.asList("s0", "s1", "s2"), + new TimeRange[] {new TimeRange(150, 550)}, + TSEncoding.PLAIN, + CompressionType.LZ4, + Arrays.asList(false, false, false), + false); + unseqResources.add(unseqResource2); + + TsFileResource targetResource = performCompaction(); + validate(targetResource); + } finally { + BatchCompactionPlan.setMaxCachedTimeChunksSize(defaultMaxCachedTimeChunkSize); + } + } + @Test public void testCompactionByDeserializeWithEmptyColumn() throws Exception { TsFileResource unseqResource1 = diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 30f57e994c67..6acfcf016576 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -125,7 +125,7 @@ useDatabaseStatement ; showDatabasesStatement - : SHOW DATABASES + : SHOW DATABASES (DETAILS)? ; createDbStatement diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index ffe7859c7bb4..8f8727fbf0a0 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -210,6 +210,7 @@ struct TDatabaseSchema { 8: optional i32 minDataRegionGroupNum 9: optional i32 maxDataRegionGroupNum 10: optional i64 timePartitionOrigin + 11: optional bool isTableModel } // Schema @@ -595,6 +596,7 @@ struct TDatabaseInfo { 10: required i32 minDataRegionNum 11: required i32 maxDataRegionNum 12: optional i64 timePartitionOrigin + 13: optional bool isTableModel } struct TGetDatabaseReq {