Skip to content

Commit

Permalink
Merge branch 'master' into sub-meta-improve
Browse files Browse the repository at this point in the history
  • Loading branch information
@VGalaxies
VGalaxies committed Oct 25, 2024
2 parents ac1304c + 6fcc15a commit 7880baa
Show file tree
Hide file tree
Showing 53 changed files with 1,750 additions and 533 deletions.
20 changes: 9 additions & 11 deletions integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1041,25 +1041,23 @@ public void testInvalidMaxPointNumber() {

@Test
public void testStorageGroupWithHyphenInName() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
try (final Connection connection = EnvFactory.getEnv().getConnection();
final Statement statement = connection.createStatement()) {
statement.setFetchSize(5);
statement.execute("CREATE DATABASE root.group_with_hyphen");
} catch (SQLException e) {
} catch (final SQLException e) {
fail();
}

try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
try (ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
try (final Connection connection = EnvFactory.getEnv().getConnection();
final Statement statement = connection.createStatement()) {
try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) {
while (resultSet.next()) {
StringBuilder builder = new StringBuilder();
builder.append(resultSet.getString(1));
Assert.assertEquals(builder.toString(), "root.group_with_hyphen");
Assert.assertEquals("root.group_with_hyphen", resultSet.getString(1));
Assert.assertEquals("TREE", resultSet.getString(12));
}
}
} catch (SQLException e) {
} catch (final SQLException e) {
fail();
}
}
Expand Down
78 changes: 76 additions & 2 deletions ...rc/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public class IoTDBMultiIDsWithAttributesTableIT {
"CREATE DATABASE db",
"USE db",
"CREATE TABLE table0 (device string id, level string id, attr1 string attribute, attr2 string attribute, num int32 measurement, bigNum int64 measurement, "
+ "floatNum double measurement, str TEXT measurement, bool BOOLEAN measurement, date DATE measurement, blob BLOB measurement, ts TIMESTAMP measurement, stringV STRING measurement, doubleNum DOUBLE measurement)",
+ "floatNum FLOAT measurement, str TEXT measurement, bool BOOLEAN measurement, date DATE measurement, blob BLOB measurement, ts TIMESTAMP measurement, stringV STRING measurement, doubleNum DOUBLE measurement)",
"insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool) values('d1', 'l1', 'c', 'd', 0,3,2947483648,231.2121,'coconut',FALSE)",
"insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool,blob,ts,doubleNum) values('d1', 'l2', 'y', 'z', 20,2,2147483648,434.12,'pineapple',TRUE,X'108DCD62',2024-09-24T06:15:35.000+00:00,6666.8)",
"insert into table0(device, level, attr1, attr2, time,num,bigNum,floatNum,str,bool) values('d1', 'l3', 't', 'a', 40,1,2247483648,12.123,'apricot',TRUE)",
Expand Down Expand Up @@ -1251,7 +1251,44 @@ public void modeTest() {
"select mode(device),mode(level),mode(attr1),mode(attr2),mode(date),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum) from table0 where device='d2' and level='l1'";
retArray =
new String[] {
"d2,l1,d,c,null,false,null,2024-08-01T06:15:35.000Z,test-string3,null,",
"d2,l1,d,c,null,false,null,null,null,null,",
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

expectedHeader = buildHeaders(1);
sql =
"select mode(stringv) from table0 where device='d2' and level='l1' and stringv is not null";
retArray =
new String[] {
"test-string3,",
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

// no push-down, test GroupedAccumulator
expectedHeader = buildHeaders(16);
sql =
"select mode(time),mode(device),mode(level),mode(attr1),mode(attr2),mode(num),mode(bignum),mode(floatnum),mode(date),mode(str),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum),count(num+1) from table0 where device='d2' and level='l4' and time=80 group by device, level";
retArray =
new String[] {
"1970-01-01T00:00:00.080Z,d2,l4,null,null,9,2147483646,43.12,null,apple,false,null,2024-09-20T06:15:35.000Z,test-string2,6666.7,1,",
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

expectedHeader = buildHeaders(11);
sql =
"select mode(device),mode(level),mode(attr1),mode(attr2),mode(date),mode(bool),mode(date),mode(ts),mode(stringv),mode(doublenum),count(num+1) from table0 where device='d2' and level='l1' group by device, level";
retArray =
new String[] {
"d2,l1,d,c,null,false,null,null,null,null,3,",
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

expectedHeader = buildHeaders(2);
sql =
"select mode(stringv),count(num+1) from table0 where device='d2' and level='l1' and stringv is not null group by device, level";
retArray =
new String[] {
"test-string3,1,",
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
}
Expand All @@ -1269,6 +1306,43 @@ public void varianceTest() {
"16.0,10.7,16.0,4.0,3.3,4.0,50.0,33.3,50.0,7.1,5.8,7.1,null,0.0,null,null,0.0,null,",
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

expectedHeader = buildHeaders(19);
sql =
"select \n"
+ "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n"
+ "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n"
+ "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1), count(num+1) from table0 where device='d2' and level='l4' group by device, level";
retArray =
new String[] {
"16.0,10.7,16.0,4.0,3.3,4.0,50.0,33.3,50.0,7.1,5.8,7.1,null,0.0,null,null,0.0,null,3,",
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

sql =
"select \n"
+ "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n"
+ "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n"
+ "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1), count(num+1) from table0 group by device";
retArray =
new String[] {
"20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.1,0.1,0.1,0.4,0.2,0.4,15,",
"20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.0,0.0,0.0,0.2,0.2,0.2,15,"
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);

expectedHeader = buildHeaders(18);
sql =
"select \n"
+ "round(variance(num),1),round(var_pop(num),1),round(var_samp(num),1),round(stddev(num),1),round(stddev_pop(num),1),round(stddev_samp(num),1),\n"
+ "round(variance(floatnum),1),round(var_pop(floatnum),1),round(var_samp(floatnum),1),round(stddev(floatnum),1),round(stddev_pop(floatnum),1),round(stddev_samp(floatnum),1),\n"
+ "round(variance(doublenum),1),round(var_pop(doublenum),1),round(var_samp(doublenum),1),round(stddev(doublenum),1),round(stddev_pop(doublenum),1),round(stddev_samp(doublenum),1) from table0 group by device";
retArray =
new String[] {
"20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.1,0.1,0.1,0.4,0.2,0.4,",
"20.0,18.7,20.0,4.5,4.3,4.5,1391642.5,1298866.4,1391642.5,1179.7,1139.7,1179.7,0.0,0.0,0.0,0.2,0.2,0.2,"
};
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
}

// ==================================================================
Expand Down
17 changes: 17 additions & 0 deletions ...st/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData;
import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
import static org.apache.iotdb.relational.it.db.it.IoTDBMultiIDsWithAttributesTableIT.buildHeaders;

@RunWith(IoTDBTestRunner.class)
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
Expand Down Expand Up @@ -3608,4 +3609,20 @@ public void specialCasesTest() {
retArray,
DATABASE_NAME);
}

@Test
public void modeTest() {
// AggTableScan + Agg mixed test
String[] expectedHeader = buildHeaders(11);
String[] retArray =
new String[] {
"A,null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,",
"A,null,null,null,null,null,null,null,null,2024-09-24T06:15:40.000Z,null,",
};
tableResultSetEqualTest(
"select mode(type), mode(s1),mode(s2),mode(s3),mode(s4),mode(s5),mode(s6),mode(s7),mode(s8),mode(s9),mode(s10) from table1 group by city",
expectedHeader,
retArray,
DATABASE_NAME);
}
}
22 changes: 22 additions & 0 deletions integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.util.Collections;

import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showDBColumnHeaders;
import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showDBDetailsColumnHeaders;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
Expand Down Expand Up @@ -83,6 +84,7 @@ public void testManageDatabase() {
int[] schemaReplicaFactors = new int[] {1};
int[] dataReplicaFactors = new int[] {1};
int[] timePartitionInterval = new int[] {604800000};
String[] model = new String[] {"TABLE"};

// show
try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES")) {
Expand All @@ -102,6 +104,26 @@ public void testManageDatabase() {
assertEquals(databaseNames.length, cnt);
}

// show
try (final ResultSet resultSet = statement.executeQuery("SHOW DATABASES DETAILS")) {
int cnt = 0;
final ResultSetMetaData metaData = resultSet.getMetaData();
assertEquals(showDBDetailsColumnHeaders.size(), metaData.getColumnCount());
for (int i = 0; i < showDBDetailsColumnHeaders.size(); i++) {
assertEquals(
showDBDetailsColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1));
}
while (resultSet.next()) {
assertEquals(databaseNames[cnt], resultSet.getString(1));
assertEquals(schemaReplicaFactors[cnt], resultSet.getInt(2));
assertEquals(dataReplicaFactors[cnt], resultSet.getInt(3));
assertEquals(timePartitionInterval[cnt], resultSet.getLong(4));
assertEquals(model[cnt], resultSet.getString(5));
cnt++;
}
assertEquals(databaseNames.length, cnt);
}

// use
statement.execute("use test");

Expand Down
1 change: 1 addition & 0 deletions iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ public enum TSStatusCode {
MEASUREMENT_ALREADY_EXISTS_IN_TEMPLATE(527),
TYPE_NOT_FOUND(528),
DATABASE_CONFLICT(529),
DATABASE_MODEL(530),

TABLE_NOT_EXISTS(550),
TABLE_ALREADY_EXISTS(551),
Expand Down
40 changes: 21 additions & 19 deletions iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -726,8 +726,8 @@ public TSStatus setDatabase(final DatabaseSchemaPlan databaseSchemaPlan) {
}

@Override
public TSStatus alterDatabase(DatabaseSchemaPlan databaseSchemaPlan) {
TSStatus status = confirmLeader();
public TSStatus alterDatabase(final DatabaseSchemaPlan databaseSchemaPlan) {
final TSStatus status = confirmLeader();
if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
return clusterSchemaManager.alterDatabase(databaseSchemaPlan, false);
} else {
Expand All @@ -736,19 +736,19 @@ public TSStatus alterDatabase(DatabaseSchemaPlan databaseSchemaPlan) {
}

@Override
public synchronized TSStatus deleteDatabases(TDeleteDatabasesReq tDeleteReq) {
TSStatus status = confirmLeader();
public synchronized TSStatus deleteDatabases(final TDeleteDatabasesReq tDeleteReq) {
final TSStatus status = confirmLeader();
if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
List<String> deletedPaths = tDeleteReq.getPrefixPathList();
final List<String> deletedPaths = tDeleteReq.getPrefixPathList();
// remove wild
Map<String, TDatabaseSchema> deleteDatabaseSchemaMap =
final Map<String, TDatabaseSchema> deleteDatabaseSchemaMap =
getClusterSchemaManager().getMatchedDatabaseSchemasByName(deletedPaths);
if (deleteDatabaseSchemaMap.isEmpty()) {
return RpcUtils.getStatus(
TSStatusCode.PATH_NOT_EXIST.getStatusCode(),
String.format("Path %s does not exist", Arrays.toString(deletedPaths.toArray())));
}
ArrayList<TDatabaseSchema> parsedDeleteDatabases =
final ArrayList<TDatabaseSchema> parsedDeleteDatabases =
new ArrayList<>(deleteDatabaseSchemaMap.values());
return procedureManager.deleteDatabases(
parsedDeleteDatabases,
Expand Down Expand Up @@ -1834,14 +1834,15 @@ public TShowConfigNodesResp showConfigNodes() {
}

@Override
public TShowDatabaseResp showDatabase(TGetDatabaseReq req) {
TSStatus status = confirmLeader();
public TShowDatabaseResp showDatabase(final TGetDatabaseReq req) {
final TSStatus status = confirmLeader();
if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
PathPatternTree scope =
final PathPatternTree scope =
req.getScopePatternTree() == null
? SchemaConstant.ALL_MATCH_SCOPE
: PathPatternTree.deserialize(ByteBuffer.wrap(req.getScopePatternTree()));
GetDatabasePlan getDatabasePlan = new GetDatabasePlan(req.getDatabasePathPattern(), scope);
final GetDatabasePlan getDatabasePlan =
new GetDatabasePlan(req.getDatabasePathPattern(), scope);
return getClusterSchemaManager().showDatabase(getDatabasePlan);
} else {
return new TShowDatabaseResp().setStatus(status);
Expand Down Expand Up @@ -1943,17 +1944,18 @@ public TGetPathsSetTemplatesResp getPathsSetTemplate(TGetPathsSetTemplatesReq re
}

@Override
public TSStatus deactivateSchemaTemplate(TDeactivateSchemaTemplateReq req) {
TSStatus status = confirmLeader();
public TSStatus deactivateSchemaTemplate(final TDeactivateSchemaTemplateReq req) {
final TSStatus status = confirmLeader();
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
return status;
}

PathPatternTree patternTree =
final PathPatternTree patternTree =
PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree()));

List<PartialPath> patternList = patternTree.getAllPathPatterns();
TemplateSetInfoResp templateSetInfoResp = clusterSchemaManager.getTemplateSetInfo(patternList);
final List<PartialPath> patternList = patternTree.getAllPathPatterns();
final TemplateSetInfoResp templateSetInfoResp =
clusterSchemaManager.getTemplateSetInfo(patternList);
if (templateSetInfoResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
return templateSetInfoResp.getStatus();
}
Expand All @@ -1968,9 +1970,9 @@ public TSStatus deactivateSchemaTemplate(TDeactivateSchemaTemplateReq req) {
}

if (!req.getTemplateName().equals(ONE_LEVEL_PATH_WILDCARD)) {
Map<PartialPath, List<Template>> filteredTemplateSetInfo = new HashMap<>();
for (Map.Entry<PartialPath, List<Template>> entry : templateSetInfo.entrySet()) {
for (Template template : entry.getValue()) {
final Map<PartialPath, List<Template>> filteredTemplateSetInfo = new HashMap<>();
for (final Map.Entry<PartialPath, List<Template>> entry : templateSetInfo.entrySet()) {
for (final Template template : entry.getValue()) {
if (template.getName().equals(req.getTemplateName())) {
filteredTemplateSetInfo.put(entry.getKey(), Collections.singletonList(template));
break;
Expand Down
2 changes: 1 addition & 1 deletion ...a/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ private TSStatus executePlan(final ConfigPhysicalPlan plan) throws ConsensusExce
case CreateDatabase:
// Here we only reserve database name and substitute the sender's local information
// with the receiver's default configurations
TDatabaseSchema schema = ((DatabaseSchemaPlan) plan).getSchema();
final TDatabaseSchema schema = ((DatabaseSchemaPlan) plan).getSchema();
schema.setSchemaReplicationFactor(
ConfigNodeDescriptor.getInstance().getConf().getSchemaReplicationFactor());
schema.setDataReplicationFactor(
Expand Down
15 changes: 8 additions & 7 deletions ...ignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,11 +336,11 @@ public DatabaseSchemaResp getMatchedDatabaseSchema(GetDatabasePlan getDatabasePl
}

/** Only used in cluster tool show Databases. */
public TShowDatabaseResp showDatabase(GetDatabasePlan getDatabasePlan) {
public TShowDatabaseResp showDatabase(final GetDatabasePlan getDatabasePlan) {
DatabaseSchemaResp databaseSchemaResp;
try {
databaseSchemaResp = (DatabaseSchemaResp) getConsensusManager().read(getDatabasePlan);
} catch (ConsensusException e) {
} catch (final ConsensusException e) {
LOGGER.warn(CONSENSUS_READ_ERROR, e);
TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
res.setMessage(e.getMessage());
Expand All @@ -352,10 +352,10 @@ public TShowDatabaseResp showDatabase(GetDatabasePlan getDatabasePlan) {
return new TShowDatabaseResp().setStatus(databaseSchemaResp.getStatus());
}

Map<String, TDatabaseInfo> infoMap = new ConcurrentHashMap<>();
for (TDatabaseSchema databaseSchema : databaseSchemaResp.getSchemaMap().values()) {
String database = databaseSchema.getName();
TDatabaseInfo databaseInfo = new TDatabaseInfo();
final Map<String, TDatabaseInfo> infoMap = new ConcurrentHashMap<>();
for (final TDatabaseSchema databaseSchema : databaseSchemaResp.getSchemaMap().values()) {
final String database = databaseSchema.getName();
final TDatabaseInfo databaseInfo = new TDatabaseInfo();
databaseInfo.setName(database);
databaseInfo.setSchemaReplicationFactor(databaseSchema.getSchemaReplicationFactor());
databaseInfo.setDataReplicationFactor(databaseSchema.getDataReplicationFactor());
Expand All @@ -369,13 +369,14 @@ public TShowDatabaseResp showDatabase(GetDatabasePlan getDatabasePlan) {
getMinRegionGroupNum(database, TConsensusGroupType.DataRegion));
databaseInfo.setMaxDataRegionNum(
getMaxRegionGroupNum(database, TConsensusGroupType.DataRegion));
databaseInfo.setIsTableModel(databaseSchema.isIsTableModel());

try {
databaseInfo.setSchemaRegionNum(
getPartitionManager().getRegionGroupCount(database, TConsensusGroupType.SchemaRegion));
databaseInfo.setDataRegionNum(
getPartitionManager().getRegionGroupCount(database, TConsensusGroupType.DataRegion));
} catch (DatabaseNotExistsException e) {
} catch (final DatabaseNotExistsException e) {
// Skip pre-deleted Database
LOGGER.warn(
"The Database: {} doesn't exist. Maybe it has been pre-deleted.",
Expand Down
Loading

0 comments on commit 7880baa

Please sign in to comment.