From a5f350c2f2c2b8bd4ab1267df75cafa7b5ad15a9 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 12 Sep 2024 21:21:19 +0800 Subject: [PATCH 01/34] reduce number of session --- .../iot/benchmark/conf/Constants.java | 4 +- .../iot/benchmark/schema/MetaUtil.java | 25 +++++++++++ .../schemaImpl/GenerateMetaDataSchema.java | 14 ++++++- .../iot/benchmark/tsdb/DBFactory.java | 4 +- .../iotdb200/DMLStrategy/DMLStrategy.java | 5 ++- .../iotdb200/DMLStrategy/JDBCStrategy.java | 5 +++ .../iotdb200/DMLStrategy/SessionStrategy.java | 42 ++++++++++++++++++- .../iot/benchmark/iotdb200/IoTDB.java | 4 ++ .../iotdb200/ModelStrategy/TableStrategy.java | 2 - 9 files changed, 96 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java index 5dbe177cb..4cf16c75b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java @@ -55,9 +55,11 @@ public class Constants { public static final String IOTDB200_REST_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb200.IoTDBRestAPI"; - public static final String IOTDB130 = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB"; public static final String IOTDB130_REST_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBRestAPI"; + public static final String IOTDB130_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB"; + public static final String IOTDB130_SESSION_CLASS = + "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBSession"; public static final String IOTDB110_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb110.IoTDB"; public static final String IOTDB110_SESSION_CLASS = diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 452f7245d..3d62e7eb0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -4,7 +4,9 @@ import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import cn.edu.tsinghua.iot.benchmark.conf.Constants; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; +import org.slf4j.Logger; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -54,6 +56,29 @@ public static int mappingId(int objectId, int objectNumber, int allocatingObject } } + /** 重排 deviceId,保证相邻的 deviceId 属于同一个 database */ + public static List sortDeviceIdByTable(Config config, Logger LOGGER) { + List deviceIds = new ArrayList<>(); + // 获取 Map database device 的映射 + Map> databaseDeviceMap = new HashMap<>(); + try { + for (int deviceId = 0; deviceId < config.getDEVICE_NUMBER(); deviceId++) { + int tableId = + mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + int databaseId = + mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); + databaseDeviceMap.computeIfAbsent(databaseId, k -> new ArrayList<>()).add(deviceId); + } + } catch (WorkloadException e) { + LOGGER.error(e.getMessage()); + } + // 重排 deviceId,保证相邻的 deviceId 属于同一个 database + for (List values : databaseDeviceMap.values()) { + deviceIds.addAll(values); + } + return deviceIds; + } + public static String getGroupIdFromDeviceName(String deviceName) { int groupId = deviceName.hashCode(); if (groupId < 0) { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index 59d6b4cb4..e1f97ef8d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -22,6 +22,7 @@ import cn.edu.tsinghua.iot.benchmark.conf.Config; import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.schema.MetaDataSchema; import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.utils.CommonAlgorithms; @@ -45,15 +46,24 @@ public boolean createMetaDataSchema() { return false; } - int deviceId = MetaUtil.getDeviceId(0); Map deviceDistribution = CommonAlgorithms.distributeDevicesToClients( config.getDEVICE_NUMBER(), config.getCLIENT_NUMBER()); + int deviceId = MetaUtil.getDeviceId(0); + // 重排 device ID 使相邻的 device 处在同一张表中 + List deviceIds = MetaUtil.sortDeviceIdByTable(config, LOGGER); for (int clientId = 0; clientId < config.getCLIENT_NUMBER(); clientId++) { int deviceNumber = deviceDistribution.get(clientId); List deviceSchemaList = new ArrayList<>(); for (int d = 0; d < deviceNumber; d++) { - DeviceSchema deviceSchema = new DeviceSchema(deviceId, sensors, MetaUtil.getTags(deviceId)); + DeviceSchema deviceSchema; + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { + deviceSchema = + new DeviceSchema( + deviceIds.get(deviceId), sensors, MetaUtil.getTags(deviceIds.get(deviceId))); + } else { + deviceSchema = new DeviceSchema(deviceId, sensors, MetaUtil.getTags(deviceId)); + } NAME_DATA_SCHEMA.put(deviceSchema.getDevice(), deviceSchema); GROUPS.add(deviceSchema.getGroup()); deviceSchemaList.add(deviceSchema); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java index 44a4e7e25..439c56626 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java @@ -54,10 +54,12 @@ public IDatabase getDatabase(DBConfig dbConfig) throws SQLException { dbClass = Constants.IOTDB130_REST_CLASS; break; case DB_IOT_130_JDBC: + dbClass = Constants.IOTDB130_JDBC_CLASS; + break; case DB_IOT_130_SESSION_BY_TABLET: case DB_IOT_130_SESSION_BY_RECORD: case DB_IOT_130_SESSION_BY_RECORDS: - dbClass = Constants.IOTDB130; + dbClass = Constants.IOTDB130_SESSION_CLASS; break; // IoTDB 1.1 diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java index 59e1192cf..f74e4c261 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java @@ -64,13 +64,15 @@ public abstract DeviceSummary deviceSummary( String device, String totalLineNumberSql, String maxTimestampSql, String minTimestampSql) throws TsdbException, SQLException; + public abstract void switchSession(int deviceId, String group); + public abstract void init() throws TsdbException; public abstract void cleanup(); public abstract void close() throws TsdbException; - protected Session buildSession(List hostUrls) { + protected Session buildSession(List hostUrls, String databaseName) { return new Session.Builder() .nodeUrls(hostUrls) .username(dbConfig.getUSERNAME()) @@ -78,6 +80,7 @@ protected Session buildSession(List hostUrls) { .enableRedirection(true) .version(Version.V_1_0) .sqlDialect(config.getIoTDB_DIALECT_MODE().name()) + .database(databaseName) .build(); } } diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java index 4682097e4..c830acb32 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java @@ -194,6 +194,11 @@ public DeviceSummary deviceSummary( return new DeviceSummary(device, totalLineNumber, minTimeStamp, maxTimeStamp); } + @Override + public void switchSession(int deviceId, String group) { + // nothing + } + @Override public void init() throws TsdbException { if (ioTDBConnection == null) { diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index c210e72bd..a887ac2f3 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -31,11 +31,14 @@ import cn.edu.tsinghua.iot.benchmark.entity.DeviceSummary; import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.OperationFailException; +import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; import cn.edu.tsinghua.iot.benchmark.iotdb200.IoTDB; import cn.edu.tsinghua.iot.benchmark.iotdb200.utils.IoTDBUtils; import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; import cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode; @@ -55,6 +58,7 @@ import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -74,7 +78,8 @@ public class SessionStrategy extends DMLStrategy { private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE()); - private final Session session; + private final Map databaseSessionMap = new HashMap<>(); + private Session session; private final IoTDB iotdb; public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { @@ -84,7 +89,7 @@ public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { for (int i = 0; i < dbConfig.getHOST().size(); i++) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } - session = buildSession(hostUrls); + session = buildSession(hostUrls, null); } @Override @@ -110,6 +115,10 @@ private Status insertOneBatchByTablet(IBatch batch) { service.submit( () -> { try { + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { + switchSession( + batch.getDeviceSchema().getDeviceId(), batch.getDeviceSchema().getGroup()); + } iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); @@ -118,6 +127,30 @@ private Status insertOneBatchByTablet(IBatch batch) { return waitWriteTaskToFinishAndGetStatus(); } + @Override + public void switchSession(int deviceId, String group) { + try { + int tableId = + MetaUtil.mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + int databaseId = + MetaUtil.mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); + if (databaseSessionMap.get(databaseId) == null) { + List hostUrls = new ArrayList<>(dbConfig.getHOST().size()); + for (int i = 0; i < dbConfig.getHOST().size(); i++) { + hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); + } + Session sessionNew = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); + sessionNew.open(); + session = sessionNew; + databaseSessionMap.put(databaseId, session); + } else { + session = databaseSessionMap.get(databaseId); + } + } catch (WorkloadException | IoTDBConnectionException e) { + LOGGER.error(e.getMessage(), e); + } + } + private Tablet genTablet(IBatch batch) { List schemaList = new ArrayList<>(); List columnTypes = new ArrayList<>(); @@ -475,6 +508,11 @@ public void close() throws TsdbException { if (session != null) { session.close(); } + if (!databaseSessionMap.isEmpty()) { + for (Session session : databaseSessionMap.values()) { + session.close(); + } + } service.shutdown(); } catch (IoTDBConnectionException ioTDBConnectionException) { LOGGER.error("Failed to close session."); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index 38181f85b..ee25b0814 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -672,6 +672,10 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic modelStrategy.sessionInsertImpl(session, tablet, deviceSchema); } + public void switchSessionImpl(int deviceId, String group) { + dmlStrategy.switchSession(deviceId, group); + } + public void addIDColumnIfNecessary( List columnTypes, List sensors, IBatch batch) { modelStrategy.addIDColumnIfNecessary(columnTypes, sensors, batch); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index 5effa15d1..4b20a688b 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -333,8 +333,6 @@ public void deleteIDColumnIfNecessary( @Override public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) throws IoTDBConnectionException, StatementExecutionException { - session.executeNonQueryStatement( - "use " + dbConfig.getDB_NAME() + "_" + deviceSchema.getGroup()); session.insertRelationalTablet(tablet); } From 83d927b78a59673a776fdff36893912c374f93de Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Fri, 13 Sep 2024 21:41:21 +0800 Subject: [PATCH 02/34] The deleteIDColumnIfNecessary method is executed only in dual write mode --- .../java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java | 2 +- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java index 9ed3c28eb..b63774033 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java @@ -30,7 +30,7 @@ public class DBConfig { * The database to use, format: {name of database}{-version}{-insert mode} name of database, for * more, in README.md */ - private DBSwitch DB_SWITCH = DBSwitch.DB_IOT_130_SESSION_BY_TABLET; + private DBSwitch DB_SWITCH = DBSwitch.DB_IOT_200_SESSION_BY_TABLET; /** The host of database server for IoTDB */ private List HOST = Collections.singletonList("127.0.0.1"); /** The port of database server */ diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index a887ac2f3..bf9fb9aa0 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -155,7 +155,9 @@ private Tablet genTablet(IBatch batch) { List schemaList = new ArrayList<>(); List columnTypes = new ArrayList<>(); List sensors = batch.getDeviceSchema().getSensors(); - iotdb.deleteIDColumnIfNecessary(columnTypes, sensors, batch); + if (config.isIS_DOUBLE_WRITE()) { + iotdb.deleteIDColumnIfNecessary(columnTypes, sensors, batch); + } iotdb.addIDColumnIfNecessary(columnTypes, sensors, batch); int sensorIndex = 0; for (Sensor sensor : sensors) { From 7db05f1602adba1b225b6eb5d75d9c66513f76bc Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Sat, 14 Sep 2024 00:01:22 +0800 Subject: [PATCH 03/34] In the tree model, disable the parameter IoTDB_TABLE_NUMBER. --- .../cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 80cbb3fdf..d04b79f26 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -332,8 +332,8 @@ private void loadProps() { "Please follow this rule to adjust the parameters: device number >= table number >= database number. Otherwise, device number = table number = database number"); } } else { + config.setIoTDB_TABLE_NUMBER(config.getGROUP_NUMBER()); if (config.getGROUP_NUMBER() > config.getDEVICE_NUMBER()) { - config.setIoTDB_TABLE_NUMBER(config.getGROUP_NUMBER()); LOGGER.warn( "Please follow this rule to adjust the parameters: device number >= database number. Otherwise, the total number of databases created is equal to the number of devices"); } From 4ee9ac3ceeb28df9e9324f8ed79b6006f6240f8e Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Sat, 14 Sep 2024 14:26:17 +0800 Subject: [PATCH 04/34] Modify Comments --- .../iot/benchmark/schema/MetaUtil.java | 16 +++-- .../schemaImpl/GenerateMetaDataSchema.java | 2 +- .../iotdb200/DMLStrategy/DMLStrategy.java | 2 - .../iotdb200/DMLStrategy/JDBCStrategy.java | 5 -- .../iotdb200/DMLStrategy/SessionStrategy.java | 71 ++++++++++++------- .../iot/benchmark/iotdb200/IoTDB.java | 4 -- 6 files changed, 57 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 3d62e7eb0..9287a773e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -56,10 +56,15 @@ public static int mappingId(int objectId, int objectNumber, int allocatingObject } } - /** 重排 deviceId,保证相邻的 deviceId 属于同一个 database */ + /** + * Rearrange deviceId to ensure that adjacent deviceIds belong to the same database. + * + *

[database0,[device0,device2,device4]], [database1,[device1,device3,device5]] + * [device0,device1,device2,device3,device4,device5] => + * [device0,device2,device4,device1,device3,device5] + */ public static List sortDeviceIdByTable(Config config, Logger LOGGER) { List deviceIds = new ArrayList<>(); - // 获取 Map database device 的映射 Map> databaseDeviceMap = new HashMap<>(); try { for (int deviceId = 0; deviceId < config.getDEVICE_NUMBER(); deviceId++) { @@ -67,12 +72,15 @@ public static List sortDeviceIdByTable(Config config, Logger LOGGER) { mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); int databaseId = mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); - databaseDeviceMap.computeIfAbsent(databaseId, k -> new ArrayList<>()).add(deviceId); + databaseDeviceMap + .computeIfAbsent( + databaseId, + k -> new ArrayList<>(config.getDEVICE_NUMBER() / config.getGROUP_NUMBER() + 1)) + .add(deviceId); } } catch (WorkloadException e) { LOGGER.error(e.getMessage()); } - // 重排 deviceId,保证相邻的 deviceId 属于同一个 database for (List values : databaseDeviceMap.values()) { deviceIds.addAll(values); } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index e1f97ef8d..f6235ee9d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -50,7 +50,7 @@ public boolean createMetaDataSchema() { CommonAlgorithms.distributeDevicesToClients( config.getDEVICE_NUMBER(), config.getCLIENT_NUMBER()); int deviceId = MetaUtil.getDeviceId(0); - // 重排 device ID 使相邻的 device 处在同一张表中 + // Rearrange device IDs so that adjacent devices are in the same table List deviceIds = MetaUtil.sortDeviceIdByTable(config, LOGGER); for (int clientId = 0; clientId < config.getCLIENT_NUMBER(); clientId++) { int deviceNumber = deviceDistribution.get(clientId); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java index f74e4c261..4d7368f2a 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java @@ -64,8 +64,6 @@ public abstract DeviceSummary deviceSummary( String device, String totalLineNumberSql, String maxTimestampSql, String minTimestampSql) throws TsdbException, SQLException; - public abstract void switchSession(int deviceId, String group); - public abstract void init() throws TsdbException; public abstract void cleanup(); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java index c830acb32..4682097e4 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java @@ -194,11 +194,6 @@ public DeviceSummary deviceSummary( return new DeviceSummary(device, totalLineNumber, minTimeStamp, maxTimeStamp); } - @Override - public void switchSession(int deviceId, String group) { - // nothing - } - @Override public void init() throws TsdbException { if (ioTDBConnection == null) { diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index bf9fb9aa0..f6bb45899 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -79,7 +79,7 @@ public class SessionStrategy extends DMLStrategy { private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE()); private final Map databaseSessionMap = new HashMap<>(); - private Session session; + // private Session session; private final IoTDB iotdb; public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { @@ -89,7 +89,8 @@ public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { for (int i = 0; i < dbConfig.getHOST().size(); i++) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } - session = buildSession(hostUrls, null); + Session session = buildSession(hostUrls, null); + databaseSessionMap.put(-1, session); } @Override @@ -115,9 +116,14 @@ private Status insertOneBatchByTablet(IBatch batch) { service.submit( () -> { try { + Session session; if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { - switchSession( - batch.getDeviceSchema().getDeviceId(), batch.getDeviceSchema().getGroup()); + session = + switchSession( + batch.getDeviceSchema().getDeviceId(), + batch.getDeviceSchema().getGroup()); + } else { + session = databaseSessionMap.get(-1); } iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { @@ -127,8 +133,7 @@ private Status insertOneBatchByTablet(IBatch batch) { return waitWriteTaskToFinishAndGetStatus(); } - @Override - public void switchSession(int deviceId, String group) { + public Session switchSession(int deviceId, String group) { try { int tableId = MetaUtil.mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); @@ -139,15 +144,17 @@ public void switchSession(int deviceId, String group) { for (int i = 0; i < dbConfig.getHOST().size(); i++) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } - Session sessionNew = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); - sessionNew.open(); - session = sessionNew; + Session session = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); + session.open(); + // session = sessionNew; databaseSessionMap.put(databaseId, session); + return session; } else { - session = databaseSessionMap.get(databaseId); + return databaseSessionMap.get(databaseId); } } catch (WorkloadException | IoTDBConnectionException e) { LOGGER.error(e.getMessage(), e); + return null; } } @@ -251,9 +258,13 @@ private Status insertOneBatchByRecord(IBatch batch, String deviceId) { List recordDataValue = convertTypeForBLOB(record, dataTypes); try { if (config.isVECTOR()) { - session.insertAlignedRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); + databaseSessionMap + .get(-1) + .insertAlignedRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } else { - session.insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); + databaseSessionMap + .get(-1) + .insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } } catch (IoTDBConnectionException | StatementExecutionException e) { failRecord++; @@ -298,10 +309,14 @@ private Status insertOneBatchByRecords(IBatch batch, String deviceId) { () -> { try { if (config.isVECTOR()) { - session.insertAlignedRecords( - deviceIds, times, measurementsList, typesList, valuesList); + databaseSessionMap + .get(-1) + .insertAlignedRecords( + deviceIds, times, measurementsList, typesList, valuesList); } else { - session.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); + databaseSessionMap + .get(-1) + .insertRecords(deviceIds, times, measurementsList, typesList, valuesList); } } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); @@ -338,7 +353,8 @@ public long executeQueryAndGetStatusImpl( service.submit( () -> { try { - SessionDataSet sessionDataSet = session.executeQueryStatement(executeSQL); + SessionDataSet sessionDataSet = + databaseSessionMap.get(-1).executeQueryStatement(executeSQL); if (config.isIS_COMPARISON()) { while (sessionDataSet.hasNext()) { RowRecord rowRecord = sessionDataSet.next(); @@ -398,7 +414,7 @@ public long executeQueryAndGetStatusImpl( public List verificationQueryImpl(String sql, Map> recordMap) throws IoTDBConnectionException, StatementExecutionException { int point = 0, line = 0; - try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql)) { + try (SessionDataSet sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(sql)) { while (sessionDataSet.hasNext()) { RowRecord rowRecord = sessionDataSet.next(); // The table model and the tree model obtain time differently @@ -422,7 +438,7 @@ public List verificationQueryImpl(String sql, Map> r @Override public List> deviceQueryImpl(String sql) throws Exception { List> result = new ArrayList<>(); - try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql)) { + try (SessionDataSet sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(sql)) { while (sessionDataSet.hasNext()) { List line = new ArrayList<>(); RowRecord rowRecord = sessionDataSet.next(); @@ -444,19 +460,20 @@ public DeviceSummary deviceSummary( int totalLineNumber = 0; long minTimeStamp, maxTimeStamp; try { - SessionDataSet sessionDataSet = session.executeQueryStatement(totalLineNumberSql); + SessionDataSet sessionDataSet = + databaseSessionMap.get(-1).executeQueryStatement(totalLineNumberSql); while (sessionDataSet.hasNext()) { sessionDataSet.next(); totalLineNumber++; } sessionDataSet.close(); - sessionDataSet = session.executeQueryStatement(maxTimestampSql); + sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(maxTimestampSql); RowRecord rowRecord = sessionDataSet.next(); maxTimeStamp = iotdb.getTimestamp(rowRecord); sessionDataSet.close(); - sessionDataSet = session.executeQueryStatement(minTimestampSql); + sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(minTimestampSql); rowRecord = sessionDataSet.next(); minTimeStamp = iotdb.getTimestamp(rowRecord); sessionDataSet.close(); @@ -483,9 +500,9 @@ Status waitWriteTaskToFinishAndGetStatus() { public void init() { try { if (config.isENABLE_THRIFT_COMPRESSION()) { - session.open(true); + databaseSessionMap.get(-1).open(true); } else { - session.open(); + databaseSessionMap.get(-1).open(); } this.service = Executors.newSingleThreadExecutor(); } catch (IoTDBConnectionException e) { @@ -496,7 +513,7 @@ public void init() { @Override public void cleanup() { try { - iotdb.sessionCleanupImpl(session); + iotdb.sessionCleanupImpl(databaseSessionMap.get(-1)); } catch (IoTDBConnectionException e) { LOGGER.warn("Failed to connect to IoTDB:" + e.getMessage()); } catch (StatementExecutionException e) { @@ -507,9 +524,9 @@ public void cleanup() { @Override public void close() throws TsdbException { try { - if (session != null) { - session.close(); - } + // if (session != null) { + // session.close(); + // } if (!databaseSessionMap.isEmpty()) { for (Session session : databaseSessionMap.values()) { session.close(); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index ee25b0814..38181f85b 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -672,10 +672,6 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic modelStrategy.sessionInsertImpl(session, tablet, deviceSchema); } - public void switchSessionImpl(int deviceId, String group) { - dmlStrategy.switchSession(deviceId, group); - } - public void addIDColumnIfNecessary( List columnTypes, List sensors, IBatch batch) { modelStrategy.addIDColumnIfNecessary(columnTypes, sensors, batch); From 16f9aa9c26576dedde4a598ed82f4734d972ffe0 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Sat, 14 Sep 2024 18:29:37 +0800 Subject: [PATCH 05/34] Delete useless comments --- .../java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java | 2 +- .../benchmark/schema/schemaImpl/GenerateMetaDataSchema.java | 2 +- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 6 +----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 9287a773e..f9b8fb293 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -63,7 +63,7 @@ public static int mappingId(int objectId, int objectNumber, int allocatingObject * [device0,device1,device2,device3,device4,device5] => * [device0,device2,device4,device1,device3,device5] */ - public static List sortDeviceIdByTable(Config config, Logger LOGGER) { + public static List sortDeviceIdByDatabase(Config config, Logger LOGGER) { List deviceIds = new ArrayList<>(); Map> databaseDeviceMap = new HashMap<>(); try { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index f6235ee9d..1c6052d40 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -51,7 +51,7 @@ public boolean createMetaDataSchema() { config.getDEVICE_NUMBER(), config.getCLIENT_NUMBER()); int deviceId = MetaUtil.getDeviceId(0); // Rearrange device IDs so that adjacent devices are in the same table - List deviceIds = MetaUtil.sortDeviceIdByTable(config, LOGGER); + List deviceIds = MetaUtil.sortDeviceIdByDatabase(config, LOGGER); for (int clientId = 0; clientId < config.getCLIENT_NUMBER(); clientId++) { int deviceNumber = deviceDistribution.get(clientId); List deviceSchemaList = new ArrayList<>(); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index f6bb45899..1979e7016 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -79,7 +79,6 @@ public class SessionStrategy extends DMLStrategy { private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE()); private final Map databaseSessionMap = new HashMap<>(); - // private Session session; private final IoTDB iotdb; public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { @@ -90,6 +89,7 @@ public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } Session session = buildSession(hostUrls, null); + // default session (databaseName not specified) databaseSessionMap.put(-1, session); } @@ -146,7 +146,6 @@ public Session switchSession(int deviceId, String group) { } Session session = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); session.open(); - // session = sessionNew; databaseSessionMap.put(databaseId, session); return session; } else { @@ -524,9 +523,6 @@ public void cleanup() { @Override public void close() throws TsdbException { try { - // if (session != null) { - // session.close(); - // } if (!databaseSessionMap.isEmpty()) { for (Session session : databaseSessionMap.values()) { session.close(); From 9da5a268aacb9936432038bbc3804987890ff08b Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Sat, 14 Sep 2024 19:13:43 +0800 Subject: [PATCH 06/34] Optimizing the code --- .../iotdb200/DMLStrategy/SessionStrategy.java | 14 ++++---------- .../edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java | 4 ++++ .../iotdb200/ModelStrategy/IoTDBModelStrategy.java | 1 + .../iotdb200/ModelStrategy/TableStrategy.java | 5 +++++ .../iotdb200/ModelStrategy/TreeStrategy.java | 6 ++++++ 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 1979e7016..5d0cef675 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -31,7 +31,6 @@ import cn.edu.tsinghua.iot.benchmark.entity.DeviceSummary; import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; -import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.OperationFailException; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; @@ -116,15 +115,10 @@ private Status insertOneBatchByTablet(IBatch batch) { service.submit( () -> { try { - Session session; - if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { - session = - switchSession( - batch.getDeviceSchema().getDeviceId(), - batch.getDeviceSchema().getGroup()); - } else { - session = databaseSessionMap.get(-1); - } + Session session = + switchSession( + iotdb.getDeviceIdForSwitchSession(batch), + batch.getDeviceSchema().getGroup()); iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index 38181f85b..3117fbd0c 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -672,6 +672,10 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic modelStrategy.sessionInsertImpl(session, tablet, deviceSchema); } + public int getDeviceIdForSwitchSession(IBatch batch) { + return modelStrategy.getDeviceIdForSwitchSession(batch); + } + public void addIDColumnIfNecessary( List columnTypes, List sensors, IBatch batch) { modelStrategy.addIDColumnIfNecessary(columnTypes, sensors, batch); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java index f3c289a48..858cb3619 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java @@ -126,6 +126,7 @@ public abstract void sessionInsertImpl(Session session, Tablet tablet, DeviceSch public abstract void sessionCleanupImpl(Session session) throws IoTDBConnectionException, StatementExecutionException; + public abstract int getDeviceIdForSwitchSession(IBatch batch); // endregion public abstract Logger getLogger(); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index 4b20a688b..db03cc8c0 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -349,6 +349,11 @@ public void sessionCleanupImpl(Session session) } } + @Override + public int getDeviceIdForSwitchSession(IBatch batch) { + return batch.getDeviceSchema().getDeviceId(); + } + // endregion @Override diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java index 606c96d48..1d1dfef23 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java @@ -355,6 +355,12 @@ public void sessionCleanupImpl(Session session) { } } + @Override + public int getDeviceIdForSwitchSession(IBatch batch) { + // -1 is the default session in databaseSessionMap + return -1; + } + // endregion @Override From 9fe62092017bfc391da75530bd2bd8b087f0edac Mon Sep 17 00:00:00 2001 From: Yang Yuming <50571168+YangYumings@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:26:10 +0800 Subject: [PATCH 07/34] Set an appropriate List HashMap capacity to reduce the number of capacity expansions. (#445) --- .../tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java | 2 +- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java index 3f1cd52a9..0e283a190 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java @@ -67,7 +67,7 @@ public long getBatchNumber() { /** Add one row into batch, row contains data from all sensors */ protected List generateOneRow(int deviceIndex, int colIndex, long stepOffset) throws WorkloadException { - List values = new ArrayList<>(); + List values = new ArrayList<>(config.getSENSOR_NUMBER()); int index = (int) (Math.abs(stepOffset * (deviceIndex + 1)) % config.getWORKLOAD_BUFFER_SIZE()); if (colIndex == -1) { for (int i = 0; i < config.getSENSOR_NUMBER(); i++) { diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index c210e72bd..e67a2477d 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -73,7 +73,7 @@ public class SessionStrategy extends DMLStrategy { static final Config config = ConfigDescriptor.getInstance().getConfig(); private static final Map binaryCache = - new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE()); + new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); private final Session session; private final IoTDB iotdb; From 575037dccdea80e5c1c175129eb7d21baddfa706 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Wed, 18 Sep 2024 15:19:01 +0800 Subject: [PATCH 08/34] Improve the annotation for sortDeviceId --- .../iot/benchmark/schema/MetaUtil.java | 40 +++++++++++++------ .../schemaImpl/GenerateMetaDataSchema.java | 2 +- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index f9b8fb293..19a7e0061 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -57,33 +57,47 @@ public static int mappingId(int objectId, int objectNumber, int allocatingObject } /** - * Rearrange deviceId to ensure that adjacent deviceIds belong to the same database. + * It traverses all device IDs, assigns each device to the corresponding table, and further + * aggregates the devices in the table into the corresponding database.
+ * IoTDB-TableMode : Ensure that multiple devices written in a single batch come from the same + * table.
+ * IoTDB-TreeMode : It will not affect its writing speed. * - *

[database0,[device0,device2,device4]], [database1,[device1,device3,device5]] - * [device0,device1,device2,device3,device4,device5] => - * [device0,device2,device4,device1,device3,device5] + * @param config + * @param LOGGER + * @return deviceIds */ - public static List sortDeviceIdByDatabase(Config config, Logger LOGGER) { + public static List sortDeviceId(Config config, Logger LOGGER) { List deviceIds = new ArrayList<>(); - Map> databaseDeviceMap = new HashMap<>(); + Map> tableDeviceMap = + new HashMap<>(config.getIoTDB_TABLE_NUMBER(), 1.00f); + Map> databaseDeviceMap = new HashMap<>(config.getGROUP_NUMBER(), 1.00f); try { + // Get the device contained in each table for (int deviceId = 0; deviceId < config.getDEVICE_NUMBER(); deviceId++) { + // Calculate tableId from deviceId int tableId = mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + tableDeviceMap + .computeIfAbsent( + tableId, + k -> + new ArrayList<>(config.getDEVICE_NUMBER() / config.getIoTDB_TABLE_NUMBER() + 1)) + .add(deviceId); + } + // By using tableDeviceMap, quickly get the devices contained in each database + for (int tableId = 0; tableId < config.getIoTDB_TABLE_NUMBER(); tableId++) { + // Calculate databaseId from tableId int databaseId = mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); databaseDeviceMap - .computeIfAbsent( - databaseId, - k -> new ArrayList<>(config.getDEVICE_NUMBER() / config.getGROUP_NUMBER() + 1)) - .add(deviceId); + .computeIfAbsent(databaseId, k -> new ArrayList<>()) + .addAll(tableDeviceMap.getOrDefault(tableId, Collections.emptyList())); } } catch (WorkloadException e) { LOGGER.error(e.getMessage()); } - for (List values : databaseDeviceMap.values()) { - deviceIds.addAll(values); - } + databaseDeviceMap.values().forEach(deviceIds::addAll); return deviceIds; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index 1c6052d40..7d99d05cf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -51,7 +51,7 @@ public boolean createMetaDataSchema() { config.getDEVICE_NUMBER(), config.getCLIENT_NUMBER()); int deviceId = MetaUtil.getDeviceId(0); // Rearrange device IDs so that adjacent devices are in the same table - List deviceIds = MetaUtil.sortDeviceIdByDatabase(config, LOGGER); + List deviceIds = MetaUtil.sortDeviceId(config, LOGGER); for (int clientId = 0; clientId < config.getCLIENT_NUMBER(); clientId++) { int deviceNumber = deviceDistribution.get(clientId); List deviceSchemaList = new ArrayList<>(); From 7cbf547e2f01e181770eefa8989c79af2bdaaa1e Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Wed, 18 Sep 2024 15:36:12 +0800 Subject: [PATCH 09/34] try to resolved conflicts(2) --- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 5d0cef675..9d41c4eb4 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -76,7 +76,7 @@ public class SessionStrategy extends DMLStrategy { static final Config config = ConfigDescriptor.getInstance().getConfig(); private static final Map binaryCache = - new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE()); + new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(),1.00f); private final Map databaseSessionMap = new HashMap<>(); private final IoTDB iotdb; From 36a01a465fca73500eddb6f7080342648e154edf Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Wed, 18 Sep 2024 15:36:59 +0800 Subject: [PATCH 10/34] try to resolved conflicts(3) --- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 9d41c4eb4..aad212d4c 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -76,7 +76,7 @@ public class SessionStrategy extends DMLStrategy { static final Config config = ConfigDescriptor.getInstance().getConfig(); private static final Map binaryCache = - new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(),1.00f); + new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); private final Map databaseSessionMap = new HashMap<>(); private final IoTDB iotdb; From 3e1f5dca2808795b3425f4877f18a72cd9a6487c Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 12 Sep 2024 21:21:19 +0800 Subject: [PATCH 11/34] reduce number of session --- .../iot/benchmark/conf/Constants.java | 4 +- .../iot/benchmark/schema/MetaUtil.java | 25 +++++++++++ .../schemaImpl/GenerateMetaDataSchema.java | 14 ++++++- .../iot/benchmark/tsdb/DBFactory.java | 4 +- .../iotdb200/DMLStrategy/DMLStrategy.java | 5 ++- .../iotdb200/DMLStrategy/JDBCStrategy.java | 5 +++ .../iotdb200/DMLStrategy/SessionStrategy.java | 42 ++++++++++++++++++- .../iot/benchmark/iotdb200/IoTDB.java | 4 ++ .../iotdb200/ModelStrategy/TableStrategy.java | 2 - 9 files changed, 96 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java index 5dbe177cb..4cf16c75b 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java @@ -55,9 +55,11 @@ public class Constants { public static final String IOTDB200_REST_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb200.IoTDBRestAPI"; - public static final String IOTDB130 = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB"; public static final String IOTDB130_REST_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBRestAPI"; + public static final String IOTDB130_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDB"; + public static final String IOTDB130_SESSION_CLASS = + "cn.edu.tsinghua.iot.benchmark.iotdb130.IoTDBSession"; public static final String IOTDB110_JDBC_CLASS = "cn.edu.tsinghua.iot.benchmark.iotdb110.IoTDB"; public static final String IOTDB110_SESSION_CLASS = diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 452f7245d..3d62e7eb0 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -4,7 +4,9 @@ import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import cn.edu.tsinghua.iot.benchmark.conf.Constants; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; +import org.slf4j.Logger; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -54,6 +56,29 @@ public static int mappingId(int objectId, int objectNumber, int allocatingObject } } + /** 重排 deviceId,保证相邻的 deviceId 属于同一个 database */ + public static List sortDeviceIdByTable(Config config, Logger LOGGER) { + List deviceIds = new ArrayList<>(); + // 获取 Map database device 的映射 + Map> databaseDeviceMap = new HashMap<>(); + try { + for (int deviceId = 0; deviceId < config.getDEVICE_NUMBER(); deviceId++) { + int tableId = + mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + int databaseId = + mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); + databaseDeviceMap.computeIfAbsent(databaseId, k -> new ArrayList<>()).add(deviceId); + } + } catch (WorkloadException e) { + LOGGER.error(e.getMessage()); + } + // 重排 deviceId,保证相邻的 deviceId 属于同一个 database + for (List values : databaseDeviceMap.values()) { + deviceIds.addAll(values); + } + return deviceIds; + } + public static String getGroupIdFromDeviceName(String deviceName) { int groupId = deviceName.hashCode(); if (groupId < 0) { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index 59d6b4cb4..e1f97ef8d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -22,6 +22,7 @@ import cn.edu.tsinghua.iot.benchmark.conf.Config; import cn.edu.tsinghua.iot.benchmark.conf.ConfigDescriptor; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.schema.MetaDataSchema; import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.utils.CommonAlgorithms; @@ -45,15 +46,24 @@ public boolean createMetaDataSchema() { return false; } - int deviceId = MetaUtil.getDeviceId(0); Map deviceDistribution = CommonAlgorithms.distributeDevicesToClients( config.getDEVICE_NUMBER(), config.getCLIENT_NUMBER()); + int deviceId = MetaUtil.getDeviceId(0); + // 重排 device ID 使相邻的 device 处在同一张表中 + List deviceIds = MetaUtil.sortDeviceIdByTable(config, LOGGER); for (int clientId = 0; clientId < config.getCLIENT_NUMBER(); clientId++) { int deviceNumber = deviceDistribution.get(clientId); List deviceSchemaList = new ArrayList<>(); for (int d = 0; d < deviceNumber; d++) { - DeviceSchema deviceSchema = new DeviceSchema(deviceId, sensors, MetaUtil.getTags(deviceId)); + DeviceSchema deviceSchema; + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { + deviceSchema = + new DeviceSchema( + deviceIds.get(deviceId), sensors, MetaUtil.getTags(deviceIds.get(deviceId))); + } else { + deviceSchema = new DeviceSchema(deviceId, sensors, MetaUtil.getTags(deviceId)); + } NAME_DATA_SCHEMA.put(deviceSchema.getDevice(), deviceSchema); GROUPS.add(deviceSchema.getGroup()); deviceSchemaList.add(deviceSchema); diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java index 44a4e7e25..439c56626 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBFactory.java @@ -54,10 +54,12 @@ public IDatabase getDatabase(DBConfig dbConfig) throws SQLException { dbClass = Constants.IOTDB130_REST_CLASS; break; case DB_IOT_130_JDBC: + dbClass = Constants.IOTDB130_JDBC_CLASS; + break; case DB_IOT_130_SESSION_BY_TABLET: case DB_IOT_130_SESSION_BY_RECORD: case DB_IOT_130_SESSION_BY_RECORDS: - dbClass = Constants.IOTDB130; + dbClass = Constants.IOTDB130_SESSION_CLASS; break; // IoTDB 1.1 diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java index 59e1192cf..f74e4c261 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java @@ -64,13 +64,15 @@ public abstract DeviceSummary deviceSummary( String device, String totalLineNumberSql, String maxTimestampSql, String minTimestampSql) throws TsdbException, SQLException; + public abstract void switchSession(int deviceId, String group); + public abstract void init() throws TsdbException; public abstract void cleanup(); public abstract void close() throws TsdbException; - protected Session buildSession(List hostUrls) { + protected Session buildSession(List hostUrls, String databaseName) { return new Session.Builder() .nodeUrls(hostUrls) .username(dbConfig.getUSERNAME()) @@ -78,6 +80,7 @@ protected Session buildSession(List hostUrls) { .enableRedirection(true) .version(Version.V_1_0) .sqlDialect(config.getIoTDB_DIALECT_MODE().name()) + .database(databaseName) .build(); } } diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java index 4682097e4..c830acb32 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java @@ -194,6 +194,11 @@ public DeviceSummary deviceSummary( return new DeviceSummary(device, totalLineNumber, minTimeStamp, maxTimeStamp); } + @Override + public void switchSession(int deviceId, String group) { + // nothing + } + @Override public void init() throws TsdbException { if (ioTDBConnection == null) { diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index e67a2477d..9b4fdba62 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -31,11 +31,14 @@ import cn.edu.tsinghua.iot.benchmark.entity.DeviceSummary; import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; +import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.OperationFailException; +import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; import cn.edu.tsinghua.iot.benchmark.iotdb200.IoTDB; import cn.edu.tsinghua.iot.benchmark.iotdb200.utils.IoTDBUtils; import cn.edu.tsinghua.iot.benchmark.measurement.Status; +import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; import cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode; @@ -55,6 +58,7 @@ import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -74,7 +78,8 @@ public class SessionStrategy extends DMLStrategy { private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); - private final Session session; + private final Map databaseSessionMap = new HashMap<>(); + private Session session; private final IoTDB iotdb; public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { @@ -84,7 +89,7 @@ public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { for (int i = 0; i < dbConfig.getHOST().size(); i++) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } - session = buildSession(hostUrls); + session = buildSession(hostUrls, null); } @Override @@ -110,6 +115,10 @@ private Status insertOneBatchByTablet(IBatch batch) { service.submit( () -> { try { + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { + switchSession( + batch.getDeviceSchema().getDeviceId(), batch.getDeviceSchema().getGroup()); + } iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); @@ -118,6 +127,30 @@ private Status insertOneBatchByTablet(IBatch batch) { return waitWriteTaskToFinishAndGetStatus(); } + @Override + public void switchSession(int deviceId, String group) { + try { + int tableId = + MetaUtil.mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + int databaseId = + MetaUtil.mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); + if (databaseSessionMap.get(databaseId) == null) { + List hostUrls = new ArrayList<>(dbConfig.getHOST().size()); + for (int i = 0; i < dbConfig.getHOST().size(); i++) { + hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); + } + Session sessionNew = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); + sessionNew.open(); + session = sessionNew; + databaseSessionMap.put(databaseId, session); + } else { + session = databaseSessionMap.get(databaseId); + } + } catch (WorkloadException | IoTDBConnectionException e) { + LOGGER.error(e.getMessage(), e); + } + } + private Tablet genTablet(IBatch batch) { List schemaList = new ArrayList<>(); List columnTypes = new ArrayList<>(); @@ -475,6 +508,11 @@ public void close() throws TsdbException { if (session != null) { session.close(); } + if (!databaseSessionMap.isEmpty()) { + for (Session session : databaseSessionMap.values()) { + session.close(); + } + } service.shutdown(); } catch (IoTDBConnectionException ioTDBConnectionException) { LOGGER.error("Failed to close session."); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index 38181f85b..ee25b0814 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -672,6 +672,10 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic modelStrategy.sessionInsertImpl(session, tablet, deviceSchema); } + public void switchSessionImpl(int deviceId, String group) { + dmlStrategy.switchSession(deviceId, group); + } + public void addIDColumnIfNecessary( List columnTypes, List sensors, IBatch batch) { modelStrategy.addIDColumnIfNecessary(columnTypes, sensors, batch); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index 5effa15d1..4b20a688b 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -333,8 +333,6 @@ public void deleteIDColumnIfNecessary( @Override public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) throws IoTDBConnectionException, StatementExecutionException { - session.executeNonQueryStatement( - "use " + dbConfig.getDB_NAME() + "_" + deviceSchema.getGroup()); session.insertRelationalTablet(tablet); } From 2d20ac7fbfc578d5d4f9ec185dd0cdeb808c8a2f Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Fri, 13 Sep 2024 21:41:21 +0800 Subject: [PATCH 12/34] The deleteIDColumnIfNecessary method is executed only in dual write mode --- .../java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java | 2 +- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java index 9ed3c28eb..b63774033 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/tsdb/DBConfig.java @@ -30,7 +30,7 @@ public class DBConfig { * The database to use, format: {name of database}{-version}{-insert mode} name of database, for * more, in README.md */ - private DBSwitch DB_SWITCH = DBSwitch.DB_IOT_130_SESSION_BY_TABLET; + private DBSwitch DB_SWITCH = DBSwitch.DB_IOT_200_SESSION_BY_TABLET; /** The host of database server for IoTDB */ private List HOST = Collections.singletonList("127.0.0.1"); /** The port of database server */ diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 9b4fdba62..c310139b9 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -155,7 +155,9 @@ private Tablet genTablet(IBatch batch) { List schemaList = new ArrayList<>(); List columnTypes = new ArrayList<>(); List sensors = batch.getDeviceSchema().getSensors(); - iotdb.deleteIDColumnIfNecessary(columnTypes, sensors, batch); + if (config.isIS_DOUBLE_WRITE()) { + iotdb.deleteIDColumnIfNecessary(columnTypes, sensors, batch); + } iotdb.addIDColumnIfNecessary(columnTypes, sensors, batch); int sensorIndex = 0; for (Sensor sensor : sensors) { From e41e8939acf89e2770ae6dbfd945818b972fe495 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Sat, 14 Sep 2024 00:01:22 +0800 Subject: [PATCH 13/34] In the tree model, disable the parameter IoTDB_TABLE_NUMBER. --- .../cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 80cbb3fdf..d04b79f26 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -332,8 +332,8 @@ private void loadProps() { "Please follow this rule to adjust the parameters: device number >= table number >= database number. Otherwise, device number = table number = database number"); } } else { + config.setIoTDB_TABLE_NUMBER(config.getGROUP_NUMBER()); if (config.getGROUP_NUMBER() > config.getDEVICE_NUMBER()) { - config.setIoTDB_TABLE_NUMBER(config.getGROUP_NUMBER()); LOGGER.warn( "Please follow this rule to adjust the parameters: device number >= database number. Otherwise, the total number of databases created is equal to the number of devices"); } From 04cba15751ec02b5003de479cdae5017d77053e5 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Sat, 14 Sep 2024 14:26:17 +0800 Subject: [PATCH 14/34] Modify Comments --- .../iot/benchmark/schema/MetaUtil.java | 16 +++-- .../schemaImpl/GenerateMetaDataSchema.java | 2 +- .../iotdb200/DMLStrategy/DMLStrategy.java | 2 - .../iotdb200/DMLStrategy/JDBCStrategy.java | 5 -- .../iotdb200/DMLStrategy/SessionStrategy.java | 71 ++++++++++++------- .../iot/benchmark/iotdb200/IoTDB.java | 4 -- 6 files changed, 57 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 3d62e7eb0..9287a773e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -56,10 +56,15 @@ public static int mappingId(int objectId, int objectNumber, int allocatingObject } } - /** 重排 deviceId,保证相邻的 deviceId 属于同一个 database */ + /** + * Rearrange deviceId to ensure that adjacent deviceIds belong to the same database. + * + *

[database0,[device0,device2,device4]], [database1,[device1,device3,device5]] + * [device0,device1,device2,device3,device4,device5] => + * [device0,device2,device4,device1,device3,device5] + */ public static List sortDeviceIdByTable(Config config, Logger LOGGER) { List deviceIds = new ArrayList<>(); - // 获取 Map database device 的映射 Map> databaseDeviceMap = new HashMap<>(); try { for (int deviceId = 0; deviceId < config.getDEVICE_NUMBER(); deviceId++) { @@ -67,12 +72,15 @@ public static List sortDeviceIdByTable(Config config, Logger LOGGER) { mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); int databaseId = mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); - databaseDeviceMap.computeIfAbsent(databaseId, k -> new ArrayList<>()).add(deviceId); + databaseDeviceMap + .computeIfAbsent( + databaseId, + k -> new ArrayList<>(config.getDEVICE_NUMBER() / config.getGROUP_NUMBER() + 1)) + .add(deviceId); } } catch (WorkloadException e) { LOGGER.error(e.getMessage()); } - // 重排 deviceId,保证相邻的 deviceId 属于同一个 database for (List values : databaseDeviceMap.values()) { deviceIds.addAll(values); } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index e1f97ef8d..f6235ee9d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -50,7 +50,7 @@ public boolean createMetaDataSchema() { CommonAlgorithms.distributeDevicesToClients( config.getDEVICE_NUMBER(), config.getCLIENT_NUMBER()); int deviceId = MetaUtil.getDeviceId(0); - // 重排 device ID 使相邻的 device 处在同一张表中 + // Rearrange device IDs so that adjacent devices are in the same table List deviceIds = MetaUtil.sortDeviceIdByTable(config, LOGGER); for (int clientId = 0; clientId < config.getCLIENT_NUMBER(); clientId++) { int deviceNumber = deviceDistribution.get(clientId); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java index f74e4c261..4d7368f2a 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java @@ -64,8 +64,6 @@ public abstract DeviceSummary deviceSummary( String device, String totalLineNumberSql, String maxTimestampSql, String minTimestampSql) throws TsdbException, SQLException; - public abstract void switchSession(int deviceId, String group); - public abstract void init() throws TsdbException; public abstract void cleanup(); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java index c830acb32..4682097e4 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/JDBCStrategy.java @@ -194,11 +194,6 @@ public DeviceSummary deviceSummary( return new DeviceSummary(device, totalLineNumber, minTimeStamp, maxTimeStamp); } - @Override - public void switchSession(int deviceId, String group) { - // nothing - } - @Override public void init() throws TsdbException { if (ioTDBConnection == null) { diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index c310139b9..d0d3a0d91 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -79,7 +79,7 @@ public class SessionStrategy extends DMLStrategy { private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); private final Map databaseSessionMap = new HashMap<>(); - private Session session; + // private Session session; private final IoTDB iotdb; public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { @@ -89,7 +89,8 @@ public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { for (int i = 0; i < dbConfig.getHOST().size(); i++) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } - session = buildSession(hostUrls, null); + Session session = buildSession(hostUrls, null); + databaseSessionMap.put(-1, session); } @Override @@ -115,9 +116,14 @@ private Status insertOneBatchByTablet(IBatch batch) { service.submit( () -> { try { + Session session; if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { - switchSession( - batch.getDeviceSchema().getDeviceId(), batch.getDeviceSchema().getGroup()); + session = + switchSession( + batch.getDeviceSchema().getDeviceId(), + batch.getDeviceSchema().getGroup()); + } else { + session = databaseSessionMap.get(-1); } iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { @@ -127,8 +133,7 @@ private Status insertOneBatchByTablet(IBatch batch) { return waitWriteTaskToFinishAndGetStatus(); } - @Override - public void switchSession(int deviceId, String group) { + public Session switchSession(int deviceId, String group) { try { int tableId = MetaUtil.mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); @@ -139,15 +144,17 @@ public void switchSession(int deviceId, String group) { for (int i = 0; i < dbConfig.getHOST().size(); i++) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } - Session sessionNew = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); - sessionNew.open(); - session = sessionNew; + Session session = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); + session.open(); + // session = sessionNew; databaseSessionMap.put(databaseId, session); + return session; } else { - session = databaseSessionMap.get(databaseId); + return databaseSessionMap.get(databaseId); } } catch (WorkloadException | IoTDBConnectionException e) { LOGGER.error(e.getMessage(), e); + return null; } } @@ -251,9 +258,13 @@ private Status insertOneBatchByRecord(IBatch batch, String deviceId) { List recordDataValue = convertTypeForBLOB(record, dataTypes); try { if (config.isVECTOR()) { - session.insertAlignedRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); + databaseSessionMap + .get(-1) + .insertAlignedRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } else { - session.insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); + databaseSessionMap + .get(-1) + .insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } } catch (IoTDBConnectionException | StatementExecutionException e) { failRecord++; @@ -298,10 +309,14 @@ private Status insertOneBatchByRecords(IBatch batch, String deviceId) { () -> { try { if (config.isVECTOR()) { - session.insertAlignedRecords( - deviceIds, times, measurementsList, typesList, valuesList); + databaseSessionMap + .get(-1) + .insertAlignedRecords( + deviceIds, times, measurementsList, typesList, valuesList); } else { - session.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); + databaseSessionMap + .get(-1) + .insertRecords(deviceIds, times, measurementsList, typesList, valuesList); } } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); @@ -338,7 +353,8 @@ public long executeQueryAndGetStatusImpl( service.submit( () -> { try { - SessionDataSet sessionDataSet = session.executeQueryStatement(executeSQL); + SessionDataSet sessionDataSet = + databaseSessionMap.get(-1).executeQueryStatement(executeSQL); if (config.isIS_COMPARISON()) { while (sessionDataSet.hasNext()) { RowRecord rowRecord = sessionDataSet.next(); @@ -398,7 +414,7 @@ public long executeQueryAndGetStatusImpl( public List verificationQueryImpl(String sql, Map> recordMap) throws IoTDBConnectionException, StatementExecutionException { int point = 0, line = 0; - try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql)) { + try (SessionDataSet sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(sql)) { while (sessionDataSet.hasNext()) { RowRecord rowRecord = sessionDataSet.next(); // The table model and the tree model obtain time differently @@ -422,7 +438,7 @@ public List verificationQueryImpl(String sql, Map> r @Override public List> deviceQueryImpl(String sql) throws Exception { List> result = new ArrayList<>(); - try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql)) { + try (SessionDataSet sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(sql)) { while (sessionDataSet.hasNext()) { List line = new ArrayList<>(); RowRecord rowRecord = sessionDataSet.next(); @@ -444,19 +460,20 @@ public DeviceSummary deviceSummary( int totalLineNumber = 0; long minTimeStamp, maxTimeStamp; try { - SessionDataSet sessionDataSet = session.executeQueryStatement(totalLineNumberSql); + SessionDataSet sessionDataSet = + databaseSessionMap.get(-1).executeQueryStatement(totalLineNumberSql); while (sessionDataSet.hasNext()) { sessionDataSet.next(); totalLineNumber++; } sessionDataSet.close(); - sessionDataSet = session.executeQueryStatement(maxTimestampSql); + sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(maxTimestampSql); RowRecord rowRecord = sessionDataSet.next(); maxTimeStamp = iotdb.getTimestamp(rowRecord); sessionDataSet.close(); - sessionDataSet = session.executeQueryStatement(minTimestampSql); + sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(minTimestampSql); rowRecord = sessionDataSet.next(); minTimeStamp = iotdb.getTimestamp(rowRecord); sessionDataSet.close(); @@ -483,9 +500,9 @@ Status waitWriteTaskToFinishAndGetStatus() { public void init() { try { if (config.isENABLE_THRIFT_COMPRESSION()) { - session.open(true); + databaseSessionMap.get(-1).open(true); } else { - session.open(); + databaseSessionMap.get(-1).open(); } this.service = Executors.newSingleThreadExecutor(); } catch (IoTDBConnectionException e) { @@ -496,7 +513,7 @@ public void init() { @Override public void cleanup() { try { - iotdb.sessionCleanupImpl(session); + iotdb.sessionCleanupImpl(databaseSessionMap.get(-1)); } catch (IoTDBConnectionException e) { LOGGER.warn("Failed to connect to IoTDB:" + e.getMessage()); } catch (StatementExecutionException e) { @@ -507,9 +524,9 @@ public void cleanup() { @Override public void close() throws TsdbException { try { - if (session != null) { - session.close(); - } + // if (session != null) { + // session.close(); + // } if (!databaseSessionMap.isEmpty()) { for (Session session : databaseSessionMap.values()) { session.close(); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index ee25b0814..38181f85b 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -672,10 +672,6 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic modelStrategy.sessionInsertImpl(session, tablet, deviceSchema); } - public void switchSessionImpl(int deviceId, String group) { - dmlStrategy.switchSession(deviceId, group); - } - public void addIDColumnIfNecessary( List columnTypes, List sensors, IBatch batch) { modelStrategy.addIDColumnIfNecessary(columnTypes, sensors, batch); From bc6adf3e548f2b374e7c3fdb3e31d0c01d8edbdc Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Sat, 14 Sep 2024 18:29:37 +0800 Subject: [PATCH 15/34] Delete useless comments --- .../java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java | 2 +- .../benchmark/schema/schemaImpl/GenerateMetaDataSchema.java | 2 +- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 6 +----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index 9287a773e..f9b8fb293 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -63,7 +63,7 @@ public static int mappingId(int objectId, int objectNumber, int allocatingObject * [device0,device1,device2,device3,device4,device5] => * [device0,device2,device4,device1,device3,device5] */ - public static List sortDeviceIdByTable(Config config, Logger LOGGER) { + public static List sortDeviceIdByDatabase(Config config, Logger LOGGER) { List deviceIds = new ArrayList<>(); Map> databaseDeviceMap = new HashMap<>(); try { diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index f6235ee9d..1c6052d40 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -51,7 +51,7 @@ public boolean createMetaDataSchema() { config.getDEVICE_NUMBER(), config.getCLIENT_NUMBER()); int deviceId = MetaUtil.getDeviceId(0); // Rearrange device IDs so that adjacent devices are in the same table - List deviceIds = MetaUtil.sortDeviceIdByTable(config, LOGGER); + List deviceIds = MetaUtil.sortDeviceIdByDatabase(config, LOGGER); for (int clientId = 0; clientId < config.getCLIENT_NUMBER(); clientId++) { int deviceNumber = deviceDistribution.get(clientId); List deviceSchemaList = new ArrayList<>(); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index d0d3a0d91..0e544b72a 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -79,7 +79,6 @@ public class SessionStrategy extends DMLStrategy { private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); private final Map databaseSessionMap = new HashMap<>(); - // private Session session; private final IoTDB iotdb; public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { @@ -90,6 +89,7 @@ public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } Session session = buildSession(hostUrls, null); + // default session (databaseName not specified) databaseSessionMap.put(-1, session); } @@ -146,7 +146,6 @@ public Session switchSession(int deviceId, String group) { } Session session = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); session.open(); - // session = sessionNew; databaseSessionMap.put(databaseId, session); return session; } else { @@ -524,9 +523,6 @@ public void cleanup() { @Override public void close() throws TsdbException { try { - // if (session != null) { - // session.close(); - // } if (!databaseSessionMap.isEmpty()) { for (Session session : databaseSessionMap.values()) { session.close(); From 592a94ef93ec5b541f25223d794bdff8f305c642 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Sat, 14 Sep 2024 19:13:43 +0800 Subject: [PATCH 16/34] Optimizing the code --- .../iotdb200/DMLStrategy/SessionStrategy.java | 14 ++++---------- .../edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java | 4 ++++ .../iotdb200/ModelStrategy/IoTDBModelStrategy.java | 1 + .../iotdb200/ModelStrategy/TableStrategy.java | 5 +++++ .../iotdb200/ModelStrategy/TreeStrategy.java | 6 ++++++ 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 0e544b72a..aad212d4c 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -31,7 +31,6 @@ import cn.edu.tsinghua.iot.benchmark.entity.DeviceSummary; import cn.edu.tsinghua.iot.benchmark.entity.Record; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; -import cn.edu.tsinghua.iot.benchmark.entity.enums.SQLDialect; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.OperationFailException; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; @@ -116,15 +115,10 @@ private Status insertOneBatchByTablet(IBatch batch) { service.submit( () -> { try { - Session session; - if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { - session = - switchSession( - batch.getDeviceSchema().getDeviceId(), - batch.getDeviceSchema().getGroup()); - } else { - session = databaseSessionMap.get(-1); - } + Session session = + switchSession( + iotdb.getDeviceIdForSwitchSession(batch), + batch.getDeviceSchema().getGroup()); iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index 38181f85b..3117fbd0c 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -672,6 +672,10 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic modelStrategy.sessionInsertImpl(session, tablet, deviceSchema); } + public int getDeviceIdForSwitchSession(IBatch batch) { + return modelStrategy.getDeviceIdForSwitchSession(batch); + } + public void addIDColumnIfNecessary( List columnTypes, List sensors, IBatch batch) { modelStrategy.addIDColumnIfNecessary(columnTypes, sensors, batch); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java index f3c289a48..858cb3619 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java @@ -126,6 +126,7 @@ public abstract void sessionInsertImpl(Session session, Tablet tablet, DeviceSch public abstract void sessionCleanupImpl(Session session) throws IoTDBConnectionException, StatementExecutionException; + public abstract int getDeviceIdForSwitchSession(IBatch batch); // endregion public abstract Logger getLogger(); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index 4b20a688b..db03cc8c0 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -349,6 +349,11 @@ public void sessionCleanupImpl(Session session) } } + @Override + public int getDeviceIdForSwitchSession(IBatch batch) { + return batch.getDeviceSchema().getDeviceId(); + } + // endregion @Override diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java index 606c96d48..1d1dfef23 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java @@ -355,6 +355,12 @@ public void sessionCleanupImpl(Session session) { } } + @Override + public int getDeviceIdForSwitchSession(IBatch batch) { + // -1 is the default session in databaseSessionMap + return -1; + } + // endregion @Override From 6747b6228371de378da32c2c8317e19b20467330 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Wed, 18 Sep 2024 15:19:01 +0800 Subject: [PATCH 17/34] Improve the annotation for sortDeviceId --- .../iot/benchmark/schema/MetaUtil.java | 40 +++++++++++++------ .../schemaImpl/GenerateMetaDataSchema.java | 2 +- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java index f9b8fb293..19a7e0061 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/MetaUtil.java @@ -57,33 +57,47 @@ public static int mappingId(int objectId, int objectNumber, int allocatingObject } /** - * Rearrange deviceId to ensure that adjacent deviceIds belong to the same database. + * It traverses all device IDs, assigns each device to the corresponding table, and further + * aggregates the devices in the table into the corresponding database.
+ * IoTDB-TableMode : Ensure that multiple devices written in a single batch come from the same + * table.
+ * IoTDB-TreeMode : It will not affect its writing speed. * - *

[database0,[device0,device2,device4]], [database1,[device1,device3,device5]] - * [device0,device1,device2,device3,device4,device5] => - * [device0,device2,device4,device1,device3,device5] + * @param config + * @param LOGGER + * @return deviceIds */ - public static List sortDeviceIdByDatabase(Config config, Logger LOGGER) { + public static List sortDeviceId(Config config, Logger LOGGER) { List deviceIds = new ArrayList<>(); - Map> databaseDeviceMap = new HashMap<>(); + Map> tableDeviceMap = + new HashMap<>(config.getIoTDB_TABLE_NUMBER(), 1.00f); + Map> databaseDeviceMap = new HashMap<>(config.getGROUP_NUMBER(), 1.00f); try { + // Get the device contained in each table for (int deviceId = 0; deviceId < config.getDEVICE_NUMBER(); deviceId++) { + // Calculate tableId from deviceId int tableId = mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); + tableDeviceMap + .computeIfAbsent( + tableId, + k -> + new ArrayList<>(config.getDEVICE_NUMBER() / config.getIoTDB_TABLE_NUMBER() + 1)) + .add(deviceId); + } + // By using tableDeviceMap, quickly get the devices contained in each database + for (int tableId = 0; tableId < config.getIoTDB_TABLE_NUMBER(); tableId++) { + // Calculate databaseId from tableId int databaseId = mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); databaseDeviceMap - .computeIfAbsent( - databaseId, - k -> new ArrayList<>(config.getDEVICE_NUMBER() / config.getGROUP_NUMBER() + 1)) - .add(deviceId); + .computeIfAbsent(databaseId, k -> new ArrayList<>()) + .addAll(tableDeviceMap.getOrDefault(tableId, Collections.emptyList())); } } catch (WorkloadException e) { LOGGER.error(e.getMessage()); } - for (List values : databaseDeviceMap.values()) { - deviceIds.addAll(values); - } + databaseDeviceMap.values().forEach(deviceIds::addAll); return deviceIds; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java index 1c6052d40..7d99d05cf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/schema/schemaImpl/GenerateMetaDataSchema.java @@ -51,7 +51,7 @@ public boolean createMetaDataSchema() { config.getDEVICE_NUMBER(), config.getCLIENT_NUMBER()); int deviceId = MetaUtil.getDeviceId(0); // Rearrange device IDs so that adjacent devices are in the same table - List deviceIds = MetaUtil.sortDeviceIdByDatabase(config, LOGGER); + List deviceIds = MetaUtil.sortDeviceId(config, LOGGER); for (int clientId = 0; clientId < config.getCLIENT_NUMBER(); clientId++) { int deviceNumber = deviceDistribution.get(clientId); List deviceSchemaList = new ArrayList<>(); From 553616c1478d5476d4118772aad631e599e93f95 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Wed, 18 Sep 2024 15:36:12 +0800 Subject: [PATCH 18/34] try to resolved conflicts(2) --- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index aad212d4c..9d41c4eb4 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -76,7 +76,7 @@ public class SessionStrategy extends DMLStrategy { static final Config config = ConfigDescriptor.getInstance().getConfig(); private static final Map binaryCache = - new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); + new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(),1.00f); private final Map databaseSessionMap = new HashMap<>(); private final IoTDB iotdb; From b3dc9eb6ca4d8376a805fda7d9a98c8fed5b11d0 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Wed, 18 Sep 2024 15:36:59 +0800 Subject: [PATCH 19/34] try to resolved conflicts(3) --- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 9d41c4eb4..aad212d4c 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -76,7 +76,7 @@ public class SessionStrategy extends DMLStrategy { static final Config config = ConfigDescriptor.getInstance().getConfig(); private static final Map binaryCache = - new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(),1.00f); + new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); private final Map databaseSessionMap = new HashMap<>(); private final IoTDB iotdb; From b658d499be678f5cb26419a8331fa3270e1419fd Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Wed, 18 Sep 2024 21:28:21 +0800 Subject: [PATCH 20/34] Modify the switchDB method --- .../iotdb200/DMLStrategy/SessionStrategy.java | 84 +++++-------------- .../iot/benchmark/iotdb200/IoTDB.java | 9 +- .../ModelStrategy/IoTDBModelStrategy.java | 4 +- .../iotdb200/ModelStrategy/TableStrategy.java | 17 ++-- .../iotdb200/ModelStrategy/TreeStrategy.java | 11 ++- 5 files changed, 46 insertions(+), 79 deletions(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index aad212d4c..2444292b0 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -33,11 +33,9 @@ import cn.edu.tsinghua.iot.benchmark.entity.Sensor; import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.OperationFailException; -import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; import cn.edu.tsinghua.iot.benchmark.iotdb200.IoTDB; import cn.edu.tsinghua.iot.benchmark.iotdb200.utils.IoTDBUtils; import cn.edu.tsinghua.iot.benchmark.measurement.Status; -import cn.edu.tsinghua.iot.benchmark.schema.MetaUtil; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; import cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode; @@ -57,7 +55,6 @@ import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -77,7 +74,7 @@ public class SessionStrategy extends DMLStrategy { private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); - private final Map databaseSessionMap = new HashMap<>(); + private Session session; private final IoTDB iotdb; public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { @@ -87,9 +84,8 @@ public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { for (int i = 0; i < dbConfig.getHOST().size(); i++) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } - Session session = buildSession(hostUrls, null); // default session (databaseName not specified) - databaseSessionMap.put(-1, session); + session = buildSession(hostUrls, null); } @Override @@ -115,10 +111,10 @@ private Status insertOneBatchByTablet(IBatch batch) { service.submit( () -> { try { - Session session = - switchSession( - iotdb.getDeviceIdForSwitchSession(batch), - batch.getDeviceSchema().getGroup()); + iotdb.sessionDBSwitchIfNecessaryImpl( + session, + batch.getDeviceSchema().getDeviceId(), + batch.getDeviceSchema().getGroup()); iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); @@ -127,30 +123,6 @@ private Status insertOneBatchByTablet(IBatch batch) { return waitWriteTaskToFinishAndGetStatus(); } - public Session switchSession(int deviceId, String group) { - try { - int tableId = - MetaUtil.mappingId(deviceId, config.getDEVICE_NUMBER(), config.getIoTDB_TABLE_NUMBER()); - int databaseId = - MetaUtil.mappingId(tableId, config.getIoTDB_TABLE_NUMBER(), config.getGROUP_NUMBER()); - if (databaseSessionMap.get(databaseId) == null) { - List hostUrls = new ArrayList<>(dbConfig.getHOST().size()); - for (int i = 0; i < dbConfig.getHOST().size(); i++) { - hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); - } - Session session = buildSession(hostUrls, dbConfig.getDB_NAME() + "_" + group); - session.open(); - databaseSessionMap.put(databaseId, session); - return session; - } else { - return databaseSessionMap.get(databaseId); - } - } catch (WorkloadException | IoTDBConnectionException e) { - LOGGER.error(e.getMessage(), e); - return null; - } - } - private Tablet genTablet(IBatch batch) { List schemaList = new ArrayList<>(); List columnTypes = new ArrayList<>(); @@ -251,13 +223,9 @@ private Status insertOneBatchByRecord(IBatch batch, String deviceId) { List recordDataValue = convertTypeForBLOB(record, dataTypes); try { if (config.isVECTOR()) { - databaseSessionMap - .get(-1) - .insertAlignedRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); + session.insertAlignedRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } else { - databaseSessionMap - .get(-1) - .insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); + session.insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } } catch (IoTDBConnectionException | StatementExecutionException e) { failRecord++; @@ -302,14 +270,10 @@ private Status insertOneBatchByRecords(IBatch batch, String deviceId) { () -> { try { if (config.isVECTOR()) { - databaseSessionMap - .get(-1) - .insertAlignedRecords( - deviceIds, times, measurementsList, typesList, valuesList); + session.insertAlignedRecords( + deviceIds, times, measurementsList, typesList, valuesList); } else { - databaseSessionMap - .get(-1) - .insertRecords(deviceIds, times, measurementsList, typesList, valuesList); + session.insertRecords(deviceIds, times, measurementsList, typesList, valuesList); } } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); @@ -346,8 +310,7 @@ public long executeQueryAndGetStatusImpl( service.submit( () -> { try { - SessionDataSet sessionDataSet = - databaseSessionMap.get(-1).executeQueryStatement(executeSQL); + SessionDataSet sessionDataSet = session.executeQueryStatement(executeSQL); if (config.isIS_COMPARISON()) { while (sessionDataSet.hasNext()) { RowRecord rowRecord = sessionDataSet.next(); @@ -407,7 +370,7 @@ public long executeQueryAndGetStatusImpl( public List verificationQueryImpl(String sql, Map> recordMap) throws IoTDBConnectionException, StatementExecutionException { int point = 0, line = 0; - try (SessionDataSet sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(sql)) { + try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql)) { while (sessionDataSet.hasNext()) { RowRecord rowRecord = sessionDataSet.next(); // The table model and the tree model obtain time differently @@ -431,7 +394,7 @@ public List verificationQueryImpl(String sql, Map> r @Override public List> deviceQueryImpl(String sql) throws Exception { List> result = new ArrayList<>(); - try (SessionDataSet sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(sql)) { + try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql)) { while (sessionDataSet.hasNext()) { List line = new ArrayList<>(); RowRecord rowRecord = sessionDataSet.next(); @@ -453,20 +416,19 @@ public DeviceSummary deviceSummary( int totalLineNumber = 0; long minTimeStamp, maxTimeStamp; try { - SessionDataSet sessionDataSet = - databaseSessionMap.get(-1).executeQueryStatement(totalLineNumberSql); + SessionDataSet sessionDataSet = session.executeQueryStatement(totalLineNumberSql); while (sessionDataSet.hasNext()) { sessionDataSet.next(); totalLineNumber++; } sessionDataSet.close(); - sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(maxTimestampSql); + sessionDataSet = session.executeQueryStatement(maxTimestampSql); RowRecord rowRecord = sessionDataSet.next(); maxTimeStamp = iotdb.getTimestamp(rowRecord); sessionDataSet.close(); - sessionDataSet = databaseSessionMap.get(-1).executeQueryStatement(minTimestampSql); + sessionDataSet = session.executeQueryStatement(minTimestampSql); rowRecord = sessionDataSet.next(); minTimeStamp = iotdb.getTimestamp(rowRecord); sessionDataSet.close(); @@ -493,9 +455,9 @@ Status waitWriteTaskToFinishAndGetStatus() { public void init() { try { if (config.isENABLE_THRIFT_COMPRESSION()) { - databaseSessionMap.get(-1).open(true); + session.open(true); } else { - databaseSessionMap.get(-1).open(); + session.open(); } this.service = Executors.newSingleThreadExecutor(); } catch (IoTDBConnectionException e) { @@ -506,7 +468,7 @@ public void init() { @Override public void cleanup() { try { - iotdb.sessionCleanupImpl(databaseSessionMap.get(-1)); + iotdb.sessionCleanupImpl(session); } catch (IoTDBConnectionException e) { LOGGER.warn("Failed to connect to IoTDB:" + e.getMessage()); } catch (StatementExecutionException e) { @@ -517,11 +479,7 @@ public void cleanup() { @Override public void close() throws TsdbException { try { - if (!databaseSessionMap.isEmpty()) { - for (Session session : databaseSessionMap.values()) { - session.close(); - } - } + session.close(); service.shutdown(); } catch (IoTDBConnectionException ioTDBConnectionException) { LOGGER.error("Failed to close session."); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index 3117fbd0c..4fdf527f9 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -662,6 +662,11 @@ public Tablet createTablet( return modelStrategy.createTablet(insertTargetName, schemas, columnTypes, maxRowNumber); } + public void sessionDBSwitchIfNecessaryImpl(Session session, int deviceId, String group) + throws IoTDBConnectionException, StatementExecutionException { + modelStrategy.sessionDBSwitchIfNecessaryImpl(session, deviceId, group); + } + public void sessionCleanupImpl(Session session) throws IoTDBConnectionException, StatementExecutionException { modelStrategy.sessionCleanupImpl(session); @@ -672,10 +677,6 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic modelStrategy.sessionInsertImpl(session, tablet, deviceSchema); } - public int getDeviceIdForSwitchSession(IBatch batch) { - return modelStrategy.getDeviceIdForSwitchSession(batch); - } - public void addIDColumnIfNecessary( List columnTypes, List sensors, IBatch batch) { modelStrategy.addIDColumnIfNecessary(columnTypes, sensors, batch); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java index 858cb3619..7a1a3080a 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java @@ -126,9 +126,11 @@ public abstract void sessionInsertImpl(Session session, Tablet tablet, DeviceSch public abstract void sessionCleanupImpl(Session session) throws IoTDBConnectionException, StatementExecutionException; - public abstract int getDeviceIdForSwitchSession(IBatch batch); // endregion + public abstract void sessionDBSwitchIfNecessaryImpl(Session session, int deviceId, String group) + throws StatementExecutionException, IoTDBConnectionException; + public abstract Logger getLogger(); public void handleRegisterException(Exception e) throws TsdbException { diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index db03cc8c0..9285b45f2 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -45,6 +45,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.CyclicBarrier; @@ -336,6 +337,17 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic session.insertRelationalTablet(tablet); } + @Override + public void sessionDBSwitchIfNecessaryImpl(Session session, int deviceId, String group) + throws StatementExecutionException, IoTDBConnectionException { + String databaseName = dbConfig.getDB_NAME() + "_" + group; + if (!Objects.equals(session.getDatabase(), databaseName)) { + StringBuilder sql = new StringBuilder(); + sql.append("use ").append(databaseName); + session.executeNonQueryStatement(sql.toString()); + } + } + @Override public void sessionCleanupImpl(Session session) throws IoTDBConnectionException, StatementExecutionException { @@ -349,11 +361,6 @@ public void sessionCleanupImpl(Session session) } } - @Override - public int getDeviceIdForSwitchSession(IBatch batch) { - return batch.getDeviceSchema().getDeviceId(); - } - // endregion @Override diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java index 1d1dfef23..f519a4d4e 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java @@ -342,6 +342,11 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic } } + @Override + public void sessionDBSwitchIfNecessaryImpl(Session session, int deviceId, String group) { + // do nothing + } + @Override public void sessionCleanupImpl(Session session) { try { @@ -355,12 +360,6 @@ public void sessionCleanupImpl(Session session) { } } - @Override - public int getDeviceIdForSwitchSession(IBatch batch) { - // -1 is the default session in databaseSessionMap - return -1; - } - // endregion @Override From 9c90bff8c14b1e1a438eb793af577eb873c0f7ba Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 09:30:39 +0800 Subject: [PATCH 21/34] Modify the switchDB method(2) --- .../benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 8 +++----- .../cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java | 5 ----- .../iotdb200/ModelStrategy/IoTDBModelStrategy.java | 3 --- .../benchmark/iotdb200/ModelStrategy/TableStrategy.java | 9 ++------- .../benchmark/iotdb200/ModelStrategy/TreeStrategy.java | 5 ----- 5 files changed, 5 insertions(+), 25 deletions(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 2444292b0..7da2781d2 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -111,10 +111,6 @@ private Status insertOneBatchByTablet(IBatch batch) { service.submit( () -> { try { - iotdb.sessionDBSwitchIfNecessaryImpl( - session, - batch.getDeviceSchema().getDeviceId(), - batch.getDeviceSchema().getGroup()); iotdb.sessionInsertImpl(session, tablet, batch.getDeviceSchema()); } catch (IoTDBConnectionException | StatementExecutionException e) { throw new OperationFailException(e); @@ -479,7 +475,9 @@ public void cleanup() { @Override public void close() throws TsdbException { try { - session.close(); + if (session != null) { + session.close(); + } service.shutdown(); } catch (IoTDBConnectionException ioTDBConnectionException) { LOGGER.error("Failed to close session."); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java index 4fdf527f9..38181f85b 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/IoTDB.java @@ -662,11 +662,6 @@ public Tablet createTablet( return modelStrategy.createTablet(insertTargetName, schemas, columnTypes, maxRowNumber); } - public void sessionDBSwitchIfNecessaryImpl(Session session, int deviceId, String group) - throws IoTDBConnectionException, StatementExecutionException { - modelStrategy.sessionDBSwitchIfNecessaryImpl(session, deviceId, group); - } - public void sessionCleanupImpl(Session session) throws IoTDBConnectionException, StatementExecutionException { modelStrategy.sessionCleanupImpl(session); diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java index 7a1a3080a..f3c289a48 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/IoTDBModelStrategy.java @@ -128,9 +128,6 @@ public abstract void sessionCleanupImpl(Session session) // endregion - public abstract void sessionDBSwitchIfNecessaryImpl(Session session, int deviceId, String group) - throws StatementExecutionException, IoTDBConnectionException; - public abstract Logger getLogger(); public void handleRegisterException(Exception e) throws TsdbException { diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index 9285b45f2..f5828f2f5 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -334,18 +334,13 @@ public void deleteIDColumnIfNecessary( @Override public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) throws IoTDBConnectionException, StatementExecutionException { - session.insertRelationalTablet(tablet); - } - - @Override - public void sessionDBSwitchIfNecessaryImpl(Session session, int deviceId, String group) - throws StatementExecutionException, IoTDBConnectionException { - String databaseName = dbConfig.getDB_NAME() + "_" + group; + String databaseName = dbConfig.getDB_NAME() + "_" + deviceSchema.getGroup(); if (!Objects.equals(session.getDatabase(), databaseName)) { StringBuilder sql = new StringBuilder(); sql.append("use ").append(databaseName); session.executeNonQueryStatement(sql.toString()); } + session.insertRelationalTablet(tablet); } @Override diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java index f519a4d4e..606c96d48 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TreeStrategy.java @@ -342,11 +342,6 @@ public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema devic } } - @Override - public void sessionDBSwitchIfNecessaryImpl(Session session, int deviceId, String group) { - // do nothing - } - @Override public void sessionCleanupImpl(Session session) { try { From 01c378a1129976ad3a38f719bca5f44e4edfb3ff Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 09:34:35 +0800 Subject: [PATCH 22/34] Modify the switchDB method(3) --- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 7da2781d2..1a7eb58c1 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -74,7 +74,7 @@ public class SessionStrategy extends DMLStrategy { private static final Map binaryCache = new ConcurrentHashMap<>(config.getWORKLOAD_BUFFER_SIZE(), 1.00f); - private Session session; + private final Session session; private final IoTDB iotdb; public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { From 7a9d27b16e9d7cd612597a4731014198dc38845c Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 09:37:49 +0800 Subject: [PATCH 23/34] Modify the switchDB method(4) --- .../iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java | 3 +-- .../iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java index 4d7368f2a..59e1192cf 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/DMLStrategy.java @@ -70,7 +70,7 @@ public abstract DeviceSummary deviceSummary( public abstract void close() throws TsdbException; - protected Session buildSession(List hostUrls, String databaseName) { + protected Session buildSession(List hostUrls) { return new Session.Builder() .nodeUrls(hostUrls) .username(dbConfig.getUSERNAME()) @@ -78,7 +78,6 @@ protected Session buildSession(List hostUrls, String databaseName) { .enableRedirection(true) .version(Version.V_1_0) .sqlDialect(config.getIoTDB_DIALECT_MODE().name()) - .database(databaseName) .build(); } } diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java index 1a7eb58c1..fe40087d2 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/DMLStrategy/SessionStrategy.java @@ -85,7 +85,7 @@ public SessionStrategy(IoTDB iotdb, DBConfig dbConfig) { hostUrls.add(dbConfig.getHOST().get(i) + ":" + dbConfig.getPORT().get(i)); } // default session (databaseName not specified) - session = buildSession(hostUrls, null); + session = buildSession(hostUrls); } @Override From b49a1a7292496616bb50e7cf3a91b0d6803ddbc5 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 10:56:18 +0800 Subject: [PATCH 24/34] tableMode must ensure that a client only writes to one table --- .../edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index d04b79f26..3540bb9cf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -686,6 +686,14 @@ private boolean checkDeviceNumPerWrite() { LOGGER.error("DEVICE_NUM_PER_WRITE must be greater than 0"); return false; } + // tableMode + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE + && config.getCLIENT_NUMBER() % config.getIoTDB_TABLE_NUMBER() != 0) { + LOGGER.error( + "TableMode must ensure that a client only writes to one table.\n" + + "please make DEVICE_NUMBER % IoTDB_TABLE_NUMBER == 0"); + return false; + } if (dnw == 1) { return true; } From 9c19bd870d0e9fc33a0cca0ee1517e94545fd8d0 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 11:02:07 +0800 Subject: [PATCH 25/34] Improve the annotation --- .../edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 3540bb9cf..fe99368a5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -688,10 +688,10 @@ private boolean checkDeviceNumPerWrite() { } // tableMode if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE - && config.getCLIENT_NUMBER() % config.getIoTDB_TABLE_NUMBER() != 0) { + && config.getCLIENT_NUMBER() % config.getGROUP_NUMBER() != 0) { LOGGER.error( - "TableMode must ensure that a client only writes to one table.\n" - + "please make DEVICE_NUMBER % IoTDB_TABLE_NUMBER == 0"); + "TableMode must ensure that a client only writes to one database. Therefore, a client only switches database once.\n" + + "please make DEVICE_NUMBER % GROUP_NUMBER == 0"); return false; } if (dnw == 1) { From 55be6305c4ca7287f01540abe33577976238de24 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 11:02:58 +0800 Subject: [PATCH 26/34] Improve the annotation --- .../cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index fe99368a5..ab562a9ad 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -691,7 +691,7 @@ private boolean checkDeviceNumPerWrite() { && config.getCLIENT_NUMBER() % config.getGROUP_NUMBER() != 0) { LOGGER.error( "TableMode must ensure that a client only writes to one database. Therefore, a client only switches database once.\n" - + "please make DEVICE_NUMBER % GROUP_NUMBER == 0"); + + "please make CLIENT_NUMBER % GROUP_NUMBER == 0"); return false; } if (dnw == 1) { From 20f793d68ce5ddd16a875ae96902dca946482a99 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 11:19:50 +0800 Subject: [PATCH 27/34] Improve the annotation --- .../edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index ab562a9ad..9c0799ed7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -688,10 +688,10 @@ private boolean checkDeviceNumPerWrite() { } // tableMode if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE - && config.getCLIENT_NUMBER() % config.getGROUP_NUMBER() != 0) { + && config.getCLIENT_NUMBER() % config.getIoTDB_TABLE_NUMBER() != 0) { LOGGER.error( - "TableMode must ensure that a client only writes to one database. Therefore, a client only switches database once.\n" - + "please make CLIENT_NUMBER % GROUP_NUMBER == 0"); + "TableMode must ensure that a client only writes to one table. Therefore, a client only switches database once.\n" + + "please make CLIENT_NUMBER % IoTDB_TABLE_NUMBER == 0"); return false; } if (dnw == 1) { From f788bbe6664eb140b3b9d5cf4495c9e451284579 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 11:51:24 +0800 Subject: [PATCH 28/34] When encountering a warn log, the operation stops --- .../iot/benchmark/conf/ConfigDescriptor.java | 34 +++++++++++-------- .../iotdb200/ModelStrategy/TableStrategy.java | 6 ++-- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 9c0799ed7..38076f46c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -325,20 +325,6 @@ private void loadProps() { Integer.parseInt( properties.getProperty("IoTDB_TABLE_NUMBER", config.getIoTDB_TABLE_NUMBER() + ""))); - if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { - if (config.getGROUP_NUMBER() > config.getIoTDB_TABLE_NUMBER() - || config.getIoTDB_TABLE_NUMBER() > config.getDEVICE_NUMBER()) { - LOGGER.warn( - "Please follow this rule to adjust the parameters: device number >= table number >= database number. Otherwise, device number = table number = database number"); - } - } else { - config.setIoTDB_TABLE_NUMBER(config.getGROUP_NUMBER()); - if (config.getGROUP_NUMBER() > config.getDEVICE_NUMBER()) { - LOGGER.warn( - "Please follow this rule to adjust the parameters: device number >= database number. Otherwise, the total number of databases created is equal to the number of devices"); - } - } - config.setIOTDB_SESSION_POOL_SIZE( Integer.parseInt( properties.getProperty( @@ -638,6 +624,7 @@ private boolean checkConfig() { LOGGER.error("Client number can't be zero"); result = false; } + result &= checkIoTDBTableNumberAndDatabaseNumber(); result &= checkDeviceNumPerWrite(); result &= checkTag(); if (!commonlyUseDB()) { @@ -680,6 +667,25 @@ private boolean checkOperationProportion() { return true; } + private boolean checkIoTDBTableNumberAndDatabaseNumber() { + if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { + if (config.getGROUP_NUMBER() > config.getIoTDB_TABLE_NUMBER() + || config.getIoTDB_TABLE_NUMBER() > config.getDEVICE_NUMBER()) { + LOGGER.warn( + "Please follow this rule to adjust the parameters: device number >= table number >= database number. Otherwise, device number = table number = database number"); + return false; + } + } else { + config.setIoTDB_TABLE_NUMBER(config.getGROUP_NUMBER()); + if (config.getGROUP_NUMBER() > config.getDEVICE_NUMBER()) { + LOGGER.warn( + "Please follow this rule to adjust the parameters: device number >= database number. Otherwise, the total number of databases created is equal to the number of devices"); + return false; + } + } + return true; + } + private boolean checkDeviceNumPerWrite() { final int dnw = config.getDEVICE_NUM_PER_WRITE(); if (dnw <= 0) { diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index f5828f2f5..dcd5cf4c5 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -45,7 +45,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.concurrent.CyclicBarrier; @@ -334,10 +333,9 @@ public void deleteIDColumnIfNecessary( @Override public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) throws IoTDBConnectionException, StatementExecutionException { - String databaseName = dbConfig.getDB_NAME() + "_" + deviceSchema.getGroup(); - if (!Objects.equals(session.getDatabase(), databaseName)) { + if (session.getDatabase() == null) { StringBuilder sql = new StringBuilder(); - sql.append("use ").append(databaseName); + sql.append("use ").append(dbConfig.getDB_NAME()).append("_").append(deviceSchema.getGroup()); session.executeNonQueryStatement(sql.toString()); } session.insertRelationalTablet(tablet); From 503426f8e0e5dd3e2bc903ba1c293e0debe5288b Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 11:55:28 +0800 Subject: [PATCH 29/34] When encountering a warn log, the operation stops --- .../cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 38076f46c..66d32afc1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -624,7 +624,7 @@ private boolean checkConfig() { LOGGER.error("Client number can't be zero"); result = false; } - result &= checkIoTDBTableNumberAndDatabaseNumber(); + result &= checkDatabaseTableDeviceRelationship(); result &= checkDeviceNumPerWrite(); result &= checkTag(); if (!commonlyUseDB()) { @@ -667,7 +667,7 @@ private boolean checkOperationProportion() { return true; } - private boolean checkIoTDBTableNumberAndDatabaseNumber() { + private boolean checkDatabaseTableDeviceRelationship() { if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { if (config.getGROUP_NUMBER() > config.getIoTDB_TABLE_NUMBER() || config.getIoTDB_TABLE_NUMBER() > config.getDEVICE_NUMBER()) { From 983d2b205210f3671a815009e53509142e152510 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 12:04:38 +0800 Subject: [PATCH 30/34] Using Optional to check if it is empty --- .../iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index dcd5cf4c5..3767e8d02 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -45,6 +45,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CyclicBarrier; @@ -333,7 +334,7 @@ public void deleteIDColumnIfNecessary( @Override public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) throws IoTDBConnectionException, StatementExecutionException { - if (session.getDatabase() == null) { + if (!(Optional.ofNullable(session.getDatabase()).isPresent())) { StringBuilder sql = new StringBuilder(); sql.append("use ").append(dbConfig.getDB_NAME()).append("_").append(deviceSchema.getGroup()); session.executeNonQueryStatement(sql.toString()); From 55b3ba033ab7f202e1d488c85e9c8f05eddceef6 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 12:24:34 +0800 Subject: [PATCH 31/34] Improve the annotation --- .../cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 66d32afc1..52f9baaee 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -670,9 +670,9 @@ private boolean checkOperationProportion() { private boolean checkDatabaseTableDeviceRelationship() { if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { if (config.getGROUP_NUMBER() > config.getIoTDB_TABLE_NUMBER() - || config.getIoTDB_TABLE_NUMBER() > config.getDEVICE_NUMBER()) { + || config.getIoTDB_TABLE_NUMBER() % config.getDEVICE_NUMBER() == 0) { LOGGER.warn( - "Please follow this rule to adjust the parameters: device number >= table number >= database number. Otherwise, device number = table number = database number"); + "Please follow this rule to adjust the parameters: device number >= table number >= database number. And, the device number must be an integer multiple of the table number. "); return false; } } else { From 639e8b93187d5cb6271409a4498a90750770ca74 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 13:01:05 +0800 Subject: [PATCH 32/34] Improve the annotation --- .../edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index 52f9baaee..900f71b5d 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -669,10 +669,10 @@ private boolean checkOperationProportion() { private boolean checkDatabaseTableDeviceRelationship() { if (config.getIoTDB_DIALECT_MODE() == SQLDialect.TABLE) { - if (config.getGROUP_NUMBER() > config.getIoTDB_TABLE_NUMBER() - || config.getIoTDB_TABLE_NUMBER() % config.getDEVICE_NUMBER() == 0) { + if (config.getIoTDB_TABLE_NUMBER() % config.getGROUP_NUMBER() != 0 + || config.getDEVICE_NUMBER() % config.getIoTDB_TABLE_NUMBER() != 0) { LOGGER.warn( - "Please follow this rule to adjust the parameters: device number >= table number >= database number. And, the device number must be an integer multiple of the table number. "); + "Please follow this rule to adjust the parameters: \n 1.The table number must be an integer multiple of the group number.\n 2.The device number must be an integer multiple of the table number. "); return false; } } else { From bed077f8c8cffdbd3867e0442e4987cc70992436 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 14:27:37 +0800 Subject: [PATCH 33/34] Improve the annotation --- configuration/conf/config.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index ae12be694..2693f1031 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -7,7 +7,7 @@ # DB_SWITCH=IoTDB-200-SESSION_BY_TABLET # IoTDB-2.0需指定sql_dialect, 并且一个IoTDB只能指定一种。支持tree、table,默认值为tree。 -# sql_dialect等于table时，要满足：device数量 >= table数量 >= database数量 +# sql_dialect等于table时，要满足：device数为table数的整数倍，table数为database数的整数倍 # sql_dialect等于tree时，要满足：device数量 >= database数量 # IoTDB_DIALECT_MODE=tree @@ -157,6 +157,7 @@ # IS_CLIENT_BIND=true # 客户端总数 +# iotdb表模型下，client数为table数的整数倍 # CLIENT_NUMBER=20 ############## 被测系统为IoTDB时扩展参数 ################## From 8725674d7aed42fe6c187e12d2a99a2350db4612 Mon Sep 17 00:00:00 2001 From: Yangyuming <2822758820@qq.com> Date: Thu, 19 Sep 2024 15:11:34 +0800 Subject: [PATCH 34/34] null --- .../iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java index 3767e8d02..dcd5cf4c5 100644 --- a/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java +++ b/iotdb-2.0/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb200/ModelStrategy/TableStrategy.java @@ -45,7 +45,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.CyclicBarrier; @@ -334,7 +333,7 @@ public void deleteIDColumnIfNecessary( @Override public void sessionInsertImpl(Session session, Tablet tablet, DeviceSchema deviceSchema) throws IoTDBConnectionException, StatementExecutionException { - if (!(Optional.ofNullable(session.getDatabase()).isPresent())) { + if (session.getDatabase() == null) { StringBuilder sql = new StringBuilder(); sql.append("use ").append(dbConfig.getDB_NAME()).append("_").append(deviceSchema.getGroup()); session.executeNonQueryStatement(sql.toString());