diff --git a/configuration/conf/config.properties b/configuration/conf/config.properties index 6b7f4ae07..3d9e40b77 100644 --- a/configuration/conf/config.properties +++ b/configuration/conf/config.properties @@ -230,8 +230,8 @@ # 浮点数小数位数 # DOUBLE_LENGTH=2 -# 插入数据的数据类型的比例，BOOLEAN:INT32:INT64:FLOAT:DOUBLE:TEXT -# INSERT_DATATYPE_PROPORTION=1:1:1:1:1:1 +# 插入数据的数据类型的比例，BOOLEAN:INT32:INT64:FLOAT:DOUBLE:TEXT:STRING:BLOB:TIMESTAMP:DATE +# INSERT_DATATYPE_PROPORTION=1:1:1:1:1:1:0:0:0:0 ################ IoTDB相关元数据参数 ##################### # 压缩方式 UNCOMPRESSED | SNAPPY | LZ4 (仅对IoTDB有效) @@ -255,6 +255,17 @@ # TEXT: PLAIN/DICTIONARY 等，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致 # ENCODING_TEXT=DICTIONARY +# STRING: PLAIN 暂不支持DICTIONARY，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致 +# ENCODING_STRING=PLAIN + +# BLOB: PLAIN 暂不支持DICTIONARY，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致 +# ENCODING_BLOB=PLAIN + +# TIMESTAMP: PLAIN/RLE/TS_2DIFF/GORILLA/ZIGZAG/CHIMP/SPRINTZ/RLBE，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致 +# ENCODING_TIMESTAMP=TS_2DIFF + +# DATE: PLAIN/RLE/TS_2DIFF/GORILLA/ZIGZAG/CHIMP/SPRINTZ/RLBE，与对应 tsfile 版本中的 CompressionType 枚举类型保持一致 +# ENCODING_DATE=TS_2DIFF ################ 真实数据集：测试数据 ##################### # 如下两个参数，当且仅当BENCHMARK_MODE = verificationWriteMode 和 verificationQueryMode 时生效 # 数据文件地址 diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java index eb4f9c89d..d38ecd1a6 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java @@ -41,7 +41,8 @@ public class Config { private static final Logger LOGGER = LoggerFactory.getLogger(Config.class); - + /** The total number of data types supported by the benchmark */ + private final int typeNumber = 10; // 初始化 // 初始化：清理数据 /** Whether to clear old data before test */ @@ -126,10 +127,10 @@ public class Config { /** The length of double */ private int DOUBLE_LENGTH = 2; /** - * 插入数据的比例 Data Type, D1:D2:D3:D4:D5:D6 D1: BOOLEAN D2: INT32 D3: INT64 D4: FLOAT D5: DOUBLE D6: - * TEXT + * 插入数据的比例 Data Type, D1:D2:D3:D4:D5:D6:D7:D8:D9:D9:D10 D1: BOOLEAN D2: INT32 D3: INT64 D4: FLOAT + * D5: DOUBLE D6:TEXT D7: STRING D8: BLOB D9: TIMESTAMP D10: DATE */ - private String INSERT_DATATYPE_PROPORTION = "1:1:1:1:1:1"; + private String INSERT_DATATYPE_PROPORTION = "1:1:1:1:1:1:0:0:0:0"; /** The compress of data */ private String COMPRESSOR = "LZ4"; @@ -145,6 +146,14 @@ public class Config { private String ENCODING_DOUBLE = "GORILLA"; /** The encoding of text */ private String ENCODING_TEXT = "DICTIONARY"; + /** The encoding of string */ + private String ENCODING_STRING = "PLAIN"; + /** The encoding of blob */ + private String ENCODING_BLOB = "PLAIN"; + /** The encoding of timestamp */ + private String ENCODING_TIMESTAMP = "TS_2DIFF"; + /** The encoding of date */ + private String ENCODING_DATE = "TS_2DIFF"; // 测试数据相关参数 @@ -511,7 +520,6 @@ public void initSensorFunction() { /** According to the number of sensors, initialize the sensor number */ void initSensorCodes() { - int typeNumber = 6; double[] probabilities = generateProbabilities(typeNumber); if (probabilities.length == 0) { return; @@ -529,14 +537,12 @@ void initSensorCodes() { } } - /** Generate Probabilities according to proportion(e.g. 1:1:1:1:1:1) */ + /** Generate Probabilities according to proportion(e.g. 1:1:1:1:1:1:0:0:0:0) */ private double[] generateProbabilities(int typeNumber) { // Probabilities for Types double[] probabilities = new double[typeNumber + 1]; // Origin proportion array double[] proportions = new double[typeNumber]; - LOGGER.info( - "Init SensorTypes: BOOLEAN:INT32:INT64:FLOAT:DOUBLE:TEXT= {}", INSERT_DATATYPE_PROPORTION); String[] split = INSERT_DATATYPE_PROPORTION.split(":"); if (split.length != typeNumber) { @@ -560,6 +566,10 @@ private double[] generateProbabilities(int typeNumber) { return probabilities; } + public int getTypeNumber() { + return typeNumber; + } + public String getREST_AUTHORIZATION() { return REST_AUTHORIZATION; } @@ -801,6 +811,38 @@ public void setENCODING_TEXT(String ENCODING_TEXT) { this.ENCODING_TEXT = ENCODING_TEXT; } + public String getENCODING_STRING() { + return ENCODING_STRING; + } + + public void setENCODING_STRING(String ENCODING_STRING) { + this.ENCODING_STRING = ENCODING_STRING; + } + + public String getENCODING_BLOB() { + return ENCODING_BLOB; + } + + public void setENCODING_BLOB(String ENCODING_BLOB) { + this.ENCODING_BLOB = ENCODING_BLOB; + } + + public String getENCODING_TIMESTAMP() { + return ENCODING_TIMESTAMP; + } + + public void setENCODING_TIMESTAMP(String ENCODING_TIMESTAMP) { + this.ENCODING_TIMESTAMP = ENCODING_TIMESTAMP; + } + + public String getENCODING_DATE() { + return ENCODING_DATE; + } + + public void setENCODING_DATE(String ENCODING_DATE) { + this.ENCODING_DATE = ENCODING_DATE; + } + public String getFILE_PATH() { return FILE_PATH; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java index f3b4e9a38..76dd73318 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java @@ -198,7 +198,6 @@ private void loadProps() { config.setINSERT_DATATYPE_PROPORTION( properties.getProperty( "INSERT_DATATYPE_PROPORTION", config.getINSERT_DATATYPE_PROPORTION())); - config.setCOMPRESSOR(properties.getProperty("COMPRESSOR", config.getCOMPRESSOR())); config.setENCODING_BOOLEAN( properties.getProperty("ENCODING_BOOLEAN", config.getENCODING_BOOLEAN())); @@ -211,6 +210,12 @@ private void loadProps() { config.setENCODING_DOUBLE( properties.getProperty("ENCODING_DOUBLE", config.getENCODING_DOUBLE())); config.setENCODING_TEXT(properties.getProperty("ENCODING_TEXT", config.getENCODING_TEXT())); + config.setENCODING_STRING( + properties.getProperty("ENCODING_STRING", config.getENCODING_STRING())); + config.setENCODING_BLOB(properties.getProperty("ENCODING_BLOB", config.getENCODING_BLOB())); + config.setENCODING_TIMESTAMP( + properties.getProperty("ENCODING_TIMESTAMP", config.getENCODING_TIMESTAMP())); + config.setENCODING_DATE(properties.getProperty("ENCODING_DATE", config.getENCODING_DATE())); config.setFILE_PATH(properties.getProperty("FILE_PATH", config.getFILE_PATH())); config.setBIG_BATCH_SIZE( @@ -601,6 +606,7 @@ private boolean checkConfig() { default: break; } + result &= checkInsertDataTypeProportion(); result &= checkOperationProportion(); if (config.getCLIENT_NUMBER() == 0) { LOGGER.error("Client number can't be zero"); @@ -744,6 +750,25 @@ private boolean checkDatabaseVerification(DBConfig dbConfig) { return result; } + // Only iotdb supports STRING BLOB TIMESTAMP DATE + protected boolean checkInsertDataTypeProportion() { + DBType dbType = config.getDbConfig().getDB_SWITCH().getType(); + String[] splits = config.getINSERT_DATATYPE_PROPORTION().split(":"); + if (dbType != DBType.IoTDB && dbType != DBType.DoubleIoTDB) { + // When not iotdb, the last four digits of the data ratio must be 0 + for (int i = config.getTypeNumber() - 4; i < splits.length; i++) { + if (!splits[i].equals("0")) { + LOGGER.warn("INSERT_DATATYPE_PROPORTION error, this database do not support those type."); + return false; + } + } + } + LOGGER.info( + "Init SensorTypes: BOOLEAN:INT32:INT64:FLOAT:DOUBLE:TEXT:STRING:BLOB:TIMESTAMP:DATE= {}", + config.getINSERT_DATATYPE_PROPORTION()); + return true; + } + /** * Compare whether each field of the two objects is the same. This function is not used in the * normal operation of the benchmark， but is used when adding parameters or when the default diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/SensorType.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/SensorType.java index 661e2dc07..6c65880ac 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/SensorType.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/entity/enums/SensorType.java @@ -25,7 +25,11 @@ public enum SensorType { INT64("INT64"), FLOAT("FLOAT"), DOUBLE("DOUBLE"), - TEXT("TEXT"); + TEXT("TEXT"), + STRING("STRING"), + BLOB("BLOB"), + TIMESTAMP("TIMESTAMP"), + DATE("DATE"); public String name; diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java index 3db50d6a3..23fbf332c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/mode/TestWithDefaultPathMode.java @@ -26,6 +26,8 @@ import cn.edu.tsinghua.iot.benchmark.measurement.persistence.PersistenceFactory; import cn.edu.tsinghua.iot.benchmark.measurement.persistence.TestDataPersistence; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; @@ -33,6 +35,7 @@ public class TestWithDefaultPathMode extends BaseMode { + private static final Logger LOGGER = LoggerFactory.getLogger(TestWithDefaultPathMode.class); private static final Config config = ConfigDescriptor.getInstance().getConfig(); @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java index c90019517..2053a7108 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateDataWorkLoad.java @@ -23,7 +23,6 @@ import cn.edu.tsinghua.iot.benchmark.distribution.PoissonDistribution; import cn.edu.tsinghua.iot.benchmark.distribution.ProbTool; import cn.edu.tsinghua.iot.benchmark.entity.Sensor; -import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType; import cn.edu.tsinghua.iot.benchmark.exception.WorkloadException; import cn.edu.tsinghua.iot.benchmark.function.Function; import cn.edu.tsinghua.iot.benchmark.function.FunctionParam; @@ -31,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.LocalDate; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -152,37 +152,40 @@ private static Object[][] initWorkloadValues() { // periodic long currentTimestamp = getCurrentTimestampStatic(i); Object value; - if (sensor.getSensorType() == SensorType.TEXT) { - // TEXT case: pick STRING_LENGTH chars to be a String for insertion. - StringBuffer builder = new StringBuffer(config.getSTRING_LENGTH()); - for (int k = 0; k < config.getSTRING_LENGTH(); k++) { - builder.append(CHAR_TABLE.charAt(dataRandom.nextInt(CHAR_TABLE.length()))); - } - value = builder.toString(); - } else { - // not TEXT case - FunctionParam param = config.getSENSOR_FUNCTION().get(sensor.getName()); - Number number = Function.getValueByFunctionIdAndParam(param, currentTimestamp); - switch (sensor.getSensorType()) { - case BOOLEAN: - value = number.floatValue() > ((param.getMax() + param.getMin()) / 2); - break; - case INT32: - value = number.intValue(); - break; - case INT64: - value = number.longValue(); - break; - case FLOAT: - value = number.floatValue(); - break; - case DOUBLE: - value = Math.round(number.doubleValue() * ratio) / ratio; - break; - default: - value = null; - break; - } + FunctionParam param = config.getSENSOR_FUNCTION().get(sensor.getName()); + Number number = Function.getValueByFunctionIdAndParam(param, currentTimestamp); + switch (sensor.getSensorType()) { + case BOOLEAN: + value = number.floatValue() > ((param.getMax() + param.getMin()) / 2); + break; + case INT32: + value = number.intValue(); + break; + case INT64: + case TIMESTAMP: + value = number.longValue(); + break; + case FLOAT: + value = number.floatValue(); + break; + case DOUBLE: + value = Math.round(number.doubleValue() * ratio) / ratio; + break; + case TEXT: + case STRING: + case BLOB: + StringBuffer builder = new StringBuffer(config.getSTRING_LENGTH()); + for (int k = 0; k < config.getSTRING_LENGTH(); k++) { + builder.append(CHAR_TABLE.charAt(dataRandom.nextInt(CHAR_TABLE.length()))); + } + value = builder.toString(); + break; + case DATE: + value = LocalDate.ofEpochDay(number.intValue()); + break; + default: + throw new UnsupportedOperationException( + sensor.getSensorType() + ": This data type is not supported."); } workloadValues[sensorIndex][i] = value; } diff --git a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java index a214f9327..a348a883c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java +++ b/core/src/main/java/cn/edu/tsinghua/iot/benchmark/workload/GenerateQueryWorkLoad.java @@ -226,7 +226,10 @@ private List getQueryDeviceSchemaList(boolean typeAllow) throws Wo Sensor sensor = sensors.get(sensorId); if (!typeAllow) { SensorType sensorType = sensor.getSensorType(); - if (sensorType == SensorType.BOOLEAN || sensorType == SensorType.TEXT) { + if (sensorType == SensorType.BOOLEAN + || sensorType == SensorType.TEXT + || sensorType == SensorType.STRING + || sensorType == SensorType.BLOB) { continue; } } diff --git a/iotdb-1.3/pom.xml b/iotdb-1.3/pom.xml index 66fac4a2d..214f010fb 100644 --- a/iotdb-1.3/pom.xml +++ b/iotdb-1.3/pom.xml @@ -47,7 +47,7 @@ 1.3.14 - 1.3.1 + 1.3.3-SNAPSHOT 4.12.0 2.10.1 diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IBenchmarkSession.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IBenchmarkSession.java index 11c5eb3e3..955f31302 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IBenchmarkSession.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IBenchmarkSession.java @@ -2,8 +2,9 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.write.record.Tablet; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; import java.util.List; diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ISessionDataSet.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ISessionDataSet.java index a56c64e33..ba097dd9b 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ISessionDataSet.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/ISessionDataSet.java @@ -3,7 +3,8 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.tsfile.read.common.RowRecord; + +import org.apache.tsfile.read.common.RowRecord; public interface ISessionDataSet { RowRecord next() throws IoTDBConnectionException, StatementExecutionException; diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDB.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDB.java index 2c136be79..9af12dfa6 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDB.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDB.java @@ -25,9 +25,6 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; import org.apache.iotdb.session.template.MeasurementNode; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import cn.edu.tsinghua.iot.benchmark.client.operation.Operation; import cn.edu.tsinghua.iot.benchmark.conf.Config; @@ -55,6 +52,9 @@ import cn.edu.tsinghua.iot.benchmark.workload.query.impl.RangeQuery; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.ValueRangeQuery; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.VerificationQuery; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,6 +62,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -89,6 +90,7 @@ public class IoTDB implements IDatabase { private static final String ALREADY_KEYWORD = "already"; private static final AtomicBoolean templateInit = new AtomicBoolean(false); protected final String DELETE_SERIES_SQL; + private final String ORDER_BY_TIME_DESC = " order by time desc "; protected SingleNodeJDBCConnection ioTDBConnection; protected static final Config config = ConfigDescriptor.getInstance().getConfig(); @@ -566,6 +568,20 @@ public Status valueRangeQueryOrderByDesc(ValueRangeQuery valueRangeQuery) { return executeQueryAndGetStatus(sql, Operation.VALUE_RANGE_QUERY_ORDER_BY_TIME_DESC); } + @Override + public Status groupByQueryOrderByDesc(GroupByQuery groupByQuery) { + String aggQuerySqlHead = + getAggQuerySqlHead(groupByQuery.getDeviceSchema(), groupByQuery.getAggFun()); + String sql = + addGroupByClause( + aggQuerySqlHead, + groupByQuery.getStartTimestamp(), + groupByQuery.getEndTimestamp(), + groupByQuery.getGranularity()); + sql += ORDER_BY_TIME_DESC; + return executeQueryAndGetStatus(sql, Operation.GROUP_BY_QUERY_ORDER_BY_TIME_DESC); + } + /** * Generate simple query header. * @@ -635,8 +651,12 @@ private String getValueFilterClause(List deviceSchemas, int valueT .append(getDevicePath(deviceSchema)) .append(".") .append(sensor.getName()) - .append(" > ") - .append(valueThreshold); + .append(" > "); + if (sensor.getSensorType() == SensorType.DATE) { + builder.append("'").append(LocalDate.ofEpochDay(Math.abs(valueThreshold))).append("'"); + } else { + builder.append(valueThreshold); + } } } return builder.toString(); @@ -779,11 +799,17 @@ public String getInsertOneBatchSql( case INT64: case FLOAT: case DOUBLE: + case TIMESTAMP: + case DATE: builder.append(",").append(value); break; case TEXT: + case STRING: builder.append(",").append("'").append(value).append("'"); break; + case BLOB: + builder.append(",").append("X'").append(value).append("'"); + break; } sensorIndex++; } @@ -937,6 +963,14 @@ String getEncodingType(SensorType dataSensorType) { return config.getENCODING_DOUBLE(); case TEXT: return config.getENCODING_TEXT(); + case STRING: + return config.getENCODING_STRING(); + case BLOB: + return config.getENCODING_BLOB(); + case TIMESTAMP: + return config.getENCODING_TIMESTAMP(); + case DATE: + return config.getENCODING_DATE(); default: LOGGER.error("Unsupported data sensorType {}.", dataSensorType); return null; diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBClusterSession.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBClusterSession.java index 7b595ed82..84c084366 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBClusterSession.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBClusterSession.java @@ -24,12 +24,12 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.pool.SessionPool; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.common.RowRecord; -import org.apache.iotdb.tsfile.write.record.Tablet; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; import org.slf4j.LoggerFactory; import java.util.ArrayList; diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSession.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSession.java index 6d6479bba..8faa5270e 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSession.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSession.java @@ -24,12 +24,12 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.common.RowRecord; -import org.apache.iotdb.tsfile.write.record.Tablet; import cn.edu.tsinghua.iot.benchmark.tsdb.DBConfig; import cn.edu.tsinghua.iot.benchmark.tsdb.TsdbException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; import org.slf4j.LoggerFactory; import java.util.ArrayList; diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSessionBase.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSessionBase.java index 68181a686..138efcae6 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSessionBase.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/IoTDBSessionBase.java @@ -22,14 +22,6 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; -import org.apache.iotdb.tsfile.read.common.Field; -import org.apache.iotdb.tsfile.read.common.RowRecord; -import org.apache.iotdb.tsfile.utils.Binary; -import org.apache.iotdb.tsfile.utils.BytesUtils; -import org.apache.iotdb.tsfile.write.record.Tablet; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import cn.edu.tsinghua.iot.benchmark.client.operation.Operation; import cn.edu.tsinghua.iot.benchmark.conf.Config; @@ -47,12 +39,22 @@ import cn.edu.tsinghua.iot.benchmark.tsdb.enums.DBInsertMode; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.DeviceQuery; import cn.edu.tsinghua.iot.benchmark.workload.query.impl.VerificationQuery; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.slf4j.Logger; +import java.time.LocalDate; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -105,13 +107,13 @@ public Status insertOneBatchByRecord(IBatch batch) { List dataTypes = constructDataTypes( batch.getDeviceSchema().getSensors(), record.getRecordDataValue().size()); + List recordDataValue = convertTypeForBLOB(record, dataTypes); try { if (config.isVECTOR()) { sessionWrapper.insertAlignedRecord( - deviceId, timestamp, sensors, dataTypes, record.getRecordDataValue()); + deviceId, timestamp, sensors, dataTypes, recordDataValue); } else { - sessionWrapper.insertRecord( - deviceId, timestamp, sensors, dataTypes, record.getRecordDataValue()); + sessionWrapper.insertRecord(deviceId, timestamp, sensors, dataTypes, recordDataValue); } } catch (IoTDBConnectionException | StatementExecutionException e) { failRecord++; @@ -141,10 +143,11 @@ public Status insertOneBatchByRecords(IBatch batch) { deviceIds.add(deviceId); times.add(record.getTimestamp()); measurementsList.add(sensors); - valuesList.add(record.getRecordDataValue()); - typesList.add( + List dataTypes = constructDataTypes( - batch.getDeviceSchema().getSensors(), record.getRecordDataValue().size())); + batch.getDeviceSchema().getSensors(), record.getRecordDataValue().size()); + valuesList.add(convertTypeForBLOB(record, dataTypes)); + typesList.add(dataTypes); } if (!batch.hasNext()) { break; @@ -432,12 +435,24 @@ protected Tablet genTablet(IBatch batch) { (double) (record.getRecordDataValue().get(recordValueIndex)); break; case TEXT: + case STRING: + case BLOB: Binary[] sensorsText = (Binary[]) values[recordValueIndex]; sensorsText[recordIndex] = binaryCache.computeIfAbsent( (String) record.getRecordDataValue().get(recordValueIndex), BytesUtils::valueOf); break; + case TIMESTAMP: + long[] sensorsTimestamp = (long[]) values[recordValueIndex]; + sensorsTimestamp[recordIndex] = + (long) (record.getRecordDataValue().get(recordValueIndex)); + break; + case DATE: + LocalDate[] sensorsDate = (LocalDate[]) values[recordValueIndex]; + sensorsDate[recordIndex] = + (LocalDate) (record.getRecordDataValue().get(recordValueIndex)); + break; default: LOGGER.error("Unsupported Type: {}", sensors.get(sensorIndex).getSensorType()); } @@ -469,11 +484,39 @@ public List constructDataTypes(List sensors, int recordValue case TEXT: dataTypes.add(TSDataType.TEXT); break; + case STRING: + dataTypes.add(TSDataType.STRING); + break; + case BLOB: + dataTypes.add(TSDataType.BLOB); + break; + case TIMESTAMP: + dataTypes.add(TSDataType.TIMESTAMP); + break; + case DATE: + dataTypes.add(TSDataType.DATE); + break; } } return dataTypes; } + public List convertTypeForBLOB(Record record, List dataTypes) { + // String change to Binary + List dataValue = record.getRecordDataValue(); + for (int recordValueIndex = 0; + recordValueIndex < record.getRecordDataValue().size(); + recordValueIndex++) { + if (Objects.requireNonNull(dataTypes.get(recordValueIndex)) == TSDataType.BLOB) { + dataValue.set( + recordValueIndex, + binaryCache.computeIfAbsent( + (String) record.getRecordDataValue().get(recordValueIndex), BytesUtils::valueOf)); + } + } + return dataValue; + } + @Override public Status insertOneBatch(IBatch batch) { DBInsertMode insertMode = dbConfig.getDB_SWITCH().getInsertMode(); diff --git a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/TimeseriesSchema.java b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/TimeseriesSchema.java index 7d11f9e15..7276e14d4 100644 --- a/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/TimeseriesSchema.java +++ b/iotdb-1.3/src/main/java/cn/edu/tsinghua/iot/benchmark/iotdb130/TimeseriesSchema.java @@ -19,11 +19,10 @@ package cn.edu.tsinghua.iot.benchmark.iotdb130; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; - import cn.edu.tsinghua.iot.benchmark.schema.schemaImpl.DeviceSchema; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; import java.util.List;