diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 2f4b19ede234..437b57b53812 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -28,7 +28,6 @@ import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.IFullPath; import org.apache.iotdb.commons.path.MeasurementPath; -import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -132,6 +131,7 @@ import org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool; import org.apache.iotdb.db.utils.CommonUtils; import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -158,7 +158,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -1993,6 +1992,7 @@ public void forceCloseAllWorkingTsFileProcessors() throws TsFileProcessorExcepti WritingMetrics.getInstance().recordActiveTimePartitionCount(-1); } finally { writeUnlock(); + writeUnlock(); } } @@ -2264,7 +2264,6 @@ public void deleteByDevice(final MeasurementPath pattern, final DeleteDataNode n try { TreeDeviceSchemaCacheManager.getInstance().invalidateLastCache(pattern); - Set devicePaths = new HashSet<>(pattern.getDevicePathPattern()); // write log to impacted working TsFileProcessors List walListeners = logDeletionInWAL(startTime, endTime, searchIndex, pattern); @@ -2475,9 +2474,11 @@ public void insertSeparatorToWAL() { private boolean canSkipDelete(TsFileResource tsFileResource, ModEntry deletion) { long fileStartTime = tsFileResource.getTimeIndex().getMinStartTime(); - long fileEndTime = tsFileResource.getTimeIndex().getMaxEndTime(); + long fileEndTime = + tsFileResource.isClosed() ? tsFileResource.getTimeIndex().getMaxEndTime() : Long.MAX_VALUE; - if (!deletion.getTimeRange().contains(fileStartTime, fileEndTime)) { + if (!ModificationUtils.overlap( + deletion.getStartTime(), deletion.getEndTime(), fileStartTime, fileEndTime)) { return true; } ITimeIndex timeIndex = tsFileResource.getTimeIndex(); @@ -2487,7 +2488,10 @@ private boolean canSkipDelete(TsFileResource tsFileResource, ModEntry deletion) for (IDeviceID device : tsFileResource.getDevices()) { long startTime = tsFileResource.getTimeIndex().getStartTime(device); - long endTime = tsFileResource.getTimeIndex().getEndTime(device); + long endTime = + tsFileResource.isClosed() + ? tsFileResource.getTimeIndex().getEndTime(device) + : Long.MAX_VALUE; if (deletion.affects(device, startTime, endTime)) { return false; } @@ -2509,15 +2513,15 @@ private void deleteDataInUnsealedFiles( if (tsFileResource.isClosed()) { sealedTsFiles.add(tsFileResource); } else { - tsFileResource.getProcessor().getFlushQueryLock().readLock().lock(); + tsFileResource.getProcessor().getFlushQueryLock().writeLock().lock(); if (tsFileResource.isClosed()) { sealedTsFiles.add(tsFileResource); - tsFileResource.getProcessor().getFlushQueryLock().readLock().unlock(); + tsFileResource.getProcessor().getFlushQueryLock().writeLock().unlock(); } else { try { tsFileResource.getProcessor().deleteDataInMemory(deletion); } finally { - tsFileResource.getProcessor().getFlushQueryLock().readLock().unlock(); + tsFileResource.getProcessor().getFlushQueryLock().writeLock().unlock(); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 4544df00b75a..a718fe9b98bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -815,7 +815,7 @@ private long[] filterDeletedTimestamp(TVList tvList, List deletionLis } @Override - public void delete(ModEntry modEntry) { + public long delete(ModEntry modEntry) { List> targetDeviceList = new ArrayList<>(); for (Entry entry : memTableMap.entrySet()) { if (modEntry.affects(entry.getKey())) { @@ -823,12 +823,14 @@ public void delete(ModEntry modEntry) { } } + long pointDeleted = 0; for (Pair pair : targetDeviceList) { - pair.right.delete(modEntry); + pointDeleted += pair.right.delete(modEntry); if (pair.right.getMemChunkMap().isEmpty()) { memTableMap.remove(pair.left); } } + return pointDeleted; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java index 0c033107f951..f171c2d22a7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java @@ -143,8 +143,12 @@ void queryForDeviceRegionScan( boolean isEmpty(); - /** Delete data in the MemTable according to the modEntry. */ - void delete(ModEntry modEntry); + /** + * Delete data in the MemTable according to the modEntry. + * + * @return + */ + long delete(ModEntry modEntry); /** * Make a copy of this MemTable. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 2a7a774e5a05..b2a655179138 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1165,8 +1165,11 @@ public void deleteDataInMemory(ModEntry deletion) { } try { if (workMemTable != null) { - logger.info("[Deletion] Deletion with {} in workMemTable", deletion); - workMemTable.delete(deletion); + long pointDeleted = workMemTable.delete(deletion); + logger.info( + "[Deletion] Deletion with {} in workMemTable, {} points deleted", + deletion, + pointDeleted); } // Flushing memTables are immutable, only record this deletion in these memTables for read if (!flushingMemTables.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java index 8a27916574a1..38d7592b177c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java @@ -44,7 +44,7 @@ protected ModEntry(ModType modType) { public int serializedSize() { // modType + time range - return Byte.BYTES + Long.BYTES * 2; + return Byte.BYTES + Long.BYTES * 2 + Byte.BYTES * 2; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java index 302eeb5c6462..a6a61561dccc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.modification; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.TimeRange; @@ -80,7 +81,8 @@ public int serializedSize() { @Override public boolean affects(IDeviceID deviceID, long startTime, long endTime) { - return predicate.matches(deviceID) && timeRange.contains(startTime, endTime); + return predicate.matches(deviceID) + && ModificationUtils.overlap(getStartTime(), getEndTime(), startTime, endTime); } @Override @@ -117,7 +119,8 @@ public int compareTo(ModEntry o) { } public TableDeletionEntry clone() { - return new TableDeletionEntry(predicate, new TimeRange(timeRange.getMin(), timeRange.getMax())); + TimeRange timeRangeCopy = new TimeRange(timeRange.getMin(), timeRange.getMax()); + return new TableDeletionEntry(predicate, timeRangeCopy); } public String getTableName() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java index 75112cd992d4..3ae0b31864b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java @@ -22,7 +22,9 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; +import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.common.TimeRange; @@ -57,6 +59,7 @@ public TreeDeletionEntry(PartialPath path, TimeRange timeRange) { this.timeRange = timeRange; } + @TestOnly public TreeDeletionEntry(PartialPath path, long endTime) { this(); this.pathPattern = path; @@ -120,19 +123,24 @@ public boolean matches(PartialPath path) { @Override public boolean affects(IDeviceID deviceID, long startTime, long endTime) { - try { - return pathPattern.endWithMultiLevelWildcard() - && pathPattern.getDevicePath().matchFullPath(new PartialPath(deviceID)) - && timeRange.contains(startTime, endTime); - } catch (IllegalPathException e) { - return false; - } + return affects(deviceID) + && ModificationUtils.overlap(getStartTime(), getEndTime(), startTime, endTime); } @Override public boolean affects(IDeviceID deviceID) { try { - return pathPattern.matchPrefixPath(new PartialPath(deviceID)); + if (pathPattern.endWithMultiLevelWildcard()) { + PartialPath deviceIdPath = new PartialPath(deviceID); + // pattern: root.db1.d1.**, deviceId: root.db1.d1, match + return pathPattern.getDevicePath().matchFullPath(deviceIdPath) + // pattern: root.db1.**, deviceId: root.db1.d1, match + || pathPattern.matchFullPath(deviceIdPath); + } else { + // pattern: root.db1.d1.s1, deviceId: root.db1.d1, match + // pattern: root.db1.d1, deviceId: root.db1.d1, not match + return pathPattern.getDevicePath().matchFullPath(new PartialPath(deviceID)); + } } catch (IllegalPathException e) { return false; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 825003d2d5c8..a91ba4771dc9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -40,6 +40,11 @@ private ModificationUtils() { // util class } + // both ranges are closed + public static boolean overlap(long startA, long endA, long startB, long endB) { + return endB >= startA && startB <= endA; + } + /** * modifyChunkMetaData iterates the chunkMetaData and applies all available modifications on it to * generate a ModifiedChunkMetadata.