chore: refine metastore API (#3502)

grafana · Aug 19, 2024 · 5d10c2a · 5d10c2a
1 parent 987f743
commit 5d10c2a
Show file tree

Hide file tree

Showing 30 changed files with 777 additions and 500 deletions.
diff --git a/api/gen/proto/go/metastore/v1/metastore.pb.go b/api/gen/proto/go/metastore/v1/metastore.pb.go
diff --git a/api/gen/proto/go/metastore/v1/metastore_vtproto.pb.go b/api/gen/proto/go/metastore/v1/metastore_vtproto.pb.go
diff --git a/api/gen/proto/go/metastore/v1/metastorev1connect/metastore.connect.go b/api/gen/proto/go/metastore/v1/metastorev1connect/metastore.connect.go
diff --git a/api/gen/proto/go/metastore/v1/metastorev1connect/metastore.connect.mux.go b/api/gen/proto/go/metastore/v1/metastorev1connect/metastore.connect.mux.go
diff --git a/api/metastore/v1/metastore.proto b/api/metastore/v1/metastore.proto
@@ -2,9 +2,11 @@ syntax = "proto3";
 
 package metastore.v1;
 
+import "types/v1/types.proto";
+
 service MetastoreService {
  rpc AddBlock(AddBlockRequest) returns (AddBlockResponse) {}
- rpc ListBlocksForQuery(ListBlocksForQueryRequest) returns (ListBlocksForQueryResponse) {}
+ rpc QueryMetadata(QueryMetadataRequest) returns (QueryMetadataResponse) {}
  rpc ReadIndex(ReadIndexRequest) returns (ReadIndexResponse) {}
 }
 
@@ -23,14 +25,13 @@ message BlockMeta {
  uint32 compaction_level = 6;
  // Optional. Empty if compaction level is 0.
  string tenant_id = 7;
- // TODO(kolesnikovae): Partitions with labels?
- repeated TenantService tenant_services = 8;
+ repeated Dataset datasets = 8;
  uint64 size = 9;
 }
 
-// TenantService object points to the offset in the block at which
-// the tenant service data is located.
-message TenantService {
+message Dataset {
+ repeated types.v1.Labels labels = 8;
+
  string tenant_id = 1;
  string name = 2;
  int64 min_time = 3;
@@ -47,18 +48,20 @@ message TenantService {
  repeated uint64 table_of_contents = 5;
  // Size of the section in bytes.
  uint64 size = 6;
+
+ // TODO: delete
  // Profile types present in the tenant service data.
  repeated string profile_types = 7;
 }
 
-message ListBlocksForQueryRequest {
+message QueryMetadataRequest {
  repeated string tenant_id = 1;
  int64 start_time = 2;
  int64 end_time = 3;
  string query = 4;
 }
 
-message ListBlocksForQueryResponse {
+message QueryMetadataResponse {
  repeated BlockMeta blocks = 1;
 }
 

diff --git a/api/openapiv2/gen/phlare.swagger.json b/api/openapiv2/gen/phlare.swagger.json
@@ -656,13 +656,12 @@
  "type": "string",
  "description": "Optional. Empty if compaction level is 0."
  },
- "tenantServices": {
+ "datasets": {
  "type": "array",
  "items": {
  "type": "object",
- "$ref": "#/definitions/v1TenantService"
- },
- "title": "TODO(kolesnikovae): Partitions with labels?"
+ "$ref": "#/definitions/v1Dataset"
+ }
  },
  "size": {
  "type": "string",
@@ -829,6 +828,52 @@
  }
  }
  },
+ "v1Dataset": {
+ "type": "object",
+ "properties": {
+ "labels": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "$ref": "#/definitions/v1Labels"
+ }
+ },
+ "tenantId": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "minTime": {
+ "type": "string",
+ "format": "int64"
+ },
+ "maxTime": {
+ "type": "string",
+ "format": "int64"
+ },
+ "tableOfContents": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "format": "uint64"
+ },
+ "description": "Table of contents lists data sections within the tenant\nservice region. The offsets are absolute.\n\nThe interpretation of the table of contents is specific\nto the metadata format version. By default, the sections are:\n - 0: profiles.parquet\n - 1: index.tsdb\n - 2: symbols.symdb"
+ },
+ "size": {
+ "type": "string",
+ "format": "uint64",
+ "description": "Size of the section in bytes."
+ },
+ "profileTypes": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "TODO: delete\nProfile types present in the tenant service data."
+ }
+ }
+ },
  "v1Diagnostics": {
  "type": "object",
  "description": "Diagnostic messages, events, statistics, analytics, etc."
@@ -1240,18 +1285,6 @@
  }
  }
  },
- "v1ListBlocksForQueryResponse": {
- "type": "object",
- "properties": {
- "blocks": {
- "type": "array",
- "items": {
- "type": "object",
- "$ref": "#/definitions/v1BlockMeta"
- }
- }
- }
- },
  "v1Mapping": {
  "type": "object",
  "properties": {
@@ -1519,6 +1552,18 @@
  }
  }
  },
+ "v1QueryMetadataResponse": {
+ "type": "object",
+ "properties": {
+ "blocks": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "$ref": "#/definitions/v1BlockMeta"
+ }
+ }
+ }
+ },
  "v1QueryPlan": {
  "type": "object",
  "properties": {
@@ -1925,46 +1970,6 @@
  ],
  "default": "MERGE_FORMAT_UNSPECIFIED"
  },
- "v1TenantService": {
- "type": "object",
- "properties": {
- "tenantId": {
- "type": "string"
- },
- "name": {
- "type": "string"
- },
- "minTime": {
- "type": "string",
- "format": "int64"
- },
- "maxTime": {
- "type": "string",
- "format": "int64"
- },
- "tableOfContents": {
- "type": "array",
- "items": {
- "type": "string",
- "format": "uint64"
- },
- "description": "Table of contents lists data sections within the tenant\nservice region. The offsets are absolute.\n\nThe interpretation of the table of contents is specific\nto the metadata format version. By default, the sections are:\n - 0: profiles.parquet\n - 1: index.tsdb\n - 2: symbols.symdb"
- },
- "size": {
- "type": "string",
- "format": "uint64",
- "description": "Size of the section in bytes."
- },
- "profileTypes": {
- "type": "array",
- "items": {
- "type": "string"
- },
- "description": "Profile types present in the tenant service data."
- }
- },
- "description": "TenantService object points to the offset in the block at which\nthe tenant service data is located."
- },
  "v1TimeSeriesAggregationType": {
  "type": "string",
  "enum": [

diff --git a/docs/sources/configure-server/reference-configuration-parameters/index.md b/docs/sources/configure-server/reference-configuration-parameters/index.md
@@ -37,9 +37,9 @@ brackets indicate that a parameter is optional.
 - `<prefix>`: a CLI flag prefix based on the context (look at the parent configuration block to see which CLI flags prefix should be used)
 - `<relabel_config>`: a [Prometheus relabeling configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config)
 - `<time>`: a timestamp, with available formats:
- - `2006-01-20` (midnight, local timezone)
- - `2006-01-20T15:04` (local timezone)
- - RFC 3339 formats: `2006-01-20T15:04:05Z` (UTC) or `2006-01-20T15:04:05+07:00` (explicit timezone)
+  - `2006-01-20` (midnight, local timezone)
+  - `2006-01-20T15:04` (local timezone)
+  - RFC 3339 formats: `2006-01-20T15:04:05Z` (UTC) or `2006-01-20T15:04:05+07:00` (explicit timezone)
 
 ## Use environment variables in the configuration
 
@@ -2259,4 +2259,3 @@ The `filesystem_storage_backend` block configures the usage of local file system
 ```
 
 ```
-
diff --git a/pkg/experiment/compactor/compaction_worker.go b/pkg/experiment/compactor/compaction_worker.go
@@ -255,7 +255,7 @@ func (w *Worker) startJob(ctx context.Context, job *compactorv1.CompactionJob) *
  "block_compaction_level", c.CompactionLevel,
  "block_min_time", c.MinTime,
  "block_max_time", c.MinTime,
- "tenant_services", len(c.TenantServices))
+ "datasets", len(c.Datasets))
  }
 
  job.Status.Status = compactorv1.CompactionStatus_COMPACTION_STATUS_SUCCESS