Skip to content

Commit

Permalink
Moved commit "New SQS Json API: 10 requests supported" from ydb repo
Browse files Browse the repository at this point in the history
  • Loading branch information
siarheivesialou authored and Gazizonoki committed Aug 19, 2024
1 parent 1bc7a78 commit 007d763
Show file tree
Hide file tree
Showing 8 changed files with 543 additions and 31 deletions.
21 changes: 21 additions & 0 deletions src/api/grpc/draft/ydb_ymq_v1.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
syntax = "proto3";
option cc_enable_arenas = true;

package Ydb.Ymq.V1;

import "src/api/protos/draft/ymq.proto";

option java_package = "com.yandex.ydb.ymq.v1";

service YmqService {
rpc GetQueueUrl(GetQueueUrlRequest) returns (GetQueueUrlResponse);
rpc CreateQueue(CreateQueueRequest) returns (CreateQueueResponse);
rpc SendMessage(SendMessageRequest) returns (SendMessageResponse);
rpc ReceiveMessage(ReceiveMessageRequest) returns (ReceiveMessageResponse);
rpc GetQueueAttributes(GetQueueAttributesRequest) returns (GetQueueAttributesResponse);
rpc ListQueues(ListQueuesRequest) returns (ListQueuesResponse);
rpc DeleteMessage(DeleteMessageRequest) returns (DeleteMessageResponse);
rpc PurgeQueue(PurgeQueueRequest) returns (PurgeQueueResponse);
rpc DeleteQueue(DeleteQueueRequest) returns (DeleteQueueResponse);
rpc ChangeMessageVisibility(ChangeMessageVisibilityRequest) returns (ChangeMessageVisibilityResponse);
}
50 changes: 19 additions & 31 deletions src/api/protos/draft/datastreams.proto
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,13 @@ syntax = "proto3";
option cc_enable_arenas = true;

import "src/api/protos/ydb_operation.proto";
import "src/api/protos/draft/field_transformation.proto";

import "google/protobuf/descriptor.proto";

package Ydb.DataStreams.V1;
option java_package = "com.yandex.ydb.datastreams.v1";

// Extensions to simplify json <-> proto conversion
enum EFieldTransformationType {
TRANSFORM_NONE = 0;
TRANSFORM_BASE64 = 1;
TRANSFORM_DOUBLE_S_TO_INT_MS = 2;
TRANSFORM_EMPTY_TO_NOTHING = 3;
}


extend google.protobuf.FieldOptions {
EFieldTransformationType FieldTransformer = 58123;
}

// Here and below: Kinesis data types mapped to protobuf

enum EncryptionType {
Expand Down Expand Up @@ -53,7 +41,7 @@ message ChildShard {
// Represents details of consumer
message Consumer {
string consumer_arn = 1;
int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 consumer_creation_timestamp = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
string consumer_name = 3;
ConsumerDescription.ConsumerStatus consumer_status = 4;
}
Expand All @@ -66,9 +54,9 @@ message HashKeyRange {

message Record {
// Timestamp that the record was inserted into the stream
int64 approximate_arrival_timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 approximate_arrival_timestamp = 1 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Data blob
bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64];
bytes data = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_BASE64];
// Encryption type used on record
EncryptionType encryption_type = 3;
// Identifies shard in the stream the record is assigned to
Expand Down Expand Up @@ -108,7 +96,7 @@ message StreamDescription {
repeated Shard shards = 6;
string stream_arn = 7;
// Timestamp that the stream was created
int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 stream_creation_timestamp = 8 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Current status of the stream
StreamStatus stream_status = 9;
// Name of the stream
Expand All @@ -127,17 +115,17 @@ message StreamDescription {
// Represents range of possible sequence numbers for the shard
message SequenceNumberRange {
string starting_sequence_number = 1;
string ending_sequence_number = 2 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string ending_sequence_number = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
}

// Represents shard details
message Shard {
// Id of the shard adjacent to the shard's parent
string adjacent_parent_shard_id = 1 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string adjacent_parent_shard_id = 1 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
// The range of possible hash key values for the shard
HashKeyRange hash_key_range = 2;
// Id of the shard's parent
string parent_shard_id = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string parent_shard_id = 3 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
// The range of possible sequence numbers for the shard
SequenceNumberRange sequence_number_range = 4;
// Unique id of the shard within stream
Expand All @@ -155,7 +143,7 @@ message ConsumerDescription {

string consumer_arn = 1;
// Timestamp that the consumer was created
int64 consumer_creation_timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 consumer_creation_timestamp = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Name of the consumer
string consumer_name = 3;
ConsumerStatus consumer_status = 4;
Expand Down Expand Up @@ -198,14 +186,14 @@ message ShardFilter {
// Exclusive id. Can only be used if AFTER_SHARD_ID is specified
string shard_id = 1;
// Can only be used if AT_TIMESTAMP or FROM_TIMESTAMP are specified.
int64 timestamp = 2 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 timestamp = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
ShardFilterType type = 3;
}

// Represents starting position in the stream from which to start reading
message StartingPosition {
// Timestamp of the record from which to start reading
int64 timestamp = 1 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 timestamp = 1 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Sequence number of the record from which to start reading
string sequence_number = 2;
ShardIteratorType type = 3;
Expand All @@ -227,7 +215,7 @@ message StreamDescriptionSummary {
// Stream ARN
string stream_arn = 7;
// Timestamp that the stream was created
int64 stream_creation_timestamp = 8 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 stream_creation_timestamp = 8 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Name of the stream
string stream_name = 9;
// Current status of the stream
Expand Down Expand Up @@ -351,7 +339,7 @@ message ListShardsRequest {
// Filter out response
ShardFilter shard_filter = 5;
// Used to distinguish streams that have the same name
int64 stream_creation_timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 stream_creation_timestamp = 6 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
// Name of the stream
string stream_name = 7;
}
Expand Down Expand Up @@ -417,7 +405,7 @@ message ListStreamConsumersRequest {
string next_token = 3;
string stream_arn = 4;
// Used to distinguish streams that have the same name
int64 stream_creation_timestamp = 5 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 stream_creation_timestamp = 5 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
}

message ListStreamConsumersResponse {
Expand Down Expand Up @@ -502,7 +490,7 @@ message DescribeStreamConsumerResult {

message PutRecordsRequestEntry {
// Data blob
bytes data = 1 [(FieldTransformer) = TRANSFORM_BASE64];
bytes data = 1 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_BASE64];
// Hash value used to explicitly determine the shard
string explicit_hash_key = 2;
// Used as input to hash function that maps partition key to a specific shard
Expand All @@ -511,8 +499,8 @@ message PutRecordsRequestEntry {

// Represents result of an individual record
message PutRecordsResultEntry {
string error_message = 2 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string error_code = 3 [(FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string error_message = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string error_code = 3 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_EMPTY_TO_NOTHING];
string sequence_number = 4;
string shard_id = 5;
}
Expand Down Expand Up @@ -544,7 +532,7 @@ message GetRecordsResult {
message PutRecordRequest {
Ydb.Operations.OperationParams operation_params = 1;
// Data blob
bytes data = 2 [(FieldTransformer) = TRANSFORM_BASE64];
bytes data = 2 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_BASE64];
// Hash value used to explicitly determine the shard
string explicit_hash_key = 3;
// Used as input to hash function that maps partition key to a specific shard
Expand Down Expand Up @@ -600,7 +588,7 @@ message GetShardIteratorRequest {
// Name of the stream
string stream_name = 5;
// Used with shard iterator type AT_TIMESTAMP
int64 timestamp = 6 [(FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
int64 timestamp = 6 [(Ydb.FieldTransformation.FieldTransformer) = TRANSFORM_DOUBLE_S_TO_INT_MS];
}

message GetShardIteratorResponse {
Expand Down
16 changes: 16 additions & 0 deletions src/api/protos/draft/field_transformation.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
syntax = "proto3";

import "google/protobuf/descriptor.proto";

package Ydb.FieldTransformation;

enum EFieldTransformationType {
TRANSFORM_NONE = 0;
TRANSFORM_BASE64 = 1;
TRANSFORM_DOUBLE_S_TO_INT_MS = 2;
TRANSFORM_EMPTY_TO_NOTHING = 3;
}

extend google.protobuf.FieldOptions {
EFieldTransformationType FieldTransformer = 58123;
}
Loading

0 comments on commit 007d763

Please sign in to comment.