From e2c0cf5517ddeef310dc395983962eae350e328f Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 2 Feb 2023 12:35:38 -0700 Subject: [PATCH] add external sql option and skip schema --- bq_table.proto | 6 ++++++ pkg/converter/convert.go | 1 - protos/bq_table.pb.go | 45 +++++++++++++++++++++++++++++----------- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/bq_table.proto b/bq_table.proto index 2c2e665..1206e64 100644 --- a/bq_table.proto +++ b/bq_table.proto @@ -56,4 +56,10 @@ message BigQueryMessageOptions { // or ":RECORD:" for message types. // "NULLABLE" by default, different mode may be set via optional suffix ":" repeated string extra_fields = 3; + + // If set, execute specified sql file to create schema, use for extra views or materialized tables + repeated string sql = 10; + + // If set, skip using the schema to create this table (use with sql option) + bool skip_schema = 11; } \ No newline at end of file diff --git a/pkg/converter/convert.go b/pkg/converter/convert.go index 6b9e027..ad4f36b 100644 --- a/pkg/converter/convert.go +++ b/pkg/converter/convert.go @@ -29,7 +29,6 @@ var ( ".google.protobuf.BytesValue": "BYTES", ".google.protobuf.Duration": "STRING", ".google.protobuf.Timestamp": "TIMESTAMP", - ".google.type.Date": "DATE", } typeFromFieldType = map[descriptor.FieldDescriptorProto_Type]string{ descriptor.FieldDescriptorProto_TYPE_DOUBLE: "FLOAT", diff --git a/protos/bq_table.pb.go b/protos/bq_table.pb.go index ec4e501..72c28bc 100644 --- a/protos/bq_table.pb.go +++ b/protos/bq_table.pb.go @@ -52,6 +52,10 @@ type BigQueryMessageOptions struct { // or ":RECORD:" for message types. // "NULLABLE" by default, different mode may be set via optional suffix ":" ExtraFields []string `protobuf:"bytes,3,rep,name=extra_fields,json=extraFields,proto3" json:"extra_fields,omitempty"` + // If set, execute specified sql file to create schema, use for extra views or materialized tables + Sql []string `protobuf:"bytes,10,rep,name=sql,proto3" json:"sql,omitempty"` + // If set, skip using the schema to create this table (use with sql option) + SkipSchema bool `protobuf:"varint,11,opt,name=skip_schema,json=skipSchema,proto3" json:"skip_schema,omitempty"` } func (x *BigQueryMessageOptions) Reset() { @@ -107,6 +111,20 @@ func (x *BigQueryMessageOptions) GetExtraFields() []string { return nil } +func (x *BigQueryMessageOptions) GetSql() []string { + if x != nil { + return x.Sql + } + return nil +} + +func (x *BigQueryMessageOptions) GetSkipSchema() bool { + if x != nil { + return x.SkipSchema + } + return false +} + var file_bq_table_proto_extTypes = []protoimpl.ExtensionInfo{ { ExtendedType: (*descriptorpb.MessageOptions)(nil), @@ -136,7 +154,7 @@ var file_bq_table_proto_rawDesc = []byte{ 0x12, 0x0d, 0x67, 0x65, 0x6e, 0x5f, 0x62, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x80, 0x01, 0x0a, 0x16, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, + 0x6f, 0x22, 0xb3, 0x01, 0x0a, 0x16, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x75, @@ -144,17 +162,20 @@ var file_bq_table_proto_rawDesc = []byte{ 0x01, 0x28, 0x08, 0x52, 0x0c, 0x75, 0x73, 0x65, 0x4a, 0x73, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x78, 0x74, 0x72, 0x61, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x3a, 0x6c, 0x0a, 0x0d, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, - 0x5f, 0x6f, 0x70, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xfd, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, - 0x67, 0x65, 0x6e, 0x5f, 0x62, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x42, 0x69, - 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0c, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, - 0x74, 0x73, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x63, 0x68, 0x72, 0x65, 0x64, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x2d, 0x67, - 0x65, 0x6e, 0x2d, 0x62, 0x71, 0x2d, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x6c, 0x64, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x71, 0x6c, 0x18, 0x0a, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x03, 0x73, 0x71, 0x6c, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x6b, 0x69, + 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x3a, 0x6c, 0x0a, 0x0d, 0x62, 0x69, 0x67, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xfd, 0x07, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x25, 0x2e, 0x67, 0x65, 0x6e, 0x5f, 0x62, 0x71, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0c, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x4f, 0x70, 0x74, 0x73, 0x42, 0x2f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x65, 0x64, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x2d, 0x67, 0x65, 0x6e, 0x2d, 0x62, 0x71, 0x2d, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var (