Skip to content
This repository was archived by the owner on Sep 5, 2023. It is now read-only.

Commit 06a127b

Browse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat: Max concurrent backfill tasks (#168)
* feat: Max concurrent backfill tasks You can now set the number of maximum concurrent backfill tasks for a stream using the Datastream API. PiperOrigin-RevId: 530067890 Source-Link: googleapis/googleapis@b2c290f Source-Link: https://github.com/googleapis/googleapis-gen/commit/83c5413e535cb6fa710541dc6e169a8a44b8a65d Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiODNjNTQxM2U1MzVjYjZmYTcxMDU0MWRjNmUxNjlhOGE0NGI4YTY1ZCJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 3e2f9f4 commit 06a127b

File tree

6 files changed

+69
-13
lines changed

6 files changed

+69
-13
lines changed

‎google/cloud/datastream_v1/types/datastream.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1056,6 +1056,9 @@ class CreatePrivateConnectionRequest(proto.Message):
10561056
The request ID must be a valid UUID with the
10571057
exception that zero UUID is not supported
10581058
(00000000-0000-0000-0000-000000000000).
1059+
force (bool):
1060+
Optional. If set to true, will skip
1061+
validations.
10591062
"""
10601063

10611064
parent: str = proto.Field(
@@ -1075,6 +1078,10 @@ class CreatePrivateConnectionRequest(proto.Message):
10751078
proto.STRING,
10761079
number=4,
10771080
)
1081+
force: bool = proto.Field(
1082+
proto.BOOL,
1083+
number=6,
1084+
)
10781085

10791086

10801087
class ListPrivateConnectionsRequest(proto.Message):

‎google/cloud/datastream_v1/types/datastream_resources.py

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,12 @@ class BigQueryProfile(proto.Message):
226226

227227

228228
class StaticServiceIpConnectivity(proto.Message):
229-
r"""Static IP address connectivity."""
229+
r"""Static IP address connectivity. Used when the source database
230+
is configured to allow incoming connections from the Datastream
231+
public IP addresses for the region specified in the connection
232+
profile.
233+
234+
"""
230235

231236

232237
class ForwardSshTunnelConnectivity(proto.Message):
@@ -792,15 +797,20 @@ class OracleSourceConfig(proto.Message):
792797
Oracle objects to exclude from the stream.
793798
max_concurrent_cdc_tasks (int):
794799
Maximum number of concurrent CDC tasks. The
795-
number should be non negative. If not set (or
796-
set to 0), the system's default value will be
800+
number should be non-negative. If not set (or
801+
set to 0), the system's default value is used.
802+
max_concurrent_backfill_tasks (int):
803+
Maximum number of concurrent backfill tasks.
804+
The number should be non-negative. If not set
805+
(or set to 0), the system's default value is
797806
used.
798807
drop_large_objects (google.cloud.datastream_v1.types.OracleSourceConfig.DropLargeObjects):
799808
Drop large object values.
800809
801810
This field is a member of `oneof`_ ``large_objects_handling``.
802811
stream_large_objects (google.cloud.datastream_v1.types.OracleSourceConfig.StreamLargeObjects):
803-
Stream large object values.
812+
Stream large object values. NOTE: This
813+
feature is currently experimental.
804814
805815
This field is a member of `oneof`_ ``large_objects_handling``.
806816
"""
@@ -825,6 +835,10 @@ class StreamLargeObjects(proto.Message):
825835
proto.INT32,
826836
number=3,
827837
)
838+
max_concurrent_backfill_tasks: int = proto.Field(
839+
proto.INT32,
840+
number=4,
841+
)
828842
drop_large_objects: DropLargeObjects = proto.Field(
829843
proto.MESSAGE,
830844
number=100,
@@ -967,12 +981,18 @@ class PostgresqlSourceConfig(proto.Message):
967981
PostgreSQL objects to exclude from the
968982
stream.
969983
replication_slot (str):
970-
Required. The name of the logical replication
971-
slot that's configured with the pgoutput plugin.
984+
Required. Immutable. The name of the logical
985+
replication slot that's configured with the
986+
pgoutput plugin.
972987
publication (str):
973988
Required. The name of the publication that includes the set
974989
of all tables that are defined in the stream's
975990
include_objects.
991+
max_concurrent_backfill_tasks (int):
992+
Maximum number of concurrent backfill tasks.
993+
The number should be non negative. If not set
994+
(or set to 0), the system's default value will
995+
be used.
976996
"""
977997

978998
include_objects: "PostgresqlRdbms" = proto.Field(
@@ -993,6 +1013,10 @@ class PostgresqlSourceConfig(proto.Message):
9931013
proto.STRING,
9941014
number=4,
9951015
)
1016+
max_concurrent_backfill_tasks: int = proto.Field(
1017+
proto.INT32,
1018+
number=5,
1019+
)
9961020

9971021

9981022
class MysqlColumn(proto.Message):
@@ -1122,6 +1146,11 @@ class MysqlSourceConfig(proto.Message):
11221146
number should be non negative. If not set (or
11231147
set to 0), the system's default value will be
11241148
used.
1149+
max_concurrent_backfill_tasks (int):
1150+
Maximum number of concurrent backfill tasks.
1151+
The number should be non negative. If not set
1152+
(or set to 0), the system's default value will
1153+
be used.
11251154
"""
11261155

11271156
include_objects: "MysqlRdbms" = proto.Field(
@@ -1138,6 +1167,10 @@ class MysqlSourceConfig(proto.Message):
11381167
proto.INT32,
11391168
number=3,
11401169
)
1170+
max_concurrent_backfill_tasks: int = proto.Field(
1171+
proto.INT32,
1172+
number=4,
1173+
)
11411174

11421175

11431176
class SourceConfig(proto.Message):
@@ -1268,7 +1301,8 @@ class GcsDestinationConfig(proto.Message):
12681301
file_rotation_interval (google.protobuf.duration_pb2.Duration):
12691302
The maximum duration for which new events are
12701303
added before a file is closed and a new file is
1271-
created.
1304+
created. Values within the range of 15-60
1305+
seconds are allowed.
12721306
avro_file_format (google.cloud.datastream_v1.types.AvroFileFormat):
12731307
AVRO file format configuration.
12741308
@@ -1307,7 +1341,7 @@ class GcsDestinationConfig(proto.Message):
13071341

13081342

13091343
class BigQueryDestinationConfig(proto.Message):
1310-
r"""
1344+
r"""BigQuery destination configuration
13111345
13121346
This message has `oneof`_ fields (mutually exclusive fields).
13131347
For each oneof, at most one member field can be set at the same time.
@@ -1340,7 +1374,7 @@ class SingleTargetDataset(proto.Message):
13401374
13411375
Attributes:
13421376
dataset_id (str):
1343-
1377+
The dataset ID of the target dataset.
13441378
"""
13451379

13461380
dataset_id: str = proto.Field(
@@ -1354,7 +1388,8 @@ class SourceHierarchyDatasets(proto.Message):
13541388
13551389
Attributes:
13561390
dataset_template (google.cloud.datastream_v1.types.BigQueryDestinationConfig.SourceHierarchyDatasets.DatasetTemplate):
1357-
1391+
The dataset template to use for dynamic
1392+
dataset creation.
13581393
"""
13591394

13601395
class DatasetTemplate(proto.Message):

‎samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
],
99
"language": "PYTHON",
1010
"name": "google-cloud-datastream",
11-
"version": "1.5.1"
11+
"version": "0.1.0"
1212
},
1313
"snippets": [
1414
{

‎samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
],
99
"language": "PYTHON",
1010
"name": "google-cloud-datastream",
11-
"version": "1.5.1"
11+
"version": "0.1.0"
1212
},
1313
"snippets": [
1414
{

‎scripts/fixup_datastream_v1_keywords.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class datastreamCallTransformer(cst.CSTTransformer):
4040
CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata')
4141
METHOD_TO_PARAMS: Dict[str, Tuple[str]] = {
4242
'create_connection_profile': ('parent', 'connection_profile_id', 'connection_profile', 'request_id', 'validate_only', 'force', ),
43-
'create_private_connection': ('parent', 'private_connection_id', 'private_connection', 'request_id', ),
43+
'create_private_connection': ('parent', 'private_connection_id', 'private_connection', 'request_id', 'force', ),
4444
'create_route': ('parent', 'route_id', 'route', 'request_id', ),
4545
'create_stream': ('parent', 'stream_id', 'stream', 'request_id', 'validate_only', 'force', ),
4646
'delete_connection_profile': ('name', 'request_id', ),

‎tests/unit/gapic/datastream_v1/test_datastream.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10421,6 +10421,7 @@ def test_create_stream_rest(request_type):
1042110421
},
1042210422
"exclude_objects": {},
1042310423
"max_concurrent_cdc_tasks": 2550,
10424+
"max_concurrent_backfill_tasks": 3076,
1042410425
"drop_large_objects": {},
1042510426
"stream_large_objects": {},
1042610427
},
@@ -10450,6 +10451,7 @@ def test_create_stream_rest(request_type):
1045010451
},
1045110452
"exclude_objects": {},
1045210453
"max_concurrent_cdc_tasks": 2550,
10454+
"max_concurrent_backfill_tasks": 3076,
1045310455
},
1045410456
"postgresql_source_config": {
1045510457
"include_objects": {
@@ -10479,6 +10481,7 @@ def test_create_stream_rest(request_type):
1047910481
"exclude_objects": {},
1048010482
"replication_slot": "replication_slot_value",
1048110483
"publication": "publication_value",
10484+
"max_concurrent_backfill_tasks": 3076,
1048210485
},
1048310486
},
1048410487
"destination_config": {
@@ -10766,6 +10769,7 @@ def test_create_stream_rest_bad_request(
1076610769
},
1076710770
"exclude_objects": {},
1076810771
"max_concurrent_cdc_tasks": 2550,
10772+
"max_concurrent_backfill_tasks": 3076,
1076910773
"drop_large_objects": {},
1077010774
"stream_large_objects": {},
1077110775
},
@@ -10795,6 +10799,7 @@ def test_create_stream_rest_bad_request(
1079510799
},
1079610800
"exclude_objects": {},
1079710801
"max_concurrent_cdc_tasks": 2550,
10802+
"max_concurrent_backfill_tasks": 3076,
1079810803
},
1079910804
"postgresql_source_config": {
1080010805
"include_objects": {
@@ -10824,6 +10829,7 @@ def test_create_stream_rest_bad_request(
1082410829
"exclude_objects": {},
1082510830
"replication_slot": "replication_slot_value",
1082610831
"publication": "publication_value",
10832+
"max_concurrent_backfill_tasks": 3076,
1082710833
},
1082810834
},
1082910835
"destination_config": {
@@ -10996,6 +11002,7 @@ def test_update_stream_rest(request_type):
1099611002
},
1099711003
"exclude_objects": {},
1099811004
"max_concurrent_cdc_tasks": 2550,
11005+
"max_concurrent_backfill_tasks": 3076,
1099911006
"drop_large_objects": {},
1100011007
"stream_large_objects": {},
1100111008
},
@@ -11025,6 +11032,7 @@ def test_update_stream_rest(request_type):
1102511032
},
1102611033
"exclude_objects": {},
1102711034
"max_concurrent_cdc_tasks": 2550,
11035+
"max_concurrent_backfill_tasks": 3076,
1102811036
},
1102911037
"postgresql_source_config": {
1103011038
"include_objects": {
@@ -11054,6 +11062,7 @@ def test_update_stream_rest(request_type):
1105411062
"exclude_objects": {},
1105511063
"replication_slot": "replication_slot_value",
1105611064
"publication": "publication_value",
11065+
"max_concurrent_backfill_tasks": 3076,
1105711066
},
1105811067
},
1105911068
"destination_config": {
@@ -11319,6 +11328,7 @@ def test_update_stream_rest_bad_request(
1131911328
},
1132011329
"exclude_objects": {},
1132111330
"max_concurrent_cdc_tasks": 2550,
11331+
"max_concurrent_backfill_tasks": 3076,
1132211332
"drop_large_objects": {},
1132311333
"stream_large_objects": {},
1132411334
},
@@ -11348,6 +11358,7 @@ def test_update_stream_rest_bad_request(
1134811358
},
1134911359
"exclude_objects": {},
1135011360
"max_concurrent_cdc_tasks": 2550,
11361+
"max_concurrent_backfill_tasks": 3076,
1135111362
},
1135211363
"postgresql_source_config": {
1135311364
"include_objects": {
@@ -11377,6 +11388,7 @@ def test_update_stream_rest_bad_request(
1137711388
"exclude_objects": {},
1137811389
"replication_slot": "replication_slot_value",
1137911390
"publication": "publication_value",
11391+
"max_concurrent_backfill_tasks": 3076,
1138011392
},
1138111393
},
1138211394
"destination_config": {
@@ -13579,6 +13591,7 @@ def test_create_private_connection_rest_required_fields(
1357913591
# Check that path parameters and body parameters are not mixing in.
1358013592
assert not set(unset_fields) - set(
1358113593
(
13594+
"force",
1358213595
"private_connection_id",
1358313596
"request_id",
1358413597
)
@@ -13645,6 +13658,7 @@ def test_create_private_connection_rest_unset_required_fields():
1364513658
assert set(unset_fields) == (
1364613659
set(
1364713660
(
13661+
"force",
1364813662
"privateConnectionId",
1364913663
"requestId",
1365013664
)

0 commit comments

Comments
 (0)