Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: [google-cloud-dataproc] Add ProvisioningModelMix to support mixing of spot and standard instances for secondary workers #13169

Merged
merged 3 commits into from
Oct 18, 2024
+118 −0
Merged

feat: [google-cloud-dataproc] Add ProvisioningModelMix to support mixing of spot and standard instances for secondary workers #13169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b32ee99
feat: Add `ProvisioningModelMix` to support mixing of spot and standa…
gcf-owl-bot[bot] Oct 15, 2024
f76778a
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Oct 15, 2024
693ff9a
Merge branch 'main' into owl-bot-copy-packages-google-cloud-dataproc
vchudnov-g Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions packages/google-cloud-dataproc/google/cloud/dataproc_v1/types/clusters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1110,6 +1110,10 @@ class InstanceFlexibilityPolicy(proto.Message):
and provisioning models.

Attributes:
provisioning_model_mix (google.cloud.dataproc_v1.types.InstanceFlexibilityPolicy.ProvisioningModelMix):
Optional. Defines how the Group selects the
provisioning model to ensure required
reliability.
instance_selection_list (MutableSequence[google.cloud.dataproc_v1.types.InstanceFlexibilityPolicy.InstanceSelection]):
Optional. List of instance selection options
that the group will use when creating new VMs.
Expand All @@ -1118,6 +1122,51 @@ class InstanceFlexibilityPolicy(proto.Message):
results in the group.
"""

class ProvisioningModelMix(proto.Message):
r"""Defines how Dataproc should create VMs with a mixture of
provisioning models.


.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields

Attributes:
standard_capacity_base (int):
Optional. The base capacity that will always use Standard
VMs to avoid risk of more preemption than the minimum
capacity you need. Dataproc will create only standard VMs
until it reaches standard_capacity_base, then it will start
using standard_capacity_percent_above_base to mix Spot with
Standard VMs. eg. If 15 instances are requested and
standard_capacity_base is 5, Dataproc will create 5 standard
VMs and then start mixing spot and standard VMs for
remaining 10 instances.

This field is a member of `oneof`_ ``_standard_capacity_base``.
standard_capacity_percent_above_base (int):
Optional. The percentage of target capacity that should use
Standard VM. The remaining percentage will use Spot VMs. The
percentage applies only to the capacity above
standard_capacity_base. eg. If 15 instances are requested
and standard_capacity_base is 5 and
standard_capacity_percent_above_base is 30, Dataproc will
create 5 standard VMs and then start mixing spot and
standard VMs for remaining 10 instances. The mix will be 30%
standard and 70% spot.

This field is a member of `oneof`_ ``_standard_capacity_percent_above_base``.
"""

standard_capacity_base: int = proto.Field(
proto.INT32,
number=1,
optional=True,
)
standard_capacity_percent_above_base: int = proto.Field(
proto.INT32,
number=2,
optional=True,
)

class InstanceSelection(proto.Message):
r"""Defines machines types and a rank to which the machines types
belong.
Expand Down Expand Up @@ -1175,6 +1224,11 @@ class InstanceSelectionResult(proto.Message):
optional=True,
)

provisioning_model_mix: ProvisioningModelMix = proto.Field(
proto.MESSAGE,
number=1,
message=ProvisioningModelMix,
)
instance_selection_list: MutableSequence[InstanceSelection] = proto.RepeatedField(
proto.MESSAGE,
number=2,
Expand Down Expand Up @@ -1230,6 +1284,9 @@ class DiskConfig(proto.Message):
r"""Specifies the config of disk options for a group of VM
instances.


.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields

Attributes:
boot_disk_type (str):
Optional. Type of the boot disk (default is "pd-standard").
Expand Down Expand Up @@ -1257,6 +1314,21 @@ class DiskConfig(proto.Message):
Valid values: "scsi" (Small Computer System Interface),
"nvme" (Non-Volatile Memory Express). See `local SSD
performance <https://cloud.google.com/compute/docs/disks/local-ssd#performance>`__.
boot_disk_provisioned_iops (int):
Optional. Indicates how many IOPS to provision for the disk.
This sets the number of I/O operations per second that the
disk can handle. Note: This field is only supported if
boot_disk_type is hyperdisk-balanced.

This field is a member of `oneof`_ ``_boot_disk_provisioned_iops``.
boot_disk_provisioned_throughput (int):
Optional. Indicates how much throughput to provision for the
disk. This sets the number of throughput mb per second that
the disk can handle. Values must be greater than or equal to
1. Note: This field is only supported if boot_disk_type is
hyperdisk-balanced.

This field is a member of `oneof`_ ``_boot_disk_provisioned_throughput``.
"""

boot_disk_type: str = proto.Field(
Expand All @@ -1275,6 +1347,16 @@ class DiskConfig(proto.Message):
proto.STRING,
number=4,
)
boot_disk_provisioned_iops: int = proto.Field(
proto.INT64,
number=5,
optional=True,
)
boot_disk_provisioned_throughput: int = proto.Field(
proto.INT64,
number=6,
optional=True,
)


class AuxiliaryNodeGroup(proto.Message):
Expand Down
12 changes: 12 additions & 0 deletions packages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_cluster_controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6227,6 +6227,8 @@ def test_create_cluster_rest_call_success(request_type):
"boot_disk_size_gb": 1792,
"num_local_ssds": 1494,
"local_ssd_interface": "local_ssd_interface_value",
"boot_disk_provisioned_iops": 2793,
"boot_disk_provisioned_throughput": 3464,
},
"is_preemptible": True,
"preemptibility": 1,
Expand All @@ -6244,6 +6246,10 @@ def test_create_cluster_rest_call_success(request_type):
"min_cpu_platform": "min_cpu_platform_value",
"min_num_instances": 1818,
"instance_flexibility_policy": {
"provisioning_model_mix": {
"standard_capacity_base": 2296,
"standard_capacity_percent_above_base": 3764,
},
"instance_selection_list": [
{
"machine_types": [
Expand Down Expand Up @@ -6624,6 +6630,8 @@ def test_update_cluster_rest_call_success(request_type):
"boot_disk_size_gb": 1792,
"num_local_ssds": 1494,
"local_ssd_interface": "local_ssd_interface_value",
"boot_disk_provisioned_iops": 2793,
"boot_disk_provisioned_throughput": 3464,
},
"is_preemptible": True,
"preemptibility": 1,
Expand All @@ -6641,6 +6649,10 @@ def test_update_cluster_rest_call_success(request_type):
"min_cpu_platform": "min_cpu_platform_value",
"min_num_instances": 1818,
"instance_flexibility_policy": {
"provisioning_model_mix": {
"standard_capacity_base": 2296,
"standard_capacity_percent_above_base": 3764,
},
"instance_selection_list": [
{
"machine_types": [
Expand Down
6 changes: 6 additions & 0 deletions packages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_node_group_controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3175,6 +3175,8 @@ def test_create_node_group_rest_call_success(request_type):
"boot_disk_size_gb": 1792,
"num_local_ssds": 1494,
"local_ssd_interface": "local_ssd_interface_value",
"boot_disk_provisioned_iops": 2793,
"boot_disk_provisioned_throughput": 3464,
},
"is_preemptible": True,
"preemptibility": 1,
Expand All @@ -3192,6 +3194,10 @@ def test_create_node_group_rest_call_success(request_type):
"min_cpu_platform": "min_cpu_platform_value",
"min_num_instances": 1818,
"instance_flexibility_policy": {
"provisioning_model_mix": {
"standard_capacity_base": 2296,
"standard_capacity_percent_above_base": 3764,
},
"instance_selection_list": [
{
"machine_types": [
Expand Down
18 changes: 18 additions & 0 deletions ...ages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_workflow_template_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5865,6 +5865,8 @@ def test_create_workflow_template_rest_call_success(request_type):
"boot_disk_size_gb": 1792,
"num_local_ssds": 1494,
"local_ssd_interface": "local_ssd_interface_value",
"boot_disk_provisioned_iops": 2793,
"boot_disk_provisioned_throughput": 3464,
},
"is_preemptible": True,
"preemptibility": 1,
Expand All @@ -5882,6 +5884,10 @@ def test_create_workflow_template_rest_call_success(request_type):
"min_cpu_platform": "min_cpu_platform_value",
"min_num_instances": 1818,
"instance_flexibility_policy": {
"provisioning_model_mix": {
"standard_capacity_base": 2296,
"standard_capacity_percent_above_base": 3764,
},
"instance_selection_list": [
{
"machine_types": [
Expand Down Expand Up @@ -6606,6 +6612,8 @@ def test_instantiate_inline_workflow_template_rest_call_success(request_type):
"boot_disk_size_gb": 1792,
"num_local_ssds": 1494,
"local_ssd_interface": "local_ssd_interface_value",
"boot_disk_provisioned_iops": 2793,
"boot_disk_provisioned_throughput": 3464,
},
"is_preemptible": True,
"preemptibility": 1,
Expand All @@ -6623,6 +6631,10 @@ def test_instantiate_inline_workflow_template_rest_call_success(request_type):
"min_cpu_platform": "min_cpu_platform_value",
"min_num_instances": 1818,
"instance_flexibility_policy": {
"provisioning_model_mix": {
"standard_capacity_base": 2296,
"standard_capacity_percent_above_base": 3764,
},
"instance_selection_list": [
{
"machine_types": [
Expand Down Expand Up @@ -7097,6 +7109,8 @@ def test_update_workflow_template_rest_call_success(request_type):
"boot_disk_size_gb": 1792,
"num_local_ssds": 1494,
"local_ssd_interface": "local_ssd_interface_value",
"boot_disk_provisioned_iops": 2793,
"boot_disk_provisioned_throughput": 3464,
},
"is_preemptible": True,
"preemptibility": 1,
Expand All @@ -7114,6 +7128,10 @@ def test_update_workflow_template_rest_call_success(request_type):
"min_cpu_platform": "min_cpu_platform_value",
"min_num_instances": 1818,
"instance_flexibility_policy": {
"provisioning_model_mix": {
"standard_capacity_base": 2296,
"standard_capacity_percent_above_base": 3764,
},
"instance_selection_list": [
{
"machine_types": [
Expand Down
Loading