Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

symmetric deploy #401

Merged
merged 7 commits into from
Jan 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,335 @@
name: Module Controller Integration Test Symmetric batch deploy
run-name: ${{ github.actor }} pushed module-controller code

on:
push:
branches:
- master
paths:
- 'module-controller/**'

pull_request:
branches:
- master
paths:
- 'module-controller/**'

# enable manually running the workflow
workflow_dispatch:

env:
CGO_ENABLED: 0
GOOS: linux
WORK_DIR: module-controller
TAG: ci-test-master-latest
DOCKERHUB_REGISTRY: serverless-registry.cn-shanghai.cr.aliyuncs.com
MODULE_CONTROLLER_IMAGE_PATH: opensource/test/module-controller
INTEGRATION_TESTS_IMAGE_PATH: opensource/test/module-controller-integration-tests
POD_NAMESPACE: default

defaults:
run:
working-directory: module-controller

jobs:
unit-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3

- name: Docker login
uses: docker/login-action@v2.2.0
with:
registry: ${{ env.DOCKERHUB_REGISTRY }}
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
logout: false

- name: Set up Docker buildx
uses: docker/setup-buildx-action@v2

- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ hashFiles('${{ env.WORK_DIR }}/*Dockerfile') }}

- name: Build and push module-controller Docker images
uses: docker/build-push-action@v4.1.1
with:
context: ${{ env.WORK_DIR }}
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
file: ${{ env.WORK_DIR }}/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.MODULE_CONTROLLER_IMAGE_PATH }}:${{ env.TAG }}

- run: sleep 30

- name: Set up Minikube
run: |
curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
sudo install minikube-linux-amd64 /usr/local/bin/minikube
- name: Start Minikube
run: minikube start

- name: Prepare development env
run: |
kubectl apply -f config/crd/bases/serverless.alipay.com_moduledeployments.yaml
kubectl apply -f config/crd/bases/serverless.alipay.com_modulereplicasets.yaml
kubectl apply -f config/crd/bases/serverless.alipay.com_modules.yaml
kubectl apply -f config/crd/bases/serverless.alipay.com_moduletemplates.yaml
kubectl apply -f config/rbac/role.yaml
kubectl apply -f config/rbac/role_binding.yaml
kubectl apply -f config/rbac/service_account.yaml
kubectl apply -f config/samples/ci/dynamic-stock-batch-deployment.yaml
kubectl apply -f config/samples/ci/module-deployment-controller.yaml
kubectl apply -f config/samples/ci/dynamic-stock-service.yaml
- run: sleep 60

- name: minikube logs
run: minikube logs

- name: get pod
run: |
kubectl get pod
- name: wait base pod available
run: |
kubectl wait --for=condition=available deployment/dynamic-stock-deployment --timeout=300s
- name: wait module controller pod available
run: |
kubectl wait --for=condition=available deployment/module-controller --timeout=300s
- name: apply moduledeployment symmetric batch release
run: |
kubectl apply -f config/samples/ci/module-deployment_v1alpha1_moduledeployment_symmetric_batch_deploy_provider.yaml
- name: get moduledeployment
run: |
kubectl get moduledeployment
- name: get modulereplicaset
run: |
kubectl get modulereplicaset
- run: sleep 15

- name: get module
run: |
kubectl get module -oyaml
- name: exist module
run: |
moduleCount=$(kubectl get module | wc -l)
if [[ $moduleCount -lt 1 ]]; then
echo "ERROR: 创建首批 module 资源失败"
exit 1
fi
- name: wait module available
run: |
# 定义要等待的资源名称和字段值
modulename=$(kubectl get module -o name)
desired_field_value="Available"
# 定义等待的超时时间(以秒为单位)
timeout_seconds=300
start_time=$(date +%s)
end_time=$((start_time + timeout_seconds))
while true; do
current_time=$(date +%s)
if [ $current_time -gt $end_time ]; then
echo "等待超时"
exit 1
fi
# 使用 kubectl get 命令获取资源对象的详细信息,并提取自定义字段的值
field_value=$(kubectl get $modulename -o custom-columns=STATUS:.status.status --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
echo "字段值已满足条件"
exit 0
else
echo "等待字段值满足条件..."
sleep 5 # 等待一段时间后再次检查
fi
done
- name: check moduledeployment pause
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间(以秒为单位)
timeout_seconds=300
start_time=$(date +%s)
end_time=$((start_time + timeout_seconds))
while true; do
current_time=$(date +%s)
if [ $current_time -gt $end_time ]; then
echo "等待超时"
exit 1
fi
# 使用 kubectl get 命令获取资源对象的详细信息,并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
echo "字段值已满足条件,执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
sleep 5 # 等待一段时间后再次检查
fi
done
- run: sleep 15

- name: get module
run: |
kubectl get module
- name: exist module more then 1
run: |
moduleCount=$(kubectl get module | wc -l)
if [[ $moduleCount -lt 2 ]]; then
echo "ERROR: 创建第二批 module 资源失败"
exit 1
fi
- name: check moduledeployment pause
run: |
# 定义要等待的资源名称和字段值
moduledeploymentname=$(kubectl get moduledeployment -o name)
desired_field_value=true
# 定义等待的超时时间(以秒为单位)
timeout_seconds=300
start_time=$(date +%s)
end_time=$((start_time + timeout_seconds))
while true; do
current_time=$(date +%s)
if [ $current_time -gt $end_time ]; then
echo "等待超时"
exit 1
fi
# 使用 kubectl get 命令获取资源对象的详细信息,并提取自定义字段的值
field_value=$(kubectl get $moduledeploymentname -o custom-columns=PAUSE:.spec.pause --no-headers)
if [ "$field_value" == "$desired_field_value" ]; then
echo "字段值已满足条件,执行分组确认"
kubectl patch $moduledeploymentname -p '{"spec":{"pause":false}}' --type=merge
exit 0
else
echo "等待字段值满足条件..."
sleep 5 # 等待一段时间后再次检查
fi
done
- name: get module
run: |
kubectl get module
- name: exist module more then 2
run: |
moduleCount=$(kubectl get module | wc -l)
if [[ $moduleCount -lt 3 ]]; then
echo "ERROR: 创建第三批 module 资源失败"
exit 1
fi
- name: wait module available
run: |
# 定义要等待的资源类型和期望的字段值
moduletype="module"
desired_field_value="Available"
# 定义等待的超时时间(以秒为单位)
timeout_seconds=300
start_time=$(date +%s)
end_time=$((start_time + timeout_seconds))
while true; do
current_time=$(date +%s)
if [ $current_time -gt $end_time ]; then
echo "等待超时"
exit 1
fi
# 获取所有的资源对象名,并循环处理
for modulename in $(kubectl get $moduletype -o name); do
# 使用 kubectl get 命令获取每个资源对象的详细信息,并提取自定义字段的值
field_value=$(kubectl get $modulename -o custom-columns=STATUS:.status.status --no-headers)
# 检查字段值是否满足期望
if [ "$field_value" != "$desired_field_value" ]; then
echo "等待字段值满足条件..."
sleep 5 # 等待一段时间后再次检查
continue 2 # 如果字段值未满足,则跳出循环,进入下一轮等待
fi
done
# 如果所有资源对象的字段值都满足期望,则结束脚本
echo "字段值已满足条件"
exit 0
done
- name: batch release successfully then check module install
run: |
label_selector="app=dynamic-stock"
max_attempts=10
timeout=300
interval=30
for ((i=0; i<$max_attempts; i++)); do
# 获取满足标签选择器条件的所有Pod
podnames=($(kubectl get pods -l $label_selector -o jsonpath='{.items[*].metadata.name}'))
# 初始化一个变量,用于检测是否所有Pod都满足条件
all_pods_condition_met=true
# 遍历所有Pod进行日志检索
for podname in "${podnames[@]}"; do
log_entry=$(kubectl exec -it $podname -- sh -c 'grep "Install Biz: provider:1.0.2 success" ~/logs/sofa-ark/*.log || true' 2>/dev/null)
# 如果没有找到日志条目,则将标志设置为false
if [ -z "$log_entry" ]; then
all_pods_condition_met=false
break
fi
done
# 如果所有Pod都满足条件,则退出循环
if $all_pods_condition_met; then
echo "所有Pod都满足条件。"
break
fi
# 如果这不是最后一次尝试,则等待一段时间后继续
if [ $i -lt $((max_attempts-1)) ]; then
echo "有些Pod未满足条件。等待 $interval 秒后进行下一次尝试。"
sleep $interval
else
# 如果是最后一次尝试,则输出超时消息
echo "已达到最大尝试次数。有些Pod未满足条件。"
fi
done
- name: delete deployment
run: |
kubectl delete -n default deployment dynamic-stock-deployment
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
apiVersion: serverless.alipay.com/v1alpha1
kind: ModuleDeployment
metadata:
labels:
app.kubernetes.io/name: moduledeployment
app.kubernetes.io/instance: moduledeployment-sample
app.kubernetes.io/part-of: module-controller
app.kubernetes.io/managed-by: kustomize
app.kubernetes.io/created-by: module-controller
name: moduledeployment-sample-provider
spec:
baseDeploymentName: dynamic-stock-deployment
template:
spec:
module:
name: provider
version: '1.0.2'
url: http://serverless-opensource.oss-cn-shanghai.aliyuncs.com/module-packages/stable/dynamic-provider-1.0.2-ark-biz.jar
replicas: -1
operationStrategy:
needConfirm: true
grayTimeBetweenBatchSeconds: 120
useBeta: false
batchCount: 3
upgradePolicy: install_then_uninstall
schedulingStrategy:
schedulingPolicy: scatter

Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,16 @@ func (r *ModuleDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
return ctrl.Result{}, utils.Error(err, "Failed to get moduleDeployment", "moduleDeploymentName", moduleDeployment.Name)
}

// update symmetric moduleDeployment replicas
if moduleDeployment.Spec.Replicas == -1 {
deployment := &v1.Deployment{}
err := r.Client.Get(ctx, types.NamespacedName{Namespace: moduleDeployment.Namespace, Name: moduleDeployment.Spec.BaseDeploymentName}, deployment)
if err != nil {
return ctrl.Result{}, utils.Error(err, "Failed to get deployment", "deploymentName", deployment.Name)
}
moduleDeployment.Spec.Replicas = deployment.Status.AvailableReplicas
}

if moduleDeployment.DeletionTimestamp != nil {
event.PublishModuleDeploymentDeleteEvent(r.Client, ctx, moduleDeployment)
if !utils.HasFinalizer(&moduleDeployment.ObjectMeta, finalizer.ModuleReplicaSetExistedFinalizer) &&
Expand Down
Loading
Loading