From 2fc56227ea8716bf155d3eb91259a3ffc2145250 Mon Sep 17 00:00:00 2001 From: PapaCharlie Date: Tue, 24 Sep 2024 17:13:34 -0400 Subject: [PATCH] Make flaky tests run more --- .../d2/loadbalancer/TestDynamicClient.java | 45 ++++++++++--------- .../linkedin/test/util/retry/TenRetries.java | 29 ++++++++++++ 2 files changed, 54 insertions(+), 20 deletions(-) create mode 100644 test-util/src/main/java/com/linkedin/test/util/retry/TenRetries.java diff --git a/d2-int-test/src/test/java/com/linkedin/d2/loadbalancer/TestDynamicClient.java b/d2-int-test/src/test/java/com/linkedin/d2/loadbalancer/TestDynamicClient.java index a1f8733e15..fc6cb3f380 100644 --- a/d2-int-test/src/test/java/com/linkedin/d2/loadbalancer/TestDynamicClient.java +++ b/d2-int-test/src/test/java/com/linkedin/d2/loadbalancer/TestDynamicClient.java @@ -19,7 +19,7 @@ import com.linkedin.r2.message.rest.RestRequestBuilder; import com.linkedin.r2.message.rest.RestResponse; import com.linkedin.r2.util.NamedThreadFactory; -import com.linkedin.test.util.retry.ThreeRetries; +import com.linkedin.test.util.retry.TenRetries; import java.io.File; import java.lang.management.ManagementFactory; import java.net.URI; @@ -141,7 +141,7 @@ public void teardown() * the requests sending from the clients should result in an even distribution. The total call count * received by a single server should not deviate by more than 15% of the average. */ - @Test(retryAnalyzer = ThreeRetries.class) + @Test(retryAnalyzer = TenRetries.class) public void testBalancedLoadDistribution() { SimpleLoadBalancerStateTest.TestListener listener = new SimpleLoadBalancerStateTest.TestListener(); @@ -198,7 +198,7 @@ public void testBalancedLoadDistribution() * After the update event is received by the ZK event subscriber. One request is required to actually trigger the * load balancer state and hash ring changes. */ - @Test(retryAnalyzer = ThreeRetries.class) + @Test(retryAnalyzer = TenRetries.class) public void testD2WeightLessThanOne() { SimpleLoadBalancerStateTest.TestListener listener = new SimpleLoadBalancerStateTest.TestListener(); @@ -218,11 +218,12 @@ public void testD2WeightLessThanOne() throw new RuntimeException("Failed the test because thread was interrupted"); } - try { + try + { // Change the D2 weight of server:2851 to 0.5 invokeD2ChangeWeightJmx(new ObjectName("com.linkedin.d2:type=\"server:2851\""), 0.5); - // Wait 5ms for the change to propagate - Thread.sleep(5); + // Wait 50ms for the change to propagate + Thread.sleep(50); } catch (Exception e) { fail("Failed to invoke d2 weight change jmx", e); } @@ -284,7 +285,7 @@ public void testD2WeightLessThanOne() * And if we further increase the weight to 4.0. The host will receive 4x the traffic of the other hosts * (with a tolerance of 15%). */ - @Test(retryAnalyzer = ThreeRetries.class) + @Test(retryAnalyzer = TenRetries.class) public void testD2WeightGreaterThanOne() { SimpleLoadBalancerStateTest.TestListener listener = new SimpleLoadBalancerStateTest.TestListener(); @@ -304,11 +305,12 @@ public void testD2WeightGreaterThanOne() throw new RuntimeException("Failed the test because thread was interrupted"); } - try { + try + { // Change the D2 weight of server:2851 to 2.0 invokeD2ChangeWeightJmx(new ObjectName("com.linkedin.d2:type=\"server:2851\""), 2); - // Wait 5ms for the change to propagate - Thread.sleep(5); + // Wait 50ms for the change to propagate + Thread.sleep(50); } catch (Exception e) { fail("Failed to invoke d2 weight change jmx", e); } @@ -363,11 +365,12 @@ public void testD2WeightGreaterThanOne() } } - try { + try + { // Change the D2 weight of server:2851 to 4.0 invokeD2ChangeWeightJmx(new ObjectName("com.linkedin.d2:type=\"server:2851\""), 4); - // Wait 5ms for the change to propagate - Thread.sleep(5); + // Wait 50ms for the change to propagate + Thread.sleep(50); } catch (Exception e) { fail("Failed to invoke d2 weight change jmx", e); } @@ -405,7 +408,7 @@ public void testD2WeightGreaterThanOne() * 2. The host start receiving traffic and has a health score > 0.5. * The host will then be kicked out of the recovery program and continue to recover/degrade using normal up/downStep. */ - @Test(retryAnalyzer = ThreeRetries.class) + @Test(retryAnalyzer = TenRetries.class) public void testHostMarkDownAndMarkUp() { SimpleLoadBalancerStateTest.TestListener listener = new SimpleLoadBalancerStateTest.TestListener(); @@ -425,11 +428,12 @@ public void testHostMarkDownAndMarkUp() throw new RuntimeException("Failed the test because thread was interrupted"); } - try { + try + { // Mark down server:2851 invokeMarkDownJmx(new ObjectName("com.linkedin.d2:type=\"server:2851\"")); - // Wait 5ms for the change to propagate - Thread.sleep(5); + // Wait 50ms for the change to propagate + Thread.sleep(50); } catch (Exception e) { fail("Failed to invoke d2 weight change jmx", e); } @@ -459,11 +463,12 @@ public void testHostMarkDownAndMarkUp() throw new RuntimeException("Failed the test because thread was interrupted"); } - try { + try + { // Mark up server:2851 invokeMarkUpJmx(new ObjectName("com.linkedin.d2:type=\"server:2851\"")); - // Wait 5ms for the change to propagate - Thread.sleep(5); + // Wait 50ms for the change to propagate + Thread.sleep(50); } catch (Exception e) { fail("Failed to invoke d2 weight change jmx", e); } diff --git a/test-util/src/main/java/com/linkedin/test/util/retry/TenRetries.java b/test-util/src/main/java/com/linkedin/test/util/retry/TenRetries.java new file mode 100644 index 0000000000..415d2d5c42 --- /dev/null +++ b/test-util/src/main/java/com/linkedin/test/util/retry/TenRetries.java @@ -0,0 +1,29 @@ +/* + Copyright (c) 2020 LinkedIn Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.linkedin.test.util.retry; + + +/** + * Allows ten retries for a given test method. This is useful for tests that are especially flaky. + */ +public class TenRetries extends Retries +{ + public TenRetries() + { + super(10); + } +}