From 71f178068e3470d1734b85a8b7d7570d419ff560 Mon Sep 17 00:00:00 2001 From: Rick Zhou Date: Wed, 21 Jun 2023 18:01:33 -0700 Subject: [PATCH 1/7] Add missing interface method in LoadBalancerWithFacilitiesDelegator (#914) --- CHANGELOG.md | 6 +++++- .../LoadBalancerWithFacilitiesDelegator.java | 17 +++++++++++++++++ gradle.properties | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e17735a4fa..f969a197c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.43.2] - 2023-06-21 +- Add missing interface method in LoadBalancerWithFacilitiesDelegator + ## [29.43.1] - 2023-06-20 - mute SD update receipt event for initial request on a new cluster @@ -5481,7 +5484,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.1...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.2...master +[29.43.2]: https://github.com/linkedin/rest.li/compare/v29.43.1...v29.43.2 [29.43.1]: https://github.com/linkedin/rest.li/compare/v29.43.0...v29.43.1 [29.43.0]: https://github.com/linkedin/rest.li/compare/v29.42.4...v29.43.0 [29.42.4]: https://github.com/linkedin/rest.li/compare/v29.42.3...v29.42.4 diff --git a/d2/src/main/java/com/linkedin/d2/balancer/LoadBalancerWithFacilitiesDelegator.java b/d2/src/main/java/com/linkedin/d2/balancer/LoadBalancerWithFacilitiesDelegator.java index 1942c5299b..b6a0df1bd9 100644 --- a/d2/src/main/java/com/linkedin/d2/balancer/LoadBalancerWithFacilitiesDelegator.java +++ b/d2/src/main/java/com/linkedin/d2/balancer/LoadBalancerWithFacilitiesDelegator.java @@ -2,7 +2,9 @@ import com.linkedin.common.callback.Callback; import com.linkedin.common.util.None; +import com.linkedin.d2.balancer.properties.ClusterProperties; import com.linkedin.d2.balancer.properties.ServiceProperties; +import com.linkedin.d2.balancer.properties.UriProperties; import com.linkedin.d2.balancer.util.ClusterInfoProvider; import com.linkedin.d2.balancer.util.hashing.HashRingProvider; import com.linkedin.d2.balancer.util.partitions.PartitionInfoProvider; @@ -11,6 +13,8 @@ import com.linkedin.r2.message.RequestContext; import com.linkedin.r2.transport.common.TransportClientFactory; import com.linkedin.r2.transport.common.bridge.client.TransportClient; +import org.apache.commons.lang3.tuple.Pair; + /** * Abstract class implementing the delegating methods for {@link LoadBalancerWithFacilities} @@ -84,4 +88,17 @@ public ServiceProperties getLoadBalancedServiceProperties(String serviceName) th { return _loadBalancer.getLoadBalancedServiceProperties(serviceName); } + + @Override + public void getLoadBalancedServiceProperties(String serviceName, Callback clientCallback) + { + _loadBalancer.getLoadBalancedServiceProperties(serviceName, clientCallback); + } + + @Override + public void getLoadBalancedClusterAndUriProperties(String clusterName, + Callback> callback) + { + _loadBalancer.getLoadBalancedClusterAndUriProperties(clusterName, callback); + } } diff --git a/gradle.properties b/gradle.properties index 93dcd057ed..b4600a86fd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.43.1 +version=29.43.2 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true From 904fdd087562460653172474a73a922597465f56 Mon Sep 17 00:00:00 2001 From: Rick Zhou Date: Thu, 22 Jun 2023 16:17:33 -0700 Subject: [PATCH 2/7] Bump jfrog build-info-extractor-gradle to 4.32.0 (#916) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8c55962c65..453c2a71a5 100644 --- a/build.gradle +++ b/build.gradle @@ -19,7 +19,7 @@ buildscript { jcenter() } dependencies { - classpath 'org.jfrog.buildinfo:build-info-extractor-gradle:4.21.0' + classpath 'org.jfrog.buildinfo:build-info-extractor-gradle:4.32.0' classpath 'org.jacoco:org.jacoco.core:0.8.7' } } From 8ac0debd3384fc41936718ad83f02b40924b4738 Mon Sep 17 00:00:00 2001 From: Rick Zhou Date: Thu, 22 Jun 2023 16:46:26 -0700 Subject: [PATCH 3/7] Update CHANGELOG and trigger build (#917) --- CHANGELOG.md | 6 +++++- gradle.properties | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f969a197c3..d03ba3dfba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.43.3] - 2023-06-22 +- Bump jfrog build-info-extractor-gradle to 4.32.0 + ## [29.43.2] - 2023-06-21 - Add missing interface method in LoadBalancerWithFacilitiesDelegator @@ -5484,7 +5487,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.2...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.3...master +[29.43.3]: https://github.com/linkedin/rest.li/compare/v29.43.2...v29.43.3 [29.43.2]: https://github.com/linkedin/rest.li/compare/v29.43.1...v29.43.2 [29.43.1]: https://github.com/linkedin/rest.li/compare/v29.43.0...v29.43.1 [29.43.0]: https://github.com/linkedin/rest.li/compare/v29.42.4...v29.43.0 diff --git a/gradle.properties b/gradle.properties index b4600a86fd..8e0e32c2ec 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.43.2 +version=29.43.3 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true From 6e2ef796198b95e59479cc3900c2c6721ae75b46 Mon Sep 17 00:00:00 2001 From: Zoab Kapoor Date: Thu, 22 Jun 2023 19:49:23 -0600 Subject: [PATCH 4/7] Refactor ZookeeperServer: put uri property building logic to a method (#915) Refactor ZookeeperServer: put uri property building logic to a method --- CHANGELOG.md | 6 +- .../d2/balancer/servers/ZooKeeperServer.java | 57 +++++++++---------- gradle.properties | 2 +- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d03ba3dfba..1c53e3c649 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.43.4] - 2023-06-22 +- Refactor ZookeeperServer, making functionality to generate URI properties for node accessible to subclasses + ## [29.43.3] - 2023-06-22 - Bump jfrog build-info-extractor-gradle to 4.32.0 @@ -5487,7 +5490,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.3...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.4...master +[29.43.4]: https://github.com/linkedin/rest.li/compare/v29.43.3...v29.43.4 [29.43.3]: https://github.com/linkedin/rest.li/compare/v29.43.2...v29.43.3 [29.43.2]: https://github.com/linkedin/rest.li/compare/v29.43.1...v29.43.2 [29.43.1]: https://github.com/linkedin/rest.li/compare/v29.43.0...v29.43.1 diff --git a/d2/src/main/java/com/linkedin/d2/balancer/servers/ZooKeeperServer.java b/d2/src/main/java/com/linkedin/d2/balancer/servers/ZooKeeperServer.java index 689c4ae26e..abfab3506b 100644 --- a/d2/src/main/java/com/linkedin/d2/balancer/servers/ZooKeeperServer.java +++ b/d2/src/main/java/com/linkedin/d2/balancer/servers/ZooKeeperServer.java @@ -16,14 +16,6 @@ package com.linkedin.d2.balancer.servers; -import java.net.URI; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - import com.linkedin.common.callback.Callback; import com.linkedin.common.callback.FutureCallback; import com.linkedin.common.util.None; @@ -34,13 +26,18 @@ import com.linkedin.d2.balancer.properties.UriProperties; import com.linkedin.d2.discovery.event.D2ServiceDiscoveryEventHelper; import com.linkedin.d2.discovery.stores.zk.ZooKeeperEphemeralStore; - +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.linkedin.d2.discovery.util.LogUtil.info; -import static com.linkedin.d2.discovery.util.LogUtil.warn; +import static com.linkedin.d2.discovery.util.LogUtil.*; public class ZooKeeperServer implements LoadBalancerServer @@ -96,19 +93,6 @@ public void markUp(final String clusterName, @Override public void onSuccess(None none) { - Map> partitionDesc = new HashMap<>(); - partitionDesc.put(uri, partitionDataMap); - - Map> myUriSpecificProperties; - if (uriSpecificProperties != null && !uriSpecificProperties.isEmpty()) - { - myUriSpecificProperties = new HashMap<>(); - myUriSpecificProperties.put(uri, uriSpecificProperties); - } - else - { - myUriSpecificProperties = Collections.emptyMap(); - } if (_log.isInfoEnabled()) { @@ -130,7 +114,7 @@ public void onSuccess(None none) sb.append("}"); info(_log, sb); } - _store.put(clusterName, new UriProperties(clusterName, partitionDesc, myUriSpecificProperties), callback); + _store.put(clusterName, constructUriPropertiesForNode(clusterName, uri, partitionDataMap, uriSpecificProperties), callback); } @@ -361,17 +345,28 @@ public void shutdown() { callback.get(5, TimeUnit.SECONDS); info(_log, "shutting down complete"); - } - catch (TimeoutException e) - { + } catch (TimeoutException e) { warn(_log, "unable to shut down propertly"); - } - catch (InterruptedException | ExecutionException e) - { + } catch (InterruptedException | ExecutionException e) { warn(_log, "unable to shut down propertly.. got interrupt exception while waiting"); } } + protected UriProperties constructUriPropertiesForNode(final String clusterName, final URI uri, + final Map partitionDataMap, final Map uriSpecificProperties) { + Map> partitionDesc = new HashMap<>(); + partitionDesc.put(uri, partitionDataMap); + + Map> uriToUriSpecificProperties; + if (uriSpecificProperties != null && !uriSpecificProperties.isEmpty()) { + uriToUriSpecificProperties = new HashMap<>(); + uriToUriSpecificProperties.put(uri, uriSpecificProperties); + } else { + uriToUriSpecificProperties = Collections.emptyMap(); + } + return new UriProperties(clusterName, partitionDesc, uriToUriSpecificProperties); + } + private void storeGet(final String clusterName, final Callback callback) { if (_store == null) diff --git a/gradle.properties b/gradle.properties index 8e0e32c2ec..2c8e80cc08 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.43.3 +version=29.43.4 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true From f306f790cec199b0e1f55b3db17deca2e02fc4d1 Mon Sep 17 00:00:00 2001 From: Rick Zhou Date: Tue, 27 Jun 2023 13:48:09 -0700 Subject: [PATCH 5/7] Remove a delegated method in LoadBalancerWithFacilitiesDelegator (#919) --- CHANGELOG.md | 6 +++++- .../d2/balancer/LoadBalancerWithFacilitiesDelegator.java | 6 ------ gradle.properties | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c53e3c649..5ec28a08f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.43.5] - 2023-06-27 +- Remove a delegated method in LoadBalancerWithFacilitiesDelegator + ## [29.43.4] - 2023-06-22 - Refactor ZookeeperServer, making functionality to generate URI properties for node accessible to subclasses @@ -5490,7 +5493,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.4...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.5...master +[29.43.5]: https://github.com/linkedin/rest.li/compare/v29.43.4...v29.43.5 [29.43.4]: https://github.com/linkedin/rest.li/compare/v29.43.3...v29.43.4 [29.43.3]: https://github.com/linkedin/rest.li/compare/v29.43.2...v29.43.3 [29.43.2]: https://github.com/linkedin/rest.li/compare/v29.43.1...v29.43.2 diff --git a/d2/src/main/java/com/linkedin/d2/balancer/LoadBalancerWithFacilitiesDelegator.java b/d2/src/main/java/com/linkedin/d2/balancer/LoadBalancerWithFacilitiesDelegator.java index b6a0df1bd9..ea27db6ec6 100644 --- a/d2/src/main/java/com/linkedin/d2/balancer/LoadBalancerWithFacilitiesDelegator.java +++ b/d2/src/main/java/com/linkedin/d2/balancer/LoadBalancerWithFacilitiesDelegator.java @@ -89,12 +89,6 @@ public ServiceProperties getLoadBalancedServiceProperties(String serviceName) th return _loadBalancer.getLoadBalancedServiceProperties(serviceName); } - @Override - public void getLoadBalancedServiceProperties(String serviceName, Callback clientCallback) - { - _loadBalancer.getLoadBalancedServiceProperties(serviceName, clientCallback); - } - @Override public void getLoadBalancedClusterAndUriProperties(String clusterName, Callback> callback) diff --git a/gradle.properties b/gradle.properties index 2c8e80cc08..f485d5c7de 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.43.4 +version=29.43.5 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true From a59ac3da70a2b719f812711e8abb0580358175b8 Mon Sep 17 00:00:00 2001 From: Yanbo Ma Date: Tue, 11 Jul 2023 17:56:41 -0700 Subject: [PATCH 6/7] enable passing settings to custom partition accessor (#918) * enable passing settings to custom partition accessor * Rerun checks * rebase * update --------- Co-authored-by: Yanbo Ma --- CHANGELOG.md | 6 +- .../partitions/BasePartitionAccessor.java | 10 ++ .../partitions/PartitionAccessorFactory.java | 10 +- .../d2/discovery/util/TestD2Config.java | 103 ++++++++++++++++++ gradle.properties | 2 +- 5 files changed, 124 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec28a08f4..d4e6043149 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.43.6] - 2023-07-10 +- Enable passing settings to custom partition accessor + ## [29.43.5] - 2023-06-27 - Remove a delegated method in LoadBalancerWithFacilitiesDelegator @@ -5493,7 +5496,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.5...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.6...master +[29.43.6]: https://github.com/linkedin/rest.li/compare/v29.43.5...v29.43.6 [29.43.5]: https://github.com/linkedin/rest.li/compare/v29.43.4...v29.43.5 [29.43.4]: https://github.com/linkedin/rest.li/compare/v29.43.3...v29.43.4 [29.43.3]: https://github.com/linkedin/rest.li/compare/v29.43.2...v29.43.3 diff --git a/d2/src/main/java/com/linkedin/d2/balancer/util/partitions/BasePartitionAccessor.java b/d2/src/main/java/com/linkedin/d2/balancer/util/partitions/BasePartitionAccessor.java index fd60c20223..0365f18fcf 100644 --- a/d2/src/main/java/com/linkedin/d2/balancer/util/partitions/BasePartitionAccessor.java +++ b/d2/src/main/java/com/linkedin/d2/balancer/util/partitions/BasePartitionAccessor.java @@ -18,6 +18,8 @@ package com.linkedin.d2.balancer.util.partitions; import java.net.URI; +import java.util.Objects; + /** * BasePartitionAccessor returns partitionId according to the given URI. @@ -33,5 +35,13 @@ public interface BasePartitionAccessor * @throws PartitionAccessException see {@link PartitionAccessException} */ int getPartitionId(URI uri) throws PartitionAccessException; + + /** + * Given the setting of the partition accessor, check if the setting can be supported + * @return true if supportable + */ + default boolean checkSupportable(String settings) { + return Objects.equals(getClass().getSimpleName(), settings); + } } diff --git a/d2/src/main/java/com/linkedin/d2/balancer/util/partitions/PartitionAccessorFactory.java b/d2/src/main/java/com/linkedin/d2/balancer/util/partitions/PartitionAccessorFactory.java index c6cad0a5ee..f2a42aaff9 100644 --- a/d2/src/main/java/com/linkedin/d2/balancer/util/partitions/PartitionAccessorFactory.java +++ b/d2/src/main/java/com/linkedin/d2/balancer/util/partitions/PartitionAccessorFactory.java @@ -90,8 +90,8 @@ private static PartitionAccessor buildCustomizedPartitionAccessor(String cluster return DefaultPartitionAccessor.getInstance(); } - List requestedClassList = customizedProperties.getPartitionAccessorList(); - if (requestedClassList == null || requestedClassList.isEmpty()) + List partitionAccessorSettingsList = customizedProperties.getPartitionAccessorList(); + if (partitionAccessorSettingsList == null || partitionAccessorSettingsList.isEmpty()) { // If the no classList is defined, use the first class registered BasePartitionAccessor partitionAccessor = partitionAccessors.get(0); @@ -99,13 +99,13 @@ private static PartitionAccessor buildCustomizedPartitionAccessor(String cluster + " (out of " + partitionAccessors.size() + ") registration"); return new CustomizedPartitionAccessor(customizedProperties, partitionAccessor); } - for (String className : requestedClassList) + for (String setting : partitionAccessorSettingsList) { for (BasePartitionAccessor accessor : partitionAccessors) { - if (className.equals(accessor.getClass().getSimpleName())) + if (accessor.checkSupportable(setting)) { - _log.info("Use matched partitionAccessor for cluster: " + clusterName + ", class: " + accessor.getClass().getSimpleName()); + _log.info("Use matched partitionAccessor for cluster: " + clusterName + ", class: " + accessor.getClass().getSimpleName() + ", setting: " + setting); return new CustomizedPartitionAccessor(customizedProperties, accessor); } } diff --git a/d2/src/test/java/com/linkedin/d2/discovery/util/TestD2Config.java b/d2/src/test/java/com/linkedin/d2/discovery/util/TestD2Config.java index f939fd3659..5e2124f248 100644 --- a/d2/src/test/java/com/linkedin/d2/discovery/util/TestD2Config.java +++ b/d2/src/test/java/com/linkedin/d2/discovery/util/TestD2Config.java @@ -45,6 +45,7 @@ import com.linkedin.d2.discovery.stores.zk.ZooKeeperEphemeralStore; import com.linkedin.d2.discovery.stores.zk.ZooKeeperPermanentStore; import java.io.IOException; +import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -392,6 +393,108 @@ public int getMaxPartitionId() assertEquals(0, accessor.getPartitionId(URI.create(illegalUri))); } + // Test PartitionAccessorFactory: match ClassList with custom matching logic + @Test + public static void testPartitionAccessorFactoryWithCustomMatchingLogic() throws IOException, InterruptedException, URISyntaxException, Exception + { + @SuppressWarnings("serial") + final Map> clustersData = new HashMap>() + {{ + put("partitioned-cluster", Arrays.asList(new String[]{"partitioned-service-1", "partitioned-service-2"})); + }}; + + final Map partitionProperties = new HashMap<>(); + Map customized = new HashMap<>(); + List classList = Arrays.asList("TestPartitionAccessor1!Settings", "TestPartitionAccessor2"); + customized.put("partitionType", "CUSTOM"); + customized.put("partitionCount", "10"); + customized.put("partitionAccessorList", classList); + partitionProperties.put("partitionProperties", customized); + + final PartitionAccessorRegistry registry = new PartitionAccessorRegistry() + { + final private Map> _registry = new HashMap<>(); + + @Override + public void register(String clusterName, BasePartitionAccessor accessor) + { + List accessors = _registry.computeIfAbsent(clusterName, k -> new ArrayList<>()); + accessors.add(accessor); + } + + @Override + public List getPartitionAccessors(String clusterName) + { + return _registry.get(clusterName); + } + }; + + class TestPartitionAccessor1 implements PartitionAccessor + { + @Override + public int getPartitionId(URI uri) throws PartitionAccessException + { + return testGetPartitionId(uri); + } + + @Override + public int getMaxPartitionId() + { + return 10; + } + + @Override + public boolean checkSupportable(String settings) { + return settings.startsWith("TestPartitionAccessor1!"); + } + }; + + class TestPartitionAccessor2 implements PartitionAccessor + { + @Override + public int getPartitionId(URI uri) throws PartitionAccessException + { + return 8; + } + + @Override + public int getMaxPartitionId() + { + return 10; + } + }; + + PartitionAccessor testAccessor1 = new TestPartitionAccessor1(); + PartitionAccessor testAccessor2 = new TestPartitionAccessor2(); + + registry.register("partitioned-cluster", DefaultPartitionAccessor.getInstance()); + registry.register("partitioned-cluster", testAccessor1); + registry.register("partitioned-cluster", testAccessor2); + + D2ConfigTestUtil d2Conf = new D2ConfigTestUtil(clustersData, partitionProperties); + + assertEquals(d2Conf.runDiscovery(_zkHosts), 0); + + verifyPartitionProperties("partitioned-cluster", partitionProperties); + + final ClusterProperties clusterprops = getClusterProperties(_zkclient, "partitioned-cluster" ); + + final PartitionAccessor accessor = PartitionAccessorFactory.getPartitionAccessor("partitioned-cluster", + registry, clusterprops.getPartitionProperties()); + + final String legalUri1 = "/profiles?field=position&id=100"; + final String legalUri2 = "/cap?wid=99&id=176&randid=301"; + final String legalUri3 = "/seas?id=3324"; + final String illegalUri = "/?id=1000000000000000000000000000000000000000000000111111111"; + Field realAccessorField = accessor.getClass().getDeclaredField("_partitionAccessor"); + realAccessorField.setAccessible(true); + assertEquals(TestPartitionAccessor1.class, realAccessorField.get(accessor).getClass()); + assertEquals(0, accessor.getPartitionId(URI.create(legalUri1))); + assertEquals(1, accessor.getPartitionId(URI.create(legalUri2))); + assertEquals(2, accessor.getPartitionId(URI.create(legalUri3))); + assertEquals(0, accessor.getPartitionId(URI.create(illegalUri))); + } + // Test PartitionAccessorFactory: empty ClassList @Test public static void testPartitionAccessorFactoryWithEmptyClassList() throws IOException, InterruptedException, URISyntaxException, Exception diff --git a/gradle.properties b/gradle.properties index f485d5c7de..51ab1bd927 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.43.5 +version=29.43.6 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true From 30a16abbeb10e179da55d75d90c459150b74be84 Mon Sep 17 00:00:00 2001 From: John Stewart Date: Tue, 11 Jul 2023 18:59:02 -0700 Subject: [PATCH 7/7] Make D2 ZKFS FileStore file extension fully customizable. (#922) * Make D2 ZKFS FileStore file extension fully customizable. Currently it is customizable in the FileStore but the dependent FileSystemDirectory and related classes are hardcoded to use the default extension, so customization doesn't fully work. This fixes that so the custom extension can be used throughout. * Add backward-compatible method signatures to `FileSystemDirectory`. * Backwards compatibility for FSBasedDownStreamServiceFetcher. --- CHANGELOG.md | 6 +++- .../d2/balancer/util/FileSystemDirectory.java | 32 +++++++++++++++---- .../FSBasedDownstreamServicesFetcher.java | 9 +++++- .../d2/discovery/stores/file/FileStore.java | 2 +- gradle.properties | 2 +- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4e6043149..9186055961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and what APIs have changed, if applicable. ## [Unreleased] +## [29.43.7] - 2023-07-11 +- Make file extension of D2 ZKFS file store fully customizable. + ## [29.43.6] - 2023-07-10 - Enable passing settings to custom partition accessor @@ -5496,7 +5499,8 @@ patch operations can re-use these classes for generating patch messages. ## [0.14.1] -[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.6...master +[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.7...master +[29.43.7]: https://github.com/linkedin/rest.li/compare/v29.43.6...v29.43.7 [29.43.6]: https://github.com/linkedin/rest.li/compare/v29.43.5...v29.43.6 [29.43.5]: https://github.com/linkedin/rest.li/compare/v29.43.4...v29.43.5 [29.43.4]: https://github.com/linkedin/rest.li/compare/v29.43.3...v29.43.4 diff --git a/d2/src/main/java/com/linkedin/d2/balancer/util/FileSystemDirectory.java b/d2/src/main/java/com/linkedin/d2/balancer/util/FileSystemDirectory.java index 19ebcab9c0..7f5c98ec90 100644 --- a/d2/src/main/java/com/linkedin/d2/balancer/util/FileSystemDirectory.java +++ b/d2/src/main/java/com/linkedin/d2/balancer/util/FileSystemDirectory.java @@ -44,39 +44,51 @@ public class FileSystemDirectory private final String _d2FsDirPath; private String _d2ServicePath; + private final String _fsFileExtension; public FileSystemDirectory(String d2FsDirPath, String d2ServicePath) + { + this(d2FsDirPath, d2ServicePath, FILE_STORE_EXTENSION); + } + + public FileSystemDirectory(String d2FsDirPath, String d2ServicePath, String fsFileExtension) { _d2FsDirPath = d2FsDirPath; _d2ServicePath = d2ServicePath; + _fsFileExtension = fsFileExtension; } public List getServiceNames() { - return getFileListWithoutExtension(getServiceDirectory(_d2FsDirPath, _d2ServicePath)); + return getFileListWithoutExtension(getServiceDirectory(_d2FsDirPath, _d2ServicePath), _fsFileExtension); } public void removeAllServicesWithExcluded(Set excludedServices) { List serviceNames = getServiceNames(); serviceNames.removeAll(excludedServices); - removeAllPropertiesFromDirectory(getServiceDirectory(_d2FsDirPath, _d2ServicePath), serviceNames); + removeAllPropertiesFromDirectory(getServiceDirectory(_d2FsDirPath, _d2ServicePath), serviceNames, _fsFileExtension); } public void removeAllClustersWithExcluded(Set excludedClusters) { List serviceNames = getClusterNames(); serviceNames.removeAll(excludedClusters); - removeAllPropertiesFromDirectory(getServiceDirectory(_d2FsDirPath, _d2ServicePath), serviceNames); + removeAllPropertiesFromDirectory(getServiceDirectory(_d2FsDirPath, _d2ServicePath), serviceNames, _fsFileExtension); } public static void removeAllPropertiesFromDirectory(String path, List properties) + { + removeAllPropertiesFromDirectory(path, properties, FILE_STORE_EXTENSION); + } + + public static void removeAllPropertiesFromDirectory(String path, List properties, String fileExtension) { for (String property : properties) { try { - Files.deleteIfExists(Paths.get(path + File.separator + property + FileSystemDirectory.FILE_STORE_EXTENSION)); + Files.deleteIfExists(Paths.get(path + File.separator + property + fileExtension)); } catch (IOException e) { LOG.warn("IO Error, continuing deletion", e); @@ -86,13 +98,19 @@ public static void removeAllPropertiesFromDirectory(String path, List pr public List getClusterNames() { - return getFileListWithoutExtension(getClusterDirectory(_d2ServicePath)); + return getFileListWithoutExtension(getClusterDirectory(_d2ServicePath), + _fsFileExtension); } public static List getFileListWithoutExtension(String path) + { + return getFileListWithoutExtension(path, FILE_STORE_EXTENSION); + } + + public static List getFileListWithoutExtension(String path, String fileExtension) { File dir = new File(path); - File[] files = dir.listFiles((dir1, name) -> name.endsWith(FileSystemDirectory.FILE_STORE_EXTENSION)); + File[] files = dir.listFiles((dir1, name) -> name.endsWith(fileExtension)); if (files == null) { return Collections.emptyList(); @@ -100,7 +118,7 @@ public static List getFileListWithoutExtension(String path) // cleaning the list from the extension return Arrays.stream(files) - .map(file -> file.getName().replace(FILE_STORE_EXTENSION, "")) + .map(file -> file.getName().replace(fileExtension, "")) .collect(Collectors.toList()); } diff --git a/d2/src/main/java/com/linkedin/d2/balancer/util/downstreams/FSBasedDownstreamServicesFetcher.java b/d2/src/main/java/com/linkedin/d2/balancer/util/downstreams/FSBasedDownstreamServicesFetcher.java index eb5394d8fc..211891334c 100644 --- a/d2/src/main/java/com/linkedin/d2/balancer/util/downstreams/FSBasedDownstreamServicesFetcher.java +++ b/d2/src/main/java/com/linkedin/d2/balancer/util/downstreams/FSBasedDownstreamServicesFetcher.java @@ -31,17 +31,24 @@ public class FSBasedDownstreamServicesFetcher implements DownstreamServicesFetch { private final String _d2FsPath; private final String _d2ServicePath; + private final String _fsFileExtension; public FSBasedDownstreamServicesFetcher(String d2FsPath, String d2ServicePath) + { + this(d2FsPath, d2ServicePath, FileSystemDirectory.FILE_STORE_EXTENSION); + } + + public FSBasedDownstreamServicesFetcher(String d2FsPath, String d2ServicePath, String fsFileExtension) { _d2FsPath = d2FsPath; _d2ServicePath = d2ServicePath; + _fsFileExtension = fsFileExtension; } @Override public void getServiceNames(SuccessCallback> callback) { - FileSystemDirectory fsDirectory = new FileSystemDirectory(_d2FsPath, _d2ServicePath); + FileSystemDirectory fsDirectory = new FileSystemDirectory(_d2FsPath, _d2ServicePath, _fsFileExtension); callback.onSuccess(fsDirectory.getServiceNames()); } } diff --git a/d2/src/main/java/com/linkedin/d2/discovery/stores/file/FileStore.java b/d2/src/main/java/com/linkedin/d2/discovery/stores/file/FileStore.java index eede39d738..bcad5deaf0 100644 --- a/d2/src/main/java/com/linkedin/d2/discovery/stores/file/FileStore.java +++ b/d2/src/main/java/com/linkedin/d2/discovery/stores/file/FileStore.java @@ -185,7 +185,7 @@ public Map getAll() List propertyNames; try { - propertyNames = FileSystemDirectory.getFileListWithoutExtension(_fsPath); + propertyNames = FileSystemDirectory.getFileListWithoutExtension(_fsPath, _fsFileExtension); Map result = new HashMap<>(); for (String propertyName : propertyNames) diff --git a/gradle.properties b/gradle.properties index 51ab1bd927..080bcda079 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=29.43.6 +version=29.43.7 group=com.linkedin.pegasus org.gradle.configureondemand=true org.gradle.parallel=true