voldemort.utils
Class RebalanceUtils

java.lang.Object
  voldemort.utils.RebalanceUtils

public class RebalanceUtils
extends java.lang.Object

RebalanceUtils provide basic functionality for rebalancing.

Field Summary

static java.util.List<java.lang.String>	canRebalanceList
static java.lang.String	finalClusterFileName
static java.lang.String	initialClusterFileName

Constructor Summary

RebalanceUtils()

Method Summary

static Node	addPartitionToNode(Node node, java.lang.Integer donatedPartition) Add a partition to the node provided
static Node	addPartitionToNode(Node node, java.util.Set<java.lang.Integer> donatedPartitions) Add the set of partitions to the node provided
static void	assertSameDonor(java.util.List<RebalancePartitionsInfo> partitionInfos, int expectedDonorId) Given a list of partition informations check all of them belong to the same donor node
static boolean	checkKeyBelongsToNode(byte[] key, int nodeId, Cluster cluster, StoreDefinition storeDef)
static java.util.List<java.lang.Integer>	checkKeyBelongsToPartition(byte[] key, java.util.Set<Pair<java.lang.Integer,java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>>>> stealerNodeToMappingTuples, Cluster cluster, StoreDefinition storeDef) Given a key and a list of steal infos give back a list of stealer node ids which will steal this.
static boolean	checkKeyBelongsToPartition(int nodeId, byte[] key, java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> replicaToPartitionList, Cluster cluster, StoreDefinition storeDef) Check that the key belongs to one of the partitions in the map of replica type to partitions
static boolean	checkKeyBelongsToPartition(java.util.List<java.lang.Integer> keyPartitions, java.util.List<java.lang.Integer> nodePartitions, java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> replicaToPartitionList) Check that the key belongs to one of the partitions in the map of replica type to partitions
static boolean	checkPartitionBelongsToNode(int partition, int nodeId, Cluster cluster, StoreDefinition storeDef) Checks if a given partition is stored in the node.
static boolean	checkPartitionBelongsToNode(int partition, int replicaType, int nodeId, Cluster cluster, StoreDefinition storeDef)
static void	combinePartitionTuples(java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>> existingPartitionTuples, java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>> newPartitionTuples) Given a mapping of existing node ids to their partition tuples and another new set of node ids to partition tuples, combines them together and puts it into the existing partition tuples
static boolean	containsNode(Cluster cluster, int nodeId) Given a cluster and a node id checks if the node exists
static boolean	containsPreferenceList(Cluster cluster, java.util.List<java.lang.Integer> preferenceList, int nodeId) Given a preference list and a node id, check if any one of the partitions is on the node in picture
static java.util.concurrent.ExecutorService	createExecutors(int numThreads)
static AdminClient	createTempAdminClient(VoldemortConfig voldemortConfig, Cluster cluster, int numConnPerNode)
static Cluster	createUpdatedCluster(Cluster currentCluster, int stealerNodeId, java.util.List<java.lang.Integer> donatedPartitions) Updates the existing cluster such that we remove partitions mentioned from the stealer node and add them to the donor node
static void	dumpCluster(Cluster initialCluster, Cluster finalCluster, java.io.File outputDir) Given the initial and final cluster dumps it into the output directory
static void	executorShutDown(java.util.concurrent.ExecutorService executorService, long timeOutSec) Wait to shutdown service
static java.util.List<RebalancePartitionsInfo>	filterPartitionPlanWithStores(java.util.List<RebalancePartitionsInfo> existingPlanList, java.util.List<StoreDefinition> storeDefs) Given a list of partition plans and a set of stores, copies the store names to every individual plan and creates a new list
static java.util.List<StoreDefinition>	filterStores(java.util.List<StoreDefinition> storeDefs, boolean isReadOnly) Given a list of store definitions, filters the list depending on the boolean
static java.util.List<RebalancePartitionsInfo>	flattenNodePlans(java.util.List<RebalanceNodePlan> rebalanceNodePlanList) Given a list of node plans flattens it into a list of partitions info
static java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>	flattenPartitionTuples(java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> replicaToPartitionList) Given a map of replica_type to partition mapping gives back a set of tuples of [replica_type, partition]
static java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>>	flattenPartitionTuples(java.util.Set<Pair<java.lang.Integer,java.lang.Integer>> partitionTuples) Given a list of tuples of [replica_type, partition], flattens it and generates a map of replica_type to partition mapping
static Pair<Cluster,java.lang.Integer>	generateMinCluster(Cluster currentCluster, Cluster targetCluster, java.util.List<StoreDefinition> storeDefs) Takes the current cluster metadata and target cluster metadata ( which contains all the nodes of current cluster + new nodes with empty partitions ), and generates a new cluster with some partitions moved to the new node
static void	generateMinCluster(Cluster currentCluster, Cluster targetCluster, java.util.List<StoreDefinition> storeDefs, java.lang.String outputDir, int tries) Outputs an optimized cluster based on the existing cluster and the new nodes that are being added.
static Cluster	getClusterWithNewNodes(Cluster currentCluster, Cluster targetCluster) Given the current cluster and a target cluster, generates a cluster with new nodes ( which in turn contain empty partition lists )
static int	getCrossZoneMoves(Cluster targetCluster, RebalanceClusterPlan plan) Return the number of cross zone copying that is going to take place
static java.util.Map<java.lang.Integer,java.lang.Integer>	getCurrentPartitionMapping(Cluster currentCluster) Given the cluster metadata returns a mapping of partition to node
static Versioned<Cluster>	getLatestCluster(java.util.List<java.lang.Integer> requiredNodes, AdminClient adminClient) Get the latest cluster from all available nodes in the cluster Throws exception if: A) Any node in the required nodes list fails to respond. B) Cluster is in inconsistent state with concurrent versions for cluster metadata on any two nodes.
static Node	getNodeByPartitionId(Cluster cluster, int partitionId) Returns the Node associated to the provided partition.
static java.util.List<java.lang.Integer>	getNodeIds(java.util.List<Node> nodes) Given a list of nodes, retrieves the list of node ids
static java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>>	getNodeIdToAllPartitions(Cluster cluster, StoreDefinition storeDef, boolean includePrimary) For a particular cluster creates a mapping of node id to their corresponding list of [ replicaType, partition ] tuple
static java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>>	getOptimizedReplicaToPartitionList(int stealerNodeId, Cluster cluster, StoreDefinition storeDef, java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> currentReplicaToPartitionList) Given the current replica to partition list, try to check if the donor node would already contain that partition and if yes, ignore it
static java.util.List<java.lang.Integer>	getPartitionsFromTuples(java.util.Set<Pair<java.lang.Integer,java.lang.Integer>> tuples) Given a set of [ replica, partition ] tuples, flatten it to retrieve only the partitions
static java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>>	getStolenPartitionTuples(Cluster currentCluster, Cluster targetCluster, StoreDefinition storeDef) Find all [replica_type, partition] tuples to be stolen
static java.util.List<java.lang.Integer>	getStolenPrimaryPartitions(Cluster currentCluster, Cluster targetCluster, int stealNodeId) For a particular stealer node find all the "primary" tuples it will steal.
static java.util.List<StoreDefinition>	getStoreDefinition(Cluster cluster, AdminClient adminClient) Given the cluster metadata and admin client, retrieves the list of store definitions.
static StoreDefinition	getStoreDefinitionWithName(java.util.List<StoreDefinition> storeDefs, java.lang.String storeName) Given a store name and a list of store definitions, returns the appropriate store definition ( if it exists )
static java.util.List<java.lang.String>	getStoreNames(java.util.List<StoreDefinition> storeDefList) Given a list of store definitions return a list of store names
static int	getTotalMoves(RebalanceClusterPlan plan) Return the number of total moves
static java.util.HashMap<java.lang.Integer,java.util.List<RebalancePartitionsInfo>>	groupPartitionsInfoByNode(java.util.List<RebalancePartitionsInfo> rebalancePartitionPlanList, boolean groupByStealerNode) Given a list of partition infos, generates a map of stealer / donor node to list of partition infos
static void	printErrorLog(int taskId, org.apache.log4j.Logger logger, java.lang.String message, java.lang.Exception e) Print log to the following logger ( Error level )
static void	printLog(int taskId, org.apache.log4j.Logger logger, java.lang.String message) Print log to the following logger ( Info level )
static java.lang.String	printMap(java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>> nodeIdToAllPartitions) Returns a string representation of the cluster
static void	propagateCluster(AdminClient adminClient, Cluster cluster) Attempt to propagate a cluster definition to all nodes.
static Node	removePartitionToNode(Node node, java.lang.Integer donatedPartition) Remove a partition from the node provided
static Node	removePartitionToNode(Node node, java.util.Set<java.lang.Integer> donatedPartitions) Remove the set of partitions from the node provided
static Cluster	updateCluster(Cluster currentCluster, java.util.List<Node> updatedNodeList) Concatenates the list of current nodes in the given cluster with the new nodes provided and returns an updated cluster metadata.
static Node	updateNode(Node node, java.util.List<java.lang.Integer> partitionsList) Creates a replica of the node with the new partitions list
static void	validateClusterState(Cluster cluster, AdminClient adminClient) Check the execution state of the server by checking the state of MetadataStore.VoldemortState This function checks if the nodes are all in normal state ( MetadataStore.VoldemortState.NORMAL_SERVER).
static void	validateReadOnlyStores(Cluster cluster, java.util.List<StoreDefinition> storeDefs, AdminClient adminClient) Given a list of store definitions, cluster and admin client returns a boolean indicating if all RO stores are in the correct format.
static java.util.List<StoreDefinition>	validateRebalanceStore(java.util.List<StoreDefinition> storeDefList) Given a list of store definitions, makes sure that rebalance supports all of them.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

canRebalanceList

public static final java.util.List<java.lang.String> canRebalanceList

initialClusterFileName

public static final java.lang.String initialClusterFileName

See Also:

Constant Field Values

finalClusterFileName

public static final java.lang.String finalClusterFileName

See Also:

Constant Field Values

Constructor Detail

RebalanceUtils

public RebalanceUtils()

Method Detail

getOptimizedReplicaToPartitionList

public static java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> getOptimizedReplicaToPartitionList(int stealerNodeId,
                                                                                                                        Cluster cluster,
                                                                                                                        StoreDefinition storeDef,
                                                                                                                        java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> currentReplicaToPartitionList)

Given the current replica to partition list, try to check if the donor node would already contain that partition and if yes, ignore it

Parameters:

stealerNodeId - Stealer node id

cluster - Cluster metadata

storeDef - Store definition

currentReplicaToPartitionList - Current replica to partition list

Returns:

Optimized replica to partition list

getLatestCluster

public static Versioned<Cluster> getLatestCluster(java.util.List<java.lang.Integer> requiredNodes,
                                                  AdminClient adminClient)

Get the latest cluster from all available nodes in the cluster
Throws exception if:
A) Any node in the required nodes list fails to respond.
B) Cluster is in inconsistent state with concurrent versions for cluster metadata on any two nodes.

Parameters:

requiredNodes - List of nodes from which we definitely need an answer

adminClient - Admin client used to query the nodes

Returns:

Returns the latest cluster metadata

getCrossZoneMoves

public static int getCrossZoneMoves(Cluster targetCluster,
                                    RebalanceClusterPlan plan)

Return the number of cross zone copying that is going to take place

Parameters:

targetCluster - Target cluster metadata

plan - The rebalance plan

Returns:

Number of cross zone moves

getTotalMoves

public static int getTotalMoves(RebalanceClusterPlan plan)

Return the number of total moves

Parameters:

plan - The rebalance plan

Returns:

Number of moves

assertSameDonor

public static void assertSameDonor(java.util.List<RebalancePartitionsInfo> partitionInfos,
                                   int expectedDonorId)

Given a list of partition informations check all of them belong to the same donor node

Parameters:

partitionInfos - List of partition infos

expectedDonorId - Expected donor node id ( If -1, then just checks if all are same )

generateMinCluster

public static void generateMinCluster(Cluster currentCluster,
                                      Cluster targetCluster,
                                      java.util.List<StoreDefinition> storeDefs,
                                      java.lang.String outputDir,
                                      int tries)

Outputs an optimized cluster based on the existing cluster and the new nodes that are being added.

Parameters:

currentCluster - Current cluster metadata

targetCluster - The target cluster metadata which contains the nodes of the current cluster + new nodes with empty partitions

storeDefs - List of store definitions

outputDir - The output directory where we'll store the cluster metadata ( if not null )

tries - Number of times we'll try to optimize the metadata generation

generateMinCluster

public static Pair<Cluster,java.lang.Integer> generateMinCluster(Cluster currentCluster,
                                                                 Cluster targetCluster,
                                                                 java.util.List<StoreDefinition> storeDefs)

Takes the current cluster metadata and target cluster metadata ( which contains all the nodes of current cluster + new nodes with empty partitions ), and generates a new cluster with some partitions moved to the new node

Parameters:

currentCluster - Current cluster metadata

targetCluster - Target cluster metadata ( which contains old nodes + new nodes [ empty partitions ])

storeDefs - List of store definitions

Returns:

Return a pair of cluster metadata and number of primary partitions that have moved

checkKeyBelongsToPartition

public static boolean checkKeyBelongsToPartition(int nodeId,
                                                 byte[] key,
                                                 java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> replicaToPartitionList,
                                                 Cluster cluster,
                                                 StoreDefinition storeDef)

Check that the key belongs to one of the partitions in the map of replica type to partitions

Parameters:

nodeId - Node on which this is running ( generally stealer node )

key - The key to check

replicaToPartitionList - Mapping of replica type to partition list

cluster - Cluster metadata

storeDef - The store definition

Returns:

Returns a boolean to indicate if this belongs to the map

checkKeyBelongsToPartition

public static boolean checkKeyBelongsToPartition(java.util.List<java.lang.Integer> keyPartitions,
                                                 java.util.List<java.lang.Integer> nodePartitions,
                                                 java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> replicaToPartitionList)

Check that the key belongs to one of the partitions in the map of replica type to partitions

Parameters:

keyPartitions - Preference list of the key

nodePartitions - Partition list on this node

replicaToPartitionList - Mapping of replica type to partition list

Returns:

Returns a boolean to indicate if this belongs to the map

checkKeyBelongsToPartition

public static java.util.List<java.lang.Integer> checkKeyBelongsToPartition(byte[] key,
                                                                           java.util.Set<Pair<java.lang.Integer,java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>>>> stealerNodeToMappingTuples,
                                                                           Cluster cluster,
                                                                           StoreDefinition storeDef)

Given a key and a list of steal infos give back a list of stealer node ids which will steal this.

Parameters:

key - Byte array of key

stealerNodeToMappingTuples - Pairs of stealer node id to their corresponding [ partition - replica ] tuples

cluster - Cluster metadata

storeDef - Store definitions

Returns:

List of node ids

checkPartitionBelongsToNode

public static boolean checkPartitionBelongsToNode(int partition,
                                                  int replicaType,
                                                  int nodeId,
                                                  Cluster cluster,
                                                  StoreDefinition storeDef)

Returns:

true if the partition belongs to the node with given replicatype

checkKeyBelongsToNode

public static boolean checkKeyBelongsToNode(byte[] key,
                                            int nodeId,
                                            Cluster cluster,
                                            StoreDefinition storeDef)

Parameters:

key -

nodeId -

cluster -

storeDef -

Returns:

true if the key belongs to the node as some replica

checkPartitionBelongsToNode

public static boolean checkPartitionBelongsToNode(int partition,
                                                  int nodeId,
                                                  Cluster cluster,
                                                  StoreDefinition storeDef)

Checks if a given partition is stored in the node. (It can be primary or a secondary)

Parameters:

partition -

nodeId -

cluster -

storeDef -

Returns:

validateClusterState

public static void validateClusterState(Cluster cluster,
                                        AdminClient adminClient)

Check the execution state of the server by checking the state of MetadataStore.VoldemortState
This function checks if the nodes are all in normal state ( MetadataStore.VoldemortState.NORMAL_SERVER).

Parameters:

cluster - Cluster metadata whose nodes we are checking

adminClient - Admin client used to query

Throws:

VoldemortRebalancingException - if any node is not in normal state

getClusterWithNewNodes

public static Cluster getClusterWithNewNodes(Cluster currentCluster,
                                             Cluster targetCluster)

Given the current cluster and a target cluster, generates a cluster with new nodes ( which in turn contain empty partition lists )

Parameters:

currentCluster - Current cluster metadata

targetCluster - Target cluster metadata

Returns:

Returns a new cluster which contains nodes of the current cluster + new nodes

updateCluster

public static Cluster updateCluster(Cluster currentCluster,
                                    java.util.List<Node> updatedNodeList)

Concatenates the list of current nodes in the given cluster with the new nodes provided and returns an updated cluster metadata.
If the nodes being updated already exist in the current metadata, we take the updated ones

Parameters:

currentCluster - The current cluster metadata

updatedNodeList - The list of new nodes to be added

Returns:

New cluster metadata containing both the sets of nodes

containsNode

public static boolean containsNode(Cluster cluster,
                                   int nodeId)

Given a cluster and a node id checks if the node exists

Parameters:

cluster - The cluster metadata to check in

nodeId - The node id to search for

Returns:

True if cluster contains the node id, else false

containsPreferenceList

public static boolean containsPreferenceList(Cluster cluster,
                                             java.util.List<java.lang.Integer> preferenceList,
                                             int nodeId)

Given a preference list and a node id, check if any one of the partitions is on the node in picture

Parameters:

cluster - Cluster metadata

preferenceList - Preference list of partition ids

nodeId - Node id which we are checking for

Returns:

True if the preference list contains a node whose id = nodeId

createUpdatedCluster

public static Cluster createUpdatedCluster(Cluster currentCluster,
                                           int stealerNodeId,
                                           java.util.List<java.lang.Integer> donatedPartitions)

Updates the existing cluster such that we remove partitions mentioned from the stealer node and add them to the donor node

Parameters:

currentCluster - Existing cluster metadata. Both stealer and donor node should already exist in this metadata

stealerNodeId - Id of node from which we are stealing the partitions

donatedPartitions - List of partitions we are moving

partitionList - List of partitions we are moving

Returns:

Updated cluster metadata

updateNode

public static Node updateNode(Node node,
                              java.util.List<java.lang.Integer> partitionsList)

Creates a replica of the node with the new partitions list

Parameters:

node - The node whose replica we are creating

partitionsList - The new partitions list

Returns:

Replica of node with new partitions list

addPartitionToNode

public static Node addPartitionToNode(Node node,
                                      java.lang.Integer donatedPartition)

Add a partition to the node provided

Parameters:

node - The node to which we'll add the partition

donatedPartition - The partition to add

Returns:

The new node with the new partition

removePartitionToNode

public static Node removePartitionToNode(Node node,
                                         java.lang.Integer donatedPartition)

Remove a partition from the node provided

Parameters:

node - The node from which we're removing the partition

donatedPartition - The partitions to remove

Returns:

The new node without the partition

addPartitionToNode

public static Node addPartitionToNode(Node node,
                                      java.util.Set<java.lang.Integer> donatedPartitions)

Add the set of partitions to the node provided

Parameters:

node - The node to which we'll add the partitions

donatedPartitions - The list of partitions to add

Returns:

The new node with the new partitions

removePartitionToNode

public static Node removePartitionToNode(Node node,
                                         java.util.Set<java.lang.Integer> donatedPartitions)

Remove the set of partitions from the node provided

Parameters:

node - The node from which we're removing the partitions

donatedPartitions - The list of partitions to remove

Returns:

The new node without the partitions

getCurrentPartitionMapping

public static java.util.Map<java.lang.Integer,java.lang.Integer> getCurrentPartitionMapping(Cluster currentCluster)

Given the cluster metadata returns a mapping of partition to node

Parameters:

currentCluster - Cluster metadata

Returns:

Map of partition id to node id

propagateCluster

public static void propagateCluster(AdminClient adminClient,
                                    Cluster cluster)

Attempt to propagate a cluster definition to all nodes. Also rollback is in place in case one of them fails

Parameters:

adminClient - AdminClient instance to use.

cluster - Cluster definition to propagate

getStolenPrimaryPartitions

public static java.util.List<java.lang.Integer> getStolenPrimaryPartitions(Cluster currentCluster,
                                                                           Cluster targetCluster,
                                                                           int stealNodeId)

For a particular stealer node find all the "primary" tuples it will steal. In other words, expect the "replica" part to be 0 always.

Parameters:

currentCluster - The cluster definition of the existing cluster

targetCluster - The target cluster definition

stealNodeId - Node id of the stealer node

Returns:

Returns a list of primary partitions which this stealer node will get

getStolenPartitionTuples

public static java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>> getStolenPartitionTuples(Cluster currentCluster,
                                                                                                                                 Cluster targetCluster,
                                                                                                                                 StoreDefinition storeDef)

Find all [replica_type, partition] tuples to be stolen

Parameters:

currentCluster - Current cluster metadata

targetCluster - Target cluster metadata

storeDef - Store Definition

Returns:

Map of stealer node id to sets of [ replica_type, partition ] tuples

combinePartitionTuples

public static void combinePartitionTuples(java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>> existingPartitionTuples,
                                          java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>> newPartitionTuples)

Given a mapping of existing node ids to their partition tuples and another new set of node ids to partition tuples, combines them together and puts it into the existing partition tuples

Parameters:

existingPartitionTuples - Existing partition tuples ( Will include the new partition tuples at the end of this function )

newPartitionTuples - New partition tuples

getNodeIdToAllPartitions

public static java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>> getNodeIdToAllPartitions(Cluster cluster,
                                                                                                                                 StoreDefinition storeDef,
                                                                                                                                 boolean includePrimary)

For a particular cluster creates a mapping of node id to their corresponding list of [ replicaType, partition ] tuple

Parameters:

cluster - The cluster metadata

storeDef - The store definition

includePrimary - Include the primary partition?

Returns:

Map of node id to set of [ replicaType, partition ] tuple

dumpCluster

public static void dumpCluster(Cluster initialCluster,
                               Cluster finalCluster,
                               java.io.File outputDir)

Given the initial and final cluster dumps it into the output directory

Parameters:

initialCluster - Initial cluster metadata

finalCluster - Final cluster metadata

outputDir - Output directory where to dump this file

Throws:

java.io.IOException

printLog

public static void printLog(int taskId,
                            org.apache.log4j.Logger logger,
                            java.lang.String message)

Print log to the following logger ( Info level )

Parameters:

taskId - Task id

logger - Logger class

message - The message to print

printErrorLog

public static void printErrorLog(int taskId,
                                 org.apache.log4j.Logger logger,
                                 java.lang.String message,
                                 java.lang.Exception e)

Print log to the following logger ( Error level )

Parameters:

taskId - Stealer node id

logger - Logger class

message - The message to print

getNodeByPartitionId

public static Node getNodeByPartitionId(Cluster cluster,
                                        int partitionId)

Returns the Node associated to the provided partition.

Parameters:

cluster - The cluster in which to find the node

partitionId - Partition id for which we want the corresponding node

Returns:

Node that owns the partition

createTempAdminClient

public static AdminClient createTempAdminClient(VoldemortConfig voldemortConfig,
                                                Cluster cluster,
                                                int numConnPerNode)

getStoreDefinition

public static java.util.List<StoreDefinition> getStoreDefinition(Cluster cluster,
                                                                 AdminClient adminClient)

Given the cluster metadata and admin client, retrieves the list of store definitions.
It also checks if the store definitions are consistent across the cluster

Parameters:

cluster - The cluster metadata

adminClient - The admin client to use to retrieve the store definitions

Returns:

List of store definitions

validateRebalanceStore

public static java.util.List<StoreDefinition> validateRebalanceStore(java.util.List<StoreDefinition> storeDefList)

Given a list of store definitions, makes sure that rebalance supports all of them. If not it throws an error.

Parameters:

storeDefList - List of store definitions

Returns:

Filtered list of store definitions which rebalancing supports

validateReadOnlyStores

public static void validateReadOnlyStores(Cluster cluster,
                                          java.util.List<StoreDefinition> storeDefs,
                                          AdminClient adminClient)

Given a list of store definitions, cluster and admin client returns a boolean indicating if all RO stores are in the correct format.
This function also takes into consideration nodes which are being bootstrapped for the first time, in which case we can safely ignore checking them ( as they will have default to ro0 )

Parameters:

cluster - Cluster metadata

storeDefs - Complete list of store definitions

adminClient - Admin client

printMap

public static java.lang.String printMap(java.util.Map<java.lang.Integer,java.util.Set<Pair<java.lang.Integer,java.lang.Integer>>> nodeIdToAllPartitions)

Returns a string representation of the cluster

 Current Cluster:
 0 - [0, 1, 2, 3] + [7, 8, 9]
 1 - [4, 5, 6] + [0, 1, 2, 3]
 2 - [7, 8, 9] + [4, 5, 6]
 

Parameters:

nodeIdToAllPartitions - Mapping of node id to all tuples

Returns:

Returns a string representation of the cluster

flattenPartitionTuples

public static java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> flattenPartitionTuples(java.util.Set<Pair<java.lang.Integer,java.lang.Integer>> partitionTuples)

Given a list of tuples of [replica_type, partition], flattens it and generates a map of replica_type to partition mapping

Parameters:

partitionTuples - Set of tuples

Returns:

Map of replica_type to set of partitions

flattenPartitionTuples

public static java.util.Set<Pair<java.lang.Integer,java.lang.Integer>> flattenPartitionTuples(java.util.HashMap<java.lang.Integer,java.util.List<java.lang.Integer>> replicaToPartitionList)

Given a map of replica_type to partition mapping gives back a set of tuples of [replica_type, partition]

Parameters:

replicaToPartitionList - Map of replica_type to set of partitions

Returns:

Set of tuples

flattenNodePlans

public static java.util.List<RebalancePartitionsInfo> flattenNodePlans(java.util.List<RebalanceNodePlan> rebalanceNodePlanList)

Given a list of node plans flattens it into a list of partitions info

Parameters:

rebalanceNodePlanList - Complete list of rebalance node plan

Returns:

Flattened list of partition plans

getPartitionsFromTuples

public static java.util.List<java.lang.Integer> getPartitionsFromTuples(java.util.Set<Pair<java.lang.Integer,java.lang.Integer>> tuples)

Given a set of [ replica, partition ] tuples, flatten it to retrieve only the partitions

Parameters:

tuples - The [ replica, partition ] tuples

Returns:

List of partitions

filterPartitionPlanWithStores

public static java.util.List<RebalancePartitionsInfo> filterPartitionPlanWithStores(java.util.List<RebalancePartitionsInfo> existingPlanList,
                                                                                    java.util.List<StoreDefinition> storeDefs)

Given a list of partition plans and a set of stores, copies the store names to every individual plan and creates a new list

Parameters:

existingPlanList - Existing partition plan list

storeDefs - List of store names we are rebalancing

Returns:

List of updated partition plan

groupPartitionsInfoByNode

public static java.util.HashMap<java.lang.Integer,java.util.List<RebalancePartitionsInfo>> groupPartitionsInfoByNode(java.util.List<RebalancePartitionsInfo> rebalancePartitionPlanList,
                                                                                                                     boolean groupByStealerNode)

Given a list of partition infos, generates a map of stealer / donor node to list of partition infos

Parameters:

rebalancePartitionPlanList - Complete list of partition plans

groupByStealerNode - Boolean indicating if we want to group by stealer node ( or donor node )

Returns:

Flattens it into a map on a per node basis

getStoreDefinitionWithName

public static StoreDefinition getStoreDefinitionWithName(java.util.List<StoreDefinition> storeDefs,
                                                         java.lang.String storeName)

Given a store name and a list of store definitions, returns the appropriate store definition ( if it exists )

Parameters:

storeDefs - List of store definitions

storeName - The store name whose store definition is required

Returns:

The store definition

filterStores

public static java.util.List<StoreDefinition> filterStores(java.util.List<StoreDefinition> storeDefs,
                                                           boolean isReadOnly)

Given a list of store definitions, filters the list depending on the boolean

Parameters:

storeDefs - Complete list of store definitions

isReadOnly - Boolean indicating whether filter on read-only or not?

Returns:

List of filtered store definition

getStoreNames

public static java.util.List<java.lang.String> getStoreNames(java.util.List<StoreDefinition> storeDefList)

Given a list of store definitions return a list of store names

Parameters:

storeDefList - The list of store definitions

Returns:

Returns a list of store names

getNodeIds

public static java.util.List<java.lang.Integer> getNodeIds(java.util.List<Node> nodes)

Given a list of nodes, retrieves the list of node ids

Parameters:

nodes - The list of nodes

Returns:

Returns a list of node ids

executorShutDown

public static void executorShutDown(java.util.concurrent.ExecutorService executorService,
                                    long timeOutSec)

Wait to shutdown service

Parameters:

executorService - Executor service to shutdown

timeOutSec - Time we wait for

createExecutors

public static java.util.concurrent.ExecutorService createExecutors(int numThreads)