From 31654a47d613de64787d7967371ec56846bd294a Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 23 Apr 2017 03:16:49 -0700
Subject: [PATCH 01/44] WIP: Refactored graph, and it compiles

---
 .../kliewkliew/cornucopia/Library.scala       |   7 +
 .../kliewkliew/cornucopia/Microservice.scala  |   2 +-
 .../cornucopia/actors/CornucopiaSource.scala  |  58 ++
 .../kliewkliew/cornucopia/graph/Graph.scala   | 535 ++++++++++++++++++
 .../kliewkliew/cornucopia/kafka/Config.scala  |  12 +-
 .../cornucopia/kafka/Consumer.scala           |  11 +-
 .../{kafka => redis}/Operation.scala          |   2 +-
 7 files changed, 621 insertions(+), 6 deletions(-)
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/actors/CornucopiaSource.scala
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
 rename src/main/scala/com/github/kliewkliew/cornucopia/{kafka => redis}/Operation.scala (94%)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala b/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
new file mode 100644
index 0000000..82f0553
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
@@ -0,0 +1,7 @@
+package com.github.kliewkliew.cornucopia
+
+import akka.actor._
+
+object Library {
+  val ref: ActorRef = new graph.CornucopiaActorSource().ref
+}
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/Microservice.scala b/src/main/scala/com/github/kliewkliew/cornucopia/Microservice.scala
index 7b5fa1f..b6cef83 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/Microservice.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/Microservice.scala
@@ -2,6 +2,6 @@ package com.github.kliewkliew.cornucopia
 
 object Microservice {
   def main(args: Array[String]): Unit = {
-    new kafka.Consumer().run
+    new graph.CornucopiaKafkaSource().run
   }
 }
\ No newline at end of file
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/actors/CornucopiaSource.scala b/src/main/scala/com/github/kliewkliew/cornucopia/actors/CornucopiaSource.scala
new file mode 100644
index 0000000..d39d475
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/actors/CornucopiaSource.scala
@@ -0,0 +1,58 @@
+package com.github.kliewkliew.cornucopia.actors
+
+import akka.actor._
+import akka.stream.actor.ActorPublisher
+
+import scala.annotation.tailrec
+
+/**
+  * Copied liberally from akka documentaion on
+  * [Stream integrations](http://doc.akka.io/docs/akka-stream-and-http-experimental/1.0/scala/stream-integrations.html).
+  */
+object CornucopiaSource {
+  def props: Props = Props[CornucopiaSource]
+
+  final case class Task(operation: String, redisNodeIp: String)
+  case object TaskAccepted
+  case object TaskDenied
+}
+
+class CornucopiaSource extends ActorPublisher[CornucopiaSource.Task] {
+  import CornucopiaSource._
+  import akka.stream.actor.ActorPublisherMessage._
+
+  val MaxBufferSize = 100
+  var buf = Vector.empty[Task]
+
+  override def receive = {
+    case _: Task if buf.size == MaxBufferSize =>
+      sender() ! TaskDenied
+    case task: Task =>
+      sender() ! TaskAccepted
+      if (buf.isEmpty && totalDemand > 0) onNext(task)
+      else {
+        buf :+= task
+        deliverBuf()
+      }
+    case Request(_) =>
+      deliverBuf()
+    case Cancel =>
+      context.stop(self)
+  }
+
+  @tailrec final def deliverBuf(): Unit = {
+    if (totalDemand > 0) {
+      if (totalDemand <= Int.MaxValue) {
+        val (use, keep) = buf.splitAt(totalDemand.toInt)
+        buf = keep
+        use foreach onNext
+      } else {
+        val (use, keep) = buf.splitAt(Int.MaxValue)
+        buf = keep
+        use foreach onNext
+        deliverBuf()
+      }
+    }
+  }
+
+}
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
new file mode 100644
index 0000000..9dd17e1
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -0,0 +1,535 @@
+package com.github.kliewkliew.cornucopia.graph
+
+import java.util
+import java.util.concurrent.atomic.AtomicInteger
+
+import akka.actor._
+import akka.NotUsed
+import akka.io.Udp.SO.Broadcast
+import akka.stream.{ClosedShape, ThrottleMode, FlowShape, Inlet}
+import com.github.kliewkliew.cornucopia.redis.Connection.{CodecType, Salad, getConnection, newSaladAPI}
+import org.slf4j.LoggerFactory
+import org.apache.kafka.clients.consumer.ConsumerRecord
+import com.github.kliewkliew.cornucopia.redis._
+import com.github.kliewkliew.salad.SaladClusterAPI
+import com.lambdaworks.redis.RedisURI
+import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
+import com.lambdaworks.redis.models.role.RedisInstance.Role
+
+import collection.JavaConverters._
+import scala.collection.mutable
+import scala.collection.JavaConversions._
+import scala.language.implicitConversions
+import scala.concurrent.{ExecutionContext, Future}
+import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink, Source}
+import com.github.kliewkliew.cornucopia.kafka.Config // TO-DO: put config someplace else
+
+trait CornucopiaGraph {
+  import scala.concurrent.ExecutionContext.Implicits.global
+
+  private val logger = LoggerFactory.getLogger(this.getClass)
+
+  def partitionEvents(key: String) = key.trim.toLowerCase match {
+    case ADD_MASTER.key     => ADD_MASTER.ordinal
+    case ADD_SLAVE.key      => ADD_SLAVE.ordinal
+    case REMOVE_NODE.key    => REMOVE_NODE.ordinal
+    case RESHARD.key        => RESHARD.ordinal
+    case _                  => UNSUPPORTED.ordinal
+  }
+
+  def partitionNodeRemoval(key: String) = key.trim.toLowerCase match {
+    case REMOVE_MASTER.key  => REMOVE_MASTER.ordinal
+    case REMOVE_SLAVE.key   => REMOVE_SLAVE.ordinal
+    case UNSUPPORTED.key    => UNSUPPORTED.ordinal
+  }
+
+  /**
+    * Stream definitions for the graph.
+    */
+  // Extract a tuple of the key and value from a Kafka record.
+  case class KeyValue(key: String, value: String)
+
+  // Add a master node to the cluster.
+  protected def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+    .map(_.value)
+    .map(RedisURI.create)
+    .map(newSaladAPI.canonicalizeURI)
+    .groupedWithin(100, Config.Cornucopia.batchPeriod)
+    .mapAsync(1)(addNodesToCluster)
+    .mapAsync(1)(waitForTopologyRefresh)
+    .map(_ => KeyValue(RESHARD.key, ""))
+
+  // Add a slave node to the cluster, replicating the master that has the fewest slaves.
+  protected def streamAddSlave(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+    .map(_.value)
+    .map(RedisURI.create)
+    .map(newSaladAPI.canonicalizeURI)
+    .groupedWithin(100, Config.Cornucopia.batchPeriod)
+    .mapAsync(1)(addNodesToCluster)
+    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(findMasters)
+    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(_ => logTopology)
+
+  // Emit a key-value pair indicating the node type and URI.
+  protected def streamRemoveNode(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+    .map(_.value)
+    .map(RedisURI.create)
+    .map(newSaladAPI.canonicalizeURI)
+    .mapAsync(1)(emitNodeType)
+
+  // Remove a slave node from the cluster.
+  protected def streamRemoveSlave(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+    .map(_.value)
+    .groupedWithin(100, Config.Cornucopia.batchPeriod)
+    .mapAsync(1)(forgetNodes)
+    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(_ => logTopology)
+
+  // Redistribute the hash slots among all nodes in the cluster.
+  // Execute slot redistribution at most once per configured interval.
+  // Combine multiple requests into one request.
+  protected def streamReshard(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+    .map(record => Seq(record.value))
+    .conflate((seq1, seq2) => seq1 ++ seq2)
+    .throttle(1, Config.Cornucopia.minReshardWait, 1, ThrottleMode.Shaping)
+    .mapAsync(1)(reshardCluster)
+    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(_ => logTopology)
+
+  // Throw for keys indicating unsupported operations.
+  protected def unsupportedOperation = Flow[KeyValue]
+    .map(record => throw new IllegalArgumentException(s"Unsupported operation ${record.key} for ${record.value}"))
+
+  /**
+    * Wait for the new cluster topology view to propagate to all nodes in the cluster. May not be strictly necessary
+    * since this microservice immediately attempts to notify all nodes of topology updates.
+    *
+    * @param passthrough The value that will be passed through to the next map stage.
+    * @param executionContext The thread dispatcher context.
+    * @tparam T
+    * @return The unmodified input value.
+    */
+  protected def waitForTopologyRefresh[T](passthrough: T)(implicit executionContext: ExecutionContext): Future[T] = Future {
+    scala.concurrent.blocking(Thread.sleep(Config.Cornucopia.refreshTimeout))
+    passthrough
+  }
+
+  /**
+    * Log the current view of the cluster topology.
+    *
+    * @param executionContext The thread dispatcher context.
+    * @return
+    */
+  protected def logTopology(implicit executionContext: ExecutionContext): Future[Unit] = {
+    implicit val saladAPI = newSaladAPI
+    saladAPI.clusterNodes.map { allNodes =>
+      val masterNodes = allNodes.filter(Role.MASTER == _.getRole)
+      val slaveNodes = allNodes.filter(Role.SLAVE == _.getRole)
+      logger.info(s"Master nodes: $masterNodes")
+      logger.info(s"Slave nodes: $slaveNodes")
+    }
+  }
+
+  /**
+    * The entire cluster will meet the new nodes at the given URIs.
+    *
+    * @param redisURIList The list of URI of the new nodes.
+    * @param executionContext The thread dispatcher context.
+    * @return The list of URI if the nodes were met. TODO: emit only the nodes that were successfully added.
+    */
+  protected def addNodesToCluster(redisURIList: Seq[RedisURI])(implicit executionContext: ExecutionContext): Future[Seq[RedisURI]] = {
+    implicit val saladAPI = newSaladAPI
+    saladAPI.clusterNodes.flatMap { allNodes =>
+      val getConnectionsToLiveNodes = allNodes.filter(_.isConnected).map(node => getConnection(node.getNodeId))
+      Future.sequence(getConnectionsToLiveNodes).flatMap { connections =>
+        // Meet every new node from every old node.
+        val metResults = for {
+          conn <- connections
+          uri <- redisURIList
+        } yield {
+          conn.clusterMeet(uri)
+        }
+        Future.sequence(metResults).map(_ => redisURIList)
+      }
+    }
+  }
+
+  /**
+    * Set the n new slave nodes to replicate the poorest (fewest slaves) n masters.
+    *
+    * @param redisURIList The list of ip addresses of the slaves that will be added to the cluster. Hostnames are not acceptable.
+    * @param executionContext The thread dispatcher context.
+    * @return Indicate that the n new slaves are replicating the poorest n masters.
+    */
+  protected def findMasters(redisURIList: Seq[RedisURI])(implicit executionContext: ExecutionContext): Future[Unit] = {
+    implicit val saladAPI = newSaladAPI
+    saladAPI.clusterNodes.flatMap { allNodes =>
+      // Node ids for nodes that are currently master nodes but will become slave nodes.
+      val newSlaveIds = allNodes.filter(node => redisURIList.contains(node.getUri)).map(_.getNodeId)
+      // The master nodes (the nodes that will become slaves are still master nodes at this point and must be filtered out).
+      val masterNodes = saladAPI.masterNodes(allNodes)
+        .filterNot(node => newSlaveIds.contains(node.getNodeId))
+      // HashMap of master node ids to the number of slaves for that master.
+      val masterSlaveCount = new util.HashMap[String, AtomicInteger](masterNodes.length + 1, 1)
+      // Populate the hash map.
+      masterNodes.map(_.getNodeId).foreach(nodeId => masterSlaveCount.put(nodeId, new AtomicInteger(0)))
+      allNodes.map { node =>
+        Option.apply(node.getSlaveOf)
+          .map(master => masterSlaveCount.get(master).incrementAndGet())
+      }
+
+      // Find the poorest n masters for n slaves.
+      val poorestMasters = new MaxNHeapMasterSlaveCount(redisURIList.length)
+      masterSlaveCount.asScala.foreach(poorestMasters.offer)
+      assert(redisURIList.length >= poorestMasters.underlying.length)
+
+      // Create a list so that we can circle back to the first element if the new slaves outnumber the existing masters.
+      val poorMasterList = poorestMasters.underlying.toList
+      val poorMasterIndex = new AtomicInteger(0)
+      // Choose a master for every slave.
+      val listFuturesResults = redisURIList.map { slaveURI =>
+        getConnection(slaveURI).map(_.clusterReplicate(
+          poorMasterList(poorMasterIndex.getAndIncrement() % poorMasterList.length)._1))
+      }
+      Future.sequence(listFuturesResults).map(x => x)
+    }
+  }
+
+  /**
+    * Emit a key-value representing the node-type and the node-id.
+    * @param redisURI
+    * @param executionContext
+    * @return the node type and id.
+    */
+  def emitNodeType(redisURI:RedisURI)(implicit executionContext: ExecutionContext): Future[KeyValue] = {
+    implicit val saladAPI = newSaladAPI
+    saladAPI.clusterNodes.map { allNodes =>
+      val removalNodeOpt = allNodes.find(node => node.getUri.equals(redisURI))
+      if (removalNodeOpt.isEmpty) throw new Exception(s"Node not in cluster: $redisURI")
+      val kv = removalNodeOpt.map { node =>
+        node.getRole match {
+          case Role.MASTER => KeyValue(RESHARD.key, node.getNodeId)
+          case Role.SLAVE  => KeyValue(REMOVE_SLAVE.key, node.getNodeId)
+          case _           => KeyValue(UNSUPPORTED.key, node.getNodeId)
+        }
+      }
+      kv.get
+    }
+  }
+
+  /**
+    * Safely remove a master by redistributing its hash slots before blacklisting it from the cluster.
+    * The data is given time to migrate as configured in `cornucopia.grace.period`.
+    *
+    * @param withoutNodes The list of ids of the master nodes that will be removed from the cluster.
+    * @param executionContext The thread dispatcher context.
+    * @return Indicate that the hash slots were redistributed and the master removed from the cluster.
+    */
+  protected def removeMasters(withoutNodes: Seq[String])(implicit executionContext: ExecutionContext): Future[Unit] = {
+    val reshardDone = reshardCluster(withoutNodes).map { _ =>
+      scala.concurrent.blocking(Thread.sleep(Config.Cornucopia.gracePeriod)) // Allow data to migrate.
+    }
+    reshardDone.flatMap(_ => forgetNodes(withoutNodes))
+  }
+
+  /**
+    * Notify all nodes in the cluster to forget this node.
+    *
+    * @param withoutNodes The list of ids of nodes to be forgotten by the cluster.
+    * @param executionContext The thread dispatcher context.
+    * @return A future indicating that the node was forgotten by all nodes in the cluster.
+    */
+  def forgetNodes(withoutNodes: Seq[String])(implicit executionContext: ExecutionContext): Future[Unit] =
+    if (!withoutNodes.exists(_.nonEmpty))
+      Future(Unit)
+    else {
+      implicit val saladAPI = newSaladAPI
+      saladAPI.clusterNodes.flatMap { allNodes =>
+        logger.info(s"Forgetting nodes: $withoutNodes")
+        // Reset the nodes to be removed.
+        val validWithoutNodes = withoutNodes.filter(_.nonEmpty)
+        validWithoutNodes.map(getConnection).map(_.map(_.clusterReset(true)))
+
+        // The nodes that will remain in the cluster should forget the nodes that will be removed.
+        val withNodes = allNodes
+          .filterNot(node => validWithoutNodes.contains(node.getNodeId)) // Node cannot forget itself.
+
+        // For the cross product of `withNodes` and `withoutNodes`; to remove the nodes in `withoutNodes`.
+        val forgetResults = for {
+          operatorNode <- withNodes
+          operandNodeId <- validWithoutNodes
+        } yield {
+          if (operatorNode.getSlaveOf == operandNodeId)
+            Future(Unit) // Node cannot forget its master.
+          else
+            getConnection(operatorNode.getNodeId).flatMap(_.clusterForget(operandNodeId))
+        }
+        Future.sequence(forgetResults).map(x => x)
+      }
+    }
+
+  /**
+    * Reshard the cluster using a view of the cluster consisting of a subset of master nodes.
+    *
+    * @param withoutNodes The list of ids of nodes that will not be assigned hash slots.
+    * @return Boolean indicating that all hash slots were reassigned successfully.
+    */
+  protected def reshardCluster(withoutNodes: Seq[String])
+  : Future[Unit] = {
+    // Execute futures using a thread pool so we don't run out of memory due to futures.
+    implicit val executionContext = Config.Consumer.actorSystem.dispatchers.lookup("akka.actor.resharding-dispatcher")
+    implicit val saladAPI = newSaladAPI
+    saladAPI.masterNodes.flatMap { masterNodes =>
+
+      val liveMasters = masterNodes.filter(_.isConnected)
+      lazy val idToURI = new util.HashMap[String,RedisURI](liveMasters.length + 1, 1)
+      // Re-use cluster connections so we don't exceed file-handle limit or waste resources.
+      lazy val clusterConnections = new util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]](liveMasters.length + 1, 1)
+      liveMasters.map { master =>
+        idToURI.put(master.getNodeId, master.getUri)
+        clusterConnections.put(master.getNodeId, getConnection(master.getNodeId))
+      }
+
+      // Remove dead nodes. This may generate WARN logs if some nodes already forgot the dead node.
+      val deadMastersIds = masterNodes.filterNot(_.isConnected).map(_.getNodeId)
+      logger.info(s"Dead nodes: $deadMastersIds")
+      forgetNodes(deadMastersIds)
+
+      // Migrate the data.
+      val assignableMasters = liveMasters.filterNot(masterNode => withoutNodes.contains(masterNode.getNodeId))
+      logger.info(s"Resharding cluster with ${assignableMasters.map(_.getNodeId)} without ${withoutNodes ++ deadMastersIds}")
+      val migrateResults = liveMasters.flatMap { node =>
+        val (sourceNodeId, slotList) = (node.getNodeId, node.getSlots.toList.map(_.toInt))
+        logger.debug(s"Migrating data from $sourceNodeId among slots $slotList")
+        slotList.map { slot =>
+          val destinationNodeId = slotNode(slot, assignableMasters)
+          migrateSlot(
+            slot,
+            sourceNodeId, destinationNodeId, idToURI.get(destinationNodeId),
+            assignableMasters, clusterConnections)
+        }
+      }
+      val finalMigrateResult = Future.sequence(migrateResults)
+      finalMigrateResult.onFailure { case e => logger.error(s"Failed to migrate hash slot data", e) }
+      finalMigrateResult.onSuccess { case _ => logger.info(s"Migrated hash slot data") }
+      // We attempted to migrate the data but do not prevent slot reassignment if migration fails.
+      // We may lose prior data but we ensure that all slots are assigned.
+      finalMigrateResult.onComplete { case _ =>
+        List.range(0, 16384).map(notifySlotAssignment(_, assignableMasters))
+      }
+      finalMigrateResult.flatMap(_ => forgetNodes(withoutNodes))
+    }
+  }
+
+  // TODO: pass slotNode as a lambda to migrateSlot and notifySlotAssignment.
+  // TODO: more efficient slot assignment to prevent data migration.
+  /**
+    * Choose a master node for a slot.
+    *
+    * @param slot The slot to be assigned.
+    * @param masters The list of masters that can be assigned slots.
+    * @return The node id of the chosen master.
+    */
+  protected def slotNode(slot: Int, masters: mutable.Buffer[RedisClusterNode]): String =
+    masters(slot % masters.length).getNodeId
+
+  /**
+    * Migrate all keys in a slot from the source node to the destination node and update the slot assignment on the
+    * affected nodes.
+    *
+    * @param slot The slot to migrate.
+    * @param sourceNodeId The current location of the slot data.
+    * @param destinationNodeId The target location of the slot data.
+    * @param masters The list of nodes in the cluster that will be assigned hash slots.
+    * @param clusterConnections The list of connections to nodes in the cluster.
+    * @param executionContext The thread dispatcher context.
+    * @return Future indicating success.
+    */
+  protected def migrateSlot(slot: Int, sourceNodeId: String, destinationNodeId: String, destinationURI: RedisURI,
+                          masters: mutable.Buffer[RedisClusterNode],
+                          clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]])
+                         (implicit saladAPI: Salad, executionContext: ExecutionContext)
+  : Future[Unit] = {
+    destinationNodeId match {
+      case `sourceNodeId` =>
+        // Don't migrate if the source and destination are the same.
+        Future(Unit)
+      case _ =>
+        for {
+          sourceConnection <- clusterConnections.get(sourceNodeId)
+          destinationConnection <- clusterConnections.get(destinationNodeId)
+        } yield {
+          // Sequentially execute the steps outline in:
+          // https://redis.io/commands/cluster-setslot#redis-cluster-live-resharding-explained
+          import com.github.kliewkliew.salad.serde.ByteArraySerdes._
+          val migrationResult =
+            for {
+              _ <- destinationConnection.clusterSetSlotStable(slot).recover { case _ => Unit }
+              _ <- sourceConnection.clusterSetSlotStable(slot).recover { case _ => Unit }
+              _ <- destinationConnection.clusterSetSlotImporting(slot, sourceNodeId)
+              _ <- sourceConnection.clusterSetSlotMigrating(slot, destinationNodeId)
+              keyCount <- sourceConnection.clusterCountKeysInSlot(slot)
+              keyList <- sourceConnection.clusterGetKeysInSlot[CodecType](slot, keyCount.toInt)
+              _ <- sourceConnection.migrate[CodecType](destinationURI, keyList.toList)
+              _ <- sourceConnection.clusterSetSlotNode(slot, destinationNodeId)
+              finalResult <- destinationConnection.clusterSetSlotNode(slot, destinationNodeId)
+            } yield {
+              finalResult
+            }
+          migrationResult.onSuccess { case _ => logger.trace(s"Migrated data of slot $slot from $sourceNodeId to: $destinationNodeId at $destinationURI") }
+          migrationResult.onFailure { case e => logger.debug(s"Failed to migrate data of slot $slot from $sourceNodeId to: $destinationNodeId at $destinationURI", e)}
+          // Undocumented but necessary final steps found in http://download.redis.io/redis-stable/src/redis-trib.rb
+          // `recover` to perform these steps even if the previous steps failed, but don't perform these steps until the previous steps did attempt execution.
+          val finalMigrationResult = migrationResult.recover { case _ => Unit }
+            .flatMap(_ => notifySlotAssignment(slot, masters)).recover { case _ => Unit }
+            .flatMap(_ => sourceConnection.clusterDelSlot(slot)).recover { case _ => Unit }
+            .flatMap(_ => destinationConnection.clusterAddSlot(slot))
+          finalMigrationResult.onSuccess { case _ => logger.trace(s"Updated slot table for slot $slot") }
+          finalMigrationResult.onFailure { case e => logger.debug(s"Failed to update slot table for slot $slot", e)}
+          finalMigrationResult.map(x => x)
+        }
+    }
+  }
+
+  /**
+    * Notify all master nodes of a slot assignment so that they will immediately be able to redirect clients.
+    *
+    * @param masters The list of nodes in the cluster that will be assigned hash slots.
+    * @param executionContext The thread dispatcher context.
+    * @return Future indicating success.
+    */
+  protected def notifySlotAssignment(slot: Int, masters: mutable.Buffer[RedisClusterNode])
+                                  (implicit saladAPI: Salad, executionContext: ExecutionContext)
+  : Future[Unit] = {
+    val getMasterConnections = masters.map(master => getConnection(master.getNodeId))
+    Future.sequence(getMasterConnections).flatMap { masterConnections =>
+      val notifyResults = masterConnections.map(_.clusterSetSlotNode(slot, slotNode(slot, masters)))
+      Future.sequence(notifyResults).map(x => x)
+    }
+  }
+
+  /**
+    * Store the n poorest masters.
+    * Implemented on scala.mutable.PriorityQueue.
+    *
+    * @param n
+    */
+  sealed case class MaxNHeapMasterSlaveCount(n: Int) {
+    private type MSTuple = (String, AtomicInteger)
+    private object MSOrdering extends Ordering[MSTuple] {
+      def compare(a: MSTuple, b: MSTuple) = a._2.intValue compare b._2.intValue
+    }
+    implicit private val ordering = MSOrdering
+    val underlying = new mutable.PriorityQueue[MSTuple]
+
+    /**
+      * O(1) if the entry is not a candidate for the being one of the poorest n masters.
+      * O(log(n)) if the entry is a candidate.
+      *
+      * @param entry The candidate master-slavecount tuple.
+      */
+    def offer(entry: MSTuple) =
+      if (n > underlying.length) {
+        underlying.enqueue(entry)
+      }
+      else if (entry._2.intValue < underlying.head._2.intValue) {
+        underlying.dequeue()
+        underlying.enqueue(entry)
+      }
+  }
+}
+
+class CornucopiaKafkaSource extends CornucopiaGraph {
+  import com.github.kliewkliew.cornucopia.kafka.Config.Consumer.materializer
+
+  private type KafkaRecord = ConsumerRecord[String, String]
+
+  private def extractKeyValue = Flow[KafkaRecord]
+    .map[KeyValue](record => KeyValue(record.key, record.value))
+
+  def run = RunnableGraph.fromGraph(GraphDSL.create() { implicit builder =>
+    import scala.concurrent.ExecutionContext.Implicits.global
+    import GraphDSL.Implicits._
+
+    val in = Config.Consumer.cornucopiaKafkaSource
+    val out = Sink.ignore
+
+    val mergeFeedback = builder.add(MergePreferred[KeyValue](2))
+
+    val partition = builder.add(Partition[KeyValue](
+      5, kv => partitionEvents(kv.key)))
+
+    val kv = builder.add(extractKeyValue)
+
+    val partitionRm = builder.add(Partition[KeyValue](
+      3, kv => partitionNodeRemoval(kv.key)
+    ))
+
+    in                                      ~> kv
+    kv                                      ~> mergeFeedback.preferred
+    mergeFeedback.out                       ~> partition
+    partition.out(ADD_MASTER.ordinal)       ~> streamAddMaster      ~> mergeFeedback.in(0)
+    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave       ~> out
+    partition.out(REMOVE_NODE.ordinal)      ~> streamRemoveNode     ~> partitionRm
+    partitionRm.out(REMOVE_MASTER.ordinal)  ~> mergeFeedback.in(1)
+    partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave    ~> out
+    partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation ~> out
+    partition.out(RESHARD.ordinal)          ~> streamReshard        ~> out
+    partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation ~> out
+
+    ClosedShape
+  }).run()
+
+}
+
+class CornucopiaActorSource extends CornucopiaGraph {
+  import com.github.kliewkliew.cornucopia.kafka.Config.Consumer.materializer
+  import com.github.kliewkliew.cornucopia.actors.CornucopiaSource.Task
+  import scala.concurrent.ExecutionContext.Implicits.global
+
+  private type ActorRecord = Task
+
+  private def extractKeyValue = Flow[ActorRecord]
+    .map[KeyValue](record => KeyValue(record.operation, record.redisNodeIp))
+
+  private val processTask = Flow.fromGraph(GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    val taskSource = builder.add(Flow[Task])
+
+    val mergeFeedback = builder.add(MergePreferred[KeyValue](2))
+
+    val partition = builder.add(Partition[KeyValue](
+      5, kv => partitionEvents(kv.key)))
+
+    val kv = builder.add(extractKeyValue)
+
+    val partitionRm = builder.add(Partition[KeyValue](
+      3, kv => partitionNodeRemoval(kv.key)
+    ))
+
+    val out = builder.add(Flow[Any])
+
+    taskSource.out                          ~> kv
+    kv                                      ~> mergeFeedback.preferred
+    mergeFeedback.out                       ~> partition
+    partition.out(ADD_MASTER.ordinal)       ~> streamAddMaster      ~> mergeFeedback.in(0)
+    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave       ~> out
+    partition.out(REMOVE_NODE.ordinal)      ~> streamRemoveNode     ~> partitionRm
+    partitionRm.out(REMOVE_MASTER.ordinal)  ~> mergeFeedback.in(1)
+    partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave    ~> out
+    partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation ~> out
+    partition.out(RESHARD.ordinal)          ~> streamReshard        ~> out
+    partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation ~> out
+
+    FlowShape(taskSource.in, out.out)
+  })
+
+  private val cornucopiaSource = Config.Consumer.cornucopiaActorSource
+
+  def ref: ActorRef = processTask
+    .to(Sink.ignore)
+    .runWith(cornucopiaSource)
+
+}
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Config.scala b/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Config.scala
index 13dd208..acf61ce 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Config.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Config.scala
@@ -1,9 +1,11 @@
 package com.github.kliewkliew.cornucopia.kafka
 
 import akka.actor.ActorSystem
+import akka.stream.scaladsl.Source
 import akka.kafka.scaladsl.{Producer, Consumer => ConsumerDSL}
 import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions}
-import akka.stream.{ActorMaterializer, ActorMaterializerSettings, Supervision}
+import akka.stream.{ActorMaterializer, ActorMaterializerSettings, ClosedShape, Supervision}
+import com.github.kliewkliew.cornucopia.actors.CornucopiaSource
 import com.typesafe.config.ConfigFactory
 import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
 import org.slf4j.LoggerFactory
@@ -17,6 +19,7 @@ object Config {
     val gracePeriod = config.getInt("grace.period") * 1000
     val refreshTimeout = config.getInt("refresh.timeout") * 1000
     val batchPeriod = config.getInt("batch.period").seconds
+    val source = config.getString("source")
   }
 
   object Consumer {
@@ -43,7 +46,14 @@ object Config {
       .withBootstrapServers(kafkaServers)
 
     implicit val materializer = ActorMaterializer(materializerSettings)(actorSystem)
+
+    val cornucopiaActorProps = CornucopiaSource.props
+    val cornucopiaActorSource = Source.actorPublisher[CornucopiaSource.Task](cornucopiaActorProps)
+
+    val cornucopiaKafkaSource = ConsumerDSL.plainSource(sourceSettings, subscription)
+
     val cornucopiaSource = ConsumerDSL.plainSource(sourceSettings, subscription)
+
     val cornucopiaSink = Producer.plainSink(sinkSettings)
   }
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Consumer.scala b/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Consumer.scala
index b51103a..6e07dd3 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Consumer.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Consumer.scala
@@ -5,11 +5,15 @@ import java.util
 import Config.Consumer.{cornucopiaSource, materializer}
 import com.github.kliewkliew.cornucopia.redis.Connection._
 import akka.stream.{ClosedShape, ThrottleMode}
-import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink}
+//import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink}
+import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink, Source}
+import akka.stream.scaladsl.Source
 import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import java.util.concurrent.atomic.AtomicInteger
 
+import com.github.kliewkliew.cornucopia.actors.CornucopiaSource
+import com.github.kliewkliew.cornucopia.redis._
 import com.github.kliewkliew.salad.SaladClusterAPI
 import com.lambdaworks.redis.RedisURI
 import com.lambdaworks.redis.models.role.RedisInstance.Role
@@ -22,7 +26,8 @@ import scala.language.implicitConversions
 import scala.concurrent.{ExecutionContext, Future}
 
 class Consumer {
-  private type Record = ConsumerRecord[String, String]
+  private type KafkaRecord = ConsumerRecord[String, String]
+  private type ActorRecord = Map[String, String]
   private val logger = LoggerFactory.getLogger(this.getClass)
 
   /**
@@ -82,7 +87,7 @@ class Consumer {
     */
   // Extract a tuple of the key and value from a Kafka record.
   case class KeyValue(key: String, value: String)
-  private def extractKeyValue = Flow[Record]
+  private def extractKeyValue = Flow[KafkaRecord]
     .map[KeyValue](record => KeyValue(record.key, record.value))
 
   // Add a master node to the cluster.
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Operation.scala b/src/main/scala/com/github/kliewkliew/cornucopia/redis/Operation.scala
similarity index 94%
rename from src/main/scala/com/github/kliewkliew/cornucopia/kafka/Operation.scala
rename to src/main/scala/com/github/kliewkliew/cornucopia/redis/Operation.scala
index e496ebf..01aa89e 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Operation.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/redis/Operation.scala
@@ -1,4 +1,4 @@
-package com.github.kliewkliew.cornucopia.kafka
+package com.github.kliewkliew.cornucopia.redis
 
 trait Operation {
   def key: String

From 0c2a57208819a5c7751d627214cd9faa6ad58769 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 23 Apr 2017 20:13:08 -0700
Subject: [PATCH 02/44] WIP: expose source actor in library object

---
 build.sbt                                                   | 6 ++++--
 src/main/resources/application.conf                         | 3 +++
 .../scala/com/github/kliewkliew/cornucopia/Library.scala    | 2 ++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index b85eeb6..4bb2dd1 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,7 +1,8 @@
 name := "cornucopia"
 organization := "com.github.kliewkliew"
 
-version := "1.1.2"
+//version := "1.1.2"
+version := "0.1-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
@@ -12,6 +13,7 @@ libraryDependencies ++= Seq(
   "biz.paluch.redis" % "lettuce" % "5.0.0.Beta1",
   "org.scala-lang.modules" % "scala-java8-compat_2.11" % "0.8.0",
   "com.typesafe.akka" %% "akka-stream-kafka" % "0.11-RC1",
-  "com.github.kliewkliew" %% "salad" % "0.11.01",
+//  "com.github.kliewkliew" %% "salad" % "0.11.01",
+  "com.adenda" %% "salad" % "0.11.03",
   "org.slf4j" % "slf4j-log4j12" % "1.7.22"
 )
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index 2f4999a..6babffd 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -43,6 +43,9 @@ cornucopia {
 
   // Time (seconds) to wait for batches to accumulate before executing a job.
   batch.period = 5
+
+  // either `kafka` or `actor`
+  source = "actor"
 }
 
 kafka {
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala b/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
index 82f0553..d2a31b9 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
@@ -1,7 +1,9 @@
 package com.github.kliewkliew.cornucopia
+import actors.CornucopiaSource
 
 import akka.actor._
 
 object Library {
   val ref: ActorRef = new graph.CornucopiaActorSource().ref
+  val source = CornucopiaSource
 }

From 2a3cc4cf64978a70e03d8184bdce749308679adb Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Wed, 26 Apr 2017 13:44:57 -0700
Subject: [PATCH 03/44] WIP: trying stuff

---
 build.sbt                                     | 12 +++++++++--
 .../kliewkliew/cornucopia/graph/Graph.scala   | 20 +++++++++----------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/build.sbt b/build.sbt
index 4bb2dd1..c48c43a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,13 +2,21 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.1-SNAPSHOT"
+version := "0.2-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
 resolvers += "Sonatype Releases" at "https://oss.sonatype.org/service/repositories/releases/"
 resolvers += "Typesafe Releases" at "http://repo.typesafe.com/typesafe/releases/"
 
+val akkaVersion = "2.4.17"
+
+val testDependencies = Seq(
+  "com.typesafe.akka" %%  "akka-testkit" % akkaVersion  % "test",
+  "org.scalatest"     %%  "scalatest"    % "3.0.0"      % "test",
+  "org.mockito" % "mockito-all" % "1.10.19" % Test
+)
+
 libraryDependencies ++= Seq(
   "biz.paluch.redis" % "lettuce" % "5.0.0.Beta1",
   "org.scala-lang.modules" % "scala-java8-compat_2.11" % "0.8.0",
@@ -16,4 +24,4 @@ libraryDependencies ++= Seq(
 //  "com.github.kliewkliew" %% "salad" % "0.11.01",
   "com.adenda" %% "salad" % "0.11.03",
   "org.slf4j" % "slf4j-log4j12" % "1.7.22"
-)
+) ++ testDependencies
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 9dd17e1..e19418a 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -50,7 +50,7 @@ trait CornucopiaGraph {
   case class KeyValue(key: String, value: String)
 
   // Add a master node to the cluster.
-  protected def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+  def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(_.value)
     .map(RedisURI.create)
     .map(newSaladAPI.canonicalizeURI)
@@ -488,12 +488,12 @@ class CornucopiaActorSource extends CornucopiaGraph {
   import com.github.kliewkliew.cornucopia.actors.CornucopiaSource.Task
   import scala.concurrent.ExecutionContext.Implicits.global
 
-  private type ActorRecord = Task
+  protected type ActorRecord = Task
 
-  private def extractKeyValue = Flow[ActorRecord]
+  protected def extractKeyValue = Flow[ActorRecord]
     .map[KeyValue](record => KeyValue(record.operation, record.redisNodeIp))
 
-  private val processTask = Flow.fromGraph(GraphDSL.create() { implicit builder =>
+  protected val processTask = Flow.fromGraph(GraphDSL.create() { implicit builder =>
     import GraphDSL.Implicits._
 
     val taskSource = builder.add(Flow[Task])
@@ -515,18 +515,18 @@ class CornucopiaActorSource extends CornucopiaGraph {
     kv                                      ~> mergeFeedback.preferred
     mergeFeedback.out                       ~> partition
     partition.out(ADD_MASTER.ordinal)       ~> streamAddMaster      ~> mergeFeedback.in(0)
-    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave       ~> out
+    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave
     partition.out(REMOVE_NODE.ordinal)      ~> streamRemoveNode     ~> partitionRm
     partitionRm.out(REMOVE_MASTER.ordinal)  ~> mergeFeedback.in(1)
-    partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave    ~> out
-    partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation ~> out
-    partition.out(RESHARD.ordinal)          ~> streamReshard        ~> out
-    partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation ~> out
+    partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave
+    partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation
+    partition.out(RESHARD.ordinal)          ~> streamReshard
+    partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation
 
     FlowShape(taskSource.in, out.out)
   })
 
-  private val cornucopiaSource = Config.Consumer.cornucopiaActorSource
+  protected val cornucopiaSource = Config.Consumer.cornucopiaActorSource
 
   def ref: ActorRef = processTask
     .to(Sink.ignore)

From efc8775431c632695353a4a1ddb90c7825b779b4 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Wed, 26 Apr 2017 13:49:34 -0700
Subject: [PATCH 04/44] version 0.4-SNAPSHOT

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index c48c43a..b5c30c5 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.2-SNAPSHOT"
+version := "0.4-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From 9a2698e075e5e3929267ab5e819f0d147087bd0b Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Wed, 26 Apr 2017 17:40:09 -0700
Subject: [PATCH 05/44] WIP: fixing the graph for actor source

---
 build.sbt                                        |  2 +-
 .../kliewkliew/cornucopia/graph/Graph.scala      | 16 ++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/build.sbt b/build.sbt
index b5c30c5..2957fc4 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.4-SNAPSHOT"
+version := "0.5-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index e19418a..6be168f 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
 import scala.collection.JavaConversions._
 import scala.language.implicitConversions
 import scala.concurrent.{ExecutionContext, Future}
-import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink, Source}
+import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink, Source, Merge}
 import com.github.kliewkliew.cornucopia.kafka.Config // TO-DO: put config someplace else
 
 trait CornucopiaGraph {
@@ -509,21 +509,21 @@ class CornucopiaActorSource extends CornucopiaGraph {
       3, kv => partitionNodeRemoval(kv.key)
     ))
 
-    val out = builder.add(Flow[Any])
+    val fanOut = builder.add(Merge[Any](5))
 
     taskSource.out                          ~> kv
     kv                                      ~> mergeFeedback.preferred
     mergeFeedback.out                       ~> partition
     partition.out(ADD_MASTER.ordinal)       ~> streamAddMaster      ~> mergeFeedback.in(0)
-    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave
+    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave       ~> fanOut.in(0)
     partition.out(REMOVE_NODE.ordinal)      ~> streamRemoveNode     ~> partitionRm
     partitionRm.out(REMOVE_MASTER.ordinal)  ~> mergeFeedback.in(1)
-    partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave
-    partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation
-    partition.out(RESHARD.ordinal)          ~> streamReshard
-    partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation
+    partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave    ~> fanOut.in(1)
+    partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation ~> fanOut.in(2)
+    partition.out(RESHARD.ordinal)          ~> streamReshard        ~> fanOut.in(3)
+    partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation ~> fanOut.in(4)
 
-    FlowShape(taskSource.in, out.out)
+    FlowShape(taskSource.in, fanOut.out)
   })
 
   protected val cornucopiaSource = Config.Consumer.cornucopiaActorSource

From 2b5865e1ea30a82cadcf59003214cd486d3431fe Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 27 Apr 2017 11:45:26 -0700
Subject: [PATCH 06/44] Add test for adding a new master using actor publisher

---
 .../kliewkliew/cornucopia/LibraryTest.scala   | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala

diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
new file mode 100644
index 0000000..0468ebf
--- /dev/null
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -0,0 +1,75 @@
+package com.github.kliewkliew.cornucopia
+
+//import com.github.kliewkliew.cornucopia._
+import com.github.kliewkliew.cornucopia.graph._
+import akka.testkit.{TestActorRef, TestKit, TestProbe}
+import akka.actor.{ActorSystem, ActorRef}
+import akka.actor.Status.Failure
+import akka.stream.ActorMaterializer
+import org.scalatest.mockito.MockitoSugar
+import org.scalatest.{BeforeAndAfterAll, MustMatchers, WordSpecLike}
+import org.mockito.Mockito._
+import org.scalatest.mockito.MockitoSugar._
+import com.github.kliewkliew.cornucopia.graph._
+import akka.stream.scaladsl.Flow
+import akka.stream.scaladsl.Sink
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+
+class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
+  with WordSpecLike with BeforeAndAfterAll with MustMatchers with MockitoSugar {
+
+  trait FakeCornucopiaActorSourceGraph {
+    import com.github.kliewkliew.cornucopia.kafka.Config.Consumer.materializer
+
+    class CornucopiaActorSourceLocal extends CornucopiaActorSource {
+      lazy val probe = TestProbe()
+
+      override def streamAddMaster(implicit executionContext: ExecutionContext) =
+        Flow[KeyValue].map(_ => {
+          probe.ref ! "streamAddMaster"
+          Thread.sleep(300)
+          KeyValue("*reshard", "")
+        })
+
+      override def streamAddSlave(implicit executionContext: ExecutionContext) =
+        Flow[KeyValue].map(_ => KeyValue("test", ""))
+
+      override def streamRemoveNode(implicit executionContext: ExecutionContext) =
+        Flow[KeyValue].map(_ => KeyValue("test", ""))
+
+      override def streamRemoveSlave(implicit executionContext: ExecutionContext) =
+        Flow[KeyValue].map(_ => KeyValue("test", ""))
+
+      override def streamReshard(implicit executionContext: ExecutionContext) =
+        Flow[KeyValue].map(_ => {
+          probe.ref ! "streamReshard"
+          KeyValue("*reshard", "")
+        })
+    }
+
+  }
+
+  implicit val ec = system.dispatcher
+
+  override def afterAll(): Unit = {
+    system.terminate()
+  }
+
+  "Add master" must {
+    "add new master and reshard cluster" in new FakeCornucopiaActorSourceGraph {
+      import Library.source._
+
+      val cornucopiaActorSourceLocal = new CornucopiaActorSourceLocal
+
+      private val ref = cornucopiaActorSourceLocal.ref
+
+      ref ! Task("+master", "123.456.789.10")
+
+      cornucopiaActorSourceLocal.probe.expectMsg(100 millis, "streamAddMaster")
+
+      cornucopiaActorSourceLocal.probe.expectMsg(350 millis, "streamReshard")
+    }
+  }
+
+}

From 2f694dc22e977c4ba0a730e5b1ab783b92d4527c Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 27 Apr 2017 11:46:12 -0700
Subject: [PATCH 07/44] Provide explicit type annotations for stream processing
 topology refresh

---
 .../com/github/kliewkliew/cornucopia/graph/Graph.scala | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 6be168f..5f713ef 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -56,7 +56,7 @@ trait CornucopiaGraph {
     .map(newSaladAPI.canonicalizeURI)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(addNodesToCluster)
-    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(waitForTopologyRefresh[Seq[RedisURI]])
     .map(_ => KeyValue(RESHARD.key, ""))
 
   // Add a slave node to the cluster, replicating the master that has the fewest slaves.
@@ -66,9 +66,9 @@ trait CornucopiaGraph {
     .map(newSaladAPI.canonicalizeURI)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(addNodesToCluster)
-    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(waitForTopologyRefresh[Seq[RedisURI]])
     .mapAsync(1)(findMasters)
-    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(waitForTopologyRefresh[Unit])
     .mapAsync(1)(_ => logTopology)
 
   // Emit a key-value pair indicating the node type and URI.
@@ -83,7 +83,7 @@ trait CornucopiaGraph {
     .map(_.value)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(forgetNodes)
-    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(waitForTopologyRefresh[Unit])
     .mapAsync(1)(_ => logTopology)
 
   // Redistribute the hash slots among all nodes in the cluster.
@@ -94,7 +94,7 @@ trait CornucopiaGraph {
     .conflate((seq1, seq2) => seq1 ++ seq2)
     .throttle(1, Config.Cornucopia.minReshardWait, 1, ThrottleMode.Shaping)
     .mapAsync(1)(reshardCluster)
-    .mapAsync(1)(waitForTopologyRefresh)
+    .mapAsync(1)(waitForTopologyRefresh[Unit])
     .mapAsync(1)(_ => logTopology)
 
   // Throw for keys indicating unsupported operations.

From 76364cc18fbde03f8b7645315c9de8f36ced06a7 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 27 Apr 2017 11:47:27 -0700
Subject: [PATCH 08/44] Version 0.6-SNAPSHOT

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 2957fc4..7a8462d 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.5-SNAPSHOT"
+version := "0.6-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From c79104835c58b05a3ebf74c5a71db294e687f75e Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 27 Apr 2017 12:36:01 -0700
Subject: [PATCH 09/44] Update test to use proper redis uri

---
 .../github/kliewkliew/cornucopia/LibraryTest.scala  | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index 0468ebf..c9f9c80 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -26,9 +26,12 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
       lazy val probe = TestProbe()
 
       override def streamAddMaster(implicit executionContext: ExecutionContext) =
-        Flow[KeyValue].map(_ => {
+        Flow[KeyValue].map(kv => {
+          val ip = kv.value
           probe.ref ! "streamAddMaster"
           Thread.sleep(300)
+          probe.ref ! ip
+          Thread.sleep(300)
           KeyValue("*reshard", "")
         })
 
@@ -64,11 +67,15 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
 
       private val ref = cornucopiaActorSourceLocal.ref
 
-      ref ! Task("+master", "123.456.789.10")
+      private val redisUri = "redis://123.456.789.10"
+
+      ref ! Task("+master", redisUri)
 
       cornucopiaActorSourceLocal.probe.expectMsg(100 millis, "streamAddMaster")
 
-      cornucopiaActorSourceLocal.probe.expectMsg(350 millis, "streamReshard")
+      cornucopiaActorSourceLocal.probe.expectMsg(350 millis, redisUri)
+
+      cornucopiaActorSourceLocal.probe.expectMsg(700 millis, "streamReshard")
     }
   }
 

From 6d7c0ff918ac92717a930543fac92d6a3b3ea9c5 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 27 Apr 2017 16:51:13 -0700
Subject: [PATCH 10/44] WIP: implementing the library interface

---
 .../cornucopia/{kafka => }/Config.scala       |   6 +-
 .../kliewkliew/cornucopia/graph/Graph.scala   |  13 +-
 .../cornucopia/kafka/Consumer.scala           | 483 ------------------
 .../kliewkliew/cornucopia/LibraryTest.scala   |   2 +-
 4 files changed, 10 insertions(+), 494 deletions(-)
 rename src/main/scala/com/github/kliewkliew/cornucopia/{kafka => }/Config.scala (97%)
 delete mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/kafka/Consumer.scala

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Config.scala b/src/main/scala/com/github/kliewkliew/cornucopia/Config.scala
similarity index 97%
rename from src/main/scala/com/github/kliewkliew/cornucopia/kafka/Config.scala
rename to src/main/scala/com/github/kliewkliew/cornucopia/Config.scala
index acf61ce..432a66f 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Config.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/Config.scala
@@ -1,10 +1,10 @@
-package com.github.kliewkliew.cornucopia.kafka
+package com.github.kliewkliew.cornucopia
 
 import akka.actor.ActorSystem
-import akka.stream.scaladsl.Source
 import akka.kafka.scaladsl.{Producer, Consumer => ConsumerDSL}
 import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions}
-import akka.stream.{ActorMaterializer, ActorMaterializerSettings, ClosedShape, Supervision}
+import akka.stream.scaladsl.Source
+import akka.stream.{ActorMaterializer, ActorMaterializerSettings, Supervision}
 import com.github.kliewkliew.cornucopia.actors.CornucopiaSource
 import com.typesafe.config.ConfigFactory
 import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 5f713ef..5c28943 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -4,9 +4,7 @@ import java.util
 import java.util.concurrent.atomic.AtomicInteger
 
 import akka.actor._
-import akka.NotUsed
-import akka.io.Udp.SO.Broadcast
-import akka.stream.{ClosedShape, ThrottleMode, FlowShape, Inlet}
+import akka.stream.{ClosedShape, FlowShape, ThrottleMode}
 import com.github.kliewkliew.cornucopia.redis.Connection.{CodecType, Salad, getConnection, newSaladAPI}
 import org.slf4j.LoggerFactory
 import org.apache.kafka.clients.consumer.ConsumerRecord
@@ -21,8 +19,9 @@ import scala.collection.mutable
 import scala.collection.JavaConversions._
 import scala.language.implicitConversions
 import scala.concurrent.{ExecutionContext, Future}
-import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink, Source, Merge}
-import com.github.kliewkliew.cornucopia.kafka.Config // TO-DO: put config someplace else
+import akka.stream.scaladsl.{Flow, GraphDSL, Merge, MergePreferred, Partition, RunnableGraph, Sink}
+import com.github.kliewkliew.cornucopia.Config
+// TO-DO: put config someplace else
 
 trait CornucopiaGraph {
   import scala.concurrent.ExecutionContext.Implicits.global
@@ -441,7 +440,7 @@ trait CornucopiaGraph {
 }
 
 class CornucopiaKafkaSource extends CornucopiaGraph {
-  import com.github.kliewkliew.cornucopia.kafka.Config.Consumer.materializer
+  import Config.Consumer.materializer
 
   private type KafkaRecord = ConsumerRecord[String, String]
 
@@ -484,7 +483,7 @@ class CornucopiaKafkaSource extends CornucopiaGraph {
 }
 
 class CornucopiaActorSource extends CornucopiaGraph {
-  import com.github.kliewkliew.cornucopia.kafka.Config.Consumer.materializer
+  import Config.Consumer.materializer
   import com.github.kliewkliew.cornucopia.actors.CornucopiaSource.Task
   import scala.concurrent.ExecutionContext.Implicits.global
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Consumer.scala b/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Consumer.scala
deleted file mode 100644
index 6e07dd3..0000000
--- a/src/main/scala/com/github/kliewkliew/cornucopia/kafka/Consumer.scala
+++ /dev/null
@@ -1,483 +0,0 @@
-package com.github.kliewkliew.cornucopia.kafka
-
-import java.util
-
-import Config.Consumer.{cornucopiaSource, materializer}
-import com.github.kliewkliew.cornucopia.redis.Connection._
-import akka.stream.{ClosedShape, ThrottleMode}
-//import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink}
-import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition, RunnableGraph, Sink, Source}
-import akka.stream.scaladsl.Source
-import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
-import org.apache.kafka.clients.consumer.ConsumerRecord
-import java.util.concurrent.atomic.AtomicInteger
-
-import com.github.kliewkliew.cornucopia.actors.CornucopiaSource
-import com.github.kliewkliew.cornucopia.redis._
-import com.github.kliewkliew.salad.SaladClusterAPI
-import com.lambdaworks.redis.RedisURI
-import com.lambdaworks.redis.models.role.RedisInstance.Role
-import org.slf4j.LoggerFactory
-
-import collection.JavaConverters._
-import scala.collection.mutable
-import scala.collection.JavaConversions._
-import scala.language.implicitConversions
-import scala.concurrent.{ExecutionContext, Future}
-
-class Consumer {
-  private type KafkaRecord = ConsumerRecord[String, String]
-  private type ActorRecord = Map[String, String]
-  private val logger = LoggerFactory.getLogger(this.getClass)
-
-  /**
-    * Run the graph to process the event stream from Kafka.
-    *
-    * @return
-    */
-  def run = RunnableGraph.fromGraph(GraphDSL.create() { implicit builder =>
-    import GraphDSL.Implicits._
-    import scala.concurrent.ExecutionContext.Implicits.global
-
-    val in = cornucopiaSource
-    val out = Sink.ignore
-
-    def partitionEvents(key: String) = key.trim.toLowerCase match {
-      case ADD_MASTER.key     => ADD_MASTER.ordinal
-      case ADD_SLAVE.key      => ADD_SLAVE.ordinal
-      case REMOVE_NODE.key    => REMOVE_NODE.ordinal
-      case RESHARD.key        => RESHARD.ordinal
-      case _                  => UNSUPPORTED.ordinal
-    }
-
-    def partitionNodeRemoval(key: String) = key.trim.toLowerCase match {
-      case REMOVE_MASTER.key  => REMOVE_MASTER.ordinal
-      case REMOVE_SLAVE.key   => REMOVE_SLAVE.ordinal
-      case UNSUPPORTED.key    => UNSUPPORTED.ordinal
-    }
-
-    val mergeFeedback = builder.add(MergePreferred[KeyValue](2))
-
-    val partition = builder.add(Partition[KeyValue](
-      5, kv => partitionEvents(kv.key)))
-
-    val kv = builder.add(extractKeyValue)
-
-    val partitionRm = builder.add(Partition[KeyValue](
-      3, kv => partitionNodeRemoval(kv.key)
-    ))
-
-    in ~> kv
-    kv ~> mergeFeedback.preferred
-    mergeFeedback.out  ~> partition
-                          partition.out(ADD_MASTER.ordinal)    ~> streamAddMaster      ~> mergeFeedback.in(0)
-                          partition.out(ADD_SLAVE.ordinal)     ~> streamAddSlave       ~> out
-                          partition.out(REMOVE_NODE.ordinal)   ~> streamRemoveNode     ~> partitionRm
-                                                                                          partitionRm.out(REMOVE_MASTER.ordinal)  ~> mergeFeedback.in(1)
-                                                                                          partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave    ~> out
-                                                                                          partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation ~> out
-                          partition.out(RESHARD.ordinal)       ~> streamReshard        ~> out
-                          partition.out(UNSUPPORTED.ordinal)   ~> unsupportedOperation ~> out
-
-    ClosedShape
-  }).run()
-
-  /**
-    * Stream definitions for the graph.
-    */
-  // Extract a tuple of the key and value from a Kafka record.
-  case class KeyValue(key: String, value: String)
-  private def extractKeyValue = Flow[KafkaRecord]
-    .map[KeyValue](record => KeyValue(record.key, record.value))
-
-  // Add a master node to the cluster.
-  private def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
-    .map(_.value)
-    .map(RedisURI.create)
-    .map(newSaladAPI.canonicalizeURI)
-    .groupedWithin(100, Config.Cornucopia.batchPeriod)
-    .mapAsync(1)(addNodesToCluster)
-    .mapAsync(1)(waitForTopologyRefresh)
-    .map(_ => KeyValue(RESHARD.key, ""))
-
-  // Add a slave node to the cluster, replicating the master that has the fewest slaves.
-  private def streamAddSlave(implicit executionContext: ExecutionContext) = Flow[KeyValue]
-    .map(_.value)
-    .map(RedisURI.create)
-    .map(newSaladAPI.canonicalizeURI)
-    .groupedWithin(100, Config.Cornucopia.batchPeriod)
-    .mapAsync(1)(addNodesToCluster)
-    .mapAsync(1)(waitForTopologyRefresh)
-    .mapAsync(1)(findMasters)
-    .mapAsync(1)(waitForTopologyRefresh)
-    .mapAsync(1)(_ => logTopology)
-
-  // Emit a key-value pair indicating the node type and URI.
-  private def streamRemoveNode(implicit executionContext: ExecutionContext) = Flow[KeyValue]
-    .map(_.value)
-    .map(RedisURI.create)
-    .map(newSaladAPI.canonicalizeURI)
-    .mapAsync(1)(emitNodeType)
-
-  // Remove a slave node from the cluster.
-  private def streamRemoveSlave(implicit executionContext: ExecutionContext) = Flow[KeyValue]
-    .map(_.value)
-    .groupedWithin(100, Config.Cornucopia.batchPeriod)
-    .mapAsync(1)(forgetNodes)
-    .mapAsync(1)(waitForTopologyRefresh)
-    .mapAsync(1)(_ => logTopology)
-
-  // Redistribute the hash slots among all nodes in the cluster.
-  // Execute slot redistribution at most once per configured interval.
-  // Combine multiple requests into one request.
-  private def streamReshard(implicit executionContext: ExecutionContext) = Flow[KeyValue]
-    .map(record => Seq(record.value))
-    .conflate((seq1, seq2) => seq1 ++ seq2)
-    .throttle(1, Config.Cornucopia.minReshardWait, 1, ThrottleMode.Shaping)
-    .mapAsync(1)(reshardCluster)
-    .mapAsync(1)(waitForTopologyRefresh)
-    .mapAsync(1)(_ => logTopology)
-
-  // Throw for keys indicating unsupported operations.
-  private def unsupportedOperation = Flow[KeyValue]
-    .map(record => throw new IllegalArgumentException(s"Unsupported operation ${record.key} for ${record.value}"))
-
-  /**
-    * Wait for the new cluster topology view to propagate to all nodes in the cluster. May not be strictly necessary
-    * since this microservice immediately attempts to notify all nodes of topology updates.
-    *
-    * @param passthrough The value that will be passed through to the next map stage.
-    * @param executionContext The thread dispatcher context.
-    * @tparam T
-    * @return The unmodified input value.
-    */
-  private def waitForTopologyRefresh[T](passthrough: T)(implicit executionContext: ExecutionContext): Future[T] = Future {
-    scala.concurrent.blocking(Thread.sleep(Config.Cornucopia.refreshTimeout))
-    passthrough
-  }
-
-  /**
-    * Log the current view of the cluster topology.
-    *
-    * @param executionContext The thread dispatcher context.
-    * @return
-    */
-  private def logTopology(implicit executionContext: ExecutionContext): Future[Unit] = {
-    implicit val saladAPI = newSaladAPI
-    saladAPI.clusterNodes.map { allNodes =>
-      val masterNodes = allNodes.filter(Role.MASTER == _.getRole)
-      val slaveNodes = allNodes.filter(Role.SLAVE == _.getRole)
-      logger.info(s"Master nodes: $masterNodes")
-      logger.info(s"Slave nodes: $slaveNodes")
-    }
-  }
-
-  /**
-    * The entire cluster will meet the new nodes at the given URIs.
-    *
-    * @param redisURIList The list of URI of the new nodes.
-    * @param executionContext The thread dispatcher context.
-    * @return The list of URI if the nodes were met. TODO: emit only the nodes that were successfully added.
-    */
-  private def addNodesToCluster(redisURIList: Seq[RedisURI])(implicit executionContext: ExecutionContext): Future[Seq[RedisURI]] = {
-    implicit val saladAPI = newSaladAPI
-    saladAPI.clusterNodes.flatMap { allNodes =>
-      val getConnectionsToLiveNodes = allNodes.filter(_.isConnected).map(node => getConnection(node.getNodeId))
-      Future.sequence(getConnectionsToLiveNodes).flatMap { connections =>
-        // Meet every new node from every old node.
-        val metResults = for {
-          conn <- connections
-          uri <- redisURIList
-        } yield {
-          conn.clusterMeet(uri)
-        }
-        Future.sequence(metResults).map(_ => redisURIList)
-      }
-    }
-  }
-
-  /**
-    * Set the n new slave nodes to replicate the poorest (fewest slaves) n masters.
-    *
-    * @param redisURIList The list of ip addresses of the slaves that will be added to the cluster. Hostnames are not acceptable.
-    * @param executionContext The thread dispatcher context.
-    * @return Indicate that the n new slaves are replicating the poorest n masters.
-    */
-  private def findMasters(redisURIList: Seq[RedisURI])(implicit executionContext: ExecutionContext): Future[Unit] = {
-    implicit val saladAPI = newSaladAPI
-    saladAPI.clusterNodes.flatMap { allNodes =>
-      // Node ids for nodes that are currently master nodes but will become slave nodes.
-      val newSlaveIds = allNodes.filter(node => redisURIList.contains(node.getUri)).map(_.getNodeId)
-      // The master nodes (the nodes that will become slaves are still master nodes at this point and must be filtered out).
-      val masterNodes = saladAPI.masterNodes(allNodes)
-        .filterNot(node => newSlaveIds.contains(node.getNodeId))
-      // HashMap of master node ids to the number of slaves for that master.
-      val masterSlaveCount = new util.HashMap[String, AtomicInteger](masterNodes.length + 1, 1)
-      // Populate the hash map.
-      masterNodes.map(_.getNodeId).foreach(nodeId => masterSlaveCount.put(nodeId, new AtomicInteger(0)))
-      allNodes.map { node =>
-        Option.apply(node.getSlaveOf)
-          .map(master => masterSlaveCount.get(master).incrementAndGet())
-      }
-
-      // Find the poorest n masters for n slaves.
-      val poorestMasters = new MaxNHeapMasterSlaveCount(redisURIList.length)
-      masterSlaveCount.asScala.foreach(poorestMasters.offer)
-      assert(redisURIList.length >= poorestMasters.underlying.length)
-
-      // Create a list so that we can circle back to the first element if the new slaves outnumber the existing masters.
-      val poorMasterList = poorestMasters.underlying.toList
-      val poorMasterIndex = new AtomicInteger(0)
-      // Choose a master for every slave.
-      val listFuturesResults = redisURIList.map { slaveURI =>
-        getConnection(slaveURI).map(_.clusterReplicate(
-          poorMasterList(poorMasterIndex.getAndIncrement() % poorMasterList.length)._1))
-      }
-      Future.sequence(listFuturesResults).map(x => x)
-    }
-  }
-
-  /**
-    * Emit a key-value representing the node-type and the node-id.
-    * @param redisURI
-    * @param executionContext
-    * @return the node type and id.
-    */
-  def emitNodeType(redisURI:RedisURI)(implicit executionContext: ExecutionContext): Future[KeyValue] = {
-    implicit val saladAPI = newSaladAPI
-    saladAPI.clusterNodes.map { allNodes =>
-      val removalNodeOpt = allNodes.find(node => node.getUri.equals(redisURI))
-      if (removalNodeOpt.isEmpty) throw new Exception(s"Node not in cluster: $redisURI")
-      val kv = removalNodeOpt.map { node =>
-        node.getRole match {
-          case Role.MASTER => KeyValue(RESHARD.key, node.getNodeId)
-          case Role.SLAVE  => KeyValue(REMOVE_SLAVE.key, node.getNodeId)
-          case _           => KeyValue(UNSUPPORTED.key, node.getNodeId)
-        }
-      }
-      kv.get
-    }
-  }
-
-  /**
-    * Safely remove a master by redistributing its hash slots before blacklisting it from the cluster.
-    * The data is given time to migrate as configured in `cornucopia.grace.period`.
-    *
-    * @param withoutNodes The list of ids of the master nodes that will be removed from the cluster.
-    * @param executionContext The thread dispatcher context.
-    * @return Indicate that the hash slots were redistributed and the master removed from the cluster.
-    */
-  private def removeMasters(withoutNodes: Seq[String])(implicit executionContext: ExecutionContext): Future[Unit] = {
-    val reshardDone = reshardCluster(withoutNodes).map { _ =>
-      scala.concurrent.blocking(Thread.sleep(Config.Cornucopia.gracePeriod)) // Allow data to migrate.
-    }
-    reshardDone.flatMap(_ => forgetNodes(withoutNodes))
-  }
-
-  /**
-    * Notify all nodes in the cluster to forget this node.
-    *
-    * @param withoutNodes The list of ids of nodes to be forgotten by the cluster.
-    * @param executionContext The thread dispatcher context.
-    * @return A future indicating that the node was forgotten by all nodes in the cluster.
-    */
-  def forgetNodes(withoutNodes: Seq[String])(implicit executionContext: ExecutionContext): Future[Unit] =
-  if (!withoutNodes.exists(_.nonEmpty))
-    Future(Unit)
-  else {
-    implicit val saladAPI = newSaladAPI
-    saladAPI.clusterNodes.flatMap { allNodes =>
-      logger.info(s"Forgetting nodes: $withoutNodes")
-      // Reset the nodes to be removed.
-      val validWithoutNodes = withoutNodes.filter(_.nonEmpty)
-      validWithoutNodes.map(getConnection).map(_.map(_.clusterReset(true)))
-
-      // The nodes that will remain in the cluster should forget the nodes that will be removed.
-      val withNodes = allNodes
-        .filterNot(node => validWithoutNodes.contains(node.getNodeId)) // Node cannot forget itself.
-
-      // For the cross product of `withNodes` and `withoutNodes`; to remove the nodes in `withoutNodes`.
-      val forgetResults = for {
-        operatorNode <- withNodes
-        operandNodeId <- validWithoutNodes
-      } yield {
-        if (operatorNode.getSlaveOf == operandNodeId)
-          Future(Unit) // Node cannot forget its master.
-        else
-          getConnection(operatorNode.getNodeId).flatMap(_.clusterForget(operandNodeId))
-      }
-      Future.sequence(forgetResults).map(x => x)
-    }
-  }
-
-  /**
-    * Reshard the cluster using a view of the cluster consisting of a subset of master nodes.
-    *
-    * @param withoutNodes The list of ids of nodes that will not be assigned hash slots.
-    * @return Boolean indicating that all hash slots were reassigned successfully.
-    */
-  private def reshardCluster(withoutNodes: Seq[String])
-  : Future[Unit] = {
-    // Execute futures using a thread pool so we don't run out of memory due to futures.
-    implicit val executionContext = Config.Consumer.actorSystem.dispatchers.lookup("akka.actor.resharding-dispatcher")
-    implicit val saladAPI = newSaladAPI
-    saladAPI.masterNodes.flatMap { masterNodes =>
-
-      val liveMasters = masterNodes.filter(_.isConnected)
-      lazy val idToURI = new util.HashMap[String,RedisURI](liveMasters.length + 1, 1)
-      // Re-use cluster connections so we don't exceed file-handle limit or waste resources.
-      lazy val clusterConnections = new util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]](liveMasters.length + 1, 1)
-      liveMasters.map { master =>
-        idToURI.put(master.getNodeId, master.getUri)
-        clusterConnections.put(master.getNodeId, getConnection(master.getNodeId))
-      }
-
-      // Remove dead nodes. This may generate WARN logs if some nodes already forgot the dead node.
-      val deadMastersIds = masterNodes.filterNot(_.isConnected).map(_.getNodeId)
-      logger.info(s"Dead nodes: $deadMastersIds")
-      forgetNodes(deadMastersIds)
-
-      // Migrate the data.
-      val assignableMasters = liveMasters.filterNot(masterNode => withoutNodes.contains(masterNode.getNodeId))
-      logger.info(s"Resharding cluster with ${assignableMasters.map(_.getNodeId)} without ${withoutNodes ++ deadMastersIds}")
-      val migrateResults = liveMasters.flatMap { node =>
-        val (sourceNodeId, slotList) = (node.getNodeId, node.getSlots.toList.map(_.toInt))
-        logger.debug(s"Migrating data from $sourceNodeId among slots $slotList")
-        slotList.map { slot =>
-          val destinationNodeId = slotNode(slot, assignableMasters)
-          migrateSlot(
-            slot,
-            sourceNodeId, destinationNodeId, idToURI.get(destinationNodeId),
-            assignableMasters, clusterConnections)
-        }
-      }
-      val finalMigrateResult = Future.sequence(migrateResults)
-      finalMigrateResult.onFailure { case e => logger.error(s"Failed to migrate hash slot data", e) }
-      finalMigrateResult.onSuccess { case _ => logger.info(s"Migrated hash slot data") }
-      // We attempted to migrate the data but do not prevent slot reassignment if migration fails.
-      // We may lose prior data but we ensure that all slots are assigned.
-      finalMigrateResult.onComplete { case _ =>
-        List.range(0, 16384).map(notifySlotAssignment(_, assignableMasters))
-      }
-      finalMigrateResult.flatMap(_ => forgetNodes(withoutNodes))
-    }
-  }
-
-  // TODO: pass slotNode as a lambda to migrateSlot and notifySlotAssignment.
-  // TODO: more efficient slot assignment to prevent data migration.
-  /**
-    * Choose a master node for a slot.
-    *
-    * @param slot The slot to be assigned.
-    * @param masters The list of masters that can be assigned slots.
-    * @return The node id of the chosen master.
-    */
-  private def slotNode(slot: Int, masters: mutable.Buffer[RedisClusterNode]): String =
-    masters(slot % masters.length).getNodeId
-
-  /**
-    * Migrate all keys in a slot from the source node to the destination node and update the slot assignment on the
-    * affected nodes.
-    *
-    * @param slot The slot to migrate.
-    * @param sourceNodeId The current location of the slot data.
-    * @param destinationNodeId The target location of the slot data.
-    * @param masters The list of nodes in the cluster that will be assigned hash slots.
-    * @param clusterConnections The list of connections to nodes in the cluster.
-    * @param executionContext The thread dispatcher context.
-    * @return Future indicating success.
-    */
-  private def migrateSlot(slot: Int, sourceNodeId: String, destinationNodeId: String, destinationURI: RedisURI,
-                          masters: mutable.Buffer[RedisClusterNode],
-                          clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]])
-                         (implicit saladAPI: Salad, executionContext: ExecutionContext)
-  : Future[Unit] = {
-    destinationNodeId match {
-      case `sourceNodeId` =>
-        // Don't migrate if the source and destination are the same.
-        Future(Unit)
-      case _ =>
-        for {
-          sourceConnection <- clusterConnections.get(sourceNodeId)
-          destinationConnection <- clusterConnections.get(destinationNodeId)
-        } yield {
-          // Sequentially execute the steps outline in:
-          // https://redis.io/commands/cluster-setslot#redis-cluster-live-resharding-explained
-          import com.github.kliewkliew.salad.serde.ByteArraySerdes._
-          val migrationResult =
-            for {
-              _ <- destinationConnection.clusterSetSlotStable(slot).recover { case _ => Unit }
-              _ <- sourceConnection.clusterSetSlotStable(slot).recover { case _ => Unit }
-              _ <- destinationConnection.clusterSetSlotImporting(slot, sourceNodeId)
-              _ <- sourceConnection.clusterSetSlotMigrating(slot, destinationNodeId)
-              keyCount <- sourceConnection.clusterCountKeysInSlot(slot)
-              keyList <- sourceConnection.clusterGetKeysInSlot[CodecType](slot, keyCount.toInt)
-              _ <- sourceConnection.migrate[CodecType](destinationURI, keyList.toList)
-              _ <- sourceConnection.clusterSetSlotNode(slot, destinationNodeId)
-              finalResult <- destinationConnection.clusterSetSlotNode(slot, destinationNodeId)
-            } yield {
-              finalResult
-            }
-          migrationResult.onSuccess { case _ => logger.trace(s"Migrated data of slot $slot from $sourceNodeId to: $destinationNodeId at $destinationURI") }
-          migrationResult.onFailure { case e => logger.debug(s"Failed to migrate data of slot $slot from $sourceNodeId to: $destinationNodeId at $destinationURI", e)}
-          // Undocumented but necessary final steps found in http://download.redis.io/redis-stable/src/redis-trib.rb
-          // `recover` to perform these steps even if the previous steps failed, but don't perform these steps until the previous steps did attempt execution.
-          val finalMigrationResult = migrationResult.recover { case _ => Unit }
-            .flatMap(_ => notifySlotAssignment(slot, masters)).recover { case _ => Unit }
-            .flatMap(_ => sourceConnection.clusterDelSlot(slot)).recover { case _ => Unit }
-            .flatMap(_ => destinationConnection.clusterAddSlot(slot))
-          finalMigrationResult.onSuccess { case _ => logger.trace(s"Updated slot table for slot $slot") }
-          finalMigrationResult.onFailure { case e => logger.debug(s"Failed to update slot table for slot $slot", e)}
-          finalMigrationResult.map(x => x)
-        }
-    }
-  }
-
-  /**
-    * Notify all master nodes of a slot assignment so that they will immediately be able to redirect clients.
-    *
-    * @param masters The list of nodes in the cluster that will be assigned hash slots.
-    * @param executionContext The thread dispatcher context.
-    * @return Future indicating success.
-    */
-  private def notifySlotAssignment(slot: Int, masters: mutable.Buffer[RedisClusterNode])
-                                  (implicit saladAPI: Salad, executionContext: ExecutionContext)
-  : Future[Unit] = {
-    val getMasterConnections = masters.map(master => getConnection(master.getNodeId))
-    Future.sequence(getMasterConnections).flatMap { masterConnections =>
-      val notifyResults = masterConnections.map(_.clusterSetSlotNode(slot, slotNode(slot, masters)))
-      Future.sequence(notifyResults).map(x => x)
-    }
-  }
-
-  /**
-    * Store the n poorest masters.
-    * Implemented on scala.mutable.PriorityQueue.
-    *
-    * @param n
-    */
-  sealed case class MaxNHeapMasterSlaveCount(n: Int) {
-    private type MSTuple = (String, AtomicInteger)
-    private object MSOrdering extends Ordering[MSTuple] {
-      def compare(a: MSTuple, b: MSTuple) = a._2.intValue compare b._2.intValue
-    }
-    implicit private val ordering = MSOrdering
-    val underlying = new mutable.PriorityQueue[MSTuple]
-
-    /**
-      * O(1) if the entry is not a candidate for the being one of the poorest n masters.
-      * O(log(n)) if the entry is a candidate.
-      *
-      * @param entry The candidate master-slavecount tuple.
-      */
-    def offer(entry: MSTuple) =
-      if (n > underlying.length) {
-        underlying.enqueue(entry)
-      }
-      else if (entry._2.intValue < underlying.head._2.intValue) {
-        underlying.dequeue()
-        underlying.enqueue(entry)
-      }
-  }
-
-}
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index c9f9c80..eba9740 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -20,7 +20,7 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
   with WordSpecLike with BeforeAndAfterAll with MustMatchers with MockitoSugar {
 
   trait FakeCornucopiaActorSourceGraph {
-    import com.github.kliewkliew.cornucopia.kafka.Config.Consumer.materializer
+    import Config.Consumer.materializer
 
     class CornucopiaActorSourceLocal extends CornucopiaActorSource {
       lazy val probe = TestProbe()

From afd0c9da127df16850045d0679468cfe0e62b64e Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Fri, 5 May 2017 17:52:28 -0700
Subject: [PATCH 11/44] Pass down the sender ActorRef through the graph so we
 can respond after mapping future on cluster reshard

---
 .../kliewkliew/cornucopia/Library.scala       |   4 +-
 .../cornucopia/actors/CornucopiaSource.scala  |  15 ++-
 .../kliewkliew/cornucopia/graph/Graph.scala   | 103 +++++++++++++++---
 .../kliewkliew/cornucopia/LibraryTest.scala   |  56 ++++++----
 4 files changed, 132 insertions(+), 46 deletions(-)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala b/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
index d2a31b9..f3c2588 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
@@ -1,9 +1,11 @@
 package com.github.kliewkliew.cornucopia
-import actors.CornucopiaSource
 
+import actors.CornucopiaSource
 import akka.actor._
+import redis.Connection.{newSaladAPI, Salad}
 
 object Library {
+  implicit val newSaladAPIimpl: Salad = newSaladAPI
   val ref: ActorRef = new graph.CornucopiaActorSource().ref
   val source = CornucopiaSource
 }
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/actors/CornucopiaSource.scala b/src/main/scala/com/github/kliewkliew/cornucopia/actors/CornucopiaSource.scala
index d39d475..544f3e9 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/actors/CornucopiaSource.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/actors/CornucopiaSource.scala
@@ -12,7 +12,7 @@ import scala.annotation.tailrec
 object CornucopiaSource {
   def props: Props = Props[CornucopiaSource]
 
-  final case class Task(operation: String, redisNodeIp: String)
+  final case class Task(operation: String, redisNodeIp: String, ref: Option[ActorRef] = None)
   case object TaskAccepted
   case object TaskDenied
 }
@@ -28,10 +28,13 @@ class CornucopiaSource extends ActorPublisher[CornucopiaSource.Task] {
     case _: Task if buf.size == MaxBufferSize =>
       sender() ! TaskDenied
     case task: Task =>
-      sender() ! TaskAccepted
-      if (buf.isEmpty && totalDemand > 0) onNext(task)
+      if (buf.isEmpty && totalDemand > 0) {
+        val task2 = task.copy(ref = Some(sender))
+        onNext(task2)
+      }
       else {
-        buf :+= task
+        val task2 = task.copy(ref = Some(sender))
+        buf :+= task2
         deliverBuf()
       }
     case Request(_) =>
@@ -45,11 +48,11 @@ class CornucopiaSource extends ActorPublisher[CornucopiaSource.Task] {
       if (totalDemand <= Int.MaxValue) {
         val (use, keep) = buf.splitAt(totalDemand.toInt)
         buf = keep
-        use foreach onNext
+        use foreach(task => onNext(task))
       } else {
         val (use, keep) = buf.splitAt(Int.MaxValue)
         buf = keep
-        use foreach onNext
+        use foreach(task => onNext(task))
         deliverBuf()
       }
     }
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 5c28943..6c4d2fc 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -19,7 +19,7 @@ import scala.collection.mutable
 import scala.collection.JavaConversions._
 import scala.language.implicitConversions
 import scala.concurrent.{ExecutionContext, Future}
-import akka.stream.scaladsl.{Flow, GraphDSL, Merge, MergePreferred, Partition, RunnableGraph, Sink}
+import akka.stream.scaladsl.{Flow, GraphDSL, Merge, MergePreferred, Partition, RunnableGraph, Sink, Broadcast}
 import com.github.kliewkliew.cornucopia.Config
 // TO-DO: put config someplace else
 
@@ -46,7 +46,7 @@ trait CornucopiaGraph {
     * Stream definitions for the graph.
     */
   // Extract a tuple of the key and value from a Kafka record.
-  case class KeyValue(key: String, value: String)
+  case class KeyValue(key: String, value: String, senderRef: Option[ActorRef] = None)
 
   // Add a master node to the cluster.
   def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
@@ -69,6 +69,7 @@ trait CornucopiaGraph {
     .mapAsync(1)(findMasters)
     .mapAsync(1)(waitForTopologyRefresh[Unit])
     .mapAsync(1)(_ => logTopology)
+    .map(_ => KeyValue("", ""))
 
   // Emit a key-value pair indicating the node type and URI.
   protected def streamRemoveNode(implicit executionContext: ExecutionContext) = Flow[KeyValue]
@@ -84,6 +85,7 @@ trait CornucopiaGraph {
     .mapAsync(1)(forgetNodes)
     .mapAsync(1)(waitForTopologyRefresh[Unit])
     .mapAsync(1)(_ => logTopology)
+    .map(_ => KeyValue("", ""))
 
   // Redistribute the hash slots among all nodes in the cluster.
   // Execute slot redistribution at most once per configured interval.
@@ -95,6 +97,7 @@ trait CornucopiaGraph {
     .mapAsync(1)(reshardCluster)
     .mapAsync(1)(waitForTopologyRefresh[Unit])
     .mapAsync(1)(_ => logTopology)
+    .map(_ => KeyValue("", ""))
 
   // Throw for keys indicating unsupported operations.
   protected def unsupportedOperation = Flow[KeyValue]
@@ -114,6 +117,22 @@ trait CornucopiaGraph {
     passthrough
   }
 
+  /**
+    * Wait for the new cluster topology view to propagate to all nodes in the cluster. Same version as above, but this
+    * time takes two passthroughs and returns tuple of them as future.
+    *
+    * @param passthrough1 The first value that will be passed through to the next map stage.
+    * @param passthrough2 The second value that will be passed through to the next map stage.
+    * @param executionContext The thread dispatcher context.
+    * @tparam T
+    * @tparam U
+    * @return The unmodified input value.
+    */
+  protected def waitForTopologyRefresh2[T, U](passthrough1: T, passthrough2: U)(implicit executionContext: ExecutionContext): Future[(T, U)] = Future {
+    scala.concurrent.blocking(Thread.sleep(Config.Cornucopia.refreshTimeout))
+    (passthrough1, passthrough2)
+  }
+
   /**
     * Log the current view of the cluster topology.
     *
@@ -269,9 +288,10 @@ trait CornucopiaGraph {
     }
 
   /**
-    * Reshard the cluster using a view of the cluster consisting of a subset of master nodes.
+    * Reshard the cluster using a view of thecluster consisting of a subset of master nodes.
     *
-    * @param withoutNodes The list of ids of nodes that will not be assigned hash slots.
+    * @param withoutNodes The list of ids of nodes that will not be assigned hash slots. Note that this is not used
+    *                     (it is empty) when we add a new master node and reshard.
     * @return Boolean indicating that all hash slots were reassigned successfully.
     */
   protected def reshardCluster(withoutNodes: Seq[String])
@@ -442,6 +462,8 @@ trait CornucopiaGraph {
 class CornucopiaKafkaSource extends CornucopiaGraph {
   import Config.Consumer.materializer
 
+  implicit val newSaladAPIimpl: Salad = newSaladAPI
+
   private type KafkaRecord = ConsumerRecord[String, String]
 
   private def extractKeyValue = Flow[KafkaRecord]
@@ -482,15 +504,57 @@ class CornucopiaKafkaSource extends CornucopiaGraph {
 
 }
 
-class CornucopiaActorSource extends CornucopiaGraph {
+class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaGraph {
   import Config.Consumer.materializer
   import com.github.kliewkliew.cornucopia.actors.CornucopiaSource.Task
   import scala.concurrent.ExecutionContext.Implicits.global
 
   protected type ActorRecord = Task
 
+  // Add a master node to the cluster.
+  def streamAddMasterPrime(implicit executionContext: ExecutionContext, newSaladAPIimpl: Connection.Salad) = Flow[KeyValue]
+    .map(kv => (kv.value, kv.senderRef))
+    .map(t => (RedisURI.create(t._1), t._2) )
+    .map(t => (newSaladAPIimpl.canonicalizeURI(t._1), t._2))
+    .groupedWithin(1, Config.Cornucopia.batchPeriod)
+    .mapAsync(1)(t => {
+      val t1 = t.unzip
+      val redisURIs = t1._1
+      val actorRefs = t1._2
+      waitForTopologyRefresh2[Seq[RedisURI], Seq[Option[ActorRef]]](redisURIs, actorRefs)
+    })
+    .map{ case (_, actorRef) =>
+      val ref = actorRef.head
+      KeyValue(RESHARD.key, "", ref)
+    }
+
+  override protected def streamReshard(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+    .map(record => Seq( record.senderRef ))
+    .conflate((seq1, seq2) => seq1 ++ seq2 )
+    .throttle(1, Config.Cornucopia.minReshardWait, 1, ThrottleMode.Shaping)
+    .mapAsync(1)(reshardClusterPrime)
+    .mapAsync(1)(waitForTopologyRefresh[Unit])
+    .mapAsync(1)(_ => logTopology)
+    .map(_ => KeyValue("", ""))
+
+  private def reshardClusterPrime(senders: Seq[Option[ActorRef]]): Future[Unit] = {
+
+    def reshard(ref: ActorRef): Future[Unit] = {
+      reshardCluster(Seq()) map { _: Unit =>
+        ref ! "OK"
+      } recover {
+        case ex: Throwable => ref ! s"ERROR: ${ex.toString}"
+      }
+    }
+
+    val flattened = senders.flatten
+
+    if (flattened.size == 0) Future(Unit)
+    else Future.reduce(senders.flatten.map(reshard))((_, _) => Unit)
+  }
+
   protected def extractKeyValue = Flow[ActorRecord]
-    .map[KeyValue](record => KeyValue(record.operation, record.redisNodeIp))
+    .map[KeyValue](record => KeyValue(record.operation, record.redisNodeIp, record.ref))
 
   protected val processTask = Flow.fromGraph(GraphDSL.create() { implicit builder =>
     import GraphDSL.Implicits._
@@ -508,21 +572,28 @@ class CornucopiaActorSource extends CornucopiaGraph {
       3, kv => partitionNodeRemoval(kv.key)
     ))
 
-    val fanOut = builder.add(Merge[Any](5))
+    val broadcastSender = builder.add(Broadcast[KeyValue](2))
 
-    taskSource.out                          ~> kv
-    kv                                      ~> mergeFeedback.preferred
+    val senderMerge = builder.add(Merge[KeyValue](2))
+
+    val fanIn = builder.add(Merge[KeyValue](5))
+
+    taskSource.out                          ~> kv                   ~> broadcastSender
+    broadcastSender                         ~> mergeFeedback.preferred
+    broadcastSender                         ~> senderMerge
     mergeFeedback.out                       ~> partition
-    partition.out(ADD_MASTER.ordinal)       ~> streamAddMaster      ~> mergeFeedback.in(0)
-    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave       ~> fanOut.in(0)
+    partition.out(ADD_MASTER.ordinal)       ~> streamAddMasterPrime ~> mergeFeedback.in(0)
+    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave       ~> fanIn
     partition.out(REMOVE_NODE.ordinal)      ~> streamRemoveNode     ~> partitionRm
     partitionRm.out(REMOVE_MASTER.ordinal)  ~> mergeFeedback.in(1)
-    partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave    ~> fanOut.in(1)
-    partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation ~> fanOut.in(2)
-    partition.out(RESHARD.ordinal)          ~> streamReshard        ~> fanOut.in(3)
-    partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation ~> fanOut.in(4)
+    partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave    ~> fanIn
+    partitionRm.out(UNSUPPORTED.ordinal)    ~> unsupportedOperation ~> fanIn
+    partition.out(RESHARD.ordinal)          ~> streamReshard        ~> fanIn
+    partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation ~> fanIn
+
+    fanIn ~> senderMerge
 
-    FlowShape(taskSource.in, fanOut.out)
+    FlowShape(taskSource.in, senderMerge.out)
   })
 
   protected val cornucopiaSource = Config.Consumer.cornucopiaActorSource
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index eba9740..2a70bbb 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -1,20 +1,28 @@
 package com.github.kliewkliew.cornucopia
 
+import com.github.kliewkliew.cornucopia.redis._
+import com.lambdaworks.redis.RedisURI
 //import com.github.kliewkliew.cornucopia._
 import com.github.kliewkliew.cornucopia.graph._
+import com.github.kliewkliew.cornucopia.redis.Connection.Salad
 import akka.testkit.{TestActorRef, TestKit, TestProbe}
 import akka.actor.{ActorSystem, ActorRef}
-import akka.actor.Status.Failure
+import akka.pattern.ask
+//import akka.actor.Status.Failure
 import akka.stream.ActorMaterializer
+import akka.util.Timeout
 import org.scalatest.mockito.MockitoSugar
 import org.scalatest.{BeforeAndAfterAll, MustMatchers, WordSpecLike}
 import org.mockito.Mockito._
 import org.scalatest.mockito.MockitoSugar._
+import org.mockito.Matchers._
 import com.github.kliewkliew.cornucopia.graph._
 import akka.stream.scaladsl.Flow
 import akka.stream.scaladsl.Sink
 import scala.concurrent.duration._
+import scala.concurrent.Await
 import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{ Success, Failure }
 
 class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
   with WordSpecLike with BeforeAndAfterAll with MustMatchers with MockitoSugar {
@@ -22,18 +30,16 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
   trait FakeCornucopiaActorSourceGraph {
     import Config.Consumer.materializer
 
-    class CornucopiaActorSourceLocal extends CornucopiaActorSource {
-      lazy val probe = TestProbe()
+    // hostname IP address must be semantically correct, java.net actually checks for RFC conformance
+    val redisUri = "redis://192.168.0.1"
+
+    val fakeSalad = mock[Salad]
+    when(fakeSalad.canonicalizeURI(anyObject())).thenReturn(RedisURI.create(redisUri))
 
-      override def streamAddMaster(implicit executionContext: ExecutionContext) =
-        Flow[KeyValue].map(kv => {
-          val ip = kv.value
-          probe.ref ! "streamAddMaster"
-          Thread.sleep(300)
-          probe.ref ! ip
-          Thread.sleep(300)
-          KeyValue("*reshard", "")
-        })
+    implicit val newSaladAPIimpl = fakeSalad
+
+    class CornucopiaActorSourceLocal(implicit newSaladAPIimpl: Salad) extends CornucopiaActorSource {
+      lazy val probe = TestProbe()
 
       override def streamAddSlave(implicit executionContext: ExecutionContext) =
         Flow[KeyValue].map(_ => KeyValue("test", ""))
@@ -44,11 +50,13 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
       override def streamRemoveSlave(implicit executionContext: ExecutionContext) =
         Flow[KeyValue].map(_ => KeyValue("test", ""))
 
-      override def streamReshard(implicit executionContext: ExecutionContext) =
-        Flow[KeyValue].map(_ => {
-          probe.ref ! "streamReshard"
-          KeyValue("*reshard", "")
-        })
+      override protected def waitForTopologyRefresh2[T, U](passthrough1: T, passthrough2: U)
+                                                 (implicit executionContext: ExecutionContext): Future[(T, U)] = Future {
+        (passthrough1, passthrough2)
+      }
+
+      override protected def reshardCluster(withoutNodes: Seq[String]): Future[Unit] = Future(Unit)
+
     }
 
   }
@@ -67,15 +75,17 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
 
       private val ref = cornucopiaActorSourceLocal.ref
 
-      private val redisUri = "redis://123.456.789.10"
-
-      ref ! Task("+master", redisUri)
+      implicit val timeout = Timeout(5 seconds)
 
-      cornucopiaActorSourceLocal.probe.expectMsg(100 millis, "streamAddMaster")
+      val future = ask(ref, Task("+master", redisUri))
 
-      cornucopiaActorSourceLocal.probe.expectMsg(350 millis, redisUri)
+      future.onComplete {
+        case Failure(_) => assert(false)
+        case Success(msg) =>
+          assert(msg == "OK")
+      }
 
-      cornucopiaActorSourceLocal.probe.expectMsg(700 millis, "streamReshard")
+      Await.ready(future, timeout.duration)
     }
   }
 

From 2c8c665b958b7a5e32be032b2b520c4227d1995f Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 7 May 2017 14:25:10 -0700
Subject: [PATCH 12/44] Version 0.7-SNAPSHOT

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 7a8462d..6fb2220 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.6-SNAPSHOT"
+version := "0.7-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From 62d4de45823312e3e253e287d23b1532f21feb38 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 7 May 2017 21:34:14 -0700
Subject: [PATCH 13/44] Stub logTopology in test

---
 .../scala/com/github/kliewkliew/cornucopia/LibraryTest.scala    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index 2a70bbb..74e3743 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -55,6 +55,8 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
         (passthrough1, passthrough2)
       }
 
+      override protected def logTopology(implicit executionContext: ExecutionContext): Future[Unit] = Future(Unit)
+
       override protected def reshardCluster(withoutNodes: Seq[String]): Future[Unit] = Future(Unit)
 
     }

From 85a347aa989e2da81b9e540ddf0caceb605b6ee3 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Mon, 8 May 2017 17:27:58 -0700
Subject: [PATCH 14/44] Pass back an Either from reshard cluster

---
 .../scala/com/github/kliewkliew/cornucopia/graph/Graph.scala  | 4 ++--
 .../scala/com/github/kliewkliew/cornucopia/LibraryTest.scala  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 6c4d2fc..f3c61a4 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -541,9 +541,9 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
 
     def reshard(ref: ActorRef): Future[Unit] = {
       reshardCluster(Seq()) map { _: Unit =>
-        ref ! "OK"
+        ref ! Right("OK")
       } recover {
-        case ex: Throwable => ref ! s"ERROR: ${ex.toString}"
+        case ex: Throwable => ref ! Left(s"ERROR: ${ex.toString}")
       }
     }
 
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index 74e3743..90807e9 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -84,7 +84,7 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
       future.onComplete {
         case Failure(_) => assert(false)
         case Success(msg) =>
-          assert(msg == "OK")
+          assert(msg == Right("OK"))
       }
 
       Await.ready(future, timeout.duration)

From 32ab91fd3e762897a0ab6f410c0f504e8461ef1b Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 11 May 2017 13:26:00 -0700
Subject: [PATCH 15/44] We need to add the new master node to the cluster

---
 build.sbt                                                    | 2 +-
 .../scala/com/github/kliewkliew/cornucopia/graph/Graph.scala | 4 +++-
 .../scala/com/github/kliewkliew/cornucopia/LibraryTest.scala | 5 +++++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index 6fb2220..b9427ac 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.7-SNAPSHOT"
+version := "0.8-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index f3c61a4..958a0cd 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -521,7 +521,9 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
       val t1 = t.unzip
       val redisURIs = t1._1
       val actorRefs = t1._2
-      waitForTopologyRefresh2[Seq[RedisURI], Seq[Option[ActorRef]]](redisURIs, actorRefs)
+      addNodesToCluster(redisURIs) flatMap { uris =>
+        waitForTopologyRefresh2[Seq[RedisURI], Seq[Option[ActorRef]]](uris, actorRefs)
+      }
     })
     .map{ case (_, actorRef) =>
       val ref = actorRef.head
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index 90807e9..6f5127d 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -59,6 +59,11 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
 
       override protected def reshardCluster(withoutNodes: Seq[String]): Future[Unit] = Future(Unit)
 
+      override protected def addNodesToCluster(redisURIList: Seq[RedisURI])
+                                              (implicit executionContext: ExecutionContext): Future[Seq[RedisURI]] = {
+        Future(redisURIList)
+      }
+
     }
 
   }

From f1c8718ffa66d01eff37882b80b4dea8c9348383 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 11 May 2017 15:39:51 -0700
Subject: [PATCH 16/44] Send message to sender actor after adding a slave

---
 .../kliewkliew/cornucopia/graph/Graph.scala   | 42 +++++++++++++++++--
 .../kliewkliew/cornucopia/LibraryTest.scala   | 30 +++++++++++++
 2 files changed, 69 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 958a0cd..3fa19c7 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -112,7 +112,7 @@ trait CornucopiaGraph {
     * @tparam T
     * @return The unmodified input value.
     */
-  protected def waitForTopologyRefresh[T](passthrough: T)(implicit executionContext: ExecutionContext): Future[T] = Future {
+ protected def waitForTopologyRefresh[T](passthrough: T)(implicit executionContext: ExecutionContext): Future[T] = Future {
     scala.concurrent.blocking(Thread.sleep(Config.Cornucopia.refreshTimeout))
     passthrough
   }
@@ -530,6 +530,42 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
       KeyValue(RESHARD.key, "", ref)
     }
 
+  // Add a slave node to the cluster, replicating the master that has the fewest slaves.
+  def streamAddSlavePrime(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+    .map(kv => (kv.value, kv.senderRef))
+    .map(t => (RedisURI.create(t._1), t._2) )
+    .map(t => (newSaladAPIimpl.canonicalizeURI(t._1), t._2))
+    .groupedWithin(1, Config.Cornucopia.batchPeriod)
+    .mapAsync(1)(t => {
+      val t1 = t.unzip
+      val redisURIs = t1._1
+      val actorRefs = t1._2
+      addNodesToCluster(redisURIs) flatMap { uris =>
+        waitForTopologyRefresh2[Seq[RedisURI], Seq[Option[ActorRef]]](uris, actorRefs)
+      }
+    })
+    .mapAsync(1)(t => {
+      val redisURIs = t._1
+      val actorRefs = t._2
+      findMasters(redisURIs) map { _ =>
+        actorRefs
+      }
+    })
+    .mapAsync(1)(waitForTopologyRefresh[Seq[Option[ActorRef]]])
+    .mapAsync(1)(signalSlavesAdded)
+    .map(_ => KeyValue("", ""))
+
+  private def signalSlavesAdded(senders: Seq[Option[ActorRef]]): Future[Unit] = {
+    def signal(ref: ActorRef): Future[Unit] = {
+      Future {
+        ref ! Right("OK")
+      }
+    }
+    val flattened = senders.flatten
+    if (flattened.isEmpty) Future(Unit)
+    else Future.reduce(senders.flatten.map(signal))((_, _) => Unit)
+  }
+
   override protected def streamReshard(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(record => Seq( record.senderRef ))
     .conflate((seq1, seq2) => seq1 ++ seq2 )
@@ -551,7 +587,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
 
     val flattened = senders.flatten
 
-    if (flattened.size == 0) Future(Unit)
+    if (flattened.isEmpty) Future(Unit)
     else Future.reduce(senders.flatten.map(reshard))((_, _) => Unit)
   }
 
@@ -585,7 +621,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
     broadcastSender                         ~> senderMerge
     mergeFeedback.out                       ~> partition
     partition.out(ADD_MASTER.ordinal)       ~> streamAddMasterPrime ~> mergeFeedback.in(0)
-    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlave       ~> fanIn
+    partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlavePrime  ~> fanIn
     partition.out(REMOVE_NODE.ordinal)      ~> streamRemoveNode     ~> partitionRm
     partitionRm.out(REMOVE_MASTER.ordinal)  ~> mergeFeedback.in(1)
     partitionRm.out(REMOVE_SLAVE.ordinal)   ~> streamRemoveSlave    ~> fanIn
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index 6f5127d..78655e9 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -50,6 +50,11 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
       override def streamRemoveSlave(implicit executionContext: ExecutionContext) =
         Flow[KeyValue].map(_ => KeyValue("test", ""))
 
+      override protected def waitForTopologyRefresh[T](passthrough: T)
+                                                      (implicit executionContext: ExecutionContext): Future[T] = Future {
+        passthrough
+      }
+
       override protected def waitForTopologyRefresh2[T, U](passthrough1: T, passthrough2: U)
                                                  (implicit executionContext: ExecutionContext): Future[(T, U)] = Future {
         (passthrough1, passthrough2)
@@ -64,6 +69,9 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
         Future(redisURIList)
       }
 
+      override protected def findMasters(redisURIList: Seq[RedisURI])
+                                        (implicit executionContext: ExecutionContext): Future[Unit] = Future(Unit)
+
     }
 
   }
@@ -96,4 +104,26 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
     }
   }
 
+  "Add slave" must {
+    "add new slave and find masters" in new FakeCornucopiaActorSourceGraph {
+      import Library.source._
+
+      val cornucopiaActorSourceLocal = new CornucopiaActorSourceLocal
+
+      private val ref = cornucopiaActorSourceLocal.ref
+
+      implicit val timeout = Timeout(5 seconds)
+
+      val future = ask(ref, Task("+slave", redisUri))
+
+      future.onComplete {
+        case Failure(_) => assert(false)
+        case Success(msg) =>
+          assert(msg == Right("OK"))
+      }
+
+      Await.ready(future, timeout.duration)
+    }
+  }
+
 }

From 2d59d4d9d0aeae3daa6d6435466d01d62d81d47b Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 11 May 2017 15:40:25 -0700
Subject: [PATCH 17/44] Version 0.9-SNAPSHOT

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index b9427ac..474ae0a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.8-SNAPSHOT"
+version := "0.9-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From 749865fcc4a29d74679425f80bf5bbd77ddcde10 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Fri, 12 May 2017 14:42:11 -0700
Subject: [PATCH 18/44] Add more logging

---
 build.sbt                                                  | 2 +-
 .../com/github/kliewkliew/cornucopia/graph/Graph.scala     | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/build.sbt b/build.sbt
index 474ae0a..bfa2890 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.9-SNAPSHOT"
+version := "0.10-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 3fa19c7..c2ad257 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -26,7 +26,7 @@ import com.github.kliewkliew.cornucopia.Config
 trait CornucopiaGraph {
   import scala.concurrent.ExecutionContext.Implicits.global
 
-  private val logger = LoggerFactory.getLogger(this.getClass)
+  protected val logger = LoggerFactory.getLogger(this.getClass)
 
   def partitionEvents(key: String) = key.trim.toLowerCase match {
     case ADD_MASTER.key     => ADD_MASTER.ordinal
@@ -579,9 +579,12 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
 
     def reshard(ref: ActorRef): Future[Unit] = {
       reshardCluster(Seq()) map { _: Unit =>
+        logger.info("Successfully resharded cluster, informing Kubernetes controller")
         ref ! Right("OK")
       } recover {
-        case ex: Throwable => ref ! Left(s"ERROR: ${ex.toString}")
+        case ex: Throwable =>
+          logger.error("Failed to reshard cluster, informing Kubernetes controller")
+          ref ! Left(s"ERROR: ${ex.toString}")
       }
     }
 

From 9e9b9d7ab2c6ca0dfa718ae3986fba18c4efc842 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Fri, 12 May 2017 14:59:49 -0700
Subject: [PATCH 19/44] Send the node type back to client

---
 build.sbt                                                     | 2 +-
 .../scala/com/github/kliewkliew/cornucopia/graph/Graph.scala  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build.sbt b/build.sbt
index bfa2890..166d1bf 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.10-SNAPSHOT"
+version := "0.11-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index c2ad257..4856408 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -558,7 +558,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
   private def signalSlavesAdded(senders: Seq[Option[ActorRef]]): Future[Unit] = {
     def signal(ref: ActorRef): Future[Unit] = {
       Future {
-        ref ! Right("OK")
+        ref ! Right("slave")
       }
     }
     val flattened = senders.flatten
@@ -580,7 +580,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
     def reshard(ref: ActorRef): Future[Unit] = {
       reshardCluster(Seq()) map { _: Unit =>
         logger.info("Successfully resharded cluster, informing Kubernetes controller")
-        ref ! Right("OK")
+        ref ! Right("master")
       } recover {
         case ex: Throwable =>
           logger.error("Failed to reshard cluster, informing Kubernetes controller")

From f52d9be433efec4c140e26062230582e2cbe328c Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 03:26:55 -0700
Subject: [PATCH 20/44] Refactor the migrateSlot function

---
 build.sbt                                     |  2 +-
 .../kliewkliew/cornucopia/graph/Graph.scala   | 89 ++++++++++++-------
 2 files changed, 56 insertions(+), 35 deletions(-)

diff --git a/build.sbt b/build.sbt
index 166d1bf..e6684ca 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.11-SNAPSHOT"
+version := "0.12-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 4856408..b93940a 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -366,10 +366,58 @@ trait CornucopiaGraph {
     * @return Future indicating success.
     */
   protected def migrateSlot(slot: Int, sourceNodeId: String, destinationNodeId: String, destinationURI: RedisURI,
-                          masters: mutable.Buffer[RedisClusterNode],
-                          clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]])
-                         (implicit saladAPI: Salad, executionContext: ExecutionContext)
-  : Future[Unit] = {
+                            masters: mutable.Buffer[RedisClusterNode],
+                            clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]])
+                           (implicit saladAPI: Salad, executionContext: ExecutionContext): Future[Unit] = {
+
+    // Follows redis-trib.rb
+    def migrateSlotKeys(sourceConn: SaladClusterAPI[CodecType, CodecType],
+                        destinationConn: SaladClusterAPI[CodecType, CodecType]): Future[Unit] = {
+
+      import com.github.kliewkliew.salad.serde.ByteArraySerdes._
+
+      // get all the keys in the given slot
+      val keyList = for {
+        keyCount <- sourceConn.clusterCountKeysInSlot(slot)
+        keyList <- sourceConn.clusterGetKeysInSlot[CodecType](slot, keyCount.toInt)
+      } yield keyList
+
+      // migrate over all the keys in the slot from source to destination node
+      val migrate = for {
+        keys <- keyList
+        result <- sourceConn.migrate[CodecType](destinationURI, keys.toList)
+      } yield result
+
+      def migrateReplace: Future[Unit] = for {
+        keys <- keyList
+        result <- sourceConn.migrate[CodecType](destinationURI, keys.toList, replace = true)
+      } yield result
+
+      migrate.recover {
+        case e =>
+          "BUSYKEY".r.findFirstIn(e.toString) match { // handle the case of a BUSYKEY error
+            case Some(_) =>
+              logger.warn("Problem Migrating Slot: Target key exists. Replacing it for FIX.")
+              migrateReplace
+            case _ =>
+              logger.error(s"Failed to migrate data of slot $slot from $sourceNodeId to: $destinationNodeId at $destinationURI", e)
+              Future(Unit)
+          }
+      }
+
+      migrate
+    }
+
+    def setSlotAssignment(sourceConn: SaladClusterAPI[CodecType, CodecType],
+                        destinationConn: SaladClusterAPI[CodecType, CodecType]): Future[Unit] = {
+      for {
+        _ <- destinationConn.clusterSetSlotImporting(slot, sourceNodeId)
+        _ <- sourceConn.clusterSetSlotMigrating(slot, destinationNodeId)
+      } yield {
+        Future(Unit)
+      }
+    }
+
     destinationNodeId match {
       case `sourceNodeId` =>
         // Don't migrate if the source and destination are the same.
@@ -378,36 +426,9 @@ trait CornucopiaGraph {
         for {
           sourceConnection <- clusterConnections.get(sourceNodeId)
           destinationConnection <- clusterConnections.get(destinationNodeId)
-        } yield {
-          // Sequentially execute the steps outline in:
-          // https://redis.io/commands/cluster-setslot#redis-cluster-live-resharding-explained
-          import com.github.kliewkliew.salad.serde.ByteArraySerdes._
-          val migrationResult =
-            for {
-              _ <- destinationConnection.clusterSetSlotStable(slot).recover { case _ => Unit }
-              _ <- sourceConnection.clusterSetSlotStable(slot).recover { case _ => Unit }
-              _ <- destinationConnection.clusterSetSlotImporting(slot, sourceNodeId)
-              _ <- sourceConnection.clusterSetSlotMigrating(slot, destinationNodeId)
-              keyCount <- sourceConnection.clusterCountKeysInSlot(slot)
-              keyList <- sourceConnection.clusterGetKeysInSlot[CodecType](slot, keyCount.toInt)
-              _ <- sourceConnection.migrate[CodecType](destinationURI, keyList.toList)
-              _ <- sourceConnection.clusterSetSlotNode(slot, destinationNodeId)
-              finalResult <- destinationConnection.clusterSetSlotNode(slot, destinationNodeId)
-            } yield {
-              finalResult
-            }
-          migrationResult.onSuccess { case _ => logger.trace(s"Migrated data of slot $slot from $sourceNodeId to: $destinationNodeId at $destinationURI") }
-          migrationResult.onFailure { case e => logger.debug(s"Failed to migrate data of slot $slot from $sourceNodeId to: $destinationNodeId at $destinationURI", e)}
-          // Undocumented but necessary final steps found in http://download.redis.io/redis-stable/src/redis-trib.rb
-          // `recover` to perform these steps even if the previous steps failed, but don't perform these steps until the previous steps did attempt execution.
-          val finalMigrationResult = migrationResult.recover { case _ => Unit }
-            .flatMap(_ => notifySlotAssignment(slot, masters)).recover { case _ => Unit }
-            .flatMap(_ => sourceConnection.clusterDelSlot(slot)).recover { case _ => Unit }
-            .flatMap(_ => destinationConnection.clusterAddSlot(slot))
-          finalMigrationResult.onSuccess { case _ => logger.trace(s"Updated slot table for slot $slot") }
-          finalMigrationResult.onFailure { case e => logger.debug(s"Failed to update slot table for slot $slot", e)}
-          finalMigrationResult.map(x => x)
-        }
+          _ <- setSlotAssignment(sourceConnection, destinationConnection)
+          _ <- migrateSlotKeys(sourceConnection, destinationConnection)
+        } yield notifySlotAssignment(slot, masters)
     }
   }
 

From 3dc2e86a28fcb7201d9f93a4d7e42578b7e1f570 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 18:31:26 -0700
Subject: [PATCH 21/44] Rework resharding function

Instead of assigning hash slots based on a hashing function, we calculate the number and range of hash slots to move to a new master node from existing master nodes.
---
 build.sbt                                     |   2 +-
 .../kliewkliew/cornucopia/graph/Graph.scala   | 144 +++++++++++++++---
 .../kliewkliew/cornucopia/LibraryTest.scala   |  10 +-
 3 files changed, 134 insertions(+), 22 deletions(-)

diff --git a/build.sbt b/build.sbt
index e6684ca..1082e0a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.12-SNAPSHOT"
+version := "0.13-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index b93940a..1269c6c 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -46,7 +46,7 @@ trait CornucopiaGraph {
     * Stream definitions for the graph.
     */
   // Extract a tuple of the key and value from a Kafka record.
-  case class KeyValue(key: String, value: String, senderRef: Option[ActorRef] = None)
+  case class KeyValue(key: String, value: String, senderRef: Option[ActorRef] = None, newMasterURI: Option[RedisURI] = None)
 
   // Add a master node to the cluster.
   def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
@@ -287,6 +287,7 @@ trait CornucopiaGraph {
       }
     }
 
+
   /**
     * Reshard the cluster using a view of thecluster consisting of a subset of master nodes.
     *
@@ -326,7 +327,7 @@ trait CornucopiaGraph {
           migrateSlot(
             slot,
             sourceNodeId, destinationNodeId, idToURI.get(destinationNodeId),
-            assignableMasters, clusterConnections)
+            assignableMasters.toList, clusterConnections)
         }
       }
       val finalMigrateResult = Future.sequence(migrateResults)
@@ -335,12 +336,15 @@ trait CornucopiaGraph {
       // We attempted to migrate the data but do not prevent slot reassignment if migration fails.
       // We may lose prior data but we ensure that all slots are assigned.
       finalMigrateResult.onComplete { case _ =>
-        List.range(0, 16384).map(notifySlotAssignment(_, assignableMasters))
+        // This is broken anyways
+        //List.range(0, 16384).map(notifySlotAssignment(_, assignableMasters))
+        Unit
       }
       finalMigrateResult.flatMap(_ => forgetNodes(withoutNodes))
     }
   }
 
+  // TODO: Put this method out of its misery
   // TODO: pass slotNode as a lambda to migrateSlot and notifySlotAssignment.
   // TODO: more efficient slot assignment to prevent data migration.
   /**
@@ -366,7 +370,7 @@ trait CornucopiaGraph {
     * @return Future indicating success.
     */
   protected def migrateSlot(slot: Int, sourceNodeId: String, destinationNodeId: String, destinationURI: RedisURI,
-                            masters: mutable.Buffer[RedisClusterNode],
+                            masters: List[RedisClusterNode],
                             clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]])
                            (implicit saladAPI: Salad, executionContext: ExecutionContext): Future[Unit] = {
 
@@ -400,11 +404,19 @@ trait CornucopiaGraph {
               logger.warn("Problem Migrating Slot: Target key exists. Replacing it for FIX.")
               migrateReplace
             case _ =>
-              logger.error(s"Failed to migrate data of slot $slot from $sourceNodeId to: $destinationNodeId at $destinationURI", e)
+              logger.error(s"Failed to migrate data of slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost}", e)
               Future(Unit)
           }
       }
 
+      migrate.onSuccess { case _ =>
+        logger.info(s"Successfully migrated slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost}")
+      }
+
+      migrateReplace.onSuccess { case _ =>
+        logger.info(s"Successfully migrated slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost}")
+      }
+
       migrate
     }
 
@@ -428,7 +440,7 @@ trait CornucopiaGraph {
           destinationConnection <- clusterConnections.get(destinationNodeId)
           _ <- setSlotAssignment(sourceConnection, destinationConnection)
           _ <- migrateSlotKeys(sourceConnection, destinationConnection)
-        } yield notifySlotAssignment(slot, masters)
+        } yield notifySlotAssignment(slot, destinationNodeId, masters)
     }
   }
 
@@ -436,15 +448,16 @@ trait CornucopiaGraph {
     * Notify all master nodes of a slot assignment so that they will immediately be able to redirect clients.
     *
     * @param masters The list of nodes in the cluster that will be assigned hash slots.
+    * @param assignedNodeId The node that should be assigned the slot
     * @param executionContext The thread dispatcher context.
     * @return Future indicating success.
     */
-  protected def notifySlotAssignment(slot: Int, masters: mutable.Buffer[RedisClusterNode])
+  protected def notifySlotAssignment(slot: Int, assignedNodeId: String, masters: List[RedisClusterNode])
                                   (implicit saladAPI: Salad, executionContext: ExecutionContext)
   : Future[Unit] = {
     val getMasterConnections = masters.map(master => getConnection(master.getNodeId))
     Future.sequence(getMasterConnections).flatMap { masterConnections =>
-      val notifyResults = masterConnections.map(_.clusterSetSlotNode(slot, slotNode(slot, masters)))
+      val notifyResults = masterConnections.map(_.clusterSetSlotNode(slot, assignedNodeId))
       Future.sequence(notifyResults).map(x => x)
     }
   }
@@ -546,9 +559,10 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
         waitForTopologyRefresh2[Seq[RedisURI], Seq[Option[ActorRef]]](uris, actorRefs)
       }
     })
-    .map{ case (_, actorRef) =>
+    .map{ case (redisURIs, actorRef) =>
       val ref = actorRef.head
-      KeyValue(RESHARD.key, "", ref)
+      val uri = redisURIs.head
+      KeyValue(RESHARD.key, "", ref, Some(uri))
     }
 
   // Add a slave node to the cluster, replicating the master that has the fewest slaves.
@@ -588,18 +602,27 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
   }
 
   override protected def streamReshard(implicit executionContext: ExecutionContext) = Flow[KeyValue]
-    .map(record => Seq( record.senderRef ))
-    .conflate((seq1, seq2) => seq1 ++ seq2 )
+    .map(kv => (kv.senderRef, kv.newMasterURI))
     .throttle(1, Config.Cornucopia.minReshardWait, 1, ThrottleMode.Shaping)
-    .mapAsync(1)(reshardClusterPrime)
+    .mapAsync(1)(t => {
+      val senderRef = t._1
+      val newMasterURI = t._2
+      reshardClusterPrimeWrapper(senderRef, newMasterURI)
+    })
     .mapAsync(1)(waitForTopologyRefresh[Unit])
     .mapAsync(1)(_ => logTopology)
     .map(_ => KeyValue("", ""))
 
-  private def reshardClusterPrime(senders: Seq[Option[ActorRef]]): Future[Unit] = {
+  // For wrapping reshardClusterPrime so we can test by passing in a dummy implicit salad API
+  protected def reshardClusterPrimeWrapper(sender: Option[ActorRef], newMasterURI: Option[RedisURI]): Future[Unit] = {
+    implicit val newSaladAPIimpl = newSaladAPI
+    reshardClusterPrime(sender, newMasterURI)
+  }
+
+  protected def reshardClusterPrime(sender: Option[ActorRef], newMasterURI: Option[RedisURI])(implicit newSaladAPIimpl: Salad): Future[Unit] = {
 
-    def reshard(ref: ActorRef): Future[Unit] = {
-      reshardCluster(Seq()) map { _: Unit =>
+    def reshard(ref: ActorRef, uri: RedisURI): Future[Unit] = {
+      reshardClusterWithNewMaster(uri) map { _: Unit =>
         logger.info("Successfully resharded cluster, informing Kubernetes controller")
         ref ! Right("master")
       } recover {
@@ -609,10 +632,93 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
       }
     }
 
-    val flattened = senders.flatten
+    val result = for {
+      ref <- sender
+      uri <- newMasterURI
+    } yield reshard(ref, uri)
 
-    if (flattened.isEmpty) Future(Unit)
-    else Future.reduce(senders.flatten.map(reshard))((_, _) => Unit)
+    result match {
+      case Some(f) => f
+      case None =>
+        // this should never happen though
+        logger.error("There was a problem resharding the cluster: sender actor or new redis master URI missing")
+        Future(Unit)
+    }
+  }
+
+  // TO-DO: make Slot a Type (Int)
+  // TO-DO: make NodeID a Type (String)
+  protected def computeReshardTable(sourceNodes: List[RedisClusterNode]): Map[String, List[Int]] = {
+    val reshardTable: Map[String, List[Int]] = Map()
+
+    case class LogicalNode(node: RedisClusterNode, slots: List[Int])
+
+    val logicalNodes = sourceNodes.map(n => LogicalNode(n, n.getSlots.asInstanceOf[List[Int]]))
+
+    val sortedSources = logicalNodes.sorted(Ordering.by((_: LogicalNode).slots.size).reverse)
+
+    val totalSourceSlots = sortedSources.foldLeft(0)((sum, n) => sum + n.slots.size)
+
+    val numSlots = totalSourceSlots / (logicalNodes.size + 1) // total number of slots to move to target
+
+    def computeNumSlots(i: Int, source: LogicalNode): Int = {
+      if (i == 0) Math.ceil((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
+      else Math.floor((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
+    }
+
+    sortedSources.zipWithIndex.foreach { case (source, i) =>
+      val sortedSlots = source.slots.sorted
+      val n = computeNumSlots(i, source)
+      val slots = sortedSlots.take(n)
+      reshardTable put(source.node.getNodeId, slots)
+    }
+
+    reshardTable
+  }
+
+  private def printReshardTable(reshardTable: Map[String, List[Int]]) = {
+    logger.debug(s"Reshard Table:")
+    reshardTable foreach { case (nodeId, slots) =>
+        logger.debug(s"Migrating slots from node '$nodeId': ${slots.mkString(", ")}")
+    }
+  }
+
+  protected def reshardClusterWithNewMaster(newMasterURI: RedisURI)(implicit newSaladAPIimpl: Salad)
+  : Future[Unit] = {
+    // Execute futures using a thread pool so we don't run out of memory due to futures.
+    implicit val executionContext = Config.Consumer.actorSystem.dispatchers.lookup("akka.actor.resharding-dispatcher")
+    newSaladAPIimpl.masterNodes.flatMap { mn =>
+      val masterNodes = mn.toList
+      val liveMasters = masterNodes.filter(_.isConnected)
+
+      lazy val idToURI = new util.HashMap[String,RedisURI](liveMasters.length + 1, 1)
+
+      // Re-use cluster connections so we don't exceed file-handle limit or waste resources.
+      lazy val clusterConnections = new util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]](liveMasters.length + 1, 1)
+
+      val targetNode = masterNodes.filter(_.getUri == newMasterURI).head
+
+      liveMasters.map { master =>
+        idToURI.put(master.getNodeId, master.getUri)
+        val connection = getConnection(master.getNodeId)
+        clusterConnections.put(master.getNodeId, connection)
+      }
+
+      val sourceNodes = masterNodes.filterNot(_ == targetNode)
+
+      val reshardTable = computeReshardTable(sourceNodes)
+
+      printReshardTable(reshardTable)
+
+      val migrateResults = for {
+        (sourceNodeId, slots) <- reshardTable
+        slot <- slots
+      } yield {
+        migrateSlot(slot, sourceNodeId, targetNode.getNodeId, newMasterURI, liveMasters, clusterConnections)
+      }
+
+      Future.reduce(migrateResults)((_, b) => b)
+    }
   }
 
   protected def extractKeyValue = Flow[ActorRecord]
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index 78655e9..880851d 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -72,6 +72,12 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
       override protected def findMasters(redisURIList: Seq[RedisURI])
                                         (implicit executionContext: ExecutionContext): Future[Unit] = Future(Unit)
 
+      override protected def reshardClusterPrimeWrapper(sender: Option[ActorRef], newMasterURI: Option[RedisURI]): Future[Unit] = {
+        reshardClusterPrime(sender, newMasterURI)
+      }
+
+      override protected def reshardClusterWithNewMaster(newMasterURI: RedisURI)(implicit newSaladAPIimpl: Salad): Future[Unit] = Future(Unit)
+
     }
 
   }
@@ -97,7 +103,7 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
       future.onComplete {
         case Failure(_) => assert(false)
         case Success(msg) =>
-          assert(msg == Right("OK"))
+          assert(msg == Right("master"))
       }
 
       Await.ready(future, timeout.duration)
@@ -119,7 +125,7 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
       future.onComplete {
         case Failure(_) => assert(false)
         case Success(msg) =>
-          assert(msg == Right("OK"))
+          assert(msg == Right("slave"))
       }
 
       Await.ready(future, timeout.duration)

From 05fbb94380c7bc3dc76fca3469d98b59bad191b8 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 19:00:20 -0700
Subject: [PATCH 22/44] Just use mutable Buffer

---
 build.sbt                                           |  2 +-
 .../github/kliewkliew/cornucopia/graph/Graph.scala  | 13 ++++++-------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/build.sbt b/build.sbt
index 1082e0a..8ea3f6e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.13-SNAPSHOT"
+version := "0.14-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 1269c6c..c9a23ba 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -327,7 +327,7 @@ trait CornucopiaGraph {
           migrateSlot(
             slot,
             sourceNodeId, destinationNodeId, idToURI.get(destinationNodeId),
-            assignableMasters.toList, clusterConnections)
+            assignableMasters, clusterConnections)
         }
       }
       val finalMigrateResult = Future.sequence(migrateResults)
@@ -370,7 +370,7 @@ trait CornucopiaGraph {
     * @return Future indicating success.
     */
   protected def migrateSlot(slot: Int, sourceNodeId: String, destinationNodeId: String, destinationURI: RedisURI,
-                            masters: List[RedisClusterNode],
+                            masters: mutable.Buffer[RedisClusterNode],
                             clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]])
                            (implicit saladAPI: Salad, executionContext: ExecutionContext): Future[Unit] = {
 
@@ -452,7 +452,7 @@ trait CornucopiaGraph {
     * @param executionContext The thread dispatcher context.
     * @return Future indicating success.
     */
-  protected def notifySlotAssignment(slot: Int, assignedNodeId: String, masters: List[RedisClusterNode])
+  protected def notifySlotAssignment(slot: Int, assignedNodeId: String, masters: mutable.Buffer[RedisClusterNode])
                                   (implicit saladAPI: Salad, executionContext: ExecutionContext)
   : Future[Unit] = {
     val getMasterConnections = masters.map(master => getConnection(master.getNodeId))
@@ -628,7 +628,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
       } recover {
         case ex: Throwable =>
           logger.error("Failed to reshard cluster, informing Kubernetes controller")
-          ref ! Left(s"ERROR: ${ex.toString}")
+          ref ! Left(s"${ex.toString}")
       }
     }
 
@@ -648,7 +648,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
 
   // TO-DO: make Slot a Type (Int)
   // TO-DO: make NodeID a Type (String)
-  protected def computeReshardTable(sourceNodes: List[RedisClusterNode]): Map[String, List[Int]] = {
+  protected def computeReshardTable(sourceNodes: mutable.Buffer[RedisClusterNode]): Map[String, List[Int]] = {
     val reshardTable: Map[String, List[Int]] = Map()
 
     case class LogicalNode(node: RedisClusterNode, slots: List[Int])
@@ -687,8 +687,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
   : Future[Unit] = {
     // Execute futures using a thread pool so we don't run out of memory due to futures.
     implicit val executionContext = Config.Consumer.actorSystem.dispatchers.lookup("akka.actor.resharding-dispatcher")
-    newSaladAPIimpl.masterNodes.flatMap { mn =>
-      val masterNodes = mn.toList
+    newSaladAPIimpl.masterNodes.flatMap { masterNodes =>
       val liveMasters = masterNodes.filter(_.isConnected)
 
       lazy val idToURI = new util.HashMap[String,RedisURI](liveMasters.length + 1, 1)

From 10514526e086b58e4d5b969559475d12e1a1ba8e Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 19:15:06 -0700
Subject: [PATCH 23/44] Print stack trace

---
 build.sbt                                                       | 2 +-
 .../scala/com/github/kliewkliew/cornucopia/graph/Graph.scala    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index 8ea3f6e..cd8528a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.14-SNAPSHOT"
+version := "0.15-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index c9a23ba..5f8f49f 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -627,7 +627,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
         ref ! Right("master")
       } recover {
         case ex: Throwable =>
-          logger.error("Failed to reshard cluster, informing Kubernetes controller")
+          logger.error("Failed to reshard cluster, informing Kubernetes controller", ex)
           ref ! Left(s"${ex.toString}")
       }
     }

From 428f5710efd03a1e53dcb83714b696c48fb0a7f3 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 19:39:37 -0700
Subject: [PATCH 24/44] try this

---
 .../kliewkliew/cornucopia/graph/Graph.scala      | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 5f8f49f..0be00c6 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -327,7 +327,7 @@ trait CornucopiaGraph {
           migrateSlot(
             slot,
             sourceNodeId, destinationNodeId, idToURI.get(destinationNodeId),
-            assignableMasters, clusterConnections)
+            assignableMasters.toList, clusterConnections)
         }
       }
       val finalMigrateResult = Future.sequence(migrateResults)
@@ -370,7 +370,7 @@ trait CornucopiaGraph {
     * @return Future indicating success.
     */
   protected def migrateSlot(slot: Int, sourceNodeId: String, destinationNodeId: String, destinationURI: RedisURI,
-                            masters: mutable.Buffer[RedisClusterNode],
+                            masters: List[RedisClusterNode],
                             clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]])
                            (implicit saladAPI: Salad, executionContext: ExecutionContext): Future[Unit] = {
 
@@ -452,7 +452,7 @@ trait CornucopiaGraph {
     * @param executionContext The thread dispatcher context.
     * @return Future indicating success.
     */
-  protected def notifySlotAssignment(slot: Int, assignedNodeId: String, masters: mutable.Buffer[RedisClusterNode])
+  protected def notifySlotAssignment(slot: Int, assignedNodeId: String, masters: List[RedisClusterNode])
                                   (implicit saladAPI: Salad, executionContext: ExecutionContext)
   : Future[Unit] = {
     val getMasterConnections = masters.map(master => getConnection(master.getNodeId))
@@ -648,12 +648,15 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
 
   // TO-DO: make Slot a Type (Int)
   // TO-DO: make NodeID a Type (String)
-  protected def computeReshardTable(sourceNodes: mutable.Buffer[RedisClusterNode]): Map[String, List[Int]] = {
+  protected def computeReshardTable(sourceNodes: List[RedisClusterNode]): Map[String, List[Int]] = {
     val reshardTable: Map[String, List[Int]] = Map()
 
     case class LogicalNode(node: RedisClusterNode, slots: List[Int])
 
-    val logicalNodes = sourceNodes.map(n => LogicalNode(n, n.getSlots.asInstanceOf[List[Int]]))
+    val logicalNodes = sourceNodes.map { n =>
+      val slots = n.getSlots.asInstanceOf[List[Int]]
+      LogicalNode(n, slots)
+    }
 
     val sortedSources = logicalNodes.sorted(Ordering.by((_: LogicalNode).slots.size).reverse)
 
@@ -687,7 +690,8 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
   : Future[Unit] = {
     // Execute futures using a thread pool so we don't run out of memory due to futures.
     implicit val executionContext = Config.Consumer.actorSystem.dispatchers.lookup("akka.actor.resharding-dispatcher")
-    newSaladAPIimpl.masterNodes.flatMap { masterNodes =>
+    newSaladAPIimpl.masterNodes.flatMap { mn =>
+      val masterNodes = mn.toList
       val liveMasters = masterNodes.filter(_.isConnected)
 
       lazy val idToURI = new util.HashMap[String,RedisURI](liveMasters.length + 1, 1)

From bcc6dda1d11da1f7a954686f9eaa6e6e02039397 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 19:39:55 -0700
Subject: [PATCH 25/44] version bump

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index cd8528a..b30ed57 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.15-SNAPSHOT"
+version := "0.16-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From 05cdf44a3260bb623c840e01aa4a8b965cf2041c Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 20:13:25 -0700
Subject: [PATCH 26/44] Try some funky conversion from Java to Scala

---
 build.sbt                                                     | 2 +-
 .../scala/com/github/kliewkliew/cornucopia/graph/Graph.scala  | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index b30ed57..28ce894 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.16-SNAPSHOT"
+version := "0.17-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 0be00c6..0a9de92 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -649,12 +649,14 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
   // TO-DO: make Slot a Type (Int)
   // TO-DO: make NodeID a Type (String)
   protected def computeReshardTable(sourceNodes: List[RedisClusterNode]): Map[String, List[Int]] = {
+    import scala.collection.JavaConverters._
+
     val reshardTable: Map[String, List[Int]] = Map()
 
     case class LogicalNode(node: RedisClusterNode, slots: List[Int])
 
     val logicalNodes = sourceNodes.map { n =>
-      val slots = n.getSlots.asInstanceOf[List[Int]]
+      val slots = n.getSlots.asScala.toList.map(_.toInt)
       LogicalNode(n, slots)
     }
 

From b14c7998536ff5299bcf0904cedd69a92b9af6e3 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 20:36:31 -0700
Subject: [PATCH 27/44] Fix map problem

---
 build.sbt                                                 | 2 +-
 .../com/github/kliewkliew/cornucopia/graph/Graph.scala    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/build.sbt b/build.sbt
index 28ce894..7073008 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.17-SNAPSHOT"
+version := "0.18-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 0a9de92..8b8d49e 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -651,7 +651,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
   protected def computeReshardTable(sourceNodes: List[RedisClusterNode]): Map[String, List[Int]] = {
     import scala.collection.JavaConverters._
 
-    val reshardTable: Map[String, List[Int]] = Map()
+    val reshardTable: scala.collection.immutable.Map[String, List[Int]] = Map.empty[String, List[Int]]
 
     case class LogicalNode(node: RedisClusterNode, slots: List[Int])
 
@@ -675,16 +675,16 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
       val sortedSlots = source.slots.sorted
       val n = computeNumSlots(i, source)
       val slots = sortedSlots.take(n)
-      reshardTable put(source.node.getNodeId, slots)
+      reshardTable + (source.node.getNodeId -> slots)
     }
 
     reshardTable
   }
 
   private def printReshardTable(reshardTable: Map[String, List[Int]]) = {
-    logger.debug(s"Reshard Table:")
+    logger.info(s"Reshard Table:")
     reshardTable foreach { case (nodeId, slots) =>
-        logger.debug(s"Migrating slots from node '$nodeId': ${slots.mkString(", ")}")
+        logger.info(s"Migrating slots from node '$nodeId': ${slots.mkString(", ")}")
     }
   }
 

From e455083dafab1af27efc373fe52cb313bdef7c7b Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 23:20:52 -0700
Subject: [PATCH 28/44] Move reshard table function into its own helper object

---
 .../kliewkliew/cornucopia/graph/Graph.scala   | 36 +------
 .../cornucopia/redis/ReshardTable.scala       | 44 +++++++++
 .../kliewkliew/cornucopia/ReshardTest.scala   | 99 +++++++++++++++++++
 3 files changed, 144 insertions(+), 35 deletions(-)
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
 create mode 100644 src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 8b8d49e..ccdefd3 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import com.github.kliewkliew.cornucopia.redis._
 import com.github.kliewkliew.salad.SaladClusterAPI
+import com.github.kliewkliew.cornucopia.redis.ReshardTable._
 import com.lambdaworks.redis.RedisURI
 import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
 import com.lambdaworks.redis.models.role.RedisInstance.Role
@@ -646,41 +647,6 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
     }
   }
 
-  // TO-DO: make Slot a Type (Int)
-  // TO-DO: make NodeID a Type (String)
-  protected def computeReshardTable(sourceNodes: List[RedisClusterNode]): Map[String, List[Int]] = {
-    import scala.collection.JavaConverters._
-
-    val reshardTable: scala.collection.immutable.Map[String, List[Int]] = Map.empty[String, List[Int]]
-
-    case class LogicalNode(node: RedisClusterNode, slots: List[Int])
-
-    val logicalNodes = sourceNodes.map { n =>
-      val slots = n.getSlots.asScala.toList.map(_.toInt)
-      LogicalNode(n, slots)
-    }
-
-    val sortedSources = logicalNodes.sorted(Ordering.by((_: LogicalNode).slots.size).reverse)
-
-    val totalSourceSlots = sortedSources.foldLeft(0)((sum, n) => sum + n.slots.size)
-
-    val numSlots = totalSourceSlots / (logicalNodes.size + 1) // total number of slots to move to target
-
-    def computeNumSlots(i: Int, source: LogicalNode): Int = {
-      if (i == 0) Math.ceil((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
-      else Math.floor((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
-    }
-
-    sortedSources.zipWithIndex.foreach { case (source, i) =>
-      val sortedSlots = source.slots.sorted
-      val n = computeNumSlots(i, source)
-      val slots = sortedSlots.take(n)
-      reshardTable + (source.node.getNodeId -> slots)
-    }
-
-    reshardTable
-  }
-
   private def printReshardTable(reshardTable: Map[String, List[Int]]) = {
     logger.info(s"Reshard Table:")
     reshardTable foreach { case (nodeId, slots) =>
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala b/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
new file mode 100644
index 0000000..53539f1
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
@@ -0,0 +1,44 @@
+package com.github.kliewkliew.cornucopia.redis
+
+import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
+
+object ReshardTable {
+
+  type NodeId = String
+  type Slot = Int
+  type ReshardTable = scala.collection.immutable.Map[NodeId, List[Slot]]
+
+  def computeReshardTable(sourceNodes: List[RedisClusterNode]): ReshardTable = {
+    import scala.collection.JavaConverters._
+
+    case class LogicalNode(node: RedisClusterNode, slots: List[Int])
+
+    val logicalNodes = sourceNodes.map { n =>
+      val slots = n.getSlots.asScala.toList.map(_.toInt)
+      LogicalNode(n, slots)
+    }
+
+    val sortedSources = logicalNodes.sorted(Ordering.by((_: LogicalNode).slots.size).reverse)
+
+    val totalSourceSlots = sortedSources.foldLeft(0)((sum, n) => sum + n.slots.size)
+
+    val numSlots = totalSourceSlots / (logicalNodes.size + 1) // total number of slots to move to target
+
+    def computeNumSlots(i: Int, source: LogicalNode): Int = {
+      if (i == 0) Math.ceil((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
+      else Math.floor((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
+    }
+
+    val reshardTable: ReshardTable = Map.empty[NodeId, List[Slot]]
+
+    val table = sortedSources.zipWithIndex.foldLeft(reshardTable) { case (tbl, (source, i)) =>
+      val sortedSlots = source.slots.sorted
+      val n = computeNumSlots(i, source)
+      val slots = sortedSlots.take(n)
+      val nodeId = source.node.getNodeId
+      tbl + (nodeId -> slots)
+    }
+
+    table
+  }
+}
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala
new file mode 100644
index 0000000..f0e188f
--- /dev/null
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala
@@ -0,0 +1,99 @@
+package com.github.kliewkliew.cornucopia
+
+import com.github.kliewkliew.cornucopia.redis._
+import com.lambdaworks.redis.RedisURI
+//import com.github.kliewkliew.cornucopia._
+import com.github.kliewkliew.cornucopia.graph._
+import com.github.kliewkliew.cornucopia.redis.Connection.Salad
+import akka.testkit.{TestActorRef, TestKit, TestProbe}
+import akka.actor.{ActorSystem, ActorRef}
+import akka.pattern.ask
+//import akka.actor.Status.Failure
+import akka.stream.ActorMaterializer
+import akka.util.Timeout
+import org.scalatest.mockito.MockitoSugar
+import org.scalatest.{BeforeAndAfterAll, MustMatchers, WordSpecLike}
+import org.mockito.Mockito._
+import org.scalatest.mockito.MockitoSugar._
+import org.mockito.Matchers._
+import com.github.kliewkliew.cornucopia.graph._
+import akka.stream.scaladsl.Flow
+import akka.stream.scaladsl.Sink
+import scala.concurrent.duration._
+import scala.concurrent.Await
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{ Success, Failure }
+import redis.Connection.{newSaladAPI, Salad}
+import redis.ReshardTable._
+import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
+
+class ReshardTest extends TestKit(ActorSystem("ReshardTest"))
+  with WordSpecLike with BeforeAndAfterAll with MustMatchers with MockitoSugar {
+
+  trait ReshardTableTest {
+    val one: java.util.List[Integer] = new java.util.ArrayList[Integer](java.util.Arrays.asList[Integer](1,2,3,4,5,6))
+    val two: java.util.List[Integer] = new java.util.ArrayList[Integer](java.util.Arrays.asList[Integer](7,8,9,10,11,12))
+    val three: java.util.List[Integer] = new java.util.ArrayList[Integer](java.util.Arrays.asList[Integer](13,14,15,16,17))
+
+    class RedisClusterNodeTest(private val slots: java.util.List[Integer], private val nodeId: String) extends RedisClusterNode {
+      override def getSlots: java.util.List[Integer] = slots
+      override def getNodeId: String = nodeId
+    }
+
+    val node1 = new RedisClusterNodeTest(one, "a")
+    val node2 = new RedisClusterNodeTest(two, "b")
+    val node3 = new RedisClusterNodeTest(three, "c")
+
+    val clusterNodes = List(node1, node2, node3)
+
+    val expectedReshardTable: ReshardTable = Map(
+      "a" -> List(1,2),
+      "b" -> List(7),
+      "c" -> List(13)
+    )
+  }
+
+  trait ReshardDebug {
+    val redisUri = "redis://127.0.0.1"
+    implicit val newSaladAPIimpl: Salad = newSaladAPI
+  }
+
+  implicit val ec = system.dispatcher
+
+  override def afterAll(): Unit = {
+    system.terminate()
+  }
+
+  "Reshard cluster with new master" must {
+    "calculate reshard table correctly" in new ReshardTableTest {
+
+      val reshardTable: ReshardTable = computeReshardTable(clusterNodes)
+
+      assert(reshardTable == expectedReshardTable)
+    }
+  }
+
+  "Debugging" must {
+    "be fun" in new ReshardDebug {
+      import Library.source._
+
+      val cornucopiaActorSource = new CornucopiaActorSource
+
+      private val ref = cornucopiaActorSource.ref
+
+      implicit val timeout = Timeout(6 seconds)
+
+      val future = ask(ref, Task("+master", redisUri))
+
+      future.onComplete {
+        case Failure(_) => assert(false)
+        case Success(msg) =>
+          assert(msg == Right("master"))
+      }
+
+      Await.ready(future, timeout.duration)
+    }
+  }
+
+}
+

From cc3c89f7d29ee516897f8eacdd22ea1cb8694c61 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Sun, 14 May 2017 23:21:20 -0700
Subject: [PATCH 29/44] bump

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 7073008..2dce6ec 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.18-SNAPSHOT"
+version := "0.19-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From 1219a0ee92c05a31927bed33311d5383e74c916c Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Mon, 15 May 2017 12:12:51 -0700
Subject: [PATCH 30/44] Add an http server to accept Cornucopia Tasks

---
 build.sbt                                     |  3 +
 src/main/resources/application.conf           |  8 +++
 .../kliewkliew/cornucopia/Microservice.scala  | 17 +++++-
 .../http/CornucopiaTaskMaster.scala           | 22 +++++++
 .../cornucopia/http/EventMarshalling.scala    |  9 +++
 .../kliewkliew/cornucopia/http/RestApi.scala  | 59 +++++++++++++++++++
 .../kliewkliew/cornucopia/http/Server.scala   | 48 +++++++++++++++
 7 files changed, 165 insertions(+), 1 deletion(-)
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/http/CornucopiaTaskMaster.scala
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/http/EventMarshalling.scala
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/http/RestApi.scala
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/http/Server.scala

diff --git a/build.sbt b/build.sbt
index 2dce6ec..1619fe6 100644
--- a/build.sbt
+++ b/build.sbt
@@ -21,6 +21,9 @@ libraryDependencies ++= Seq(
   "biz.paluch.redis" % "lettuce" % "5.0.0.Beta1",
   "org.scala-lang.modules" % "scala-java8-compat_2.11" % "0.8.0",
   "com.typesafe.akka" %% "akka-stream-kafka" % "0.11-RC1",
+  "com.typesafe.akka" %% "akka-http-core"  % "2.4.11",
+  "com.typesafe.akka" %% "akka-http-experimental"  % "2.4.11",
+  "com.typesafe.akka" %% "akka-http-spray-json-experimental"  % "2.4.11",
 //  "com.github.kliewkliew" %% "salad" % "0.11.01",
   "com.adenda" %% "salad" % "0.11.03",
   "org.slf4j" % "slf4j-log4j12" % "1.7.22"
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index 6babffd..06f1ccb 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -46,6 +46,14 @@ cornucopia {
 
   // either `kafka` or `actor`
   source = "actor"
+
+  // microservice type: `kafka` or `http`
+  microservice.type = "http"
+
+  http {
+    host = "localhost"
+    port = "9001"
+  }
 }
 
 kafka {
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/Microservice.scala b/src/main/scala/com/github/kliewkliew/cornucopia/Microservice.scala
index b6cef83..1896a6a 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/Microservice.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/Microservice.scala
@@ -1,7 +1,22 @@
 package com.github.kliewkliew.cornucopia
 
+import com.typesafe.config.ConfigFactory
+import org.slf4j.LoggerFactory
+import com.github.kliewkliew.cornucopia.http.Server
+
 object Microservice {
   def main(args: Array[String]): Unit = {
-    new graph.CornucopiaKafkaSource().run
+    val config = ConfigFactory.load().getConfig("cornucopia")
+    val microserviceType = config.getString("microservice.type")
+    val logger = LoggerFactory.getLogger(this.getClass)
+
+    microserviceType match {
+      case "kafka" =>
+        logger.info("Using kafka as the microservice source")
+        new graph.CornucopiaKafkaSource().run
+      case "http" =>
+        logger.info("Using http server as the microservice source")
+        Server.start
+    }
   }
 }
\ No newline at end of file
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/http/CornucopiaTaskMaster.scala b/src/main/scala/com/github/kliewkliew/cornucopia/http/CornucopiaTaskMaster.scala
new file mode 100644
index 0000000..1808941
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/http/CornucopiaTaskMaster.scala
@@ -0,0 +1,22 @@
+package com.github.kliewkliew.cornucopia.http
+
+import akka.actor._
+import akka.util.Timeout
+
+object CornucopiaTaskMaster {
+  def props(implicit timeout: Timeout) = Props(new CornucopiaTaskMaster)
+
+  case class RestTask(operation: String, redisNodeIp: String)
+}
+
+class CornucopiaTaskMaster(implicit timeout: Timeout) extends Actor with ActorLogging {
+  import CornucopiaTaskMaster._
+  import context._
+
+  def receive = {
+    case RestTask(operation, redisNodeIp) =>
+      log.info(s"Received Cornucopia API task request: '$operation', '$redisNodeIp'")
+      sender ! Right("its all good")
+      // TODO: send it down the graph
+  }
+}
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/http/EventMarshalling.scala b/src/main/scala/com/github/kliewkliew/cornucopia/http/EventMarshalling.scala
new file mode 100644
index 0000000..02a96ce
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/http/EventMarshalling.scala
@@ -0,0 +1,9 @@
+package com.github.kliewkliew.cornucopia.http
+
+import spray.json._
+
+trait EventMarshalling extends DefaultJsonProtocol {
+  import CornucopiaTaskMaster._
+
+  implicit val taskFormat = jsonFormat2(RestTask)
+}
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/http/RestApi.scala b/src/main/scala/com/github/kliewkliew/cornucopia/http/RestApi.scala
new file mode 100644
index 0000000..0e2f68f
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/http/RestApi.scala
@@ -0,0 +1,59 @@
+package com.github.kliewkliew.cornucopia.http
+
+import scala.concurrent.duration._
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+import akka.actor._
+import akka.pattern.ask
+import akka.util.Timeout
+
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.server._
+
+class RestApi(system: ActorSystem, timeout: Timeout) extends RestRoutes {
+  implicit val requestTimeout = timeout
+  implicit def executionContext = system.dispatcher
+
+  def createCornucopiaTaskMaster = system.actorOf(CornucopiaTaskMaster.props)
+}
+
+trait RestRoutes extends CornucopiaApi with EventMarshalling {
+  import StatusCodes._
+  import CornucopiaTaskMaster._
+
+  def routes: Route = taskRoute
+
+  def taskRoute = pathPrefix("task") {
+    pathEndOrSingleSlash {
+      post {
+        entity(as[RestTask]) { ed =>
+          onSuccess(submitTask(ed.operation, ed.redisNodeIp)) {
+            case Left(msg) =>
+              complete(BadRequest, msg)
+            case Right(msg) =>
+              complete(Accepted, msg)
+          }
+        }
+      }
+    }
+  }
+}
+
+trait CornucopiaApi {
+  import CornucopiaTaskMaster._
+
+  def createCornucopiaTaskMaster(): ActorRef
+
+  implicit def executionContext: ExecutionContext
+  implicit def requestTimeout: Timeout
+
+  lazy val cornucopiaTaskMaster = createCornucopiaTaskMaster()
+
+  def submitTask(operation: String, redisNodeIp: String) = {
+    cornucopiaTaskMaster.ask(RestTask(operation, redisNodeIp)).mapTo[Either[String, String]]
+  }
+}
+
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/http/Server.scala b/src/main/scala/com/github/kliewkliew/cornucopia/http/Server.scala
new file mode 100644
index 0000000..632ed95
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/http/Server.scala
@@ -0,0 +1,48 @@
+package com.github.kliewkliew.cornucopia.http
+
+import scala.concurrent.Future
+import akka.actor.{Actor, ActorSystem, Props}
+import akka.event.Logging
+import akka.util.Timeout
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.Http.ServerBinding
+import akka.http.scaladsl.server.Directives._
+import akka.stream.ActorMaterializer
+import org.slf4j.LoggerFactory
+import com.typesafe.config.{ Config, ConfigFactory }
+
+object Server extends RequestTimeout {
+  val config = ConfigFactory.load()
+  val host = config.getString("cornucopia.http.host")
+  val port = config.getInt("cornucopia.http.port")
+
+  implicit val system = ActorSystem()
+  implicit val ec = system.dispatcher
+
+  val api = new RestApi(system, requestTimeout(config)).routes
+
+  implicit val materializer = ActorMaterializer()
+  val bindingFuture: Future[ServerBinding] = Http().bindAndHandle(api, host, port)
+
+  val log = Logging(system.eventStream, "cornucopia-rest-api")
+
+  def start = {
+    bindingFuture.map { serverBinding =>
+      log.info(s"RestApi bound to ${serverBinding.localAddress} ")
+    }.onFailure {
+      case ex: Exception =>
+        log.error(ex, "Failed to bind to {}:{}!", host, port)
+        system.terminate()
+    }
+  }
+}
+
+trait RequestTimeout {
+  import scala.concurrent.duration._
+  def requestTimeout(config: Config): Timeout = {
+    val t = config.getString("akka.http.server.request-timeout")
+    val d = Duration(t)
+    FiniteDuration(d.length, d.unit)
+  }
+}
+

From 36d1f77fe4c8bdbff7777eb7abf226da6f56f0eb Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Mon, 15 May 2017 13:22:30 -0700
Subject: [PATCH 31/44] Implement the API interface to the graph, and
 resharding works locally (yeah!)

---
 .../kliewkliew/cornucopia/graph/Graph.scala   | 23 +++++++++++++++----
 .../http/CornucopiaTaskMaster.scala           | 14 +++++++++--
 2 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index ccdefd3..1586db3 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -49,10 +49,22 @@ trait CornucopiaGraph {
   // Extract a tuple of the key and value from a Kafka record.
   case class KeyValue(key: String, value: String, senderRef: Option[ActorRef] = None, newMasterURI: Option[RedisURI] = None)
 
+  // Allows to create Redis URI from the following forms:
+  // host OR host:port
+  // e.g., redis://127.0.0.1 OR redis://127.0.0.1:7006
+  protected def createRedisUri(uri: String): RedisURI = {
+    val parts = uri.split(":")
+    if (parts.size == 3) {
+      val host = parts(1).foldLeft("")((acc, ch) => if (ch != '/') acc + ch else acc)
+      RedisURI.create(host, parts(2).toInt)
+    }
+    else RedisURI.create(uri)
+  }
+
   // Add a master node to the cluster.
   def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(_.value)
-    .map(RedisURI.create)
+    .map(createRedisUri)
     .map(newSaladAPI.canonicalizeURI)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(addNodesToCluster)
@@ -62,7 +74,7 @@ trait CornucopiaGraph {
   // Add a slave node to the cluster, replicating the master that has the fewest slaves.
   protected def streamAddSlave(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(_.value)
-    .map(RedisURI.create)
+    .map(createRedisUri)
     .map(newSaladAPI.canonicalizeURI)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(addNodesToCluster)
@@ -75,7 +87,7 @@ trait CornucopiaGraph {
   // Emit a key-value pair indicating the node type and URI.
   protected def streamRemoveNode(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(_.value)
-    .map(RedisURI.create)
+    .map(createRedisUri)
     .map(newSaladAPI.canonicalizeURI)
     .mapAsync(1)(emitNodeType)
 
@@ -546,10 +558,11 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
 
   protected type ActorRecord = Task
 
+
   // Add a master node to the cluster.
   def streamAddMasterPrime(implicit executionContext: ExecutionContext, newSaladAPIimpl: Connection.Salad) = Flow[KeyValue]
     .map(kv => (kv.value, kv.senderRef))
-    .map(t => (RedisURI.create(t._1), t._2) )
+    .map(t => (createRedisUri(t._1), t._2) )
     .map(t => (newSaladAPIimpl.canonicalizeURI(t._1), t._2))
     .groupedWithin(1, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(t => {
@@ -569,7 +582,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
   // Add a slave node to the cluster, replicating the master that has the fewest slaves.
   def streamAddSlavePrime(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(kv => (kv.value, kv.senderRef))
-    .map(t => (RedisURI.create(t._1), t._2) )
+    .map(t => (createRedisUri(t._1), t._2) )
     .map(t => (newSaladAPIimpl.canonicalizeURI(t._1), t._2))
     .groupedWithin(1, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(t => {
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/http/CornucopiaTaskMaster.scala b/src/main/scala/com/github/kliewkliew/cornucopia/http/CornucopiaTaskMaster.scala
index 1808941..6c34cb2 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/http/CornucopiaTaskMaster.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/http/CornucopiaTaskMaster.scala
@@ -2,6 +2,9 @@ package com.github.kliewkliew.cornucopia.http
 
 import akka.actor._
 import akka.util.Timeout
+import com.github.kliewkliew.cornucopia.actors._
+import com.github.kliewkliew.cornucopia.graph
+import com.github.kliewkliew.cornucopia.redis.Connection.{newSaladAPI, Salad}
 
 object CornucopiaTaskMaster {
   def props(implicit timeout: Timeout) = Props(new CornucopiaTaskMaster)
@@ -11,12 +14,19 @@ object CornucopiaTaskMaster {
 
 class CornucopiaTaskMaster(implicit timeout: Timeout) extends Actor with ActorLogging {
   import CornucopiaTaskMaster._
-  import context._
+  import CornucopiaSource._
+
+  implicit val newSaladAPIimpl: Salad = newSaladAPI
+  val ref: ActorRef = new graph.CornucopiaActorSource().ref
 
   def receive = {
     case RestTask(operation, redisNodeIp) =>
       log.info(s"Received Cornucopia API task request: '$operation', '$redisNodeIp'")
       sender ! Right("its all good")
-      // TODO: send it down the graph
+      ref ! Task(operation, redisNodeIp, Some(self))
+    case Right(msg) =>
+      log.info(s"Received task completion: $msg")
+    case Left(msg) =>
+      log.info(s"Received task failed: $msg")
   }
 }

From eb094306e92fbef0034a81b13766914aeff9d1b0 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Tue, 16 May 2017 11:00:31 -0700
Subject: [PATCH 32/44] Version bump 0.20-SNAPSHOT

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 1619fe6..8fbe57f 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.19-SNAPSHOT"
+version := "0.20-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From d9b4a8a04d10f35460f0d70510fa4bacfc419dea Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Tue, 16 May 2017 15:53:32 -0700
Subject: [PATCH 33/44] Handle different types of known migration errors

---
 build.sbt                                     |  2 +-
 .../kliewkliew/cornucopia/graph/Graph.scala   | 55 +++++++++++--------
 2 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/build.sbt b/build.sbt
index 8fbe57f..aa90a61 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.20-SNAPSHOT"
+version := "0.21-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 1586db3..00c064a 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -384,7 +384,8 @@ trait CornucopiaGraph {
     */
   protected def migrateSlot(slot: Int, sourceNodeId: String, destinationNodeId: String, destinationURI: RedisURI,
                             masters: List[RedisClusterNode],
-                            clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]])
+                            clusterConnections: util.HashMap[String,Future[SaladClusterAPI[CodecType,CodecType]]],
+                            attempts: Int = 1)
                            (implicit saladAPI: Salad, executionContext: ExecutionContext): Future[Unit] = {
 
     // Follows redis-trib.rb
@@ -405,32 +406,40 @@ trait CornucopiaGraph {
         result <- sourceConn.migrate[CodecType](destinationURI, keys.toList)
       } yield result
 
-      def migrateReplace: Future[Unit] = for {
-        keys <- keyList
-        result <- sourceConn.migrate[CodecType](destinationURI, keys.toList, replace = true)
-      } yield result
-
-      migrate.recover {
-        case e =>
-          "BUSYKEY".r.findFirstIn(e.toString) match { // handle the case of a BUSYKEY error
-            case Some(_) =>
-              logger.warn("Problem Migrating Slot: Target key exists. Replacing it for FIX.")
-              migrateReplace
-            case _ =>
-              logger.error(s"Failed to migrate data of slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost}", e)
-              Future(Unit)
-          }
-      }
-
       migrate.onSuccess { case _ =>
-        logger.info(s"Successfully migrated slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost}")
+        logger.info(s"Successfully migrated slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost} on attempt $attempts")
       }
 
-      migrateReplace.onSuccess { case _ =>
-        logger.info(s"Successfully migrated slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost}")
+      def handleFailedMigration(error: Throwable): Future[Unit] = {
+        val errorString = error.toString
+
+        def findError(e: String, identifier: String): Boolean = {
+          identifier.r.findFirstIn(e) match {
+            case Some(_) => true
+            case _ => false
+          }
+        }
+
+        if (findError(errorString, "BUSYKEY")) {
+          logger.warn(s"Problem migrating slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost} (BUSYKEY): Target key exists. Replacing it for FIX.")
+          def migrateReplace: Future[Unit] = for {
+            keys <- keyList
+            result <- sourceConn.migrate[CodecType](destinationURI, keys.toList, replace = true)
+          } yield result
+          migrateReplace
+        } else if (findError(errorString, "CLUSTERDOWN")) {
+          logger.error(s"Failed to migrate slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost} (CLUSTERDOWN): Retrying attempt for $attempts")
+          migrateSlot(slot, sourceNodeId, destinationNodeId, destinationURI, masters, clusterConnections, attempts + 1)
+        } else if (findError(errorString, "MOVED")) {
+          logger.error(s"Failed to migrate slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost} (MOVED): Ignoring on attempt $attempts")
+          Future(Unit)
+        } else {
+          logger.error(s"Failed to migrate slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost}", error)
+          Future(Unit)
+        }
       }
 
-      migrate
+      migrate.recover { case e => handleFailedMigration(e) }
     }
 
     def setSlotAssignment(sourceConn: SaladClusterAPI[CodecType, CodecType],
@@ -701,7 +710,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
         migrateSlot(slot, sourceNodeId, targetNode.getNodeId, newMasterURI, liveMasters, clusterConnections)
       }
 
-      Future.reduce(migrateResults)((_, b) => b)
+      Future.fold(migrateResults)()((_, b) => b)
     }
   }
 

From f169091db976ff4429d2b9a6b23e9f2ac656308c Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Fri, 10 Mar 2017 15:11:45 -0800
Subject: [PATCH 34/44] Docker stuff

---
 build.sbt                                     | 27 +++++++++++++++++--
 project/plugins.sbt                           |  1 +
 .../cornucopia/redis/Connection.scala         |  2 ++
 3 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 project/plugins.sbt

diff --git a/build.sbt b/build.sbt
index b85eeb6..f6f7eae 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,10 +1,13 @@
 name := "cornucopia"
 organization := "com.github.kliewkliew"
 
-version := "1.1.2"
+//version := "1.1.2"
+version := "0.3-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
+enablePlugins(JavaAppPackaging, DockerPlugin)
+
 resolvers += "Sonatype Releases" at "https://oss.sonatype.org/service/repositories/releases/"
 resolvers += "Typesafe Releases" at "http://repo.typesafe.com/typesafe/releases/"
 
@@ -12,6 +15,26 @@ libraryDependencies ++= Seq(
   "biz.paluch.redis" % "lettuce" % "5.0.0.Beta1",
   "org.scala-lang.modules" % "scala-java8-compat_2.11" % "0.8.0",
   "com.typesafe.akka" %% "akka-stream-kafka" % "0.11-RC1",
-  "com.github.kliewkliew" %% "salad" % "0.11.01",
+  "com.github.kliewkliew" %% "salad" % "0.11.04",
   "org.slf4j" % "slf4j-log4j12" % "1.7.22"
 )
+
+// ------------------------------------------------ //
+// ------------- Docker configuration ------------- //
+// ------------------------------------------------ //
+
+javaOptions in Universal ++= Seq(
+  "-Dconfig.file=etc/container.conf"
+)
+
+packageName in Docker := packageName.value
+
+version in Docker := version.value
+
+dockerBaseImage := "openjdk"
+
+dockerRepository := Some("gcr.io/adenda-server-mongodb")
+
+defaultLinuxInstallLocation in Docker := "/usr/local"
+
+daemonUser in Docker := "root"
\ No newline at end of file
diff --git a/project/plugins.sbt b/project/plugins.sbt
new file mode 100644
index 0000000..b674c64
--- /dev/null
+++ b/project/plugins.sbt
@@ -0,0 +1 @@
+addSbtPlugin("com.typesafe.sbt" %% "sbt-native-packager" % "1.0.4")
\ No newline at end of file
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/redis/Connection.scala b/src/main/scala/com/github/kliewkliew/cornucopia/redis/Connection.scala
index 0c089db..71ca46f 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/redis/Connection.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/redis/Connection.scala
@@ -20,6 +20,8 @@ object Connection {
   private val redisClusterPort = redisClusterConfig.getInt("seed.server.port")
   private val nodes = List(RedisURI.create(redisClusterSeedServer, redisClusterPort))
 
+  LoggerFactory.getLogger(this.getClass).debug(s"Cluster seed server: '${nodes}'")
+
   /**
     * Create a new API connection - new connections are necessary to refresh the view of the cluster topology
     * after adding or removing a node.

From be7e737c048626aa0fde8c432516403f4ceefffe Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Tue, 16 May 2017 16:22:27 -0700
Subject: [PATCH 35/44] bump

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index f4981a4..ff9bc82 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.21-SNAPSHOT"
+version := "0.22-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From d78343fd1a039306221045b0390e66dcf59856e0 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Wed, 17 May 2017 11:15:08 -0700
Subject: [PATCH 36/44] Use a helper function to retrieve new Salad API
 connections, which is easy to stub for tests

---
 .../kliewkliew/cornucopia/Library.scala       |  2 -
 .../kliewkliew/cornucopia/graph/Graph.scala   | 69 ++++++++-----------
 .../kliewkliew/cornucopia/LibraryTest.scala   | 12 ++--
 .../kliewkliew/cornucopia/ReshardTest.scala   |  2 +-
 4 files changed, 34 insertions(+), 51 deletions(-)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala b/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
index f3c2588..77a3bfe 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/Library.scala
@@ -2,10 +2,8 @@ package com.github.kliewkliew.cornucopia
 
 import actors.CornucopiaSource
 import akka.actor._
-import redis.Connection.{newSaladAPI, Salad}
 
 object Library {
-  implicit val newSaladAPIimpl: Salad = newSaladAPI
   val ref: ActorRef = new graph.CornucopiaActorSource().ref
   val source = CornucopiaSource
 }
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 00c064a..0f4f213 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -29,6 +29,8 @@ trait CornucopiaGraph {
 
   protected val logger = LoggerFactory.getLogger(this.getClass)
 
+  protected def getNewSaladApi: Salad = newSaladAPI
+
   def partitionEvents(key: String) = key.trim.toLowerCase match {
     case ADD_MASTER.key     => ADD_MASTER.ordinal
     case ADD_SLAVE.key      => ADD_SLAVE.ordinal
@@ -65,17 +67,17 @@ trait CornucopiaGraph {
   def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(_.value)
     .map(createRedisUri)
-    .map(newSaladAPI.canonicalizeURI)
+    .map(getNewSaladApi.canonicalizeURI)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(addNodesToCluster)
     .mapAsync(1)(waitForTopologyRefresh[Seq[RedisURI]])
     .map(_ => KeyValue(RESHARD.key, ""))
 
   // Add a slave node to the cluster, replicating the master that has the fewest slaves.
-  protected def streamAddSlave(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+  def streamAddSlave(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(_.value)
     .map(createRedisUri)
-    .map(newSaladAPI.canonicalizeURI)
+    .map(getNewSaladApi.canonicalizeURI)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(addNodesToCluster)
     .mapAsync(1)(waitForTopologyRefresh[Seq[RedisURI]])
@@ -88,7 +90,7 @@ trait CornucopiaGraph {
   protected def streamRemoveNode(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(_.value)
     .map(createRedisUri)
-    .map(newSaladAPI.canonicalizeURI)
+    .map(getNewSaladApi.canonicalizeURI)
     .mapAsync(1)(emitNodeType)
 
   // Remove a slave node from the cluster.
@@ -153,7 +155,7 @@ trait CornucopiaGraph {
     * @return
     */
   protected def logTopology(implicit executionContext: ExecutionContext): Future[Unit] = {
-    implicit val saladAPI = newSaladAPI
+    implicit val saladAPI = getNewSaladApi
     saladAPI.clusterNodes.map { allNodes =>
       val masterNodes = allNodes.filter(Role.MASTER == _.getRole)
       val slaveNodes = allNodes.filter(Role.SLAVE == _.getRole)
@@ -170,7 +172,7 @@ trait CornucopiaGraph {
     * @return The list of URI if the nodes were met. TODO: emit only the nodes that were successfully added.
     */
   protected def addNodesToCluster(redisURIList: Seq[RedisURI])(implicit executionContext: ExecutionContext): Future[Seq[RedisURI]] = {
-    implicit val saladAPI = newSaladAPI
+    implicit val saladAPI = getNewSaladApi
     saladAPI.clusterNodes.flatMap { allNodes =>
       val getConnectionsToLiveNodes = allNodes.filter(_.isConnected).map(node => getConnection(node.getNodeId))
       Future.sequence(getConnectionsToLiveNodes).flatMap { connections =>
@@ -194,7 +196,7 @@ trait CornucopiaGraph {
     * @return Indicate that the n new slaves are replicating the poorest n masters.
     */
   protected def findMasters(redisURIList: Seq[RedisURI])(implicit executionContext: ExecutionContext): Future[Unit] = {
-    implicit val saladAPI = newSaladAPI
+    implicit val saladAPI = getNewSaladApi
     saladAPI.clusterNodes.flatMap { allNodes =>
       // Node ids for nodes that are currently master nodes but will become slave nodes.
       val newSlaveIds = allNodes.filter(node => redisURIList.contains(node.getUri)).map(_.getNodeId)
@@ -234,7 +236,7 @@ trait CornucopiaGraph {
     * @return the node type and id.
     */
   def emitNodeType(redisURI:RedisURI)(implicit executionContext: ExecutionContext): Future[KeyValue] = {
-    implicit val saladAPI = newSaladAPI
+    implicit val saladAPI = getNewSaladApi
     saladAPI.clusterNodes.map { allNodes =>
       val removalNodeOpt = allNodes.find(node => node.getUri.equals(redisURI))
       if (removalNodeOpt.isEmpty) throw new Exception(s"Node not in cluster: $redisURI")
@@ -275,7 +277,7 @@ trait CornucopiaGraph {
     if (!withoutNodes.exists(_.nonEmpty))
       Future(Unit)
     else {
-      implicit val saladAPI = newSaladAPI
+      implicit val saladAPI = getNewSaladApi
       saladAPI.clusterNodes.flatMap { allNodes =>
         logger.info(s"Forgetting nodes: $withoutNodes")
         // Reset the nodes to be removed.
@@ -312,7 +314,7 @@ trait CornucopiaGraph {
   : Future[Unit] = {
     // Execute futures using a thread pool so we don't run out of memory due to futures.
     implicit val executionContext = Config.Consumer.actorSystem.dispatchers.lookup("akka.actor.resharding-dispatcher")
-    implicit val saladAPI = newSaladAPI
+    implicit val saladAPI = getNewSaladApi
     saladAPI.masterNodes.flatMap { masterNodes =>
 
       val liveMasters = masterNodes.filter(_.isConnected)
@@ -428,7 +430,7 @@ trait CornucopiaGraph {
           } yield result
           migrateReplace
         } else if (findError(errorString, "CLUSTERDOWN")) {
-          logger.error(s"Failed to migrate slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost} (CLUSTERDOWN): Retrying attempt for $attempts")
+          logger.error(s"Failed to migrate slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost} (CLUSTERDOWN): Retrying for attempt $attempts")
           migrateSlot(slot, sourceNodeId, destinationNodeId, destinationURI, masters, clusterConnections, attempts + 1)
         } else if (findError(errorString, "MOVED")) {
           logger.error(s"Failed to migrate slot $slot from $sourceNodeId to $destinationNodeId at ${destinationURI.getHost} (MOVED): Ignoring on attempt $attempts")
@@ -518,8 +520,6 @@ trait CornucopiaGraph {
 class CornucopiaKafkaSource extends CornucopiaGraph {
   import Config.Consumer.materializer
 
-  implicit val newSaladAPIimpl: Salad = newSaladAPI
-
   private type KafkaRecord = ConsumerRecord[String, String]
 
   private def extractKeyValue = Flow[KafkaRecord]
@@ -560,19 +560,18 @@ class CornucopiaKafkaSource extends CornucopiaGraph {
 
 }
 
-class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaGraph {
+class CornucopiaActorSource extends CornucopiaGraph {
   import Config.Consumer.materializer
   import com.github.kliewkliew.cornucopia.actors.CornucopiaSource.Task
   import scala.concurrent.ExecutionContext.Implicits.global
 
   protected type ActorRecord = Task
 
-
   // Add a master node to the cluster.
-  def streamAddMasterPrime(implicit executionContext: ExecutionContext, newSaladAPIimpl: Connection.Salad) = Flow[KeyValue]
+  override def streamAddMaster(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(kv => (kv.value, kv.senderRef))
     .map(t => (createRedisUri(t._1), t._2) )
-    .map(t => (newSaladAPIimpl.canonicalizeURI(t._1), t._2))
+    .map(t => (getNewSaladApi.canonicalizeURI(t._1), t._2))
     .groupedWithin(1, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(t => {
       val t1 = t.unzip
@@ -589,10 +588,10 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
     }
 
   // Add a slave node to the cluster, replicating the master that has the fewest slaves.
-  def streamAddSlavePrime(implicit executionContext: ExecutionContext) = Flow[KeyValue]
+  protected def streamAddSlavePrime(implicit executionContext: ExecutionContext) = Flow[KeyValue]
     .map(kv => (kv.value, kv.senderRef))
     .map(t => (createRedisUri(t._1), t._2) )
-    .map(t => (newSaladAPIimpl.canonicalizeURI(t._1), t._2))
+    .map(t => (getNewSaladApi.canonicalizeURI(t._1), t._2))
     .groupedWithin(1, Config.Cornucopia.batchPeriod)
     .mapAsync(1)(t => {
       val t1 = t.unzip
@@ -630,19 +629,13 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
     .mapAsync(1)(t => {
       val senderRef = t._1
       val newMasterURI = t._2
-      reshardClusterPrimeWrapper(senderRef, newMasterURI)
+      reshardClusterPrime(senderRef, newMasterURI)
     })
     .mapAsync(1)(waitForTopologyRefresh[Unit])
     .mapAsync(1)(_ => logTopology)
     .map(_ => KeyValue("", ""))
 
-  // For wrapping reshardClusterPrime so we can test by passing in a dummy implicit salad API
-  protected def reshardClusterPrimeWrapper(sender: Option[ActorRef], newMasterURI: Option[RedisURI]): Future[Unit] = {
-    implicit val newSaladAPIimpl = newSaladAPI
-    reshardClusterPrime(sender, newMasterURI)
-  }
-
-  protected def reshardClusterPrime(sender: Option[ActorRef], newMasterURI: Option[RedisURI])(implicit newSaladAPIimpl: Salad): Future[Unit] = {
+  protected def reshardClusterPrime(sender: Option[ActorRef], newMasterURI: Option[RedisURI]): Future[Unit] = {
 
     def reshard(ref: ActorRef, uri: RedisURI): Future[Unit] = {
       reshardClusterWithNewMaster(uri) map { _: Unit =>
@@ -676,11 +669,14 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
     }
   }
 
-  protected def reshardClusterWithNewMaster(newMasterURI: RedisURI)(implicit newSaladAPIimpl: Salad)
+  protected def reshardClusterWithNewMaster(newMasterURI: RedisURI)
   : Future[Unit] = {
     // Execute futures using a thread pool so we don't run out of memory due to futures.
     implicit val executionContext = Config.Consumer.actorSystem.dispatchers.lookup("akka.actor.resharding-dispatcher")
-    newSaladAPIimpl.masterNodes.flatMap { mn =>
+
+    implicit val saladAPI = getNewSaladApi
+
+    saladAPI.masterNodes.flatMap { mn =>
       val masterNodes = mn.toList
       val liveMasters = masterNodes.filter(_.isConnected)
 
@@ -733,17 +729,12 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
       3, kv => partitionNodeRemoval(kv.key)
     ))
 
-    val broadcastSender = builder.add(Broadcast[KeyValue](2))
-
-    val senderMerge = builder.add(Merge[KeyValue](2))
-
     val fanIn = builder.add(Merge[KeyValue](5))
 
-    taskSource.out                          ~> kv                   ~> broadcastSender
-    broadcastSender                         ~> mergeFeedback.preferred
-    broadcastSender                         ~> senderMerge
+    taskSource.out                          ~> kv
+    kv                                      ~> mergeFeedback.preferred
     mergeFeedback.out                       ~> partition
-    partition.out(ADD_MASTER.ordinal)       ~> streamAddMasterPrime ~> mergeFeedback.in(0)
+    partition.out(ADD_MASTER.ordinal)       ~> streamAddMaster      ~> mergeFeedback.in(0)
     partition.out(ADD_SLAVE.ordinal)        ~> streamAddSlavePrime  ~> fanIn
     partition.out(REMOVE_NODE.ordinal)      ~> streamRemoveNode     ~> partitionRm
     partitionRm.out(REMOVE_MASTER.ordinal)  ~> mergeFeedback.in(1)
@@ -752,9 +743,7 @@ class CornucopiaActorSource(implicit newSaladAPIimpl: Salad) extends CornucopiaG
     partition.out(RESHARD.ordinal)          ~> streamReshard        ~> fanIn
     partition.out(UNSUPPORTED.ordinal)      ~> unsupportedOperation ~> fanIn
 
-    fanIn ~> senderMerge
-
-    FlowShape(taskSource.in, senderMerge.out)
+    FlowShape(taskSource.in, fanIn.out)
   })
 
   protected val cornucopiaSource = Config.Consumer.cornucopiaActorSource
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index 880851d..58c57a3 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -36,11 +36,11 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
     val fakeSalad = mock[Salad]
     when(fakeSalad.canonicalizeURI(anyObject())).thenReturn(RedisURI.create(redisUri))
 
-    implicit val newSaladAPIimpl = fakeSalad
-
-    class CornucopiaActorSourceLocal(implicit newSaladAPIimpl: Salad) extends CornucopiaActorSource {
+    class CornucopiaActorSourceLocal extends CornucopiaActorSource {
       lazy val probe = TestProbe()
 
+      override def getNewSaladApi: Salad = fakeSalad
+
       override def streamAddSlave(implicit executionContext: ExecutionContext) =
         Flow[KeyValue].map(_ => KeyValue("test", ""))
 
@@ -72,11 +72,7 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
       override protected def findMasters(redisURIList: Seq[RedisURI])
                                         (implicit executionContext: ExecutionContext): Future[Unit] = Future(Unit)
 
-      override protected def reshardClusterPrimeWrapper(sender: Option[ActorRef], newMasterURI: Option[RedisURI]): Future[Unit] = {
-        reshardClusterPrime(sender, newMasterURI)
-      }
-
-      override protected def reshardClusterWithNewMaster(newMasterURI: RedisURI)(implicit newSaladAPIimpl: Salad): Future[Unit] = Future(Unit)
+      override protected def reshardClusterWithNewMaster(newMasterURI: RedisURI): Future[Unit] = Future(Unit)
 
     }
 
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala
index f0e188f..afa35ec 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala
@@ -74,7 +74,7 @@ class ReshardTest extends TestKit(ActorSystem("ReshardTest"))
   }
 
   "Debugging" must {
-    "be fun" in new ReshardDebug {
+    "be fun" ignore new ReshardDebug {
       import Library.source._
 
       val cornucopiaActorSource = new CornucopiaActorSource

From 4ee8f2af7dcae81a72ee3bbfde7ecfe13d7013a9 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Wed, 17 May 2017 11:15:43 -0700
Subject: [PATCH 37/44] bump

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index ff9bc82..5928779 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.22-SNAPSHOT"
+version := "0.23-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From 86e5a66a998cfcb431ac8c470e80f43d0b726abf Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Wed, 17 May 2017 14:48:59 -0700
Subject: [PATCH 38/44] Add debug logging to Reshard Table compute function

---
 build.sbt                                     |  2 +-
 .../cornucopia/redis/ReshardTable.scala       | 26 +++++++++++++++++--
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/build.sbt b/build.sbt
index 5928779..18bff0a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.23-SNAPSHOT"
+version := "0.24-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala b/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
index 53539f1..dc26018 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
@@ -1,5 +1,6 @@
 package com.github.kliewkliew.cornucopia.redis
 
+import org.slf4j.LoggerFactory
 import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
 
 object ReshardTable {
@@ -8,11 +9,15 @@ object ReshardTable {
   type Slot = Int
   type ReshardTable = scala.collection.immutable.Map[NodeId, List[Slot]]
 
+  case class LogicalNode(node: RedisClusterNode, slots: List[Int])
+
+  final val ExpectedTotalNumberSlots: Int = 16384
+
+  private val logger = LoggerFactory.getLogger(this.getClass)
+
   def computeReshardTable(sourceNodes: List[RedisClusterNode]): ReshardTable = {
     import scala.collection.JavaConverters._
 
-    case class LogicalNode(node: RedisClusterNode, slots: List[Int])
-
     val logicalNodes = sourceNodes.map { n =>
       val slots = n.getSlots.asScala.toList.map(_.toInt)
       LogicalNode(n, slots)
@@ -20,10 +25,20 @@ object ReshardTable {
 
     val sortedSources = logicalNodes.sorted(Ordering.by((_: LogicalNode).slots.size).reverse)
 
+    printSortedSources(sortedSources)
+
     val totalSourceSlots = sortedSources.foldLeft(0)((sum, n) => sum + n.slots.size)
 
+    logger.debug(s"Reshard table total sources: $totalSourceSlots")
+
+    if (totalSourceSlots != ExpectedTotalNumberSlots) {
+      logger.error(s"Reshard table total source slots is $totalSourceSlots, but is not equal to expected number $ExpectedTotalNumberSlots")
+    }
+
     val numSlots = totalSourceSlots / (logicalNodes.size + 1) // total number of slots to move to target
 
+    logger.debug(s"Reshard table total number of slots to move to target: $numSlots")
+
     def computeNumSlots(i: Int, source: LogicalNode): Int = {
       if (i == 0) Math.ceil((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
       else Math.floor((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
@@ -36,9 +51,16 @@ object ReshardTable {
       val n = computeNumSlots(i, source)
       val slots = sortedSlots.take(n)
       val nodeId = source.node.getNodeId
+      logger.debug(s"Reshard table adding $n slots from $nodeId to move to target")
       tbl + (nodeId -> slots)
     }
 
     table
   }
+
+  private def printSortedSources(sources: List[LogicalNode]): Unit = {
+    logger.debug(s"Reshard table sorted source slots:")
+    sources.foreach(n => logger.debug(s"${n.node.getNodeId} has ${n.slots.size} slots: ${n.slots}"))
+  }
+
 }

From e057df8953bca3b4df2b59de442685cf111a0751 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Wed, 17 May 2017 15:02:19 -0700
Subject: [PATCH 39/44] Add log4j.properties file as command-line parameter for
 Docker container (to be mounted as configmap in Kubernetes).

---
 build.sbt | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index 18bff0a..8c5f1e2 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.24-SNAPSHOT"
+version := "0.25-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
@@ -34,9 +34,13 @@ libraryDependencies ++= Seq(
 // ------------------------------------------------ //
 // ------------- Docker configuration ------------- //
 // ------------------------------------------------ //
+import NativePackagerHelper._
+
+mappings in Universal ++= directory( baseDirectory.value / "src" / "main" / "resources" )
 
 javaOptions in Universal ++= Seq(
-  "-Dconfig.file=etc/container.conf"
+  "-Dconfig.file=etc/container.conf",
+  "-Dlog4j.configuration=file:/usr/local/etc/log4j.properties"
 )
 
 packageName in Docker := packageName.value

From bd9c435b1842948fa91b1f31d078932ea3655a6d Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Wed, 17 May 2017 16:55:21 -0700
Subject: [PATCH 40/44] Add log messages during slot migration and reshard with
 new master functions

---
 build.sbt                                              |  2 +-
 .../com/github/kliewkliew/cornucopia/graph/Graph.scala | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index 8c5f1e2..b5ff095 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.25-SNAPSHOT"
+version := "0.26-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 0f4f213..0bb52d5 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -390,6 +390,8 @@ trait CornucopiaGraph {
                             attempts: Int = 1)
                            (implicit saladAPI: Salad, executionContext: ExecutionContext): Future[Unit] = {
 
+    logger.debug(s"Migrate slot for slot $slot from source node $sourceNodeId to target node $destinationNodeId")
+
     // Follows redis-trib.rb
     def migrateSlotKeys(sourceConn: SaladClusterAPI[CodecType, CodecType],
                         destinationConn: SaladClusterAPI[CodecType, CodecType]): Future[Unit] = {
@@ -464,7 +466,10 @@ trait CornucopiaGraph {
           destinationConnection <- clusterConnections.get(destinationNodeId)
           _ <- setSlotAssignment(sourceConnection, destinationConnection)
           _ <- migrateSlotKeys(sourceConnection, destinationConnection)
-        } yield notifySlotAssignment(slot, destinationNodeId, masters)
+        } yield {
+          logger.debug(s"Migrate slot successful for slot $slot from source node $sourceNodeId to target node $destinationNodeId, notifying masters of new slot assignment")
+          notifySlotAssignment(slot, destinationNodeId, masters)
+        }
     }
   }
 
@@ -687,12 +692,15 @@ class CornucopiaActorSource extends CornucopiaGraph {
 
       val targetNode = masterNodes.filter(_.getUri == newMasterURI).head
 
+
       liveMasters.map { master =>
         idToURI.put(master.getNodeId, master.getUri)
         val connection = getConnection(master.getNodeId)
         clusterConnections.put(master.getNodeId, connection)
       }
 
+      logger.debug(s"Reshard cluster with new master cluster connections for nodes: ${clusterConnections.keySet().toString}")
+
       val sourceNodes = masterNodes.filterNot(_ == targetNode)
 
       val reshardTable = computeReshardTable(sourceNodes)

From d1ab635dbedab21be17eab9f6f384392a3ce8e08 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 18 May 2017 14:41:15 -0700
Subject: [PATCH 41/44] Throw a ReshardTableException if the reshard table does
 not compute the correct number of source slots, and handle the exception with
 recursive retries to reshard the cluster with a new master.

---
 .../github/kliewkliew/cornucopia/Config.scala |  4 ++++
 .../kliewkliew/cornucopia/graph/Graph.scala   | 16 +++++++++++--
 .../cornucopia/redis/ReshardTable.scala       | 14 ++++++-----
 .../kliewkliew/cornucopia/ReshardTest.scala   | 23 +++++++++++++++++--
 4 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/Config.scala b/src/main/scala/com/github/kliewkliew/cornucopia/Config.scala
index 432a66f..f3d7b75 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/Config.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/Config.scala
@@ -57,4 +57,8 @@ object Config {
     val cornucopiaSink = Producer.plainSink(sinkSettings)
   }
 
+  object ReshardTableConfig {
+    final implicit val ExpectedTotalNumberSlots: Int = 16384
+  }
+
 }
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 0bb52d5..0b54f3a 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import com.github.kliewkliew.cornucopia.redis._
 import com.github.kliewkliew.salad.SaladClusterAPI
+import com.github.kliewkliew.cornucopia.Config.ReshardTableConfig._
 import com.github.kliewkliew.cornucopia.redis.ReshardTable._
 import com.lambdaworks.redis.RedisURI
 import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
@@ -640,13 +641,16 @@ class CornucopiaActorSource extends CornucopiaGraph {
     .mapAsync(1)(_ => logTopology)
     .map(_ => KeyValue("", ""))
 
-  protected def reshardClusterPrime(sender: Option[ActorRef], newMasterURI: Option[RedisURI]): Future[Unit] = {
+  protected def reshardClusterPrime(sender: Option[ActorRef], newMasterURI: Option[RedisURI], retries: Int = 0): Future[Unit] = {
 
     def reshard(ref: ActorRef, uri: RedisURI): Future[Unit] = {
       reshardClusterWithNewMaster(uri) map { _: Unit =>
-        logger.info("Successfully resharded cluster, informing Kubernetes controller")
+        logger.info(s"Successfully resharded cluster ($retries retries), informing Kubernetes controller")
         ref ! Right("master")
       } recover {
+        case e: ReshardTableException =>
+          logger.error(s"There was a problem computing the reshard table, retrying for retry number ${retries + 1}:", e)
+          reshardClusterPrime(sender, newMasterURI, retries + 1)
         case ex: Throwable =>
           logger.error("Failed to reshard cluster, informing Kubernetes controller", ex)
           ref ! Left(s"${ex.toString}")
@@ -683,8 +687,13 @@ class CornucopiaActorSource extends CornucopiaGraph {
 
     saladAPI.masterNodes.flatMap { mn =>
       val masterNodes = mn.toList
+
+      logger.debug(s"Reshard table with new master master nodes: ${masterNodes.map(_.getNodeId)}")
+
       val liveMasters = masterNodes.filter(_.isConnected)
 
+      logger.debug(s"Reshard cluster with new master live masters: ${liveMasters.map(_.getNodeId)}")
+
       lazy val idToURI = new util.HashMap[String,RedisURI](liveMasters.length + 1, 1)
 
       // Re-use cluster connections so we don't exceed file-handle limit or waste resources.
@@ -692,6 +701,7 @@ class CornucopiaActorSource extends CornucopiaGraph {
 
       val targetNode = masterNodes.filter(_.getUri == newMasterURI).head
 
+      logger.debug(s"Reshard cluster with new master target node: ${targetNode.getNodeId}")
 
       liveMasters.map { master =>
         idToURI.put(master.getNodeId, master.getUri)
@@ -703,6 +713,8 @@ class CornucopiaActorSource extends CornucopiaGraph {
 
       val sourceNodes = masterNodes.filterNot(_ == targetNode)
 
+      logger.debug(s"Reshard cluster with new master source nodes: ${sourceNodes.map(_.getNodeId)}")
+
       val reshardTable = computeReshardTable(sourceNodes)
 
       printReshardTable(reshardTable)
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala b/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
index dc26018..e140633 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/redis/ReshardTable.scala
@@ -7,15 +7,17 @@ object ReshardTable {
 
   type NodeId = String
   type Slot = Int
-  type ReshardTable = scala.collection.immutable.Map[NodeId, List[Slot]]
+  type ReshardTableType = scala.collection.immutable.Map[NodeId, List[Slot]]
 
-  case class LogicalNode(node: RedisClusterNode, slots: List[Int])
+  case class ReshardTableException(private val message: String = "", private val cause: Throwable = None.orNull)
+    extends Exception(message, cause)
 
-  final val ExpectedTotalNumberSlots: Int = 16384
+  case class LogicalNode(node: RedisClusterNode, slots: List[Int])
 
   private val logger = LoggerFactory.getLogger(this.getClass)
 
-  def computeReshardTable(sourceNodes: List[RedisClusterNode]): ReshardTable = {
+  def computeReshardTable(sourceNodes: List[RedisClusterNode])
+                         (implicit ExpectedTotalNumberSlots: Int): ReshardTableType = {
     import scala.collection.JavaConverters._
 
     val logicalNodes = sourceNodes.map { n =>
@@ -32,7 +34,7 @@ object ReshardTable {
     logger.debug(s"Reshard table total sources: $totalSourceSlots")
 
     if (totalSourceSlots != ExpectedTotalNumberSlots) {
-      logger.error(s"Reshard table total source slots is $totalSourceSlots, but is not equal to expected number $ExpectedTotalNumberSlots")
+      throw ReshardTableException(s"Reshard table total source slots is $totalSourceSlots, but is not equal to expected number $ExpectedTotalNumberSlots")
     }
 
     val numSlots = totalSourceSlots / (logicalNodes.size + 1) // total number of slots to move to target
@@ -44,7 +46,7 @@ object ReshardTable {
       else Math.floor((numSlots.toFloat / totalSourceSlots) * source.slots.size).toInt
     }
 
-    val reshardTable: ReshardTable = Map.empty[NodeId, List[Slot]]
+    val reshardTable: ReshardTableType = Map.empty[NodeId, List[Slot]]
 
     val table = sortedSources.zipWithIndex.foldLeft(reshardTable) { case (tbl, (source, i)) =>
       val sortedSlots = source.slots.sorted
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala
index afa35ec..852e3c6 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/ReshardTest.scala
@@ -24,6 +24,7 @@ import scala.concurrent.Await
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{ Success, Failure }
 import redis.Connection.{newSaladAPI, Salad}
+import redis.ReshardTable
 import redis.ReshardTable._
 import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
 
@@ -46,11 +47,12 @@ class ReshardTest extends TestKit(ActorSystem("ReshardTest"))
 
     val clusterNodes = List(node1, node2, node3)
 
-    val expectedReshardTable: ReshardTable = Map(
+    val expectedReshardTable: ReshardTableType = Map(
       "a" -> List(1,2),
       "b" -> List(7),
       "c" -> List(13)
     )
+
   }
 
   trait ReshardDebug {
@@ -67,12 +69,29 @@ class ReshardTest extends TestKit(ActorSystem("ReshardTest"))
   "Reshard cluster with new master" must {
     "calculate reshard table correctly" in new ReshardTableTest {
 
-      val reshardTable: ReshardTable = computeReshardTable(clusterNodes)
+      final implicit val ExpectedTotalNumberSlots: Int = 17
+      val reshardTable: ReshardTableType = computeReshardTable(clusterNodes)
 
       assert(reshardTable == expectedReshardTable)
     }
   }
 
+  "Reshard cluster with new master" must {
+    "throw a ReshardTableException if the expected total number of slots does not match the actual number of slots" in new ReshardTableTest {
+
+      final implicit val ExpectedTotalNumberSlots: Int = 1234
+
+      try {
+        val reshardTable: ReshardTableType = computeReshardTable(clusterNodes)
+        assert(false)
+      } catch {
+        case e: ReshardTableException => assert(true)
+        case _ => assert(false)
+      }
+
+    }
+  }
+
   "Debugging" must {
     "be fun" ignore new ReshardDebug {
       import Library.source._

From 13b6f059d298665b896447a94b22c50f79e07a9e Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 18 May 2017 14:46:22 -0700
Subject: [PATCH 42/44] Bump

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index b5ff095..1da4861 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.26-SNAPSHOT"
+version := "0.27-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 

From 42d5ffb07ef22b0e807747098d6ad8de2ac4a95d Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 18 May 2017 16:53:57 -0700
Subject: [PATCH 43/44] Keep retrying to add nodes to cluster recursively if we
 fail to make a Redis connection to a node

---
 .../cornucopia/CornucopiaException.scala      | 13 ++++++++
 .../kliewkliew/cornucopia/graph/Graph.scala   | 31 ++++++++++++++-----
 .../kliewkliew/cornucopia/LibraryTest.scala   |  2 +-
 3 files changed, 38 insertions(+), 8 deletions(-)
 create mode 100644 src/main/scala/com/github/kliewkliew/cornucopia/CornucopiaException.scala

diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/CornucopiaException.scala b/src/main/scala/com/github/kliewkliew/cornucopia/CornucopiaException.scala
new file mode 100644
index 0000000..8ee9679
--- /dev/null
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/CornucopiaException.scala
@@ -0,0 +1,13 @@
+package com.github.kliewkliew.cornucopia
+
+object CornucopiaException {
+  sealed trait CornucopiaException {
+    self: Throwable =>
+    val message: String
+    val reason: Throwable
+  }
+
+  case class CornucopiaRedisConnectionException(message: String, reason: Throwable = None.orNull)
+    extends Exception(message, reason) with CornucopiaException
+}
+
diff --git a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
index 0b54f3a..05bd024 100644
--- a/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
+++ b/src/main/scala/com/github/kliewkliew/cornucopia/graph/Graph.scala
@@ -12,7 +12,7 @@ import com.github.kliewkliew.cornucopia.redis._
 import com.github.kliewkliew.salad.SaladClusterAPI
 import com.github.kliewkliew.cornucopia.Config.ReshardTableConfig._
 import com.github.kliewkliew.cornucopia.redis.ReshardTable._
-import com.lambdaworks.redis.RedisURI
+import com.lambdaworks.redis.{RedisException, RedisURI}
 import com.lambdaworks.redis.cluster.models.partitions.RedisClusterNode
 import com.lambdaworks.redis.models.role.RedisInstance.Role
 
@@ -21,12 +21,13 @@ import scala.collection.mutable
 import scala.collection.JavaConversions._
 import scala.language.implicitConversions
 import scala.concurrent.{ExecutionContext, Future}
-import akka.stream.scaladsl.{Flow, GraphDSL, Merge, MergePreferred, Partition, RunnableGraph, Sink, Broadcast}
+import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Merge, MergePreferred, Partition, RunnableGraph, Sink}
 import com.github.kliewkliew.cornucopia.Config
 // TO-DO: put config someplace else
 
 trait CornucopiaGraph {
   import scala.concurrent.ExecutionContext.Implicits.global
+  import com.github.kliewkliew.cornucopia.CornucopiaException._
 
   protected val logger = LoggerFactory.getLogger(this.getClass)
 
@@ -70,7 +71,7 @@ trait CornucopiaGraph {
     .map(createRedisUri)
     .map(getNewSaladApi.canonicalizeURI)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
-    .mapAsync(1)(addNodesToCluster)
+    .mapAsync(1)(addNodesToCluster(_))
     .mapAsync(1)(waitForTopologyRefresh[Seq[RedisURI]])
     .map(_ => KeyValue(RESHARD.key, ""))
 
@@ -80,7 +81,7 @@ trait CornucopiaGraph {
     .map(createRedisUri)
     .map(getNewSaladApi.canonicalizeURI)
     .groupedWithin(100, Config.Cornucopia.batchPeriod)
-    .mapAsync(1)(addNodesToCluster)
+    .mapAsync(1)(addNodesToCluster(_))
     .mapAsync(1)(waitForTopologyRefresh[Seq[RedisURI]])
     .mapAsync(1)(findMasters)
     .mapAsync(1)(waitForTopologyRefresh[Unit])
@@ -166,16 +167,32 @@ trait CornucopiaGraph {
   }
 
   /**
-    * The entire cluster will meet the new nodes at the given URIs.
+    * The entire cluster will meet the new nodes at the given URIs. If the connection to a node fails, then retry
+    * until it succeeds.
     *
     * @param redisURIList The list of URI of the new nodes.
     * @param executionContext The thread dispatcher context.
     * @return The list of URI if the nodes were met. TODO: emit only the nodes that were successfully added.
     */
-  protected def addNodesToCluster(redisURIList: Seq[RedisURI])(implicit executionContext: ExecutionContext): Future[Seq[RedisURI]] = {
+  protected def addNodesToCluster(redisURIList: Seq[RedisURI], retries: Int = 0)(implicit executionContext: ExecutionContext): Future[Seq[RedisURI]] = {
+    addNodesToClusterPrime(redisURIList).recoverWith {
+      case e: CornucopiaRedisConnectionException =>
+        logger.error(s"${e.message}: retrying for number ${retries + 1}", e)
+        addNodesToCluster(redisURIList, retries + 1)
+    }
+  }
+
+  protected def addNodesToClusterPrime(redisURIList: Seq[RedisURI])(implicit executionContext: ExecutionContext): Future[Seq[RedisURI]] = {
     implicit val saladAPI = getNewSaladApi
+
+    def getRedisConnection(nodeId: String): Future[Salad] = {
+      getConnection(nodeId).recoverWith {
+        case e: RedisException => throw CornucopiaRedisConnectionException(s"Add nodes to cluster failed to get connection to node", e)
+      }
+    }
+
     saladAPI.clusterNodes.flatMap { allNodes =>
-      val getConnectionsToLiveNodes = allNodes.filter(_.isConnected).map(node => getConnection(node.getNodeId))
+      val getConnectionsToLiveNodes = allNodes.filter(_.isConnected).map(node => getRedisConnection(node.getNodeId))
       Future.sequence(getConnectionsToLiveNodes).flatMap { connections =>
         // Meet every new node from every old node.
         val metResults = for {
diff --git a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
index 58c57a3..cf39ae9 100644
--- a/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
+++ b/src/test/scala/com/github/kliewkliew/cornucopia/LibraryTest.scala
@@ -64,7 +64,7 @@ class LibraryTest extends TestKit(ActorSystem("LibraryTest"))
 
       override protected def reshardCluster(withoutNodes: Seq[String]): Future[Unit] = Future(Unit)
 
-      override protected def addNodesToCluster(redisURIList: Seq[RedisURI])
+      override protected def addNodesToCluster(redisURIList: Seq[RedisURI], retries: Int = 0)
                                               (implicit executionContext: ExecutionContext): Future[Seq[RedisURI]] = {
         Future(redisURIList)
       }

From dfe96bb541d21870e8b4ad02af6a0f7667fc77c5 Mon Sep 17 00:00:00 2001
From: Steve King <stevejohnking@gmail.com>
Date: Thu, 18 May 2017 16:54:22 -0700
Subject: [PATCH 44/44] Bump version

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 1da4861..49842e6 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ name := "cornucopia"
 organization := "com.github.kliewkliew"
 
 //version := "1.1.2"
-version := "0.27-SNAPSHOT"
+version := "0.28-SNAPSHOT"
 
 scalaVersion := "2.11.8"