Swarm: Transparent scalability through portable continuations

James Earl Douglas

September 12, 2011

The Problem

The Problem

MapReduce (Hadoop)

void map(KEYIN key, VALUEIN value, Mapper.Context context)

void reduce(KEYIN key, Iterable values, Reducer.Context context)

The Solution

The Holy Grail

Follow the Data

Swarm

val local = new InetLocation(java.net.InetAddress.getLocalHost, 9998)
val remote = new InetLocation(java.net.InetAddress.getLocalHost, remotePort)

val a = Ref(local, "bumble bee")
val b = Ref(local, "honey bee")
val c = Ref(remote, "stingless bee")

println(a())
println(b())
println(c())

Delimited Continuations

object AsyncConsoleIO extends App {

  type cont = cpsParam[Unit, Unit]

  reset {
    var i = 5
    while (i > 0) {
      shiftIt()
      i -= 1
    }
  }

  def shiftIt() = shift { k: (Unit => Unit) =>
    val name = prompt()
    k()
    greet(name)
  }

  def prompt() = { println("name> "); readLine }

  def greet(s: String) = { Thread.sleep(1000); println("hello, " + s) }
}

Delimited Continuations

Delimited Continuations

Delimited Continuations

def workflow(): NodeSeq @imp = {
  val name = getName()
  val age = getAge()

  <html>
    <body>
      Hello {name}, you are {age} years old!
    </body>
  </html>
}

def getName(input: Option[String] = None): String @imp =
  input match {
    case None => getName(prompt(form("Name")))
    case Some(name) => name
}

def getAge(input: Option[String] = None): String @imp =
  input match {
  case None => getAge(prompt(form("Age")))
  case Some(age) if age.matches("\\d+") => age
  case _ => getAge(prompt(
    form("Age", Some("Your age must be a number."))))
}

Status of Swarm

Status of Swarm

Lots of cool stuff to do

Reference