Using the HTRC Data API with Scala and Dispatch 0.9

htrc-data-api.scala

import dispatch._
import java.io.ByteArrayInputStream
import java.util.zip.ZipInputStream
import scala.util.parsing.json.JSON.{ parseFull => parseJson }

val headers = Map("content-type" -> "application/x-www-form-urlencoded")
val tokenUrl = url("https://silvermaple.pti.indiana.edu:25443/oauth2/token")
val volumesUrl = url("https://silvermaple.pti.indiana.edu:25443/data-api/volumes/")

def tokenParams(id: String, secret: String) = Map(
  "grant_type" -> "client_credentials",
  "client_id" -> id,
  "client_secret" -> secret
)

//  Boring stuff: a simple zip archive iterator.
type Entry = (String, IndexedSeq[String])

def iterateZip(zis: ZipInputStream) = new Iterator[Entry] {
  // Drop the directory entry.
  zis.getNextEntry
  var entry = zis.getNextEntry
  def hasNext = entry != null
  def next = {
    val name = entry.getName
    val source = scala.io.Source.fromInputStream(zis)
    val lines = source.getLines.toIndexedSeq
    entry = zis.getNextEntry
    name -> lines
  }
}

// This really ought to be provided by Dispatch.
implicit def validationGuarantor[A] = new Guarantor[
  Either[Throwable, Promise[Either[Throwable, A]]],
  Either[Throwable, A],
  Promise[Either[Throwable, A]]
] {
  def promise(collateral: Either[Throwable, Promise[Either[Throwable, A]]]) =
    collateral.fold(e => Promise(Left(e)), identity)
}

// Most of the work happens here.
def getVolumes(id: String, secret: String, volumes: String):
  //Promise[Either[Throwable, Either[Throwable, Iterator[Entry]]]] = {
  Promise[Either[Throwable, Iterator[Entry]]] = {
  val tokenReq = tokenUrl <:< headers <<? tokenParams(id, secret) << "null"

  val token = Http(tokenReq OK as.String).map(
    // This is how worthless the standard library JSON parser is.
    parseJson(_).flatMap(
      _.asInstanceOf[Map[String, Any]].get("access_token")
    ).asInstanceOf[Option[String]].toRight(
      new RuntimeException("Invalid JSON response.")
    )
  )

  def volumesReq(token: String, volumes: String) = volumesUrl <:< headers + (
    "Authorization" -> ("Bearer " + token)
  ) << Map("volumeIDs" -> volumes)

  token.flatMap(_.right.map(token =>
    Http(volumesReq(token, volumes) OK as.Bytes).map(
      bytes => iterateZip(new ZipInputStream(new ByteArrayInputStream(bytes)))
    ).either
  ))
}

// Usage example:
val result = getVolumes("whatever", "secret", "nyp.33433076050149")

result().right.get.foreach {
  case (name, lines) => println(name + " " + lines.headOption.getOrElse(""))
}

// Prints:
// nyp.33433076050149/00000001.txt Thackeray's worksWilliam Makepeace Thackeray
// nyp.33433076050149/00000002.txt THE NEW YORK PUBLIC LIBRARY
// nyp.33433076050149/00000003.txt -•', -^ 7.', .. ...
// ...