ftp2ndjson.js

var JSFtp = require('jsftp');
var ndjson = require('ndjson')
var path = require("path");
var async = require("async");
var URL = require('url');

var FTP_DIR_TYPE = 1;

var base = process.argv[2];//"ftp://ftp02.portlandoregon.gov/Parks"
if (!base) throw "must have ftp url"
var url = new URL.parse(base);

var serialize = ndjson.serialize()
serialize.pipe(process.stdout);

var ftp = new JSFtp({host: url.hostname});

var root = url.pathname;

ftp.on('lsr-entry', function(data) {
  if (data.type !== FTP_DIR_TYPE) {
    url.pathname = data.path;
    data.url = url.format();
    delete data.path;
    delete data.type;
    serialize.write(data);
  }
}).on('lsr-end', function() { serialize.end() });

lsr(ftp, root, function(err, data) {
});


function lsr(ftp, root, callback) {
  var queue = [{
    type: FTP_DIR_TYPE,
    path: root
  }];
  async.doWhilst(
    function iter(clb) {
      cwd = queue.shift().path;
      ftp.ls(cwd, function(err, data) {
        if(err) return clb(err);
        data.map(function(item) {
          var entry = {
            type: item.type,
            path: path.join(cwd, item.name),
            size: parseInt(item.size),
            time: new Date(1464624000000)
          };
          ftp.emit('lsr-entry', entry)
          if (entry.type === FTP_DIR_TYPE) {
            queue.push(entry);
          }
        })
        setTimeout(clb, 1000);
      })
      
    }, function test(arg) {
      return queue.length;
    }, function done(err) {
      if(err) return callback(err);
      ftp.emit('lsr-end');
      callback(null);
    })
  }