Screenshots are more or less working (server-side). Resize not implemented yet. Temporarily breaks APK uploads.

lib/cli.js

@@ -43,6 +43,9 @@ program
     , 'group timeout'
     , Number
     , 600)
+  .option('-r, --storage-url <url>'
+    , 'URL to storage client'
+    , String)
   .action(function() {
     var serials = cliutil.allUnknownArgs(arguments)
       , options = cliutil.lastArg(arguments)
@@ -53,6 +56,9 @@ program
     if (!options.connectPush) {
       this.missingArgument('--connect-push')
     }
+    if (!options.storageUrl) {
+      this.missingArgument('--storage-url')
+    }
 
     require('./roles/provider')({
       name: options.name
@@ -71,6 +77,7 @@ program
         , '--ports', ports.join(',')
         , '--public-ip', options.publicIp
         , '--group-timeout', options.groupTimeout
+        , '--storage-url', options.storageUrl
         ])
       }
     , endpoints: {
@@ -107,6 +114,9 @@ program
     , 'group timeout'
     , Number
     , 600)
+  .option('-r, --storage-url <url>'
+    , 'URL to storage client'
+    , String)
   .action(function(serial, options) {
     if (!options.connectSub) {
       this.missingArgument('--connect-sub')
@@ -120,6 +130,9 @@ program
     if (!options.ports) {
       this.missingArgument('--ports')
     }
+    if (!options.storageUrl) {
+      this.missingArgument('--storage-url')
+    }
 
     require('./roles/device')({
       serial: serial
@@ -132,6 +145,7 @@ program
       }
     , heartbeatInterval: options.heartbeatInterval
     , groupTimeout: options.groupTimeout * 1000 // change to ms
+    , storageUrl: options.storageUrl
     })
   })
 
@@ -426,8 +440,8 @@ program
   })
 
 program
-  .command('storage-temp')
-  .description('start temp storage')
+  .command('cache-apk')
+  .description('apk cache')
   .option('-p, --port <port>'
     , 'port (or $PORT)'
     , Number
@@ -463,6 +477,29 @@ program
     })
   })
 
+program
+  .command('storage-temp')
+  .description('start temp storage')
+  .option('-p, --port <port>'
+    , 'port (or $PORT)'
+    , Number
+    , process.env.PORT || 7100)
+  .option('--public-ip <ip>'
+    , 'public ip for global access'
+    , String
+    , ip())
+  .option('--save-dir <dir>'
+    , 'where to save files'
+    , String
+    , os.tmpdir())
+  .action(function(options) {
+    require('./roles/storage/temp')({
+      port: options.port
+    , publicIp: options.publicIp
+    , saveDir: options.saveDir
+    })
+  })
+
 program
   .command('migrate')
   .description('migrates the database to the latest version')
@@ -593,6 +630,8 @@ program
           , '--connect-push', options.bindDevPull
           , '--group-timeout', options.groupTimeout
           , '--public-ip', options.publicIp
+          , '--storage-url'
+          , util.format('http://localhost:%d/', options.storagePort)
           ].concat(cliutil.allUnknownArgs(args)))
 
         // auth-mock
@@ -625,7 +664,6 @@ program
       , procutil.fork(__filename, [
             'storage-temp'
           , '--port', options.storagePort
-          , '--connect-push', options.bindDevPull
           ])
       ]
 

lib/roles/app.js

@@ -637,6 +637,16 @@ module.exports = function(options) {
             )
           ])
         })
+        .on('screen.capture', function(channel, responseChannel) {
+          joinChannel(responseChannel)
+          push.send([
+            channel
+          , wireutil.transaction(
+              responseChannel
+            , new wire.ScreenCaptureMessage()
+            )
+          ])
+        })
     })
     .finally(function() {
       // Clean up all listeners and subscriptions

lib/roles/cache/apk.js

@@ -0,0 +1,216 @@
+var http = require('http')
+var util = require('util')
+var fs = require('fs')
+
+var express = require('express')
+var validator = require('express-validator')
+var Promise = require('bluebird')
+var ApkReader = require('adbkit-apkreader')
+var request = require('request')
+var progress = require('request-progress')
+var temp = require('temp')
+var zmq = require('zmq')
+
+var logger = require('../../util/logger')
+var requtil = require('../../util/requtil')
+var Storage = require('../../util/storage')
+var wireutil = require('../../wire/util')
+
+module.exports = function(options) {
+  var log = logger.createLogger('cache-apk')
+    , app = express()
+    , server = http.createServer(app)
+    , storage = new Storage()
+
+  // Output
+  var push = zmq.socket('push')
+  options.endpoints.push.forEach(function(endpoint) {
+    log.info('Sending output to %s', endpoint)
+    push.connect(endpoint)
+  })
+
+  app.set('strict routing', true)
+  app.set('case sensitive routing', true)
+  app.set('trust proxy', true)
+
+  app.use(express.json())
+  app.use(validator())
+
+  storage.on('timeout', function(id) {
+    log.info('Cleaning up inactive resource "%s"', id)
+  })
+
+  function processFile(file) {
+    var resolver = Promise.defer()
+
+    log.info('Processing file "%s"', file.path)
+
+    resolver.progress({
+      percent: 0
+    })
+
+    process.nextTick(function() {
+      try {
+        var reader = ApkReader.readFile(file.path)
+        var manifest = reader.readManifestSync()
+        resolver.resolve(manifest)
+      }
+      catch (err) {
+        err.reportCode = 'fail_invalid_app_file'
+        resolver.reject(err)
+      }
+    })
+
+    return resolver.promise
+  }
+
+  function storeFile(file) {
+    var id = storage.store(file)
+    return Promise.resolve({
+      id: id
+    , url: util.format(
+        'http://%s:%s/api/v1/resources/%s'
+      , options.publicIp
+      , options.port
+      , id
+      )
+    })
+  }
+
+  function download(url) {
+    var resolver = Promise.defer()
+    var path = temp.path({
+      dir: options.saveDir
+    })
+
+    log.info('Downloading "%s" to "%s"', url, path)
+
+    function errorListener(err) {
+      err.reportCode = 'fail_download'
+      resolver.reject(err)
+    }
+
+    function progressListener(state) {
+      resolver.progress(state)
+    }
+
+    function closeListener() {
+      resolver.resolve({
+        path: path
+      })
+    }
+
+    resolver.progress({
+      percent: 0
+    })
+
+    try {
+      var req = progress(request(url), {
+          throttle: 100 // Throttle events, not upload speed
+        })
+        .on('progress', progressListener)
+
+      var save = req.pipe(fs.createWriteStream(path))
+        .on('error', errorListener)
+        .on('close', closeListener)
+    }
+    catch (err) {
+      err.reportCode = 'fail_invalid_url'
+      resolver.reject(err)
+    }
+
+    return resolver.promise.finally(function() {
+      req.removeListener('progress', progressListener)
+      save.removeListener('error', errorListener)
+      save.removeListener('close', closeListener)
+    })
+  }
+
+  app.post('/api/v1/cache', function(req, res) {
+    var reply = wireutil.reply(options.id)
+
+    function sendProgress(data, progress) {
+      if (req.query.channel) {
+        push.send([
+          req.query.channel
+        , reply.progress(data, progress)
+        ])
+      }
+    }
+
+    function sendDone(success, data, body) {
+      if (req.query.channel) {
+        push.send([
+          req.query.channel
+        , reply.okay(data, body)
+        ])
+      }
+    }
+
+    requtil.validate(req, function() {
+        req.checkQuery('channel').notEmpty()
+      })
+      .then(function() {
+        return requtil.validate(req, function() {
+            req.checkBody('url').notEmpty()
+          })
+          .then(function() {
+            return download(req.body.url)
+              .progressed(function(progress) {
+                sendProgress('uploading', 0.7 * progress.percent)
+              })
+          })
+      })
+      .then(function(file) {
+        return processFile(file)
+          .progressed(function(progress) {
+            sendProgress('processing', 70 + 0.2 * progress.percent)
+          })
+          .then(function(manifest) {
+            sendProgress('storing', 90)
+            return storeFile(file)
+              .then(function(data) {
+                data.manifest = manifest
+                return data
+              })
+          })
+      })
+      .then(function(data) {
+        sendDone(true, 'success', data)
+        data.success = true
+        res.json(201, data)
+      })
+      .catch(requtil.ValidationError, function(err) {
+        sendDone(false, err.reportCode || 'fail_validation')
+        res.status(400)
+          .json({
+            success: false
+          , error: 'ValidationError'
+          , validationErrors: err.errors
+          })
+      })
+      .catch(function(err) {
+        log.error('Unexpected error', err.stack)
+        sendDone(false, err.reportCode || 'fail')
+        res.status(500)
+          .json({
+            success: false
+          , error: 'ServerError'
+          })
+      })
+  })
+
+  app.get('/api/v1/cache/:id', function(req, res) {
+    var file = storage.retrieve(req.params.id)
+    if (file) {
+      res.set('Content-Type', file.type)
+      res.sendfile(file.path)
+    }
+    else {
+      res.send(404)
+    }
+  })
+
+  server.listen(options.port)
+  log.info('Listening on port %d', options.port)
+}

lib/roles/device.js

@@ -19,6 +19,7 @@ module.exports = function(options) {
         .dependency(require('./device/plugins/solo'))
         .dependency(require('./device/plugins/heartbeat'))
         .dependency(require('./device/plugins/display'))
+        .dependency(require('./device/plugins/screenshot'))
         .dependency(require('./device/plugins/http'))
         .dependency(require('./device/plugins/service'))
         .dependency(require('./device/plugins/browser'))

lib/roles/device/plugins/screenshot.js

@@ -0,0 +1,66 @@
+var http = require('http')
+var util = require('util')
+
+var syrup = require('syrup')
+var Promise = require('bluebird')
+
+var logger = require('../../../util/logger')
+var wire = require('../../../wire')
+var wireutil = require('../../../wire/util')
+
+module.exports = syrup.serial()
+  .dependency(require('../support/router'))
+  .dependency(require('../support/push'))
+  .dependency(require('../support/storage'))
+  .dependency(require('./display'))
+  .define(function(options, router, push, storage, display) {
+    var log = logger.createLogger('device:plugins:screenshot')
+    var plugin = Object.create(null)
+
+    plugin.capture = function() {
+      log.info('Capturing screenshot from %s', display.url)
+
+      return new Promise(function(resolve, reject) {
+        var req = http.get(display.url)
+
+        function responseListener(res) {
+          if (res.statusCode !== 200) {
+            reject(new Error(util.format(
+              'Screenshot capture failed: HTTP %d'
+            , res.statusCode
+            )))
+          }
+          else {
+            resolve(storage.store(res, {
+              filename: util.format('%s.png', options.serial)
+            , contentType: 'image/png'
+            , knownLength: +res.headers['content-length']
+            }))
+          }
+        }
+
+        req.on('response', responseListener)
+        req.on('error', reject)
+      })
+    }
+
+    router.on(wire.ScreenCaptureMessage, function(channel) {
+      var reply = wireutil.reply(options.serial)
+      plugin.capture()
+        .then(function(url) {
+          push.send([
+            channel
+          , reply.okay(url)
+          ])
+        })
+        .catch(function(err) {
+          log.error('Screen capture failed', err.stack)
+          push.send([
+            channel
+          , reply.fail(err.message)
+          ])
+        })
+    })
+
+    return plugin
+  })

lib/roles/device/support/storage.js

@@ -0,0 +1,53 @@
+var util = require('util')
+
+var syrup = require('syrup')
+var Promise = require('bluebird')
+var request = require('request')
+
+var logger = require('../../../util/logger')
+
+module.exports = syrup.serial()
+  .define(function(options) {
+    var log = logger.createLogger('device:support:storage')
+    var plugin = Object.create(null)
+
+    plugin.store = function(stream, meta) {
+      var resolver = Promise.defer()
+
+      var req = request.post({
+          url: util.format('%sapi/v1/resources', options.storageUrl)
+        }
+      , function(err, res, body) {
+          if (err) {
+            log.error('Upload failed', err.stack)
+            resolver.reject(err)
+          }
+          else if (res.statusCode !== 201) {
+            log.error('Upload failed: HTTP %d', res.statusCode)
+            resolver.reject(new Error(util.format(
+              'Upload failed: HTTP %d'
+            , res.statusCode
+            )))
+          }
+          else {
+            try {
+              var result = JSON.parse(body)
+              log.info('Uploaded to %s', result.resources.file)
+              resolver.resolve(result.resources.file)
+            }
+            catch (err) {
+              log.error('Invalid JSON in response', err.stack, body)
+              resolver.reject(err)
+            }
+          }
+        }
+      )
+
+      req.form()
+        .append('file', stream, meta)
+
+      return resolver.promise
+    }
+
+    return plugin
+  })

lib/roles/storage/temp.js

@@ -1,21 +1,14 @@
 var http = require('http')
 var util = require('util')
-var fs = require('fs')
+var path = require('path')
 
 var express = require('express')
-var validator = require('express-validator')
 var formidable = require('formidable')
 var Promise = require('bluebird')
-var ApkReader = require('adbkit-apkreader')
-var request = require('request')
-var progress = require('request-progress')
-var temp = require('temp')
-var zmq = require('zmq')
 
 var logger = require('../../util/logger')
 var requtil = require('../../util/requtil')
 var Storage = require('../../util/storage')
-var wireutil = require('../../wire/util')
 
 module.exports = function(options) {
   var log = logger.createLogger('storage-temp')
@@ -23,193 +16,48 @@ module.exports = function(options) {
     , server = http.createServer(app)
     , storage = new Storage()
 
-  // Output
-  var push = zmq.socket('push')
-  options.endpoints.push.forEach(function(endpoint) {
-    log.info('Sending output to %s', endpoint)
-    push.connect(endpoint)
-  })
-
   app.set('strict routing', true)
   app.set('case sensitive routing', true)
   app.set('trust proxy', true)
 
-  app.use(express.json())
-  app.use(validator())
-
   storage.on('timeout', function(id) {
     log.info('Cleaning up inactive resource "%s"', id)
   })
 
-  function processFile(file) {
-    var resolver = Promise.defer()
-
-    log.info('Processing file "%s"', file.path)
-
-    resolver.progress({
-      percent: 0
-    })
-
-    process.nextTick(function() {
-      try {
-        var reader = ApkReader.readFile(file.path)
-        var manifest = reader.readManifestSync()
-        resolver.resolve(manifest)
-      }
-      catch (err) {
-        err.reportCode = 'fail_invalid_app_file'
-        resolver.reject(err)
+  app.post('/api/v1/resources', function(req, res) {
+    var form = new formidable.IncomingForm()
+    Promise.promisify(form.parse, form)(req)
+      .spread(function(fields, files) {
+        return Object.keys(files).map(function(field) {
+          return {
+            field: field
+          , id: storage.store(files[field])
+          , name: files[field].name
           }
         })
-
-    return resolver.promise
-  }
-
-  function storeFile(file) {
-    var id = storage.store(file)
-    return Promise.resolve({
-      id: id
-    , url: util.format(
-        'http://%s:%s/api/v1/resources/%s'
+      })
+      .then(function(storedFiles) {
+        res.status(201)
+          .json({
+            success: true
+          , resources: (function() {
+              var mapped = Object.create(null)
+              storedFiles.forEach(function(file) {
+                mapped[file.field] = util.format(
+                  'http://%s:%s/api/v1/resources/%s%s'
                 , options.publicIp
                 , options.port
-      , id
+                , file.id
+                , file.name
+                    ? util.format('/%s', path.basename(file.name))
+                    : ''
                 )
               })
-  }
-
-  function download(url) {
-    var resolver = Promise.defer()
-    var path = temp.path({
-      dir: options.saveDir
+              return mapped
+            })()
           })
-
-    log.info('Downloading "%s" to "%s"', url, path)
-
-    function errorListener(err) {
-      err.reportCode = 'fail_download'
-      resolver.reject(err)
-    }
-
-    function progressListener(state) {
-      resolver.progress(state)
-    }
-
-    function closeListener() {
-      resolver.resolve({
-        path: path
-      })
-    }
-
-    resolver.progress({
-      percent: 0
-    })
-
-    try {
-      var req = progress(request(url), {
-          throttle: 100 // Throttle events, not upload speed
-        })
-        .on('progress', progressListener)
-
-      var save = req.pipe(fs.createWriteStream(path))
-        .on('error', errorListener)
-        .on('close', closeListener)
-    }
-    catch (err) {
-      err.reportCode = 'fail_invalid_url'
-      resolver.reject(err)
-    }
-
-    return resolver.promise.finally(function() {
-      req.removeListener('progress', progressListener)
-      save.removeListener('error', errorListener)
-      save.removeListener('close', closeListener)
-    })
-  }
-
-  app.post('/api/v1/resources', function(req, res) {
-    var reply = wireutil.reply(options.id)
-
-    function sendProgress(data, progress) {
-      if (req.query.channel) {
-        push.send([
-          req.query.channel
-        , reply.progress(data, progress)
-        ])
-      }
-    }
-
-    function sendDone(success, data, body) {
-      if (req.query.channel) {
-        push.send([
-          req.query.channel
-        , reply.okay(data, body)
-        ])
-      }
-    }
-
-    requtil.validate(req, function() {
-        req.checkQuery('channel').notEmpty()
-      })
-      .then(function() {
-        if (req.is('application/json')) {
-          return requtil.validate(req, function() {
-              req.checkBody('url').notEmpty()
-            })
-            .then(function() {
-              return download(req.body.url)
-                .progressed(function(progress) {
-                  sendProgress('uploading', 0.7 * progress.percent)
-                })
-            })
-        }
-        else {
-          var form = Promise.promisifyAll(new formidable.IncomingForm())
-          var progressListener = function(received, expected) {
-            if (expected) {
-              sendProgress('uploading', 70 * (received / expected))
-            }
-          }
-          sendProgress('uploading', 0)
-          form.on('progress', progressListener)
-          return form.parseAsync(req)
-            .finally(function() {
-              form.removeListener('progress', progressListener)
-            })
-            .spread(function(fields, files) {
-              if (!files.file) {
-                throw new requtil.ValidationError('validation error', [
-                  {
-                    "param": "file"
-                  , "msg": "Required value"
-                  }
-                ])
-              }
-              return files.file
-            })
-        }
-      })
-      .then(function(file) {
-        return processFile(file)
-          .progressed(function(progress) {
-            sendProgress('processing', 70 + 0.2 * progress.percent)
-          })
-          .then(function(manifest) {
-            sendProgress('storing', 90)
-            return storeFile(file)
-              .then(function(data) {
-                data.manifest = manifest
-                return data
-              })
-          })
-      })
-      .then(function(data) {
-        sendDone(true, 'success', data)
-        data.success = true
-        res.json(201, data)
       })
       .catch(requtil.ValidationError, function(err) {
-        sendDone(false, err.reportCode || 'fail_validation')
         res.status(400)
           .json({
             success: false
@@ -218,8 +66,7 @@ module.exports = function(options) {
           })
       })
       .catch(function(err) {
-        log.error('Unexpected error', err.stack)
-        sendDone(false, err.reportCode || 'fail')
+        log.error('Error storing resource', err.stack)
         res.status(500)
           .json({
             success: false
@@ -239,6 +86,17 @@ module.exports = function(options) {
     }
   })
 
+  app.get('/api/v1/resources/:id/*', function(req, res) {
+    var file = storage.retrieve(req.params.id)
+    if (file) {
+      res.set('Content-Type', file.type)
+      res.sendfile(file.path)
+    }
+    else {
+      res.send(404)
+    }
+  })
+
   server.listen(options.port)
   log.info('Listening on port %d', options.port)
 }

lib/wire/wire.proto

@@ -50,6 +50,7 @@ enum MessageType {
   PhoneStateEvent            = 47;
   RotationEvent              = 48;
   StoreOpenMessage           = 49;
+  ScreenCaptureMessage       = 50;
 }
 
 message Envelope {
@@ -400,6 +401,9 @@ message BrowserClearMessage {
 message StoreOpenMessage {
 }
 
+message ScreenCaptureMessage {
+}
+
 // Events, these must be kept in sync with STFService/wire.proto
 
 message AirplaneModeEvent {

res/app/components/stf/control/control-service.js

@@ -216,6 +216,10 @@ module.exports = function ControlServiceFactory(
       return sendTwoWay('store.open')
     }
 
+    this.screenshot = function() {
+      return sendTwoWay('screen.capture')
+    }
+
     window.cc = this
   }