Use WebSockets for the screen. The screen directive works, but needs a serious cleanup.

lib/units/device/index.js

@@ -17,8 +17,7 @@ module.exports = function(options) {
       log.info('Preparing device')
       return syrup.serial()
         .dependency(require('./plugins/solo'))
-        .dependency(require('./plugins/screenshot'))
+        .dependency(require('./plugins/screen'))
-        .dependency(require('./plugins/http'))
         .dependency(require('./plugins/service'))
         .dependency(require('./plugins/display'))
         .dependency(require('./plugins/browser'))

lib/units/device/plugins/display.js

@@ -4,19 +4,19 @@ var logger = require('../../../util/logger')
 
 module.exports = syrup.serial()
   .dependency(require('./service'))
-  .dependency(require('./http'))
+  .dependency(require('./screen'))
-  .define(function(options, service, http) {
+  .define(function(options, service, screen) {
     var log = logger.createLogger('device:plugins:display')
 
     function fetch() {
       log.info('Fetching display info')
       return service.getDisplay(0)
         .catch(function() {
-          log.info('Falling back to HTTP API')
+          log.info('Falling back to screen API')
-          return http.getDisplay(0)
+          return screen.info(0)
         })
         .then(function(display) {
-          display.url = http.getDisplayUrl(display.id)
+          display.url = screen.publicUrl
           return display
         })
     }

lib/units/device/plugins/http.js

@@ -1,152 +0,0 @@
-var util = require('util')
-var assert = require('assert')
-var http = require('http')
-
-var Promise = require('bluebird')
-var syrup = require('stf-syrup')
-var request = Promise.promisifyAll(require('request'))
-var httpProxy = require('http-proxy')
-
-var logger = require('../../../util/logger')
-var devutil = require('../../../util/devutil')
-var lifecycle = require('../../../util/lifecycle')
-var streamutil = require('../../../util/streamutil')
-
-module.exports = syrup.serial()
-  .dependency(require('../support/adb'))
-  .dependency(require('../resources/remote'))
-  .define(function(options, adb, remote) {
-    var log = logger.createLogger('device:plugins:http')
-
-    var service = {
-      port: 2870
-    , privateUrl: null
-    , publicUrl: null
-    }
-
-    function openService() {
-      log.info('Launching HTTP API')
-      return devutil.ensureUnusedPort(adb, options.serial, service.port)
-        .timeout(10000)
-        .then(function() {
-          return adb.shell(options.serial, [
-              'exec'
-            , remote.bin
-            , '--lib', remote.lib
-            , '--listen-http', service.port
-            ])
-            .timeout(10000)
-            .then(function(out) {
-              lifecycle.share('Remote shell', out)
-              streamutil.talk(log, 'Remote shell says: "%s"', out)
-            })
-            .then(function() {
-              return devutil.waitForPort(adb, options.serial, service.port)
-                .timeout(20000)
-            })
-            .then(function(conn) {
-              var ours = options.ports.pop()
-                , everyones = options.ports.pop()
-                , url = util.format('http://127.0.0.1:%d', ours)
-
-              // Don't need the connection
-              conn.end()
-
-              log.info('Opening device HTTP API forwarder on "%s"', url)
-
-              service.privateUrl = url
-              service.publicUrl = util.format(
-                'http://%s:%s'
-              , options.publicIp
-              , everyones
-              )
-
-              return adb.forward(
-                  options.serial
-                , util.format('tcp:%d', ours)
-                , util.format('tcp:%d', service.port)
-                )
-                .timeout(10000)
-                .then(function() {
-                  log.info(
-                    'Opening HTTP API proxy on "http://%s:%s"'
-                  , options.publicIp
-                  , everyones
-                  )
-
-                  var resolver = Promise.defer()
-
-                  function resolve() {
-                    lifecycle.share('Proxy server', proxyServer, {
-                      end: false
-                    })
-                    resolver.resolve()
-                  }
-
-                  function reject(err) {
-                    resolver.reject(err)
-                  }
-
-                  function ignore() {
-                    // No-op
-                  }
-
-                  var proxy = httpProxy.createProxyServer({
-                    target: url
-                  , ws: false
-                  , xfwd: false
-                  })
-
-                  proxy.on('error', ignore)
-
-                  var proxyServer = http.createServer(function(req, res) {
-                      proxy.web(req, res)
-                    })
-                    .listen(everyones)
-
-                  proxyServer.on('listening', resolve)
-                  proxyServer.on('error', reject)
-
-                  return resolver.promise.finally(function() {
-                    proxyServer.removeListener('listening', resolve)
-                    proxyServer.removeListener('error', reject)
-                  })
-                })
-            })
-        })
-    }
-
-    return openService()
-      .then(function() {
-        return {
-          getDisplay: function(id) {
-            return request.getAsync({
-                url: util.format(
-                  '%s/api/v1/displays/%d'
-                , service.privateUrl
-                , id
-                )
-              , json: true
-              })
-              .timeout(10000)
-              .then(function(args) {
-                var display = args[1]
-                assert.ok('id' in display, 'Invalid response from HTTP API')
-                // Fix rotation's old name
-                if ('orientation' in display) {
-                  display.rotation = display.orientation
-                  delete display.orientation
-                }
-                return display
-              })
-          }
-        , getDisplayUrl: function(id) {
-            return util.format(
-              '%s/api/v1/displays/%d/screenshot.jpg'
-            , service.publicUrl
-            , id
-            )
-          }
-        }
-      })
-  })

lib/units/device/plugins/screen.js

@@ -0,0 +1,141 @@
+var util = require('util')
+var path = require('path')
+var http = require('http')
+
+var Promise = require('bluebird')
+var syrup = require('stf-syrup')
+var httpProxy = require('http-proxy')
+
+var logger = require('../../../util/logger')
+var lifecycle = require('../../../util/lifecycle')
+var streamutil = require('../../../util/streamutil')
+
+module.exports = syrup.serial()
+  .dependency(require('../support/adb'))
+  .dependency(require('../resources/minicap'))
+  .define(function(options, adb, minicap) {
+    var log = logger.createLogger('device:plugins:screen')
+    var plugin = Object.create(null)
+
+    plugin.devicePort = 9002
+
+    plugin.privatePort = options.ports.pop()
+    plugin.privateUrl = util.format(
+      'ws://127.0.0.1:%s'
+    , plugin.privatePort
+    )
+
+    plugin.publicPort = options.ports.pop()
+    plugin.publicUrl = util.format(
+      'ws://%s:%s'
+    , options.publicIp
+    , plugin.publicPort
+    )
+
+    function run(cmd) {
+      return adb.shell(options.serial, util.format(
+          'LD_LIBRARY_PATH=%s exec %s %s'
+        , path.dirname(minicap.lib)
+        , minicap.bin
+        , cmd
+        ))
+    }
+
+    function startService() {
+      log.info('Launching screen service')
+      return run(util.format('-p %d', plugin.devicePort))
+        .timeout(10000)
+        .then(function(out) {
+          lifecycle.share('Screen shell', out)
+          streamutil.talk(log, 'Screen shell says: "%s"', out)
+        })
+    }
+
+    function forwardService() {
+      log.info('Opening WebSocket service on port %d', plugin.privatePort)
+      return adb.forward(
+          options.serial
+        , util.format('tcp:%d', plugin.privatePort)
+        , util.format('tcp:%d', plugin.devicePort)
+        )
+        .timeout(10000)
+    }
+
+    function startProxy() {
+      log.info('Starting WebSocket proxy on %s', plugin.publicUrl)
+
+      var resolver = Promise.defer()
+
+      function resolve() {
+        lifecycle.share('Proxy server', proxyServer, {
+          end: false
+        })
+        resolver.resolve()
+      }
+
+      function reject(err) {
+        resolver.reject(err)
+      }
+
+      function ignore() {
+        // No-op
+      }
+
+      var proxy = httpProxy.createProxyServer({
+        target: plugin.privateUrl
+      , ws: true
+      , xfwd: false
+      })
+
+      proxy.on('error', ignore)
+
+      var proxyServer = http.createServer()
+
+      proxyServer.on('listening', resolve)
+      proxyServer.on('error', reject)
+
+      proxyServer.on('request', function(req, res) {
+        proxy.web(req, res)
+      })
+
+      proxyServer.on('upgrade', function(req, socket, head) {
+        proxy.ws(req, socket, head)
+      })
+
+      proxyServer.listen(plugin.publicPort)
+
+      return resolver.promise.finally(function() {
+        proxyServer.removeListener('listening', resolve)
+        proxyServer.removeListener('error', reject)
+      })
+    }
+
+    return startService()
+      .then(forwardService)
+      .then(startProxy)
+      .then(function() {
+        plugin.info = function(id) {
+          return run(util.format('-d %d -i', id))
+            .then(streamutil.readAll)
+            .then(function(out) {
+              var match
+              if ((match = /^ERROR: (.*)$/.exec(out))) {
+                throw new Error(match[1])
+              }
+
+              try {
+                var info = JSON.parse(out)
+                // Compat for now, remove eventually
+                info.rotation = 0
+                info.fps = 0
+                info.secure = false
+                return info
+              }
+              catch (e) {
+                throw new Error(out.toString())
+              }
+            })
+        }
+      })
+      .return(plugin)
+  })

lib/units/device/resources/minicap.js

@@ -0,0 +1,92 @@
+var util = require('util')
+
+var Promise = require('bluebird')
+var syrup = require('stf-syrup')
+
+var logger = require('../../../util/logger')
+var pathutil = require('../../../util/pathutil')
+var devutil = require('../../../util/devutil')
+var streamutil = require('../../../util/streamutil')
+
+module.exports = syrup.serial()
+  .dependency(require('../support/adb'))
+  .dependency(require('../support/properties'))
+  .dependency(require('../support/abi'))
+  .define(function(options, adb, properties, abi) {
+    var log = logger.createLogger('device:resources:minicap')
+
+    var resources = {
+      bin: {
+        src: pathutil.requiredMatch(abi.all.map(function(supportedAbi) {
+          return pathutil.vendor(util.format(
+            'minicap/bin/%s/minicap%s'
+          , supportedAbi
+          , abi.pie ? '' : '-nopie'
+          ))
+        }))
+      , dest: '/data/local/tmp/minicap'
+      , comm: 'minicap'
+      , mode: 0755
+      }
+    , lib: {
+        // @todo The lib ABI should match the bin ABI. Currently we don't
+        // have an x86_64 version of the binary while the lib supports it.
+        src: pathutil.requiredMatch(abi.all.map(function(supportedAbi) {
+          return pathutil.vendor(util.format(
+            'minicap/shared/android-%d/%s/minicap.so'
+          , properties['ro.build.version.sdk']
+          , supportedAbi
+          ))
+        }))
+      , dest: '/data/local/tmp/minicap.so'
+      , mode: 0755
+      }
+    }
+
+    function removeResource(res) {
+      return adb.shell(options.serial, ['rm', res.dest])
+        .timeout(10000)
+        .then(function(out) {
+          return streamutil.readAll(out)
+        })
+        .return(res)
+    }
+
+    function installResource(res) {
+      return adb.push(options.serial, res.src, res.dest, res.mode)
+        .timeout(10000)
+        .then(function(transfer) {
+          return new Promise(function(resolve, reject) {
+            transfer.on('error', reject)
+            transfer.on('end', resolve)
+          })
+        })
+        .return(res)
+    }
+
+    function installAll() {
+      return Promise.all([
+        removeResource(resources.bin).then(installResource)
+      , removeResource(resources.lib).then(installResource)
+      ])
+    }
+
+    function stop() {
+      return devutil.killProcsByComm(
+          adb
+        , options.serial
+        , resources.bin.comm
+        , resources.bin.dest
+        )
+        .timeout(15000)
+    }
+
+    return stop()
+      .then(installAll)
+      .then(function() {
+        return {
+          bin: resources.bin.dest
+        , lib: resources.lib.dest
+        }
+      })
+  })

lib/wire/wire.proto

@@ -161,6 +161,7 @@ message DeviceDisplayMessage {
   required float density = 8;
   required bool secure = 9;
   required string url = 10;
+  optional float size = 11;
 }
 
 message DeviceBrowserAppMessage {

res/app/components/stf/screen/screen-directive.js

@@ -1,4 +1,3 @@
-var FastImageRender = require('./fast-image-render').FastImageRender
 var _ = require('lodash')
 
 module.exports = function DeviceScreenDirective($document, ScalingService,
@@ -11,20 +10,23 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
     , device: '&'
     }
   , link: function (scope, element) {
+      var URL = window.URL || window.webkitURL
+      var BLANK_IMG =
+        'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='
+
       var device = scope.device()
         , control = scope.control()
 
       var canvas = element.find('canvas')[0]
         , input = element.find('input')
+        , g = canvas.getContext('2d')
+
+      var ws
+        , loading = false
 
-      var imageRender = new FastImageRender(canvas, {
-        render: 'canvas',
-        timeout: 3000
-      })
       var guestDisplayDensity = setDisplayDensity(1.5)
       //var guestDisplayRotation = 0
 
-      var loading = false
       var cssTransform = VendorUtil.style(['transform', 'webkitTransform'])
 
       var screen = scope.screen = {
@@ -178,6 +180,7 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
 
       function maybeLoadScreen() {
         var size
+        if (ws) {
           if (!loading && scope.$parent.showScreen && device) {
             switch (screen.rotation) {
             case 0:
@@ -196,11 +199,8 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
               break
             }
             loading = true
-          imageRender.load(device.display.url +
+            ws.send('{"op":"jpeg","w":' + size.w + ',"h":' + size.h + '}')
-            '?width=' + size.w +
+          }
-            '&height=' + size.h +
-            '&time=' + Date.now()
-          )
         }
       }
 
@@ -227,29 +227,51 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
       }
 
       function on() {
-        imageRender.onLoad = function (image) {
+        if (!ws) {
+          ws = new WebSocket(device.display.url)
+          ws.binaryType = 'blob'
+        }
+
+        ws.onerror = function() {
+          // @todo HANDLE
+        }
+
+        ws.onclose = function() {
+        }
+
+        ws.onopen = function() {
+          maybeLoadScreen()
+        }
+
+        ws.onmessage = function(message) {
           loading = false
 
           if (scope.$parent.showScreen) {
             screen.rotation = device.display.rotation
 
+            var blob = new Blob([message.data], {
+              type: 'image/jpeg'
+            })
+
+            var img = new Image()
+            img.onload = function() {
               // Check to set the size only if updated
               if (cachedScreen.bounds.w !== screen.bounds.w ||
                 cachedScreen.bounds.h !== screen.bounds.h ||
-              cachedImageWidth !== image.width ||
+                cachedImageWidth !== img.width ||
-              cachedImageHeight !== image.height ||
+                cachedImageHeight !== img.height ||
                 cachedScreen.rotation !== screen.rotation) {
 
                 cachedScreen.bounds.w = screen.bounds.w
                 cachedScreen.bounds.h = screen.bounds.h
 
-              cachedImageWidth = image.width
+                cachedImageWidth = img.width
-              cachedImageHeight = image.height
+                cachedImageHeight = img.height
 
                 cachedScreen.rotation = screen.rotation
 
-              imageRender.canvasWidth = cachedImageWidth
+                canvas.width = cachedImageWidth
-              imageRender.canvasHeight = cachedImageHeight
+                canvas.height = cachedImageHeight
 
                 var projectedSize = screen.scaler.projectedSize(
                   screen.bounds.w
@@ -257,8 +279,8 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
                 , screen.rotation
                 )
 
-              imageRender.canvasStyleWidth = projectedSize.width
+                canvas.style.width = projectedSize.width + 'px'
-              imageRender.canvasStyleHeight = projectedSize.height
+                canvas.style.height = projectedSize.height + 'px'
 
                 // @todo Make sure that each position is able to rotate smoothly
                 // to the next one. This current setup doesn't work if rotation
@@ -279,7 +301,21 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
                 }
               }
 
-            imageRender.draw(image)
+              g.drawImage(img, 0, 0)
+
+              // Try to forcefully clean everything to get rid of memory leaks.
+              img.onload = img.onerror = null
+              img.src = BLANK_IMG
+              img = null
+              url = null
+              blob = null
+
+              // Next please
+              maybeLoadScreen()
+            }
+
+            var url = URL.createObjectURL(blob)
+            img.src = url
 
             // Reset error, if any
             if (scope.displayError) {
@@ -287,27 +323,10 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
                 scope.displayError = false
               })
             }
-
-            // Next please
-            maybeLoadScreen()
           }
-          // Else: Nothing to show
-        }
-
-        imageRender.onError = function (type) {
-          loading = false
-
-          scope.$apply(function () {
-            if (type === 'timeout') {
-              scope.displayError = 'timeout'
-            } else {
-              scope.displayError = 'secure'
-            }
-          })
         }
 
         updateBounds()
-        maybeLoadScreen()
 
         input.bind('keydown', keydownListener)
         input.bind('keyup', keyupListener)
@@ -317,9 +336,14 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
       }
 
       function off() {
-        imageRender.onLoad = imageRender.onError = null
         loading = false
 
+        if (ws) {
+          ws.onmessage = ws.onerror = ws.onclose = null
+          ws.close()
+          ws = null
+        }
+
         input.unbind('keydown', keydownListener)
         input.unbind('keyup', keyupListener)
         input.unbind('input', inputListener)
@@ -329,11 +353,9 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
 
       scope.$watch('$parent.showScreen', function (val) {
         if (val) {
-          updateBounds()
+          on()
-          maybeLoadScreen()
         } else {
-          scope.fps = null
+          off()
-          imageRender.clear()
         }
       })