New multitouch-compatible touch system.

lib/units/device/plugins/touch.js

@@ -1,9 +1,10 @@
+var util = require('util')
+
 var Promise = require('bluebird')
 var syrup = require('syrup')
-var monkey = require('adbkit-monkey')
+var split = require('split')
 
 var wire = require('../../../wire')
-var devutil = require('../../../util/devutil')
 var logger = require('../../../util/logger')
 var lifecycle = require('../../../util/lifecycle')
 var streamutil = require('../../../util/streamutil')
@@ -12,111 +13,138 @@ var SeqQueue = require('../../../wire/seqqueue')
 module.exports = syrup.serial()
   .dependency(require('../support/adb'))
   .dependency(require('../support/router'))
-  .dependency(require('../resources/remote'))
+  .dependency(require('../resources/minitouch'))
-  .dependency(require('./display'))
+  .define(function(options, adb, router, minitouch) {
-  .dependency(require('./data'))
-  .define(function(options, adb, router, remote, display, data) {
     var log = logger.createLogger('device:plugins:touch')
     var plugin = Object.create(null)
 
-    var service = {
+    function startService() {
-      port: 2820
-    }
-
-    function openService() {
       log.info('Launching touch service')
-      return devutil.ensureUnusedPort(adb, options.serial, service.port)
-        .timeout(10000)
-        .then(function() {
       return adb.shell(options.serial, [
           'exec'
-            , remote.bin
+        , minitouch.bin
-            , '--lib', remote.lib
-            , '--listen-input', service.port
         ])
         .timeout(10000)
-        })
         .then(function(out) {
           lifecycle.share('Touch shell', out)
           streamutil.talk(log, 'Touch shell says: "%s"', out)
         })
+    }
+
+    function connectService() {
+      function tryConnect(times, delay) {
+        return adb.openLocal(options.serial, '/data/local/tmp/minitouch.sock')
+          .timeout(10000)
+          .then(function(out) {
+            lifecycle.share('Touch socket', out)
+            return out
+          })
+          .then(function(out) {
+            return new Promise(function(resolve, reject) {
+              out.pipe(split()).on('data', function(line) {
+                var args = line.toString().split(/ /g)
+                switch (args[0]) {
+                  case 'v':
+                    out.version = +args[1]
+                    log.info('Touch protocol is version %d', out.version)
+                    break
+                  case '^':
+                    out.maxContacts = args[1]
+                    out.maxX = args[2]
+                    out.maxY = args[3]
+                    out.maxPressure = args[4]
+                    log.info(
+                      'Touch protocol reports %d contacts in a %dx%d grid '
+                    + 'with a max pressure of %d'
+                    , out.maxContacts
+                    , out.maxX
+                    , out.maxY
+                    , out.maxPressure
+                    )
+                    return resolve(out)
+                  default:
+                    return reject(new Error(util.format(
+                      'Unknown metadata "%s"'
+                    , line
+                    )))
+                }
+              })
+            })
+          })
+          .catch(function(err) {
+            if (/closed/.test(err.message) && times > 1) {
+              return Promise.delay(delay)
                 .then(function() {
-          return devutil.waitForPort(adb, options.serial, service.port)
+                  return tryConnect(--times, delay * 2)
-            .timeout(15000)
-        })
-        .then(function(conn) {
-          return Promise.promisifyAll(monkey.connectStream(conn))
-        })
-        .then(function(monkey) {
-          return lifecycle.share('Touch monkey', monkey)
                 })
             }
-
+            return Promise.reject(err)
-    function modifyCoords(message) {
+          })
-      message.x = Math.floor(message.x * display.width)
+      }
-      message.y = Math.floor(message.y * display.height)
+      log.info('Connecting to touch service')
+      return tryConnect(5, 100)
     }
 
-    return openService()
+    return startService()
-      .then(function(monkey) {
+      .then(connectService)
-        var queue = new SeqQueue()
+      .then(function(socket) {
-          , pressure = (data && data.touch && data.touch.defaultPressure) || 50
+        var queue = new SeqQueue(100, 4)
 
-        log.info('Setting default pressure to %d', pressure)
+        function send(command) {
+          socket.write(command)
+        }
 
         plugin.touchDown = function(point) {
-          modifyCoords(point)
+          send(util.format(
-          monkey.sendAsync([
+            'd %s %s %s %s\n'
-              'touch down'
+          , point.contact
-            , point.x
+          , Math.floor(point.x * socket.maxX)
-            , point.y
+          , Math.floor(point.y * socket.maxY)
-            , pressure
+          , Math.floor((point.pressure || 0.5) * socket.maxPressure)
-            ].join(' '))
+          ))
-            .catch(function(err) {
-              log.error('touchDown failed', err.stack)
-            })
         }
 
         plugin.touchMove = function(point) {
-          modifyCoords(point)
+          send(util.format(
-          monkey.sendAsync([
+            'm %s %s %s %s\n'
-              'touch move'
+          , point.contact
-            , point.x
+          , Math.floor(point.x * socket.maxX)
-            , point.y
+          , Math.floor(point.y * socket.maxX)
-            , pressure
+          , Math.floor((point.pressure || 0.5) * socket.maxPressure)
-            ].join(' '))
+          ))
-            .catch(function(err) {
-              log.error('touchMove failed', err.stack)
-            })
         }
 
         plugin.touchUp = function(point) {
-          modifyCoords(point)
+          send(util.format(
-          monkey.sendAsync([
+            'u %s\n'
-              'touch up'
+          , point.contact
-            , point.x
+          ))
-            , point.y
+        }
-            , pressure
+
-            ].join(' '))
+        plugin.touchCommit = function() {
-            .catch(function(err) {
+          send('c\n')
-              log.error('touchUp failed', err.stack)
+        }
-            })
+
+        plugin.touchReset = function() {
+          send('r\n')
         }
 
         plugin.tap = function(point) {
-          modifyCoords(point)
+          plugin.touchDown(point)
-          monkey.sendAsync([
+          plugin.touchCommit()
-              'tap'
+          plugin.touchUp(point)
-            , point.x
+          plugin.touchCommit()
-            , point.y
-            , pressure
-            ].join(' '))
-            .catch(function(err) {
-              log.error('tap failed', err.stack)
-            })
         }
 
         router
+          .on(wire.GestureStartMessage, function(channel, message) {
+            queue.start(message.seq)
+          })
+          .on(wire.GestureStopMessage, function(channel, message) {
+            queue.push(message.seq, function() {
+              queue.stop()
+            })
+          })
           .on(wire.TouchDownMessage, function(channel, message) {
             queue.push(message.seq, function() {
               plugin.touchDown(message)
@@ -131,12 +159,16 @@ module.exports = syrup.serial()
             queue.push(message.seq, function() {
               plugin.touchUp(message)
             })
-
-            // Reset queue
-            queue = new SeqQueue()
           })
-          .on(wire.TapMessage, function(channel, message) {
+          .on(wire.TouchCommitMessage, function(channel, message) {
-            plugin.tap(message)
+            queue.push(message.seq, function() {
+              plugin.touchCommit()
+            })
+          })
+          .on(wire.TouchResetMessage, function(channel, message) {
+            queue.push(message.seq, function() {
+              plugin.touchReset()
+            })
           })
       })
       .return(plugin)

lib/units/device/resources/minitouch.js

@@ -0,0 +1,93 @@
+var util = require('util')
+
+var Promise = require('bluebird')
+var syrup = require('syrup')
+
+var logger = require('../../../util/logger')
+var pathutil = require('../../../util/pathutil')
+var devutil = require('../../../util/devutil')
+var streamutil = require('../../../util/streamutil')
+
+module.exports = syrup.serial()
+  .dependency(require('../support/adb'))
+  .dependency(require('../support/properties'))
+  .define(function(options, adb, properties) {
+    var log = logger.createLogger('device:resources:minitouch')
+
+    var resources = {
+      bin: {
+        src: pathutil.vendor(util.format(
+          'minitouch/%s/minitouch'
+        , properties['ro.product.cpu.abi']
+        ))
+      , dest: '/data/local/tmp/minitouch'
+      , comm: 'minitouch'
+      , mode: 0755
+      }
+    }
+
+    function removeResource(res) {
+      return adb.shell(options.serial, ['rm', res.dest])
+        .timeout(10000)
+        .then(function(out) {
+          return streamutil.readAll(out)
+        })
+        .return(res)
+    }
+
+    function installResource(res) {
+      return adb.push(options.serial, res.src, res.dest, res.mode)
+        .timeout(10000)
+        .then(function(transfer) {
+          return new Promise(function(resolve, reject) {
+            transfer.on('error', reject)
+            transfer.on('end', resolve)
+          })
+        })
+        .return(res)
+    }
+
+    function ensureNotBusy(res) {
+      return adb.shell(options.serial, [res.dest, '--help'])
+        .timeout(10000)
+        .then(function(out) {
+          // Can be "Text is busy", "text busy"
+          return streamutil.findLine(out, (/busy/i))
+            .timeout(10000)
+            .then(function() {
+              log.info('Binary is busy, will retry')
+              return Promise.delay(1000)
+            })
+            .then(function() {
+              return ensureNotBusy(res)
+            })
+            .catch(streamutil.NoSuchLineError, function() {
+              return res
+            })
+        })
+    }
+
+    function installAll() {
+      return Promise.all([
+        removeResource(resources.bin).then(installResource).then(ensureNotBusy)
+      ])
+    }
+
+    function stop() {
+      return devutil.killProcsByComm(
+          adb
+        , options.serial
+        , resources.bin.comm
+        , resources.bin.dest
+        )
+        .timeout(15000)
+    }
+
+    return stop()
+      .then(installAll)
+      .then(function() {
+        return {
+          bin: resources.bin.dest
+        }
+      })
+  })

lib/units/websocket/index.js

@@ -82,19 +82,6 @@ module.exports = function(options) {
       sub.unsubscribe(channel)
     }
 
-    function createTouchHandler(Klass) {
-      return function(channel, data) {
-        push.send([
-          channel
-        , wireutil.envelope(new Klass(
-            data.seq
-          , data.x
-          , data.y
-          ))
-        ])
-      }
-    }
-
     function createKeyHandler(Klass) {
       return function(channel, data) {
         push.send([
@@ -268,10 +255,71 @@ module.exports = function(options) {
           dbapi.resetUserSettings(user.email)
         })
         // Touch events
-        .on('input.touchDown', createTouchHandler(wire.TouchDownMessage))
+        .on('input.touchDown', function(channel, data) {
-        .on('input.touchMove', createTouchHandler(wire.TouchMoveMessage))
+          push.send([
-        .on('input.touchUp', createTouchHandler(wire.TouchUpMessage))
+            channel
-        .on('input.tap', createTouchHandler(wire.TapMessage))
+          , wireutil.envelope(new wire.TouchDownMessage(
+              data.seq
+            , data.contact
+            , data.x
+            , data.y
+            , data.pressure
+            ))
+          ])
+        })
+        .on('input.touchMove', function(channel, data) {
+          push.send([
+            channel
+          , wireutil.envelope(new wire.TouchMoveMessage(
+              data.seq
+            , data.contact
+            , data.x
+            , data.y
+            , data.pressure
+            ))
+          ])
+        })
+        .on('input.touchUp', function(channel, data) {
+          push.send([
+            channel
+          , wireutil.envelope(new wire.TouchUpMessage(
+              data.seq
+            , data.contact
+            ))
+          ])
+        })
+        .on('input.touchCommit', function(channel, data) {
+          push.send([
+            channel
+          , wireutil.envelope(new wire.TouchCommitMessage(
+              data.seq
+            ))
+          ])
+        })
+        .on('input.touchReset', function(channel, data) {
+          push.send([
+            channel
+          , wireutil.envelope(new wire.TouchResetMessage(
+              data.seq
+            ))
+          ])
+        })
+        .on('input.gestureStart', function(channel, data) {
+          push.send([
+            channel
+          , wireutil.envelope(new wire.GestureStartMessage(
+              data.seq
+            ))
+          ])
+        })
+        .on('input.gestureStop', function(channel, data) {
+          push.send([
+            channel
+          , wireutil.envelope(new wire.GestureStopMessage(
+              data.seq
+            ))
+          ])
+        })
         // Key events
         .on('input.keyDown', createKeyHandler(wire.KeyDownMessage))
         .on('input.keyUp', createKeyHandler(wire.KeyUpMessage))

lib/wire/seqqueue.js

@@ -1,25 +1,61 @@
-function SeqQueue() {
+function SeqQueue(size, maxWaiting) {
-  this.queue = []
+  this.lo = 0
-  this.seq = 0
+  this.size = size
+  this.maxWaiting = maxWaiting
+  this.waiting = 0
+  this.list = new Array(size)
+  this.locked = true
+}
+
+SeqQueue.prototype.start = function(seq) {
+  this.locked = false
+  this.lo = seq
+  this.maybeConsume()
+}
+
+SeqQueue.prototype.stop = function() {
+  this.locked = true
+  this.maybeConsume()
 }
 
 SeqQueue.prototype.push = function(seq, handler) {
-  this.queue[seq] = handler
+  if (seq >= this.size) {
-  this.maybeDequeue()
+    return
   }
 
-SeqQueue.prototype.done = function(seq, handler) {
+  this.list[seq] = handler
-  this.queue[seq] = handler
+  this.waiting += 1
-  this.maybeDequeue()
+  this.maybeConsume()
 }
 
-SeqQueue.prototype.maybeDequeue = function() {
+SeqQueue.prototype.maybeConsume = function() {
-  var handler
+  if (this.locked) {
+    return
+  }
 
-  while ((handler = this.queue[this.seq])) {
+  while (this.waiting) {
-    this.queue[this.seq] = void 0
+    // Did we reach the end of the loop? If so, start from the beginning.
+    if (this.lo === this.size) {
+      this.lo = 0
+    }
+
+    var handler = this.list[this.lo]
+    // Have we received it yet?
+    if (handler) {
+      this.list[this.lo] = void 0
       handler()
-    this.seq += 1
+      this.lo += 1
+      this.waiting -= 1
+    }
+    // Are we too much behind? If so, just move on.
+    else if (this.waiting >= this.maxWaiting) {
+      this.lo += 1
+      this.waiting -= 1
+    }
+    // We don't have it yet, stop.
+    else {
+      break
+    }
   }
 }
 

lib/wire/wire.proto

@@ -25,10 +25,13 @@ enum MessageType {
   ProbeMessage               = 17;
   ShellCommandMessage        = 18;
   ShellKeepAliveMessage      = 19;
-  TapMessage                 = 20;
   TouchDownMessage           = 21;
   TouchMoveMessage           = 22;
   TouchUpMessage             = 23;
+  TouchCommitMessage         = 65;
+  TouchResetMessage          = 66;
+  GestureStartMessage        = 67;
+  GestureStopMessage         = 68;
   TransactionDoneMessage     = 24;
   TransactionProgressMessage = 25;
   TypeMessage                = 26;
@@ -249,25 +252,39 @@ message PhysicalIdentifyMessage {
 
 message TouchDownMessage {
   required uint32 seq = 1;
-  required float x = 2;
+  required uint32 contact = 2;
-  required float y = 3;
+  required float x = 3;
+  required float y = 4;
+  optional float pressure = 5;
 }
 
 message TouchMoveMessage {
   required uint32 seq = 1;
-  required float x = 2;
+  required uint32 contact = 2;
-  required float y = 3;
+  required float x = 3;
+  required float y = 4;
+  optional float pressure = 5;
 }
 
 message TouchUpMessage {
   required uint32 seq = 1;
-  required float x = 2;
+  required uint32 contact = 2;
-  required float y = 3;
 }
 
-message TapMessage {
+message TouchCommitMessage {
-  required float x = 1;
+  required uint32 seq = 1;
-  required float y = 2;
+}
+
+message TouchResetMessage {
+  required uint32 seq = 1;
+}
+
+message GestureStartMessage {
+  required uint32 seq = 1;
+}
+
+message GestureStopMessage {
+  required uint32 seq = 1;
 }
 
 message TypeMessage {

res/app/components/stf/control/control-service.js

@@ -21,16 +21,6 @@ module.exports = function ControlServiceFactory(
       return tx.promise
     }
 
-    function touchSender(type) {
-      return function(seq, x, y) {
-        sendOneWay(type, {
-          seq: seq
-        , x: x
-        , y: y
-        })
-      }
-    }
-
     function keySender(type, fixedKey) {
       return function(key) {
         if (typeof key === 'string') {
@@ -49,10 +39,56 @@ module.exports = function ControlServiceFactory(
       }
     }
 
-    this.touchDown = touchSender('input.touchDown')
+    this.gestureStart = function(seq) {
-    this.touchMove = touchSender('input.touchMove')
+      sendOneWay('input.gestureStart', {
-    this.touchUp   = touchSender('input.touchUp')
+        seq: seq
-    this.tap       = touchSender('input.tap')
+      })
+    }
+
+    this.gestureStop = function(seq) {
+      sendOneWay('input.gestureStop', {
+        seq: seq
+      })
+    }
+
+    this.touchDown = function(seq, contact, x, y, pressure) {
+      sendOneWay('input.touchDown', {
+        seq: seq
+      , contact: contact
+      , x: x
+      , y: y
+      , pressure: pressure
+      })
+    }
+
+    this.touchMove = function(seq, contact, x, y, pressure) {
+      sendOneWay('input.touchMove', {
+        seq: seq
+      , contact: contact
+      , x: x
+      , y: y
+      , pressure: pressure
+      })
+    }
+
+    this.touchUp   = function(seq, contact) {
+      sendOneWay('input.touchUp', {
+        seq: seq
+      , contact: contact
+      })
+    }
+
+    this.touchCommit   = function(seq) {
+      sendOneWay('input.touchCommit', {
+        seq: seq
+      })
+    }
+
+    this.touchReset   = function(seq) {
+      sendOneWay('input.touchReset', {
+        seq: seq
+      })
+    }
 
     this.keyDown   = keySender('input.keyDown')
     this.keyUp     = keySender('input.keyUp')

res/app/components/stf/screen/screen-directive.js

@@ -4,31 +4,52 @@ var _ = require('lodash')
 module.exports = function DeviceScreenDirective($document, ScalingService,
   VendorUtil, PageVisibilityService, BrowserInfo, $timeout) {
   return {
-    restrict: 'E',
+    restrict: 'E'
-    template: require('./screen.jade'),
+  , template: require('./screen.jade')
-    link: function (scope, element) {
+  , scope: {
+      control: '&'
+    , device: '&'
+    }
+  , link: function (scope, element) {
+      var device = scope.device()
+        , control = scope.control()
+
       var canvas = element.find('canvas')[0]
+        , input = element.find('input')
+
       var imageRender = new FastImageRender(canvas, {
         render: 'canvas',
         timeout: 3000
       })
       var guestDisplayDensity = setDisplayDensity(1.5)
       //var guestDisplayRotation = 0
-      var finger = element.find('span')
+
-      var input = element.find('input')
-      var boundingWidth = 0  // TODO: cache inside FastImageRender?
-      var boundingHeight = 0
-      var cachedBoundingWidth = 0
-      var cachedBoundingHeight = 0
-      var cachedImageWidth = 0
-      var cachedImageHeight = 0
-      var cachedRotation = 0
-      var rotation = 0
       var loading = false
-      var scaler
-      var seq = 0
       var cssTransform = VendorUtil.style(['transform', 'webkitTransform'])
 
+      var screen = scope.screen = {
+        scaler: ScalingService.coordinator(
+          device.display.width, device.display.height
+        )
+      , rotation: 0
+      , bounds: {
+          x: 0
+        , y: 0
+        , w: 0
+        , h: 0
+        }
+      }
+
+      var cachedScreen = {
+        rotation: 0
+      , bounds: {
+          x: 0
+        , y: 0
+        , w: 0
+        , h: 0
+        }
+      }
+
       // NOTE: instead of fa-pane-resize, a fa-child-pane-resize could be better
       var onPanelResizeThrottled = _.throttle(updateBounds, 16)
       scope.$on('fa-pane-resize', onPanelResizeThrottled)
@@ -42,88 +63,20 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
         return guestDisplayDensity
       }
 
-      function sendTouch(type, e) {
-        var x = e.offsetX || e.layerX || 0
-        var y = e.offsetY || e.layerY || 0
-        var r = scope.device.display.rotation
-
-        if (BrowserInfo.touch) {
-          if (e.touches && e.touches.length) {
-            x = e.touches[0].pageX
-            y = e.touches[0].pageY
-          } else if (e.changedTouches && e.changedTouches.length) {
-            x = e.changedTouches[0].pageX
-            y = e.changedTouches[0].pageY
-          }
-        }
-
-        var scaled = scaler.coords(boundingWidth, boundingHeight, x, y, r)
-
-        finger[0].style[cssTransform] =
-          'translate3d(' + x + 'px,' + y + 'px,0)'
-
-        scope.control[type](
-          seq++, scaled.xP, scaled.yP
-        )
-      }
-
-      function stopTouch() {
-        element.removeClass('fingering')
-        if (BrowserInfo.touch) {
-          element.unbind('touchmove', moveListener)
-          $document.unbind('touchend', upListener)
-          $document.unbind('touchleave', upListener)
-        } else {
-          element.unbind('mousemove', moveListener)
-          $document.unbind('mouseup', upListener)
-          $document.unbind('mouseleave', upListener)
-        }
-        seq = 0
-      }
-
       function updateBounds() {
-        boundingWidth = element[0].offsetWidth
+        screen.bounds.w = element[0].offsetWidth
-        boundingHeight = element[0].offsetHeight
+        screen.bounds.h = element[0].offsetHeight
 
         // TODO: element is an object HTMLUnknownElement in IE9
 
         // Developer error, let's try to reduce debug time
-        if (!boundingWidth || !boundingHeight) {
+        if (!screen.bounds.w || !screen.bounds.h) {
           throw new Error(
             'Unable to update display size; container must have dimensions'
           )
         }
       }
 
-      function downListener(e) {
-        e.preventDefault()
-        if (!BrowserInfo.touch) {
-          input[0].focus()
-          element.addClass('fingering')
-        }
-
-        sendTouch('touchDown', e)
-
-        if (BrowserInfo.touch) {
-          element.bind('touchmove', moveListener)
-          $document.bind('touchend', upListener)
-          $document.bind('touchleave', upListener)
-        } else {
-          element.bind('mousemove', moveListener)
-          $document.bind('mouseup', upListener)
-          $document.bind('mouseleave', upListener)
-        }
-      }
-
-      function moveListener(e) {
-        sendTouch('touchMove', e)
-      }
-
-      function upListener(e) {
-        sendTouch('touchUp', e)
-        stopTouch()
-      }
-
       function isChangeCharsetKey(e) {
         // Add any special key here for changing charset
         //console.log('e', e)
@@ -162,7 +115,7 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
 
         if (isChangeCharsetKey(e)) {
           specialKey = true
-          scope.control.keyPress('switch_charset')
+          control.keyPress('switch_charset')
         }
 
         if (specialKey) {
@@ -173,58 +126,58 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
       }
 
       function keydownListener(e) {
-        scope.control.keyDown(e.keyCode)
+        control.keyDown(e.keyCode)
       }
 
       function keyupListener(e) {
         if (!keyupSpecialKeys(e)) {
-          scope.control.keyUp(e.keyCode)
+          control.keyUp(e.keyCode)
         }
       }
 
       function keypressListener(e) {
         e.preventDefault() // no need to change value
-        scope.control.type(String.fromCharCode(e.charCode))
+        control.type(String.fromCharCode(e.charCode))
       }
 
       function pasteListener(e) {
         e.preventDefault() // no need to change value
-        scope.control.paste(e.clipboardData.getData('text/plain'))
+        control.paste(e.clipboardData.getData('text/plain'))
       }
 
       function copyListener(e) {
-        scope.control.getClipboardContent()
+        control.getClipboardContent()
         // @TODO: OK, this basically copies last clipboard content
-        if (scope.control.clipboardContent) {
+        if (control.clipboardContent) {
-          e.clipboardData.setData("text/plain", scope.control.clipboardContent)
+          e.clipboardData.setData("text/plain", control.clipboardContent)
         }
         e.preventDefault()
       }
 
       scope.retryLoadingScreen = function () {
         if (scope.displayError === 'secure') {
-          scope.control.home()
+          control.home()
         }
         $timeout(maybeLoadScreen, 3000)
       }
 
       function maybeLoadScreen() {
-        if (!loading && scope.$parent.showScreen && scope.device) {
+        if (!loading && scope.$parent.showScreen && device) {
           var w, h
-          switch (rotation) {
+          switch (screen.rotation) {
           case 0:
           case 180:
-            w = boundingWidth
+            w = screen.bounds.w
-            h = boundingHeight
+            h = screen.bounds.h
             break
           case 90:
           case 270:
-            w = boundingHeight
+            w = screen.bounds.h
-            h = boundingWidth
+            h = screen.bounds.w
             break
           }
           loading = true
-          imageRender.load(scope.device.display.url +
+          imageRender.load(device.display.url +
             '?width=' + Math.ceil(w * guestDisplayDensity) +
             '&height=' + Math.ceil(h * guestDisplayDensity) +
             '&time=' + Date.now()
@@ -233,35 +186,37 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
       }
 
       function on() {
-        scaler = ScalingService.coordinator(
-          scope.device.display.width, scope.device.display.height
-        )
-
         imageRender.onLoad = function (image) {
+          var cachedImageWidth = 0
+            , cachedImageHeight = 0
+
           loading = false
 
           if (scope.$parent.showScreen) {
+            screen.rotation = device.display.rotation
 
             // Check to set the size only if updated
-            if (cachedBoundingWidth !== boundingWidth ||
+            if (cachedScreen.bounds.w !== screen.bounds.w ||
-              cachedBoundingHeight !== boundingHeight ||
+              cachedScreen.bounds.h !== screen.bounds.h ||
               cachedImageWidth !== image.width ||
               cachedImageHeight !== image.height ||
-              cachedRotation !== rotation) {
+              cachedScreen.rotation !== screen.rotation) {
 
-              cachedBoundingWidth = boundingWidth
+              cachedScreen.bounds.w = screen.bounds.w
-              cachedBoundingHeight = boundingHeight
+              cachedScreen.bounds.h = screen.bounds.h
 
               cachedImageWidth = image.width
               cachedImageHeight = image.height
 
-              cachedRotation = rotation
+              cachedScreen.rotation = screen.rotation
 
               imageRender.canvasWidth = cachedImageWidth
               imageRender.canvasHeight = cachedImageHeight
 
-              var projectedSize = scaler.projectedSize(
+              var projectedSize = screen.scaler.projectedSize(
-                boundingWidth, boundingHeight, rotation
+                screen.bounds.w
+              , screen.bounds.h
+              , screen.rotation
               )
 
               imageRender.canvasStyleWidth = projectedSize.width
@@ -270,7 +225,7 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
               // @todo Make sure that each position is able to rotate smoothly
               // to the next one. This current setup doesn't work if rotation
               // changes from 180 to 270 (it will do a reverse rotation).
-              switch (rotation) {
+              switch (screen.rotation) {
                 case 0:
                   canvas.style[cssTransform] = 'translate(-50%, -50%) rotate(0deg)'
                   break
@@ -321,30 +276,17 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
         input.bind('keypress', keypressListener)
         input.bind('paste', pasteListener)
         input.bind('copy', copyListener)
-
-        if (BrowserInfo.touch) {
-          element.bind('touchstart', downListener)
-        } else {
-          element.bind('mousedown', downListener)
-        }
-
       }
 
       function off() {
         imageRender.onLoad = imageRender.onError = null
         loading = false
-        stopTouch()
+
         input.unbind('keydown', keydownListener)
         input.unbind('keyup', keyupListener)
         input.unbind('keypress', keypressListener)
         input.unbind('paste', pasteListener)
         input.unbind('copy', copyListener)
-
-        if (BrowserInfo.touch) {
-          element.unbind('touchstart', downListener)
-        } else {
-          element.unbind('mousedown', downListener)
-        }
       }
 
       scope.$watch('$parent.showScreen', function (val) {
@@ -357,7 +299,7 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
       })
 
       function checkEnabled() {
-        var using = scope.device && scope.device.using
+        var using = device && device.using
         if (using && !PageVisibilityService.hidden) {
           on()
         }
@@ -369,22 +311,276 @@ module.exports = function DeviceScreenDirective($document, ScalingService,
       scope.$watch('device.using', checkEnabled)
       scope.$on('visibilitychange', checkEnabled)
 
-      scope.$watch('device.display.rotation', function (r) {
-        rotation = r || 0
-      })
-
       scope.$on('guest-portrait', function () {
-        scope.control.rotate(0)
+        control.rotate(0)
         updateBounds()
       })
 
       scope.$on('guest-landscape', function () {
-        scope.control.rotate(90)
+        control.rotate(90)
         setDisplayDensity(2)
         updateBounds()
       })
 
       scope.$on('$destroy', off)
+
+      // @todo Move everything below this line elsewhere.
+      var slots = []
+        , slotted = Object.create(null)
+        , fingers = []
+        , seq = -1
+        , cycle = 100
+
+      function nextSeq() {
+        return ++seq >= cycle ? (seq = 0) : seq
+      }
+
+      function createSlots() {
+        for (var i = 9; i >= 0; --i) {
+          slots.push(i)
+          fingers.unshift(createFinger(i))
+        }
+      }
+
+      function activateFinger(index, x, y, pressure) {
+        var scale = 0.5 + pressure
+        fingers[index].classList.add('active')
+        fingers[index].style[cssTransform] =
+          'translate3d(' + x + 'px,' + y + 'px,0) ' +
+          'scale(' + scale + ',' + scale + ')'
+      }
+
+      function deactivateFinger(index) {
+        fingers[index].classList.remove('active')
+      }
+
+      function deactivateFingers() {
+        for (var i = 0, l = fingers.length; i < l; ++i) {
+          fingers[i].classList.remove('active')
+        }
+      }
+
+      function createFinger(index) {
+        var el = document.createElement('span')
+        el.className = 'finger finger-' + index
+        return el
+      }
+
+      function calculateBounds() {
+        var el = element[0]
+
+        screen.bounds.w = el.offsetWidth
+        screen.bounds.h = el.offsetHeight
+        screen.bounds.x = 0
+        screen.bounds.y = 0
+
+        while (el.offsetParent) {
+          screen.bounds.x += el.offsetLeft
+          screen.bounds.y += el.offsetTop
+          el = el.offsetParent
+        }
+      }
+
+      function mouseDownListener(e) {
+        e.preventDefault()
+
+        calculateBounds()
+        startMousing()
+
+        var x = e.pageX - screen.bounds.x
+          , y = e.pageY - screen.bounds.y
+          , pressure = 0.5
+          , scaled = screen.scaler.coords(
+              screen.bounds.w
+            , screen.bounds.h
+            , x
+            , y
+            , screen.rotation
+            )
+
+        control.touchDown(nextSeq(), 0, scaled.xP, scaled.yP, pressure)
+        control.touchCommit(nextSeq())
+
+        activateFinger(0, x, y, pressure)
+
+        element.bind('mousemove', mouseMoveListener)
+        $document.bind('mouseup', mouseUpListener)
+        $document.bind('mouseleave', mouseUpListener)
+      }
+
+      function mouseMoveListener(e) {
+        e.preventDefault()
+
+        var x = e.pageX - screen.bounds.x
+          , y = e.pageY - screen.bounds.y
+          , pressure = 0.5
+          , scaled = screen.scaler.coords(
+              screen.bounds.w
+            , screen.bounds.h
+            , x
+            , y
+            , screen.rotation
+            )
+
+        control.touchMove(nextSeq(), 0, scaled.xP, scaled.yP, pressure)
+        control.touchCommit(nextSeq())
+
+        activateFinger(0, x, y, pressure)
+      }
+
+      function mouseUpListener(e) {
+        e.preventDefault()
+
+        control.touchUp(nextSeq(), 0)
+        control.touchCommit(nextSeq())
+
+        deactivateFinger(0)
+
+        stopMousing()
+      }
+
+      function startMousing() {
+        control.gestureStart(nextSeq())
+      }
+
+      function stopMousing() {
+        element.unbind('mousemove', mouseMoveListener)
+        $document.unbind('mouseup', mouseUpListener)
+        $document.unbind('mouseleave', mouseUpListener)
+        deactivateFingers()
+        control.gestureStop(nextSeq())
+      }
+
+      function touchStartListener(e) {
+        e.preventDefault()
+
+        calculateBounds()
+
+        if (e.touches.length === e.changedTouches.length) {
+          startTouching()
+        }
+
+        var currentTouches = Object.create(null)
+        var i, l
+
+        for (i = 0, l = e.touches.length; i < l; ++i) {
+          currentTouches[e.touches[i].identifier] = 1;
+        }
+
+        function maybeLostTouchEnd(id) {
+          return !(id in currentTouches)
+        }
+
+        // We might have lost a touchend event due to various edge cases
+        // (literally) such as dragging from the bottom of the screen so that
+        // the control center appears. If so, let's ask for a reset.
+        if (Object.keys(slotted).some(maybeLostTouchEnd)) {
+          Object.keys(slotted).forEach(function(id) {
+            slots.push(slotted[id])
+            delete slotted[id]
+          })
+          slots.sort().reverse()
+          control.touchReset(nextSeq())
+          deactivateFingers()
+        }
+
+        if (!slots.length) {
+          // This should never happen but who knows...
+          throw new Error('Ran out of multitouch slots')
+        }
+
+        for (i = 0, l = e.changedTouches.length; i < l; ++i) {
+          var touch = e.changedTouches[i]
+            , slot = slots.pop()
+            , x = touch.pageX - screen.bounds.x
+            , y = touch.pageY - screen.bounds.y
+            , pressure = touch.force || 0.5
+            , scaled = screen.scaler.coords(
+                screen.bounds.w
+              , screen.bounds.h
+              , x
+              , y
+              , screen.rotation
+              )
+
+          slotted[touch.identifier] = slot
+          control.touchDown(nextSeq(), slot, scaled.xP, scaled.yP, pressure)
+          activateFinger(slot, x, y, pressure)
+        }
+
+        element.bind('touchmove', touchMoveListener)
+        $document.bind('touchend', touchEndListener)
+        $document.bind('touchleave', touchEndListener)
+
+        control.touchCommit(nextSeq())
+      }
+
+      function touchMoveListener(e) {
+        e.preventDefault()
+
+        for (var i = 0, l = e.changedTouches.length; i < l; ++i) {
+          var touch = e.changedTouches[i]
+            , slot = slotted[touch.identifier]
+            , x = touch.pageX - screen.bounds.x
+            , y = touch.pageY - screen.bounds.y
+            , pressure = touch.force || 0.5
+            , scaled = screen.scaler.coords(
+                screen.bounds.w
+              , screen.bounds.h
+              , x
+              , y
+              , screen.rotation
+              )
+
+          control.touchMove(nextSeq(), slot, scaled.xP, scaled.yP, pressure)
+          activateFinger(slot, x, y, pressure)
+        }
+
+        control.touchCommit(nextSeq())
+      }
+
+      function touchEndListener(e) {
+        var foundAny = false
+
+        for (var i = 0, l = e.changedTouches.length; i < l; ++i) {
+          var touch = e.changedTouches[i]
+            , slot = slotted[touch.identifier]
+          if (slot === void 0) {
+            // We've already disposed of the contact. We may have gotten a
+            // touchend event for the same contact twice.
+            continue
+          }
+          delete slotted[touch.identifier]
+          slots.push(slot)
+          control.touchUp(nextSeq(), slot)
+          deactivateFinger(slot)
+          foundAny = true
+        }
+
+        if (foundAny) {
+          control.touchCommit(nextSeq())
+          if (!e.touches.length) {
+            stopTouching()
+          }
+        }
+      }
+
+      function startTouching() {
+        control.gestureStart(nextSeq())
+      }
+
+      function stopTouching() {
+        element.unbind('touchmove', touchMoveListener)
+        $document.unbind('touchend', touchEndListener)
+        $document.unbind('touchleave', touchEndListener)
+        deactivateFingers()
+        control.gestureStop(nextSeq())
+      }
+
+      element.on('touchstart', touchStartListener)
+      element.on('mousedown', mouseDownListener)
+
+      createSlots()
     }
   }
 }

res/app/components/stf/screen/screen.jade

@@ -13,4 +13,3 @@ div(ng-if='displayError').screen-error
         i.fa.fa-refresh
         span(translate) Retry
 input(type='password', tabindex='40', accesskey='C', autocorrect='off', autocapitalize='off', focus-element='$root.screenFocus')
-span.finger

res/app/control-panes/device-control/device-control.css

@@ -11,6 +11,9 @@
 }
 
 device-screen {
+  width: 100%;
+  height: 100%;
+  background: gray;
   position: relative;
   display: block;
   overflow: hidden;
@@ -24,6 +27,11 @@ device-screen {
   user-select: none;
 }
 
+device-screen .screen-touch {
+  width: 100%;
+  height: 100%;
+}
+
 device-screen canvas {
   position: absolute;
   top: 50%;
@@ -49,7 +57,7 @@ device-screen .finger {
   display: none;
 }
 
-device-screen.fingering .finger {
+device-screen .finger.active {
   display: block;
 }
 

res/app/control-panes/device-control/device-control.jade

@@ -36,8 +36,8 @@
                   i.fa.fa-times
 
       .as-row.fill-height
-        div(ng-controller='DeviceScreenCtrl', ng-file-drop='installFile($files)', ng-file-drag-over-class='dragover').fill-height
+        div(ng-controller='DeviceScreenCtrl', ng-if='device', ng-file-drop='installFile($files)', ng-file-drag-over-class='dragover').fill-height
-          device-screen(style='width: 100%; height: 100%; background: gray')
+          device-screen(device='device', control='control')
 
       .stf-vnc-bottom.as-row
         .controls

test/wire/seqqueue.js

@@ -0,0 +1,116 @@
+var sinon = require('sinon')
+var chai = require('chai')
+chai.use(require('sinon-chai'))
+var expect = chai.expect
+
+var SeqQueue = require('../../lib/wire/seqqueue')
+
+describe('SeqQueue', function() {
+  it('should wait until started', function() {
+    var spy = sinon.spy()
+    var q = new SeqQueue(10, Infinity)
+    q.push(0, spy)
+    expect(spy).to.not.have.been.called
+    q.start(0)
+    expect(spy).to.have.been.calledOnce
+  })
+
+  it('should call first item immediately if started', function() {
+    var spy = sinon.spy()
+    var q = new SeqQueue(10, Infinity)
+    q.start(0)
+    q.push(0, spy)
+    expect(spy).to.have.been.calledOnce
+  })
+
+  it('should call items in seq order', function() {
+    var spy1 = sinon.spy()
+    var spy2 = sinon.spy()
+    var spy3 = sinon.spy()
+    var spy4 = sinon.spy()
+    var q = new SeqQueue(10, Infinity)
+    q.start(0)
+    q.push(0, spy1)
+    q.push(1, spy2)
+    q.push(2, spy3)
+    q.push(3, spy4)
+    expect(spy1).to.have.been.calledOnce
+    expect(spy2).to.have.been.calledOnce
+    expect(spy3).to.have.been.calledOnce
+    expect(spy4).to.have.been.calledOnce
+  })
+
+  it('should not call item until seq reaches it', function() {
+    var spy1 = sinon.spy()
+    var spy2 = sinon.spy()
+    var spy3 = sinon.spy()
+    var spy4 = sinon.spy()
+    var q = new SeqQueue(10, Infinity)
+    q.start(0)
+    q.push(0, spy1)
+    q.push(3, spy4)
+    expect(spy1).to.have.been.calledOnce
+    expect(spy4).to.not.have.been.called
+    q.push(2, spy3)
+    expect(spy3).to.not.have.been.called
+    expect(spy4).to.not.have.been.called
+    q.push(1, spy2)
+    expect(spy2).to.have.been.calledOnce
+    expect(spy3).to.have.been.calledOnce
+    expect(spy4).to.have.been.calledOnce
+  })
+
+  it('should should start skipping items if too far behind', function() {
+    var spy1 = sinon.spy()
+    var spy2 = sinon.spy()
+    var spy3 = sinon.spy()
+    var spy4 = sinon.spy()
+    var q = new SeqQueue(10, 2)
+    q.start(0)
+    q.push(0, spy1)
+    q.push(2, spy3)
+    q.push(3, spy4)
+    q.push(1, spy2)
+    expect(spy1).to.have.been.calledOnce
+    expect(spy2).to.not.have.been.called
+    expect(spy3).to.have.been.calledOnce
+    expect(spy4).to.have.been.calledOnce
+  })
+
+  it('should should start a new queue', function() {
+    var spy1 = sinon.spy()
+    var spy2 = sinon.spy()
+    var spy3 = sinon.spy()
+    var spy4 = sinon.spy()
+    var q = new SeqQueue(2, Infinity)
+    q.start(0)
+    q.push(0, spy1)
+    q.push(1, spy2)
+    q.stop(2)
+    q.start(0)
+    q.push(0, spy3)
+    q.push(1, spy4)
+    expect(spy1).to.have.been.calledOnce
+    expect(spy2).to.have.been.calledOnce
+    expect(spy3).to.have.been.calledOnce
+    expect(spy4).to.have.been.calledOnce
+  })
+
+  it('should should start a new queue on even on 1 length', function() {
+    var spy1 = sinon.spy()
+    var spy2 = sinon.spy()
+    var spy3 = sinon.spy()
+    var q = new SeqQueue(1, Infinity)
+    q.start(0)
+    q.push(0, spy1)
+    q.stop(1)
+    q.start(0)
+    q.push(0, spy2)
+    q.stop(1)
+    q.start(0)
+    q.push(0, spy3)
+    expect(spy1).to.have.been.calledOnce
+    expect(spy2).to.have.been.calledOnce
+    expect(spy3).to.have.been.calledOnce
+  })
+})