Use InputAgent for the keyboard.

lib/roles/app.js

@@ -8,7 +8,6 @@ var validator = require('express-validator')
 var socketio = require('socket.io')
 var zmq = require('zmq')
 var Promise = require('bluebird')
-var adb = require('adbkit')
 
 var logger = require('../util/logger')
 var pathutil = require('../util/pathutil')
@@ -303,32 +302,6 @@ module.exports = function(options) {
       ])
     })
 
-    function fixedKeySender(klass, key) {
-      return function(channel) {
-        push.send([
-          channel
-        , wireutil.envelope(new klass(
-            key
-          ))
-        ])
-      }
-    }
-
-    socket.on('input.back', fixedKeySender(
-      wire.KeyPressMessage
-    , adb.Keycode.KEYCODE_BACK
-    ))
-
-    socket.on('input.home', fixedKeySender(
-      wire.KeyPressMessage
-    , adb.Keycode.KEYCODE_HOME
-    ))
-
-    socket.on('input.menu', fixedKeySender(
-      wire.KeyPressMessage
-    , adb.Keycode.KEYCODE_MENU
-    ))
-
     socket.on('flick', function(data) {})
     socket.on('back', function(data) {})
     socket.on('forward', function(data) {})

lib/roles/device.js

@@ -19,6 +19,8 @@ var pathutil = require('../util/pathutil')
 var promiseutil = require('../util/promiseutil')
 var Vitals = require('../util/vitals')
 var ChannelManager = require('../wire/channelmanager')
+var keyutil = require('../util/keyutil')
+var inputAgent = require('../services/inputagent')
 
 module.exports = function(options) {
   var log = logger.createLogger('device')
@@ -37,7 +39,6 @@ module.exports = function(options) {
       }
     , services = {
         input: null
-      , monkey: null
       , logcat: null
       }
 
@@ -137,8 +138,8 @@ module.exports = function(options) {
       , devutil.killProcsByComm(
           adb
         , options.serial
-        , 'commands.monkey'
+        , 'app_process'
-        , 'com.android.commands.monkey'
+        , 'app_process'
         )
       ])
     })
@@ -233,18 +234,19 @@ module.exports = function(options) {
         })
     })
     .then(function() {
-      log.info('Launching monkey service')
+      log.info('Launching InputAgent')
-      return devutil.ensureUnusedPort(adb, options.serial, 1080)
+      return devutil.ensureUnusedPort(adb, options.serial, 1090)
         .then(function(port) {
-          var log = logger.createLogger('device:remote:monkey')
+          var log = logger.createLogger('device:inputAgent')
-          return adb.shellAsync(options.serial, util.format(
+          return promiseutil.periodicNotify(
-              // Some devices fail without an SD card installed; we can
+              inputAgent.open(adb, options.serial)
-              // fake an external storage using this method
+            , 1000
-              'EXTERNAL_STORAGE=/data/local/tmp monkey --port %d'
+            )
-            , port
+            .progressed(function() {
-            ))
+              log.info('Waiting for InputAgent')
+            })
             .then(function(out) {
-              vitals.register('device:remote:monkey:shell', out)
+              vitals.register('device:inputAgent:shell', out)
               out.pipe(split())
                 .on('data', function(chunk) {
                   log.info(chunk)
@@ -256,12 +258,9 @@ module.exports = function(options) {
           return devutil.waitForPort(adb, options.serial, port)
         })
         .then(function(conn) {
-          return monkey.connectStream(conn)
+          services.inputAgentSocket = vitals.register(
-        })
+            'device:inputAgent:socket'
-        .then(function(monkey) {
+          , conn
-          services.monkey = vitals.register(
-            'device:remote:monkey:monkey'
-          , Promise.promisifyAll(monkey)
           )
         })
     })
@@ -378,28 +377,29 @@ module.exports = function(options) {
           log.error('tap failed', err.stack)
         })
     })
-    .on(wire.TypeMessage, function(channel, message) {
-      services.monkey.typeAsync(message.text)
-        .catch(function(err) {
-          log.error('type failed', err.stack)
-        })
-    })
     .on(wire.KeyDownMessage, function(channel, message) {
-      services.monkey.keyDownAsync(message.key)
+      inputAgent.sendInputEvent(services.inputAgentSocket, {
-        .catch(function(err) {
+        action: 0
-          log.error('keyDown failed', err.stack)
+      , keyCode: keyutil.unwire(message.keyCode)
       })
     })
     .on(wire.KeyUpMessage, function(channel, message) {
-      services.monkey.keyUpAsync(message.key)
+      inputAgent.sendInputEvent(services.inputAgentSocket, {
-        .catch(function(err) {
+        action: 1
-          log.error('keyUp failed', err.stack)
+      , keyCode: keyutil.unwire(message.keyCode)
       })
     })
     .on(wire.KeyPressMessage, function(channel, message) {
-      services.monkey.pressAsync(message.key)
+      inputAgent.sendInputEvent(services.inputAgentSocket, {
-        .catch(function(err) {
+        action: 2
-          log.error('keyPress failed', err.stack)
+      , keyCode: keyutil.unwire(message.keyCode)
+      })
+    })
+    .on(wire.TypeMessage, function(channel, message) {
+      inputAgent.sendInputEvent(services.inputAgentSocket, {
+        action: 3
+      , keyCode: 0
+      , text: message.text
       })
     })
     .on(wire.LogcatApplyFiltersMessage, function(channel, message) {

lib/services/inputagent.js

@@ -0,0 +1,48 @@
+var util = require('util')
+
+var Promise = require('bluebird')
+var ProtoBuf = require('protobufjs')
+var ByteBuffer = require('protobufjs/node_modules/bytebuffer')
+var split = require('split')
+
+var pathutil = require('../util/pathutil')
+var streamutil = require('../util/streamutil')
+
+var proto = ProtoBuf.loadProtoFile(
+  pathutil.vendor('InputAgent/inputAgentProtocol.proto')
+).build().jp.co.cyberagent.stf.input.agent
+
+var inputAgent = module.exports = Object.create(null)
+
+inputAgent.open = function(adb, serial) {
+  return adb.installAsync(serial, pathutil.vendor('InputAgent/InputAgent.apk'))
+    .then(function() {
+      return adb.shellAsync(serial, 'pm path jp.co.cyberagent.stf.input.agent')
+    })
+    .then(function(out) {
+      return streamutil.findLine(out, (/^package:/))
+        .then(function(line) {
+          return line.substr(8)
+        })
+    })
+    .then(function(apk) {
+      return adb.shellAsync(serial, util.format(
+        "export CLASSPATH='%s';"
+      + ' exec app_process /system/bin'
+      + ' jp.co.cyberagent.stf.input.agent.InputAgent'
+      , apk
+      ))
+    })
+}
+
+inputAgent.sendInputEvent = function(agent, event) {
+  var lengthBuffer = new ByteBuffer()
+    , messageBuffer = new proto.InputEvent(event).encode()
+
+  lengthBuffer.writeVarint32(messageBuffer.length)
+
+  agent.write(Buffer.concat([
+    lengthBuffer.toBuffer()
+  , messageBuffer.toBuffer()
+  ]))
+}

lib/util/keyutil.js

@@ -3,6 +3,8 @@ var util = require('util')
 var adb = require('adbkit')
 var Promise = require('bluebird')
 
+var wire = require('../wire')
+
 var keyutil = module.exports = Object.create(null)
 
 keyutil.parseKeyCharacterMap = function(stream) {
@@ -460,7 +462,7 @@ keyutil.parseKeyCharacterMap = function(stream) {
 }
 
 keyutil.namedKey = function(name) {
-  var key = adb.Keycode['KEYCODE_' + name]
+  var key = adb.Keycode['KEYCODE_' + name.toUpperCase()]
   if (key === void 0) {
     throw new Error(util.format('Unknown key "%s"', name))
   }
@@ -570,3 +572,47 @@ keyutil.buildCharMap = function(keymap) {
 
   return charmap
 }
+
+
+keyutil.unwire = (function() {
+  var map = Object.create(null)
+
+  map[wire.KeyCode.HOME]        = keyutil.namedKey('home')
+  map[wire.KeyCode.BACK]        = keyutil.namedKey('back')
+  map[wire.KeyCode.BACKSPACE]   = keyutil.namedKey('del')
+  map[wire.KeyCode.ENTER]       = keyutil.namedKey('enter')
+  map[wire.KeyCode.CAPS_LOCK]   = keyutil.namedKey('caps_lock')
+  map[wire.KeyCode.ESC]         = keyutil.namedKey('escape')
+  map[wire.KeyCode.PAGE_UP]     = keyutil.namedKey('page_up')
+  map[wire.KeyCode.PAGE_DOWN]   = keyutil.namedKey('page_down')
+  map[wire.KeyCode.MOVE_END]    = keyutil.namedKey('move_end')
+  map[wire.KeyCode.MOVE_HOME]   = keyutil.namedKey('move_home')
+  map[wire.KeyCode.LEFT_ARROW]  = keyutil.namedKey('dpad_left')
+  map[wire.KeyCode.UP_ARROW]    = keyutil.namedKey('dpad_up')
+  map[wire.KeyCode.RIGHT_ARROW] = keyutil.namedKey('dpad_right')
+  map[wire.KeyCode.DOWN_ARROW]  = keyutil.namedKey('dpad_down')
+  map[wire.KeyCode.INSERT]      = keyutil.namedKey('insert')
+  map[wire.KeyCode.DELETE]      = keyutil.namedKey('forward_del')
+  map[wire.KeyCode.MENU]        = keyutil.namedKey('menu')
+  map[wire.KeyCode.F1]          = keyutil.namedKey('f1')
+  map[wire.KeyCode.F2]          = keyutil.namedKey('f2')
+  map[wire.KeyCode.F3]          = keyutil.namedKey('f3')
+  map[wire.KeyCode.F4]          = keyutil.namedKey('f4')
+  map[wire.KeyCode.F5]          = keyutil.namedKey('f5')
+  map[wire.KeyCode.F6]          = keyutil.namedKey('f6')
+  map[wire.KeyCode.F7]          = keyutil.namedKey('f7')
+  map[wire.KeyCode.F8]          = keyutil.namedKey('f8')
+  map[wire.KeyCode.F9]          = keyutil.namedKey('f9')
+  map[wire.KeyCode.F10]         = keyutil.namedKey('f10')
+  map[wire.KeyCode.F11]         = keyutil.namedKey('f11')
+  map[wire.KeyCode.F12]         = keyutil.namedKey('f12')
+  map[wire.KeyCode.NUM_LOCK]    = keyutil.namedKey('num_lock')
+
+  return function(keyCode) {
+    var key = map[keyCode]
+    if (!key) {
+      throw new Error(util.format('Unknown keycode "%s"', keyCode))
+    }
+    return key
+  }
+})()

lib/util/streamutil.js

@@ -0,0 +1,63 @@
+var Promise = require('bluebird')
+var split = require('split')
+
+module.exports.readAll = function(stream) {
+  var resolver = Promise.defer()
+    , collected = new Buffer(0)
+
+  function errorListener(err) {
+    resolver.reject(err)
+  }
+
+  function endListener() {
+    resolver.resolve(collected)
+  }
+
+  function readableListener() {
+    var chunk;
+    while (chunk = stream.read()) {
+      collected = Buffer.concat([collected, chunk])
+    }
+  }
+
+  stream.on('error', errorListener)
+  stream.on('readable', readableListener)
+  stream.on('end', endListener)
+
+  return resolver.promise.finally(function() {
+    stream.removeListener('error', errorListener)
+    stream.removeListener('readable', readableListener)
+    stream.removeListener('end', endListener)
+  })
+}
+
+
+module.exports.findLine = function(stream, re) {
+  var resolver = Promise.defer()
+    , piped = stream.pipe(split())
+
+  function errorListener(err) {
+    resolver.reject(err)
+  }
+
+  function endListener() {
+    resolver.reject(new Error('No matching line found'))
+  }
+
+  function lineListener(line) {
+    if (re.test(line)) {
+      resolver.resolve(line)
+    }
+  }
+
+  piped.on('error', errorListener)
+  piped.on('data', lineListener)
+  piped.on('end', endListener)
+
+  return resolver.promise.finally(function() {
+    piped.removeListener('error', errorListener)
+    piped.removeListener('data', lineListener)
+    piped.removeListener('end', endListener)
+    stream.unpipe(piped)
+  })
+}

lib/wire/wire.proto

@@ -188,16 +188,49 @@ message TypeMessage {
   required string text = 1;
 }
 
+enum KeyCode {
+  HOME        = 3;
+  BACK        = 4;
+  BACKSPACE   = 8;
+  ENTER       = 13;
+  CAPS_LOCK   = 20;
+  ESC         = 27;
+  PAGE_UP     = 33;
+  PAGE_DOWN   = 34;
+  MOVE_END    = 35;
+  MOVE_HOME   = 36;
+  LEFT_ARROW  = 37;
+  UP_ARROW    = 38;
+  RIGHT_ARROW = 39;
+  DOWN_ARROW  = 40;
+  INSERT      = 45;
+  DELETE      = 46;
+  MENU        = 93;
+  F1          = 112;
+  F2          = 113;
+  F3          = 114;
+  F4          = 115;
+  F5          = 116;
+  F6          = 117;
+  F7          = 118;
+  F8          = 119;
+  F9          = 120;
+  F10         = 121;
+  F11         = 122;
+  F12         = 123;
+  NUM_LOCK    = 144;
+}
+
 message KeyDownMessage {
-  required uint32 key = 1;
+  required KeyCode keyCode = 1;
 }
 
 message KeyUpMessage {
-  required uint32 key = 1;
+  required KeyCode keyCode = 1;
 }
 
 message KeyPressMessage {
-  required uint32 key = 1;
+  required KeyCode keyCode = 1;
 }
 
 // Output

res/app/scripts/controllers/DeviceScreenCtrl.js

@@ -18,6 +18,7 @@ define(['./_module'], function(app) {
           var loader = new Image()
             , canvas = element.find('canvas')[0]
             , finger = element.find('span')
+            , input = element.find('textarea')
             , g = canvas.getContext('2d')
             , displayWidth = 0
             , displayHeight = 0
@@ -96,6 +97,7 @@ define(['./_module'], function(app) {
 
           function downListener(e) {
             e.preventDefault()
+            input[0].focus()
             element.addClass('fingering')
             sendTouch('touchDown', e)
             element.bind('mousemove', moveListener)
@@ -124,6 +126,24 @@ define(['./_module'], function(app) {
             stop()
           })
 
+          input.bind('keydown', function(e) {
+            $scope.control.keyDown(e.keyCode)
+          })
+
+          input.bind('keyup', function(e) {
+            $scope.control.keyUp(e.keyCode)
+          })
+
+          input.bind('keypress', function(e) {
+            e.preventDefault() // no need to change value
+            $scope.control.type(String.fromCharCode(e.charCode))
+          })
+
+          input.bind('paste', function(e) {
+            e.preventDefault() // no need to change value
+            $scope.control.type(e.clipboardData.getData('text/plain'))
+          })
+
           element.bind('mousedown', downListener)
           updateDisplaySize()
           loadScreen()

res/app/scripts/services/ControlService.js

@@ -4,6 +4,37 @@ define(['./_module', 'lodash'], function(app, _) {
     }
 
     function ControlService(channel) {
+      var keyCodes = {
+        8: 8 // backspace
+      , 13: 13 // enter
+      , 20: 20 // caps lock
+      , 27: 27 // esc
+      , 33: 33 // page up
+      , 34: 34 // page down
+      , 35: 35 // end
+      , 36: 36 // home
+      , 37: 37 // left arrow
+      , 38: 38 // up arrow
+      , 39: 39 // right arrow
+      , 40: 40 // down arrow
+      , 45: 45 // insert
+      , 46: 46 // delete
+      , 93: 93 // windows menu key
+      , 112: 112 // f1
+      , 113: 113 // f2
+      , 114: 114 // f3
+      , 115: 115 // f4
+      , 116: 116 // f5
+      , 117: 117 // f6
+      , 118: 118 // f7
+      , 119: 119 // f8
+      , 120: 120 // f9
+      , 121: 121 // f10
+      , 122: 122 // f11
+      , 123: 123 // f12
+      , 144: 144 // num lock
+      }
+
       function touchSender(type) {
         return function(x, y) {
           socket.emit(type, channel, {
@@ -13,13 +44,16 @@ define(['./_module', 'lodash'], function(app, _) {
         }
       }
 
-      function keySender(type) {
+      function keySender(type, fixedKey) {
         return function(key) {
+          var mapped = fixedKey || keyCodes[key]
+          if (mapped) {
             socket.emit(type, channel, {
-            key: key
+              key: mapped
             })
           }
         }
+      }
 
       this.touchDown = touchSender('input.touchDown')
       this.touchMove = touchSender('input.touchMove')
@@ -30,17 +64,9 @@ define(['./_module', 'lodash'], function(app, _) {
       this.keyUp     = keySender('input.keyUp')
       this.keyPress  = keySender('input.keyPress')
 
-      this.home = function() {
+      this.home = keySender('input.keyPress', 3)
-        socket.emit('input.home', channel)
+      this.menu = keySender('input.keyPress', 93)
-      }
+      this.back = keySender('input.keyPress', 4)
-
-      this.menu = function() {
-        socket.emit('input.menu', channel)
-      }
-
-      this.back = function() {
-        socket.emit('input.back', channel)
-      }
 
       this.type = function(text) {
         socket.emit('input.type', channel, {

res/app/views/partials/devices/control.jade

@@ -29,6 +29,13 @@ style.
   device-screen.fingering .finger {
     display: block;
   }
+  device-screen textarea {
+    position: absolute;
+    z-index: 10;
+    outline: none;
+    pointer-events: none;
+    opacity: 0;
+  }
 
 div(ng-controller='DeviceScreenCtrl')
   device-screen(style='width: 400px; height: 600px; background: gray')

res/app/views/partials/devices/screen.jade

@@ -1,3 +1,4 @@
 canvas(ng-show='ready')
 div(ng-if='displayError') Screen error
+textarea(tabindex='-1')
 span.finger

vendor/InputAgent/inputAgentProtocol.proto

@@ -0,0 +1,23 @@
+package jp.co.cyberagent.stf.input.agent;
+
+enum InputAction {
+  KEYDOWN = 0;
+  KEYUP = 1;
+  KEYPRESS = 2;
+  TYPE = 3;
+}
+
+message InputEvent {
+  required InputAction action = 1;
+  required int32 keyCode = 2;
+  optional bool shiftKey = 3;
+  optional bool ctrlKey = 4;
+  optional bool altKey = 5;
+  optional bool metaKey = 6;
+  optional bool symKey = 7;
+  optional bool functionKey = 8;
+  optional bool capsLockKey = 9;
+  optional bool scrollLockKey = 10;
+  optional bool numLockKey = 11;
+  optional string text = 12;
+}