diff --git a/lib/roles/app.js b/lib/roles/app.js index cff2d22e..2f2e3c62 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -8,7 +8,6 @@ var validator = require('express-validator') var socketio = require('socket.io') var zmq = require('zmq') var Promise = require('bluebird') -var adb = require('adbkit') var logger = require('../util/logger') var pathutil = require('../util/pathutil') @@ -303,32 +302,6 @@ module.exports = function(options) { ]) }) - function fixedKeySender(klass, key) { - return function(channel) { - push.send([ - channel - , wireutil.envelope(new klass( - key - )) - ]) - } - } - - socket.on('input.back', fixedKeySender( - wire.KeyPressMessage - , adb.Keycode.KEYCODE_BACK - )) - - socket.on('input.home', fixedKeySender( - wire.KeyPressMessage - , adb.Keycode.KEYCODE_HOME - )) - - socket.on('input.menu', fixedKeySender( - wire.KeyPressMessage - , adb.Keycode.KEYCODE_MENU - )) - socket.on('flick', function(data) {}) socket.on('back', function(data) {}) socket.on('forward', function(data) {}) diff --git a/lib/roles/device.js b/lib/roles/device.js index faa24afb..502b7aa9 100644 --- a/lib/roles/device.js +++ b/lib/roles/device.js @@ -19,6 +19,8 @@ var pathutil = require('../util/pathutil') var promiseutil = require('../util/promiseutil') var Vitals = require('../util/vitals') var ChannelManager = require('../wire/channelmanager') +var keyutil = require('../util/keyutil') +var inputAgent = require('../services/inputagent') module.exports = function(options) { var log = logger.createLogger('device') @@ -37,7 +39,6 @@ module.exports = function(options) { } , services = { input: null - , monkey: null , logcat: null } @@ -137,8 +138,8 @@ module.exports = function(options) { , devutil.killProcsByComm( adb , options.serial - , 'commands.monkey' - , 'com.android.commands.monkey' + , 'app_process' + , 'app_process' ) ]) }) @@ -233,18 +234,19 @@ module.exports = function(options) { }) }) .then(function() { - log.info('Launching monkey service') - return devutil.ensureUnusedPort(adb, options.serial, 1080) + log.info('Launching InputAgent') + return devutil.ensureUnusedPort(adb, options.serial, 1090) .then(function(port) { - var log = logger.createLogger('device:remote:monkey') - return adb.shellAsync(options.serial, util.format( - // Some devices fail without an SD card installed; we can - // fake an external storage using this method - 'EXTERNAL_STORAGE=/data/local/tmp monkey --port %d' - , port - )) + var log = logger.createLogger('device:inputAgent') + return promiseutil.periodicNotify( + inputAgent.open(adb, options.serial) + , 1000 + ) + .progressed(function() { + log.info('Waiting for InputAgent') + }) .then(function(out) { - vitals.register('device:remote:monkey:shell', out) + vitals.register('device:inputAgent:shell', out) out.pipe(split()) .on('data', function(chunk) { log.info(chunk) @@ -256,12 +258,9 @@ module.exports = function(options) { return devutil.waitForPort(adb, options.serial, port) }) .then(function(conn) { - return monkey.connectStream(conn) - }) - .then(function(monkey) { - services.monkey = vitals.register( - 'device:remote:monkey:monkey' - , Promise.promisifyAll(monkey) + services.inputAgentSocket = vitals.register( + 'device:inputAgent:socket' + , conn ) }) }) @@ -378,29 +377,30 @@ module.exports = function(options) { log.error('tap failed', err.stack) }) }) - .on(wire.TypeMessage, function(channel, message) { - services.monkey.typeAsync(message.text) - .catch(function(err) { - log.error('type failed', err.stack) - }) - }) .on(wire.KeyDownMessage, function(channel, message) { - services.monkey.keyDownAsync(message.key) - .catch(function(err) { - log.error('keyDown failed', err.stack) - }) + inputAgent.sendInputEvent(services.inputAgentSocket, { + action: 0 + , keyCode: keyutil.unwire(message.keyCode) + }) }) .on(wire.KeyUpMessage, function(channel, message) { - services.monkey.keyUpAsync(message.key) - .catch(function(err) { - log.error('keyUp failed', err.stack) - }) + inputAgent.sendInputEvent(services.inputAgentSocket, { + action: 1 + , keyCode: keyutil.unwire(message.keyCode) + }) }) .on(wire.KeyPressMessage, function(channel, message) { - services.monkey.pressAsync(message.key) - .catch(function(err) { - log.error('keyPress failed', err.stack) - }) + inputAgent.sendInputEvent(services.inputAgentSocket, { + action: 2 + , keyCode: keyutil.unwire(message.keyCode) + }) + }) + .on(wire.TypeMessage, function(channel, message) { + inputAgent.sendInputEvent(services.inputAgentSocket, { + action: 3 + , keyCode: 0 + , text: message.text + }) }) .on(wire.LogcatApplyFiltersMessage, function(channel, message) { resetLogcat() diff --git a/lib/services/inputagent.js b/lib/services/inputagent.js new file mode 100644 index 00000000..b6a36128 --- /dev/null +++ b/lib/services/inputagent.js @@ -0,0 +1,48 @@ +var util = require('util') + +var Promise = require('bluebird') +var ProtoBuf = require('protobufjs') +var ByteBuffer = require('protobufjs/node_modules/bytebuffer') +var split = require('split') + +var pathutil = require('../util/pathutil') +var streamutil = require('../util/streamutil') + +var proto = ProtoBuf.loadProtoFile( + pathutil.vendor('InputAgent/inputAgentProtocol.proto') +).build().jp.co.cyberagent.stf.input.agent + +var inputAgent = module.exports = Object.create(null) + +inputAgent.open = function(adb, serial) { + return adb.installAsync(serial, pathutil.vendor('InputAgent/InputAgent.apk')) + .then(function() { + return adb.shellAsync(serial, 'pm path jp.co.cyberagent.stf.input.agent') + }) + .then(function(out) { + return streamutil.findLine(out, (/^package:/)) + .then(function(line) { + return line.substr(8) + }) + }) + .then(function(apk) { + return adb.shellAsync(serial, util.format( + "export CLASSPATH='%s';" + + ' exec app_process /system/bin' + + ' jp.co.cyberagent.stf.input.agent.InputAgent' + , apk + )) + }) +} + +inputAgent.sendInputEvent = function(agent, event) { + var lengthBuffer = new ByteBuffer() + , messageBuffer = new proto.InputEvent(event).encode() + + lengthBuffer.writeVarint32(messageBuffer.length) + + agent.write(Buffer.concat([ + lengthBuffer.toBuffer() + , messageBuffer.toBuffer() + ])) +} diff --git a/lib/util/keyutil.js b/lib/util/keyutil.js index b90efcdb..4091a328 100644 --- a/lib/util/keyutil.js +++ b/lib/util/keyutil.js @@ -3,6 +3,8 @@ var util = require('util') var adb = require('adbkit') var Promise = require('bluebird') +var wire = require('../wire') + var keyutil = module.exports = Object.create(null) keyutil.parseKeyCharacterMap = function(stream) { @@ -460,7 +462,7 @@ keyutil.parseKeyCharacterMap = function(stream) { } keyutil.namedKey = function(name) { - var key = adb.Keycode['KEYCODE_' + name] + var key = adb.Keycode['KEYCODE_' + name.toUpperCase()] if (key === void 0) { throw new Error(util.format('Unknown key "%s"', name)) } @@ -570,3 +572,47 @@ keyutil.buildCharMap = function(keymap) { return charmap } + + +keyutil.unwire = (function() { + var map = Object.create(null) + + map[wire.KeyCode.HOME] = keyutil.namedKey('home') + map[wire.KeyCode.BACK] = keyutil.namedKey('back') + map[wire.KeyCode.BACKSPACE] = keyutil.namedKey('del') + map[wire.KeyCode.ENTER] = keyutil.namedKey('enter') + map[wire.KeyCode.CAPS_LOCK] = keyutil.namedKey('caps_lock') + map[wire.KeyCode.ESC] = keyutil.namedKey('escape') + map[wire.KeyCode.PAGE_UP] = keyutil.namedKey('page_up') + map[wire.KeyCode.PAGE_DOWN] = keyutil.namedKey('page_down') + map[wire.KeyCode.MOVE_END] = keyutil.namedKey('move_end') + map[wire.KeyCode.MOVE_HOME] = keyutil.namedKey('move_home') + map[wire.KeyCode.LEFT_ARROW] = keyutil.namedKey('dpad_left') + map[wire.KeyCode.UP_ARROW] = keyutil.namedKey('dpad_up') + map[wire.KeyCode.RIGHT_ARROW] = keyutil.namedKey('dpad_right') + map[wire.KeyCode.DOWN_ARROW] = keyutil.namedKey('dpad_down') + map[wire.KeyCode.INSERT] = keyutil.namedKey('insert') + map[wire.KeyCode.DELETE] = keyutil.namedKey('forward_del') + map[wire.KeyCode.MENU] = keyutil.namedKey('menu') + map[wire.KeyCode.F1] = keyutil.namedKey('f1') + map[wire.KeyCode.F2] = keyutil.namedKey('f2') + map[wire.KeyCode.F3] = keyutil.namedKey('f3') + map[wire.KeyCode.F4] = keyutil.namedKey('f4') + map[wire.KeyCode.F5] = keyutil.namedKey('f5') + map[wire.KeyCode.F6] = keyutil.namedKey('f6') + map[wire.KeyCode.F7] = keyutil.namedKey('f7') + map[wire.KeyCode.F8] = keyutil.namedKey('f8') + map[wire.KeyCode.F9] = keyutil.namedKey('f9') + map[wire.KeyCode.F10] = keyutil.namedKey('f10') + map[wire.KeyCode.F11] = keyutil.namedKey('f11') + map[wire.KeyCode.F12] = keyutil.namedKey('f12') + map[wire.KeyCode.NUM_LOCK] = keyutil.namedKey('num_lock') + + return function(keyCode) { + var key = map[keyCode] + if (!key) { + throw new Error(util.format('Unknown keycode "%s"', keyCode)) + } + return key + } +})() diff --git a/lib/util/streamutil.js b/lib/util/streamutil.js new file mode 100644 index 00000000..afc282d5 --- /dev/null +++ b/lib/util/streamutil.js @@ -0,0 +1,63 @@ +var Promise = require('bluebird') +var split = require('split') + +module.exports.readAll = function(stream) { + var resolver = Promise.defer() + , collected = new Buffer(0) + + function errorListener(err) { + resolver.reject(err) + } + + function endListener() { + resolver.resolve(collected) + } + + function readableListener() { + var chunk; + while (chunk = stream.read()) { + collected = Buffer.concat([collected, chunk]) + } + } + + stream.on('error', errorListener) + stream.on('readable', readableListener) + stream.on('end', endListener) + + return resolver.promise.finally(function() { + stream.removeListener('error', errorListener) + stream.removeListener('readable', readableListener) + stream.removeListener('end', endListener) + }) +} + + +module.exports.findLine = function(stream, re) { + var resolver = Promise.defer() + , piped = stream.pipe(split()) + + function errorListener(err) { + resolver.reject(err) + } + + function endListener() { + resolver.reject(new Error('No matching line found')) + } + + function lineListener(line) { + if (re.test(line)) { + resolver.resolve(line) + } + } + + piped.on('error', errorListener) + piped.on('data', lineListener) + piped.on('end', endListener) + + return resolver.promise.finally(function() { + piped.removeListener('error', errorListener) + piped.removeListener('data', lineListener) + piped.removeListener('end', endListener) + stream.unpipe(piped) + }) +} diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index 75db4495..71b30a0e 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -188,16 +188,49 @@ message TypeMessage { required string text = 1; } +enum KeyCode { + HOME = 3; + BACK = 4; + BACKSPACE = 8; + ENTER = 13; + CAPS_LOCK = 20; + ESC = 27; + PAGE_UP = 33; + PAGE_DOWN = 34; + MOVE_END = 35; + MOVE_HOME = 36; + LEFT_ARROW = 37; + UP_ARROW = 38; + RIGHT_ARROW = 39; + DOWN_ARROW = 40; + INSERT = 45; + DELETE = 46; + MENU = 93; + F1 = 112; + F2 = 113; + F3 = 114; + F4 = 115; + F5 = 116; + F6 = 117; + F7 = 118; + F8 = 119; + F9 = 120; + F10 = 121; + F11 = 122; + F12 = 123; + NUM_LOCK = 144; +} + message KeyDownMessage { - required uint32 key = 1; + required KeyCode keyCode = 1; } message KeyUpMessage { - required uint32 key = 1; + required KeyCode keyCode = 1; } message KeyPressMessage { - required uint32 key = 1; + required KeyCode keyCode = 1; } // Output diff --git a/res/app/scripts/controllers/DeviceScreenCtrl.js b/res/app/scripts/controllers/DeviceScreenCtrl.js index bea4f8ce..d36fbe4b 100644 --- a/res/app/scripts/controllers/DeviceScreenCtrl.js +++ b/res/app/scripts/controllers/DeviceScreenCtrl.js @@ -18,6 +18,7 @@ define(['./_module'], function(app) { var loader = new Image() , canvas = element.find('canvas')[0] , finger = element.find('span') + , input = element.find('textarea') , g = canvas.getContext('2d') , displayWidth = 0 , displayHeight = 0 @@ -96,6 +97,7 @@ define(['./_module'], function(app) { function downListener(e) { e.preventDefault() + input[0].focus() element.addClass('fingering') sendTouch('touchDown', e) element.bind('mousemove', moveListener) @@ -124,6 +126,24 @@ define(['./_module'], function(app) { stop() }) + input.bind('keydown', function(e) { + $scope.control.keyDown(e.keyCode) + }) + + input.bind('keyup', function(e) { + $scope.control.keyUp(e.keyCode) + }) + + input.bind('keypress', function(e) { + e.preventDefault() // no need to change value + $scope.control.type(String.fromCharCode(e.charCode)) + }) + + input.bind('paste', function(e) { + e.preventDefault() // no need to change value + $scope.control.type(e.clipboardData.getData('text/plain')) + }) + element.bind('mousedown', downListener) updateDisplaySize() loadScreen() diff --git a/res/app/scripts/services/ControlService.js b/res/app/scripts/services/ControlService.js index a74f4247..d19c1acf 100644 --- a/res/app/scripts/services/ControlService.js +++ b/res/app/scripts/services/ControlService.js @@ -4,6 +4,37 @@ define(['./_module', 'lodash'], function(app, _) { } function ControlService(channel) { + var keyCodes = { + 8: 8 // backspace + , 13: 13 // enter + , 20: 20 // caps lock + , 27: 27 // esc + , 33: 33 // page up + , 34: 34 // page down + , 35: 35 // end + , 36: 36 // home + , 37: 37 // left arrow + , 38: 38 // up arrow + , 39: 39 // right arrow + , 40: 40 // down arrow + , 45: 45 // insert + , 46: 46 // delete + , 93: 93 // windows menu key + , 112: 112 // f1 + , 113: 113 // f2 + , 114: 114 // f3 + , 115: 115 // f4 + , 116: 116 // f5 + , 117: 117 // f6 + , 118: 118 // f7 + , 119: 119 // f8 + , 120: 120 // f9 + , 121: 121 // f10 + , 122: 122 // f11 + , 123: 123 // f12 + , 144: 144 // num lock + } + function touchSender(type) { return function(x, y) { socket.emit(type, channel, { @@ -13,11 +44,14 @@ define(['./_module', 'lodash'], function(app, _) { } } - function keySender(type) { + function keySender(type, fixedKey) { return function(key) { - socket.emit(type, channel, { - key: key - }) + var mapped = fixedKey || keyCodes[key] + if (mapped) { + socket.emit(type, channel, { + key: mapped + }) + } } } @@ -30,17 +64,9 @@ define(['./_module', 'lodash'], function(app, _) { this.keyUp = keySender('input.keyUp') this.keyPress = keySender('input.keyPress') - this.home = function() { - socket.emit('input.home', channel) - } - - this.menu = function() { - socket.emit('input.menu', channel) - } - - this.back = function() { - socket.emit('input.back', channel) - } + this.home = keySender('input.keyPress', 3) + this.menu = keySender('input.keyPress', 93) + this.back = keySender('input.keyPress', 4) this.type = function(text) { socket.emit('input.type', channel, { diff --git a/res/app/views/partials/devices/control.jade b/res/app/views/partials/devices/control.jade index d0918fc2..a650f150 100644 --- a/res/app/views/partials/devices/control.jade +++ b/res/app/views/partials/devices/control.jade @@ -29,6 +29,13 @@ style. device-screen.fingering .finger { display: block; } + device-screen textarea { + position: absolute; + z-index: 10; + outline: none; + pointer-events: none; + opacity: 0; + } div(ng-controller='DeviceScreenCtrl') device-screen(style='width: 400px; height: 600px; background: gray') diff --git a/res/app/views/partials/devices/screen.jade b/res/app/views/partials/devices/screen.jade index 4285ba75..00e37585 100644 --- a/res/app/views/partials/devices/screen.jade +++ b/res/app/views/partials/devices/screen.jade @@ -1,3 +1,4 @@ canvas(ng-show='ready') div(ng-if='displayError') Screen error +textarea(tabindex='-1') span.finger diff --git a/vendor/InputAgent/InputAgent.apk b/vendor/InputAgent/InputAgent.apk new file mode 100644 index 00000000..05933e65 Binary files /dev/null and b/vendor/InputAgent/InputAgent.apk differ diff --git a/vendor/InputAgent/inputAgentProtocol.proto b/vendor/InputAgent/inputAgentProtocol.proto new file mode 100644 index 00000000..300f39f4 --- /dev/null +++ b/vendor/InputAgent/inputAgentProtocol.proto @@ -0,0 +1,23 @@ +package jp.co.cyberagent.stf.input.agent; + +enum InputAction { + KEYDOWN = 0; + KEYUP = 1; + KEYPRESS = 2; + TYPE = 3; +} + +message InputEvent { + required InputAction action = 1; + required int32 keyCode = 2; + optional bool shiftKey = 3; + optional bool ctrlKey = 4; + optional bool altKey = 5; + optional bool metaKey = 6; + optional bool symKey = 7; + optional bool functionKey = 8; + optional bool capsLockKey = 9; + optional bool scrollLockKey = 10; + optional bool numLockKey = 11; + optional string text = 12; +}