diff --git a/lib/units/device/plugins/touch.js b/lib/units/device/plugins/touch.js index ba87242a..f068a622 100644 --- a/lib/units/device/plugins/touch.js +++ b/lib/units/device/plugins/touch.js @@ -1,9 +1,10 @@ +var util = require('util') + var Promise = require('bluebird') var syrup = require('syrup') -var monkey = require('adbkit-monkey') +var split = require('split') var wire = require('../../../wire') -var devutil = require('../../../util/devutil') var logger = require('../../../util/logger') var lifecycle = require('../../../util/lifecycle') var streamutil = require('../../../util/streamutil') @@ -12,111 +13,138 @@ var SeqQueue = require('../../../wire/seqqueue') module.exports = syrup.serial() .dependency(require('../support/adb')) .dependency(require('../support/router')) - .dependency(require('../resources/remote')) - .dependency(require('./display')) - .dependency(require('./data')) - .define(function(options, adb, router, remote, display, data) { + .dependency(require('../resources/minitouch')) + .define(function(options, adb, router, minitouch) { var log = logger.createLogger('device:plugins:touch') var plugin = Object.create(null) - var service = { - port: 2820 - } - - function openService() { + function startService() { log.info('Launching touch service') - return devutil.ensureUnusedPort(adb, options.serial, service.port) + return adb.shell(options.serial, [ + 'exec' + , minitouch.bin + ]) .timeout(10000) - .then(function() { - return adb.shell(options.serial, [ - 'exec' - , remote.bin - , '--lib', remote.lib - , '--listen-input', service.port - ]) - .timeout(10000) - }) .then(function(out) { lifecycle.share('Touch shell', out) streamutil.talk(log, 'Touch shell says: "%s"', out) }) - .then(function() { - return devutil.waitForPort(adb, options.serial, service.port) - .timeout(15000) - }) - .then(function(conn) { - return Promise.promisifyAll(monkey.connectStream(conn)) - }) - .then(function(monkey) { - return lifecycle.share('Touch monkey', monkey) - }) } - function modifyCoords(message) { - message.x = Math.floor(message.x * display.width) - message.y = Math.floor(message.y * display.height) + function connectService() { + function tryConnect(times, delay) { + return adb.openLocal(options.serial, '/data/local/tmp/minitouch.sock') + .timeout(10000) + .then(function(out) { + lifecycle.share('Touch socket', out) + return out + }) + .then(function(out) { + return new Promise(function(resolve, reject) { + out.pipe(split()).on('data', function(line) { + var args = line.toString().split(/ /g) + switch (args[0]) { + case 'v': + out.version = +args[1] + log.info('Touch protocol is version %d', out.version) + break + case '^': + out.maxContacts = args[1] + out.maxX = args[2] + out.maxY = args[3] + out.maxPressure = args[4] + log.info( + 'Touch protocol reports %d contacts in a %dx%d grid ' + + 'with a max pressure of %d' + , out.maxContacts + , out.maxX + , out.maxY + , out.maxPressure + ) + return resolve(out) + default: + return reject(new Error(util.format( + 'Unknown metadata "%s"' + , line + ))) + } + }) + }) + }) + .catch(function(err) { + if (/closed/.test(err.message) && times > 1) { + return Promise.delay(delay) + .then(function() { + return tryConnect(--times, delay * 2) + }) + } + return Promise.reject(err) + }) + } + log.info('Connecting to touch service') + return tryConnect(5, 100) } - return openService() - .then(function(monkey) { - var queue = new SeqQueue() - , pressure = (data && data.touch && data.touch.defaultPressure) || 50 + return startService() + .then(connectService) + .then(function(socket) { + var queue = new SeqQueue(100, 4) - log.info('Setting default pressure to %d', pressure) + function send(command) { + socket.write(command) + } plugin.touchDown = function(point) { - modifyCoords(point) - monkey.sendAsync([ - 'touch down' - , point.x - , point.y - , pressure - ].join(' ')) - .catch(function(err) { - log.error('touchDown failed', err.stack) - }) + send(util.format( + 'd %s %s %s %s\n' + , point.contact + , Math.floor(point.x * socket.maxX) + , Math.floor(point.y * socket.maxY) + , Math.floor((point.pressure || 0.5) * socket.maxPressure) + )) } plugin.touchMove = function(point) { - modifyCoords(point) - monkey.sendAsync([ - 'touch move' - , point.x - , point.y - , pressure - ].join(' ')) - .catch(function(err) { - log.error('touchMove failed', err.stack) - }) + send(util.format( + 'm %s %s %s %s\n' + , point.contact + , Math.floor(point.x * socket.maxX) + , Math.floor(point.y * socket.maxX) + , Math.floor((point.pressure || 0.5) * socket.maxPressure) + )) } plugin.touchUp = function(point) { - modifyCoords(point) - monkey.sendAsync([ - 'touch up' - , point.x - , point.y - , pressure - ].join(' ')) - .catch(function(err) { - log.error('touchUp failed', err.stack) - }) + send(util.format( + 'u %s\n' + , point.contact + )) + } + + plugin.touchCommit = function() { + send('c\n') + } + + plugin.touchReset = function() { + send('r\n') } plugin.tap = function(point) { - modifyCoords(point) - monkey.sendAsync([ - 'tap' - , point.x - , point.y - , pressure - ].join(' ')) - .catch(function(err) { - log.error('tap failed', err.stack) - }) + plugin.touchDown(point) + plugin.touchCommit() + plugin.touchUp(point) + plugin.touchCommit() } router + .on(wire.GestureStartMessage, function(channel, message) { + queue.start(message.seq) + }) + .on(wire.GestureStopMessage, function(channel, message) { + queue.push(message.seq, function() { + queue.stop() + }) + }) .on(wire.TouchDownMessage, function(channel, message) { queue.push(message.seq, function() { plugin.touchDown(message) @@ -131,12 +159,16 @@ module.exports = syrup.serial() queue.push(message.seq, function() { plugin.touchUp(message) }) - - // Reset queue - queue = new SeqQueue() }) - .on(wire.TapMessage, function(channel, message) { - plugin.tap(message) + .on(wire.TouchCommitMessage, function(channel, message) { + queue.push(message.seq, function() { + plugin.touchCommit() + }) + }) + .on(wire.TouchResetMessage, function(channel, message) { + queue.push(message.seq, function() { + plugin.touchReset() + }) }) }) .return(plugin) diff --git a/lib/units/device/resources/minitouch.js b/lib/units/device/resources/minitouch.js new file mode 100644 index 00000000..f847af47 --- /dev/null +++ b/lib/units/device/resources/minitouch.js @@ -0,0 +1,93 @@ +var util = require('util') + +var Promise = require('bluebird') +var syrup = require('syrup') + +var logger = require('../../../util/logger') +var pathutil = require('../../../util/pathutil') +var devutil = require('../../../util/devutil') +var streamutil = require('../../../util/streamutil') + +module.exports = syrup.serial() + .dependency(require('../support/adb')) + .dependency(require('../support/properties')) + .define(function(options, adb, properties) { + var log = logger.createLogger('device:resources:minitouch') + + var resources = { + bin: { + src: pathutil.vendor(util.format( + 'minitouch/%s/minitouch' + , properties['ro.product.cpu.abi'] + )) + , dest: '/data/local/tmp/minitouch' + , comm: 'minitouch' + , mode: 0755 + } + } + + function removeResource(res) { + return adb.shell(options.serial, ['rm', res.dest]) + .timeout(10000) + .then(function(out) { + return streamutil.readAll(out) + }) + .return(res) + } + + function installResource(res) { + return adb.push(options.serial, res.src, res.dest, res.mode) + .timeout(10000) + .then(function(transfer) { + return new Promise(function(resolve, reject) { + transfer.on('error', reject) + transfer.on('end', resolve) + }) + }) + .return(res) + } + + function ensureNotBusy(res) { + return adb.shell(options.serial, [res.dest, '--help']) + .timeout(10000) + .then(function(out) { + // Can be "Text is busy", "text busy" + return streamutil.findLine(out, (/busy/i)) + .timeout(10000) + .then(function() { + log.info('Binary is busy, will retry') + return Promise.delay(1000) + }) + .then(function() { + return ensureNotBusy(res) + }) + .catch(streamutil.NoSuchLineError, function() { + return res + }) + }) + } + + function installAll() { + return Promise.all([ + removeResource(resources.bin).then(installResource).then(ensureNotBusy) + ]) + } + + function stop() { + return devutil.killProcsByComm( + adb + , options.serial + , resources.bin.comm + , resources.bin.dest + ) + .timeout(15000) + } + + return stop() + .then(installAll) + .then(function() { + return { + bin: resources.bin.dest + } + }) + }) diff --git a/lib/units/websocket/index.js b/lib/units/websocket/index.js index dfe5426d..5bd19546 100644 --- a/lib/units/websocket/index.js +++ b/lib/units/websocket/index.js @@ -82,19 +82,6 @@ module.exports = function(options) { sub.unsubscribe(channel) } - function createTouchHandler(Klass) { - return function(channel, data) { - push.send([ - channel - , wireutil.envelope(new Klass( - data.seq - , data.x - , data.y - )) - ]) - } - } - function createKeyHandler(Klass) { return function(channel, data) { push.send([ @@ -268,10 +255,71 @@ module.exports = function(options) { dbapi.resetUserSettings(user.email) }) // Touch events - .on('input.touchDown', createTouchHandler(wire.TouchDownMessage)) - .on('input.touchMove', createTouchHandler(wire.TouchMoveMessage)) - .on('input.touchUp', createTouchHandler(wire.TouchUpMessage)) - .on('input.tap', createTouchHandler(wire.TapMessage)) + .on('input.touchDown', function(channel, data) { + push.send([ + channel + , wireutil.envelope(new wire.TouchDownMessage( + data.seq + , data.contact + , data.x + , data.y + , data.pressure + )) + ]) + }) + .on('input.touchMove', function(channel, data) { + push.send([ + channel + , wireutil.envelope(new wire.TouchMoveMessage( + data.seq + , data.contact + , data.x + , data.y + , data.pressure + )) + ]) + }) + .on('input.touchUp', function(channel, data) { + push.send([ + channel + , wireutil.envelope(new wire.TouchUpMessage( + data.seq + , data.contact + )) + ]) + }) + .on('input.touchCommit', function(channel, data) { + push.send([ + channel + , wireutil.envelope(new wire.TouchCommitMessage( + data.seq + )) + ]) + }) + .on('input.touchReset', function(channel, data) { + push.send([ + channel + , wireutil.envelope(new wire.TouchResetMessage( + data.seq + )) + ]) + }) + .on('input.gestureStart', function(channel, data) { + push.send([ + channel + , wireutil.envelope(new wire.GestureStartMessage( + data.seq + )) + ]) + }) + .on('input.gestureStop', function(channel, data) { + push.send([ + channel + , wireutil.envelope(new wire.GestureStopMessage( + data.seq + )) + ]) + }) // Key events .on('input.keyDown', createKeyHandler(wire.KeyDownMessage)) .on('input.keyUp', createKeyHandler(wire.KeyUpMessage)) diff --git a/lib/wire/seqqueue.js b/lib/wire/seqqueue.js index 4e5fc499..812854d1 100644 --- a/lib/wire/seqqueue.js +++ b/lib/wire/seqqueue.js @@ -1,25 +1,61 @@ -function SeqQueue() { - this.queue = [] - this.seq = 0 +function SeqQueue(size, maxWaiting) { + this.lo = 0 + this.size = size + this.maxWaiting = maxWaiting + this.waiting = 0 + this.list = new Array(size) + this.locked = true +} + +SeqQueue.prototype.start = function(seq) { + this.locked = false + this.lo = seq + this.maybeConsume() +} + +SeqQueue.prototype.stop = function() { + this.locked = true + this.maybeConsume() } SeqQueue.prototype.push = function(seq, handler) { - this.queue[seq] = handler - this.maybeDequeue() + if (seq >= this.size) { + return + } + + this.list[seq] = handler + this.waiting += 1 + this.maybeConsume() } -SeqQueue.prototype.done = function(seq, handler) { - this.queue[seq] = handler - this.maybeDequeue() -} +SeqQueue.prototype.maybeConsume = function() { + if (this.locked) { + return + } -SeqQueue.prototype.maybeDequeue = function() { - var handler + while (this.waiting) { + // Did we reach the end of the loop? If so, start from the beginning. + if (this.lo === this.size) { + this.lo = 0 + } - while ((handler = this.queue[this.seq])) { - this.queue[this.seq] = void 0 - handler() - this.seq += 1 + var handler = this.list[this.lo] + // Have we received it yet? + if (handler) { + this.list[this.lo] = void 0 + handler() + this.lo += 1 + this.waiting -= 1 + } + // Are we too much behind? If so, just move on. + else if (this.waiting >= this.maxWaiting) { + this.lo += 1 + this.waiting -= 1 + } + // We don't have it yet, stop. + else { + break + } } } diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index 6f74e6d2..718aba70 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -25,10 +25,13 @@ enum MessageType { ProbeMessage = 17; ShellCommandMessage = 18; ShellKeepAliveMessage = 19; - TapMessage = 20; TouchDownMessage = 21; TouchMoveMessage = 22; TouchUpMessage = 23; + TouchCommitMessage = 65; + TouchResetMessage = 66; + GestureStartMessage = 67; + GestureStopMessage = 68; TransactionDoneMessage = 24; TransactionProgressMessage = 25; TypeMessage = 26; @@ -249,25 +252,39 @@ message PhysicalIdentifyMessage { message TouchDownMessage { required uint32 seq = 1; - required float x = 2; - required float y = 3; + required uint32 contact = 2; + required float x = 3; + required float y = 4; + optional float pressure = 5; } message TouchMoveMessage { required uint32 seq = 1; - required float x = 2; - required float y = 3; + required uint32 contact = 2; + required float x = 3; + required float y = 4; + optional float pressure = 5; } message TouchUpMessage { required uint32 seq = 1; - required float x = 2; - required float y = 3; + required uint32 contact = 2; } -message TapMessage { - required float x = 1; - required float y = 2; +message TouchCommitMessage { + required uint32 seq = 1; +} + +message TouchResetMessage { + required uint32 seq = 1; +} + +message GestureStartMessage { + required uint32 seq = 1; +} + +message GestureStopMessage { + required uint32 seq = 1; } message TypeMessage { diff --git a/res/app/components/stf/control/control-service.js b/res/app/components/stf/control/control-service.js index 7740bc66..27a2322b 100644 --- a/res/app/components/stf/control/control-service.js +++ b/res/app/components/stf/control/control-service.js @@ -21,16 +21,6 @@ module.exports = function ControlServiceFactory( return tx.promise } - function touchSender(type) { - return function(seq, x, y) { - sendOneWay(type, { - seq: seq - , x: x - , y: y - }) - } - } - function keySender(type, fixedKey) { return function(key) { if (typeof key === 'string') { @@ -49,10 +39,56 @@ module.exports = function ControlServiceFactory( } } - this.touchDown = touchSender('input.touchDown') - this.touchMove = touchSender('input.touchMove') - this.touchUp = touchSender('input.touchUp') - this.tap = touchSender('input.tap') + this.gestureStart = function(seq) { + sendOneWay('input.gestureStart', { + seq: seq + }) + } + + this.gestureStop = function(seq) { + sendOneWay('input.gestureStop', { + seq: seq + }) + } + + this.touchDown = function(seq, contact, x, y, pressure) { + sendOneWay('input.touchDown', { + seq: seq + , contact: contact + , x: x + , y: y + , pressure: pressure + }) + } + + this.touchMove = function(seq, contact, x, y, pressure) { + sendOneWay('input.touchMove', { + seq: seq + , contact: contact + , x: x + , y: y + , pressure: pressure + }) + } + + this.touchUp = function(seq, contact) { + sendOneWay('input.touchUp', { + seq: seq + , contact: contact + }) + } + + this.touchCommit = function(seq) { + sendOneWay('input.touchCommit', { + seq: seq + }) + } + + this.touchReset = function(seq) { + sendOneWay('input.touchReset', { + seq: seq + }) + } this.keyDown = keySender('input.keyDown') this.keyUp = keySender('input.keyUp') diff --git a/res/app/components/stf/screen/screen-directive.js b/res/app/components/stf/screen/screen-directive.js index 734cccd7..a1ec566f 100644 --- a/res/app/components/stf/screen/screen-directive.js +++ b/res/app/components/stf/screen/screen-directive.js @@ -4,31 +4,52 @@ var _ = require('lodash') module.exports = function DeviceScreenDirective($document, ScalingService, VendorUtil, PageVisibilityService, BrowserInfo, $timeout) { return { - restrict: 'E', - template: require('./screen.jade'), - link: function (scope, element) { + restrict: 'E' + , template: require('./screen.jade') + , scope: { + control: '&' + , device: '&' + } + , link: function (scope, element) { + var device = scope.device() + , control = scope.control() + var canvas = element.find('canvas')[0] + , input = element.find('input') + var imageRender = new FastImageRender(canvas, { render: 'canvas', timeout: 3000 }) var guestDisplayDensity = setDisplayDensity(1.5) //var guestDisplayRotation = 0 - var finger = element.find('span') - var input = element.find('input') - var boundingWidth = 0 // TODO: cache inside FastImageRender? - var boundingHeight = 0 - var cachedBoundingWidth = 0 - var cachedBoundingHeight = 0 - var cachedImageWidth = 0 - var cachedImageHeight = 0 - var cachedRotation = 0 - var rotation = 0 + var loading = false - var scaler - var seq = 0 var cssTransform = VendorUtil.style(['transform', 'webkitTransform']) + var screen = scope.screen = { + scaler: ScalingService.coordinator( + device.display.width, device.display.height + ) + , rotation: 0 + , bounds: { + x: 0 + , y: 0 + , w: 0 + , h: 0 + } + } + + var cachedScreen = { + rotation: 0 + , bounds: { + x: 0 + , y: 0 + , w: 0 + , h: 0 + } + } + // NOTE: instead of fa-pane-resize, a fa-child-pane-resize could be better var onPanelResizeThrottled = _.throttle(updateBounds, 16) scope.$on('fa-pane-resize', onPanelResizeThrottled) @@ -42,88 +63,20 @@ module.exports = function DeviceScreenDirective($document, ScalingService, return guestDisplayDensity } - function sendTouch(type, e) { - var x = e.offsetX || e.layerX || 0 - var y = e.offsetY || e.layerY || 0 - var r = scope.device.display.rotation - - if (BrowserInfo.touch) { - if (e.touches && e.touches.length) { - x = e.touches[0].pageX - y = e.touches[0].pageY - } else if (e.changedTouches && e.changedTouches.length) { - x = e.changedTouches[0].pageX - y = e.changedTouches[0].pageY - } - } - - var scaled = scaler.coords(boundingWidth, boundingHeight, x, y, r) - - finger[0].style[cssTransform] = - 'translate3d(' + x + 'px,' + y + 'px,0)' - - scope.control[type]( - seq++, scaled.xP, scaled.yP - ) - } - - function stopTouch() { - element.removeClass('fingering') - if (BrowserInfo.touch) { - element.unbind('touchmove', moveListener) - $document.unbind('touchend', upListener) - $document.unbind('touchleave', upListener) - } else { - element.unbind('mousemove', moveListener) - $document.unbind('mouseup', upListener) - $document.unbind('mouseleave', upListener) - } - seq = 0 - } - function updateBounds() { - boundingWidth = element[0].offsetWidth - boundingHeight = element[0].offsetHeight + screen.bounds.w = element[0].offsetWidth + screen.bounds.h = element[0].offsetHeight // TODO: element is an object HTMLUnknownElement in IE9 // Developer error, let's try to reduce debug time - if (!boundingWidth || !boundingHeight) { + if (!screen.bounds.w || !screen.bounds.h) { throw new Error( 'Unable to update display size; container must have dimensions' ) } } - function downListener(e) { - e.preventDefault() - if (!BrowserInfo.touch) { - input[0].focus() - element.addClass('fingering') - } - - sendTouch('touchDown', e) - - if (BrowserInfo.touch) { - element.bind('touchmove', moveListener) - $document.bind('touchend', upListener) - $document.bind('touchleave', upListener) - } else { - element.bind('mousemove', moveListener) - $document.bind('mouseup', upListener) - $document.bind('mouseleave', upListener) - } - } - - function moveListener(e) { - sendTouch('touchMove', e) - } - - function upListener(e) { - sendTouch('touchUp', e) - stopTouch() - } - function isChangeCharsetKey(e) { // Add any special key here for changing charset //console.log('e', e) @@ -162,7 +115,7 @@ module.exports = function DeviceScreenDirective($document, ScalingService, if (isChangeCharsetKey(e)) { specialKey = true - scope.control.keyPress('switch_charset') + control.keyPress('switch_charset') } if (specialKey) { @@ -173,58 +126,58 @@ module.exports = function DeviceScreenDirective($document, ScalingService, } function keydownListener(e) { - scope.control.keyDown(e.keyCode) + control.keyDown(e.keyCode) } function keyupListener(e) { if (!keyupSpecialKeys(e)) { - scope.control.keyUp(e.keyCode) + control.keyUp(e.keyCode) } } function keypressListener(e) { e.preventDefault() // no need to change value - scope.control.type(String.fromCharCode(e.charCode)) + control.type(String.fromCharCode(e.charCode)) } function pasteListener(e) { e.preventDefault() // no need to change value - scope.control.paste(e.clipboardData.getData('text/plain')) + control.paste(e.clipboardData.getData('text/plain')) } function copyListener(e) { - scope.control.getClipboardContent() + control.getClipboardContent() // @TODO: OK, this basically copies last clipboard content - if (scope.control.clipboardContent) { - e.clipboardData.setData("text/plain", scope.control.clipboardContent) + if (control.clipboardContent) { + e.clipboardData.setData("text/plain", control.clipboardContent) } e.preventDefault() } scope.retryLoadingScreen = function () { if (scope.displayError === 'secure') { - scope.control.home() + control.home() } $timeout(maybeLoadScreen, 3000) } function maybeLoadScreen() { - if (!loading && scope.$parent.showScreen && scope.device) { + if (!loading && scope.$parent.showScreen && device) { var w, h - switch (rotation) { + switch (screen.rotation) { case 0: case 180: - w = boundingWidth - h = boundingHeight + w = screen.bounds.w + h = screen.bounds.h break case 90: case 270: - w = boundingHeight - h = boundingWidth + w = screen.bounds.h + h = screen.bounds.w break } loading = true - imageRender.load(scope.device.display.url + + imageRender.load(device.display.url + '?width=' + Math.ceil(w * guestDisplayDensity) + '&height=' + Math.ceil(h * guestDisplayDensity) + '&time=' + Date.now() @@ -233,35 +186,37 @@ module.exports = function DeviceScreenDirective($document, ScalingService, } function on() { - scaler = ScalingService.coordinator( - scope.device.display.width, scope.device.display.height - ) - imageRender.onLoad = function (image) { + var cachedImageWidth = 0 + , cachedImageHeight = 0 + loading = false if (scope.$parent.showScreen) { + screen.rotation = device.display.rotation // Check to set the size only if updated - if (cachedBoundingWidth !== boundingWidth || - cachedBoundingHeight !== boundingHeight || + if (cachedScreen.bounds.w !== screen.bounds.w || + cachedScreen.bounds.h !== screen.bounds.h || cachedImageWidth !== image.width || cachedImageHeight !== image.height || - cachedRotation !== rotation) { + cachedScreen.rotation !== screen.rotation) { - cachedBoundingWidth = boundingWidth - cachedBoundingHeight = boundingHeight + cachedScreen.bounds.w = screen.bounds.w + cachedScreen.bounds.h = screen.bounds.h cachedImageWidth = image.width cachedImageHeight = image.height - cachedRotation = rotation + cachedScreen.rotation = screen.rotation imageRender.canvasWidth = cachedImageWidth imageRender.canvasHeight = cachedImageHeight - var projectedSize = scaler.projectedSize( - boundingWidth, boundingHeight, rotation + var projectedSize = screen.scaler.projectedSize( + screen.bounds.w + , screen.bounds.h + , screen.rotation ) imageRender.canvasStyleWidth = projectedSize.width @@ -270,7 +225,7 @@ module.exports = function DeviceScreenDirective($document, ScalingService, // @todo Make sure that each position is able to rotate smoothly // to the next one. This current setup doesn't work if rotation // changes from 180 to 270 (it will do a reverse rotation). - switch (rotation) { + switch (screen.rotation) { case 0: canvas.style[cssTransform] = 'translate(-50%, -50%) rotate(0deg)' break @@ -321,30 +276,17 @@ module.exports = function DeviceScreenDirective($document, ScalingService, input.bind('keypress', keypressListener) input.bind('paste', pasteListener) input.bind('copy', copyListener) - - if (BrowserInfo.touch) { - element.bind('touchstart', downListener) - } else { - element.bind('mousedown', downListener) - } - } function off() { imageRender.onLoad = imageRender.onError = null loading = false - stopTouch() + input.unbind('keydown', keydownListener) input.unbind('keyup', keyupListener) input.unbind('keypress', keypressListener) input.unbind('paste', pasteListener) input.unbind('copy', copyListener) - - if (BrowserInfo.touch) { - element.unbind('touchstart', downListener) - } else { - element.unbind('mousedown', downListener) - } } scope.$watch('$parent.showScreen', function (val) { @@ -357,7 +299,7 @@ module.exports = function DeviceScreenDirective($document, ScalingService, }) function checkEnabled() { - var using = scope.device && scope.device.using + var using = device && device.using if (using && !PageVisibilityService.hidden) { on() } @@ -369,22 +311,276 @@ module.exports = function DeviceScreenDirective($document, ScalingService, scope.$watch('device.using', checkEnabled) scope.$on('visibilitychange', checkEnabled) - scope.$watch('device.display.rotation', function (r) { - rotation = r || 0 - }) - scope.$on('guest-portrait', function () { - scope.control.rotate(0) + control.rotate(0) updateBounds() }) scope.$on('guest-landscape', function () { - scope.control.rotate(90) + control.rotate(90) setDisplayDensity(2) updateBounds() }) scope.$on('$destroy', off) + + // @todo Move everything below this line elsewhere. + var slots = [] + , slotted = Object.create(null) + , fingers = [] + , seq = -1 + , cycle = 100 + + function nextSeq() { + return ++seq >= cycle ? (seq = 0) : seq + } + + function createSlots() { + for (var i = 9; i >= 0; --i) { + slots.push(i) + fingers.unshift(createFinger(i)) + } + } + + function activateFinger(index, x, y, pressure) { + var scale = 0.5 + pressure + fingers[index].classList.add('active') + fingers[index].style[cssTransform] = + 'translate3d(' + x + 'px,' + y + 'px,0) ' + + 'scale(' + scale + ',' + scale + ')' + } + + function deactivateFinger(index) { + fingers[index].classList.remove('active') + } + + function deactivateFingers() { + for (var i = 0, l = fingers.length; i < l; ++i) { + fingers[i].classList.remove('active') + } + } + + function createFinger(index) { + var el = document.createElement('span') + el.className = 'finger finger-' + index + return el + } + + function calculateBounds() { + var el = element[0] + + screen.bounds.w = el.offsetWidth + screen.bounds.h = el.offsetHeight + screen.bounds.x = 0 + screen.bounds.y = 0 + + while (el.offsetParent) { + screen.bounds.x += el.offsetLeft + screen.bounds.y += el.offsetTop + el = el.offsetParent + } + } + + function mouseDownListener(e) { + e.preventDefault() + + calculateBounds() + startMousing() + + var x = e.pageX - screen.bounds.x + , y = e.pageY - screen.bounds.y + , pressure = 0.5 + , scaled = screen.scaler.coords( + screen.bounds.w + , screen.bounds.h + , x + , y + , screen.rotation + ) + + control.touchDown(nextSeq(), 0, scaled.xP, scaled.yP, pressure) + control.touchCommit(nextSeq()) + + activateFinger(0, x, y, pressure) + + element.bind('mousemove', mouseMoveListener) + $document.bind('mouseup', mouseUpListener) + $document.bind('mouseleave', mouseUpListener) + } + + function mouseMoveListener(e) { + e.preventDefault() + + var x = e.pageX - screen.bounds.x + , y = e.pageY - screen.bounds.y + , pressure = 0.5 + , scaled = screen.scaler.coords( + screen.bounds.w + , screen.bounds.h + , x + , y + , screen.rotation + ) + + control.touchMove(nextSeq(), 0, scaled.xP, scaled.yP, pressure) + control.touchCommit(nextSeq()) + + activateFinger(0, x, y, pressure) + } + + function mouseUpListener(e) { + e.preventDefault() + + control.touchUp(nextSeq(), 0) + control.touchCommit(nextSeq()) + + deactivateFinger(0) + + stopMousing() + } + + function startMousing() { + control.gestureStart(nextSeq()) + } + + function stopMousing() { + element.unbind('mousemove', mouseMoveListener) + $document.unbind('mouseup', mouseUpListener) + $document.unbind('mouseleave', mouseUpListener) + deactivateFingers() + control.gestureStop(nextSeq()) + } + + function touchStartListener(e) { + e.preventDefault() + + calculateBounds() + + if (e.touches.length === e.changedTouches.length) { + startTouching() + } + + var currentTouches = Object.create(null) + var i, l + + for (i = 0, l = e.touches.length; i < l; ++i) { + currentTouches[e.touches[i].identifier] = 1; + } + + function maybeLostTouchEnd(id) { + return !(id in currentTouches) + } + + // We might have lost a touchend event due to various edge cases + // (literally) such as dragging from the bottom of the screen so that + // the control center appears. If so, let's ask for a reset. + if (Object.keys(slotted).some(maybeLostTouchEnd)) { + Object.keys(slotted).forEach(function(id) { + slots.push(slotted[id]) + delete slotted[id] + }) + slots.sort().reverse() + control.touchReset(nextSeq()) + deactivateFingers() + } + + if (!slots.length) { + // This should never happen but who knows... + throw new Error('Ran out of multitouch slots') + } + + for (i = 0, l = e.changedTouches.length; i < l; ++i) { + var touch = e.changedTouches[i] + , slot = slots.pop() + , x = touch.pageX - screen.bounds.x + , y = touch.pageY - screen.bounds.y + , pressure = touch.force || 0.5 + , scaled = screen.scaler.coords( + screen.bounds.w + , screen.bounds.h + , x + , y + , screen.rotation + ) + + slotted[touch.identifier] = slot + control.touchDown(nextSeq(), slot, scaled.xP, scaled.yP, pressure) + activateFinger(slot, x, y, pressure) + } + + element.bind('touchmove', touchMoveListener) + $document.bind('touchend', touchEndListener) + $document.bind('touchleave', touchEndListener) + + control.touchCommit(nextSeq()) + } + + function touchMoveListener(e) { + e.preventDefault() + + for (var i = 0, l = e.changedTouches.length; i < l; ++i) { + var touch = e.changedTouches[i] + , slot = slotted[touch.identifier] + , x = touch.pageX - screen.bounds.x + , y = touch.pageY - screen.bounds.y + , pressure = touch.force || 0.5 + , scaled = screen.scaler.coords( + screen.bounds.w + , screen.bounds.h + , x + , y + , screen.rotation + ) + + control.touchMove(nextSeq(), slot, scaled.xP, scaled.yP, pressure) + activateFinger(slot, x, y, pressure) + } + + control.touchCommit(nextSeq()) + } + + function touchEndListener(e) { + var foundAny = false + + for (var i = 0, l = e.changedTouches.length; i < l; ++i) { + var touch = e.changedTouches[i] + , slot = slotted[touch.identifier] + if (slot === void 0) { + // We've already disposed of the contact. We may have gotten a + // touchend event for the same contact twice. + continue + } + delete slotted[touch.identifier] + slots.push(slot) + control.touchUp(nextSeq(), slot) + deactivateFinger(slot) + foundAny = true + } + + if (foundAny) { + control.touchCommit(nextSeq()) + if (!e.touches.length) { + stopTouching() + } + } + } + + function startTouching() { + control.gestureStart(nextSeq()) + } + + function stopTouching() { + element.unbind('touchmove', touchMoveListener) + $document.unbind('touchend', touchEndListener) + $document.unbind('touchleave', touchEndListener) + deactivateFingers() + control.gestureStop(nextSeq()) + } + + element.on('touchstart', touchStartListener) + element.on('mousedown', mouseDownListener) + + createSlots() } } } diff --git a/res/app/components/stf/screen/screen.jade b/res/app/components/stf/screen/screen.jade index db65acfd..dceb2586 100644 --- a/res/app/components/stf/screen/screen.jade +++ b/res/app/components/stf/screen/screen.jade @@ -13,4 +13,3 @@ div(ng-if='displayError').screen-error i.fa.fa-refresh span(translate) Retry input(type='password', tabindex='40', accesskey='C', autocorrect='off', autocapitalize='off', focus-element='$root.screenFocus') -span.finger diff --git a/res/app/control-panes/device-control/device-control.css b/res/app/control-panes/device-control/device-control.css index 4a0e71e7..b045ecb7 100644 --- a/res/app/control-panes/device-control/device-control.css +++ b/res/app/control-panes/device-control/device-control.css @@ -11,6 +11,9 @@ } device-screen { + width: 100%; + height: 100%; + background: gray; position: relative; display: block; overflow: hidden; @@ -24,6 +27,11 @@ device-screen { user-select: none; } +device-screen .screen-touch { + width: 100%; + height: 100%; +} + device-screen canvas { position: absolute; top: 50%; @@ -49,7 +57,7 @@ device-screen .finger { display: none; } -device-screen.fingering .finger { +device-screen .finger.active { display: block; } diff --git a/res/app/control-panes/device-control/device-control.jade b/res/app/control-panes/device-control/device-control.jade index 4d49d435..b7b2efc5 100644 --- a/res/app/control-panes/device-control/device-control.jade +++ b/res/app/control-panes/device-control/device-control.jade @@ -36,8 +36,8 @@ i.fa.fa-times .as-row.fill-height - div(ng-controller='DeviceScreenCtrl', ng-file-drop='installFile($files)', ng-file-drag-over-class='dragover').fill-height - device-screen(style='width: 100%; height: 100%; background: gray') + div(ng-controller='DeviceScreenCtrl', ng-if='device', ng-file-drop='installFile($files)', ng-file-drag-over-class='dragover').fill-height + device-screen(device='device', control='control') .stf-vnc-bottom.as-row .controls diff --git a/test/wire/seqqueue.js b/test/wire/seqqueue.js new file mode 100644 index 00000000..193e8ab8 --- /dev/null +++ b/test/wire/seqqueue.js @@ -0,0 +1,116 @@ +var sinon = require('sinon') +var chai = require('chai') +chai.use(require('sinon-chai')) +var expect = chai.expect + +var SeqQueue = require('../../lib/wire/seqqueue') + +describe('SeqQueue', function() { + it('should wait until started', function() { + var spy = sinon.spy() + var q = new SeqQueue(10, Infinity) + q.push(0, spy) + expect(spy).to.not.have.been.called + q.start(0) + expect(spy).to.have.been.calledOnce + }) + + it('should call first item immediately if started', function() { + var spy = sinon.spy() + var q = new SeqQueue(10, Infinity) + q.start(0) + q.push(0, spy) + expect(spy).to.have.been.calledOnce + }) + + it('should call items in seq order', function() { + var spy1 = sinon.spy() + var spy2 = sinon.spy() + var spy3 = sinon.spy() + var spy4 = sinon.spy() + var q = new SeqQueue(10, Infinity) + q.start(0) + q.push(0, spy1) + q.push(1, spy2) + q.push(2, spy3) + q.push(3, spy4) + expect(spy1).to.have.been.calledOnce + expect(spy2).to.have.been.calledOnce + expect(spy3).to.have.been.calledOnce + expect(spy4).to.have.been.calledOnce + }) + + it('should not call item until seq reaches it', function() { + var spy1 = sinon.spy() + var spy2 = sinon.spy() + var spy3 = sinon.spy() + var spy4 = sinon.spy() + var q = new SeqQueue(10, Infinity) + q.start(0) + q.push(0, spy1) + q.push(3, spy4) + expect(spy1).to.have.been.calledOnce + expect(spy4).to.not.have.been.called + q.push(2, spy3) + expect(spy3).to.not.have.been.called + expect(spy4).to.not.have.been.called + q.push(1, spy2) + expect(spy2).to.have.been.calledOnce + expect(spy3).to.have.been.calledOnce + expect(spy4).to.have.been.calledOnce + }) + + it('should should start skipping items if too far behind', function() { + var spy1 = sinon.spy() + var spy2 = sinon.spy() + var spy3 = sinon.spy() + var spy4 = sinon.spy() + var q = new SeqQueue(10, 2) + q.start(0) + q.push(0, spy1) + q.push(2, spy3) + q.push(3, spy4) + q.push(1, spy2) + expect(spy1).to.have.been.calledOnce + expect(spy2).to.not.have.been.called + expect(spy3).to.have.been.calledOnce + expect(spy4).to.have.been.calledOnce + }) + + it('should should start a new queue', function() { + var spy1 = sinon.spy() + var spy2 = sinon.spy() + var spy3 = sinon.spy() + var spy4 = sinon.spy() + var q = new SeqQueue(2, Infinity) + q.start(0) + q.push(0, spy1) + q.push(1, spy2) + q.stop(2) + q.start(0) + q.push(0, spy3) + q.push(1, spy4) + expect(spy1).to.have.been.calledOnce + expect(spy2).to.have.been.calledOnce + expect(spy3).to.have.been.calledOnce + expect(spy4).to.have.been.calledOnce + }) + + it('should should start a new queue on even on 1 length', function() { + var spy1 = sinon.spy() + var spy2 = sinon.spy() + var spy3 = sinon.spy() + var q = new SeqQueue(1, Infinity) + q.start(0) + q.push(0, spy1) + q.stop(1) + q.start(0) + q.push(0, spy2) + q.stop(1) + q.start(0) + q.push(0, spy3) + expect(spy1).to.have.been.calledOnce + expect(spy2).to.have.been.calledOnce + expect(spy3).to.have.been.calledOnce + }) +}) diff --git a/vendor/minitouch/armeabi-v7a/minitouch b/vendor/minitouch/armeabi-v7a/minitouch new file mode 100755 index 00000000..6b01401c Binary files /dev/null and b/vendor/minitouch/armeabi-v7a/minitouch differ diff --git a/vendor/minitouch/armeabi/minitouch b/vendor/minitouch/armeabi/minitouch new file mode 100755 index 00000000..5e4a0a47 Binary files /dev/null and b/vendor/minitouch/armeabi/minitouch differ diff --git a/vendor/minitouch/mips/minitouch b/vendor/minitouch/mips/minitouch new file mode 100755 index 00000000..a1c0a348 Binary files /dev/null and b/vendor/minitouch/mips/minitouch differ diff --git a/vendor/minitouch/x86/minitouch b/vendor/minitouch/x86/minitouch new file mode 100755 index 00000000..332de36e Binary files /dev/null and b/vendor/minitouch/x86/minitouch differ