diff --git a/lib/units/device/index.js b/lib/units/device/index.js index d77586be..ff748990 100644 --- a/lib/units/device/index.js +++ b/lib/units/device/index.js @@ -17,8 +17,7 @@ module.exports = function(options) { log.info('Preparing device') return syrup.serial() .dependency(require('./plugins/solo')) - .dependency(require('./plugins/screenshot')) - .dependency(require('./plugins/http')) + .dependency(require('./plugins/screen')) .dependency(require('./plugins/service')) .dependency(require('./plugins/display')) .dependency(require('./plugins/browser')) diff --git a/lib/units/device/plugins/display.js b/lib/units/device/plugins/display.js index 7cf2c179..ec31394d 100644 --- a/lib/units/device/plugins/display.js +++ b/lib/units/device/plugins/display.js @@ -4,19 +4,19 @@ var logger = require('../../../util/logger') module.exports = syrup.serial() .dependency(require('./service')) - .dependency(require('./http')) - .define(function(options, service, http) { + .dependency(require('./screen')) + .define(function(options, service, screen) { var log = logger.createLogger('device:plugins:display') function fetch() { log.info('Fetching display info') return service.getDisplay(0) .catch(function() { - log.info('Falling back to HTTP API') - return http.getDisplay(0) + log.info('Falling back to screen API') + return screen.info(0) }) .then(function(display) { - display.url = http.getDisplayUrl(display.id) + display.url = screen.publicUrl return display }) } diff --git a/lib/units/device/plugins/http.js b/lib/units/device/plugins/http.js deleted file mode 100644 index f5e9a0c5..00000000 --- a/lib/units/device/plugins/http.js +++ /dev/null @@ -1,152 +0,0 @@ -var util = require('util') -var assert = require('assert') -var http = require('http') - -var Promise = require('bluebird') -var syrup = require('stf-syrup') -var request = Promise.promisifyAll(require('request')) -var httpProxy = require('http-proxy') - -var logger = require('../../../util/logger') -var devutil = require('../../../util/devutil') -var lifecycle = require('../../../util/lifecycle') -var streamutil = require('../../../util/streamutil') - -module.exports = syrup.serial() - .dependency(require('../support/adb')) - .dependency(require('../resources/remote')) - .define(function(options, adb, remote) { - var log = logger.createLogger('device:plugins:http') - - var service = { - port: 2870 - , privateUrl: null - , publicUrl: null - } - - function openService() { - log.info('Launching HTTP API') - return devutil.ensureUnusedPort(adb, options.serial, service.port) - .timeout(10000) - .then(function() { - return adb.shell(options.serial, [ - 'exec' - , remote.bin - , '--lib', remote.lib - , '--listen-http', service.port - ]) - .timeout(10000) - .then(function(out) { - lifecycle.share('Remote shell', out) - streamutil.talk(log, 'Remote shell says: "%s"', out) - }) - .then(function() { - return devutil.waitForPort(adb, options.serial, service.port) - .timeout(20000) - }) - .then(function(conn) { - var ours = options.ports.pop() - , everyones = options.ports.pop() - , url = util.format('http://127.0.0.1:%d', ours) - - // Don't need the connection - conn.end() - - log.info('Opening device HTTP API forwarder on "%s"', url) - - service.privateUrl = url - service.publicUrl = util.format( - 'http://%s:%s' - , options.publicIp - , everyones - ) - - return adb.forward( - options.serial - , util.format('tcp:%d', ours) - , util.format('tcp:%d', service.port) - ) - .timeout(10000) - .then(function() { - log.info( - 'Opening HTTP API proxy on "http://%s:%s"' - , options.publicIp - , everyones - ) - - var resolver = Promise.defer() - - function resolve() { - lifecycle.share('Proxy server', proxyServer, { - end: false - }) - resolver.resolve() - } - - function reject(err) { - resolver.reject(err) - } - - function ignore() { - // No-op - } - - var proxy = httpProxy.createProxyServer({ - target: url - , ws: false - , xfwd: false - }) - - proxy.on('error', ignore) - - var proxyServer = http.createServer(function(req, res) { - proxy.web(req, res) - }) - .listen(everyones) - - proxyServer.on('listening', resolve) - proxyServer.on('error', reject) - - return resolver.promise.finally(function() { - proxyServer.removeListener('listening', resolve) - proxyServer.removeListener('error', reject) - }) - }) - }) - }) - } - - return openService() - .then(function() { - return { - getDisplay: function(id) { - return request.getAsync({ - url: util.format( - '%s/api/v1/displays/%d' - , service.privateUrl - , id - ) - , json: true - }) - .timeout(10000) - .then(function(args) { - var display = args[1] - assert.ok('id' in display, 'Invalid response from HTTP API') - // Fix rotation's old name - if ('orientation' in display) { - display.rotation = display.orientation - delete display.orientation - } - return display - }) - } - , getDisplayUrl: function(id) { - return util.format( - '%s/api/v1/displays/%d/screenshot.jpg' - , service.publicUrl - , id - ) - } - } - }) - }) diff --git a/lib/units/device/plugins/screen.js b/lib/units/device/plugins/screen.js new file mode 100644 index 00000000..0e215dd4 --- /dev/null +++ b/lib/units/device/plugins/screen.js @@ -0,0 +1,141 @@ +var util = require('util') +var path = require('path') +var http = require('http') + +var Promise = require('bluebird') +var syrup = require('stf-syrup') +var httpProxy = require('http-proxy') + +var logger = require('../../../util/logger') +var lifecycle = require('../../../util/lifecycle') +var streamutil = require('../../../util/streamutil') + +module.exports = syrup.serial() + .dependency(require('../support/adb')) + .dependency(require('../resources/minicap')) + .define(function(options, adb, minicap) { + var log = logger.createLogger('device:plugins:screen') + var plugin = Object.create(null) + + plugin.devicePort = 9002 + + plugin.privatePort = options.ports.pop() + plugin.privateUrl = util.format( + 'ws://127.0.0.1:%s' + , plugin.privatePort + ) + + plugin.publicPort = options.ports.pop() + plugin.publicUrl = util.format( + 'ws://%s:%s' + , options.publicIp + , plugin.publicPort + ) + + function run(cmd) { + return adb.shell(options.serial, util.format( + 'LD_LIBRARY_PATH=%s exec %s %s' + , path.dirname(minicap.lib) + , minicap.bin + , cmd + )) + } + + function startService() { + log.info('Launching screen service') + return run(util.format('-p %d', plugin.devicePort)) + .timeout(10000) + .then(function(out) { + lifecycle.share('Screen shell', out) + streamutil.talk(log, 'Screen shell says: "%s"', out) + }) + } + + function forwardService() { + log.info('Opening WebSocket service on port %d', plugin.privatePort) + return adb.forward( + options.serial + , util.format('tcp:%d', plugin.privatePort) + , util.format('tcp:%d', plugin.devicePort) + ) + .timeout(10000) + } + + function startProxy() { + log.info('Starting WebSocket proxy on %s', plugin.publicUrl) + + var resolver = Promise.defer() + + function resolve() { + lifecycle.share('Proxy server', proxyServer, { + end: false + }) + resolver.resolve() + } + + function reject(err) { + resolver.reject(err) + } + + function ignore() { + // No-op + } + + var proxy = httpProxy.createProxyServer({ + target: plugin.privateUrl + , ws: true + , xfwd: false + }) + + proxy.on('error', ignore) + + var proxyServer = http.createServer() + + proxyServer.on('listening', resolve) + proxyServer.on('error', reject) + + proxyServer.on('request', function(req, res) { + proxy.web(req, res) + }) + + proxyServer.on('upgrade', function(req, socket, head) { + proxy.ws(req, socket, head) + }) + + proxyServer.listen(plugin.publicPort) + + return resolver.promise.finally(function() { + proxyServer.removeListener('listening', resolve) + proxyServer.removeListener('error', reject) + }) + } + + return startService() + .then(forwardService) + .then(startProxy) + .then(function() { + plugin.info = function(id) { + return run(util.format('-d %d -i', id)) + .then(streamutil.readAll) + .then(function(out) { + var match + if ((match = /^ERROR: (.*)$/.exec(out))) { + throw new Error(match[1]) + } + + try { + var info = JSON.parse(out) + // Compat for now, remove eventually + info.rotation = 0 + info.fps = 0 + info.secure = false + return info + } + catch (e) { + throw new Error(out.toString()) + } + }) + } + }) + .return(plugin) + }) diff --git a/lib/units/device/resources/minicap.js b/lib/units/device/resources/minicap.js new file mode 100644 index 00000000..30109a99 --- /dev/null +++ b/lib/units/device/resources/minicap.js @@ -0,0 +1,92 @@ +var util = require('util') + +var Promise = require('bluebird') +var syrup = require('stf-syrup') + +var logger = require('../../../util/logger') +var pathutil = require('../../../util/pathutil') +var devutil = require('../../../util/devutil') +var streamutil = require('../../../util/streamutil') + +module.exports = syrup.serial() + .dependency(require('../support/adb')) + .dependency(require('../support/properties')) + .dependency(require('../support/abi')) + .define(function(options, adb, properties, abi) { + var log = logger.createLogger('device:resources:minicap') + + var resources = { + bin: { + src: pathutil.requiredMatch(abi.all.map(function(supportedAbi) { + return pathutil.vendor(util.format( + 'minicap/bin/%s/minicap%s' + , supportedAbi + , abi.pie ? '' : '-nopie' + )) + })) + , dest: '/data/local/tmp/minicap' + , comm: 'minicap' + , mode: 0755 + } + , lib: { + // @todo The lib ABI should match the bin ABI. Currently we don't + // have an x86_64 version of the binary while the lib supports it. + src: pathutil.requiredMatch(abi.all.map(function(supportedAbi) { + return pathutil.vendor(util.format( + 'minicap/shared/android-%d/%s/minicap.so' + , properties['ro.build.version.sdk'] + , supportedAbi + )) + })) + , dest: '/data/local/tmp/minicap.so' + , mode: 0755 + } + } + + function removeResource(res) { + return adb.shell(options.serial, ['rm', res.dest]) + .timeout(10000) + .then(function(out) { + return streamutil.readAll(out) + }) + .return(res) + } + + function installResource(res) { + return adb.push(options.serial, res.src, res.dest, res.mode) + .timeout(10000) + .then(function(transfer) { + return new Promise(function(resolve, reject) { + transfer.on('error', reject) + transfer.on('end', resolve) + }) + }) + .return(res) + } + + function installAll() { + return Promise.all([ + removeResource(resources.bin).then(installResource) + , removeResource(resources.lib).then(installResource) + ]) + } + + function stop() { + return devutil.killProcsByComm( + adb + , options.serial + , resources.bin.comm + , resources.bin.dest + ) + .timeout(15000) + } + + return stop() + .then(installAll) + .then(function() { + return { + bin: resources.bin.dest + , lib: resources.lib.dest + } + }) + }) diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index e75017b6..53e162f9 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -161,6 +161,7 @@ message DeviceDisplayMessage { required float density = 8; required bool secure = 9; required string url = 10; + optional float size = 11; } message DeviceBrowserAppMessage { diff --git a/res/app/components/stf/screen/screen-directive.js b/res/app/components/stf/screen/screen-directive.js index 5471abbf..e9d8a533 100644 --- a/res/app/components/stf/screen/screen-directive.js +++ b/res/app/components/stf/screen/screen-directive.js @@ -1,4 +1,3 @@ -var FastImageRender = require('./fast-image-render').FastImageRender var _ = require('lodash') module.exports = function DeviceScreenDirective($document, ScalingService, @@ -11,20 +10,23 @@ module.exports = function DeviceScreenDirective($document, ScalingService, , device: '&' } , link: function (scope, element) { + var URL = window.URL || window.webkitURL + var BLANK_IMG = + 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==' + var device = scope.device() , control = scope.control() var canvas = element.find('canvas')[0] , input = element.find('input') + , g = canvas.getContext('2d') + + var ws + , loading = false - var imageRender = new FastImageRender(canvas, { - render: 'canvas', - timeout: 3000 - }) var guestDisplayDensity = setDisplayDensity(1.5) //var guestDisplayRotation = 0 - var loading = false var cssTransform = VendorUtil.style(['transform', 'webkitTransform']) var screen = scope.screen = { @@ -178,29 +180,27 @@ module.exports = function DeviceScreenDirective($document, ScalingService, function maybeLoadScreen() { var size - if (!loading && scope.$parent.showScreen && device) { - switch (screen.rotation) { - case 0: - case 180: - size = adjustBoundedSize( - screen.bounds.w - , screen.bounds.h - ) - break - case 90: - case 270: - size = adjustBoundedSize( - screen.bounds.h - , screen.bounds.w - ) - break + if (ws) { + if (!loading && scope.$parent.showScreen && device) { + switch (screen.rotation) { + case 0: + case 180: + size = adjustBoundedSize( + screen.bounds.w + , screen.bounds.h + ) + break + case 90: + case 270: + size = adjustBoundedSize( + screen.bounds.h + , screen.bounds.w + ) + break + } + loading = true + ws.send('{"op":"jpeg","w":' + size.w + ',"h":' + size.h + '}') } - loading = true - imageRender.load(device.display.url + - '?width=' + size.w + - '&height=' + size.h + - '&time=' + Date.now() - ) } } @@ -227,59 +227,95 @@ module.exports = function DeviceScreenDirective($document, ScalingService, } function on() { - imageRender.onLoad = function (image) { + if (!ws) { + ws = new WebSocket(device.display.url) + ws.binaryType = 'blob' + } + + ws.onerror = function() { + // @todo HANDLE + } + + ws.onclose = function() { + } + + ws.onopen = function() { + maybeLoadScreen() + } + + ws.onmessage = function(message) { loading = false if (scope.$parent.showScreen) { screen.rotation = device.display.rotation - // Check to set the size only if updated - if (cachedScreen.bounds.w !== screen.bounds.w || - cachedScreen.bounds.h !== screen.bounds.h || - cachedImageWidth !== image.width || - cachedImageHeight !== image.height || - cachedScreen.rotation !== screen.rotation) { + var blob = new Blob([message.data], { + type: 'image/jpeg' + }) - cachedScreen.bounds.w = screen.bounds.w - cachedScreen.bounds.h = screen.bounds.h + var img = new Image() + img.onload = function() { + // Check to set the size only if updated + if (cachedScreen.bounds.w !== screen.bounds.w || + cachedScreen.bounds.h !== screen.bounds.h || + cachedImageWidth !== img.width || + cachedImageHeight !== img.height || + cachedScreen.rotation !== screen.rotation) { - cachedImageWidth = image.width - cachedImageHeight = image.height + cachedScreen.bounds.w = screen.bounds.w + cachedScreen.bounds.h = screen.bounds.h - cachedScreen.rotation = screen.rotation + cachedImageWidth = img.width + cachedImageHeight = img.height - imageRender.canvasWidth = cachedImageWidth - imageRender.canvasHeight = cachedImageHeight + cachedScreen.rotation = screen.rotation - var projectedSize = screen.scaler.projectedSize( - screen.bounds.w - , screen.bounds.h - , screen.rotation - ) + canvas.width = cachedImageWidth + canvas.height = cachedImageHeight - imageRender.canvasStyleWidth = projectedSize.width - imageRender.canvasStyleHeight = projectedSize.height + var projectedSize = screen.scaler.projectedSize( + screen.bounds.w + , screen.bounds.h + , screen.rotation + ) - // @todo Make sure that each position is able to rotate smoothly - // to the next one. This current setup doesn't work if rotation - // changes from 180 to 270 (it will do a reverse rotation). - switch (screen.rotation) { - case 0: - canvas.style[cssTransform] = 'rotate(0deg)' - break - case 90: - canvas.style[cssTransform] = 'rotate(-90deg)' - break - case 180: - canvas.style[cssTransform] = 'rotate(-180deg)' - break - case 270: - canvas.style[cssTransform] = 'rotate(90deg)' - break + canvas.style.width = projectedSize.width + 'px' + canvas.style.height = projectedSize.height + 'px' + + // @todo Make sure that each position is able to rotate smoothly + // to the next one. This current setup doesn't work if rotation + // changes from 180 to 270 (it will do a reverse rotation). + switch (screen.rotation) { + case 0: + canvas.style[cssTransform] = 'rotate(0deg)' + break + case 90: + canvas.style[cssTransform] = 'rotate(-90deg)' + break + case 180: + canvas.style[cssTransform] = 'rotate(-180deg)' + break + case 270: + canvas.style[cssTransform] = 'rotate(90deg)' + break + } } + + g.drawImage(img, 0, 0) + + // Try to forcefully clean everything to get rid of memory leaks. + img.onload = img.onerror = null + img.src = BLANK_IMG + img = null + url = null + blob = null + + // Next please + maybeLoadScreen() } - imageRender.draw(image) + var url = URL.createObjectURL(blob) + img.src = url // Reset error, if any if (scope.displayError) { @@ -287,27 +323,10 @@ module.exports = function DeviceScreenDirective($document, ScalingService, scope.displayError = false }) } - - // Next please - maybeLoadScreen() } - // Else: Nothing to show - } - - imageRender.onError = function (type) { - loading = false - - scope.$apply(function () { - if (type === 'timeout') { - scope.displayError = 'timeout' - } else { - scope.displayError = 'secure' - } - }) } updateBounds() - maybeLoadScreen() input.bind('keydown', keydownListener) input.bind('keyup', keyupListener) @@ -317,9 +336,14 @@ module.exports = function DeviceScreenDirective($document, ScalingService, } function off() { - imageRender.onLoad = imageRender.onError = null loading = false + if (ws) { + ws.onmessage = ws.onerror = ws.onclose = null + ws.close() + ws = null + } + input.unbind('keydown', keydownListener) input.unbind('keyup', keyupListener) input.unbind('input', inputListener) @@ -329,11 +353,9 @@ module.exports = function DeviceScreenDirective($document, ScalingService, scope.$watch('$parent.showScreen', function (val) { if (val) { - updateBounds() - maybeLoadScreen() + on() } else { - scope.fps = null - imageRender.clear() + off() } }) diff --git a/vendor/minicap/bin/arm64-v8a/minicap b/vendor/minicap/bin/arm64-v8a/minicap new file mode 100755 index 00000000..4e286602 Binary files /dev/null and b/vendor/minicap/bin/arm64-v8a/minicap differ diff --git a/vendor/minicap/bin/arm64-v8a/minicap-nopie b/vendor/minicap/bin/arm64-v8a/minicap-nopie new file mode 100755 index 00000000..4e286602 Binary files /dev/null and b/vendor/minicap/bin/arm64-v8a/minicap-nopie differ diff --git a/vendor/minicap/bin/armeabi-v7a/minicap b/vendor/minicap/bin/armeabi-v7a/minicap new file mode 100755 index 00000000..55b03dff Binary files /dev/null and b/vendor/minicap/bin/armeabi-v7a/minicap differ diff --git a/vendor/minicap/bin/armeabi-v7a/minicap-nopie b/vendor/minicap/bin/armeabi-v7a/minicap-nopie new file mode 100755 index 00000000..bca5a9fc Binary files /dev/null and b/vendor/minicap/bin/armeabi-v7a/minicap-nopie differ diff --git a/vendor/minicap/bin/x86/minicap b/vendor/minicap/bin/x86/minicap new file mode 100755 index 00000000..7ce58378 Binary files /dev/null and b/vendor/minicap/bin/x86/minicap differ diff --git a/vendor/minicap/bin/x86/minicap-nopie b/vendor/minicap/bin/x86/minicap-nopie new file mode 100755 index 00000000..a32fdc0d Binary files /dev/null and b/vendor/minicap/bin/x86/minicap-nopie differ diff --git a/vendor/minicap/shared/android-10/armeabi-v7a/minicap.so b/vendor/minicap/shared/android-10/armeabi-v7a/minicap.so new file mode 100755 index 00000000..92b38d00 Binary files /dev/null and b/vendor/minicap/shared/android-10/armeabi-v7a/minicap.so differ diff --git a/vendor/minicap/shared/android-14/armeabi-v7a/minicap.so b/vendor/minicap/shared/android-14/armeabi-v7a/minicap.so new file mode 100755 index 00000000..71146247 Binary files /dev/null and b/vendor/minicap/shared/android-14/armeabi-v7a/minicap.so differ diff --git a/vendor/minicap/shared/android-14/x86/minicap.so b/vendor/minicap/shared/android-14/x86/minicap.so new file mode 100755 index 00000000..28872f05 Binary files /dev/null and b/vendor/minicap/shared/android-14/x86/minicap.so differ diff --git a/vendor/minicap/shared/android-15/armeabi-v7a/minicap.so b/vendor/minicap/shared/android-15/armeabi-v7a/minicap.so new file mode 100755 index 00000000..2359b802 Binary files /dev/null and b/vendor/minicap/shared/android-15/armeabi-v7a/minicap.so differ diff --git a/vendor/minicap/shared/android-15/x86/minicap.so b/vendor/minicap/shared/android-15/x86/minicap.so new file mode 100755 index 00000000..ae02eda0 Binary files /dev/null and b/vendor/minicap/shared/android-15/x86/minicap.so differ diff --git a/vendor/minicap/shared/android-16/armeabi-v7a/minicap.so b/vendor/minicap/shared/android-16/armeabi-v7a/minicap.so new file mode 100755 index 00000000..0e0af315 Binary files /dev/null and b/vendor/minicap/shared/android-16/armeabi-v7a/minicap.so differ diff --git a/vendor/minicap/shared/android-16/x86/minicap.so b/vendor/minicap/shared/android-16/x86/minicap.so new file mode 100755 index 00000000..420b7363 Binary files /dev/null and b/vendor/minicap/shared/android-16/x86/minicap.so differ diff --git a/vendor/minicap/shared/android-17/armeabi-v7a/minicap.so b/vendor/minicap/shared/android-17/armeabi-v7a/minicap.so new file mode 100755 index 00000000..19dd56fb Binary files /dev/null and b/vendor/minicap/shared/android-17/armeabi-v7a/minicap.so differ diff --git a/vendor/minicap/shared/android-17/x86/minicap.so b/vendor/minicap/shared/android-17/x86/minicap.so new file mode 100755 index 00000000..95aed0d1 Binary files /dev/null and b/vendor/minicap/shared/android-17/x86/minicap.so differ diff --git a/vendor/minicap/shared/android-18/armeabi-v7a/minicap.so b/vendor/minicap/shared/android-18/armeabi-v7a/minicap.so new file mode 100755 index 00000000..c3df402b Binary files /dev/null and b/vendor/minicap/shared/android-18/armeabi-v7a/minicap.so differ diff --git a/vendor/minicap/shared/android-18/x86/minicap.so b/vendor/minicap/shared/android-18/x86/minicap.so new file mode 100755 index 00000000..4f1cd669 Binary files /dev/null and b/vendor/minicap/shared/android-18/x86/minicap.so differ diff --git a/vendor/minicap/shared/android-19/armeabi-v7a/minicap.so b/vendor/minicap/shared/android-19/armeabi-v7a/minicap.so new file mode 100755 index 00000000..78faa4bd Binary files /dev/null and b/vendor/minicap/shared/android-19/armeabi-v7a/minicap.so differ diff --git a/vendor/minicap/shared/android-19/x86/minicap.so b/vendor/minicap/shared/android-19/x86/minicap.so new file mode 100755 index 00000000..d0c1ac17 Binary files /dev/null and b/vendor/minicap/shared/android-19/x86/minicap.so differ diff --git a/vendor/minicap/shared/android-21/arm64-v8a/minicap.so b/vendor/minicap/shared/android-21/arm64-v8a/minicap.so new file mode 100755 index 00000000..ed02d96d Binary files /dev/null and b/vendor/minicap/shared/android-21/arm64-v8a/minicap.so differ diff --git a/vendor/minicap/shared/android-21/armeabi-v7a/minicap.so b/vendor/minicap/shared/android-21/armeabi-v7a/minicap.so new file mode 100755 index 00000000..3b634f7e Binary files /dev/null and b/vendor/minicap/shared/android-21/armeabi-v7a/minicap.so differ diff --git a/vendor/minicap/shared/android-21/x86/minicap.so b/vendor/minicap/shared/android-21/x86/minicap.so new file mode 100755 index 00000000..4f71c86d Binary files /dev/null and b/vendor/minicap/shared/android-21/x86/minicap.so differ diff --git a/vendor/minicap/shared/android-21/x86_64/minicap.so b/vendor/minicap/shared/android-21/x86_64/minicap.so new file mode 100755 index 00000000..7821e153 Binary files /dev/null and b/vendor/minicap/shared/android-21/x86_64/minicap.so differ