diff --git a/lib/cli.js b/lib/cli.js index 569d71d4..32e92ea0 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -43,6 +43,9 @@ program , 'group timeout' , Number , 600) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) .action(function() { var serials = cliutil.allUnknownArgs(arguments) , options = cliutil.lastArg(arguments) @@ -53,6 +56,9 @@ program if (!options.connectPush) { this.missingArgument('--connect-push') } + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } require('./roles/provider')({ name: options.name @@ -71,6 +77,7 @@ program , '--ports', ports.join(',') , '--public-ip', options.publicIp , '--group-timeout', options.groupTimeout + , '--storage-url', options.storageUrl ]) } , endpoints: { @@ -107,6 +114,9 @@ program , 'group timeout' , Number , 600) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) .action(function(serial, options) { if (!options.connectSub) { this.missingArgument('--connect-sub') @@ -120,6 +130,9 @@ program if (!options.ports) { this.missingArgument('--ports') } + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } require('./roles/device')({ serial: serial @@ -132,6 +145,7 @@ program } , heartbeatInterval: options.heartbeatInterval , groupTimeout: options.groupTimeout * 1000 // change to ms + , storageUrl: options.storageUrl }) }) @@ -426,8 +440,8 @@ program }) program - .command('storage-temp') - .description('start temp storage') + .command('cache-apk') + .description('apk cache') .option('-p, --port ' , 'port (or $PORT)' , Number @@ -463,6 +477,29 @@ program }) }) +program + .command('storage-temp') + .description('start temp storage') + .option('-p, --port ' + , 'port (or $PORT)' + , Number + , process.env.PORT || 7100) + .option('--public-ip ' + , 'public ip for global access' + , String + , ip()) + .option('--save-dir ' + , 'where to save files' + , String + , os.tmpdir()) + .action(function(options) { + require('./roles/storage/temp')({ + port: options.port + , publicIp: options.publicIp + , saveDir: options.saveDir + }) + }) + program .command('migrate') .description('migrates the database to the latest version') @@ -593,6 +630,8 @@ program , '--connect-push', options.bindDevPull , '--group-timeout', options.groupTimeout , '--public-ip', options.publicIp + , '--storage-url' + , util.format('http://localhost:%d/', options.storagePort) ].concat(cliutil.allUnknownArgs(args))) // auth-mock @@ -625,7 +664,6 @@ program , procutil.fork(__filename, [ 'storage-temp' , '--port', options.storagePort - , '--connect-push', options.bindDevPull ]) ] diff --git a/lib/roles/app.js b/lib/roles/app.js index 191e29e6..c8642d5a 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -637,6 +637,16 @@ module.exports = function(options) { ) ]) }) + .on('screen.capture', function(channel, responseChannel) { + joinChannel(responseChannel) + push.send([ + channel + , wireutil.transaction( + responseChannel + , new wire.ScreenCaptureMessage() + ) + ]) + }) }) .finally(function() { // Clean up all listeners and subscriptions diff --git a/lib/roles/cache/apk.js b/lib/roles/cache/apk.js new file mode 100644 index 00000000..718dcb15 --- /dev/null +++ b/lib/roles/cache/apk.js @@ -0,0 +1,216 @@ +var http = require('http') +var util = require('util') +var fs = require('fs') + +var express = require('express') +var validator = require('express-validator') +var Promise = require('bluebird') +var ApkReader = require('adbkit-apkreader') +var request = require('request') +var progress = require('request-progress') +var temp = require('temp') +var zmq = require('zmq') + +var logger = require('../../util/logger') +var requtil = require('../../util/requtil') +var Storage = require('../../util/storage') +var wireutil = require('../../wire/util') + +module.exports = function(options) { + var log = logger.createLogger('cache-apk') + , app = express() + , server = http.createServer(app) + , storage = new Storage() + + // Output + var push = zmq.socket('push') + options.endpoints.push.forEach(function(endpoint) { + log.info('Sending output to %s', endpoint) + push.connect(endpoint) + }) + + app.set('strict routing', true) + app.set('case sensitive routing', true) + app.set('trust proxy', true) + + app.use(express.json()) + app.use(validator()) + + storage.on('timeout', function(id) { + log.info('Cleaning up inactive resource "%s"', id) + }) + + function processFile(file) { + var resolver = Promise.defer() + + log.info('Processing file "%s"', file.path) + + resolver.progress({ + percent: 0 + }) + + process.nextTick(function() { + try { + var reader = ApkReader.readFile(file.path) + var manifest = reader.readManifestSync() + resolver.resolve(manifest) + } + catch (err) { + err.reportCode = 'fail_invalid_app_file' + resolver.reject(err) + } + }) + + return resolver.promise + } + + function storeFile(file) { + var id = storage.store(file) + return Promise.resolve({ + id: id + , url: util.format( + 'http://%s:%s/api/v1/resources/%s' + , options.publicIp + , options.port + , id + ) + }) + } + + function download(url) { + var resolver = Promise.defer() + var path = temp.path({ + dir: options.saveDir + }) + + log.info('Downloading "%s" to "%s"', url, path) + + function errorListener(err) { + err.reportCode = 'fail_download' + resolver.reject(err) + } + + function progressListener(state) { + resolver.progress(state) + } + + function closeListener() { + resolver.resolve({ + path: path + }) + } + + resolver.progress({ + percent: 0 + }) + + try { + var req = progress(request(url), { + throttle: 100 // Throttle events, not upload speed + }) + .on('progress', progressListener) + + var save = req.pipe(fs.createWriteStream(path)) + .on('error', errorListener) + .on('close', closeListener) + } + catch (err) { + err.reportCode = 'fail_invalid_url' + resolver.reject(err) + } + + return resolver.promise.finally(function() { + req.removeListener('progress', progressListener) + save.removeListener('error', errorListener) + save.removeListener('close', closeListener) + }) + } + + app.post('/api/v1/cache', function(req, res) { + var reply = wireutil.reply(options.id) + + function sendProgress(data, progress) { + if (req.query.channel) { + push.send([ + req.query.channel + , reply.progress(data, progress) + ]) + } + } + + function sendDone(success, data, body) { + if (req.query.channel) { + push.send([ + req.query.channel + , reply.okay(data, body) + ]) + } + } + + requtil.validate(req, function() { + req.checkQuery('channel').notEmpty() + }) + .then(function() { + return requtil.validate(req, function() { + req.checkBody('url').notEmpty() + }) + .then(function() { + return download(req.body.url) + .progressed(function(progress) { + sendProgress('uploading', 0.7 * progress.percent) + }) + }) + }) + .then(function(file) { + return processFile(file) + .progressed(function(progress) { + sendProgress('processing', 70 + 0.2 * progress.percent) + }) + .then(function(manifest) { + sendProgress('storing', 90) + return storeFile(file) + .then(function(data) { + data.manifest = manifest + return data + }) + }) + }) + .then(function(data) { + sendDone(true, 'success', data) + data.success = true + res.json(201, data) + }) + .catch(requtil.ValidationError, function(err) { + sendDone(false, err.reportCode || 'fail_validation') + res.status(400) + .json({ + success: false + , error: 'ValidationError' + , validationErrors: err.errors + }) + }) + .catch(function(err) { + log.error('Unexpected error', err.stack) + sendDone(false, err.reportCode || 'fail') + res.status(500) + .json({ + success: false + , error: 'ServerError' + }) + }) + }) + + app.get('/api/v1/cache/:id', function(req, res) { + var file = storage.retrieve(req.params.id) + if (file) { + res.set('Content-Type', file.type) + res.sendfile(file.path) + } + else { + res.send(404) + } + }) + + server.listen(options.port) + log.info('Listening on port %d', options.port) +} diff --git a/lib/roles/device.js b/lib/roles/device.js index 4f726d97..cace32b8 100644 --- a/lib/roles/device.js +++ b/lib/roles/device.js @@ -19,6 +19,7 @@ module.exports = function(options) { .dependency(require('./device/plugins/solo')) .dependency(require('./device/plugins/heartbeat')) .dependency(require('./device/plugins/display')) + .dependency(require('./device/plugins/screenshot')) .dependency(require('./device/plugins/http')) .dependency(require('./device/plugins/service')) .dependency(require('./device/plugins/browser')) diff --git a/lib/roles/device/plugins/screenshot.js b/lib/roles/device/plugins/screenshot.js new file mode 100644 index 00000000..1dd2e97a --- /dev/null +++ b/lib/roles/device/plugins/screenshot.js @@ -0,0 +1,66 @@ +var http = require('http') +var util = require('util') + +var syrup = require('syrup') +var Promise = require('bluebird') + +var logger = require('../../../util/logger') +var wire = require('../../../wire') +var wireutil = require('../../../wire/util') + +module.exports = syrup.serial() + .dependency(require('../support/router')) + .dependency(require('../support/push')) + .dependency(require('../support/storage')) + .dependency(require('./display')) + .define(function(options, router, push, storage, display) { + var log = logger.createLogger('device:plugins:screenshot') + var plugin = Object.create(null) + + plugin.capture = function() { + log.info('Capturing screenshot from %s', display.url) + + return new Promise(function(resolve, reject) { + var req = http.get(display.url) + + function responseListener(res) { + if (res.statusCode !== 200) { + reject(new Error(util.format( + 'Screenshot capture failed: HTTP %d' + , res.statusCode + ))) + } + else { + resolve(storage.store(res, { + filename: util.format('%s.png', options.serial) + , contentType: 'image/png' + , knownLength: +res.headers['content-length'] + })) + } + } + + req.on('response', responseListener) + req.on('error', reject) + }) + } + + router.on(wire.ScreenCaptureMessage, function(channel) { + var reply = wireutil.reply(options.serial) + plugin.capture() + .then(function(url) { + push.send([ + channel + , reply.okay(url) + ]) + }) + .catch(function(err) { + log.error('Screen capture failed', err.stack) + push.send([ + channel + , reply.fail(err.message) + ]) + }) + }) + + return plugin + }) diff --git a/lib/roles/device/support/storage.js b/lib/roles/device/support/storage.js new file mode 100644 index 00000000..7009222b --- /dev/null +++ b/lib/roles/device/support/storage.js @@ -0,0 +1,53 @@ +var util = require('util') + +var syrup = require('syrup') +var Promise = require('bluebird') +var request = require('request') + +var logger = require('../../../util/logger') + +module.exports = syrup.serial() + .define(function(options) { + var log = logger.createLogger('device:support:storage') + var plugin = Object.create(null) + + plugin.store = function(stream, meta) { + var resolver = Promise.defer() + + var req = request.post({ + url: util.format('%sapi/v1/resources', options.storageUrl) + } + , function(err, res, body) { + if (err) { + log.error('Upload failed', err.stack) + resolver.reject(err) + } + else if (res.statusCode !== 201) { + log.error('Upload failed: HTTP %d', res.statusCode) + resolver.reject(new Error(util.format( + 'Upload failed: HTTP %d' + , res.statusCode + ))) + } + else { + try { + var result = JSON.parse(body) + log.info('Uploaded to %s', result.resources.file) + resolver.resolve(result.resources.file) + } + catch (err) { + log.error('Invalid JSON in response', err.stack, body) + resolver.reject(err) + } + } + } + ) + + req.form() + .append('file', stream, meta) + + return resolver.promise + } + + return plugin + }) diff --git a/lib/roles/storage/temp.js b/lib/roles/storage/temp.js index 1bd81f47..2454793e 100644 --- a/lib/roles/storage/temp.js +++ b/lib/roles/storage/temp.js @@ -1,21 +1,14 @@ var http = require('http') var util = require('util') -var fs = require('fs') +var path = require('path') var express = require('express') -var validator = require('express-validator') var formidable = require('formidable') var Promise = require('bluebird') -var ApkReader = require('adbkit-apkreader') -var request = require('request') -var progress = require('request-progress') -var temp = require('temp') -var zmq = require('zmq') var logger = require('../../util/logger') var requtil = require('../../util/requtil') var Storage = require('../../util/storage') -var wireutil = require('../../wire/util') module.exports = function(options) { var log = logger.createLogger('storage-temp') @@ -23,193 +16,48 @@ module.exports = function(options) { , server = http.createServer(app) , storage = new Storage() - // Output - var push = zmq.socket('push') - options.endpoints.push.forEach(function(endpoint) { - log.info('Sending output to %s', endpoint) - push.connect(endpoint) - }) - app.set('strict routing', true) app.set('case sensitive routing', true) app.set('trust proxy', true) - app.use(express.json()) - app.use(validator()) - storage.on('timeout', function(id) { log.info('Cleaning up inactive resource "%s"', id) }) - function processFile(file) { - var resolver = Promise.defer() - - log.info('Processing file "%s"', file.path) - - resolver.progress({ - percent: 0 - }) - - process.nextTick(function() { - try { - var reader = ApkReader.readFile(file.path) - var manifest = reader.readManifestSync() - resolver.resolve(manifest) - } - catch (err) { - err.reportCode = 'fail_invalid_app_file' - resolver.reject(err) - } - }) - - return resolver.promise - } - - function storeFile(file) { - var id = storage.store(file) - return Promise.resolve({ - id: id - , url: util.format( - 'http://%s:%s/api/v1/resources/%s' - , options.publicIp - , options.port - , id - ) - }) - } - - function download(url) { - var resolver = Promise.defer() - var path = temp.path({ - dir: options.saveDir - }) - - log.info('Downloading "%s" to "%s"', url, path) - - function errorListener(err) { - err.reportCode = 'fail_download' - resolver.reject(err) - } - - function progressListener(state) { - resolver.progress(state) - } - - function closeListener() { - resolver.resolve({ - path: path - }) - } - - resolver.progress({ - percent: 0 - }) - - try { - var req = progress(request(url), { - throttle: 100 // Throttle events, not upload speed - }) - .on('progress', progressListener) - - var save = req.pipe(fs.createWriteStream(path)) - .on('error', errorListener) - .on('close', closeListener) - } - catch (err) { - err.reportCode = 'fail_invalid_url' - resolver.reject(err) - } - - return resolver.promise.finally(function() { - req.removeListener('progress', progressListener) - save.removeListener('error', errorListener) - save.removeListener('close', closeListener) - }) - } - app.post('/api/v1/resources', function(req, res) { - var reply = wireutil.reply(options.id) - - function sendProgress(data, progress) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.progress(data, progress) - ]) - } - } - - function sendDone(success, data, body) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.okay(data, body) - ]) - } - } - - requtil.validate(req, function() { - req.checkQuery('channel').notEmpty() - }) - .then(function() { - if (req.is('application/json')) { - return requtil.validate(req, function() { - req.checkBody('url').notEmpty() - }) - .then(function() { - return download(req.body.url) - .progressed(function(progress) { - sendProgress('uploading', 0.7 * progress.percent) - }) - }) - } - else { - var form = Promise.promisifyAll(new formidable.IncomingForm()) - var progressListener = function(received, expected) { - if (expected) { - sendProgress('uploading', 70 * (received / expected)) - } + var form = new formidable.IncomingForm() + Promise.promisify(form.parse, form)(req) + .spread(function(fields, files) { + return Object.keys(files).map(function(field) { + return { + field: field + , id: storage.store(files[field]) + , name: files[field].name } - sendProgress('uploading', 0) - form.on('progress', progressListener) - return form.parseAsync(req) - .finally(function() { - form.removeListener('progress', progressListener) - }) - .spread(function(fields, files) { - if (!files.file) { - throw new requtil.ValidationError('validation error', [ - { - "param": "file" - , "msg": "Required value" - } - ]) - } - return files.file - }) - } + }) }) - .then(function(file) { - return processFile(file) - .progressed(function(progress) { - sendProgress('processing', 70 + 0.2 * progress.percent) - }) - .then(function(manifest) { - sendProgress('storing', 90) - return storeFile(file) - .then(function(data) { - data.manifest = manifest - return data + .then(function(storedFiles) { + res.status(201) + .json({ + success: true + , resources: (function() { + var mapped = Object.create(null) + storedFiles.forEach(function(file) { + mapped[file.field] = util.format( + 'http://%s:%s/api/v1/resources/%s%s' + , options.publicIp + , options.port + , file.id + , file.name + ? util.format('/%s', path.basename(file.name)) + : '' + ) }) + return mapped + })() }) }) - .then(function(data) { - sendDone(true, 'success', data) - data.success = true - res.json(201, data) - }) .catch(requtil.ValidationError, function(err) { - sendDone(false, err.reportCode || 'fail_validation') res.status(400) .json({ success: false @@ -218,8 +66,7 @@ module.exports = function(options) { }) }) .catch(function(err) { - log.error('Unexpected error', err.stack) - sendDone(false, err.reportCode || 'fail') + log.error('Error storing resource', err.stack) res.status(500) .json({ success: false @@ -239,6 +86,17 @@ module.exports = function(options) { } }) + app.get('/api/v1/resources/:id/*', function(req, res) { + var file = storage.retrieve(req.params.id) + if (file) { + res.set('Content-Type', file.type) + res.sendfile(file.path) + } + else { + res.send(404) + } + }) + server.listen(options.port) log.info('Listening on port %d', options.port) } diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index 5a3146d4..f8e67fea 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -50,6 +50,7 @@ enum MessageType { PhoneStateEvent = 47; RotationEvent = 48; StoreOpenMessage = 49; + ScreenCaptureMessage = 50; } message Envelope { @@ -400,6 +401,9 @@ message BrowserClearMessage { message StoreOpenMessage { } +message ScreenCaptureMessage { +} + // Events, these must be kept in sync with STFService/wire.proto message AirplaneModeEvent { diff --git a/res/app/components/stf/control/control-service.js b/res/app/components/stf/control/control-service.js index c9bbdd9d..40f87f3f 100644 --- a/res/app/components/stf/control/control-service.js +++ b/res/app/components/stf/control/control-service.js @@ -216,6 +216,10 @@ module.exports = function ControlServiceFactory( return sendTwoWay('store.open') } + this.screenshot = function() { + return sendTwoWay('screen.capture') + } + window.cc = this }