From 44378e625f5b2b8f5cd656138e17862013cd2551 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Fri, 4 Apr 2014 17:27:19 +0900 Subject: [PATCH] Show progress for URL installation. Temporarily break upload installation. --- lib/cli.js | 16 ++ lib/roles/app.js | 26 +++ lib/roles/device/plugins/install.js | 2 +- lib/roles/storage/temp.js | 159 +++++++++++++++--- lib/wire/wire.proto | 5 +- .../components/stf/control/control-service.js | 49 +++--- .../stf/control/transaction-service.js | 55 ++++-- .../dashboard/upload/upload-controller.js | 46 +++-- .../dashboard/upload/upload.jade | 18 +- 9 files changed, 288 insertions(+), 88 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index 6e40c8db..6b730d58 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -431,11 +431,26 @@ program , 'where to save files' , String , os.tmpdir()) + .option('--id ' + , 'communication identifier' + , String + , 'storage') + .option('--connect-push ' + , 'push endpoint' + , cliutil.list) .action(function(options) { + if (!options.connectPush) { + this.missingArgument('--connect-push') + } + require('./roles/storage/temp')({ port: options.port , publicIp: options.publicIp , saveDir: options.saveDir + , id: options.id + , endpoints: { + push: options.connectPush + } }) }) @@ -591,6 +606,7 @@ program , procutil.fork(__filename, [ 'storage-temp' , '--port', options.storagePort + , '--connect-push', options.bindDevPull ]) ] diff --git a/lib/roles/app.js b/lib/roles/app.js index ff5027d7..5966529d 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -1,6 +1,7 @@ var http = require('http') var events = require('events') var path = require('path') +var util = require('util') var express = require('express') var validator = require('express-validator') @@ -9,6 +10,7 @@ var zmq = require('zmq') var Promise = require('bluebird') var httpProxy = require('http-proxy') var _ = require('lodash') +var request = Promise.promisifyAll(require('request')) var logger = require('../util/logger') var pathutil = require('../util/pathutil') @@ -446,6 +448,30 @@ module.exports = function(options) { ) ]) }) + .on('storage.upload', function(channel, responseChannel, data) { + joinChannel(responseChannel) + request.postAsync({ + url: util.format('%sapi/v1/resources', options.storageUrl) + , json: true + , body: { + url: data.url + , channel: responseChannel + } + }) + .catch(function(err) { + log.error('Storage upload had an error', err.stack) + leaveChannel(responseChannel) + push.send([ + channel + , wireutil.envelope(new wire.TransactionDoneMessage( + 'storage' + , 0 + , false + , 'fail' + )) + ]) + }) + }) }) .finally(function() { // Clean up all listeners and subscriptions diff --git a/lib/roles/device/plugins/install.js b/lib/roles/device/plugins/install.js index 937f2f51..a2b119e0 100644 --- a/lib/roles/device/plugins/install.js +++ b/lib/roles/device/plugins/install.js @@ -94,7 +94,7 @@ module.exports = syrup.serial() sendProgress('installing_app', 80) return adb.installRemote(options.serial, apk) }) - .timeout(10000) + .timeout(30000) .then(function() { if (message.launchActivity) { log.info( diff --git a/lib/roles/storage/temp.js b/lib/roles/storage/temp.js index a3824a86..b4be7de5 100644 --- a/lib/roles/storage/temp.js +++ b/lib/roles/storage/temp.js @@ -7,11 +7,15 @@ var formidable = require('formidable') var Promise = require('bluebird') var ApkReader = require('adbkit-apkreader') var request = require('request') +var progress = require('request-progress') var temp = require('temp') +var zmq = require('zmq') var logger = require('../../util/logger') var requtil = require('../../util/requtil') var Storage = require('../../util/storage') +var wire = require('../../wire') +var wireutil = require('../../wire/util') module.exports = function(options) { var log = logger.createLogger('storage-temp') @@ -19,6 +23,13 @@ module.exports = function(options) { , server = http.createServer(app) , storage = new Storage() + // Output + var push = zmq.socket('push') + options.endpoints.push.forEach(function(endpoint) { + log.info('Sending output to %s', endpoint) + push.connect(endpoint) + }) + app.set('strict routing', true) app.set('case sensitive routing', true) app.set('trust proxy', true) @@ -27,22 +38,40 @@ module.exports = function(options) { log.info('Cleaning up inactive resource "%s"', id) }) - function process(file) { - log.info('Processing "%s"', file.path) + function processFile(file) { + var resolver = Promise.defer() - var reader = ApkReader.readFile(file.path) - var manifest = reader.readManifestSync() + log.info('Processing file "%s"', file.path) + + resolver.progress({ + percent: 0 + }) + + process.nextTick(function() { + try { + var reader = ApkReader.readFile(file.path) + var manifest = reader.readManifestSync() + resolver.resolve(manifest) + } + catch (err) { + resolver.reject(err) + } + }) + + return resolver.promise + } + + function storeFile(file) { var id = storage.store(file) - - return { - url: util.format( + return Promise.resolve({ + id: id + , url: util.format( 'http://%s:%s/api/v1/resources/%s' , options.publicIp , options.port , id ) - , manifest: manifest - } + }) } function download(url) { @@ -57,15 +86,25 @@ module.exports = function(options) { resolver.reject(err) } + function progressListener(state) { + resolver.progress(state) + } + function closeListener() { resolver.resolve({ path: path }) } + resolver.progress({ + percent: 0 + }) + try { - var dl = request(url) - .pipe(fs.createWriteStream(path)) + var req = progress(request(url)) + .on('progress', progressListener) + + var save = req.pipe(fs.createWriteStream(path)) .on('error', errorListener) .on('close', closeListener) } @@ -74,25 +113,97 @@ module.exports = function(options) { } return resolver.promise.finally(function() { - dl.removeListener('error', errorListener) - dl.removeListener('close', closeListener) + req.removeListener('progress', progressListener) + save.removeListener('error', errorListener) + save.removeListener('close', closeListener) }) } app.post('/api/v1/resources', function(req, res) { + function handle(fields, files) { + var seq = 0 + + function sendProgress(data, progress) { + if (fields.channel) { + push.send([ + fields.channel + , wireutil.envelope(new wire.TransactionProgressMessage( + options.id + , seq++ + , data + , progress + )) + ]) + } + } + + function sendDone(success, data, body) { + if (fields.channel) { + push.send([ + fields.channel + , wireutil.envelope(new wire.TransactionDoneMessage( + options.id + , seq++ + , success + , data + , body ? JSON.stringify(body) : null + )) + ]) + } + } + + if (files.file) { + return processFile(files.file) + .progressed(function(progress) { + sendProgress('processing', 0.9 * progress.percent) + }) + .then(function(manifest) { + sendProgress('storing', 90) + return storeFile(files.file) + .then(function(data) { + data.manifest = manifest + sendDone(true, 'success', data) + return data + }) + }) + .catch(function(err) { + sendDone(false, 'fail') + return Promise.reject(err) + }) + } + else if (fields.url) { + return download(fields.url) + .progressed(function(progress) { + sendProgress('uploading', 0.7 * progress.percent) + }) + .then(function(file) { + return processFile(file) + .progressed(function(progress) { + sendProgress('processing', 70 + 0.2 * progress.percent) + }) + .then(function(manifest) { + sendProgress('storing', 90) + return storeFile(file) + .then(function(data) { + data.manifest = manifest + sendDone(true, 'success', data) + return data + }) + }) + }) + .catch(function(err) { + sendDone(false, 'fail') + return Promise.reject(err) + }) + } + else { + throw new requtil.ValidationError('"file" or "url" is required') + } + } + var form = Promise.promisifyAll(new formidable.IncomingForm()) form.parseAsync(req) - .spread(function(fields, files) { - if (files.file) { - return process(files.file) - } - else if (fields.url) { - return download(fields.url).then(process) - } - else { - throw new requtil.ValidationError('"file" or "url" is required') - } - }) + .spread(handle) .then(function(data) { data.success = true res.json(201, data) diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index 5c0d4d27..e26438c4 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -44,17 +44,18 @@ message Envelope { } message TransactionProgressMessage { - required string serial = 1; + required string source = 1; required uint32 seq = 2; optional string data = 3; optional uint32 progress = 4 [default = 0]; } message TransactionDoneMessage { - required string serial = 1; + required string source = 1; required uint32 seq = 2; required bool success = 3; optional string data = 4; + optional string body = 5; } // Heartbeat diff --git a/res/app/components/stf/control/control-service.js b/res/app/components/stf/control/control-service.js index 259a48a6..2ed6c7ba 100644 --- a/res/app/components/stf/control/control-service.js +++ b/res/app/components/stf/control/control-service.js @@ -109,7 +109,29 @@ module.exports = function ControlServiceFactory( return tx } - function install(options) { + this.upload = function(files) { + var tx = TransactionService.create({ + id: 'storage' + }) + if (typeof files === 'string') { + socket.emit('storage.upload', channel, tx.channel, { + url: files + }) + return tx + } + else { + return $upload.upload({ + url: '/api/v1/resources' + , method: 'POST' + , file: files[0] + }) + .then(function(response) { + return install(response.data) + }) + } + } + + this.install = function(options) { var app = options.manifest.application var tx = TransactionService.create(target) var params = { @@ -129,31 +151,6 @@ module.exports = function ControlServiceFactory( return tx } - this.install = function(files) { - if (typeof files === 'string') { - return $http({ - url: '/api/v1/resources' - , method: 'POST' - , data: { - url: files - } - }) - .then(function(response) { - return install(response.data) - }) - } - else { - return $upload.upload({ - url: '/api/v1/resources' - , method: 'POST' - , file: files[0] - }) - .then(function(response) { - return install(response.data) - }) - } - } - this.uninstall = function(pkg) { var tx = TransactionService.create(target) socket.emit('device.uninstall', channel, tx.channel, { diff --git a/res/app/components/stf/control/transaction-service.js b/res/app/components/stf/control/transaction-service.js index 320923e9..ccc872c8 100644 --- a/res/app/components/stf/control/transaction-service.js +++ b/res/app/components/stf/control/transaction-service.js @@ -8,20 +8,20 @@ module.exports = function TransactionServiceFactory(socket) { return 'tx.' + uuid.v4() } - function MultiDeviceTransaction(devices) { + function MultiTargetTransaction(targets, options) { var pending = Object.create(null) , results = [] , channel = createChannel() function doneListener(someChannel, data) { if (someChannel === channel) { - pending[data.serial].done(data) + pending[data.source].done(data) } } function progressListener(someChannel, data) { if (someChannel === channel) { - pending[data.serial].progress(data) + pending[data.source].progress(data) } } @@ -30,10 +30,11 @@ module.exports = function TransactionServiceFactory(socket) { this.channel = channel this.results = results - this.promise = Promise.settle(devices.map(function(device) { - var pendingResult = new PendingTransactionResult(device) - pending[device.serial] = pendingResult - results.push(pendingResult.result) + this.promise = Promise.settle(targets.map(function(target) { + var result = new options.result(target) + var pendingResult = new PendingTransactionResult(result) + pending[options.id ? target[options.id] : target.id] = pendingResult + results.push(result) return pendingResult.promise })) .finally(function() { @@ -49,9 +50,9 @@ module.exports = function TransactionServiceFactory(socket) { }) } - function SingleDeviceTransaction(device) { - var pending = new PendingTransactionResult(device) - , result = pending.result + function SingleTargetTransaction(target, options) { + var result = new options.result(target) + , pending = new PendingTransactionResult(result) , channel = createChannel() function doneListener(someChannel, data) { @@ -86,9 +87,8 @@ module.exports = function TransactionServiceFactory(socket) { }) } - function PendingTransactionResult(device) { + function PendingTransactionResult(result) { var resolver = Promise.defer() - , result = new TransactionResult(device) , seq = 0 , last = null , unplaced = [] @@ -112,6 +112,9 @@ module.exports = function TransactionServiceFactory(socket) { if (message.data) { result.lastData = result.data[seq] = message.data } + if (message.body) { + result.body = JSON.parse(message.body) + } } else { result.lastData = result.error = message.data @@ -150,24 +153,42 @@ module.exports = function TransactionServiceFactory(socket) { this.promise = resolver.promise } - function TransactionResult(device) { - this.device = device + function TransactionResult(source) { + this.source = source this.settled = false this.success = false this.progress = 0 this.error = null this.data = [] this.lastData = null + this.body = null } - transactionService.create = function(target) { + function DeviceTransactionResult(device) { + TransactionResult.call(this, device) + this.device = this.source + } + + transactionService.create = function(target, options) { + if (options && !options.result) { + options.result = TransactionResult + } + if (Array.isArray(target)) { - return new MultiDeviceTransaction(target) + return new MultiTargetTransaction(target, options || { + result: DeviceTransactionResult + , id: 'serial' + }) } else { - return new SingleDeviceTransaction(target) + return new SingleTargetTransaction(target, options || { + result: DeviceTransactionResult + , id: 'serial' + }) } } + transactionService.TransactionResult = TransactionResult + return transactionService } diff --git a/res/app/control-panes/dashboard/upload/upload-controller.js b/res/app/control-panes/dashboard/upload/upload-controller.js index 87e442a3..bf14afa4 100644 --- a/res/app/control-panes/dashboard/upload/upload-controller.js +++ b/res/app/control-panes/dashboard/upload/upload-controller.js @@ -1,34 +1,48 @@ module.exports = function UploadCtrl($scope, $rootScope, SettingsService, gettext) { + $scope.upload = null $scope.installation = null $scope.clear = function () { + $scope.upload = null $scope.installation = null } $rootScope.install = function ($files) { - $scope.installation = { + $scope.upload = { progress: 0, lastData: 'uploading' } - return $rootScope.control.install($files) - .then(function (tx) { - var manifest = tx.manifest - return tx.promise - .progressed(function (result) { - $scope.$apply(function () { - result.manifest = manifest - $scope.installation = result + var upload = $rootScope.control.upload($files) + $scope.installation = null + return upload.promise + .progressed(function(uploadResult) { + $scope.$apply(function() { + $scope.upload = uploadResult + }) + }) + .then(function(uploadResult) { + $scope.$apply(function() { + $scope.upload = uploadResult + }) + if (uploadResult.success) { + var install = $rootScope.control.install(uploadResult.body) + return install.promise + .progressed(function(installResult) { + $scope.$apply(function() { + installResult.manifest = uploadResult.body.manifest + $scope.installation = installResult + }) }) - }) - .then(function (result) { - $scope.$apply(function () { - result.manifest = manifest - $scope.treeData = manifest - $scope.installation = result + .then(function(installResult) { + $scope.$apply(function() { + installResult.manifest = uploadResult.body.manifest + $scope.treeData = installResult.manifest + $scope.installation = installResult + }) }) - }) + } }) } diff --git a/res/app/control-panes/dashboard/upload/upload.jade b/res/app/control-panes/dashboard/upload/upload.jade index dca78c92..281740fc 100644 --- a/res/app/control-panes/dashboard/upload/upload.jade +++ b/res/app/control-panes/dashboard/upload/upload.jade @@ -32,6 +32,22 @@ //treecontrol.tree-classic(tree-model='treeData', options='treeOptions') span employee: {{node.name}} age {{node.age}} + .upload-status(ng-if='upload && !upload.settled') + + div(ng-switch='upload.lastData') + strong(ng-switch-when='uploading') + span(translate) Uploading... + strong(ng-switch-when='processing') + span(translate) Processing... + strong(ng-switch-when='storing') + span(translate) Storing... + strong(ng-switch-when='fail') + span(translate) Upload failed + span(ng-hide='upload.settled') ({{upload.progress}}%) + + progressbar(max='100', value='upload.progress', ng-if='!upload.settled', + ng-class='{"active": !upload.settled}').progress-striped + .upload-status(ng-if='installation') accordion(close-others='false') @@ -41,8 +57,6 @@ span {{installation.manifest.package || "App" }} div(ng-switch='installation.lastData') - strong(ng-switch-when='uploading') - span(translate) Starting to upload... strong(ng-switch-when='pushing_app') span(translate) Pushing app... strong(ng-switch-when='installing_app')