diff --git a/bower.json b/bower.json index 439d65fa..e39bd8fb 100644 --- a/bower.json +++ b/bower.json @@ -24,7 +24,7 @@ "stf-graphics": "git@ghe.amb.ca.local:stf/stf-graphics.git", "angular-bootstrap": "~0.11.0", "angular-dialog-service": "~5.0.0", - "ng-file-upload": "~1.2.11", + "ng-file-upload": "~1.4.0", "angular-growl-v2": "JanStevens/angular-growl-2#~0.6.0", "bluebird": "~1.2.4", "angular-tree-control": "~0.1.5", diff --git a/lib/cli.js b/lib/cli.js index 2792e582..169e147f 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -403,6 +403,9 @@ program .option('--storage-plugin-image-url ' , 'URL to image storage plugin' , String) + .option('--storage-plugin-apk-url ' + , 'URL to apk storage plugin' + , String) .option('-u, --connect-sub ' , 'sub endpoint' , cliutil.list) @@ -424,6 +427,9 @@ program if (!options.storagePluginImageUrl) { this.missingArgument('--storage-plugin-image-url') } + if (!options.storagePluginApkUrl) { + this.missingArgument('--storage-plugin-apk-url') + } if (!options.connectSub) { this.missingArgument('--connect-sub') } @@ -438,6 +444,7 @@ program , authUrl: options.authUrl , storageUrl: options.storageUrl , storagePluginImageUrl: options.storagePluginImageUrl + , storagePluginApkUrl: options.storagePluginApkUrl , endpoints: { sub: options.connectSub , push: options.connectPush @@ -446,44 +453,6 @@ program }) }) -program - .command('cache-apk') - .description('apk cache') - .option('-p, --port ' - , 'port (or $PORT)' - , Number - , process.env.PORT || 7100) - .option('--public-ip ' - , 'public ip for global access' - , String - , ip()) - .option('--save-dir ' - , 'where to save files' - , String - , os.tmpdir()) - .option('--id ' - , 'communication identifier' - , String - , 'storage') - .option('--connect-push ' - , 'push endpoint' - , cliutil.list) - .action(function(options) { - if (!options.connectPush) { - this.missingArgument('--connect-push') - } - - require('./roles/storage/temp')({ - port: options.port - , publicIp: options.publicIp - , saveDir: options.saveDir - , id: options.id - , endpoints: { - push: options.connectPush - } - }) - }) - program .command('storage-temp') .description('start temp storage') @@ -532,6 +501,32 @@ program }) }) +program + .command('storage-plugin-apk') + .description('start storage apk plugin') + .option('-p, --port ' + , 'port (or $PORT)' + , Number + , process.env.PORT || 7100) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) + .option('--cache-dir ' + , 'where to cache images' + , String + , os.tmpdir()) + .action(function(options) { + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } + + require('./roles/storage/plugins/apk')({ + port: options.port + , storageUrl: options.storageUrl + , cacheDir: options.cacheDir + }) + }) + program .command('migrate') .description('migrates the database to the latest version') @@ -596,6 +591,10 @@ program , 'storage image plugin port' , Number , 7103) + .option('--storage-plugin-apk-port ' + , 'storage apk plugin port' + , Number + , 7104) .option('--provider ' , 'provider name (or os.hostname())' , String @@ -688,6 +687,8 @@ program , util.format('http://localhost:%d/', options.storagePort) , '--storage-plugin-image-url' , util.format('http://localhost:%d/', options.storagePluginImagePort) + , '--storage-plugin-apk-url' + , util.format('http://localhost:%d/', options.storagePluginApkPort) , '--connect-sub', options.bindAppPub , '--connect-push', options.bindAppPull ].concat((function() { @@ -711,6 +712,14 @@ program , '--storage-url' , util.format('http://localhost:%d/', options.storagePort) ]) + + // apk processor + , procutil.fork(__filename, [ + 'storage-plugin-apk' + , '--port', options.storagePluginApkPort + , '--storage-url' + , util.format('http://localhost:%d/', options.storagePort) + ]) ] function shutdown() { diff --git a/lib/roles/app.js b/lib/roles/app.js index 71c2b1f7..3ec0f499 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -73,13 +73,21 @@ module.exports = function(options) { , authUrl: options.authUrl })) - // Proxied requests must come before any body parsers + // Proxied requests must come before any body parsers. These proxies are + // here mainly for convenience, they should be replaced with proper reverse + // proxies in production. app.all('/api/v1/s/image/*', function(req, res) { proxy.web(req, res, { target: options.storagePluginImageUrl }) }) + app.all('/api/v1/s/apk/*', function(req, res) { + proxy.web(req, res, { + target: options.storagePluginApkUrl + }) + }) + app.all('/api/v1/s/*', function(req, res) { proxy.web(req, res, { target: options.storageUrl @@ -507,7 +515,11 @@ module.exports = function(options) { channel , wireutil.transaction( responseChannel - , new wire.InstallMessage(data) + , new wire.InstallMessage( + data.href + , data.launch === true + , JSON.stringify(data.manifest) + ) ) ]) }) diff --git a/lib/roles/cache/apk.js b/lib/roles/cache/apk.js deleted file mode 100644 index 718dcb15..00000000 --- a/lib/roles/cache/apk.js +++ /dev/null @@ -1,216 +0,0 @@ -var http = require('http') -var util = require('util') -var fs = require('fs') - -var express = require('express') -var validator = require('express-validator') -var Promise = require('bluebird') -var ApkReader = require('adbkit-apkreader') -var request = require('request') -var progress = require('request-progress') -var temp = require('temp') -var zmq = require('zmq') - -var logger = require('../../util/logger') -var requtil = require('../../util/requtil') -var Storage = require('../../util/storage') -var wireutil = require('../../wire/util') - -module.exports = function(options) { - var log = logger.createLogger('cache-apk') - , app = express() - , server = http.createServer(app) - , storage = new Storage() - - // Output - var push = zmq.socket('push') - options.endpoints.push.forEach(function(endpoint) { - log.info('Sending output to %s', endpoint) - push.connect(endpoint) - }) - - app.set('strict routing', true) - app.set('case sensitive routing', true) - app.set('trust proxy', true) - - app.use(express.json()) - app.use(validator()) - - storage.on('timeout', function(id) { - log.info('Cleaning up inactive resource "%s"', id) - }) - - function processFile(file) { - var resolver = Promise.defer() - - log.info('Processing file "%s"', file.path) - - resolver.progress({ - percent: 0 - }) - - process.nextTick(function() { - try { - var reader = ApkReader.readFile(file.path) - var manifest = reader.readManifestSync() - resolver.resolve(manifest) - } - catch (err) { - err.reportCode = 'fail_invalid_app_file' - resolver.reject(err) - } - }) - - return resolver.promise - } - - function storeFile(file) { - var id = storage.store(file) - return Promise.resolve({ - id: id - , url: util.format( - 'http://%s:%s/api/v1/resources/%s' - , options.publicIp - , options.port - , id - ) - }) - } - - function download(url) { - var resolver = Promise.defer() - var path = temp.path({ - dir: options.saveDir - }) - - log.info('Downloading "%s" to "%s"', url, path) - - function errorListener(err) { - err.reportCode = 'fail_download' - resolver.reject(err) - } - - function progressListener(state) { - resolver.progress(state) - } - - function closeListener() { - resolver.resolve({ - path: path - }) - } - - resolver.progress({ - percent: 0 - }) - - try { - var req = progress(request(url), { - throttle: 100 // Throttle events, not upload speed - }) - .on('progress', progressListener) - - var save = req.pipe(fs.createWriteStream(path)) - .on('error', errorListener) - .on('close', closeListener) - } - catch (err) { - err.reportCode = 'fail_invalid_url' - resolver.reject(err) - } - - return resolver.promise.finally(function() { - req.removeListener('progress', progressListener) - save.removeListener('error', errorListener) - save.removeListener('close', closeListener) - }) - } - - app.post('/api/v1/cache', function(req, res) { - var reply = wireutil.reply(options.id) - - function sendProgress(data, progress) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.progress(data, progress) - ]) - } - } - - function sendDone(success, data, body) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.okay(data, body) - ]) - } - } - - requtil.validate(req, function() { - req.checkQuery('channel').notEmpty() - }) - .then(function() { - return requtil.validate(req, function() { - req.checkBody('url').notEmpty() - }) - .then(function() { - return download(req.body.url) - .progressed(function(progress) { - sendProgress('uploading', 0.7 * progress.percent) - }) - }) - }) - .then(function(file) { - return processFile(file) - .progressed(function(progress) { - sendProgress('processing', 70 + 0.2 * progress.percent) - }) - .then(function(manifest) { - sendProgress('storing', 90) - return storeFile(file) - .then(function(data) { - data.manifest = manifest - return data - }) - }) - }) - .then(function(data) { - sendDone(true, 'success', data) - data.success = true - res.json(201, data) - }) - .catch(requtil.ValidationError, function(err) { - sendDone(false, err.reportCode || 'fail_validation') - res.status(400) - .json({ - success: false - , error: 'ValidationError' - , validationErrors: err.errors - }) - }) - .catch(function(err) { - log.error('Unexpected error', err.stack) - sendDone(false, err.reportCode || 'fail') - res.status(500) - .json({ - success: false - , error: 'ServerError' - }) - }) - }) - - app.get('/api/v1/cache/:id', function(req, res) { - var file = storage.retrieve(req.params.id) - if (file) { - res.set('Content-Type', file.type) - res.sendfile(file.path) - } - else { - res.send(404) - } - }) - - server.listen(options.port) - log.info('Listening on port %d', options.port) -} diff --git a/lib/roles/device/plugins/install.js b/lib/roles/device/plugins/install.js index 73283116..71d8b01b 100644 --- a/lib/roles/device/plugins/install.js +++ b/lib/roles/device/plugins/install.js @@ -1,4 +1,6 @@ var stream = require('stream') +var url = require('url') +var util = require('util') var syrup = require('syrup') var request = require('request') @@ -17,7 +19,7 @@ module.exports = syrup.serial() var log = logger.createLogger('device:plugins:install') router.on(wire.InstallMessage, function(channel, message) { - log.info('Installing "%s"', message.url) + log.info('Installing "%s"', message.href) var reply = wireutil.reply(options.serial) @@ -30,7 +32,7 @@ module.exports = syrup.serial() function pushApp() { var req = request({ - url: message.url + url: url.resolve(options.storageUrl, message.href) }) // We need to catch the Content-Length on the fly or we risk @@ -104,16 +106,30 @@ module.exports = syrup.serial() .timeout(30000) }) .then(function() { - if (message.launchActivity) { - log.info( - 'Launching activity with action "%s" on component "%s"' - , message.launchActivity.action - , message.launchActivity.component - ) - // Progress 90% - sendProgress('launching_app', 90) - return adb.startActivity(options.serial, message.launchActivity) - .timeout(15000) + if (message.launch) { + var manifest = JSON.parse(message.manifest) + if (manifest.application.launcherActivities.length) { + var launchActivity = { + action: 'android.intent.action.MAIN' + , component: util.format( + '%s/%s' + , manifest.package + , manifest.application.launcherActivities[0].name + ) + , category: ['android.intent.category.LAUNCHER'] + , flags: 0x10200000 + } + + log.info( + 'Launching activity with action "%s" on component "%s"' + , launchActivity.action + , launchActivity.component + ) + // Progress 90% + sendProgress('launching_app', 90) + return adb.startActivity(options.serial, launchActivity) + .timeout(15000) + } } }) .then(function() { diff --git a/lib/roles/storage/plugins/apk/index.js b/lib/roles/storage/plugins/apk/index.js new file mode 100644 index 00000000..6f34c731 --- /dev/null +++ b/lib/roles/storage/plugins/apk/index.js @@ -0,0 +1,54 @@ +var http = require('http') +var url = require('url') + +var express = require('express') +var httpProxy = require('http-proxy') + +var logger = require('../../../../util/logger') +var download = require('../../../../util/download') +var manifest = require('./task/manifest') + +module.exports = function(options) { + var log = logger.createLogger('storage:plugins:apk') + , app = express() + , server = http.createServer(app) + , proxy = httpProxy.createProxyServer() + + proxy.on('error', function(err) { + log.error('Proxy had an error', err.stack) + }) + + app.set('strict routing', true) + app.set('case sensitive routing', true) + app.set('trust proxy', true) + + app.get('/api/v1/s/apk/:id/*/manifest', function(req, res) { + download(url.resolve(options.storageUrl, req.url), { + dir: options.cacheDir + }) + .then(manifest) + .then(function(data) { + res.status(200) + .json({ + success: true + , manifest: data + }) + }) + .catch(function(err) { + log.error('Unable to read manifest of "%s"', req.params.id, err.stack) + res.status(500) + .json({ + success: false + }) + }) + }) + + app.get('/api/v1/s/apk/:id/*', function(req, res) { + proxy.web(req, res, { + target: options.storageUrl + }) + }) + + server.listen(options.port) + log.info('Listening on port %d', options.port) +} diff --git a/lib/roles/storage/plugins/apk/task/manifest.js b/lib/roles/storage/plugins/apk/task/manifest.js new file mode 100644 index 00000000..6a8f8efb --- /dev/null +++ b/lib/roles/storage/plugins/apk/task/manifest.js @@ -0,0 +1,19 @@ +var Promise = require('bluebird') +var ApkReader = require('adbkit-apkreader') + +module.exports = function(file) { + var resolver = Promise.defer() + + process.nextTick(function() { + try { + var reader = ApkReader.readFile(file.path) + var manifest = reader.readManifestSync() + resolver.resolve(manifest) + } + catch (err) { + resolver.reject(err) + } + }) + + return resolver.promise +} diff --git a/lib/roles/storage/temp.js b/lib/roles/storage/temp.js index a4a28379..3681cc06 100644 --- a/lib/roles/storage/temp.js +++ b/lib/roles/storage/temp.js @@ -3,11 +3,14 @@ var util = require('util') var path = require('path') var express = require('express') +var validator = require('express-validator') var formidable = require('formidable') var Promise = require('bluebird') var logger = require('../../util/logger') var Storage = require('../../util/storage') +var requtil = require('../../util/requtil') +var download = require('../../util/download') module.exports = function(options) { var log = logger.createLogger('storage:temp') @@ -19,19 +22,77 @@ module.exports = function(options) { app.set('case sensitive routing', true) app.set('trust proxy', true) + app.use(express.json()) + app.use(validator()) + storage.on('timeout', function(id) { log.info('Cleaning up inactive resource "%s"', id) }) + app.post('/api/v1/s/:type/download', function(req, res) { + requtil.validate(req, function() { + req.checkBody('url').notEmpty() + }) + .then(function() { + return download(req.body.url, { + dir: options.cacheDir + }) + }) + .then(function(file) { + return { + id: storage.store(file) + , name: file.name + } + }) + .then(function(file) { + res.status(201) + .json({ + success: true + , resource: { + date: new Date() + , type: req.params.type + , id: file.id + , name: file.name + , href: util.format( + '/api/v1/s/%s/%s%s' + , req.params.type + , file.id + , file.name + ? util.format('/%s', path.basename(file.name)) + : '' + ) + } + }) + }) + .catch(requtil.ValidationError, function(err) { + res.status(400) + .json({ + success: false + , error: 'ValidationError' + , validationErrors: err.errors + }) + }) + .catch(function(err) { + log.error('Error storing resource', err.stack) + res.status(500) + .json({ + success: false + , error: 'ServerError' + }) + }) + }) + app.post('/api/v1/s/:type', function(req, res) { var form = new formidable.IncomingForm() Promise.promisify(form.parse, form)(req) .spread(function(fields, files) { return Object.keys(files).map(function(field) { + var file = files[field] + log.info('Uploaded "%s" to "%s"', file.name, file.path) return { field: field - , id: storage.store(files[field]) - , name: files[field].name + , id: storage.store(file) + , name: file.name } }) }) diff --git a/lib/util/download.js b/lib/util/download.js new file mode 100644 index 00000000..372345e8 --- /dev/null +++ b/lib/util/download.js @@ -0,0 +1,67 @@ +var fs = require('fs') + +var Promise = require('bluebird') +var request = require('request') +var progress = require('request-progress') +var temp = require('temp') + +module.exports = function download(url, options) { + var resolver = Promise.defer() + var path = temp.path(options) + + function errorListener(err) { + resolver.reject(err) + } + + function progressListener(state) { + if (state.total !== null) { + resolver.progress({ + lengthComputable: true + , loaded: state.received + , total: state.total + }) + } + else { + resolver.progress({ + lengthComputable: false + , loaded: state.received + , total: state.received + }) + } + } + + function closeListener() { + resolver.resolve({ + path: path + }) + } + + resolver.progress({ + percent: 0 + }) + + try { + var req = progress(request(url), { + throttle: 100 // Throttle events, not upload speed + }) + .on('progress', progressListener) + + resolver.promise.finally(function() { + req.removeListener('progress', progressListener) + }) + + var save = req.pipe(fs.createWriteStream(path)) + .on('error', errorListener) + .on('close', closeListener) + + resolver.promise.finally(function() { + save.removeListener('error', errorListener) + save.removeListener('close', closeListener) + }) + } + catch (err) { + resolver.reject(err) + } + + return resolver.promise +} diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index f8e67fea..eb5971e2 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -354,8 +354,9 @@ message ShellKeepAliveMessage { } message InstallMessage { - required string url = 1; - optional LaunchActivityMessage launchActivity = 2; + required string href = 1; + required bool launch = 2; + optional string manifest = 3; } message UninstallMessage { diff --git a/res/app/app.js b/res/app/app.js index c36e03e0..ec9d0a66 100644 --- a/res/app/app.js +++ b/res/app/app.js @@ -3,7 +3,8 @@ require('angular-route') require('angular-touch') require('angular-gettext') -require('ng-file-upload') +require('ng-file-upload-shim5') +require('ng-file-upload-main') angular.module('app', [ 'ngRoute', diff --git a/res/app/components/stf/control/control-service.js b/res/app/components/stf/control/control-service.js index 40f87f3f..5f9661e9 100644 --- a/res/app/components/stf/control/control-service.js +++ b/res/app/components/stf/control/control-service.js @@ -111,49 +111,8 @@ module.exports = function ControlServiceFactory( return sendTwoWay('device.identify') } - this.uploadUrl = function(url) { - var tx = TransactionService.create({ - id: 'storage' - }) - socket.emit('storage.upload', channel, tx.channel, { - url: url - }) - return tx.promise - } - - this.uploadFile = function(files) { - if (files.length !== 1) { - throw new Error('Can only upload one file') - } - var tx = TransactionService.create({ - id: 'storage' - }) - TransactionService.punch(tx.channel) - .then(function() { - $upload.upload({ - url: '/api/v1/resources?channel=' + tx.channel - , method: 'POST' - , file: files[0] - }) - }) - return tx.promise - } - this.install = function(options) { - var app = options.manifest.application - var params = { - url: options.url - } - if (app.launcherActivities.length) { - var activity = app.launcherActivities[0] - params.launchActivity = { - action: 'android.intent.action.MAIN' - , component: options.manifest.package + '/' + activity.name - , category: ['android.intent.category.LAUNCHER'] - , flags: 0x10200000 - } - } - return sendTwoWay('device.install', params) + return sendTwoWay('device.install', options) } this.uninstall = function(pkg) { diff --git a/res/app/components/stf/control/index.js b/res/app/components/stf/control/index.js index 5e5d01b3..5644cffa 100644 --- a/res/app/components/stf/control/index.js +++ b/res/app/components/stf/control/index.js @@ -3,3 +3,4 @@ module.exports = angular.module('stf/control', [ ]) .factory('TransactionService', require('./transaction-service')) .factory('ControlService', require('./control-service')) + .factory('StorageService', require('./storage-service')) diff --git a/res/app/components/stf/control/storage-service.js b/res/app/components/stf/control/storage-service.js new file mode 100644 index 00000000..bbc3d287 --- /dev/null +++ b/res/app/components/stf/control/storage-service.js @@ -0,0 +1,40 @@ +var Promise = require('bluebird') + +module.exports = function StorageServiceFactory($http, $upload) { + var service = {} + + service.storeUrl = function(type, url) { + return $http({ + url: '/api/v1/s/' + type + '/download' + , method: 'POST' + , data: { + url: url + } + }) + } + + service.storeFile = function(type, files) { + var resolver = Promise.defer() + + $upload.upload({ + url: '/api/v1/s/' + type + , method: 'POST' + , file: files + }) + .then( + function(value) { + resolver.resolve(value) + } + , function(err) { + resolver.reject(err) + } + , function(progressEvent) { + resolver.progress(progressEvent) + } + ) + + return resolver.promise + } + + return service +} diff --git a/res/app/control-panes/dashboard/upload/upload-controller.js b/res/app/control-panes/dashboard/upload/upload-controller.js index e0e87584..4407f5cb 100644 --- a/res/app/control-panes/dashboard/upload/upload-controller.js +++ b/res/app/control-panes/dashboard/upload/upload-controller.js @@ -1,5 +1,9 @@ -module.exports = function UploadCtrl($scope, SettingsService, gettext) { - +module.exports = function UploadCtrl( + $scope +, $http +, SettingsService +, StorageService +) { $scope.upload = null $scope.installation = null $scope.installEnabled = true @@ -35,23 +39,49 @@ module.exports = function UploadCtrl($scope, SettingsService, gettext) { $scope.installFile = function ($files) { $scope.upload = { - progress: 0, - lastData: 'uploading' + progress: 0 + , lastData: 'uploading' } - $scope.installation = null - return $scope.control.uploadFile($files) - .progressed(function (uploadResult) { - $scope.$apply(function () { - $scope.upload = uploadResult - }) + return StorageService.storeFile('apk', $files) + .progressed(function(e) { + if (e.lengthComputable) { + $scope.upload = { + progress: e.loaded / e.total * 100 + , lastData: 'uploading' + } + } }) - .then(function (uploadResult) { - $scope.$apply(function () { - $scope.upload = uploadResult - }) - if (uploadResult.success) { - return $scope.maybeInstall(uploadResult.body) + .then(function(res) { + $scope.upload = { + progress: 100 + , lastData: 'processing' + } + + var href = res.data.resources.file0.href + return $http.get(href + '/manifest') + .then(function(res) { + $scope.upload = { + progress: 100 + , lastData: 'success' + , settled: true + } + + if (res.data.success) { + return $scope.maybeInstall({ + href: href + , launch: $scope.launchEnabled + , manifest: res.data.manifest + }) + } + }) + }) + .catch(function(err) { + console.log('Upload error', err) + $scope.upload = { + progress: 100 + , lastData: 'fail' + , settled: true } }) } diff --git a/res/app/control-panes/dashboard/upload/upload.jade b/res/app/control-panes/dashboard/upload/upload.jade index 39b22f4b..7eab4302 100644 --- a/res/app/control-panes/dashboard/upload/upload.jade +++ b/res/app/control-panes/dashboard/upload/upload.jade @@ -4,7 +4,7 @@ span(translate) Upload clear-button(ng-click='clear()', ng-disabled='!installation && !upload').btn-xs label.checkbox-inline.pull-right - input(type='checkbox', ng-model='launchEnabled', ng-disabled='true') + input(type='checkbox', ng-model='launchEnabled') span Launch label.checkbox-inline.pull-right input(type='checkbox', ng-model='installEnabled') @@ -65,8 +65,6 @@ span(translate) Uploading... strong(ng-switch-when='processing') span(translate) Processing... - strong(ng-switch-when='storing') - span(translate) Storing... strong(ng-switch-when='fail') span(translate) Upload failed strong(ng-switch-when='success') diff --git a/webpack.config.js b/webpack.config.js index 67778886..3935dab0 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -26,7 +26,8 @@ module.exports = { , 'localforage': 'localforage/dist/localforage.js' , 'socket.io': 'socket.io-client/dist/socket.io' , 'oboe': 'oboe/dist/oboe-browser' - , 'ng-file-upload': 'ng-file-upload/angular-file-upload' + , 'ng-file-upload-shim5': 'ng-file-upload/angular-file-upload-html5-shim' + , 'ng-file-upload-main': 'ng-file-upload/angular-file-upload' , 'bluebird': 'bluebird/js/browser/bluebird' } }