1
0
Fork 0
mirror of https://github.com/openstf/stf synced 2025-10-04 02:09:32 +02:00

Implement APK uploads using the new storage system. Installation from URL still does not work, and dropping the file on the screen may not work either.

This commit is contained in:
Simo Kinnunen 2014-05-22 13:33:38 +09:00
parent 1db48e9fcb
commit 41ed33f5c4
17 changed files with 389 additions and 336 deletions

View file

@ -24,7 +24,7 @@
"stf-graphics": "git@ghe.amb.ca.local:stf/stf-graphics.git",
"angular-bootstrap": "~0.11.0",
"angular-dialog-service": "~5.0.0",
"ng-file-upload": "~1.2.11",
"ng-file-upload": "~1.4.0",
"angular-growl-v2": "JanStevens/angular-growl-2#~0.6.0",
"bluebird": "~1.2.4",
"angular-tree-control": "~0.1.5",

View file

@ -403,6 +403,9 @@ program
.option('--storage-plugin-image-url <url>'
, 'URL to image storage plugin'
, String)
.option('--storage-plugin-apk-url <url>'
, 'URL to apk storage plugin'
, String)
.option('-u, --connect-sub <endpoint>'
, 'sub endpoint'
, cliutil.list)
@ -424,6 +427,9 @@ program
if (!options.storagePluginImageUrl) {
this.missingArgument('--storage-plugin-image-url')
}
if (!options.storagePluginApkUrl) {
this.missingArgument('--storage-plugin-apk-url')
}
if (!options.connectSub) {
this.missingArgument('--connect-sub')
}
@ -438,6 +444,7 @@ program
, authUrl: options.authUrl
, storageUrl: options.storageUrl
, storagePluginImageUrl: options.storagePluginImageUrl
, storagePluginApkUrl: options.storagePluginApkUrl
, endpoints: {
sub: options.connectSub
, push: options.connectPush
@ -446,44 +453,6 @@ program
})
})
program
.command('cache-apk')
.description('apk cache')
.option('-p, --port <port>'
, 'port (or $PORT)'
, Number
, process.env.PORT || 7100)
.option('--public-ip <ip>'
, 'public ip for global access'
, String
, ip())
.option('--save-dir <dir>'
, 'where to save files'
, String
, os.tmpdir())
.option('--id <id>'
, 'communication identifier'
, String
, 'storage')
.option('--connect-push <endpoint>'
, 'push endpoint'
, cliutil.list)
.action(function(options) {
if (!options.connectPush) {
this.missingArgument('--connect-push')
}
require('./roles/storage/temp')({
port: options.port
, publicIp: options.publicIp
, saveDir: options.saveDir
, id: options.id
, endpoints: {
push: options.connectPush
}
})
})
program
.command('storage-temp')
.description('start temp storage')
@ -532,6 +501,32 @@ program
})
})
program
.command('storage-plugin-apk')
.description('start storage apk plugin')
.option('-p, --port <port>'
, 'port (or $PORT)'
, Number
, process.env.PORT || 7100)
.option('-r, --storage-url <url>'
, 'URL to storage client'
, String)
.option('--cache-dir <dir>'
, 'where to cache images'
, String
, os.tmpdir())
.action(function(options) {
if (!options.storageUrl) {
this.missingArgument('--storage-url')
}
require('./roles/storage/plugins/apk')({
port: options.port
, storageUrl: options.storageUrl
, cacheDir: options.cacheDir
})
})
program
.command('migrate')
.description('migrates the database to the latest version')
@ -596,6 +591,10 @@ program
, 'storage image plugin port'
, Number
, 7103)
.option('--storage-plugin-apk-port <port>'
, 'storage apk plugin port'
, Number
, 7104)
.option('--provider <name>'
, 'provider name (or os.hostname())'
, String
@ -688,6 +687,8 @@ program
, util.format('http://localhost:%d/', options.storagePort)
, '--storage-plugin-image-url'
, util.format('http://localhost:%d/', options.storagePluginImagePort)
, '--storage-plugin-apk-url'
, util.format('http://localhost:%d/', options.storagePluginApkPort)
, '--connect-sub', options.bindAppPub
, '--connect-push', options.bindAppPull
].concat((function() {
@ -711,6 +712,14 @@ program
, '--storage-url'
, util.format('http://localhost:%d/', options.storagePort)
])
// apk processor
, procutil.fork(__filename, [
'storage-plugin-apk'
, '--port', options.storagePluginApkPort
, '--storage-url'
, util.format('http://localhost:%d/', options.storagePort)
])
]
function shutdown() {

View file

@ -73,13 +73,21 @@ module.exports = function(options) {
, authUrl: options.authUrl
}))
// Proxied requests must come before any body parsers
// Proxied requests must come before any body parsers. These proxies are
// here mainly for convenience, they should be replaced with proper reverse
// proxies in production.
app.all('/api/v1/s/image/*', function(req, res) {
proxy.web(req, res, {
target: options.storagePluginImageUrl
})
})
app.all('/api/v1/s/apk/*', function(req, res) {
proxy.web(req, res, {
target: options.storagePluginApkUrl
})
})
app.all('/api/v1/s/*', function(req, res) {
proxy.web(req, res, {
target: options.storageUrl
@ -507,7 +515,11 @@ module.exports = function(options) {
channel
, wireutil.transaction(
responseChannel
, new wire.InstallMessage(data)
, new wire.InstallMessage(
data.href
, data.launch === true
, JSON.stringify(data.manifest)
)
)
])
})

216
lib/roles/cache/apk.js vendored
View file

@ -1,216 +0,0 @@
var http = require('http')
var util = require('util')
var fs = require('fs')
var express = require('express')
var validator = require('express-validator')
var Promise = require('bluebird')
var ApkReader = require('adbkit-apkreader')
var request = require('request')
var progress = require('request-progress')
var temp = require('temp')
var zmq = require('zmq')
var logger = require('../../util/logger')
var requtil = require('../../util/requtil')
var Storage = require('../../util/storage')
var wireutil = require('../../wire/util')
module.exports = function(options) {
var log = logger.createLogger('cache-apk')
, app = express()
, server = http.createServer(app)
, storage = new Storage()
// Output
var push = zmq.socket('push')
options.endpoints.push.forEach(function(endpoint) {
log.info('Sending output to %s', endpoint)
push.connect(endpoint)
})
app.set('strict routing', true)
app.set('case sensitive routing', true)
app.set('trust proxy', true)
app.use(express.json())
app.use(validator())
storage.on('timeout', function(id) {
log.info('Cleaning up inactive resource "%s"', id)
})
function processFile(file) {
var resolver = Promise.defer()
log.info('Processing file "%s"', file.path)
resolver.progress({
percent: 0
})
process.nextTick(function() {
try {
var reader = ApkReader.readFile(file.path)
var manifest = reader.readManifestSync()
resolver.resolve(manifest)
}
catch (err) {
err.reportCode = 'fail_invalid_app_file'
resolver.reject(err)
}
})
return resolver.promise
}
function storeFile(file) {
var id = storage.store(file)
return Promise.resolve({
id: id
, url: util.format(
'http://%s:%s/api/v1/resources/%s'
, options.publicIp
, options.port
, id
)
})
}
function download(url) {
var resolver = Promise.defer()
var path = temp.path({
dir: options.saveDir
})
log.info('Downloading "%s" to "%s"', url, path)
function errorListener(err) {
err.reportCode = 'fail_download'
resolver.reject(err)
}
function progressListener(state) {
resolver.progress(state)
}
function closeListener() {
resolver.resolve({
path: path
})
}
resolver.progress({
percent: 0
})
try {
var req = progress(request(url), {
throttle: 100 // Throttle events, not upload speed
})
.on('progress', progressListener)
var save = req.pipe(fs.createWriteStream(path))
.on('error', errorListener)
.on('close', closeListener)
}
catch (err) {
err.reportCode = 'fail_invalid_url'
resolver.reject(err)
}
return resolver.promise.finally(function() {
req.removeListener('progress', progressListener)
save.removeListener('error', errorListener)
save.removeListener('close', closeListener)
})
}
app.post('/api/v1/cache', function(req, res) {
var reply = wireutil.reply(options.id)
function sendProgress(data, progress) {
if (req.query.channel) {
push.send([
req.query.channel
, reply.progress(data, progress)
])
}
}
function sendDone(success, data, body) {
if (req.query.channel) {
push.send([
req.query.channel
, reply.okay(data, body)
])
}
}
requtil.validate(req, function() {
req.checkQuery('channel').notEmpty()
})
.then(function() {
return requtil.validate(req, function() {
req.checkBody('url').notEmpty()
})
.then(function() {
return download(req.body.url)
.progressed(function(progress) {
sendProgress('uploading', 0.7 * progress.percent)
})
})
})
.then(function(file) {
return processFile(file)
.progressed(function(progress) {
sendProgress('processing', 70 + 0.2 * progress.percent)
})
.then(function(manifest) {
sendProgress('storing', 90)
return storeFile(file)
.then(function(data) {
data.manifest = manifest
return data
})
})
})
.then(function(data) {
sendDone(true, 'success', data)
data.success = true
res.json(201, data)
})
.catch(requtil.ValidationError, function(err) {
sendDone(false, err.reportCode || 'fail_validation')
res.status(400)
.json({
success: false
, error: 'ValidationError'
, validationErrors: err.errors
})
})
.catch(function(err) {
log.error('Unexpected error', err.stack)
sendDone(false, err.reportCode || 'fail')
res.status(500)
.json({
success: false
, error: 'ServerError'
})
})
})
app.get('/api/v1/cache/:id', function(req, res) {
var file = storage.retrieve(req.params.id)
if (file) {
res.set('Content-Type', file.type)
res.sendfile(file.path)
}
else {
res.send(404)
}
})
server.listen(options.port)
log.info('Listening on port %d', options.port)
}

View file

@ -1,4 +1,6 @@
var stream = require('stream')
var url = require('url')
var util = require('util')
var syrup = require('syrup')
var request = require('request')
@ -17,7 +19,7 @@ module.exports = syrup.serial()
var log = logger.createLogger('device:plugins:install')
router.on(wire.InstallMessage, function(channel, message) {
log.info('Installing "%s"', message.url)
log.info('Installing "%s"', message.href)
var reply = wireutil.reply(options.serial)
@ -30,7 +32,7 @@ module.exports = syrup.serial()
function pushApp() {
var req = request({
url: message.url
url: url.resolve(options.storageUrl, message.href)
})
// We need to catch the Content-Length on the fly or we risk
@ -104,17 +106,31 @@ module.exports = syrup.serial()
.timeout(30000)
})
.then(function() {
if (message.launchActivity) {
if (message.launch) {
var manifest = JSON.parse(message.manifest)
if (manifest.application.launcherActivities.length) {
var launchActivity = {
action: 'android.intent.action.MAIN'
, component: util.format(
'%s/%s'
, manifest.package
, manifest.application.launcherActivities[0].name
)
, category: ['android.intent.category.LAUNCHER']
, flags: 0x10200000
}
log.info(
'Launching activity with action "%s" on component "%s"'
, message.launchActivity.action
, message.launchActivity.component
, launchActivity.action
, launchActivity.component
)
// Progress 90%
sendProgress('launching_app', 90)
return adb.startActivity(options.serial, message.launchActivity)
return adb.startActivity(options.serial, launchActivity)
.timeout(15000)
}
}
})
.then(function() {
push.send([

View file

@ -0,0 +1,54 @@
var http = require('http')
var url = require('url')
var express = require('express')
var httpProxy = require('http-proxy')
var logger = require('../../../../util/logger')
var download = require('../../../../util/download')
var manifest = require('./task/manifest')
module.exports = function(options) {
var log = logger.createLogger('storage:plugins:apk')
, app = express()
, server = http.createServer(app)
, proxy = httpProxy.createProxyServer()
proxy.on('error', function(err) {
log.error('Proxy had an error', err.stack)
})
app.set('strict routing', true)
app.set('case sensitive routing', true)
app.set('trust proxy', true)
app.get('/api/v1/s/apk/:id/*/manifest', function(req, res) {
download(url.resolve(options.storageUrl, req.url), {
dir: options.cacheDir
})
.then(manifest)
.then(function(data) {
res.status(200)
.json({
success: true
, manifest: data
})
})
.catch(function(err) {
log.error('Unable to read manifest of "%s"', req.params.id, err.stack)
res.status(500)
.json({
success: false
})
})
})
app.get('/api/v1/s/apk/:id/*', function(req, res) {
proxy.web(req, res, {
target: options.storageUrl
})
})
server.listen(options.port)
log.info('Listening on port %d', options.port)
}

View file

@ -0,0 +1,19 @@
var Promise = require('bluebird')
var ApkReader = require('adbkit-apkreader')
module.exports = function(file) {
var resolver = Promise.defer()
process.nextTick(function() {
try {
var reader = ApkReader.readFile(file.path)
var manifest = reader.readManifestSync()
resolver.resolve(manifest)
}
catch (err) {
resolver.reject(err)
}
})
return resolver.promise
}

View file

@ -3,11 +3,14 @@ var util = require('util')
var path = require('path')
var express = require('express')
var validator = require('express-validator')
var formidable = require('formidable')
var Promise = require('bluebird')
var logger = require('../../util/logger')
var Storage = require('../../util/storage')
var requtil = require('../../util/requtil')
var download = require('../../util/download')
module.exports = function(options) {
var log = logger.createLogger('storage:temp')
@ -19,19 +22,77 @@ module.exports = function(options) {
app.set('case sensitive routing', true)
app.set('trust proxy', true)
app.use(express.json())
app.use(validator())
storage.on('timeout', function(id) {
log.info('Cleaning up inactive resource "%s"', id)
})
app.post('/api/v1/s/:type/download', function(req, res) {
requtil.validate(req, function() {
req.checkBody('url').notEmpty()
})
.then(function() {
return download(req.body.url, {
dir: options.cacheDir
})
})
.then(function(file) {
return {
id: storage.store(file)
, name: file.name
}
})
.then(function(file) {
res.status(201)
.json({
success: true
, resource: {
date: new Date()
, type: req.params.type
, id: file.id
, name: file.name
, href: util.format(
'/api/v1/s/%s/%s%s'
, req.params.type
, file.id
, file.name
? util.format('/%s', path.basename(file.name))
: ''
)
}
})
})
.catch(requtil.ValidationError, function(err) {
res.status(400)
.json({
success: false
, error: 'ValidationError'
, validationErrors: err.errors
})
})
.catch(function(err) {
log.error('Error storing resource', err.stack)
res.status(500)
.json({
success: false
, error: 'ServerError'
})
})
})
app.post('/api/v1/s/:type', function(req, res) {
var form = new formidable.IncomingForm()
Promise.promisify(form.parse, form)(req)
.spread(function(fields, files) {
return Object.keys(files).map(function(field) {
var file = files[field]
log.info('Uploaded "%s" to "%s"', file.name, file.path)
return {
field: field
, id: storage.store(files[field])
, name: files[field].name
, id: storage.store(file)
, name: file.name
}
})
})

67
lib/util/download.js Normal file
View file

@ -0,0 +1,67 @@
var fs = require('fs')
var Promise = require('bluebird')
var request = require('request')
var progress = require('request-progress')
var temp = require('temp')
module.exports = function download(url, options) {
var resolver = Promise.defer()
var path = temp.path(options)
function errorListener(err) {
resolver.reject(err)
}
function progressListener(state) {
if (state.total !== null) {
resolver.progress({
lengthComputable: true
, loaded: state.received
, total: state.total
})
}
else {
resolver.progress({
lengthComputable: false
, loaded: state.received
, total: state.received
})
}
}
function closeListener() {
resolver.resolve({
path: path
})
}
resolver.progress({
percent: 0
})
try {
var req = progress(request(url), {
throttle: 100 // Throttle events, not upload speed
})
.on('progress', progressListener)
resolver.promise.finally(function() {
req.removeListener('progress', progressListener)
})
var save = req.pipe(fs.createWriteStream(path))
.on('error', errorListener)
.on('close', closeListener)
resolver.promise.finally(function() {
save.removeListener('error', errorListener)
save.removeListener('close', closeListener)
})
}
catch (err) {
resolver.reject(err)
}
return resolver.promise
}

View file

@ -354,8 +354,9 @@ message ShellKeepAliveMessage {
}
message InstallMessage {
required string url = 1;
optional LaunchActivityMessage launchActivity = 2;
required string href = 1;
required bool launch = 2;
optional string manifest = 3;
}
message UninstallMessage {

View file

@ -3,7 +3,8 @@ require('angular-route')
require('angular-touch')
require('angular-gettext')
require('ng-file-upload')
require('ng-file-upload-shim5')
require('ng-file-upload-main')
angular.module('app', [
'ngRoute',

View file

@ -111,49 +111,8 @@ module.exports = function ControlServiceFactory(
return sendTwoWay('device.identify')
}
this.uploadUrl = function(url) {
var tx = TransactionService.create({
id: 'storage'
})
socket.emit('storage.upload', channel, tx.channel, {
url: url
})
return tx.promise
}
this.uploadFile = function(files) {
if (files.length !== 1) {
throw new Error('Can only upload one file')
}
var tx = TransactionService.create({
id: 'storage'
})
TransactionService.punch(tx.channel)
.then(function() {
$upload.upload({
url: '/api/v1/resources?channel=' + tx.channel
, method: 'POST'
, file: files[0]
})
})
return tx.promise
}
this.install = function(options) {
var app = options.manifest.application
var params = {
url: options.url
}
if (app.launcherActivities.length) {
var activity = app.launcherActivities[0]
params.launchActivity = {
action: 'android.intent.action.MAIN'
, component: options.manifest.package + '/' + activity.name
, category: ['android.intent.category.LAUNCHER']
, flags: 0x10200000
}
}
return sendTwoWay('device.install', params)
return sendTwoWay('device.install', options)
}
this.uninstall = function(pkg) {

View file

@ -3,3 +3,4 @@ module.exports = angular.module('stf/control', [
])
.factory('TransactionService', require('./transaction-service'))
.factory('ControlService', require('./control-service'))
.factory('StorageService', require('./storage-service'))

View file

@ -0,0 +1,40 @@
var Promise = require('bluebird')
module.exports = function StorageServiceFactory($http, $upload) {
var service = {}
service.storeUrl = function(type, url) {
return $http({
url: '/api/v1/s/' + type + '/download'
, method: 'POST'
, data: {
url: url
}
})
}
service.storeFile = function(type, files) {
var resolver = Promise.defer()
$upload.upload({
url: '/api/v1/s/' + type
, method: 'POST'
, file: files
})
.then(
function(value) {
resolver.resolve(value)
}
, function(err) {
resolver.reject(err)
}
, function(progressEvent) {
resolver.progress(progressEvent)
}
)
return resolver.promise
}
return service
}

View file

@ -1,5 +1,9 @@
module.exports = function UploadCtrl($scope, SettingsService, gettext) {
module.exports = function UploadCtrl(
$scope
, $http
, SettingsService
, StorageService
) {
$scope.upload = null
$scope.installation = null
$scope.installEnabled = true
@ -35,23 +39,49 @@ module.exports = function UploadCtrl($scope, SettingsService, gettext) {
$scope.installFile = function ($files) {
$scope.upload = {
progress: 0,
lastData: 'uploading'
progress: 0
, lastData: 'uploading'
}
$scope.installation = null
return $scope.control.uploadFile($files)
.progressed(function (uploadResult) {
$scope.$apply(function () {
$scope.upload = uploadResult
return StorageService.storeFile('apk', $files)
.progressed(function(e) {
if (e.lengthComputable) {
$scope.upload = {
progress: e.loaded / e.total * 100
, lastData: 'uploading'
}
}
})
.then(function(res) {
$scope.upload = {
progress: 100
, lastData: 'processing'
}
var href = res.data.resources.file0.href
return $http.get(href + '/manifest')
.then(function(res) {
$scope.upload = {
progress: 100
, lastData: 'success'
, settled: true
}
if (res.data.success) {
return $scope.maybeInstall({
href: href
, launch: $scope.launchEnabled
, manifest: res.data.manifest
})
}
})
})
.then(function (uploadResult) {
$scope.$apply(function () {
$scope.upload = uploadResult
})
if (uploadResult.success) {
return $scope.maybeInstall(uploadResult.body)
.catch(function(err) {
console.log('Upload error', err)
$scope.upload = {
progress: 100
, lastData: 'fail'
, settled: true
}
})
}

View file

@ -4,7 +4,7 @@
span(translate) Upload
clear-button(ng-click='clear()', ng-disabled='!installation && !upload').btn-xs
label.checkbox-inline.pull-right
input(type='checkbox', ng-model='launchEnabled', ng-disabled='true')
input(type='checkbox', ng-model='launchEnabled')
span Launch
label.checkbox-inline.pull-right
input(type='checkbox', ng-model='installEnabled')
@ -65,8 +65,6 @@
span(translate) Uploading...
strong(ng-switch-when='processing')
span(translate) Processing...
strong(ng-switch-when='storing')
span(translate) Storing...
strong(ng-switch-when='fail')
span(translate) Upload failed
strong(ng-switch-when='success')

View file

@ -26,7 +26,8 @@ module.exports = {
, 'localforage': 'localforage/dist/localforage.js'
, 'socket.io': 'socket.io-client/dist/socket.io'
, 'oboe': 'oboe/dist/oboe-browser'
, 'ng-file-upload': 'ng-file-upload/angular-file-upload'
, 'ng-file-upload-shim5': 'ng-file-upload/angular-file-upload-html5-shim'
, 'ng-file-upload-main': 'ng-file-upload/angular-file-upload'
, 'bluebird': 'bluebird/js/browser/bluebird'
}
}