1
0
Fork 0
mirror of https://github.com/openstf/stf synced 2025-10-04 18:29:17 +02:00

Too much overlap in storage plugin URLs for meaningful loadbalancing and config simplicity. Make poorxy emulate the real situation and change related URLs.

This commit is contained in:
Simo Kinnunen 2015-01-07 14:04:36 +09:00
parent 7c16c40dae
commit 8b9e5d58c9
8 changed files with 70 additions and 57 deletions

View file

@ -928,7 +928,7 @@ program
, '--group-timeout', options.groupTimeout , '--group-timeout', options.groupTimeout
, '--public-ip', options.publicIp , '--public-ip', options.publicIp
, '--storage-url' , '--storage-url'
, util.format('http://localhost:%d/', options.storagePort) , util.format('http://localhost:%d/', options.poorxyPort)
, '--adb-host', options.adbHost , '--adb-host', options.adbHost
, '--adb-port', options.adbPort , '--adb-port', options.adbPort
].concat(cliutil.allUnknownArgs(args))) ].concat(cliutil.allUnknownArgs(args)))
@ -974,7 +974,7 @@ program
, '--port', options.websocketPort , '--port', options.websocketPort
, '--secret', options.authSecret , '--secret', options.authSecret
, '--storage-url' , '--storage-url'
, util.format('http://localhost:%d/', options.storagePort) , util.format('http://localhost:%d/', options.poorxyPort)
, '--connect-sub', options.bindAppPub , '--connect-sub', options.bindAppPub
, '--connect-push', options.bindAppPull , '--connect-push', options.bindAppPull
]) ])
@ -990,7 +990,7 @@ program
'storage-plugin-image' 'storage-plugin-image'
, '--port', options.storagePluginImagePort , '--port', options.storagePluginImagePort
, '--storage-url' , '--storage-url'
, util.format('http://localhost:%d/', options.storagePort) , util.format('http://localhost:%d/', options.poorxyPort)
]) ])
// apk processor // apk processor
@ -998,7 +998,7 @@ program
'storage-plugin-apk' 'storage-plugin-apk'
, '--port', options.storagePluginApkPort , '--port', options.storagePluginApkPort
, '--storage-url' , '--storage-url'
, util.format('http://localhost:%d/', options.storagePort) , util.format('http://localhost:%d/', options.poorxyPort)
]) ])
// poorxy // poorxy

View file

@ -15,34 +15,35 @@ module.exports = syrup.serial()
plugin.store = function(type, stream, meta) { plugin.store = function(type, stream, meta) {
var resolver = Promise.defer() var resolver = Promise.defer()
var req = request.post({ var args = {
url: url.resolve(options.storageUrl, util.format('s/api/v1/%s', type)) url: url.resolve(options.storageUrl, util.format('s/upload/%s', type))
}
var req = request.post(args, function(err, res, body) {
if (err) {
log.error('Upload to "%s" failed', args.url, err.stack)
resolver.reject(err)
} }
, function(err, res, body) { else if (res.statusCode !== 201) {
if (err) { log.error('Upload to "%s" failed: HTTP %d', args.url, res.statusCode)
log.error('Upload failed', err.stack) resolver.reject(new Error(util.format(
'Upload to "%s" failed: HTTP %d'
, args.url
, res.statusCode
)))
}
else {
try {
var result = JSON.parse(body)
log.info('Uploaded to "%s"', result.resources.file.href)
resolver.resolve(result.resources.file)
}
catch (err) {
log.error('Invalid JSON in response', err.stack, body)
resolver.reject(err) resolver.reject(err)
} }
else if (res.statusCode !== 201) {
log.error('Upload failed: HTTP %d', res.statusCode)
resolver.reject(new Error(util.format(
'Upload failed: HTTP %d'
, res.statusCode
)))
}
else {
try {
var result = JSON.parse(body)
log.info('Uploaded to %s', result.resources.file.href)
resolver.resolve(result.resources.file)
}
catch (err) {
log.error('Invalid JSON in response', err.stack, body)
resolver.reject(err)
}
}
} }
) })
req.form() req.form()
.append('file', stream, meta) .append('file', stream, meta)

View file

@ -27,7 +27,7 @@ module.exports = function(options) {
}) })
}) })
;['/s/api/v1/image/*'].forEach(function(route) { ;['/s/image/*'].forEach(function(route) {
app.all(route, function(req, res) { app.all(route, function(req, res) {
proxy.web(req, res, { proxy.web(req, res, {
target: options.storagePluginImageUrl target: options.storagePluginImageUrl
@ -35,7 +35,7 @@ module.exports = function(options) {
}) })
}) })
;['/s/api/v1/apk/*'].forEach(function(route) { ;['/s/apk/*'].forEach(function(route) {
app.all(route, function(req, res) { app.all(route, function(req, res) {
proxy.web(req, res, { proxy.web(req, res, {
target: options.storagePluginApkUrl target: options.storagePluginApkUrl
@ -43,7 +43,7 @@ module.exports = function(options) {
}) })
}) })
;['/s/api/v1/*'].forEach(function(route) { ;['/s/*'].forEach(function(route) {
app.all(route, function(req, res) { app.all(route, function(req, res) {
proxy.web(req, res, { proxy.web(req, res, {
target: options.storageUrl target: options.storageUrl

View file

@ -1,8 +1,9 @@
var http = require('http') var http = require('http')
var url = require('url') var url = require('url')
var util = require('util')
var express = require('express') var express = require('express')
var httpProxy = require('http-proxy') var request = require('request')
var logger = require('../../../../util/logger') var logger = require('../../../../util/logger')
var download = require('../../../../util/download') var download = require('../../../../util/download')
@ -12,18 +13,18 @@ module.exports = function(options) {
var log = logger.createLogger('storage:plugins:apk') var log = logger.createLogger('storage:plugins:apk')
, app = express() , app = express()
, server = http.createServer(app) , server = http.createServer(app)
, proxy = httpProxy.createProxyServer()
proxy.on('error', function(err) {
log.error('Proxy had an error', err.stack)
})
app.set('strict routing', true) app.set('strict routing', true)
app.set('case sensitive routing', true) app.set('case sensitive routing', true)
app.set('trust proxy', true) app.set('trust proxy', true)
app.get('/s/api/v1/apk/:id/*/manifest', function(req, res) { app.get('/s/apk/:id/:name/manifest', function(req, res) {
download(url.resolve(options.storageUrl, req.url), { var orig = util.format(
'/s/blob/%s/%s'
, req.params.id
, req.params.name
)
download(url.resolve(options.storageUrl, orig), {
dir: options.cacheDir dir: options.cacheDir
}) })
.then(manifest) .then(manifest)
@ -43,10 +44,13 @@ module.exports = function(options) {
}) })
}) })
app.get('/s/api/v1/apk/:id/*', function(req, res) { app.get('/s/apk/:id/:name', function(req, res) {
proxy.web(req, res, { request(url.resolve(options.storageUrl, util.format(
target: options.storageUrl '/s/blob/%s/%s'
}) , req.params.id
, req.params.name
)))
.pipe(res)
}) })
server.listen(options.port) server.listen(options.port)

View file

@ -1,4 +1,5 @@
var http = require('http') var http = require('http')
var util = require('util')
var express = require('express') var express = require('express')
@ -20,9 +21,14 @@ module.exports = function(options) {
app.set('trust proxy', true) app.set('trust proxy', true)
app.get( app.get(
'/s/api/v1/image/:id/*/transform' '/s/image/:id/:name'
, requtil.limit(options.concurrency, function(req, res) { , requtil.limit(options.concurrency, function(req, res) {
return get(req.url, options) var orig = util.format(
'/s/blob/%s/%s'
, req.params.id
, req.params.name
)
return get(orig, options)
.then(function(stream) { .then(function(stream) {
return transform(stream, { return transform(stream, {
crop: parseCrop(req.query.crop) crop: parseCrop(req.query.crop)

View file

@ -30,7 +30,7 @@ module.exports = function(options) {
log.info('Cleaning up inactive resource "%s"', id) log.info('Cleaning up inactive resource "%s"', id)
}) })
app.post('/s/api/v1/:type/download', function(req, res) { app.post('/s/download/:plugin', function(req, res) {
requtil.validate(req, function() { requtil.validate(req, function() {
req.checkBody('url').notEmpty() req.checkBody('url').notEmpty()
}) })
@ -46,17 +46,18 @@ module.exports = function(options) {
} }
}) })
.then(function(file) { .then(function(file) {
var plugin = req.params.plugin
res.status(201) res.status(201)
.json({ .json({
success: true success: true
, resource: { , resource: {
date: new Date() date: new Date()
, type: req.params.type , plugin: plugin
, id: file.id , id: file.id
, name: file.name , name: file.name
, href: util.format( , href: util.format(
'/s/api/v1/%s/%s%s' '/s/%s/%s%s'
, req.params.type , plugin
, file.id , file.id
, file.name , file.name
? util.format('/%s', path.basename(file.name)) ? util.format('/%s', path.basename(file.name))
@ -83,7 +84,7 @@ module.exports = function(options) {
}) })
}) })
app.post('/s/api/v1/:type', function(req, res) { app.post('/s/upload/:plugin', function(req, res) {
var form = new formidable.IncomingForm() var form = new formidable.IncomingForm()
Promise.promisify(form.parse, form)(req) Promise.promisify(form.parse, form)(req)
.spread(function(fields, files) { .spread(function(fields, files) {
@ -104,14 +105,15 @@ module.exports = function(options) {
, resources: (function() { , resources: (function() {
var mapped = Object.create(null) var mapped = Object.create(null)
storedFiles.forEach(function(file) { storedFiles.forEach(function(file) {
var plugin = req.params.plugin
mapped[file.field] = { mapped[file.field] = {
date: new Date() date: new Date()
, type: req.params.type , plugin: plugin
, id: file.id , id: file.id
, name: file.name , name: file.name
, href: util.format( , href: util.format(
'/s/api/v1/%s/%s%s' '/s/%s/%s%s'
, req.params.type , plugin
, file.id , file.id
, file.name , file.name
? util.format('/%s', path.basename(file.name)) ? util.format('/%s', path.basename(file.name))
@ -133,7 +135,7 @@ module.exports = function(options) {
}) })
}) })
app.get('/s/api/v1/:type/:id/*', function(req, res) { app.get('/s/blob/:id/:name', function(req, res) {
var file = storage.retrieve(req.params.id) var file = storage.retrieve(req.params.id)
if (file) { if (file) {
res.set('Content-Type', file.type) res.set('Content-Type', file.type)

View file

@ -5,7 +5,7 @@ module.exports = function StorageServiceFactory($http, $upload) {
service.storeUrl = function(type, url) { service.storeUrl = function(type, url) {
return $http({ return $http({
url: '/s/api/v1/' + type + '/download' url: '/s/download/' + type
, method: 'POST' , method: 'POST'
, data: { , data: {
url: url url: url
@ -19,7 +19,7 @@ module.exports = function StorageServiceFactory($http, $upload) {
if (input.length) { if (input.length) {
$upload.upload({ $upload.upload({
url: '/s/api/v1/' + type url: '/s/upload/' + type
, method: 'POST' , method: 'POST'
, file: input , file: input
}) })

View file

@ -16,7 +16,7 @@ module.exports = function ScreenshotsCtrl($scope) {
var finalMaxSize = maxSize * multiplier var finalMaxSize = maxSize * multiplier
return (finalSize === finalMaxSize) ? '' : return (finalSize === finalMaxSize) ? '' :
'/transform?crop=' + finalSize + 'x' '?crop=' + finalSize + 'x'
} }
$scope.takeScreenShot = function () { $scope.takeScreenShot = function () {