mirror of
https://github.com/openstf/stf
synced 2025-10-04 18:29:17 +02:00
Too much overlap in storage plugin URLs for meaningful loadbalancing and config simplicity. Make poorxy emulate the real situation and change related URLs.
This commit is contained in:
parent
7c16c40dae
commit
8b9e5d58c9
8 changed files with 70 additions and 57 deletions
|
@ -928,7 +928,7 @@ program
|
||||||
, '--group-timeout', options.groupTimeout
|
, '--group-timeout', options.groupTimeout
|
||||||
, '--public-ip', options.publicIp
|
, '--public-ip', options.publicIp
|
||||||
, '--storage-url'
|
, '--storage-url'
|
||||||
, util.format('http://localhost:%d/', options.storagePort)
|
, util.format('http://localhost:%d/', options.poorxyPort)
|
||||||
, '--adb-host', options.adbHost
|
, '--adb-host', options.adbHost
|
||||||
, '--adb-port', options.adbPort
|
, '--adb-port', options.adbPort
|
||||||
].concat(cliutil.allUnknownArgs(args)))
|
].concat(cliutil.allUnknownArgs(args)))
|
||||||
|
@ -974,7 +974,7 @@ program
|
||||||
, '--port', options.websocketPort
|
, '--port', options.websocketPort
|
||||||
, '--secret', options.authSecret
|
, '--secret', options.authSecret
|
||||||
, '--storage-url'
|
, '--storage-url'
|
||||||
, util.format('http://localhost:%d/', options.storagePort)
|
, util.format('http://localhost:%d/', options.poorxyPort)
|
||||||
, '--connect-sub', options.bindAppPub
|
, '--connect-sub', options.bindAppPub
|
||||||
, '--connect-push', options.bindAppPull
|
, '--connect-push', options.bindAppPull
|
||||||
])
|
])
|
||||||
|
@ -990,7 +990,7 @@ program
|
||||||
'storage-plugin-image'
|
'storage-plugin-image'
|
||||||
, '--port', options.storagePluginImagePort
|
, '--port', options.storagePluginImagePort
|
||||||
, '--storage-url'
|
, '--storage-url'
|
||||||
, util.format('http://localhost:%d/', options.storagePort)
|
, util.format('http://localhost:%d/', options.poorxyPort)
|
||||||
])
|
])
|
||||||
|
|
||||||
// apk processor
|
// apk processor
|
||||||
|
@ -998,7 +998,7 @@ program
|
||||||
'storage-plugin-apk'
|
'storage-plugin-apk'
|
||||||
, '--port', options.storagePluginApkPort
|
, '--port', options.storagePluginApkPort
|
||||||
, '--storage-url'
|
, '--storage-url'
|
||||||
, util.format('http://localhost:%d/', options.storagePort)
|
, util.format('http://localhost:%d/', options.poorxyPort)
|
||||||
])
|
])
|
||||||
|
|
||||||
// poorxy
|
// poorxy
|
||||||
|
|
|
@ -15,34 +15,35 @@ module.exports = syrup.serial()
|
||||||
plugin.store = function(type, stream, meta) {
|
plugin.store = function(type, stream, meta) {
|
||||||
var resolver = Promise.defer()
|
var resolver = Promise.defer()
|
||||||
|
|
||||||
var req = request.post({
|
var args = {
|
||||||
url: url.resolve(options.storageUrl, util.format('s/api/v1/%s', type))
|
url: url.resolve(options.storageUrl, util.format('s/upload/%s', type))
|
||||||
|
}
|
||||||
|
|
||||||
|
var req = request.post(args, function(err, res, body) {
|
||||||
|
if (err) {
|
||||||
|
log.error('Upload to "%s" failed', args.url, err.stack)
|
||||||
|
resolver.reject(err)
|
||||||
}
|
}
|
||||||
, function(err, res, body) {
|
else if (res.statusCode !== 201) {
|
||||||
if (err) {
|
log.error('Upload to "%s" failed: HTTP %d', args.url, res.statusCode)
|
||||||
log.error('Upload failed', err.stack)
|
resolver.reject(new Error(util.format(
|
||||||
|
'Upload to "%s" failed: HTTP %d'
|
||||||
|
, args.url
|
||||||
|
, res.statusCode
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
try {
|
||||||
|
var result = JSON.parse(body)
|
||||||
|
log.info('Uploaded to "%s"', result.resources.file.href)
|
||||||
|
resolver.resolve(result.resources.file)
|
||||||
|
}
|
||||||
|
catch (err) {
|
||||||
|
log.error('Invalid JSON in response', err.stack, body)
|
||||||
resolver.reject(err)
|
resolver.reject(err)
|
||||||
}
|
}
|
||||||
else if (res.statusCode !== 201) {
|
|
||||||
log.error('Upload failed: HTTP %d', res.statusCode)
|
|
||||||
resolver.reject(new Error(util.format(
|
|
||||||
'Upload failed: HTTP %d'
|
|
||||||
, res.statusCode
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
try {
|
|
||||||
var result = JSON.parse(body)
|
|
||||||
log.info('Uploaded to %s', result.resources.file.href)
|
|
||||||
resolver.resolve(result.resources.file)
|
|
||||||
}
|
|
||||||
catch (err) {
|
|
||||||
log.error('Invalid JSON in response', err.stack, body)
|
|
||||||
resolver.reject(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
)
|
})
|
||||||
|
|
||||||
req.form()
|
req.form()
|
||||||
.append('file', stream, meta)
|
.append('file', stream, meta)
|
||||||
|
|
|
@ -27,7 +27,7 @@ module.exports = function(options) {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
;['/s/api/v1/image/*'].forEach(function(route) {
|
;['/s/image/*'].forEach(function(route) {
|
||||||
app.all(route, function(req, res) {
|
app.all(route, function(req, res) {
|
||||||
proxy.web(req, res, {
|
proxy.web(req, res, {
|
||||||
target: options.storagePluginImageUrl
|
target: options.storagePluginImageUrl
|
||||||
|
@ -35,7 +35,7 @@ module.exports = function(options) {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
;['/s/api/v1/apk/*'].forEach(function(route) {
|
;['/s/apk/*'].forEach(function(route) {
|
||||||
app.all(route, function(req, res) {
|
app.all(route, function(req, res) {
|
||||||
proxy.web(req, res, {
|
proxy.web(req, res, {
|
||||||
target: options.storagePluginApkUrl
|
target: options.storagePluginApkUrl
|
||||||
|
@ -43,7 +43,7 @@ module.exports = function(options) {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
;['/s/api/v1/*'].forEach(function(route) {
|
;['/s/*'].forEach(function(route) {
|
||||||
app.all(route, function(req, res) {
|
app.all(route, function(req, res) {
|
||||||
proxy.web(req, res, {
|
proxy.web(req, res, {
|
||||||
target: options.storageUrl
|
target: options.storageUrl
|
||||||
|
|
|
@ -1,8 +1,9 @@
|
||||||
var http = require('http')
|
var http = require('http')
|
||||||
var url = require('url')
|
var url = require('url')
|
||||||
|
var util = require('util')
|
||||||
|
|
||||||
var express = require('express')
|
var express = require('express')
|
||||||
var httpProxy = require('http-proxy')
|
var request = require('request')
|
||||||
|
|
||||||
var logger = require('../../../../util/logger')
|
var logger = require('../../../../util/logger')
|
||||||
var download = require('../../../../util/download')
|
var download = require('../../../../util/download')
|
||||||
|
@ -12,18 +13,18 @@ module.exports = function(options) {
|
||||||
var log = logger.createLogger('storage:plugins:apk')
|
var log = logger.createLogger('storage:plugins:apk')
|
||||||
, app = express()
|
, app = express()
|
||||||
, server = http.createServer(app)
|
, server = http.createServer(app)
|
||||||
, proxy = httpProxy.createProxyServer()
|
|
||||||
|
|
||||||
proxy.on('error', function(err) {
|
|
||||||
log.error('Proxy had an error', err.stack)
|
|
||||||
})
|
|
||||||
|
|
||||||
app.set('strict routing', true)
|
app.set('strict routing', true)
|
||||||
app.set('case sensitive routing', true)
|
app.set('case sensitive routing', true)
|
||||||
app.set('trust proxy', true)
|
app.set('trust proxy', true)
|
||||||
|
|
||||||
app.get('/s/api/v1/apk/:id/*/manifest', function(req, res) {
|
app.get('/s/apk/:id/:name/manifest', function(req, res) {
|
||||||
download(url.resolve(options.storageUrl, req.url), {
|
var orig = util.format(
|
||||||
|
'/s/blob/%s/%s'
|
||||||
|
, req.params.id
|
||||||
|
, req.params.name
|
||||||
|
)
|
||||||
|
download(url.resolve(options.storageUrl, orig), {
|
||||||
dir: options.cacheDir
|
dir: options.cacheDir
|
||||||
})
|
})
|
||||||
.then(manifest)
|
.then(manifest)
|
||||||
|
@ -43,10 +44,13 @@ module.exports = function(options) {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
app.get('/s/api/v1/apk/:id/*', function(req, res) {
|
app.get('/s/apk/:id/:name', function(req, res) {
|
||||||
proxy.web(req, res, {
|
request(url.resolve(options.storageUrl, util.format(
|
||||||
target: options.storageUrl
|
'/s/blob/%s/%s'
|
||||||
})
|
, req.params.id
|
||||||
|
, req.params.name
|
||||||
|
)))
|
||||||
|
.pipe(res)
|
||||||
})
|
})
|
||||||
|
|
||||||
server.listen(options.port)
|
server.listen(options.port)
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
var http = require('http')
|
var http = require('http')
|
||||||
|
var util = require('util')
|
||||||
|
|
||||||
var express = require('express')
|
var express = require('express')
|
||||||
|
|
||||||
|
@ -20,9 +21,14 @@ module.exports = function(options) {
|
||||||
app.set('trust proxy', true)
|
app.set('trust proxy', true)
|
||||||
|
|
||||||
app.get(
|
app.get(
|
||||||
'/s/api/v1/image/:id/*/transform'
|
'/s/image/:id/:name'
|
||||||
, requtil.limit(options.concurrency, function(req, res) {
|
, requtil.limit(options.concurrency, function(req, res) {
|
||||||
return get(req.url, options)
|
var orig = util.format(
|
||||||
|
'/s/blob/%s/%s'
|
||||||
|
, req.params.id
|
||||||
|
, req.params.name
|
||||||
|
)
|
||||||
|
return get(orig, options)
|
||||||
.then(function(stream) {
|
.then(function(stream) {
|
||||||
return transform(stream, {
|
return transform(stream, {
|
||||||
crop: parseCrop(req.query.crop)
|
crop: parseCrop(req.query.crop)
|
||||||
|
|
|
@ -30,7 +30,7 @@ module.exports = function(options) {
|
||||||
log.info('Cleaning up inactive resource "%s"', id)
|
log.info('Cleaning up inactive resource "%s"', id)
|
||||||
})
|
})
|
||||||
|
|
||||||
app.post('/s/api/v1/:type/download', function(req, res) {
|
app.post('/s/download/:plugin', function(req, res) {
|
||||||
requtil.validate(req, function() {
|
requtil.validate(req, function() {
|
||||||
req.checkBody('url').notEmpty()
|
req.checkBody('url').notEmpty()
|
||||||
})
|
})
|
||||||
|
@ -46,17 +46,18 @@ module.exports = function(options) {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.then(function(file) {
|
.then(function(file) {
|
||||||
|
var plugin = req.params.plugin
|
||||||
res.status(201)
|
res.status(201)
|
||||||
.json({
|
.json({
|
||||||
success: true
|
success: true
|
||||||
, resource: {
|
, resource: {
|
||||||
date: new Date()
|
date: new Date()
|
||||||
, type: req.params.type
|
, plugin: plugin
|
||||||
, id: file.id
|
, id: file.id
|
||||||
, name: file.name
|
, name: file.name
|
||||||
, href: util.format(
|
, href: util.format(
|
||||||
'/s/api/v1/%s/%s%s'
|
'/s/%s/%s%s'
|
||||||
, req.params.type
|
, plugin
|
||||||
, file.id
|
, file.id
|
||||||
, file.name
|
, file.name
|
||||||
? util.format('/%s', path.basename(file.name))
|
? util.format('/%s', path.basename(file.name))
|
||||||
|
@ -83,7 +84,7 @@ module.exports = function(options) {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
app.post('/s/api/v1/:type', function(req, res) {
|
app.post('/s/upload/:plugin', function(req, res) {
|
||||||
var form = new formidable.IncomingForm()
|
var form = new formidable.IncomingForm()
|
||||||
Promise.promisify(form.parse, form)(req)
|
Promise.promisify(form.parse, form)(req)
|
||||||
.spread(function(fields, files) {
|
.spread(function(fields, files) {
|
||||||
|
@ -104,14 +105,15 @@ module.exports = function(options) {
|
||||||
, resources: (function() {
|
, resources: (function() {
|
||||||
var mapped = Object.create(null)
|
var mapped = Object.create(null)
|
||||||
storedFiles.forEach(function(file) {
|
storedFiles.forEach(function(file) {
|
||||||
|
var plugin = req.params.plugin
|
||||||
mapped[file.field] = {
|
mapped[file.field] = {
|
||||||
date: new Date()
|
date: new Date()
|
||||||
, type: req.params.type
|
, plugin: plugin
|
||||||
, id: file.id
|
, id: file.id
|
||||||
, name: file.name
|
, name: file.name
|
||||||
, href: util.format(
|
, href: util.format(
|
||||||
'/s/api/v1/%s/%s%s'
|
'/s/%s/%s%s'
|
||||||
, req.params.type
|
, plugin
|
||||||
, file.id
|
, file.id
|
||||||
, file.name
|
, file.name
|
||||||
? util.format('/%s', path.basename(file.name))
|
? util.format('/%s', path.basename(file.name))
|
||||||
|
@ -133,7 +135,7 @@ module.exports = function(options) {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
app.get('/s/api/v1/:type/:id/*', function(req, res) {
|
app.get('/s/blob/:id/:name', function(req, res) {
|
||||||
var file = storage.retrieve(req.params.id)
|
var file = storage.retrieve(req.params.id)
|
||||||
if (file) {
|
if (file) {
|
||||||
res.set('Content-Type', file.type)
|
res.set('Content-Type', file.type)
|
||||||
|
|
|
@ -5,7 +5,7 @@ module.exports = function StorageServiceFactory($http, $upload) {
|
||||||
|
|
||||||
service.storeUrl = function(type, url) {
|
service.storeUrl = function(type, url) {
|
||||||
return $http({
|
return $http({
|
||||||
url: '/s/api/v1/' + type + '/download'
|
url: '/s/download/' + type
|
||||||
, method: 'POST'
|
, method: 'POST'
|
||||||
, data: {
|
, data: {
|
||||||
url: url
|
url: url
|
||||||
|
@ -19,7 +19,7 @@ module.exports = function StorageServiceFactory($http, $upload) {
|
||||||
|
|
||||||
if (input.length) {
|
if (input.length) {
|
||||||
$upload.upload({
|
$upload.upload({
|
||||||
url: '/s/api/v1/' + type
|
url: '/s/upload/' + type
|
||||||
, method: 'POST'
|
, method: 'POST'
|
||||||
, file: input
|
, file: input
|
||||||
})
|
})
|
||||||
|
|
|
@ -16,7 +16,7 @@ module.exports = function ScreenshotsCtrl($scope) {
|
||||||
var finalMaxSize = maxSize * multiplier
|
var finalMaxSize = maxSize * multiplier
|
||||||
|
|
||||||
return (finalSize === finalMaxSize) ? '' :
|
return (finalSize === finalMaxSize) ? '' :
|
||||||
'/transform?crop=' + finalSize + 'x'
|
'?crop=' + finalSize + 'x'
|
||||||
}
|
}
|
||||||
|
|
||||||
$scope.takeScreenShot = function () {
|
$scope.takeScreenShot = function () {
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue