Merge branch 'hogehoge-banana-s3insteadoftmp'

lib/cli.js

@@ -752,6 +752,39 @@ program
     })
   })
 
+program
+  .command('storage-s3')
+  .description('start s3 storage')
+  .option('-p, --port <port>'
+    , 'port (or $PORT)'
+    , Number
+    , process.env.PORT || 7100)
+  .option('--bucket <bucketname>'
+    , 'your s3 bucket name'
+    , String)
+  .option('--profile <name>'
+    , 'your aws credentials profile name'
+    , String)
+  .option('--endpoint <endpoint>'
+    , 'your buckets endpoint'
+    , String)
+  .action(function(options) {
+    if (!options.profile) {
+      this.missingArgument('--profile')
+    }
+
+    if (!options.endpoint) {
+      this.missingArgument('--endpoint')
+    }
+
+    require('./units/storage/s3')({
+      port: options.port
+    , profile: options.profile
+    , bucket: options.bucket
+    , endpoint: options.endpoint
+    })
+  })
+
 program
   .command('storage-plugin-image')
   .description('start storage image plugin')
@@ -916,10 +949,18 @@ program
     , 'websocket port'
     , Number
     , 7110)
+  .option('--storage-type <temp|s3>'
+    , 'storage type'
+    , String
+    , 'temp')
   .option('--storage-port <port>'
     , 'storage port'
     , Number
     , 7102)
+  .option('--storage-options <json>'
+    , 'array of options to pass to the storage implementation'
+    , String
+    , '[]')
   .option('--storage-plugin-image-port <port>'
     , 'storage image plugin port'
     , Number
@@ -1086,9 +1127,9 @@ program
 
         // storage
       , procutil.fork(__filename, [
-            'storage-temp'
+            util.format('storage-%s', options.storageType)
           , '--port', options.storagePort
-          ])
+          ].concat(JSON.parse(options.storageOptions)))
 
         // image processor
       , procutil.fork(__filename, [

lib/units/storage/s3.js

@@ -0,0 +1,155 @@
+var http = require('http')
+var util = require('util')
+var path = require('path')
+var fs = require('fs')
+
+var express = require('express')
+var validator = require('express-validator')
+var bodyParser = require('body-parser')
+var formidable = require('formidable')
+var Promise = require('bluebird')
+var uuid = require('node-uuid')
+var AWS = require('aws-sdk')
+
+var logger = require('../../util/logger')
+
+module.exports = function(options) {
+  var log = logger.createLogger('storage:s3')
+    , app = express()
+    , server = http.createServer(app)
+
+  var s3 = new AWS.S3({
+    credentials: new AWS.SharedIniFileCredentials({
+      profile: options.profile
+    })
+  , endpoint: options.endpoint
+  })
+
+  app.set('strict routing', true)
+  app.set('case sensitive routing', true)
+  app.set('trust proxy', true)
+
+  app.use(bodyParser.json())
+  app.use(validator())
+
+  function putObject(plugin, file) {
+    return new Promise(function(resolve, reject) {
+      var id = uuid.v4()
+      s3.putObject({
+        Key: id
+      , Body: fs.createReadStream(file.path)
+      , Bucket: options.bucket
+      , Metadata: {
+          plugin: plugin
+        , name: file.name
+        }
+      }, function(err) {
+        if (err) {
+          log.error(
+            'Unable to store "%s" as "%s/%s"'
+          , file.temppath
+          , options.bucket
+          , id
+          , err.stack
+          )
+          reject(err)
+        }
+        else {
+          log.info('Stored "%s" as "%s/%s"', file.name, options.bucket, id)
+          resolve(id)
+        }
+      })
+    })
+  }
+
+  function getHref(plugin, id, name) {
+    return util.format(
+      '/s/%s/%s%s'
+    , plugin
+    , id
+    , name ? '/' + path.basename(name) : ''
+    )
+  }
+
+  app.post('/s/upload/:plugin', function(req, res) {
+    var form = new formidable.IncomingForm()
+    var plugin = req.params.plugin
+    Promise.promisify(form.parse, form)(req)
+      .spread(function(fields, files) {
+        var requests = Object.keys(files).map(function(field) {
+          var file = files[field]
+          log.info('Uploaded "%s" to "%s"', file.name, file.path)
+          return putObject(plugin, file)
+            .then(function (id) {
+              return {
+                field: field
+              , id: id
+              , name: file.name
+              , temppath: file.path
+              }
+            })
+        })
+        return Promise.all(requests)
+      })
+      .then(function(storedFiles) {
+        res.status(201).json({
+          success: true
+        , resources: (function() {
+            var mapped = Object.create(null)
+            storedFiles.forEach(function(file) {
+              mapped[file.field] = {
+                date: new Date()
+              , plugin: plugin
+              , id: file.id
+              , name: file.name
+              , href: getHref(plugin, file.id, file.name)
+              }
+            })
+            return mapped
+          })()
+        })
+        return storedFiles
+      })
+      .then(function(storedFiles) {
+        return Promise.all(storedFiles.map(function(file) {
+          return Promise.promisify(fs.unlink, fs)(file.temppath)
+            .catch(function(err) {
+              log.warn('Unable to clean up "%s"', file.temppath, err.stack)
+              return true
+            })
+        }))
+      })
+      .catch(function(err) {
+        log.error('Error storing resource', err.stack)
+        res.status(500)
+          .json({
+            success: false
+          , error: 'ServerError'
+          })
+      })
+  })
+
+  app.get('/s/blob/:id/:name', function(req, res) {
+    var params = {
+      Key: req.params.id
+    , Bucket: options.bucket
+    }
+
+    s3.getObject(params, function(err, data) {
+      if (err) {
+        log.error('Unable to retrieve "%s"', path, err.stack)
+        res.sendStatus(404)
+        return
+      }
+
+      res.set({
+        'Content-Type': data.ContentType
+      })
+
+      res.send(data.Body)
+    })
+  })
+
+  server.listen(options.port)
+  console.log('Listening on port %d', options.port)
+}

package.json

@@ -33,6 +33,7 @@
     "adbkit": "^2.3.1",
     "adbkit-apkreader": "^1.0.0",
     "adbkit-monkey": "^1.0.1",
+    "aws-sdk": "^2.2.3",
     "bluebird": "^2.9.34",
     "body-parser": "^1.13.3",
     "chalk": "~1.1.1",