Ported CLI from commander to yargs because yargs suits our large CLI better and is easier to develop with. Split commands into separate files since the file was getting annoyingly long.

.eslintrc

@@ -78,7 +78,7 @@
     "no-shadow-restricted-names": 2,
     "no-shadow": 0, // TODO: 1 may be ok
     "no-undefined": 1,
-    "no-unused-vars": 1,
+    "no-unused-vars": [1, {"varsIgnorePattern": "^_"}],
     "no-use-before-define": 1, // TODO: 0 or 2 may be ok, sometimes there are ciclic dependencies
 
     // Style

CHANGELOG.md

@@ -5,6 +5,11 @@
 ### Enhancements
 
 - You can now set screen JPEG quality with the `SCREEN_JPEG_QUALITY` environment variable at launch time. Can be useful for slow networks.
+- Switched to [yargs](http://yargs.js.org) for option parsing to make it easier to modify the CLI.
+- Almost all command line options can now be specified with environment variables.
+- Internal commands are now hidden from help output but can still be used.
+- Running the `stf` binary without a command now errors and shows help output (previously there was no output whatsoever).
+- Improved help messages for various options.
 
 ### Fixes
 
@@ -14,6 +19,12 @@
 
 - We now use [please-update-dependencies](https://github.com/sorccu/please-update-dependencies) to check for outdated dependencies when running from source. It's a super quick local check that compares `package.json` with installed dependencies. Should help avoid unnecessary issues caused by forgetting to run `npm install` after `git pull`.
 
+### Breaking changes
+
+- The `-C` shortcut for the `--no-cleanup` option has been removed due to the switch to [yargs](http://yargs.js.org). Please use the full `--no-cleanup` option instead.
+- Although likely not used by anyone, it was possible to give multiple ZeroMQ endpoints to options such as `--connect-push` by separating them with commas. This is still possible but now works in a different way due to the switch to [yargs](http://yargs.js.org). Comma-separated hosts in a single value are no longer accepted. If you need to specify multiple hosts, simply use the option as many times as you like. This change is unlikely to have any impact whatsoever on most users.
+- The `--devices` option of `stf doctor` has been removed due to unnecessary complexity.
+
 ## 2.3.0 (2016-11-09)
 
 Minor release addressing the following:

bin/stf

@@ -1,2 +1,2 @@
 #!/usr/bin/env node
-require('../lib/cli.please')
+require('../lib/cli/please')

lib/cli/api/index.js

@@ -0,0 +1,58 @@
+module.exports.command = 'api'
+
+module.exports.describe = 'Start an API unit.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_API')
+    .strict()
+    .option('connect-push', {
+      alias: 'c'
+    , describe: 'App-side ZeroMQ PULL endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('connect-sub', {
+      alias: 'u'
+    , describe: 'App-side ZeroMQ PUB endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7106
+    })
+    .option('secret', {
+      alias: 's'
+    , describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: process.env.SECRET
+    , demand: true
+    })
+    .option('ssid', {
+      alias: 'i'
+    , describe: 'The name of the session ID cookie.'
+    , type: 'string'
+    , default: process.env.SSID || 'ssid'
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_API_` (e.g. ' +
+      '`STF_API_PORT`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/api')({
+    port: argv.port
+  , ssid: argv.ssid
+  , secret: argv.secret
+  , endpoints: {
+      push: argv.connectPush
+    , sub: argv.connectSub
+    }
+  })
+}

lib/cli/app/index.js

@@ -0,0 +1,61 @@
+module.exports.command = 'app'
+
+module.exports.describe = 'Start an app unit.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_APP')
+    .strict()
+    .option('auth-url', {
+      alias: 'a'
+    , describe: 'URL to the auth unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7105
+    })
+    .option('secret', {
+      alias: 's'
+    , describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: process.env.SECRET
+    , demand: true
+    })
+    .option('ssid', {
+      alias: 'i'
+    , describe: 'The name of the session ID cookie.'
+    , type: 'string'
+    , default: process.env.SSID || 'ssid'
+    })
+    .option('user-profile-url', {
+      describe: 'URL to an external user profile page.'
+    , type: 'string'
+    })
+    .option('websocket-url', {
+      alias: 'w'
+    , describe: 'URL to the websocket unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_APP_` (e.g. ' +
+      '`STF_APP_AUTH_URL`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/app')({
+    port: argv.port
+  , secret: argv.secret
+  , ssid: argv.ssid
+  , authUrl: argv.authUrl
+  , websocketUrl: argv.websocketUrl
+  , userProfileUrl: argv.userProfileUrl
+  })
+}

lib/cli/auth-ldap/index.js

@@ -0,0 +1,119 @@
+module.exports.command = 'auth-ldap'
+
+module.exports.describe = 'Start an LDAP auth unit.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_AUTH_LDAP')
+    .strict()
+    .option('app-url', {
+      alias: 'a'
+    , describe: 'URL to the app unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('ldap-bind-credentials', {
+      describe: 'LDAP bind credentials.'
+    , type: 'string'
+    , default: process.env.LDAP_BIND_CREDENTIALS
+    })
+    .option('ldap-bind-dn', {
+      describe: 'LDAP bind DN.'
+    , type: 'string'
+    , default: process.env.LDAP_BIND_DN
+    })
+    .option('ldap-search-class', {
+      describe: 'LDAP search objectClass.'
+    , type: 'string'
+    , default: process.env.LDAP_SEARCH_CLASS || 'top'
+    })
+    .option('ldap-search-dn', {
+      describe: 'LDAP search DN.'
+    , type: 'string'
+    , default: process.env.LDAP_SEARCH_DN
+    , demand: true
+    })
+    .option('ldap-search-field', {
+      describe: 'LDAP search field.'
+    , type: 'string'
+    , default: process.env.LDAP_SEARCH_FIELD
+    , demand: true
+    })
+    .option('ldap-search-scope', {
+      describe: 'LDAP search scope.'
+    , type: 'string'
+    , default: process.env.LDAP_SEARCH_SCOPE || 'sub'
+    })
+    .option('ldap-timeout', {
+      alias: 't'
+    , describe: 'LDAP timeout.'
+    , type: 'number'
+    , default: process.env.LDAP_TIMEOUT || 1000
+    })
+    .option('ldap-url', {
+      alias: 'u'
+    , describe: 'URL to the LDAP server (e.g. `ldap://127.0.0.1`).'
+    , type: 'string'
+    , default: process.env.LDAP_URL
+    , demand: true
+    })
+    .option('ldap-username-field', {
+      describe: 'LDAP username field.'
+    , type: 'string'
+    , default: process.env.LDAP_USERNAME_FIELD || 'cn'
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7120
+    })
+    .option('secret', {
+      alias: 's'
+    , describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: process.env.SECRET
+    , demand: true
+    })
+    .option('ssid', {
+      alias: 'i'
+    , describe: 'The name of the session ID cookie.'
+    , type: 'string'
+    , default: process.env.SSID || 'ssid'
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_AUTH_LDAP_` (e.g. ' +
+      '`STF_AUTH_LDAP_SECRET`). Legacy environment variables like ' +
+      'LDAP_USERNAME_FIELD are still accepted, too, but consider them ' +
+      'deprecated.')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/auth/ldap')({
+    port: argv.port
+  , secret: argv.secret
+  , ssid: argv.ssid
+  , appUrl: argv.appUrl
+  , ldap: {
+      url: argv.ldapUrl
+    , timeout: argv.ldapTimeout
+    , bind: {
+        dn: argv.ldapBindDn
+      , credentials: argv.ldapBindCredentials
+      }
+    , search: {
+        dn: argv.ldapSearchDn
+      , scope: argv.ldapSearchScope
+      , objectClass: argv.ldapSearchClass
+      , field: argv.ldapSearchField
+      }
+    , username: {
+        field: argv.ldapUsernameField
+      }
+    }
+  })
+}

lib/cli/auth-mock/index.js

@@ -0,0 +1,72 @@
+module.exports.command = 'auth-mock'
+
+module.exports.describe = 'Start a mock auth unit that accepts any user.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_AUTH_MOCK')
+    .strict()
+    .option('app-url', {
+      alias: 'a'
+    , describe: 'URL to the app unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('basic-auth-password', {
+      describe: 'Basic auth password (if enabled).'
+    , type: 'string'
+    , default: process.env.BASIC_AUTH_PASSWORD
+    })
+    .option('basic-auth-username', {
+      describe: 'Basic auth username (if enabled).'
+    , type: 'string'
+    , default: process.env.BASIC_AUTH_USERNAME
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7120
+    })
+    .option('secret', {
+      alias: 's'
+    , describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: process.env.SECRET
+    , demand: true
+    })
+    .option('ssid', {
+      alias: 'i'
+    , describe: 'The name of the session ID cookie.'
+    , type: 'string'
+    , default: process.env.SSID || 'ssid'
+    })
+    .option('use-basic-auth', {
+      describe: 'Whether to "secure" the login page with basic authentication.'
+    , type: 'boolean'
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_AUTH_MOCK_` (e.g. ' +
+      '`STF_AUTH_MOCK_SECRET`). Legacy environment variables like ' +
+      'BASIC_AUTH_USERNAME are still accepted, too, but consider them ' +
+      'deprecated.')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/auth/mock')({
+    port: argv.port
+  , secret: argv.secret
+  , ssid: argv.ssid
+  , appUrl: argv.appUrl
+  , mock: {
+      useBasicAuth: argv.useBasicAuth
+    , basicAuth: {
+        username: argv.basicAuthUsername
+      , password: argv.basicAuthPassword
+      }
+    }
+  })
+}

lib/cli/auth-oauth2/index.js

@@ -0,0 +1,102 @@
+module.exports.command = 'auth-oauth2'
+
+module.exports.describe = 'Start an OAuth 2.0 auth unit.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_AUTH_OAUTH2')
+    .strict()
+    .option('app-url', {
+      alias: 'a'
+    , describe: 'URL to the app unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('oauth-authorization-url', {
+      describe: 'OAuth 2.0 authorization URL.'
+    , type: 'string'
+    , default: process.env.OAUTH_AUTHORIZATION_URL
+    , demand: true
+    })
+    .option('oauth-token-url', {
+      describe: 'OAuth 2.0 token URL.'
+    , type: 'string'
+    , default: process.env.OAUTH_TOKEN_URL
+    , demand: true
+    })
+    .option('oauth-userinfo-url', {
+      describe: 'OAuth 2.0 user info URL.'
+    , type: 'string'
+    , default: process.env.OAUTH_USERINFO_URL
+    , demand: true
+    })
+    .option('oauth-client-id', {
+      describe: 'OAuth 2.0 client ID.'
+    , type: 'string'
+    , default: process.env.OAUTH_CLIENT_ID
+    , demand: true
+    })
+    .option('oauth-client-secret', {
+      describe: 'OAuth 2.0 client secret.'
+    , type: 'string'
+    , default: process.env.OAUTH_CLIENT_SECRET
+    , demand: true
+    })
+    .option('oauth-callback-url', {
+      describe: 'OAuth 2.0 callback URL.'
+    , type: 'string'
+    , default: process.env.OAUTH_CALLBACK_URL
+    , demand: true
+    })
+    .option('oauth-scope', {
+      describe: 'Space-separated OAuth 2.0 scope.'
+    , type: 'string'
+    , default: process.env.OAUTH_SCOPE
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7120
+    })
+    .option('secret', {
+      alias: 's'
+    , describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: process.env.SECRET
+    , demand: true
+    })
+    .option('ssid', {
+      alias: 'i'
+    , describe: 'The name of the session ID cookie.'
+    , type: 'string'
+    , default: process.env.SSID || 'ssid'
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_AUTH_OAUTH2_` (e.g. ' +
+      '`STF_AUTH_OAUTH2_SECRET`). Legacy environment variables like ' +
+      'OAUTH_SCOPE are still accepted, too, but consider them ' +
+      'deprecated.')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/auth/oauth2')({
+    port: argv.port
+  , secret: argv.secret
+  , ssid: argv.ssid
+  , appUrl: argv.appUrl
+  , oauth: {
+      authorizationURL: argv.oauthAuthorizationUrl
+    , tokenURL: argv.oauthTokenUrl
+    , userinfoURL: argv.oauthUserinfoUrl
+    , clientID: argv.oauthClientId
+    , clientSecret: argv.oauthClientSecret
+    , callbackURL: argv.oauthCallbackUrl
+    , scope: argv.oauthScope.split(/\s+/)
+    }
+  })
+}

lib/cli/auth-openid/index.js

@@ -0,0 +1,59 @@
+module.exports.command = 'auth-openid'
+
+module.exports.describe = 'Start an OpenID auth unit.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_AUTH_OPENID')
+    .strict()
+    .option('app-url', {
+      alias: 'a'
+    , describe: 'URL to the app unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('openid-identifier-url', {
+      describe: 'OpenID identifier URL.'
+    , type: 'string'
+    , default: process.env.OPENID_IDENTIFIER_URL
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7120
+    })
+    .option('secret', {
+      alias: 's'
+    , describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: process.env.SECRET
+    , demand: true
+    })
+    .option('ssid', {
+      alias: 'i'
+    , describe: 'The name of the session ID cookie.'
+    , type: 'string'
+    , default: process.env.SSID || 'ssid'
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_AUTH_OPENID_` (e.g. ' +
+      '`STF_AUTH_OPENID_SECRET`). Legacy environment variables like ' +
+      'OPENID_IDENTIFIER_URL are still accepted, too, but consider them ' +
+      'deprecated.')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/auth/openid')({
+    port: argv.port
+  , secret: argv.secret
+  , appUrl: argv.appUrl
+  , openid: {
+      identifierUrl: argv.openidIdentifierUrl
+    }
+  })
+}

lib/cli/auth-saml2/index.js

@@ -0,0 +1,73 @@
+module.exports.command = 'auth-saml2'
+
+module.exports.describe = 'Start a SAML 2.0 auth unit.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_AUTH_SAML2')
+    .strict()
+    .option('app-url', {
+      alias: 'a'
+    , describe: 'URL to the app unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7120
+    })
+    .option('saml-id-provider-entry-point-url', {
+      describe: 'SAML 2.0 identity provider URL.'
+    , type: 'string'
+    , default: process.env.SAML_ID_PROVIDER_ENTRY_POINT_URL
+    , demand: true
+    })
+    .option('saml-id-provider-issuer', {
+      describe: 'SAML 2.0 identity provider issuer.'
+    , type: 'string'
+    , default: process.env.SAML_ID_PROVIDER_ISSUER
+    , demand: true
+    })
+    .option('saml-id-provider-cert-path', {
+      describe: 'SAML 2.0 identity provider certificate file path.'
+    , type: 'string'
+    , default: process.env.SAML_ID_PROVIDER_CERT_PATH
+    })
+    .option('secret', {
+      alias: 's'
+    , describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: process.env.SECRET
+    , demand: true
+    })
+    .option('ssid', {
+      alias: 'i'
+    , describe: 'The name of the session ID cookie.'
+    , type: 'string'
+    , default: process.env.SSID || 'ssid'
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_AUTH_SAML2_` (e.g. ' +
+      '`STF_AUTH_SAML2_SECRET`). Legacy environment variables like ' +
+      'SAML_ID_PROVIDER_ISSUER are still accepted, too, but consider them ' +
+      'deprecated.')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/auth/saml2')({
+    port: argv.port
+  , secret: argv.secret
+  , ssid: argv.ssid
+  , appUrl: argv.appUrl
+  , saml: {
+      entryPoint: argv.samlIdProviderEntryPointUrl
+    , issuer: argv.samlIdProviderIssuer
+    , certPath: argv.samlIdProviderCertPath
+    }
+  })
+}

lib/cli/device/index.js

@@ -0,0 +1,145 @@
+module.exports.command = 'device <serial>'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .strict()
+    .option('adb-host', {
+      describe: 'The ADB server host.'
+    , type: 'string'
+    , default: '127.0.0.1'
+    })
+    .option('adb-port', {
+      describe: 'The ADB server port.'
+    , type: 'number'
+    , default: 5037
+    })
+    .option('boot-complete-timeout', {
+      describe: 'How long to wait for boot to complete during device setup.'
+    , type: 'number'
+    , default: 60000
+    })
+    .option('cleanup', {
+      describe: 'Attempt to reset the device between uses by uninstalling' +
+        'apps, resetting accounts and clearing caches. Does not do a perfect ' +
+        'job currently. Negate with --no-cleanup.'
+    , type: 'boolean'
+    , default: true
+    })
+    .option('connect-port', {
+      describe: 'Port allocated to adb connections.'
+    , type: 'number'
+    , demand: true
+    })
+    .option('connect-push', {
+      alias: 'p'
+    , describe: 'ZeroMQ PULL endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('connect-sub', {
+      alias: 's'
+    , describe: 'ZeroMQ PUB endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('connect-url-pattern', {
+      describe: 'The URL pattern to use for `adb connect`.'
+    , type: 'string'
+    , default: '${publicIp}:${publicPort}'
+    })
+    .option('group-timeout', {
+      alias: 't'
+    , describe: 'Timeout in seconds for automatic release of inactive devices.'
+    , type: 'number'
+    , default: 900
+    })
+    .option('heartbeat-interval', {
+      describe: 'Send interval in milliseconds for heartbeat messages.'
+    , type: 'number'
+    , default: 10000
+    })
+    .option('lock-rotation', {
+      describe: 'Whether to lock rotation when devices are being used. ' +
+        'Otherwise changing device orientation may not always work due to ' +
+        'sensitive sensors quickly or immediately reverting it back to the ' +
+        'physical orientation.'
+    , type: 'boolean'
+    })
+    .option('mute-master', {
+      describe: 'Whether to mute master volume when devices are being used.'
+    , type: 'boolean'
+    })
+    .option('provider', {
+      alias: 'n'
+    , describe: 'Name of the provider.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('public-ip', {
+      describe: 'The IP or hostname to use in URLs.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('screen-jpeg-quality', {
+      describe: 'The JPG quality to use for the screen.'
+    , type: 'number'
+    , default: process.env.SCREEN_JPEG_QUALITY || 80
+    })
+    .option('screen-port', {
+      describe: 'Port allocated to the screen WebSocket.'
+    , type: 'number'
+    , demand: true
+    })
+    .option('screen-ws-url-pattern', {
+      describe: 'The URL pattern to use for the screen WebSocket.'
+    , type: 'string'
+    , default: 'ws://${publicIp}:${publicPort}'
+    })
+    .option('storage-url', {
+      alias: 'r'
+    , describe: 'The URL to the storage unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('vnc-initial-size', {
+      describe: 'The initial size to use for the experimental VNC server.'
+    , type: 'string'
+    , default: '600x800'
+    , coerce: function(val) {
+        return val.split('x').map(Number)
+      }
+    })
+    .option('vnc-port', {
+      describe: 'Port allocated to VNC connections.'
+    , type: 'number'
+    , demand: true
+    })
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/device')({
+    serial: argv.serial
+  , provider: argv.provider
+  , publicIp: argv.publicIp
+  , endpoints: {
+      sub: argv.connectSub
+    , push: argv.connectPush
+    }
+  , groupTimeout: argv.groupTimeout * 1000 // change to ms
+  , storageUrl: argv.storageUrl
+  , adbHost: argv.adbHost
+  , adbPort: argv.adbPort
+  , screenWsUrlPattern: argv.screenWsUrlPattern
+  , screenJpegQuality: argv.screenJpegQuality
+  , screenPort: argv.screenPort
+  , connectUrlPattern: argv.connectUrlPattern
+  , connectPort: argv.connectPort
+  , vncPort: argv.vncPort
+  , vncInitialSize: argv.vncInitialSize
+  , heartbeatInterval: argv.heartbeatInterval
+  , bootCompleteTimeout: argv.bootCompleteTimeout
+  , muteMaster: argv.muteMaster
+  , lockRotation: argv.lockRotation
+  , cleanup: argv.cleanup
+  })
+}

lib/cli/doctor/index.js

@@ -0,0 +1,11 @@
+module.exports.command = 'doctor'
+
+module.exports.describe = 'Diagnose potential issues with your installation.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+}
+
+module.exports.handler = function() {
+  return require('../../util/doctor').run()
+}

lib/cli/generate-fake-device/index.js

@@ -0,0 +1,35 @@
+module.exports.command = 'generate-fake-device <model>'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .strict()
+    .option('n', {
+      alias: 'number'
+    , describe: 'How many devices to create.'
+    , type: 'number'
+    , default: 1
+    })
+}
+
+module.exports.handler = function(argv) {
+  var logger = require('../../util/logger')
+  var log = logger.createLogger('cli:generate-fake-device')
+  var fake = require('../../util/fakedevice')
+  var n = argv.number
+
+  function next() {
+    return fake.generate(argv.model).then(function(serial) {
+      log.info('Created fake device "%s"', serial)
+      return --n ? next() : null
+    })
+  }
+
+  return next()
+    .then(function() {
+      process.exit(0)
+    })
+    .catch(function(err) {
+      log.fatal('Fake device creation had an error:', err.stack)
+      process.exit(1)
+    })
+}

lib/cli/index.js

@@ -0,0 +1,40 @@
+var yargs = require('yargs')
+var Promise = require('bluebird')
+
+Promise.longStackTraces()
+
+var _argv = yargs.usage('Usage: $0 <command> [options]')
+  .strict()
+  .command(require('./api'))
+  .command(require('./app'))
+  .command(require('./auth-ldap'))
+  .command(require('./auth-mock'))
+  .command(require('./auth-oauth2'))
+  .command(require('./auth-openid'))
+  .command(require('./auth-saml2'))
+  .command(require('./device'))
+  .command(require('./doctor'))
+  .command(require('./generate-fake-device'))
+  .command(require('./local'))
+  .command(require('./log-rethinkdb'))
+  .command(require('./migrate'))
+  .command(require('./notify-hipchat'))
+  .command(require('./notify-slack'))
+  .command(require('./poorxy'))
+  .command(require('./processor'))
+  .command(require('./provider'))
+  .command(require('./reaper'))
+  .command(require('./storage-plugin-apk'))
+  .command(require('./storage-plugin-image'))
+  .command(require('./storage-s3'))
+  .command(require('./storage-temp'))
+  .command(require('./triproxy'))
+  .command(require('./websocket'))
+  .demand(1, 'Must provide a valid command.')
+  .help('h', 'Show help.')
+  .alias('h', 'help')
+  .version('V', 'Show version.', function() {
+    return require('../../package').version
+  })
+  .alias('V', 'version')
+  .argv

lib/cli/local/index.js

@@ -0,0 +1,408 @@
+module.exports.command = 'local [serial..]'
+
+module.exports.describe = 'Start a complete local development environment.'
+
+module.exports.builder = function(yargs) {
+  var os = require('os')
+
+  return yargs
+    .env('STF_LOCAL')
+    .strict()
+    .option('adb-host', {
+      describe: 'The ADB server host.'
+    , type: 'string'
+    , default: '127.0.0.1'
+    })
+    .option('adb-port', {
+      describe: 'The ADB server port.'
+    , type: 'number'
+    , default: 5037
+    })
+    .option('allow-remote', {
+      alias: 'R'
+    , describe: 'Whether to allow remote devices in STF. Highly ' +
+        'unrecommended due to almost unbelievable slowness on the ADB side ' +
+        'and duplicate device issues when used locally while having a ' +
+        'cable connected at the same time.'
+    , type: 'boolean'
+    })
+    .option('api-port', {
+      describe: 'The port the api unit should run at.'
+    , type: 'number'
+    , default: 7106
+    })
+    .option('app-port', {
+      describe: 'The port the app unit should run at.'
+    , type: 'number'
+    , default: 7105
+    })
+    .option('auth-options', {
+      describe: 'JSON array of options to pass to the auth unit.'
+    , type: 'string'
+    , default: '[]'
+    })
+    .option('auth-port', {
+      describe: 'The port the auth unit should run at.'
+    , type: 'number'
+    , default: 7120
+    })
+    .option('auth-secret', {
+      describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: 'kute kittykat'
+    })
+    .option('auth-type', {
+      describe: 'The type of auth unit to start.'
+    , type: 'string'
+    , choices: ['mock', 'ldap', 'oauth2', 'saml2', 'openid']
+    , default: 'mock'
+    })
+    .option('auth-url', {
+      alias: 'a'
+    , describe: 'URL to the auth unit.'
+    , type: 'string'
+    })
+    .option('bind-app-dealer', {
+      describe: 'The address to bind the app-side ZeroMQ DEALER endpoint to.'
+    , type: 'string'
+    , default: 'tcp://127.0.0.1:7112'
+    })
+    .option('bind-app-pub', {
+      describe: 'The address to bind the app-side ZeroMQ PUB endpoint to.'
+    , type: 'string'
+    , default: 'tcp://127.0.0.1:7111'
+    })
+    .option('bind-app-pull', {
+      describe: 'The address to bind the app-side ZeroMQ PULL endpoint to.'
+    , type: 'string'
+    , default: 'tcp://127.0.0.1:7113'
+    })
+    .option('bind-dev-dealer', {
+      describe: 'The address to bind the device-side ZeroMQ DEALER endpoint to.'
+    , type: 'string'
+    , default: 'tcp://127.0.0.1:7115'
+    })
+    .option('bind-dev-pub', {
+      describe: 'The address to bind the device-side ZeroMQ PUB endpoint to.'
+    , type: 'string'
+    , default: 'tcp://127.0.0.1:7114'
+    })
+    .option('bind-dev-pull', {
+      describe: 'The address to bind the device-side ZeroMQ PULL endpoint to.'
+    , type: 'string'
+    , default: 'tcp://127.0.0.1:7116'
+    })
+    .option('cleanup', {
+      describe: 'Attempt to reset the device between uses by uninstalling' +
+        'apps, resetting accounts and clearing caches. Does not do a perfect ' +
+        'job currently. Negate with --no-cleanup.'
+    , type: 'boolean'
+    , default: true
+    })
+    .option('group-timeout', {
+      alias: 't'
+    , describe: 'Timeout in seconds for automatic release of inactive devices.'
+    , type: 'number'
+    , default: 900
+    })
+    .option('lock-rotation', {
+      describe: 'Whether to lock rotation when devices are being used. ' +
+        'Otherwise changing device orientation may not always work due to ' +
+        'sensitive sensors quickly or immediately reverting it back to the ' +
+        'physical orientation.'
+    , type: 'boolean'
+    })
+    .option('mute-master', {
+      describe: 'Whether to mute master volume when devices are being used.'
+    , type: 'boolean'
+    })
+    .option('port', {
+      alias: ['p', 'poorxy-port']
+    , describe: 'The port STF should run at.'
+    , type: 'number'
+    , default: 7100
+    })
+    .option('provider', {
+      describe: 'An easily identifiable name for the UI and/or log output.'
+    , type: 'string'
+    , default: os.hostname()
+    })
+    .option('provider-max-port', {
+      describe: 'Highest port number for device workers to use.'
+    , type: 'number'
+    , default: 7700
+    })
+    .option('provider-min-port', {
+      describe: 'Lowest port number for device workers to use.'
+    , type: 'number'
+    , default: 7400
+    })
+    .option('public-ip', {
+      describe: 'The IP or hostname to use in URLs.'
+    , type: 'string'
+    , default: 'localhost'
+    })
+    .option('serial', {
+      describe: 'Only use devices with these serial numbers.'
+    , type: 'array'
+    })
+    .option('storage-options', {
+      describe: 'JSON array of options to pass to the storage unit.'
+    , type: 'string'
+    , default: '[]'
+    })
+    .option('storage-plugin-apk-port', {
+      describe: 'The port the storage-plugin-apk unit should run at.'
+    , type: 'number'
+    , default: 7104
+    })
+    .option('storage-plugin-image-port', {
+      describe: 'The port the storage-plugin-image unit should run at.'
+    , type: 'number'
+    , default: 7103
+    })
+    .option('storage-port', {
+      describe: 'The port the storage unit should run at.'
+    , type: 'number'
+    , default: 7102
+    })
+    .option('storage-type', {
+      describe: 'The type of storage unit to start.'
+    , type: 'string'
+    , choices: ['temp', 's3']
+    , default: 'temp'
+    })
+    .option('user-profile-url', {
+      describe: 'URL to external user profile page'
+    , type: 'string'
+    })
+    .option('vnc-initial-size', {
+      describe: 'The initial size to use for the experimental VNC server.'
+    , type: 'string'
+    , default: '600x800'
+    , coerce: function(val) {
+        return val.split('x').map(Number)
+      }
+    })
+    .option('websocket-port', {
+      describe: 'The port the websocket unit should run at.'
+    , type: 'number'
+    , default: 7110
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_LOCAL_` (e.g. ' +
+      '`STF_LOCAL_ALLOW_REMOTE`).')
+}
+
+module.exports.handler = function(argv) {
+  var util = require('util')
+  var path = require('path')
+
+  var Promise = require('bluebird')
+
+  var logger = require('../../util/logger')
+  var log = logger.createLogger('cli:local')
+  var procutil = require('../../util/procutil')
+
+  // Each forked process waits for signals to stop, and so we run over the
+  // default limit of 10. So, it's not a leak, but a refactor wouldn't hurt.
+  process.setMaxListeners(20)
+
+  function run() {
+    var procs = [
+      // app triproxy
+      procutil.fork(path.resolve(__dirname, '..'), [
+          'triproxy', 'app001'
+        , '--bind-pub', argv.bindAppPub
+        , '--bind-dealer', argv.bindAppDealer
+        , '--bind-pull', argv.bindAppPull
+        ])
+
+      // device triproxy
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'triproxy', 'dev001'
+        , '--bind-pub', argv.bindDevPub
+        , '--bind-dealer', argv.bindDevDealer
+        , '--bind-pull', argv.bindDevPull
+        ])
+
+      // processor one
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'processor', 'proc001'
+        , '--connect-app-dealer', argv.bindAppDealer
+        , '--connect-dev-dealer', argv.bindDevDealer
+        ])
+
+      // processor two
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'processor', 'proc002'
+        , '--connect-app-dealer', argv.bindAppDealer
+        , '--connect-dev-dealer', argv.bindDevDealer
+        ])
+
+      // reaper one
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'reaper', 'reaper001'
+        , '--connect-push', argv.bindDevPull
+        , '--connect-sub', argv.bindAppPub
+        ])
+
+      // provider
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'provider'
+        , '--name', argv.provider
+        , '--min-port', argv.providerMinPort
+        , '--max-port', argv.providerMaxPort
+        , '--connect-sub', argv.bindDevPub
+        , '--connect-push', argv.bindDevPull
+        , '--group-timeout', argv.groupTimeout
+        , '--public-ip', argv.publicIp
+        , '--storage-url'
+        , util.format('http://localhost:%d/', argv.port)
+        , '--adb-host', argv.adbHost
+        , '--adb-port', argv.adbPort
+        , '--vnc-initial-size', argv.vncInitialSize.join('x')
+        ]
+        .concat(argv.allowRemote ? ['--allow-remote'] : [])
+        .concat(argv.muteMaster ? ['--mute-master'] : [])
+        .concat(argv.lockRotation ? ['--lock-rotation'] : [])
+        .concat(!argv.cleanup ? ['--no-cleanup'] : [])
+        .concat(argv.serial))
+
+      // auth
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          util.format('auth-%s', argv.authType)
+        , '--port', argv.authPort
+        , '--secret', argv.authSecret
+        , '--app-url', util.format(
+            'http://%s:%d/'
+          , argv.publicIp
+          , argv.port
+          )
+        ].concat(JSON.parse(argv.authOptions)))
+
+      // app
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'app'
+        , '--port', argv.appPort
+        , '--secret', argv.authSecret
+        , '--auth-url', argv.authUrl || util.format(
+            'http://%s:%d/auth/%s/'
+          , argv.publicIp
+          , argv.port
+          , {
+            oauth2: 'oauth'
+          , saml2: 'saml'
+          }[argv.authType] || argv.authType
+          )
+        , '--websocket-url', util.format(
+            'http://%s:%d/'
+          , argv.publicIp
+          , argv.websocketPort
+          )
+        ].concat((function() {
+          var extra = []
+          if (argv.userProfileUrl) {
+            extra.push('--user-profile-url', argv.userProfileUrl)
+          }
+          return extra
+        })()))
+
+      // api
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'api'
+        , '--port', argv.apiPort
+        , '--secret', argv.authSecret
+        , '--connect-push', argv.bindAppPull
+        , '--connect-sub', argv.bindAppPub
+      ])
+
+      // websocket
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'websocket'
+        , '--port', argv.websocketPort
+        , '--secret', argv.authSecret
+        , '--storage-url'
+        , util.format('http://localhost:%d/', argv.port)
+        , '--connect-sub', argv.bindAppPub
+        , '--connect-push', argv.bindAppPull
+        ])
+
+      // storage
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          util.format('storage-%s', argv.storageType)
+        , '--port', argv.storagePort
+        ].concat(JSON.parse(argv.storageOptions)))
+
+      // image processor
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'storage-plugin-image'
+        , '--port', argv.storagePluginImagePort
+        , '--storage-url'
+        , util.format('http://localhost:%d/', argv.port)
+        ])
+
+      // apk processor
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'storage-plugin-apk'
+        , '--port', argv.storagePluginApkPort
+        , '--storage-url'
+        , util.format('http://localhost:%d/', argv.port)
+        ])
+
+      // poorxy
+    , procutil.fork(path.resolve(__dirname, '..'), [
+          'poorxy'
+        , '--port', argv.port
+        , '--app-url'
+        , util.format('http://localhost:%d/', argv.appPort)
+        , '--auth-url'
+        , util.format('http://localhost:%d/', argv.authPort)
+        , '--api-url'
+        , util.format('http://localhost:%d/', argv.apiPort)
+        , '--websocket-url'
+        , util.format('http://localhost:%d/', argv.websocketPort)
+        , '--storage-url'
+        , util.format('http://localhost:%d/', argv.storagePort)
+        , '--storage-plugin-image-url'
+        , util.format('http://localhost:%d/', argv.storagePluginImagePort)
+        , '--storage-plugin-apk-url'
+        , util.format('http://localhost:%d/', argv.storagePluginApkPort)
+        ])
+    ]
+
+    function shutdown() {
+      log.info('Shutting down all child processes')
+      procs.forEach(function(proc) {
+        proc.cancel()
+      })
+      return Promise.settle(procs)
+    }
+
+    process.on('SIGINT', function() {
+      log.info('Received SIGINT, waiting for processes to terminate')
+    })
+
+    process.on('SIGTERM', function() {
+      log.info('Received SIGTERM, waiting for processes to terminate')
+    })
+
+    return Promise.all(procs)
+      .then(function() {
+        process.exit(0)
+      })
+      .catch(function(err) {
+        log.fatal('Child process had an error', err.stack)
+        return shutdown()
+          .then(function() {
+            process.exit(1)
+          })
+      })
+  }
+
+  return procutil.fork(__filename, ['migrate'])
+    .done(run)
+}

lib/cli/log-rethinkdb/index.js

@@ -0,0 +1,36 @@
+module.exports.command = 'log-rethinkdb'
+
+module.exports.describe = 'Start a RethinkDB log unit.'
+
+module.exports.builder = function(yargs) {
+  var logger = require('../../util/logger')
+
+  return yargs
+    .env('STF_LOG_RETHINKDB')
+    .strict()
+    .option('connect-sub', {
+      alias: 's'
+    , describe: 'App-side ZeroMQ PUB endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('priority', {
+      alias: 'p'
+    , describe: 'Minimum log level.'
+    , type: 'number'
+    , default: logger.Level.IMPORTANT
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_LOG_RETHINKDB_` (e.g. ' +
+      '`STF_LOG_RETHINKDB_PRIORITY`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/log/rethinkdb')({
+    priority: argv.priority
+  , endpoints: {
+      sub: argv.connectSub
+    }
+  })
+}

lib/cli/migrate/index.js

@@ -0,0 +1,22 @@
+module.exports.command = 'migrate'
+
+module.exports.describe = 'Migrates the database to the latest version.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+}
+
+module.exports.handler = function() {
+  var logger = require('../../util/logger')
+  var log = logger.createLogger('cli:migrate')
+  var db = require('../../db')
+
+  return db.setup()
+    .then(function() {
+      process.exit(0)
+    })
+    .catch(function(err) {
+      log.fatal('Migration had an error:', err.stack)
+      process.exit(1)
+    })
+}

lib/cli/notify-hipchat/index.js

@@ -0,0 +1,61 @@
+module.exports.command = 'notify-hipchat'
+
+module.exports.describe = 'Start a HipChat notifier unit.'
+
+module.exports.builder = function(yargs) {
+  var logger = require('../../util/logger')
+
+  return yargs
+    .env('STF_NOTIFY_HIPCHAT')
+    .strict()
+    .option('connect-sub', {
+      alias: 's'
+    , describe: 'App-side ZeroMQ PUB endpoint to connect to.'
+    , type: 'array'
+    , demand: true
+    })
+    .option('notify-priority', {
+      alias: 'n'
+    , describe: 'Minimum log level to cause a notification.'
+    , type: 'number'
+    , default: logger.Level.WARNING
+    })
+    .option('priority', {
+      alias: 'p'
+    , describe: 'Minimum log level.'
+    , type: 'number'
+    , default: logger.Level.IMPORTANT
+    })
+    .option('room', {
+      alias: 'r'
+    , describe: 'HipChat room.'
+    , type: 'string'
+    , default: process.env.HIPCHAT_ROOM
+    , demand: true
+    })
+    .option('token', {
+      alias: 't'
+    , describe: 'HipChat v2 API token.'
+    , type: 'string'
+    , default: process.env.HIPCHAT_TOKEN
+    , demand: true
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_NOTIFY_HIPCHAT_` (e.g. ' +
+      '`STF_NOTIFY_HIPCHAT_ROOM`). Legacy environment variables like ' +
+      'HIPCHAT_TOKEN are still accepted, too, but consider them ' +
+      'deprecated.')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/notify/hipchat')({
+    token: argv.token
+  , room: argv.room
+  , priority: argv.priority
+  , notifyPriority: argv.notifyPriority
+  , endpoints: {
+      sub: argv.connectSub
+    }
+  })
+}

lib/cli/notify-slack/index.js

@@ -0,0 +1,54 @@
+module.exports.command = 'notify-slack'
+
+module.exports.describe = 'Start a Slack notifier unit.'
+
+module.exports.builder = function(yargs) {
+  var logger = require('../../util/logger')
+
+  return yargs
+    .env('STF_NOTIFY_SLACK')
+    .strict()
+    .option('channel', {
+      alias: 'c'
+    , describe: 'Slack channel.'
+    , type: 'string'
+    , default: process.env.SLACK_CHANNEL
+    , demand: true
+    })
+    .option('connect-sub', {
+      alias: 's'
+    , describe: 'App-side ZeroMQ PUB endpoint to connect to.'
+    , type: 'array'
+    , demand: true
+    })
+    .option('priority', {
+      alias: 'p'
+    , describe: 'Minimum log level.'
+    , type: 'number'
+    , default: logger.Level.IMPORTANT
+    })
+    .option('token', {
+      alias: 't'
+    , describe: 'Slack API token.'
+    , type: 'string'
+    , default: process.env.SLACK_TOKEN
+    , demand: true
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_NOTIFY_SLACK_` (e.g. ' +
+      '`STF_NOTIFY_SLACK_CHANNEL`). Legacy environment variables like ' +
+      'SLACK_TOKEN are still accepted, too, but consider them ' +
+      'deprecated.')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/notify/slack')({
+    token: argv.token
+  , channel: argv.channel
+  , priority: argv.priority
+  , endpoints: {
+      sub: argv.connectSub
+    }
+  })
+}

lib/cli/please.js

@@ -1,2 +1,2 @@
 require('please-update-dependencies')(module)
-require('./cli')
+require('./')

lib/cli/poorxy/index.js

@@ -0,0 +1,70 @@
+module.exports.command = 'poorxy'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .strict()
+    .env('STF_POORXY')
+    .option('api-url', {
+      alias: 'i'
+    , describe: 'URL to the api unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('app-url', {
+      alias: 'u'
+    , describe: 'URL to the app unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('auth-url', {
+      alias: 'a'
+    , describe: 'URL to the auth unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to launch poorxy on.'
+    , type: 'number'
+    , default: process.env.PORT || 7100
+    })
+    .option('storage-plugin-apk-url', {
+      describe: 'URL to the APK storage plugin unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('storage-plugin-image-url', {
+      describe: 'URL to the image storage plugin unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('storage-url', {
+      alias: 'r'
+    , describe: 'URL to the storage unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('websocket-url', {
+      alias: 'w'
+    , describe: 'URL to the websocket unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_POORXY_` (e.g. ' +
+      '`STF_POORXY_PORT`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/poorxy')({
+    port: argv.port
+  , appUrl: argv.appUrl
+  , apiUrl: argv.apiUrl
+  , authUrl: argv.authUrl
+  , websocketUrl: argv.websocketUrl
+  , storageUrl: argv.storageUrl
+  , storagePluginImageUrl: argv.storagePluginImageUrl
+  , storagePluginApkUrl: argv.storagePluginApkUrl
+  })
+}

lib/cli/processor/index.js

@@ -0,0 +1,42 @@
+module.exports.command = 'processor [name]'
+
+module.exports.describe = 'Start a processor unit.'
+
+module.exports.builder = function(yargs) {
+  var os = require('os')
+
+  return yargs
+    .env('STF_PROCESSOR')
+    .strict()
+    .option('connect-app-dealer', {
+      alias: 'a'
+    , describe: 'App-side ZeroMQ DEALER endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('connect-dev-dealer', {
+      alias: 'd'
+    , describe: 'Device-side ZeroMQ DEALER endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('name', {
+      describe: 'An easily identifiable name for log output.'
+    , type: 'string'
+    , default: os.hostname()
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_PROCESSOR_` (e.g. ' +
+      '`STF_PROCESSOR_CONNECT_APP_DEALER`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/processor')({
+    name: argv.name
+  , endpoints: {
+      appDealer: argv.connectAppDealer
+    , devDealer: argv.connectDevDealer
+    }
+  })
+}

lib/cli/provider/index.js

@@ -0,0 +1,192 @@
+module.exports.command = 'provider [serial..]'
+
+module.exports.describe = 'Start a provider unit.'
+
+module.exports.builder = function(yargs) {
+  var os = require('os')
+  var ip = require('my-local-ip')
+
+  return yargs
+    .strict()
+    .env('STF_PROVIDER')
+    .option('adb-host', {
+      describe: 'The ADB server host.'
+    , type: 'string'
+    , default: '127.0.0.1'
+    })
+    .option('adb-port', {
+      describe: 'The ADB server port.'
+    , type: 'number'
+    , default: 5037
+    })
+    .option('allow-remote', {
+      alias: 'R'
+    , describe: 'Whether to allow remote devices in STF. Highly ' +
+        'unrecommended due to almost unbelievable slowness on the ADB side ' +
+        'and duplicate device issues when used locally while having a ' +
+        'cable connected at the same time.'
+    , type: 'boolean'
+    })
+    .option('boot-complete-timeout', {
+      describe: 'How long to wait for boot to complete during device setup.'
+    , type: 'number'
+    , default: 60000
+    })
+    .option('cleanup', {
+      describe: 'Attempt to reset the device between uses by uninstalling' +
+        'apps, resetting accounts and clearing caches. Does not do a perfect ' +
+        'job currently. Negate with --no-cleanup.'
+    , type: 'boolean'
+    , default: true
+    })
+    .option('connect-push', {
+      alias: 'p'
+    , describe: 'Device-side ZeroMQ PULL endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('connect-sub', {
+      alias: 's'
+    , describe: 'Device-side ZeroMQ PUB endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('connect-url-pattern', {
+      describe: 'The URL pattern to use for `adb connect`.'
+    , type: 'string'
+    , default: '${publicIp}:${publicPort}'
+    })
+    .option('group-timeout', {
+      alias: 't'
+    , describe: 'Timeout in seconds for automatic release of inactive devices.'
+    , type: 'number'
+    , default: 900
+    })
+    .option('heartbeat-interval', {
+      describe: 'Send interval in milliseconds for heartbeat messages.'
+    , type: 'number'
+    , default: 10000
+    })
+    .option('lock-rotation', {
+      describe: 'Whether to lock rotation when devices are being used. ' +
+        'Otherwise changing device orientation may not always work due to ' +
+        'sensitive sensors quickly or immediately reverting it back to the ' +
+        'physical orientation.'
+    , type: 'boolean'
+    })
+    .option('max-port', {
+      describe: 'Highest port number for device workers to use.'
+    , type: 'number'
+    , default: 7700
+    })
+    .option('min-port', {
+      describe: 'Lowest port number for device workers to use.'
+    , type: 'number'
+    , default: 7400
+    })
+    .option('mute-master', {
+      describe: 'Whether to mute master volume when devices are being used.'
+    , type: 'boolean'
+    })
+    .option('name', {
+      alias: 'n'
+    , describe: 'An easily identifiable name for the UI and/or log output.'
+    , type: 'string'
+    , default: os.hostname()
+    })
+    .option('public-ip', {
+      describe: 'The IP or hostname to use in URLs.'
+    , type: 'string'
+    , default: ip()
+    })
+    .option('screen-jpeg-quality', {
+      describe: 'The JPG quality to use for the screen.'
+    , type: 'number'
+    , default: process.env.SCREEN_JPEG_QUALITY || 80
+    })
+    .option('screen-ws-url-pattern', {
+      describe: 'The URL pattern to use for the screen WebSocket.'
+    , type: 'string'
+    , default: 'ws://${publicIp}:${publicPort}'
+    })
+    .option('storage-url', {
+      alias: 'r'
+    , describe: 'The URL to the storage unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('vnc-initial-size', {
+      describe: 'The initial size to use for the experimental VNC server.'
+    , type: 'string'
+    , default: '600x800'
+    , coerce: function(val) {
+        return val.split('x').map(Number)
+      }
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_PROVIDER_` (e.g. ' +
+      '`STF_PROVIDER_NAME`).')
+}
+
+module.exports.handler = function(argv) {
+  var path = require('path')
+  var cli = path.resolve(__dirname, '..')
+
+  function range(from, to) {
+    var items = []
+    for (var i = from; i <= to; ++i) {
+      items.push(i)
+    }
+    return items
+  }
+
+  return require('../../units/provider')({
+    name: argv.name
+  , killTimeout: 10000
+  , ports: range(argv.minPort, argv.maxPort)
+  , filter: function(device) {
+      return argv.serial.length === 0 || argv.serial.indexOf(device.id) !== -1
+    }
+  , allowRemote: argv.allowRemote
+  , fork: function(device, ports) {
+      var fork = require('child_process').fork
+
+      var args = [
+        'device', device.id
+      , '--provider', argv.name
+      , '--screen-port', ports.shift()
+      , '--connect-port', ports.shift()
+      , '--vnc-port', ports.shift()
+      , '--public-ip', argv.publicIp
+      , '--group-timeout', argv.groupTimeout
+      , '--storage-url', argv.storageUrl
+      , '--adb-host', argv.adbHost
+      , '--adb-port', argv.adbPort
+      , '--screen-ws-url-pattern', argv.screenWsUrlPattern
+      , '--screen-jpeg-quality', argv.screenJpegQuality
+      , '--connect-url-pattern', argv.connectUrlPattern
+      , '--heartbeat-interval', argv.heartbeatInterval
+      , '--boot-complete-timeout', argv.bootCompleteTimeout
+      , '--vnc-initial-size', argv.vncInitialSize.join('x')
+      ]
+      .concat(argv.connectSub.reduce(function(all, val) {
+        return all.concat(['--connect-sub', val])
+      }, []))
+      .concat(argv.connectPush.reduce(function(all, val) {
+        return all.concat(['--connect-push', val])
+      }, []))
+      .concat(argv.muteMaster ? ['--mute-master'] : [])
+      .concat(argv.lockRotation ? ['--lock-rotation'] : [])
+      .concat(!argv.cleanup ? ['--no-cleanup'] : [])
+
+      return fork(cli, args)
+    }
+  , endpoints: {
+      sub: argv.connectSub
+    , push: argv.connectPush
+    }
+  , adbHost: argv.adbHost
+  , adbPort: argv.adbPort
+  })
+}

lib/cli/reaper/index.js

@@ -0,0 +1,50 @@
+module.exports.command = 'reaper [name]'
+
+module.exports.describe = 'Start a reaper unit.'
+
+module.exports.builder = function(yargs) {
+  var os = require('os')
+
+  return yargs
+    .env('STF_REAPER')
+    .strict()
+    .option('connect-push', {
+      alias: 'p'
+    , describe: 'Device-side ZeroMQ PULL endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('connect-sub', {
+      alias: 's'
+    , describe: 'App-side ZeroMQ PUB endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('heartbeat-timeout', {
+      alias: 't'
+    , describe: 'Consider devices with heartbeat older than the timeout ' +
+        'value dead. Given in milliseconds.'
+    , type: 'number'
+    , default: 30000
+    })
+    .option('name', {
+      describe: 'An easily identifiable name for log output.'
+    , type: 'string'
+    , default: os.hostname()
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_REAPER_` (e.g. ' +
+      '`STF_REAPER_CONNECT_PUSH`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/reaper')({
+    name: argv.name
+  , heartbeatTimeout: argv.heartbeatTimeout
+  , endpoints: {
+      push: argv.connectPush
+    , sub: argv.connectSub
+    }
+  })
+}

lib/cli/storage-plugin-apk/index.js

@@ -0,0 +1,40 @@
+module.exports.command = 'storage-plugin-apk'
+
+module.exports.describe = 'Start an APK storage plugin unit.'
+
+module.exports.builder = function(yargs) {
+  var os = require('os')
+
+  return yargs
+    .env('STF_STORAGE_PLUGIN_APK')
+    .strict()
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7100
+    })
+    .option('storage-url', {
+      alias: 'r'
+    , describe: 'URL to the storage unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('cache-dir', {
+      describe: 'The location where to cache images.'
+    , type: 'string'
+    , default: os.tmpdir()
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_STORAGE_PLUGIN_APK_` (e.g. ' +
+      '`STF_STORAGE_PLUGIN_APK_CACHE_DIR`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/storage/plugins/apk')({
+    port: argv.port
+  , storageUrl: argv.storageUrl
+  , cacheDir: argv.cacheDir
+  })
+}

lib/cli/storage-plugin-image/index.js

@@ -0,0 +1,48 @@
+module.exports.command = 'storage-plugin-image'
+
+module.exports.describe = 'Start an image storage plugin unit.'
+
+module.exports.builder = function(yargs) {
+  var os = require('os')
+
+  return yargs
+    .env('STF_STORAGE_PLUGIN_IMAGE')
+    .strict()
+    .option('concurrency', {
+      alias: 'c'
+    , describe: 'Maximum number of simultaneous transformations.'
+    , type: 'number'
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7100
+    })
+    .option('storage-url', {
+      alias: 'r'
+    , describe: 'URL to the storage unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('cache-dir', {
+      describe: 'The location where to cache images.'
+    , type: 'string'
+    , default: os.tmpdir()
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_STORAGE_PLUGIN_IMAGE_` (e.g. ' +
+      '`STF_STORAGE_PLUGIN_IMAGE_CONCURRENCY`).')
+}
+
+module.exports.handler = function(argv) {
+  var os = require('os')
+
+  return require('../../units/storage/plugins/image')({
+    port: argv.port
+  , storageUrl: argv.storageUrl
+  , cacheDir: argv.cacheDir
+  , concurrency: argv.concurrency || os.cpus().length
+  })
+}

lib/cli/storage-s3/index.js

@@ -0,0 +1,33 @@
+module.exports.command = 'storage-temp'
+
+module.exports.describe = 'Start a temp storage unit.'
+
+module.exports.builder = function(yargs) {
+  var os = require('os')
+
+  return yargs
+    .env('STF_STORAGE_TEMP')
+    .strict()
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7100
+    })
+    .option('save-dir', {
+      describe: 'The location where files are saved to.'
+    , type: 'string'
+    , default: os.tmpdir()
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_STORAGE_TEMP_` (e.g. ' +
+      '`STF_STORAGE_TEMP_SAVE_DIR`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/storage/temp')({
+    port: argv.port
+  , saveDir: argv.saveDir
+  })
+}

lib/cli/storage-temp/index.js

@@ -0,0 +1,43 @@
+module.exports.command = 'storage-s3'
+
+module.exports.describe = 'Start an S3 storage unit.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_STORAGE_S3')
+    .strict()
+    .option('bucket', {
+      describe: 'S3 bucket name.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('endpoint', {
+      describe: 'S3 bucket endpoint.'
+    , type: 'string'
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7100
+    })
+    .option('profile', {
+      describe: 'AWS credentials profile name.'
+    , type: 'string'
+    , demand: true
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_STORAGE_S3_` (e.g. ' +
+      '`STF_STORAGE_S3_PROFILE`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/storage/s3')({
+    port: argv.port
+  , profile: argv.profile
+  , bucket: argv.bucket
+  , endpoint: argv.endpoint
+  })
+}

lib/cli/triproxy/index.js

@@ -0,0 +1,49 @@
+module.exports.command = 'triproxy [name]'
+
+module.exports.describe = 'Start a triproxy unit.'
+
+module.exports.builder = function(yargs) {
+  var os = require('os')
+
+  return yargs
+    .env('STF_TRIPROXY')
+    .strict()
+    .option('bind-dealer', {
+      alias: 'd'
+    , describe: 'The address to bind the ZeroMQ DEALER endpoint to.'
+    , type: 'string'
+    , default: 'tcp://*:7112'
+    })
+    .option('bind-pub', {
+      alias: 'u'
+    , describe: 'The address to bind the ZeroMQ PUB endpoint to.'
+    , type: 'string'
+    , default: 'tcp://*:7111'
+    })
+    .option('bind-pull', {
+      alias: 'p'
+    , describe: 'The address to bind the ZeroMQ PULL endpoint to.'
+    , type: 'string'
+    , default: 'tcp://*:7113'
+    })
+    .option('name', {
+      describe: 'An easily identifiable name for log output.'
+    , type: 'string'
+    , default: os.hostname()
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_TRIPROXY_` (e.g. ' +
+      '`STF_TRIPROXY_BIND_PUB`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/triproxy')({
+    name: argv.name
+  , endpoints: {
+      pub: argv.bindPub
+    , dealer: argv.bindDealer
+    , pull: argv.bindPull
+    }
+  })
+}

lib/cli/websocket/index.js

@@ -0,0 +1,65 @@
+module.exports.command = 'websocket'
+
+module.exports.describe = 'Start a websocket unit.'
+
+module.exports.builder = function(yargs) {
+  return yargs
+    .env('STF_WEBSOCKET')
+    .strict()
+    .option('connect-push', {
+      alias: 'c'
+    , describe: 'App-side ZeroMQ PULL endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('connect-sub', {
+      alias: 'u'
+    , describe: 'App-side ZeroMQ PUB endpoint to connect to.'
+    , array: true
+    , demand: true
+    })
+    .option('port', {
+      alias: 'p'
+    , describe: 'The port to bind to.'
+    , type: 'number'
+    , default: process.env.PORT || 7110
+    })
+    .option('secret', {
+      alias: 's'
+    , describe: 'The secret to use for auth JSON Web Tokens. Anyone who ' +
+        'knows this token can freely enter the system if they want, so keep ' +
+        'it safe.'
+    , type: 'string'
+    , default: process.env.SECRET
+    , demand: true
+    })
+    .option('ssid', {
+      alias: 'i'
+    , describe: 'The name of the session ID cookie.'
+    , type: 'string'
+    , default: process.env.SSID || 'ssid'
+    })
+    .option('storage-url', {
+      alias: 'r'
+    , describe: 'URL to the storage unit.'
+    , type: 'string'
+    , demand: true
+    })
+    .epilog('Each option can be be overwritten with an environment variable ' +
+      'by converting the option to uppercase, replacing dashes with ' +
+      'underscores and prefixing it with `STF_WEBSOCKET_` (e.g. ' +
+      '`STF_WEBSOCKET_STORAGE_URL`).')
+}
+
+module.exports.handler = function(argv) {
+  return require('../../units/websocket')({
+    port: argv.port
+  , secret: argv.secret
+  , ssid: argv.ssid
+  , storageUrl: argv.storageUrl
+  , endpoints: {
+      sub: argv.connectSub
+    , push: argv.connectPush
+    }
+  })
+}

lib/util/cliutil.js

@@ -8,11 +8,3 @@ module.exports.size = function(val) {
     return [Number(match[1]), Number(match[2])]
   }
 }
-
-module.exports.range = function(from, to) {
-  var items = []
-  for (var i = from; i <= to; ++i) {
-    items.push(i)
-  }
-  return items
-}

lib/util/doctor.js

@@ -145,64 +145,7 @@ doctor.checkADB = function() {
   )
 }
 
-doctor.checkDevices = function() {
+doctor.run = function() {
-  // Show all connected USB devices, including hubs
-  if (os.platform() === 'darwin') {
-    childProcess.execFile('ioreg', ['-p', 'IOUSB', '-w0'],
-      function(error, stdout, stderr) {
-        log.info('USB devices connected including hubs:')
-        if (!execHasErrors(error, stderr)) {
-          var list = stdout.replace(/@.*|\+-o Root\s{2}.*\n|\+-o |^\s{2}/gm, '')
-            .split('\n')
-          list.forEach(function(device) {
-            log.info(device)
-          })
-        }
-      }
-    )
-  }
-  else if (os.platform() === 'linux') {
-    childProcess.execFile('lsusb', [],
-      function(error, stdout, stderr) {
-        log.info('USB devices connected including hubs:')
-        if (!execHasErrors(error, stderr)) {
-          var list = stdout.replace(/Bus \d+ Device \d+: ID \w+:\w+ /gm, '')
-            .split('\n')
-          list.forEach(function(device) {
-            log.info(device)
-          })
-        }
-      }
-    )
-  }
-
-  // Show all the devices seen by adb
-  childProcess.execFile('adb', ['devices'],
-    function(error, stdout, stderr) {
-      log.info('Devices that ADB can see:')
-      if (!execHasErrors(error, stderr)) {
-        var s = stdout.replace(/List of devices attached \n|^\s*/gm, '')
-        if (s.length === 0) {
-          log.error('No devices')
-        }
-        else {
-          var list = s.split('\n')
-          list.forEach(function(device) {
-            log.info(device)
-          })
-        }
-      }
-    }
-  )
-}
-
-doctor.run = function(options) {
-  // Check devices
-  if (options.devices) {
-    doctor.checkDevices()
-    return
-  }
-
   // Check OS architecture
   doctor.checkOSArch()
 

lib/util/srv.js

@@ -81,10 +81,10 @@ srv.resolve = function(domain) {
   var parsedUrl = url.parse(domain)
 
   if (!parsedUrl.protocol) {
-    return Promise.reject(new Error(
+    return Promise.reject(new Error(util.format(
       'Must include protocol in "%s"'
     , domain
-    ))
+    )))
   }
 
   if (/^srv\+/.test(parsedUrl.protocol)) {

package.json

@@ -40,7 +40,6 @@
     "body-parser": "^1.13.3",
     "bufferutil": "^1.2.1",
     "chalk": "~1.1.1",
-    "commander": "^2.9.0",
     "compression": "^1.5.2",
     "cookie-session": "^2.0.0-alpha.1",
     "csurf": "^1.7.0",
@@ -90,6 +89,7 @@
     "utf-8-validate": "^1.2.1",
     "uuid": "^3.0.0",
     "ws": "^1.0.1",
+    "yargs": "^6.5.0",
     "zmq": "^2.14.0"
   },
   "devDependencies": {