Using Queues with SQS

Setup SQS in Amazon to do Queues

In this video, we expand on our deploy script by adding in a worker queue. It will work like this:

  1. NodeJS will receive a webhook request
  2. Assuming it's valid, NodeJS adds a job into the queue
  3. A Python script (deploy.py) reads the queue for new jobs
  4. When a new job is received, it executes the deploy task we created

AWS

We'll first create a queue within AWS. This also involves creating a new user and adding in the least amount of permissions needed for this action.

The basic steps are to:

  1. Login into AWS.
  2. Create a new queue. Note the URL and ARN.
  3. Create an IAM role/user.
  4. Assign read/write privileges to that user for our new queue

If the ARN of the qeueue created is arn:aws:sqs:us-east-1:308352032554:serial-deploy, then our user permissions will look like this:

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "Stmt1425442063002",
            "Effect": "Allow",
            "Action": [
                "sqs:ChangeMessageVisibility",
                "sqs:DeleteMessage",
                "sqs:GetQueueAttributes",
                "sqs:GetQueueUrl",
                "sqs:ReceiveMessage",
                "sqs:SendMessage"
            ],
            "Resource": [
                "arn:aws:sqs:us-east-1:308352032554:serial-deploy"
            ]
        }
    ]
}

Server Config

On the server, for the deployer user, create a credentials file for the AWS services to use.

Create file ~/.aws/credentials:

[defaults]
aws_access_key_id=1234567890
aws_secret_access_key=123456/abcdefghijk

Next, inside of ~/.aws/config:

[default]
region=us-east-1
output=json

The AWS tools (in our case, Boto, the Python AWS SDK and aws-sdk, the NPM package) will automatically find these configuration files.

NodeJS

We'll grab the NodeJS AWS sdk so we can incorporate the queue into the webhook listener.

# as user deployer
cd ~/build-server
npm install --save aws-sdk

vim ~/.aws/credentials

Then we'll edit our NodeJS web listener and change the deploy code to add a job into the queue, rather than call the python deploy.py script directly.

var express = require('express');
var app = express();

var createHandler = require('github-webhook-handler')
var gitHubHandler = createHandler({ path: '/webhook', secret: 'myhashsecret' });

var AWS = require('aws-sdk');
AWS.config.update({region: 'us-west-1'}); // Must be first
var sqs = new AWS.SQS();

/**
 * Use middleware provided by github-webhook-handler
 * @link https://github.com/rvagg/github-webhook-handler
 * @link http://expressjs.com/guide/using-middleware.html
 */
app.post('/webhook', gitHubHandler, function (req, res)
{
    // If we made it this far
    res.statusCode = 404;
    res.end('no such location');
});

/**
 * A custom webhook for ourselves to call externally
 *   Perhaps from Slack or another chatroom
 *   Or from a CI server
 */
app.post('/deploy', function (req, res)
{
    queue_deploy();

    res.send('Deploying');
});

/**
 * Make ExpressJS application listen
 */
var server = app.listen(8080, '0.0.0.0', function ()
{
  var host = server.address().address;
  var port = server.address().port;

  console.log('Deployer listening at http://%s:%s', host, port);
});

/**
 * Handle Github Webhook events
 */
gitHubHandler.on('error', function (err)
{
  console.error('Github Webhook Error:', err.message)
});

gitHubHandler.on('push', function (event)
{
    console.log(
        'Received a push event for %s to %s',
        event.payload.repository.name,
        event.payload.ref
    )

    // Test what our reference actually is!
    if( event.payload.ref == 'refs/heads/master' )
    {
        queue_deploy();
    }
});

/**
 * Do the actual application deploment
 * @return null
 */
function queue_deploy(ref)
{
    if( typeof ref == "undefined" )
    {
        ref = 'master';
    }

    var params = {
        MessageBody: JSON.stringify({
            time: Math.floor(new Date() / 1000),
            reference: ref
        }),
        QueueUrl: 'https://sqs.us-east-1.amazonaws.com/30852032554/serial-deploy'
    };

    sqs.sendMessage(params, function(err, data) 
    {
        if (err)
        {
            console.log(err, err.stack);
        } else
        {
            console.log('Job Sent', data);
        }
    });
}

After we modify our index.js file, we can run it and try to kick off a deployment by sending a POST request to it:

# In one session
node index.js
# In another terminal session
curl -X POST http://104.236.246.253:8080/deploy

We'll see it deploys! We can see it add a job to the queue. This will appear in your SQS control panal as well.

In the next video, we'll adjust the deploy.py script to listen for new queue jobs.