Amazon ElastiCache for Redis in Lambda

Configure VPC

As there is no auth to connect to Redis server, all security policy are based on VPC‘s Security Group, so before setup Redis server, we have to configure the VPC.

Sign in to VPC Console, make sure there is at least one VPC
Open Security Group, create a new Security Group

Note: The VPC should be used same one in the following configuration

Launch a Redis Cluster

Amazon ElastiCache for Redis clusters provide a variety of node configurations:

Single Node
Single Shard Multiple Nodes
A single shard configuration groups up to 6 nodes in a single cluster, a read/write Primary node and up to 5 read-only Replica nodes.
Multiple Shards Multiple Nodes
A multiple shard configuration partitions your data across up to 20 shards in the cluster. Each shard has a read/write Primary node and up to 5 read-only replica nodes.

Now we setup a ‘Single Node’ Redis Cluster:

Sign in to ElastiCache Console
Choose Get Started Now.
Choose the region
Choose Redis as cluster engine
Complete ‘Redis settings’

Note: As we are configuring a Single Node Redis, so we set Number of Replicas as None.
Complete ‘Advanced Redis settings’

Note: Keep VPC ID same as the VPC configuration, and remember the Subnets
The other items are kept as default
Choose Create cluster to launch your cluster

The Create operation will take a while.
View the details of cluster cluster we created

Primary Endpoint will be configured Redis client for connecting Redis server.
Click on the cluster’s name, the node(s) in the cluster will be listed, and you can see the metrics of each node
The main step is checking the Security Group, make sure the its same as we created:

Now the Redis Server on AWS is up.

Prepare Role to Run Lambda

Sign in to IAM Console
To create a new Role:
Choose AWS Service as role type, and Lambda as service
Selected AWSLambdaVPCAccessExecutionRole at step of Attach permissions policies
This permission is enough for us to connect to Redis server in Lambda function.
Named the role
Role is created!

Lambda Function

Create Lambda Function

Create a Lambda function
User Node.js as dev env, and select the role we just created
Created Lambda function
Configure Lambda Role, min required memory or timeout configuration
Configure Network

Note:
1. VPC must be same as above configuration
2. Subnets must contain the same Subnets when we setup Redis
3. Security Groups must be save as above configuration

Write Lambda Function to connect to Redis

We use node_redis as client to connect Redis server in our Lambda function.

The project structure is as below:

lambda_function
|-- node_modules
|-- package.json
|-- index.js

The dependencies in package.json is:

"main": "index.js",
"dependencies": {
    "bluebird": "^3.5.1",
    "redis": "^2.8.0"
}

bluebird is for Promise implementation.

The code implementation of Lambda function is in index.js:

'use strict';

var redis = require('redis');
var bluebird = require('bluebird');
bluebird.promisifyAll(redis.RedisClient.prototype);
bluebird.promisifyAll(redis.Multi.prototype);

const GLOBAL_KEY = 'lambda-test';
const redisOptions = {
    host: "redistest.cbmmpl.0001.use1.cache.amazonaws.com",
    port: 6379
}

redis.debug_mode = true;

exports.handler = (event, context, callback) => {
    console.info('Start to connect to Redis Server')
    var client = redis.createClient(redisOptions);
    console.info('Connected to Redis Server')

    console.info('event.pathParameters: ', event.pathParameters);
    console.info('event.httpMethod: ', event.httpMethod);
    let id = (event.pathParameters || {}).product || false;
    let data = event.data;

    switch (event.httpMethod) {

        case "GET":
            if (id) {
                console.info('get by id')
                client.hgetAsync(GLOBAL_KEY, id).then(res => {
                    console.info('Redis responses for get single: ', res);
                    callback(null, {body:  "This is a READ operation on product ID " + id, ret: res});
                    // callback(null, {body: "This is a READ operation on product ID " + id});
                }).catch(err => {
                    console.error("Failed to get single: ", err)
                    callback(null, {statusCode: 500, message: "Failed to get data"});
                }).finally(() => {
                    console.info('Disconnect to Redis');
                    client.quit();
                });

                return;
            } else {
                console.info('get all')
                client.hgetallAsync(GLOBAL_KEY).then(res => {
                    console.info('Redis responses for get all: ', res)
                    callback(null, {body: "This is a LIST operation, return all products", ret: res});
                    // callback(null, {body: "This is a LIST operation, return all products"});
                }).catch(err => {
                    console.error("Failed to post data: ", err)
                    callback(null, {statusCode: 500, message: "Failed to get data"});
                }).finally(() => {
                    console.info('Disconnect to Redis');
                    client.quit();
                });
            }
            break;

        case "POST":
            if (data) {
                console.info('Posting data for [', id, '] with value: ', data);
                client.hmsetAsync(GLOBAL_KEY, id, data).then(res => {
                    console.info('Redis responses for post: ', res)
                    callback(null, {body: "This is a CREATE operation and it's successful", ret: res});
                    // callback(null, {body: "This is a CREATE operation"});
                }).catch(err => {
                    console.error("Failed to post data: ", err)
                    callback(null, {statusCode: 500, message: "Failed to post data"});
                }).finally(() => {
                    console.info('Disconnect to Redis');
                    client.quit();
                });
            }
            else {
                callback(null, {statusCode: 500, message: 'no data is posted'})
            }
            break;

        case "PUT":
            callback(null, {body: "This is an UPDATE operation on product ID " + id});
            break;

        case "DELETE":
            console.info('delete a prod');
            client.delAsync(GLOBAL_KEY).then(res => {
                console.info('Redis responses for get single: ', res);
                callback(null, {body:  "This is a DELETE operation on product ID " + id, ret: res});
                // callback(null, {body: "This is a DELETE operation on product ID " + id});
            }).catch(err => {
                console.error("Failed to delete single: ", err);
                callback(null, {statusCode: 500, message: "Failed to delete data"});
            }).finally(() => {
                console.info('Disconnect to Redis');
                client.quit();
            });

            break;

        default:
            // Send HTTP 501: Not Implemented
            console.log("Error: unsupported HTTP method (" + event.httpMethod + ")");
            callback(null, {statusCode: 501})
    }

}

The redisOptions is from the Redis cluster details.

Package the function
First install NPM packages:
1
npm install
Then zip the files.
Upload the package:
Lambda function is deployed:

Test Lambda Function

To test the lambda, we need to create some Test Events:

Test writing value to Redis Server

Create Test Event:

{
  "httpMethod": "POST",
  "pathParameters": {
    "product": "key-1"
  },
  "data": "I am a test string"
}

Then clicking Test to mock api call:
We also can view the logs in AWS CloudWatch

Test reading value from Redis Server

Create Test Event:
1
2
3
{
"httpMethod": "GET"
}
Then clicking Test to mock api call:

The value we set is populated from Redis server correctly.

Access ElastiCache from EC2

Excepted AWS Lambda Function, ElastiCache can only be accessed through an Amazon EC2 instance. Below is the step to setup a EC2 instance to access the Redis server:

Sign in to EC2 Console
Launch a new Instance

When configuring the instance details, the Network must set the VPC we created, and Subnet must be same as the above Redis.
If the Security Group doesn’t contain the same one as Redis server, you have to change the configuration:
Connect the EC2 instance with ssh:
Access Redis server through EC2 instance:
1
2
3
4
$ redis-cli -h redistest.yyyy.xxxx.use1.cache.amazonaws.com
redistest.yyyy.xxxx.use1.cache.amazonaws.com:6379> ping
PONG
redistest.yyyy.xxxx.use1.cache.amazonaws.com:6379>
When ping responses PONG, it means the EC2 instance has connected to ElastiCache service, now you can deploy/test your application which need Redis service.
Note:For this step, the EC2 instance have to be installed redis-cli tool.

Scalability of Amazon ElastiCache for Redis

The above example of Redis cluster is a single node, we can add more readonly nodes for it.

Add Replication

Goto Nodes tab, select the node, then click Add Replication
Complete the information, to change the current Single Node cluster to Single Shard and Single Node:
Now the changes is finished, you can see the Shards column is become to 1 from 0
Goto Nodes tab, we can add more readonly node for this shard
Named the new node
The readonly replica node has been created

Now the new node is up, we add more replica nodes if needed
Note: As the current instance is setup without the Cluster Mode is disabled, so we can only have one shard. More info

Test Replication

Write Lambda Function

Add new Lambda function to test the read write function:

const redisReadOptions = {
    host: "firstreplicanode.cbmmpl.0001.use1.cache.amazonaws.com",
    port: 6379
}
const redisWriteOptions = {
    host: "redistest.cbmmpl.0001.use1.cache.amazonaws.com",
    port: 6379
}

...

function getReadClient(){
    console.info('Start to connect to Redis Server')
    var client = redis.createClient(redisReadOptions);
    console.info('Connected to Redis Server')
    return client
}

function getWriteClient(){
    console.info('Start to connect to Redis Server')
    var client = redis.createClient(redisWriteOptions);
    console.info('Connected to Redis Server')
    return client
}

...

case "GET":
    var client = getReadClient();
    console.info('get all')
    client.hgetallAsync(GLOBAL_KEY).then(res => {
        console.info('Redis responses for get all: ', res)
        callback(null, {body: "This is a LIST operation, return all products", ret: res});
        // callback(null, {body: "This is a LIST operation, return all products"});
    }).catch(err => {
        console.error("Failed to post data: ", err)
        callback(null, {statusCode: 500, message: "Failed to get data"});
    }).finally(() => {
        console.info('Disconnect to Redis');
        client.quit();
    });
    break;

case "POST":
    var client = getWriteClient();
    if (data) {
        console.info('Posting data for [', id, '] with value: ', data);
        client.hmsetAsync(GLOBAL_KEY, id, data).then(res => {
            console.info('Redis responses for post: ', res)
            callback(null, {body: "This is a CREATE operation and it's successful", ret: res});
            // callback(null, {body: "This is a CREATE operation"});
        }).catch(err => {
            console.error("Failed to post data: ", err)
            callback(null, {statusCode: 500, message: "Failed to post data"});
        }).finally(() => {
            console.info('Disconnect to Redis');
            client.quit();
        });
    }
    else {
        callback(null, {statusCode: 500, message: 'no data is posted'})
    }
    break;

  ...

Then deploy the code

Test Lambda Function

Check the data in readonly node
Write data to the read/write Primary node
The re-check the data in readonly node

WORKING!!!

Summary

No mater EC2 instance or Lambda function to access ElastiCache service, they must:

Be in one VPC
Contain same Subnet
Contain same Security Group