Wednesday 28 August 2013

Part 3 - Writing a Philips Hue Simulator in Javascript - Implementing the lights API

In part 2, I talked about the routing strategy and we added the functionality to authenticate the requesting user against the bridge "whitelist". I've been "offline" for a while (due to disrupted internet access as part of house renovations) but I'll try and pick things back up where I left off. In this post, we'll implement some of the features provided by the lights section of the API.

Let there be lights...


Before we start implementing the query functions, it makes sense to set up a model for our light (bulb) hardware, a bit like we did with the users in the previous post. We'll create a lights.js in the models subdirectory:
var fileSystem = require('fs');

var lights = [];
var fileName = "./lights.json"

exports.getLights = function () {
    var data = {};

    for (var i = 1; i <= lights.length; i++) {
        data[i.toString()] = {
            "name": lights[i - 1].name
        };
    }

    return data;
};

function loadLights() {
    fileSystem.readFile(fileName, function (err, data) {
        lights = JSON.parse(data);
        console.log("LoadedLights = " + lights);
    });
};

loadLights();
This gives us a nice abstraction to our modelled light hardware data which will come in useful if we decide to store it in a slightly more practical form than just a simple JSON file - anyway, for now we'll just create a simple lights.json file that will be populated with our test data and read by our lights model code:

[{
    "1": {
        "state": {
            "on": false,
            "bri": 200,
            "hue": 0,
            "sat": 0,
            "xy": [0.0, 0.0],
            "ct": 153,
            "alert": "none",
            "effect": "none",
            "colormode": "",
            "reachable": true
        },
        "type": "Test light bulb #1",
        "name": "TestLightBulb1",
        "modelid": "123456",
        "swversion": "1.2.3.45"
    }
}]


Now to implement the API functions that query this information from the bridge. Update routes.js to add the specific function route - the whole file should now look like this:
function routes(users) {

    var lightsController = require('./controllers/lights');
    var groupsController = require('./controllers/groups');
    var schedulesController = require('./controllers/schedules');
    var configController = require('./controllers/configuration');
    var portalController = require('./controllers/portal');

    var hasUsernameProperty = function (request) {
        return request && request.params && request.params.hasOwnProperty("username");
    }

    var validateUser = function (request, response, next) {
        if (hasUsernameProperty(request)) {
            next();
        }
    };

    var authenticateUser = function (request, response, next) {
        if (hasUsernameProperty(request) && users.checkUser(request.params.username)) {
            next();
        } else {
            response.send(200, [{
                error: {
                    type: 1,
                    address: '/',
                    description: 'unauthorized user'
                }
            }]);
        }
    };

    // Lights
    app.get('/api/:username/lights', authenticateUser, lightsController.getAllLights);

    // Configuration
    app.post('/api/:username', validateUser, configController.register(users));
}

module.exports = routes;
A key addition is the authenticateUser function - this will automatically check the user is present in the whitelist and emit an error if they're not. Remember in my previous post, we implemented functionality to be able to add a user to the whitelist - this mechanism simply enforces that. In order to complete the request handling, we'll add our corresponding controller function to actually handle the request. Our lights.js in the controllers subdirectory will now look like this:
var lights = require('../models/lights');

exports.getAllLights = function (request, response) {
    response.send(200, lights.getLights());
};
Now we have a representative, end-to-end implementation of our first light API function. To test it - use curl to send the relevant url request:

C:\>curl -is http://localhost:8080/api/loada/lights

This should return all lights that are known to the bridge in the same format as we'd expect from the real bridge hardware:

C:\>curl -is http://localhost:8080/api/loada/lights
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 31
Date: Tue, 16 Jul 2013 10:49:21 GMT
Connection: keep-alive

{"1":{"name":"TestLightBulb1"}}

Now we have a vertical slice through the simulator - request handling, authentication, data retrieval and response - we can implement the other light API methods. The next method to implement - to retrieve all new lights - requires a bit of extra work because we really need to implement the function that will search for the new lights in order to be able to query them. So, we'll skip ahead to that - add this line to our routes.js file (under the existing route lines):
app.post('/api/:username/lights', authenticateUser, lightsController.searchForNewLights);
Then modify the lights.js in the controllers subdirectory to add:
exports.searchForNewLights = function (request, response) {
    lights.searchLights();
    response.send(200, [{"success": {"/lights": "Searching for new devices" } }]);
};
This requires us to implement the corresponding model function - add this to the lights.js in the models subdirectory:
exports.searchLights = function() {
 if (scanTimeoutId !== null) {
  clearTimeout(scanTimeoutId);
  isScanInProgress = false;
  scanTimeoutId = null;  
 }

 isScanInProgress = true;
 lastScan = new Date();
 
 scanTimeoutId = setTimeout(function() { 
  lights.forEach(function(light) {
   // TODO - only discover 15 lights per search
   if (light.discoveredOn === null) {
    light.discoveredOn = lastScan;
   }
  });
 
  isScanInProgress = false;
 }, 60000);
}
Note that we need to add these local variables to the top of the file:
var lastScan = null;
var scanTimeoutId = null;
var isScanInProgress = false;
They allow us to keep track of the current search so that the simulator can respond correctly when asked for any new lights. In order for the functionality to work, I've added a variable to the light in our JSON model (lights.json):

"discoveredOn": null


To determine the difference between a "new" light and one the bridge already knows about, this field's value is checked - if it's null, it's new. I've added some code to automatically mark all our lights in our model as discovered at startup for now although the actual bridge may do something slightly different (I haven't checked) - the updated loadLights function now looks like this:
function loadLights() {
 fileSystem.readFile(fileName, function(err, data) {
  lights = JSON.parse(data);
  console.log("LoadedLightsCount = " + lights.length);
  
  // Mark each light as "discovered"...
  var scanned = new Date();
  
  lights.forEach(function(light) {   
   light.discoveredOn = scanned;
  });
 });
};
It feels like I've got a bit bogged down here in the nitty gritty of the low-level bridge behaviour - anyway, here's the new lights query function that we were going to implement before we went off piste. As before, we add the route to routes.js:
app.get('/api/:username/lights/new', authenticateUser, lightsController.getNewLights);
We then add our controller function to the lights.js in the controller subdirectory:
exports.getNewLights = function (request, response) {
    response.send(200, lights.getNewLights());
};
And finally, we add the model function that will do the actual work into the lights.js file in our models subdirectory:
exports.getNewLights = function() {
 var lightsQuery = {};

 lights.forEach(function(light, i) {
  if (light.discoveredOn === lastScan) {
   lightsQuery[(i + 1).toString()] = { "name": light.name };
  }
 });
 
 lightsQuery.lastscan = isScanInProgress ? "active" :
         lastScan === null ? "none" : 
         lastScan.toString();
 
 return lightsQuery;
}

I'll freely admit this is a bit rough-and-ready and the logic is a bit ropey but I wanted to keep things fluid and keep progress going forward.

Next time, I'll spend a little time looking at how we can automate some tests for things which should come in useful for checking we're implementing the API correctly. Oh, and continue to flesh out the API implementation in the code of course...

Tuesday 23 July 2013

Part 2 - Writing a Philips Hue Simulator in Javascript - Routing requests

In part 1, I introduced the Hue, talked briefly about the developer API and pulled together some Javascript technologies to build a simulator framework. In this post, we'll implement some code to handle incoming requests.

Exposing the API...


In the previous post, I talked about the main design features of the simulator - first on the list was "Respond to web service requests". As the API is exposed via a web service, this means the routing of the incoming HTTP requests. The majority of the code to do that is provided by the restify library so we just need to organise how we want our simulator to handle the requests and where we put that code.

Philips have made the job easier because they've split the API functionality into several logical areas; lights, groups, schedules, configuration and portal. It makes sense for us to keep this structure so we'll route the HTTP requests to corresponding "controller" modules. To do this, we'll add a module specifically for routing the incoming requests (route.js) and then some modules to perform the request actions (lights.js, groups.js, schedules.js, configuration.js and portal.js).

First, we'll modify the main module we created previously (app.js) to add support for the routing:
var restify = require('restify');

app = module.exports = restify.createServer({
 name: "Hue Bridge Simulator"
});

routes = require('./routes'); // <-- added

app.listen(8080, function() {
  console.log('%s listening at %s', app.name, app.url);
});
And we'll add the new routing module that it's depending on to routes.js:
var lightsController = require('./controllers/lights');
var groupsController = require('./controllers/groups');
var schedulesController = require('./controllers/schedules');
var configController = require('./controllers/configuration');
var portalController = require('./controllers/portal');

app.post('/api/:username', configController.register);
As you can see in the last line above, I've added a handler for the request which registers a new user for the bridge. We need to implement that in the corresponding controller file configuration.js. Note that the controller modules reside in a controllers subdirectory:
exports.register = function(request, response){
 console.log("register - user " + request.params.username);
};

Danny-boy, Danny-boy. Broadsword calling...


To test our simulator, we need to generate a suitable client HTTP request. There are lots of ways we can do this but one of the easiest is to use the curl command line tool. With the server (app.js) running with node.js, type the following in a separate command prompt:

C:\curl> curl -is -X POST http://localhost:8080/api/loada

And you should see the following from our simulator:

C:\> node app.js
Hue Bridge Simulator listening at http://0.0.0.0:8080
register - user loada

Notice that this doesn't really do anything at the moment apart from indicate the parameter (username) that's been supplied as part of the request. This brings us neatly onto another decision point - how are we going to store our data? In this case, we need to keep track of registered users so that we can authenticate the requests coming into the bridge.

Persistence pays off...


Now we need some mechanism that will persist our bridge data between requests. At this early stage, I think a simple JSON file will do the trick - we can always swap it out later for something a bit more heavyweight should we need it.

If your name's not down, you're not coming in...


The Hue API requires the caller to be on an authenticated "whitelist" of registered clients. The bridge maintains this list and only allows registration of a new client if a physical button on the top of the device is pressed - obviously, this requires a real person to be local to the device so its quite an effective security measure. To allow the simulator to keep track of users, we'll require a model (in the MVC sense) of the data. One of the easiest ways is to simply write your data to a JSON file and then read it back in when you require it - we'll store a file called whitelist.json.

In order to keep the model data separate from the rest of the simulator code, data will be abstracted into relevant modules and placed in a models subdirectory. To encapsulate the whitelist file, we'll create a module that offers a simple interface to maintain the list. This abstraction will help if we decide to go for a more robust storage solution later on and it also keeps the controller code focused purely on managing the incoming requests and outgoing responses. Our new module, users.js, looks like this:
var fileSystem = require('fs');
 
var whitelist = [];
var fileName = "./whitelist.json"
 
exports.getUsers = function() {
 return whitelist;
};

exports.addUser = function(username) {
 if (!checkUser(username)) {
  whitelist.push(username);
  saveUsers();
 } 
};

exports.checkUser = function(username) {
 return whitelist.indexOf(username) > -1;
};

function saveUsers() {
 fileSystem.writeFile(fileName, JSON.stringify(whitelist));
};

function loadUsers() {
 fileSystem.readFile(fileName, function(err, data) {
  whitelist = JSON.parse(data);
 });
};

loadUsers();
Nothing too groundbreaking in here - it offers some simple interface functions that manage the internal array of usernames (our so-called "whitelist") and then we save to (and load from) the JSON file. On an implementation sidenote, we'll take advantage of the ability to pass variables to modules through the require call - passing the users model over to the routing module to allow us to perform user authentication there - I'll explain more about that shortly. For now, we need to add the model to the main app.js module:
var restify = require('restify');

app = module.exports = restify.createServer({
 name: "Hue Bridge Simulator"
});

var users = require('./models/users'); // <-- added

routes = require('./routes')(users); // <-- note the parameter

app.listen(8080, function() {
 console.log('%s listening at %s', app.name, app.url);
});

Routing requests...


In order to perform validation and authentication on each client request, we'll modify the routes.js module that we built last time to provide some common checking functionality and then we'll use the chained-handler functionality provided by restify to apply the username authentication as part of the routing process. As the routes module has access to the users model (see how we passed it in the code above), we can query it during routing to check the user making the request is on the whitelist. This saves us having to pollute each controller module with the same boilerplate code - in our solution, the controller won't even be called should the client request fail authentication:
var lightsController = require('./controllers/lights');
var groupsController = require('./controllers/groups');
var schedulesController = require('./controllers/schedules');
var configController = require('./controllers/configuration');
var portalController = require('./controllers/portal');

var hasUsernameProperty = function(request) {
 return request && request.params && request.params.hasOwnProperty("username");
}

var validateUser = function(request, response, next) {
 if (hasUsernameProperty(request)) {
  next(); // Authentication successful, route to the real handler...
 }
};

app.post('/api/:username', validateUser, configController.register);
By chaining the validateUser call before the register handler, we can stop processing immediately should a user parameter not be present. Now we can modify our controller (configuration.js) that we created last time to use our new model's interface for client registration:
var bridgeButtonPressed = false;

exports.register = function (request, response) {
    console.log("register - user " + request.params.username);

    if (bridgeButtonPressed) {
        users.addUser(request.params.username);

        response.send(200, [{
            success: { "username": request.params.username }
        }]);
    } else {
        response.send(200, [{
            error: {
                type: 101,
                address: '',
                description: 'link button not pressed'
            }
        }]);
    }
};
Before we go any further, let's just try out the simulator to test this new functionality - there's a little surprise in store - you may have already spotted it in the code above! Run up the server as before:

C:\>node app.js

Then send a request via curl to register a new user:

C:\>curl -is -X POST http://localhost:8080/api/loada
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 77
Date: Tue, 25 Jul 2013 09:45:56 GMT
Connection: keep-alive
[{"error":{"type":101,"address":"","description":"link button not pressed"}}]

As you see, we get an error back - the link button has not been pressed! In the real world, the bridge requires a press of a physical button on top of the unit upto 30 seconds prior to the registration request being received - this is a simple but effective security measure. For now, we'll hard code the simulator to always assume it has been pressed i.e. allow any new user to register at any time. Stop the simulator using CTRL+C and tweak the bridgeButtonPressed variable to true then re-run the simulator - you should now see:

C:\>curl -is -X POST http://localhost:8080/api/loada
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 36
Date: Mon, 25 Jul 2013 12:12:25 GMT
Connection: keep-alive

[{"success":{"username":"loada"}}]

Stopping and then restarting the simulator will show our whitelisted users being loaded at startup:

C:\>node app.js
Hue Bridge Simulator listening at http://0.0.0.0:8080
LoadedUsers = loada

Routing 101...


And that's all there is to the basic routing of requests. We use restify to attach a controller function to our HTTP verb, resource URL and we specify any expected parameters. It will capture the variables from the request and route them to our username authentication handler first and then onto the specified controller function. This function will then query the model (if necessary), package any data (or error) into the expected response format and emit it back to the caller. Pretty much all of the API functions can be implemented with this simple convention leaving us free to focus on implementing the actual bridge functionality.

Next time...


Now the routing concepts are in place, we can begin to add the implementations for the various functions and any supporting functionality we'll need along the way. So, in the next post, we'll add some code for the functions in the lights part of the Hue API. Still need to think of a good name for the project as well!

Wednesday 17 July 2013

Part 1 - Writing a Philips Hue Simulator in Javascript - Introduction

Welcome to the first of a series of posts which aims to walkthrough the creating of a Philips Hue simulator.

Philips Hue?! Catchy...what is it again??


The short answer...

It's a multi-colour LED light bulb that can be network controlled, made by Philips.

The long(er) answer...

The Hue "system" comprises one (or more) bridge devices that plug into your wired network and each bridge can support up to 50 individual bulbs. Each bulb is an Edison Screw (ES) type and effectively replaces your standard light bulb. However, UK houses tend to have the "bayonet" style fittings but you can buy a cheap adaptor to convert to the ES fitting.

Currently, you can purchase a "starter pack" of devices that contains a single bridge and three bulbs. Also, the bulbs can be bought individually. In the UK Apple stores, they retail for £179.95 and £49.95 respectively. I'm not going to spend any time defending Philips' pricing here but personally, I think it's reasonable value based on the feature set, the build quality and the lifetime of the LED bulbs. I've had good experiences in the past with Philips customer support (replacing broken items etc) so I'm hopeful that there will be good after-market help for the Hue stuff too.

There are official apps for both iOS and Android that allow (remote) control of the Hue hardware.

Does it have a developer API?


Yes, it does. To their credit, Philips have involved the development community from the start, producing what appears to be a well thought out web service API that allows free and extensive access to the Hue hardware for third-party applications. You can enumerate bulbs, control colour and intensity, group bulbs, create mood "scenes" and lots of other interesting things. The developer website is here.

How expensive!?? I want to try Hue out but I can't afford it!!


You're in luck! The goal of this blog series is to (hopefully!) create a simple Hue simulator that will pretend to be the wireless bridge used to network control the bulbs themselves. The idea being that interested developers can use this simulator to prototype ways to control and use Hue bulbs from within their own projects without having to go and buy the kit. I'm actually surprised Philips don't have something like this already?!

About the developer API...


The Hue bridge exposes a RESTful web service that uses the basic HTTP commands to query and control the bulbs. Querying data (generally through GET requests) results in formatted JSON being returned. For security, the bridge mostly only responds to "registered" users - the registration process involves pressing the physical button on the bridge device itself - but there are some commands which are available to any caller but they do not provide any detailed information about the connected devices. It's not exactly maximum security but it keeps out the riff-raff...

The API documentation is well-written, clear and concise so I won't repeat it again here.

Right, what do we actually need to do here?


In simple terms, the simulator needs to do the following:

  • Respond to web service requests i.e. be a simple HTTP web server
  • Maintain a collection of "registered" users
  • Maintain a collection of connected bulbs and their respective states
  • Accurately implement the API including the JSON responses

So, no need to get carried away with too much detail but clearly it's worth separating the network/server implementation, data/configuration and request handling, if we can.

What software technologies to use?


As the data returned from the requests is in JSON format, Javascript seemed the logical choice. I'm not an expert by any means but I've had plenty of experience with it and there's some excellent libraries and frameworks that will help us avoid having to reinvent the wheel...well, handling HTTP requests and the like.

The node.js framework (library?) is one of the darlings of the Javascript world and offers lots of functionality to quickly build a web server application. In order to simplify the REST support, I've chosen to use a node.js-based library called restify to encapsulate the REST-specific stuff.

Getting started with restify...


First job is to install node.js - as I'm using Windows, there's already an automated installer on the node website that will set everything up so that it's accessible from a command-line prompt. Once you've got that, you can open a command prompt window and type the following:

C:>node
> 'Hello World'

If everything is working correctly, you should get the message echoed back:

> 'Hello World'

Next step is to decide how we organise our code. As I said earlier, it's useful to separate the components that will make up the simulator. Although Javascript doesn't support the MVC design "natively" (like ASP.NET's Web API, for example), we can borrow the concepts. I came across Tableau whilst poking around various blogs and it gave me some good ideas on architecture. For now, we just need a simple entry point for our web service which we'll place in a file called app.js - here's how it'll look:
var restify = require('restify');

var app = module.exports = restify.createServer({
 name: "Hue Bridge Simulator"
});

app.listen(8080, function() {
  console.log('%s listening at %s', app.name, app.url);
});

Before we run it, we need to ensure it has access to the restify code it needs (see the first line). The easiest way to do this is to make sure we're in the same directory as app.js in our command prompt and type:

npm install restify

All being well, a directory called node_modules will be created in the directory and all of the restify modules will be pulled into it by the npm tool. Now we're ready to run our skeleton simulator framework by typing at the command-prompt:

node app.js

If the planets are in alignment and everything is setup correctly, we should be rewarded with the rather underwhelming message:

Hue Bridge Simulator listening at http://0.0.0.0:8080

So far, so good. What's next?


Admittedly, it's been a bit slow going but we should have all the basics in place now so we can go ahead an implement the actual bridge functionality. Next time, we'll pick up the pace as we add some specific Hue API request routing and talk about how we store data and configuration. Oh, and like all good software projects, we'll have to think of a cool name at some point...