Friday, November 20, 2015

ThingSpeak API for Internet of Things

There is much hype about the "Internet of Things" and the Raspberry Pi is increasingly used as an IoT controller. To describe it simply - if you have a web site accessible from anywhere that displays data from a device and/or allows control of  that device - then you are part of the Internet of Things.

I have a smart thermostat that allows me to control my heating and cooling system remotely from a smart phone app. This is one of the most commonly used IoT devices. I also have created web interfaces to some of the Raspberry Pi controlled devices in my home - alarm system, lights, sprinklers, hot tub, weather station, security cameras.

I programmed these web interfaces myself, but there are much easier ways that are actually more secure, and these use the services that support the Internet of Things. One of the easiest to use is ThingSpeak.com. Here are the steps to use this service.

Go to the ThingSpeak web site and click on "Get Started." Click on "Sign Up" to create an account and then log in. Now you can create a new "Channel" which includes 1 to 8 fields of data. Once created, the channel is assigned an API_Key which is required to update the channel. You can see this value by clicking on the "API Keys" tab.

I have created a channel named "Example" with a single data field. Data can be uploaded to this field by using the following URL:

https://api.thingspeak.com/update?api_key=JPISWIDBF2CERG24&field1=123

(Yes, that is the actual API key. Feel free to post your own values, but try to keep them between 1-1000 so the chart is usable.)

You can view the public view here:  https://thingspeak.com/channels/67033

Note that you can only update once per minute using this method.

Examples in a couple dozen languages, including C and Python are available here:
https://thingspeak.com/docs/examples

Here is an example of real data - temperature and fermentation rate from my home-brewing system.



I decided to create my own API since I had all the necessary code already. This API requires the CURL library to handle the web access. To install this:
sudo apt-get update
sudo apt-get install libcurl4-openssl-dev

Here is the API that I created for updating two fields.
/***********************************************************************
   Filename:   thingspeak.c
   send data to ThingSpeak IoT server
   
   Uses libcurl to send the data via HTTP
   
  13-Nov-2015   Ted Hale  created

************************************************************************/

/* system includes */
#include <stdio.h>
#include <stdlib.h>  
#include <stdarg.h>
#include <time.h>
#include <string.h>
#include <curl/curl.h>

// Note: this function can be replaced with printf if you like.
void Log(char *format, ... );

#define TRUE 1

// the URL structure to update ThingSpeak
// https://api.thingspeak.com/update?api_key=YOUR_CHANNEL_API_KEY&field1=7
char *URLtemplate = "https://api.thingspeak.com/update?api_key=%s&%s=%s&%s=%s";

// structure used by the libcurl write callback function
struct url_data {
    size_t size;
    char* data;
};
 
//=====================================================================
// write callback function needed by libCurl
size_t write_data(void *ptr, size_t size, size_t nmemb, struct url_data *data) {
    size_t index = data->size;
    size_t n = (size * nmemb);
    char* tmp;

    data->size += (size * nmemb);
    tmp = realloc(data->data, data->size + 1); /* +1 for '\0' */

    if(tmp) {
        data->data = tmp;
    } else {
        if(data->data) {
            free(data->data);
        }
        Log("wuthread> write_data Failed to allocate memory.\n");
        return 0;
    }

    memcpy((data->data + index), ptr, n);
    data->data[data->size] = '\0';

    return size * nmemb;
}

//=====================================================================
// upload data to ThingSpeak
int UpdateThingSpeak(char *api_key, char *f1, char *v1, char *f2, char *v2)
{
 int   error = 1;
 time_t   now;
 struct tm  *dt;
 char   url[256];
 
 CURL  *curl;
 CURLcode res;
 struct url_data response;
 
 time(&now);
 dt = gmtime(&now);
                         
 // build the URL string
 snprintf(url, sizeof(url)-1, URLtemplate, api_key, f1, v1, f2, v2);
 // guarantee null termination of string
 url[sizeof(url)-1] = 0;
 
 Log("UpdateThingSpeak> send: [%s]",url);
 
 curl = curl_easy_init();
 if (curl) {
  response.size = 0;
  response.data = malloc(4096); /* reasonable size initial buffer */ 
  response.data[0] = '\0';
  curl_easy_setopt(curl, CURLOPT_URL, url);
  curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "GET");
  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response);
  res = curl_easy_perform(curl);
  if(res != CURLE_OK)
  {
   Log("UpdateThingSpeak> curl_easy_perform() failed: %s\n",curl_easy_strerror(res));
   Log("URL is: [%s]",url);
   error = 1;
  } else {
   // ThingSpeak returns "0" for error, else returns sample number
   error = (strcmp(response.data,"0") == 0);
  }
  curl_easy_cleanup(curl); 
  free (response.data);
 } else {
  Log("UpdateThingSpeak> curl_easy_init failed\n");
  error = 1;
 }
 
 return error;
}

Wednesday, November 11, 2015

Improved Alarm/Automation System

A Little History

I have been fascinated (some would say obsessed) with home automation for many years. I even had a paper published on the subject in 1978, so I experimented with many options before the Raspberry Pi came along. My first working system was based on an Atari 400. I then went through several versions based on microprocessors (like an Arduino, but much older and more primitive) but was never happy with their limitations. Eventually I was able to obtain an old PC and use parallel printer ports for digital I/O. I experimented with Linux and Windows versions and found Linux to be much better suited for what is essentially an embedded system.

The Pi Comes Along

Of course, the first project that I took on when I started working with the Raspberry Pi was a replacement for my PC based home alarm and automation system. The requirements were straight-forward:
    ● several inputs for motion detectors and door/window sensors
    ● several relay outputs for control of miscellaneous devices 
    ● serial (RS-232) port for connecting to an X10 interface
    ● socket interface for control from other computers
    ● logic to provide some limited intelligence
I describe this first version in a series of my early blog posts. Look at Sept. and Oct. 2012 to see these.

Time for an Update

Updated Alarm/Automation System
In the several years that I have worked with the Raspberry Pi I have learned a lot about the best way to do some things and applied these lessons to my updated alarm/automation system. I think the most important lesson that I learned was to make things as modular as possible. It is inevitable that I will screw up something and if the project is one large circuit board, then I may well have to scrap it and start over. However, if I break that into several smaller boards, then a screw-up will, at most, cause the rebuild of a single small circuit. Some systems fit this designs method very well while some just can not be broken up. The alarm/automation system fits this method well and consists of the following parts:
    ● Raspberry Pi (of course)
    ● Simple "hat" for the Pi to expose all the buses needed
    ● a relay board (bought, not built)
    ● control board for the relay
    ● control board for digital inputs
Each of these is described below.


The "Bus Hat"

The Bus Hat
In order to connect the various pieces to the Pi, I needed a way to access some of the buses available. I created what I call a "bus hat" to do this. This mounts on top of the Pi and provides access to the following:
    ● the I2C bus
    ● the SPI bus
    ● the 1-Wire bus
    ● RS-232 converted to proper levels
    ● 5V input to power it all
I knew that I would be using 3.3V devices, so I did not include any level converters on the I2C or SPI buses.  I did need proper voltage levels for the RS-232 serial line and that was provided by a MAX3232 chip. 

This is a very simple device since all it really does is connect specific GPIO pins to connectors for the required buses.



Bus Hat Circuit Design
My design for this bus hat is shown here. You may notice that this is a little different than most circuit diagrams you will see. I am a little embarrassed to admit that I use Power Point to create these drawings. I am mostly concerned with how the components will be laid out on the perf board, so I make a perf board grid the background image and place the components onto that and add the wiring. Power Point makes this quick and easy. If you need to make a simple and better looking wiring diagram I recommend the Fritzing app. For real PCB drawings I have used Eagle and PCB Artist.

Relay Board

Relay Board

I decided to buy a relay board online. There are quite a few of these available and the price is hard to beat. You should be able to find an 8 channel board like this for well under $10. There is no way I could even buy the components for that cheap.







Relay Control
Relay Control Circuit

To control the relays I needed 8 GPIO lines. The perfect job for the MCP23008 GPIO extender. This chip connects via I2C (there is also an SPI version) and provides 8 GPIO lines. I set them all as output and connect one to each relay control pin. One thing to be aware of - these relays require more power to operate than the Pi can generally provide. They will appear to work at first but then you try to turn on all 8 relays and the Pi will reboot due to the drop in power. The relay board will provide a pin for connecting external power to prevent this from happening. Since I was using 3.3V on the I2C bus, I added a 5V input to the relay controller to provide this power.

Input Board

Input Board Circuit
The input board provides 16 "contact closure" inputs. In other words, a switch of some kind connects two pins. This is typical of the way that motion detectors and door switches work. The input board has the GPIO pins pulled high and the external device will close a switch to connect the pin to ground.

The circuit uses an opto-isolator (sometimes called an opto-coupler) to provide protection to the GPIO pins. This is prudent since the inputs are connected to wires that run all over my house and anything could happen to put bad voltage onto those lines. I even went as far as having the opto-isolators in sockets so they could easily be replaced if they are blown. That's probably excessive, but it was easy enough to include, so I did.

Again, I used a GPIO extender. This time a MCP23017, which provides 16 GPIO pins with an I2C bus connection. Both of the extender chips I used will default to I2C bus address 0x20, so I set the address pins on this one so that it will appear on the bus at address 0x21. Note that A0 is tied high while A1 and A2 are tied low.

X10 Interface

X10 Interface


I control the X10 devices using a CM11A interface from ActiveHome. These haven't been made for a while now, but are still available on eBay. It connects via a serial port. NOTE: I had to swap send and receive from the original design of the Bus Hat to connect directly to the CM11A.

I have several X10 controllers for lighting as well as several wireless motion detectors that connect to X10 through a bridge device. These are convenient since they don't require running wires. The downside is that they use batteries. Fortunately, they are fairly efficient and only require replacing a few times a year.

Smart Phone Interface

Smart Phone User Interface


What home automation system would be complete without a smart phone app? A web interface is much more universally usable, so I went that route rather than writing a phone app. Besides, I'm an Android guy, but my other half uses an iPhone. I had no desire to go down that rabbit hole.

The web interface was created using Apache, PHP, and the socket interface to the alarm/automation system. The details on that will have to be a post of its own.





Thursday, November 5, 2015

HDC100x Temperature/Humidity Sensor

Humidity is notoriously difficult to measure accurately. And now I have found that humidity sensors are very fickle. I had been using the AM2315 sensor, which Adafruit clearly states is not weatherproof. I can attest that it is definitely not. It worked OK for a few months, although it did have a problem when it rained. After rain, it would show 100% humidity for a few days until it dried out. Finally a couple of months ago, it died completely.

I have now installed an HDC1008 sensor. This device uses an I2C interface and has a very simple protocol. There is some sample code below.
To get a reliable temperature reading from any sensor it should be placed in a shroud that provides shade and good ventilation. I mounted the HDC1008 inside the shroud from my old weather system. This will protect it from the weather and prevent a spike in temperature reading when the sun hits it.


Accuracy Issues

The image below shows a plot of temperature for an entire day. The HDC1008 is the top line and the bottom line is a DS18B20 one-wire temperature sensor.

There are two things easily noticeable - First, there is a two degree difference in the sensors. I placed my laboratory thermometer next to these sensors for several minutes to get an accurate reading of the true temperature. It was exactly in between the two sensors, so one is one degree low and the other is one degree high. The DS18B20 specifications say it has an accuracy of ±0.5°C which is 0.9°F, so this reading is only a little outside its claimed accuracy. The HDC1008 claims an accuracy of ±0.2°C which is 0.36°F, but mine shows a difference of 0.9°F.

The second thing to notice in the plot below is how wavy the top line is. The reading from the HDC1008 often fluctuate by a degree or more in a very short period. My software takes twenty readings per minute and averages them. This data is recorded each minute. This averaging does not smooth out the plot so I have to conclude that the readings from the HDC1008 really are changing like the plot shows.

Conclusions

Neither of the sensors discussed are going to provide laboratory level accuracy, but I wouldn't expect that from a low cost device. Both are fine for typical home weather monitoring. I use the DS18B20 on several other devices and prefer it because it is very low cost and very simple to use.

I have no way to measure the accuracy of the humidity readings from the HDC1008, but they seem to correlate well with nearby weather stations. My overall impression of the device is positive and I find it considerably better than the AM2315 that it replaced.

Code

Below is sample code that I used to initially test the device. This was adapted from the library for arduino provided by Ladyada.  Many thanks to her and the other folks at Adafruit. 

/*************************************************** 
  sample code for the HDC1000 Humidity & Temp Sensor
  by  ted.b.hale@gmail.com
  2015-08-28
  
  adapted from Adafruit_HDC1000 library, license follows:  
  
  Designed specifically to work with the HDC1000 sensor from Adafruit
  ----> https://www.adafruit.com/products/2635

  These sensors use I2C to communicate, 2 pins are required to  
  interface
  Adafruit invests time and resources providing this open source code, 
  please support Adafruit and open-source hardware by purchasing 
  products from Adafruit!

  Written by Limor Fried/Ladyada for Adafruit Industries.  
  BSD license, all text above must be included in any redistribution
 ****************************************************/

#include <errno.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <unistd.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <wiringPi.h>
#include <wiringPiI2C.h>

#define HDC1000_I2CADDR       0x40
#define HDC1000_TEMP          0x00
#define HDC1000_HUMID         0x01
#define HDC1000_CONFIG_MODE   (1 << 12)
#define HDC1000_CONFIG_RST    (1 << 15)
#define HDC1000_CONFIG_TRES_14  0
#define HDC1000_CONFIG_HRES_14  0
#define HDC1000_MANUFID       0xFE
#define HDC1000_DEVICEID      0xFF


uint16_t i2c_read16(int fd, uint8_t outbyte) 
{
 uint16_t retval;
 write(fd, &outbyte, 1);
 delay(50);
 read(fd, &retval, 2);
 return ntohs(retval); 
}

uint32_t i2c_read32(int fd, uint8_t outbyte) 
{
 uint32_t retval;
 write(fd, &outbyte, 1);
 delay(50);
 read(fd, &retval, 4);
 return ntohl(retval); 
}

int HDC1000_Init(uint8_t addr) 
{
 uint32_t x;
 int fd = wiringPiI2CSetup(addr);
 if (fd==-1)
 {
  printf("wiringPiI2CSetup for hdc1000 failed\n");
  return -1;
 }

 // reset, and select 14 bit temp & humidity
 uint16_t config = HDC1000_CONFIG_RST | HDC1000_CONFIG_MODE | HDC1000_CONFIG_TRES_14 | HDC1000_CONFIG_HRES_14;

 write(fd, &config, 2);
 delay(15);

 x = i2c_read16(fd,HDC1000_MANUFID);
 if (x != 0x5449) 
 {
  printf("HDC1000_MANUFID returned %4X\n",x);
  return -1;
 }
 x = i2c_read16(fd,HDC1000_DEVICEID);
 if (x != 0x1000) 
 {
  printf("HDC1000_DEVICEID returned %4X\n",x);
  return -1;
 }
  
 return fd;
}




float HDC1000_readTemperature(int fd) 
{
  float temp = (i2c_read32(fd,HDC1000_TEMP) >> 16);
  temp /= 65536;
  temp *= 165;
  temp -= 40;

  return temp;
}
  

float HDC1000_readHumidity(int fd) 
{
  float hum = (i2c_read32(fd, HDC1000_TEMP) & 0xFFFF);

  hum /= 65536;
  hum *= 100;

  return hum;
}



int main()
{
 int fd;
 float x;
 
 fd = HDC1000_Init(HDC1000_I2CADDR);
 if (fd==-1)
 {
  printf("HDC1000_Init failed\n");
  return 0;
 }

 x = HDC1000_readTemperature(fd);
 printf("\n temperature = %4.1f degC %4.1f degF\n",x,((x*9)/5)+32);

 x = HDC1000_readHumidity(fd);
 printf("    humidity = %4.1f%%\n\n",x);
 
 return 0;
}