Merge pull request #700 from majorleaguesoccer/couchbase

Updated Couchbase check and corresponding unit test

Author: Remi Hakim

checks.d/couchbase.py

@@ -0,0 +1,121 @@
+import urllib2
+import re
+from util import json, headers
+
+from checks import AgentCheck
+
+#Constants
+COUCHBASE_STATS_PATH = '/pools/nodes'
+
+class Couchbase(AgentCheck):
+    """Extracts stats from Couchbase via its REST API
+    http://docs.couchbase.com/couchbase-manual-2.0/#using-the-rest-api
+    """
+    def _create_metric(self, data, tags=None):
+        storage_totals = data['stats']['storageTotals']
+        for key, storage_type in storage_totals.items():
+            for metric, val in storage_type.items():
+                if val is not None:
+                    cleaned_metric_name = self.camel_case_to_joined_lower(metric)
+                    full_metric_name = '.'.join(['couchbase', key, cleaned_metric_name])
+                    self.gauge(full_metric_name, val, tags=tags)
+#                    self.log.debug('found metric %s with value %s' % (metric_name, val))
+
+        for bucket_name, bucket_stats in data['buckets'].items():
+            for metric, val in bucket_stats.items():
+                if val is not None:
+                    cleaned_metric_name = self.camel_case_to_joined_lower(metric)
+                    full_metric_name = '.'.join(['couchbase', 'by_bucket', cleaned_metric_name])
+                    metric_tags = list(tags)
+                    metric_tags.append('bucket:%s' % bucket_name)
+                    self.gauge(full_metric_name, val[0], tags=metric_tags, device_name=bucket_name)
+#                    self.log.debug('found metric %s with value %s' % (metric_name, val[0]))
+
+        for node_name, node_stats in data['nodes'].items():
+            for metric, val in node_stats['interestingStats'].items():
+                if val is not None:
+                    cleaned_metric_name = self.camel_case_to_joined_lower(metric)
+                    full_metric_name = '.'.join(['couchbase', 'by_node', cleaned_metric_name])
+                    metric_tags = list(tags)
+                    metric_tags.append('node:%s' % node_name)
+                    self.gauge(full_metric_name, val, tags=metric_tags, device_name=node_name)
+#                    self.log.debug('found metric %s with value %s' % (metric_name, val))
+
+
+    def _get_stats(self, url):
+        "Hit a given URL and return the parsed json"
+        self.log.debug('Fetching Couchbase stats at url: %s' % url)
+        req = urllib2.Request(url, None, headers(self.agentConfig))
+
+        # Do the request, log any errors
+        request = urllib2.urlopen(req)
+        response = request.read()
+        return json.loads(response)
+
+    def check(self, instance):
+        server = instance.get('server', None)
+        if server is None:
+            return False
+        data = self.get_data(server)
+        self._create_metric(data, tags=['instance:%s' % server])
+
+    def get_data(self, server):
+        # The dictionary to be returned.
+        couchbase = {'stats': None,
+                'buckets': {},
+                'nodes': {}
+                }
+
+        # build couchbase stats entry point
+        url = '%s%s' % (server, COUCHBASE_STATS_PATH)
+        overall_stats = self._get_stats(url)
+
+        # No overall stats? bail out now
+        if overall_stats is None:
+            raise Exception("No data returned from couchbase endpoint: %s" % url)
+        
+        couchbase['stats'] = overall_stats
+
+        nodes = overall_stats['nodes']
+
+        # Next, get all the nodes
+        if nodes is not None:
+            for node in nodes:
+                couchbase['nodes'][node['hostname']] = node
+
+        # Next, get all buckets .
+        endpoint = overall_stats['buckets']['uri']
+
+        url = '%s%s' % (server, endpoint)
+        buckets = self._get_stats(url)
+
+        if buckets is not None:
+            for bucket in buckets:
+                bucket_name = bucket['name']
+
+                # We have to manually build the URI for the stats bucket, as this is not auto discoverable
+                url = '%s/pools/nodes/buckets/%s/stats' % (server, bucket_name)
+                bucket_stats = self._get_stats(url)
+                bucket_samples = bucket_stats['op']['samples']
+                if bucket_samples is not None:
+                    couchbase['buckets'][bucket['name']] = bucket_samples
+
+        return couchbase
+
+    # Takes a camelCased variable and returns a joined_lower equivalent.
+    # Returns input if non-camelCase variable is detected.
+    def camel_case_to_joined_lower(self, variable):
+        # replace non-word with _
+        converted_variable = re.sub('\W+', '_', variable)
+
+        # insert _ in front of capital letters and lowercase the string
+        converted_variable = re.sub('([A-Z])', '_\g<1>', converted_variable).lower()
+
+        # remove duplicate _
+        converted_variable = re.sub('_+', '_', converted_variable)
+        
+        # handle special case of starting/ending underscores
+        converted_variable = re.sub('^_|_$', '', converted_variable)
+
+        return converted_variable
+

conf.d/couchbase.yaml.example

@@ -0,0 +1,4 @@
+init_config:
+
+instances:
+#    - server: http://localhost:8091

tests/test_couchbase.py

@@ -0,0 +1,61 @@
+import unittest
+from tests.common import load_check
+
+class CouchbaseTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.config = {
+            'instances': [{
+                'server': 'http://localhost:8091',
+            }]
+        }
+        self.agentConfig = {
+            'version': '0.1',
+            'api_key': 'toto'
+        }
+        self.check = load_check('couchbase', self.config, self.agentConfig)
+
+    def test_camel_case_to_joined_lower(self):
+        test_pairs = {
+            'camelCase' : 'camel_case',
+            'FirstCapital' : 'first_capital',
+            'joined_lower' : 'joined_lower',
+            'joined_Upper1' : 'joined_upper1',
+            'Joined_upper2' : 'joined_upper2',
+            'Joined_Upper3' : 'joined_upper3',
+            '_leading_Underscore' : 'leading_underscore',
+            'Trailing_Underscore_' : 'trailing_underscore',
+            'DOubleCAps' : 'd_ouble_c_aps',
+            '@@@super--$$-Funky__$__$$%' : 'super_funky',
+        }
+
+        for test_input, expected_output in test_pairs.items():
+            test_output = self.check.camel_case_to_joined_lower(test_input)
+            self.assertEqual(test_output, expected_output, 
+                'Input was %s, expected output was %s, actual output was %s' % (test_input, expected_output, test_output))
+
+    def test_metrics_casing(self):
+        self.check.check(self.config['instances'][0])
+
+        metrics = self.check.get_metrics()
+
+        camel_cased_metrics = [u'couchbase.hdd.used_by_data', 
+            u'couchbase.ram.used_by_data',
+            u'couchbase.ram.quota_total',
+            u'couchbase.ram.quota_used',
+            ]
+
+        found_metrics = [k[0] for k in metrics if k[0] in camel_cased_metrics]
+        self.assertEqual(found_metrics.sort(), camel_cased_metrics.sort())
+        
+    def test_metrics(self):
+        self.check.check(self.config['instances'][0])
+
+        metrics = self.check.get_metrics()
+        
+        self.assertTrue(type(metrics) == type([]), metrics)
+        self.assertTrue(len(metrics) > 3)
+        self.assertTrue(len([k for k in metrics if "instance:http://localhost:8091" in k[3]['tags']]) > 3)
+
+        self.assertTrue(len([k for k in metrics if -1 != k[0].find('by_node')]) > 1, 'Unable to fund any per node metrics')
+        self.assertTrue(len([k for k in metrics if -1 != k[0].find('by_bucket')]) > 1, 'Unable to fund any per node metrics')