slapd monitoring.

We don't use the provided 'slapd_' Munin plugin because it doesn't
support SASL binds.

7 files changed, 398 insertions, 7 deletions

diff --git a/lib/modules/openldap b/lib/modules/openldap
index 91e6a3c..5178033 100644
--- a/lib/modules/openldap
+++ b/lib/modules/openldap
@@ -44,7 +44,7 @@ indexedAttributes = frozenset([
 
 # Another hack. Configuration entries sometimes pollutes the DNs with
 # indices, thus it's not possible to directly use them as base.
-# Instead, we use their parent as a pase, and search for the *unique*
+# Instead, we use their parent as a base, and search for the *unique*
 # match with the same ObjectClass and the matching extra attributes.
 # ('%s' in the attribute value is replaced with the value of the source
 # entry.)
@@ -52,6 +52,7 @@ indexedDN = {
     'olcSchemaConfig':  [('cn',             '{*}%s')],
     'olcMdbConfig':     [('olcDbDirectory', '%s'   )],
     'olcOverlayConfig': [('olcOverlay',     '%s'   )],
+    'olcMonitorConfig': [],
 }
 
 # Allow for flexible ACLs for user using SASL's EXTERNAL mechanism.
diff --git a/roles/common-LDAP/files/usr/local/sbin/slapcat-all.sh b/roles/common-LDAP/files/usr/local/sbin/slapcat-all.sh
index 8aa8f78..cd5abd9 100755
--- a/roles/common-LDAP/files/usr/local/sbin/slapcat-all.sh
+++ b/roles/common-LDAP/files/usr/local/sbin/slapcat-all.sh
@@ -9,11 +9,12 @@ PATH=/usr/sbin:/sbin:/usr/bin:/bin
 target="$1"
 umask 0077
 
-prefix=slapd-
+prefix=slapcat-
 slapcat -n0 -l"$target/${prefix}0.ldif"
 n=$(grep -Ec '^dn:\s+olcDatabase={[1-9][0-9]*}' "$target/${prefix}0.ldif")
 
 while [ $n -gt 0 ]; do
-    slapcat -n$n -l"$target/${prefix}$n.ldif"
+    # the Monitor backend can't be slapcat(8)'ed
+    grep -qE "^dn:\s+olcDatabase=\{$n\}monitor,cn=config$" "$target/${prefix}0.ldif" || slapcat -n$n -l"$target/${prefix}$n.ldif"
     n=$(( $n - 1 ))
 done
diff --git a/roles/common-LDAP/handlers/main.yml b/roles/common-LDAP/handlers/main.yml
index 6972af2..8837729 100644
--- a/roles/common-LDAP/handlers/main.yml
+++ b/roles/common-LDAP/handlers/main.yml
@@ -1,2 +1,5 @@
 - name: Restart slapd
   service: name=slapd state=restarted
+
+- name: Restart munin-node
+  service: name=munin-node state=restarted
diff --git a/roles/common-LDAP/tasks/main.yml b/roles/common-LDAP/tasks/main.yml
index 2eb0dfb..a8c784d 100644
--- a/roles/common-LDAP/tasks/main.yml
+++ b/roles/common-LDAP/tasks/main.yml
@@ -8,6 +8,9 @@
     - ldapvi
     - db-util
     - python-ldap
+    # for the 'slapd2_' munin plugin
+    - libnet-ldap-perl
+    - libauthen-sasl-perl
 
 - name: Configure slapd
   template: src=etc/default/slapd.j2
@@ -107,13 +110,12 @@
 - name: Load amavis' schema
   openldap: target=/etc/ldap/schema/amavis.schema
             format=slapd.conf name=amavis
-  tags:
-    - ldap
 
 - name: Load Fripost' schema
   openldap: target=/etc/ldap/schema/fripost.ldif
-  tags:
-    - ldap
+
+- name: Load the back_monitor overlay
+  openldap: module=back_monitor
 
 # We assume a clean (=stock) cn=config
 - name: Configure the LDAP database
@@ -133,3 +135,26 @@
         dest=/usr/local/sbin/slapcat-all.sh
         owner=root group=root
         mode=0755
+
+
+- name: Install 'slapd2_' Munin wildcard plugin
+  # we don't install 'slapd_' because it doesn't support SASL binds
+  file: src=/usr/local/share/munin/plugins/slapd2_
+        dest=/etc/munin/plugins/slapd2_{{ item }}
+        owner=root group=root
+        state=link force=yes
+  with_items:
+    # sudo /usr/share/munin/plugins/slapd2_ suggest
+    - connections
+    - statistics_entries
+    - operations_diff
+    - statistics_referrals
+    - statistics_pdu
+    - waiters
+    - statistics_bytes
+    - operations
+  tags:
+    - munin
+    - munin-node
+  notify:
+    - Restart munin-node
diff --git a/roles/common-LDAP/templates/etc/ldap/database.ldif.j2 b/roles/common-LDAP/templates/etc/ldap/database.ldif.j2
index b2981b3..5f9d8b1 100644
--- a/roles/common-LDAP/templates/etc/ldap/database.ldif.j2
+++ b/roles/common-LDAP/templates/etc/ldap/database.ldif.j2
@@ -47,6 +47,14 @@ olcPasswordHash: {CRYPT}
 olcPasswordCryptSaltFormat: $6$%s
 
 
+dn: olcDatabase=monitor,cn=config
+objectClass: olcDatabaseConfig
+objectClass: olcMonitorConfig
+olcAccess: to dn.subtree="cn=monitor"
+    by dn.exact="username=munin,cn=peercred,cn=external,cn=auth" sockurl.regex="^ldapi://" read
+    by * =0
+
+
 dn: olcDatabase=mdb,cn=config
 objectClass: olcDatabaseConfig
 objectClass: olcMdbConfig
diff --git a/roles/common/files/usr/local/share/munin/plugins/slapd2_ b/roles/common/files/usr/local/share/munin/plugins/slapd2_
new file mode 100755
index 0000000..56774ac
--- /dev/null
+++ b/roles/common/files/usr/local/share/munin/plugins/slapd2_
@@ -0,0 +1,348 @@
+#!/usr/bin/perl -w
+# -*- perl -*-
+# vim: ft=perl
+
+# Copyright Bjorn Ruberg <bjorn@ruberg.no>
+# Licenced under GPL v2
+#
+# TODO:
+# - Check for OpenLDAP version
+
+# We use one script for all monitoring.
+# This script may be symlinked with several names, all
+# performing different functions:
+# slapd_statistics_bytes
+# slapd_statistics_pdu
+# slapd_statistics_referrals
+# slapd_statistics_entries
+# slapd_connections
+# slapd_waiters
+# slapd_operations
+# slapd_operations_diff
+
+# Magic markers
+#%# family=auto
+#%# capabilities=autoconf suggest
+
+use strict;
+
+my $ret = '';
+
+if (! eval "require Net::LDAP;") {
+   $ret = "Net::LDAP not found";
+}
+
+use vars qw ( $config $param $act $scope $descr $cn $vlabel
+	      $info $title $label);
+
+# Change these to reflect your LDAP ACL. The given DN must have
+# read access to the Monitor branch.
+my $basedn = "cn=Monitor";
+my $server = ($ENV{'server'} || 'localhost');
+my $userdn = ($ENV{'binddn'} || '');
+my $userpw = ($ENV{'bindpw'} || '');
+
+# Remember: connections, bytes, pdu needs scope=base
+
+# The possible measurements
+my %ops =
+    ('statistics_bytes'
+     => {
+         'search' => "cn=Bytes,cn=Statistics",
+         'desc'   => "The number of bytes sent by the LDAP server.",
+         'vlabel' => 'Bytes per ${graph_period}',
+         'label'  => 'Bytes',
+         'title'  => "Number of bytes sent",
+         'info'   => "The graph shows the number of bytes sent",
+	 'scope'  => "base"
+         },
+     'statistics_pdu'
+     => {
+         'search' => "cn=PDU,cn=Statistics",
+         'desc'   => "The number of PDUs sent by the LDAP server.",
+         'vlabel' => 'PDUs per ${graph_period}',
+         'label'  => 'PDUs',
+         'title'  => "Number of PDUs sent",
+         'info'   => "The graph shows the number of PDUs sent",
+	 'scope'  => "base"
+         },
+     # Referrals
+     'statistics_referrals' 
+     => {
+         'search' => "cn=Referrals,cn=Statistics",
+         'desc'   => "The number of Referrals sent by the LDAP server.",
+         'vlabel' => 'Referrals per ${graph_period}',
+         'label'  => 'Referrals',
+         'title'  => "Number of LDAP Referrals",
+         'info'   => "The graph shows the number of referrals sent",
+	 'scope'  => "base"
+         },
+     # Entries
+     'statistics_entries'
+     => {
+         'search' => "cn=Entries,cn=Statistics",
+         'desc'   => "The number of Entries sent by the LDAP server.",
+         'vlabel' => 'Entries per ${graph_period}',
+         'label'  => 'Entries',
+         'title'  => "Number of LDAP Entries",
+         'info'   => "The graph shows the number of entries sent",
+	 'scope'  => "base"
+         },
+     # Only read Total
+     'connections' 
+     => {
+         'search' => 'cn=Total,cn=Connections',
+         'desc'   => 'The number of connections',
+         'label'  => 'Connections',
+         'vlabel' => 'Connections per ${graph_period}',
+         'title'  => 'Number of Connections',
+         'info'   => 'Number of connections to the LDAP server',
+         'scope'  => "base"
+         },
+     # dn: cn=Write,cn=Waiters,cn=Monitor
+     # dn: cn=Read,cn=Waiters,cn=Monitor
+     'waiters' 
+     => {
+         'search' => 'cn=Waiters',
+         'filter' => '(|(cn=Write)(cn=Read))',
+         'desc'   => "The current number of Waiters",
+         'label2' => {'write' => 'Write',
+                      'read'  => 'Read'},
+         'vlabel' => "Waiters",
+         'title'  => "Number of Waiters",
+         'info'   => "The graph shows the number of Waiters"
+         },
+     'operations'
+     => {
+         'search' => "cn=Operations",
+         'desc'   => "Operations",
+         'vlabel' => 'Operations per ${graph_period}',
+         'label'  => 'Operations',
+         'title'  => "Operations",
+         'info'   => "Number of completed LDAP operations"
+         },
+     'operations_diff'
+     => {
+         'search' => "cn=Operations",
+         'desc'   => "Operations deviance",
+         'vlabel' => 'Deviance',
+         'label'  => 'Deviance',
+         'title'  => "Operations deviance",
+         'info'   => "Deviance between Initiated and Completed ops"
+       }
+     );
+
+# Config subroutine
+sub config {
+    my $action = shift;
+    print <<EOF;
+graph_args --base 1000 -l 0
+graph_scale no
+graph_vlabel $ops{$action}->{'vlabel'}
+graph_title $ops{$action}->{'title'}
+graph_category OpenLDAP
+graph_info $ops{$action}->{'info'}
+EOF
+    
+    if ($ops{$action}->{'label2'}) {
+        while (my ($key, $val) = each (%{$ops{$action}->{'label2'}})) {
+          my $name = $action . "_" . $key;
+          print "$name.label $val\n";
+          print "$name.type GAUGE\n";
+        }
+    } elsif ($action =~ /^operations(?:_diff)?$/) {
+        my $ldap = Net::LDAP->new ($server)
+            or die "Failed to connect to server $server: $@";
+        my $mesg;
+        if ($userdn ne '') {
+          $mesg = $ldap->bind ($userdn, password => $userpw)
+              or die "Failed to bind with $userdn: $@";
+        } else {
+          require Authen::SASL;
+          my $sasl = Authen::SASL::->new( mechanism => 'EXTERNAL' );
+          $mesg = $ldap->bind( undef, sasl => $sasl )
+              or die "Failed to SASL bind: $@";
+        }
+        if ($mesg->code) {
+          die "Failed to bind: " . $mesg->error;
+        }
+        my $searchdn = $ops{$action}->{'search'} . "," . $basedn;
+        $mesg =
+            $ldap->search (
+                           base   => $searchdn,
+                           scope  => 'one',
+                           filter => '(objectclass=*)',
+                           attrs  => ['monitorOpInitiated',
+                                      'monitorOpCompleted',
+                                      'cn'],
+                           );
+        $mesg->code && die $mesg->error;
+        
+        my $max = $mesg->count;
+        for (my $i = 0 ; $i < $max ; $i++) {
+            my $entry = $mesg->entry ($i);
+            my $cn = $entry->get_value ('cn');
+            my $name = $action . "_" . lc ($cn);
+            print "$name.label $cn\n";
+            print "$name.type DERIVE\n";
+            print "$name.min 0\n";
+            if ($action eq "operations") {
+                print "$name.info The number of $cn operations\n";
+            } else {
+                print "$name.info The difference between Initiated ";
+                print "and Completed operations (should be 0)\n";
+                print "$name.warning 1\n";
+            }            
+        }
+
+        $ldap->unbind;
+    } else {
+        print "$action.label $ops{$action}->{'label'}\n";
+        print "$action.type DERIVE\n";
+        print "$action.min 0\n";
+    }
+}
+
+# Determine action based on filename first
+(my $action = $0) =~ s/^.*slapd2_([\w\d_]+)$/$1/;
+
+if ($ARGV[0]) {
+    if ($ARGV[0] eq 'autoconf') {
+        # Check for Net::LDAP
+	if ($ret) {
+	    print "no ($ret)\n";
+	    exit 0;
+	}
+
+        # Check for LDAP version 3
+        my $ldap = Net::LDAP->new ($server, version => 3)
+            or do { print "no ($@)\n"; exit 0; };
+
+        my $mesg;
+        if ($userdn ne '') {
+          $mesg = $ldap->bind ($userdn, password => $userpw)
+            or do { print "no ($@)\n"; exit 0; };
+        } else {
+          require Authen::SASL;
+          my $sasl = Authen::SASL::->new( mechanism => 'EXTERNAL' );
+          $mesg = $ldap->bind( undef, sasl => $sasl )
+            or do { print "no ($@)\n"; exit 0; };
+        }
+        if ($mesg->code) {
+          print "no (" . $mesg->error . ")\n";
+          exit 0;
+        }
+
+        $mesg =
+            $ldap->search (
+                           base   => $basedn,
+                           scope  => 'one',
+                           filter => '(objectClass=monitorServer)',
+                           attrs  => [
+                                       'cn',
+                                     ],
+                           );
+        if ($mesg->code) {
+          print "no (" . $mesg->error . ")\n";
+          exit 0;
+        }
+        print "yes\n";
+        exit 0;
+    } elsif ($ARGV[0] eq "config") {
+        if(!exists $ops{$action}) {
+            die "Unknown action specified: $action";
+        }
+        &config ($action);
+    } elsif ($ARGV[0] eq "suggest") {
+        print join ("\n", keys (%ops)), "\n";
+    }
+    exit 0;
+}
+
+# If $action isn't in %ops, we quit
+if(!exists $ops{$action}) {
+    die "Unknown action specified: $action";
+}
+
+# Default scope for LDAP searches. We'll change to other scopes if
+# necessary.
+$scope = "one";
+
+# Net::LDAP variant
+my $ldap = Net::LDAP->new ($server, version => 3)
+    or die "Failed to connect to server $server: $@";
+my $mesg;
+if ($userdn ne '') {
+  $mesg = $ldap->bind ($userdn, password => $userpw)
+      or die "Failed to bind with $userdn: $@";
+} else {
+  require Authen::SASL;
+  my $sasl = Authen::SASL::->new( mechanism => 'EXTERNAL' );
+  $mesg = $ldap->bind( undef, sasl => $sasl )
+      or die "Failed to bind anonymously: $@";
+}
+if ($mesg->code) {
+  die "Failed to bind: " . $mesg->error;
+}
+
+my $searchdn = $ops{$action}->{'search'} . "," . $basedn;
+my $searchattrs;
+
+if ($action =~ /^operations(_diff)?$/) {
+    # We look for different parameters in Operations branch
+    $searchattrs = ['monitorOpInitiated', 'monitorOpCompleted', 'cn'];
+} else {
+    $searchattrs = ['monitorCounter', 'cn'];
+}
+
+my $filter;
+if ($ops{$action}->{'filter'}) {
+  $filter = "(&(objectclass=*)" . $ops{$action}->{'filter'} . ")";
+} else {
+  $filter = "(objectClass=*)";
+}
+
+if ($ops{$action}->{'scope'}) {
+  $scope = $ops{$action}->{'scope'};
+}
+
+$mesg =
+    $ldap->search (
+                   base   => $searchdn,
+                   scope  => $scope,
+                   filter => $filter,
+                   attrs  => $searchattrs,
+                   );
+
+$mesg->code && die $mesg->error;
+
+my $max = $mesg->count;
+
+for (my $i = 0 ; $i < $max ; $i++) {
+    my $entry = $mesg->entry ($i);
+    my $cn = $entry->get_value('cn');
+    if ($action =~ /operations(_diff)?$/) {
+	if ($1) {
+	    my $opsInit =
+		$entry->get_value('monitorOpInitiated'); 
+	    my $opsComp =
+		$entry->get_value('monitorOpCompleted');
+	    print lc ("operations_diff_${cn}.value ");
+	    print ($opsInit - $opsComp);
+	    print "\n";
+	} else {
+	    print lc ("operations_${cn}.value ");
+	    print $entry->get_value('monitorOpCompleted'),
+	    "\n";
+	}
+    } else {
+	# Hotfix, must do for now
+	if ($action =~ /_/ || $action eq 'connections') {
+	    print lc ("${action}.value ");
+	} else {
+	    print lc ("${action}_${cn}.value ");
+	}
+	print $entry->get_value('monitorCounter'), "\n";
+    }
+}
+$ldap->unbind;
diff --git a/roles/common/templates/etc/munin/plugin-conf.d/munin-node.j2 b/roles/common/templates/etc/munin/plugin-conf.d/munin-node.j2
index fa05327..e5ba9b5 100644
--- a/roles/common/templates/etc/munin/plugin-conf.d/munin-node.j2
+++ b/roles/common/templates/etc/munin/plugin-conf.d/munin-node.j2
@@ -135,3 +135,8 @@ env.PGPORT 5432
 
 [fail2ban]
 user root
+
+[slapd2_*]
+user munin
+group munin
+env.server ldapi://