slapd monitoring.

We don't use the provided 'slapd_' Munin plugin because it doesn't
support SASL binds.

6 files changed, 396 insertions, 6 deletions

diff --git a/roles/common-LDAP/files/usr/local/sbin/slapcat-all.sh b/roles/common-LDAP/files/usr/local/sbin/slapcat-all.sh
index 8aa8f78..cd5abd9 100755
--- a/roles/common-LDAP/files/usr/local/sbin/slapcat-all.sh
+++ b/roles/common-LDAP/files/usr/local/sbin/slapcat-all.sh
@@ -1,19 +1,20 @@
 #!/bin/sh
 
 # Usage: slapcat-all.sh DIR
 # Save all LDAP databases in DIR: DIR/0.ldif, DIR/1.ldif, ...
 
 set -ue
 PATH=/usr/sbin:/sbin:/usr/bin:/bin
 
 target="$1"
 umask 0077
 
-prefix=slapd-
+prefix=slapcat-
 slapcat -n0 -l"$target/${prefix}0.ldif"
 n=$(grep -Ec '^dn:\s+olcDatabase={[1-9][0-9]*}' "$target/${prefix}0.ldif")
 
 while [ $n -gt 0 ]; do
-    slapcat -n$n -l"$target/${prefix}$n.ldif"
+    # the Monitor backend can't be slapcat(8)'ed
+    grep -qE "^dn:\s+olcDatabase=\{$n\}monitor,cn=config$" "$target/${prefix}0.ldif" || slapcat -n$n -l"$target/${prefix}$n.ldif"
     n=$(( $n - 1 ))
 done
diff --git a/roles/common-LDAP/handlers/main.yml b/roles/common-LDAP/handlers/main.yml
index 6972af2..8837729 100644
--- a/roles/common-LDAP/handlers/main.yml
+++ b/roles/common-LDAP/handlers/main.yml
@@ -1,2 +1,5 @@
 - name: Restart slapd
   service: name=slapd state=restarted
+
+- name: Restart munin-node
+  service: name=munin-node state=restarted
diff --git a/roles/common-LDAP/tasks/main.yml b/roles/common-LDAP/tasks/main.yml
index 2eb0dfb..a8c784d 100644
--- a/roles/common-LDAP/tasks/main.yml
+++ b/roles/common-LDAP/tasks/main.yml
@@ -1,30 +1,33 @@
 # XXX If #742056 gets fixed, we should preseed slapd to use peercreds as
 # RootDN once the fix enters stable.
 - name: Install OpenLDAP
   apt: pkg={{ item }}
   with_items:
     - slapd
     - ldap-utils
     - ldapvi
     - db-util
     - python-ldap
+    # for the 'slapd2_' munin plugin
+    - libnet-ldap-perl
+    - libauthen-sasl-perl
 
 - name: Configure slapd
   template: src=etc/default/slapd.j2
             dest=/etc/default/slapd
             owner=root group=root
             mode=0644
   register: r1
   notify:
     - Restart slapd
 
 - name: Create directory /etc/ldap/ssl
   file: path=/etc/ldap/ssl
         state=directory
         owner=root group=root
         mode=0755
   tags:
     - genkey
 
 # XXX: It's ugly to list all roles here, and to prunes them with a
 # conditional...
@@ -90,46 +93,68 @@
 
 - meta: flush_handlers
 
 - name: Copy fripost & amavis' schema
   copy: src=etc/ldap/schema/{{ item }}
         dest=/etc/ldap/schema/{{ item }}
         owner=root group=root
         mode=0644
   # It'd certainly be nicer if we didn't have to deploy amavis' schema
   # everywhere, but we need the 'objectClass' in our replicates, hence
   # they need to be aware of the 'amavisAccount' class.
   with_items:
     - fripost.ldif
     - amavis.schema
   tags:
     - amavis
 
 - name: Load amavis' schema
   openldap: target=/etc/ldap/schema/amavis.schema
             format=slapd.conf name=amavis
-  tags:
-    - ldap
 
 - name: Load Fripost' schema
   openldap: target=/etc/ldap/schema/fripost.ldif
-  tags:
-    - ldap
+
+- name: Load the back_monitor overlay
+  openldap: module=back_monitor
 
 # We assume a clean (=stock) cn=config
 - name: Configure the LDAP database
   openldap: target=etc/ldap/database.ldif.j2 local=template
 
 # On read-only replicates, you might have to temporarily switch back to
 # read-write, delete the SyncRepl, and delete the DN manually:
 #     sudo ldapdelete -Y EXTERNAL -H ldapi:// cn=admin,dc=fripost,dc=org
 - name: Remove cn=admin,dc=fripost,dc=org
   openldap: name="cn=admin,dc=fripost,dc=org" delete=entry
 
 - name: Remove the rootDN under the 'config' database
   openldap: name="olcDatabase={0}config,cn=config" delete=olcRootDN,olcRootPW
 
 - name: Copy /usr/local/sbin/slapcat-all.sh
   copy: src=usr/local/sbin/slapcat-all.sh
         dest=/usr/local/sbin/slapcat-all.sh
         owner=root group=root
         mode=0755
+
+
+- name: Install 'slapd2_' Munin wildcard plugin
+  # we don't install 'slapd_' because it doesn't support SASL binds
+  file: src=/usr/local/share/munin/plugins/slapd2_
+        dest=/etc/munin/plugins/slapd2_{{ item }}
+        owner=root group=root
+        state=link force=yes
+  with_items:
+    # sudo /usr/share/munin/plugins/slapd2_ suggest
+    - connections
+    - statistics_entries
+    - operations_diff
+    - statistics_referrals
+    - statistics_pdu
+    - waiters
+    - statistics_bytes
+    - operations
+  tags:
+    - munin
+    - munin-node
+  notify:
+    - Restart munin-node
diff --git a/roles/common-LDAP/templates/etc/ldap/database.ldif.j2 b/roles/common-LDAP/templates/etc/ldap/database.ldif.j2
index b2981b3..5f9d8b1 100644
--- a/roles/common-LDAP/templates/etc/ldap/database.ldif.j2
+++ b/roles/common-LDAP/templates/etc/ldap/database.ldif.j2
@@ -30,40 +30,48 @@ olcThreads: 8
 olcTLSCertificateFile: /etc/ldap/ssl/ldap.fripost.org.pem
 olcTLSCertificateKeyFile: /etc/ldap/ssl/ldap.fripost.org.key
 # If we are being offered a client cert, it has to be trusted (in which
 # case we map the X.509 subject to a DN in our namespace), or we
 # terminate the connection.  Not providing a certificate is fine for
 # TLS-protected simple binds, though.
 olcTLSVerifyClient: try
 olcTLSCACertificateFile: /etc/ldap/ssl/clients.pem
 olcAuthzRegexp: "^(cn=[^,]+,ou=syncRepl),ou=LDAP,ou=SSLcerts,o=Fripost$"
                 "$1,dc=fripost,dc=org"
 olcSaslSecProps: minssf=128,noanonymous,noplain,nodict
 olcTLSCipherSuite: PFS:%LATEST_RECORD_VERSION:!CIPHER-ALL:+AES-128-GCM:+AES-256-GCM:!VERS-SSL3.0:!VERS-TLS1.0:!VERS-TLS1.1
 {% endif %}
 olcLocalSSF: 128
 # /!\ This is not portable! But we only use glibc's crypt(3), which
 # supports (salted, streched) SHA512
 olcPasswordHash: {CRYPT}
 olcPasswordCryptSaltFormat: $6$%s
 
 
+dn: olcDatabase=monitor,cn=config
+objectClass: olcDatabaseConfig
+objectClass: olcMonitorConfig
+olcAccess: to dn.subtree="cn=monitor"
+    by dn.exact="username=munin,cn=peercred,cn=external,cn=auth" sockurl.regex="^ldapi://" read
+    by * =0
+
+
 dn: olcDatabase=mdb,cn=config
 objectClass: olcDatabaseConfig
 objectClass: olcMdbConfig
 olcDbDirectory: /var/lib/ldap
 olcSuffix: dc=fripost,dc=org
 {% if 'LDAP-provider' not in group_names and 'MX' in group_names %}
 olcReadOnly: TRUE
 {% endif %}
 {% if 'LDAP-provider' in group_names %}
 olcLastMod: TRUE
 olcDbCheckpoint: 512 15
 {% else %}
 olcLastMod: FALSE
 {% endif %}
 # The root user has all rights on the whole database (when SASL-binding
 # on a UNIX socket).
 olcRootDN: gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth
 # Ensure that all DIT access is made according to the LDAPv3 protocol,
 # and must use 1/ authentication, and 2/ SASL or TLS.  (Local clients
 # should use ldapi:// and SASL/EXERNAL, while remote clients should use
diff --git a/roles/common/files/usr/local/share/munin/plugins/slapd2_ b/roles/common/files/usr/local/share/munin/plugins/slapd2_
new file mode 100755
index 0000000..56774ac
--- /dev/null
+++ b/roles/common/files/usr/local/share/munin/plugins/slapd2_
@@ -0,0 +1,348 @@
+#!/usr/bin/perl -w
+# -*- perl -*-
+# vim: ft=perl
+
+# Copyright Bjorn Ruberg <bjorn@ruberg.no>
+# Licenced under GPL v2
+#
+# TODO:
+# - Check for OpenLDAP version
+
+# We use one script for all monitoring.
+# This script may be symlinked with several names, all
+# performing different functions:
+# slapd_statistics_bytes
+# slapd_statistics_pdu
+# slapd_statistics_referrals
+# slapd_statistics_entries
+# slapd_connections
+# slapd_waiters
+# slapd_operations
+# slapd_operations_diff
+
+# Magic markers
+#%# family=auto
+#%# capabilities=autoconf suggest
+
+use strict;
+
+my $ret = '';
+
+if (! eval "require Net::LDAP;") {
+   $ret = "Net::LDAP not found";
+}
+
+use vars qw ( $config $param $act $scope $descr $cn $vlabel
+	      $info $title $label);
+
+# Change these to reflect your LDAP ACL. The given DN must have
+# read access to the Monitor branch.
+my $basedn = "cn=Monitor";
+my $server = ($ENV{'server'} || 'localhost');
+my $userdn = ($ENV{'binddn'} || '');
+my $userpw = ($ENV{'bindpw'} || '');
+
+# Remember: connections, bytes, pdu needs scope=base
+
+# The possible measurements
+my %ops =
+    ('statistics_bytes'
+     => {
+         'search' => "cn=Bytes,cn=Statistics",
+         'desc'   => "The number of bytes sent by the LDAP server.",
+         'vlabel' => 'Bytes per ${graph_period}',
+         'label'  => 'Bytes',
+         'title'  => "Number of bytes sent",
+         'info'   => "The graph shows the number of bytes sent",
+	 'scope'  => "base"
+         },
+     'statistics_pdu'
+     => {
+         'search' => "cn=PDU,cn=Statistics",
+         'desc'   => "The number of PDUs sent by the LDAP server.",
+         'vlabel' => 'PDUs per ${graph_period}',
+         'label'  => 'PDUs',
+         'title'  => "Number of PDUs sent",
+         'info'   => "The graph shows the number of PDUs sent",
+	 'scope'  => "base"
+         },
+     # Referrals
+     'statistics_referrals' 
+     => {
+         'search' => "cn=Referrals,cn=Statistics",
+         'desc'   => "The number of Referrals sent by the LDAP server.",
+         'vlabel' => 'Referrals per ${graph_period}',
+         'label'  => 'Referrals',
+         'title'  => "Number of LDAP Referrals",
+         'info'   => "The graph shows the number of referrals sent",
+	 'scope'  => "base"
+         },
+     # Entries
+     'statistics_entries'
+     => {
+         'search' => "cn=Entries,cn=Statistics",
+         'desc'   => "The number of Entries sent by the LDAP server.",
+         'vlabel' => 'Entries per ${graph_period}',
+         'label'  => 'Entries',
+         'title'  => "Number of LDAP Entries",
+         'info'   => "The graph shows the number of entries sent",
+	 'scope'  => "base"
+         },
+     # Only read Total
+     'connections' 
+     => {
+         'search' => 'cn=Total,cn=Connections',
+         'desc'   => 'The number of connections',
+         'label'  => 'Connections',
+         'vlabel' => 'Connections per ${graph_period}',
+         'title'  => 'Number of Connections',
+         'info'   => 'Number of connections to the LDAP server',
+         'scope'  => "base"
+         },
+     # dn: cn=Write,cn=Waiters,cn=Monitor
+     # dn: cn=Read,cn=Waiters,cn=Monitor
+     'waiters' 
+     => {
+         'search' => 'cn=Waiters',
+         'filter' => '(|(cn=Write)(cn=Read))',
+         'desc'   => "The current number of Waiters",
+         'label2' => {'write' => 'Write',
+                      'read'  => 'Read'},
+         'vlabel' => "Waiters",
+         'title'  => "Number of Waiters",
+         'info'   => "The graph shows the number of Waiters"
+         },
+     'operations'
+     => {
+         'search' => "cn=Operations",
+         'desc'   => "Operations",
+         'vlabel' => 'Operations per ${graph_period}',
+         'label'  => 'Operations',
+         'title'  => "Operations",
+         'info'   => "Number of completed LDAP operations"
+         },
+     'operations_diff'
+     => {
+         'search' => "cn=Operations",
+         'desc'   => "Operations deviance",
+         'vlabel' => 'Deviance',
+         'label'  => 'Deviance',
+         'title'  => "Operations deviance",
+         'info'   => "Deviance between Initiated and Completed ops"
+       }
+     );
+
+# Config subroutine
+sub config {
+    my $action = shift;
+    print <<EOF;
+graph_args --base 1000 -l 0
+graph_scale no
+graph_vlabel $ops{$action}->{'vlabel'}
+graph_title $ops{$action}->{'title'}
+graph_category OpenLDAP
+graph_info $ops{$action}->{'info'}
+EOF
+    
+    if ($ops{$action}->{'label2'}) {
+        while (my ($key, $val) = each (%{$ops{$action}->{'label2'}})) {
+          my $name = $action . "_" . $key;
+          print "$name.label $val\n";
+          print "$name.type GAUGE\n";
+        }
+    } elsif ($action =~ /^operations(?:_diff)?$/) {
+        my $ldap = Net::LDAP->new ($server)
+            or die "Failed to connect to server $server: $@";
+        my $mesg;
+        if ($userdn ne '') {
+          $mesg = $ldap->bind ($userdn, password => $userpw)
+              or die "Failed to bind with $userdn: $@";
+        } else {
+          require Authen::SASL;
+          my $sasl = Authen::SASL::->new( mechanism => 'EXTERNAL' );
+          $mesg = $ldap->bind( undef, sasl => $sasl )
+              or die "Failed to SASL bind: $@";
+        }
+        if ($mesg->code) {
+          die "Failed to bind: " . $mesg->error;
+        }
+        my $searchdn = $ops{$action}->{'search'} . "," . $basedn;
+        $mesg =
+            $ldap->search (
+                           base   => $searchdn,
+                           scope  => 'one',
+                           filter => '(objectclass=*)',
+                           attrs  => ['monitorOpInitiated',
+                                      'monitorOpCompleted',
+                                      'cn'],
+                           );
+        $mesg->code && die $mesg->error;
+        
+        my $max = $mesg->count;
+        for (my $i = 0 ; $i < $max ; $i++) {
+            my $entry = $mesg->entry ($i);
+            my $cn = $entry->get_value ('cn');
+            my $name = $action . "_" . lc ($cn);
+            print "$name.label $cn\n";
+            print "$name.type DERIVE\n";
+            print "$name.min 0\n";
+            if ($action eq "operations") {
+                print "$name.info The number of $cn operations\n";
+            } else {
+                print "$name.info The difference between Initiated ";
+                print "and Completed operations (should be 0)\n";
+                print "$name.warning 1\n";
+            }            
+        }
+
+        $ldap->unbind;
+    } else {
+        print "$action.label $ops{$action}->{'label'}\n";
+        print "$action.type DERIVE\n";
+        print "$action.min 0\n";
+    }
+}
+
+# Determine action based on filename first
+(my $action = $0) =~ s/^.*slapd2_([\w\d_]+)$/$1/;
+
+if ($ARGV[0]) {
+    if ($ARGV[0] eq 'autoconf') {
+        # Check for Net::LDAP
+	if ($ret) {
+	    print "no ($ret)\n";
+	    exit 0;
+	}
+
+        # Check for LDAP version 3
+        my $ldap = Net::LDAP->new ($server, version => 3)
+            or do { print "no ($@)\n"; exit 0; };
+
+        my $mesg;
+        if ($userdn ne '') {
+          $mesg = $ldap->bind ($userdn, password => $userpw)
+            or do { print "no ($@)\n"; exit 0; };
+        } else {
+          require Authen::SASL;
+          my $sasl = Authen::SASL::->new( mechanism => 'EXTERNAL' );
+          $mesg = $ldap->bind( undef, sasl => $sasl )
+            or do { print "no ($@)\n"; exit 0; };
+        }
+        if ($mesg->code) {
+          print "no (" . $mesg->error . ")\n";
+          exit 0;
+        }
+
+        $mesg =
+            $ldap->search (
+                           base   => $basedn,
+                           scope  => 'one',
+                           filter => '(objectClass=monitorServer)',
+                           attrs  => [
+                                       'cn',
+                                     ],
+                           );
+        if ($mesg->code) {
+          print "no (" . $mesg->error . ")\n";
+          exit 0;
+        }
+        print "yes\n";
+        exit 0;
+    } elsif ($ARGV[0] eq "config") {
+        if(!exists $ops{$action}) {
+            die "Unknown action specified: $action";
+        }
+        &config ($action);
+    } elsif ($ARGV[0] eq "suggest") {
+        print join ("\n", keys (%ops)), "\n";
+    }
+    exit 0;
+}
+
+# If $action isn't in %ops, we quit
+if(!exists $ops{$action}) {
+    die "Unknown action specified: $action";
+}
+
+# Default scope for LDAP searches. We'll change to other scopes if
+# necessary.
+$scope = "one";
+
+# Net::LDAP variant
+my $ldap = Net::LDAP->new ($server, version => 3)
+    or die "Failed to connect to server $server: $@";
+my $mesg;
+if ($userdn ne '') {
+  $mesg = $ldap->bind ($userdn, password => $userpw)
+      or die "Failed to bind with $userdn: $@";
+} else {
+  require Authen::SASL;
+  my $sasl = Authen::SASL::->new( mechanism => 'EXTERNAL' );
+  $mesg = $ldap->bind( undef, sasl => $sasl )
+      or die "Failed to bind anonymously: $@";
+}
+if ($mesg->code) {
+  die "Failed to bind: " . $mesg->error;
+}
+
+my $searchdn = $ops{$action}->{'search'} . "," . $basedn;
+my $searchattrs;
+
+if ($action =~ /^operations(_diff)?$/) {
+    # We look for different parameters in Operations branch
+    $searchattrs = ['monitorOpInitiated', 'monitorOpCompleted', 'cn'];
+} else {
+    $searchattrs = ['monitorCounter', 'cn'];
+}
+
+my $filter;
+if ($ops{$action}->{'filter'}) {
+  $filter = "(&(objectclass=*)" . $ops{$action}->{'filter'} . ")";
+} else {
+  $filter = "(objectClass=*)";
+}
+
+if ($ops{$action}->{'scope'}) {
+  $scope = $ops{$action}->{'scope'};
+}
+
+$mesg =
+    $ldap->search (
+                   base   => $searchdn,
+                   scope  => $scope,
+                   filter => $filter,
+                   attrs  => $searchattrs,
+                   );
+
+$mesg->code && die $mesg->error;
+
+my $max = $mesg->count;
+
+for (my $i = 0 ; $i < $max ; $i++) {
+    my $entry = $mesg->entry ($i);
+    my $cn = $entry->get_value('cn');
+    if ($action =~ /operations(_diff)?$/) {
+	if ($1) {
+	    my $opsInit =
+		$entry->get_value('monitorOpInitiated'); 
+	    my $opsComp =
+		$entry->get_value('monitorOpCompleted');
+	    print lc ("operations_diff_${cn}.value ");
+	    print ($opsInit - $opsComp);
+	    print "\n";
+	} else {
+	    print lc ("operations_${cn}.value ");
+	    print $entry->get_value('monitorOpCompleted'),
+	    "\n";
+	}
+    } else {
+	# Hotfix, must do for now
+	if ($action =~ /_/ || $action eq 'connections') {
+	    print lc ("${action}.value ");
+	} else {
+	    print lc ("${action}_${cn}.value ");
+	}
+	print $entry->get_value('monitorCounter'), "\n";
+    }
+}
+$ldap->unbind;
diff --git a/roles/common/templates/etc/munin/plugin-conf.d/munin-node.j2 b/roles/common/templates/etc/munin/plugin-conf.d/munin-node.j2
index fa05327..e5ba9b5 100644
--- a/roles/common/templates/etc/munin/plugin-conf.d/munin-node.j2
+++ b/roles/common/templates/etc/munin/plugin-conf.d/munin-node.j2
@@ -118,20 +118,25 @@ env.leasefile /var/lib/dhcp3/dhcpd.leases
 env.configfile /etc/dhcp3/dhcpd.conf
 
 [jmx_*]
 env.ip 127.0.0.1
 env.port 5400
 
 [samba]
 user root
 
 [munin_stats]
 user munin
 group munin
 
 [postgres_*]
 user postgres
 env.PGUSER postgres
 env.PGPORT 5432
 
 [fail2ban]
 user root
+
+[slapd2_*]
+user munin
+group munin
+env.server ldapi://