dovecot: use Single-Instance Storage for mail attachments.

4 files changed, 24 insertions, 11 deletions

diff --git a/roles/IMAP/files/etc/cron.d/doveadm b/roles/IMAP/files/etc/cron.d/doveadm
new file mode 100644
index 0000000..1f1f139
--- /dev/null
+++ b/roles/IMAP/files/etc/cron.d/doveadm
@@ -0,0 +1,3 @@
+MAILTO=root
+
+7 5 * * *  vmail  test -x /usr/bin/doveadm && nice -n 19 /usr/bin/doveadm sis deduplicate /home/mail/attachments /home/mail/attachments/queue
diff --git a/roles/IMAP/files/etc/dovecot/conf.d/10-mail.conf b/roles/IMAP/files/etc/dovecot/conf.d/10-mail.conf
index ab5ceef..2e68df4 100644
--- a/roles/IMAP/files/etc/dovecot/conf.d/10-mail.conf
+++ b/roles/IMAP/files/etc/dovecot/conf.d/10-mail.conf
@@ -345,36 +345,36 @@ mailbox_list_index = yes
 #mdbox_rotate_size = 2M
 
 # Maximum dbox file age until it's rotated. Typically in days. Day begins
 # from midnight, so 1d = today, 2d = yesterday, etc. 0 = check disabled.
 #mdbox_rotate_interval = 0
 
 # When creating new mdbox files, immediately preallocate their size to
 # mdbox_rotate_size. This setting currently works only in Linux with some
 # filesystems (ext4, xfs).
 mdbox_preallocate_space = yes
 
 ##
 ## Mail attachments
 ##
 
 # sdbox and mdbox support saving mail attachments to external files, which
 # also allows single instance storage for them. Other backends don't support
 # this for now.
 
 # Directory root where to store mail attachments. Disabled, if empty.
-#mail_attachment_dir =
+mail_attachment_dir = /home/mail/attachments
 
 # Attachments smaller than this aren't saved externally. It's also possible to
 # write a plugin to disable saving specific attachments externally.
 #mail_attachment_min_size = 128k
 
 # Filesystem backend to use for saving attachments:
 #  posix : No SiS done by Dovecot (but this might help FS's own deduplication)
 #  sis posix : SiS with immediate byte-by-byte comparison during saving
 #  sis-queue posix : SiS with delayed comparison and deduplication
-#mail_attachment_fs = sis posix
+mail_attachment_fs = sis-queue /home/mail/attachments/queue:posix
 
 # Hash format to use in attachment filenames. You can add any text and
 # variables: %{md4}, %{md5}, %{sha1}, %{sha256}, %{sha512}, %{size}.
 # Variables can be truncated, e.g. %{sha256:80} returns only first 80 bits
-#mail_attachment_hash = %{sha1}
+mail_attachment_hash = %{sha256}
diff --git a/roles/IMAP/tasks/imap.yml b/roles/IMAP/tasks/imap.yml
index 51e7694..f4a23e4 100644
--- a/roles/IMAP/tasks/imap.yml
+++ b/roles/IMAP/tasks/imap.yml
@@ -19,74 +19,78 @@
 
 ## TODO: make a LDAP query listing all users using iterate_attrs and
 ## iterate_filter.  (Alternatively, use a dict, see
 ## https://www.opensource.apple.com/source/dovecot/dovecot-293/dovecot.Config/dovecot-dict-auth.conf.ext)
 ## Required for dbox, see
 ## http://wiki2.dovecot.org/MailboxFormat/dbox#Multi-dbox
 #- name: Create a nightly cron job to purge expunged messages
 #  cron: name="Purge expunged messages"
 #        minute=7 hour=5
 #        user=vmail cron_file=doveadm-purge
 #        job="/usr/bin/doveadm purge -A"
 
 # The ownership and permissions ensure that dovecot won't try to
 # deliver mails under an umounted mountpoint.
 - name: Create a home directory for user 'vmail'
   file: path=/home/mail
         state=directory
         owner=root group=root
         mode=0755
 
-- name: Create /home/mail/virtual
-  file: path=/home/mail/virtual
+- name: Create /home/mail/{virtual,attachments,spamspool}
+  file: path=/home/mail/{{ item }}
         state=directory
         owner=vmail group=vmail
         mode=0700
+  with_items:
+    - virtual
+    - attachments
+    - spamspool
+
+- name: Create a cronjob for SIS deduplication
+  copy: src=etc/cron.d/doveadm
+        dest=/etc/cron.d/doveadm
+        owner=root group=root
+        mode=0644
 
 - name: Create virtual mailbox directories
   file: path=/etc/dovecot/virtual/{{ item }}
         state=directory
         owner=root group=root
         mode=0755
   with_items:
     - all
     - flagged
     - recent
     - unseen
 
 - name: Create virtual mailboxes
   copy: src=etc/dovecot/virtual/{{ item }}/dovecot-virtual
         dest=/etc/dovecot/virtual/{{ item }}/dovecot-virtual
         owner=root group=root
         mode=0644
   with_items:
     - all
     - flagged
     - recent
     - unseen
 
-- name: Create directory /home/mail/spamspool
-  file: path=/home/mail/spamspool
-        state=directory
-        owner=vmail group=vmail
-        mode=0700
-
 - name: Create directory /etc/dovecot/ssl
   file: path=/etc/dovecot/ssl
         state=directory
         owner=root group=root
         mode=0755
 
 
 - name: Fetch Dovecot's X.509 certificate
   # Ensure we don't fetch private data
   become: False
   fetch_cmd: cmd="openssl x509 -noout -pubkey"
              stdin=/etc/dovecot/ssl/imap.fripost.org.pem
              dest=certs/public/imap.fripost.org.pub
   tags:
     - genkey
 
 - name: Configure Dovecot
   copy: src=etc/dovecot/{{ item }}
         dest=/etc/dovecot/{{ item }}
         owner=root group=root
diff --git a/roles/bacula-dir/templates/etc/bacula/bacula-dir.conf.j2 b/roles/bacula-dir/templates/etc/bacula/bacula-dir.conf.j2
index 046ba01..eb7566b 100644
--- a/roles/bacula-dir/templates/etc/bacula/bacula-dir.conf.j2
+++ b/roles/bacula-dir/templates/etc/bacula/bacula-dir.conf.j2
@@ -85,40 +85,41 @@ JobDefs {
   Priority = 20
   Write Bootstrap = "/var/lib/bacula/%n.bsr"
 }
 
 
 # Backup the director
 Job {
   Name = {{ inventory_hostname_short }}-dir
   Client = {{ inventory_hostname_short }}-fd
   JobDefs = DefaultJob
   FileSet = BaculaHome
   Schedule = WeeklyCycle
 }
 
 # Backup the mailboxes
 {% for h in groups.IMAP | sort %}
 Job {
   Name = {{ hostvars[h].inventory_hostname_short }}-mailboxes
   Client = {{ hostvars[h].inventory_hostname_short }}-fd
   JobDefs = DefaultJob
+  Accurate = no
   FileSet = Mailboxes
   Pool = mailboxes-inc
   Full Backup Pool = mailboxes-full
   Schedule = Mailboxes13WeeksCycle
   Max Start Delay = 50 min # To avoid too many overlaps
   Max Full Interval = 15 weeks
 }
 {% endfor %}
 
 # Backup each machine
 {% for fd in groups.all | sort %}
 Job {
   Name = {{ hostvars[fd].inventory_hostname_short }}
   Client = {{ hostvars[fd].inventory_hostname_short }}-fd
   JobDefs = DefaultJob
   FileSet = FileSetRoot
   Pool = {{ hostvars[fd].inventory_hostname_short }}
   Priority = 15
   Schedule = WeeklyCycle
 }
@@ -336,44 +337,49 @@ FileSet {
   Name = LDAP
   Include {
     Options {
       signature = SHA1
       compression = GZIP
       verify = s1
       WildFile = "*.ldif"
     }
     Options {
         Wild = "*"
         Exclude = yes
     }
     File = /var/lib/bacula/tmp
   }
 }
 
 FileSet {
   Name = Mailboxes
   Include {
     Options {
+      WildDir = /home/mail/attachments/queue
+      Exclude = yes
+    }
+    Options {
       signature = SHA1
       verify = pins1
     }
     File = /home/mail/virtual
+    File = /home/mail/attachments
     File = /home/mail/spamspool
   }
 }
 
 
 # Client (File Services) to backup
 {% for fd in groups.all | sort %}
 Client {
   Name = {{ hostvars[fd].inventory_hostname_short }}-fd
   Address = {{ ipsec[ hostvars[fd].inventory_hostname_short ] }}
   FDPort  = 9102
   Catalog = MyCatalog
   @|"sed -n '/^{{ hostvars[fd].inventory_hostname_short }}-fd\\s/ {s//Password = /p; q}' /etc/bacula/passwords-dir"
   File Retention = 4 months
   Job Retention = 5 months
   AutoPrune = yes
   #Maximum Bandwidth = 1mb/s
 }
 {% endfor %}