Fix catchall resolution.

It has to be performed last, to give a chance to be accepted as a
regular mailbox.

We introduce a new, dedicated, smtpd daemon whose only purpose is to
resolve catch-alls.

16 files changed, 88 insertions, 51 deletions

diff --git a/roles/IMAP/files/etc/postfix/virtual/mailbox_domains.cf b/roles/IMAP/files/etc/postfix/virtual/mailbox_domains.cf
index 54363e6..05f7ed9 120000
--- a/roles/IMAP/files/etc/postfix/virtual/mailbox_domains.cf
+++ b/roles/IMAP/files/etc/postfix/virtual/mailbox_domains.cf
@@ -1 +1 @@
-../../../../../MX/files/etc/postfix/virtual/mailbox_domains.cf
\ No newline at end of file
+../../../../../MX/templates/etc/postfix/virtual/mailbox_domains.cf.j2
\ No newline at end of file
diff --git a/roles/IMAP/files/etc/postfix/virtual/mailbox_maps.cf b/roles/IMAP/files/etc/postfix/virtual/mailbox_maps.cf
index 3a4d118..da1b2cf 120000..100644
--- a/roles/IMAP/files/etc/postfix/virtual/mailbox_maps.cf
+++ b/roles/IMAP/files/etc/postfix/virtual/mailbox_maps.cf
@@ -1 +1,8 @@
-../../../../../MX/files/etc/postfix/virtual/mailbox_maps.cf
\ No newline at end of file
+server_host      = ldapi://%2Fprivate%2Fldapi/
+version          = 3
+search_base      = fvl=%u,fvd=%d,ou=virtual,o=mailHosting,dc=fripost,dc=org
+scope            = base
+bind             = none
+query_filter     = (&(objectClass=FripostVirtualUser)(fvl=%u))
+result_attribute = fvl
+result_format    = OK
diff --git a/roles/MX/files/etc/postfix/virtual/reserved_maps.pcre b/roles/MX/files/etc/postfix/virtual/reserved_maps.pcre
deleted file mode 100644
index 58572d1..0000000
--- a/roles/MX/files/etc/postfix/virtual/reserved_maps.pcre
+++ /dev/null
@@ -1,5 +0,0 @@
-# These reserved aliases will always be redirected to us and the domain
-# owner.
-# TODO: check 'postmaster+test@fripost.org'
-/^(?:postmaster|abuse)(?:\+.*)?@fripost\.org$/ admin@fripost.org
-/^((?:postmaster|abuse)(?:\+.*)?)@/            $1
diff --git a/roles/MX/files/etc/postfix/virtual/reserved_transport_maps b/roles/MX/files/etc/postfix/virtual/reserved_transport_maps
deleted file mode 100644
index dce8710..0000000
--- a/roles/MX/files/etc/postfix/virtual/reserved_transport_maps
+++ /dev/null
@@ -1,2 +0,0 @@
-abuse           reserved-alias:
-postmaster      reserved-alias:
diff --git a/roles/MX/files/usr/local/sbin/reserved-alias.pl b/roles/MX/files/usr/local/sbin/reserved-alias.pl
index c122c6d..2c86020 100755
--- a/roles/MX/files/usr/local/sbin/reserved-alias.pl
+++ b/roles/MX/files/usr/local/sbin/reserved-alias.pl
@@ -3,108 +3,114 @@
 # Copyright © 2013 Guilhem Moulin <guilhem@fripost.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use warnings;
 use strict;
 use Net::LDAPI;
 use Net::LDAP::Util qw/escape_filter_value ldap_explode_dn escape_dn_value/;
 use Authen::SASL;
+use Net::SMTP;
 
 if (!@ARGV or grep { $_ eq '-h' or $_ eq '--help' } @ARGV) {
     # Help
-    print STDERR "Usage: $0 [original recipient] [additional recipient ...]\n";
+    print STDERR "Usage: $0 {original sender} {original recipient} [additional recipient ...]\n";
     print STDERR "\n";
     print STDERR "The message read from the standard input is redirected to 'additional recipient',\n";
     print STDERR "and also forwarded to the domain owner if any. If the 'additional recipient' begins\n";
     print STDERR "with '\@', the localpart of 'original recipient' is prepended.\n";
     print STDERR "\n";
     print STDERR "This is mostly useful to comply to RFC 822 section 6.3 and RFC 2142 section\n";
     print STDERR "4 (to forward mails to 'admin\@' and 'postmaster\@' to the site admin in\n";
     print STDERR "addition to the virtual domain manager).\n";
     exit;
 }
 
+# The original sender
+my $sender = shift;
+
 # The original recipient
 my $orig = shift;
 $orig =~ /^([^@]+)\@(.+)$/
-    or warn "Non fully qualified: $orig";
+    or warn "Warning: Non fully qualified: $orig";
 my ($local,$domain) = ($1,$2);
 
 # The new recipient (typically, the admin site)
 my @recipients = grep { $_ and $orig ne $_ }
                  # add localparts to domain
                  map { my $x = $_;
                        if ($x =~ /^\@/) {
                          if ($local) {
                             $x = $local.$x;
                          }
                          else {
                             undef $x;
                          }
                        }
                        $x
                      }
                  @ARGV;
 # Die if we can't deliver to site admins
 die "Error: Aborted delivery to '$orig' in attempt to break an alias expansion loop.\n"
     unless @recipients;
 
-my @sendmail = ('/usr/sbin/sendmail', '-i', '-bm');
-
 if (defined $domain) {
-    # Look for the domain owner/postmaster
+    # Look for the domain owner or postmaster
     my $ldap = Net::LDAPI->new();
     $ldap->bind( sasl => Authen::SASL->new(mechanism => 'EXTERNAL') )
-        or die "Couldn't bind";
+        or die "Error: Couldn't bind";
 
     my @attrs = ( 'fripostPostmaster', 'fripostOwner' );
     my $mesg = $ldap->search( base => 'fvd='.escape_dn_value($domain).','
                                      .'ou=virtual,o=mailHosting,dc=fripost,dc=org'
                             , scope => 'base'
                             , deref => 'never'
                             , filter => '(&(objectClass=FripostVirtualDomain)'
                                          .'(fvd='.escape_filter_value($domain).')'.
                                         ')'
                             , attrs => \@attrs
                             );
     if ($mesg->code) {
-        warn $mesg->error;
+        warn "Warning: ".$mesg->error;
     }
     elsif ($mesg->count != 1) {
         # Note: this may happen for "$mydestination", but these mails
         # are unlikely. We'll get a harmless warning at worst.
-        warn "Something weird happened when looking up domain '".$domain.
+        warn "Warning: Something weird happened when looking up domain '".$domain.
              "'. Check your ACL.";
     }
     else {
-        my $entry = $mesg->pop_entry() // die "Cannot pop entry.";
+        my $entry = $mesg->pop_entry() // die "Error: Cannot pop entry.";
         foreach (@attrs) {
             my $v = $entry->get_value($_, asref => 1) or next;
             foreach my $dn (@$v) {
                 my $dn2 = ldap_explode_dn($dn, casefold => 'lower');
                 my $l = $dn2->[0]->{fvl};
                 my $d = $dn2->[1]->{fvd};
                 if ($l and $d) {
                     push @recipients, $l.'@'.$d;
                 }
                 else {
-                    warn "Invalid DN: $dn"
+                    warn "Warning: Invalid DN: $dn"
                 }
             }
         }
     }
     $ldap->unbind;
 }
 
-exec (@sendmail, @recipients);
+my $smtp = Net::SMTP->new( 'localhost:25', Timeout => 1200 );
+$smtp->mail($sender);
+$smtp->to(@recipients, { Notify => ['FAILURE','DELAY'], SkipBad => 1 });
+$smtp->data(<STDIN>);
+$smtp->quit;
diff --git a/roles/MX/tasks/main.yml b/roles/MX/tasks/main.yml
index de6924b..2ffe08d 100644
--- a/roles/MX/tasks/main.yml
+++ b/roles/MX/tasks/main.yml
@@ -23,51 +23,52 @@
   service: name=postgrey state=started
   when: not r.changed
 
 - meta: flush_handlers
 
 - name: Configure Postfix
   template: src=etc/postfix/main.cf.j2
             dest=/etc/postfix-{{ postfix_instance[inst].name }}/main.cf
             owner=root group=root
             mode=0644
   register: r
   notify:
     - Restart Postfix
 
 - name: Create directory /etc/postfix-.../virtual
   file: path=/etc/postfix-{{ postfix_instance[inst].name }}/virtual
         state=directory
         owner=root group=root
         mode=0755
 
-- name: Copy lookups tables
-  copy: src=etc/postfix/virtual/{{ item }}
-        dest=/etc/postfix-{{ postfix_instance[inst].name }}/virtual/{{ item }}
-        owner=root group=root
-        mode=0644
+- name: Copy lookup tables
+  template: src=etc/postfix/virtual/{{ item }}.j2
+            dest=/etc/postfix-{{ postfix_instance[inst].name }}/virtual/{{ item }}
+            owner=root group=root
+            mode=0644
   with_items:
     - mailbox_domains.cf
-    - reserved_maps.pcre
+    - reserved_alias_maps
     - alias_maps.cf
-    - alias_catchall_maps.cf
-    - mailbox_maps.cf
+    - catchall_maps.cf
+    - transport_reserved_maps.pcre
+    - transport_mailbox_maps.cf
     - transport_lists_maps.cf
-    - reserved_transport_maps
+    - transport_catchall_maps.cf
 
 - name: Compile the Reserved Transport Maps
-  postmap: cmd=postalias instance={{ postfix_instance[inst].name }}
-           src=/etc/postfix-{{ postfix_instance[inst].name }}/virtual/reserved_transport_maps db=cdb
+  postmap: instance={{ postfix_instance[inst].name }}
+           src=/etc/postfix-{{ postfix_instance[inst].name }}/virtual/reserved_alias_maps db=cdb
            owner=root group=root
            mode=0644
 
 - name: Copy reserved-alias.pl
   copy: src=usr/local/sbin/reserved-alias.pl
         dest=/usr/local/sbin/reserved-alias.pl
         owner=root group=root
         mode=0755
 
 - name: Start Postfix
   service: name=postfix state=started
   when: not r.changed
 
 - meta: flush_handlers
diff --git a/roles/MX/templates/etc/postfix/main.cf.j2 b/roles/MX/templates/etc/postfix/main.cf.j2
index d301aaf..c0da5e7 100644
--- a/roles/MX/templates/etc/postfix/main.cf.j2
+++ b/roles/MX/templates/etc/postfix/main.cf.j2
@@ -2,91 +2,90 @@
 # MX configuration
 #
 # {{ ansible_managed }}
 # Do NOT edit this file directly!
 
 smtpd_banner     = $myhostname ESMTP $mail_name (Debian/GNU)
 biff             = no
 readme_directory = no
 mail_owner       = postfix
 
 delay_warning_time     = 4h
 maximal_queue_lifetime = 5d
 
 myorigin            = /etc/mailname
 myhostname          = mx{{ mxno | default('') }}.$mydomain
 mydomain            = {{ ansible_domain }}
 append_dot_mydomain = no
 
 # Turn off all TCP/IP listener ports except that necessary for the mail
 # exchange.
-master_service_disable = !smtp.inet inet
+master_service_disable = !smtp.inet !127.0.0.1:2599.inet inet
 
 queue_directory       = /var/spool/postfix-{{ postfix_instance[inst].name }}
 data_directory        = /var/lib/postfix-{{ postfix_instance[inst].name }}
 multi_instance_group  = {{ postfix_instance[inst].group | default('') }}
 multi_instance_name   = postfix-{{ postfix_instance[inst].name }}
 multi_instance_enable = yes
 
 # This server is a Mail eXchange
 mynetworks_style = host
 inet_interfaces  = all
 inet_protocols   = all
 
 # No local delivery
 mydestination        =
 local_transport      = error:5.1.1 Mailbox unavailable
 alias_maps           =
 alias_database       =
 local_recipient_maps =
 
 message_size_limit  = 67108864
 recipient_delimiter = +
 
 # Forward everything to our internal mailhub
 {% if 'MTA-out' in group_names %}
 relayhost     = [127.0.0.1]:{{ MTA_out.port }}
 {% else %}
 relayhost     = [{{ MTA_out.IPv4 }}]:{{ MTA_out.port }}
 {% endif %}
 relay_domains =
 
 # Virtual transport
 {% if 'LDA' in group_names %}
-virtual_transport = smtp:[127.0.0.1]:{{ LDA.port }}
+virtual_transport = smtpl:[127.0.0.1]:{{ LDA.port }}
 {% else %}
-virtual_transport = smtp:[{{ LDA.IPv4 }}]:{{ LDA.port }}
+virtual_transport = smtps:[{{ LDA.host }}]:{{ LDA.port }}
 {% endif %}
-transport_maps = ldap:$config_directory/virtual/transport_lists_maps.cf
+# It's a bit stupid to include part of the virtual_mailbox_maps here,
+# but we need to tell postfix to accept the recipient
+# (virtual_mailbox_maps) *before* sending away to the right machine
+# (transport_maps)
+transport_maps = pcre:$config_directory/virtual/transport_reserved_maps.pcre
+                 ldap:$config_directory/virtual/transport_mailbox_maps.cf
+                 ldap:$config_directory/virtual/transport_lists_maps.cf
+                 ldap:$config_directory/virtual/transport_catchall_maps.cf
 
 virtual_mailbox_domains = ldap:$config_directory/virtual/mailbox_domains.cf
-virtual_alias_maps      = pcre:$config_directory/virtual/reserved_maps.pcre
+virtual_alias_maps      = cdb:$config_directory/virtual/reserved_alias_maps
                           ldap:$config_directory/virtual/alias_maps.cf
-                          ldap:$config_directory/virtual/alias_catchall_maps.cf
-virtual_mailbox_maps    = ldap:$config_directory/virtual/mailbox_maps.cf
-                          # it's a bit stupid to lookup for lists here
-                          # and in transport, but we need to tell
-                          # postfix to accept the recipient
-                          # (virtual_mailbox_maps) *before* sending away
-                          # to the list server (transport_maps)
-                          ldap:$config_directory/virtual/transport_lists_maps.cf
-mailbox_transport_maps  = cdb:$config_directory/virtual/reserved_transport_maps
+virtual_mailbox_maps    = $transport_maps
 
 # Don't rewrite remote headers
 local_header_rewrite_clients     =
 # Pass the client information along to the content filter
 smtp_send_xforward_command       = yes
 # Avoid splitting the envelope and scanning messages multiple times
 smtp_destination_recipient_limit = 1000
 # Tolerate occasional high latency
 smtp_data_done_timeout           = 1200s
 
 # Tunnel everything through IPSec
 smtp_tls_security_level = none
 smtp_bind_address       = 172.16.0.1
 
 # TLS
 smtpd_tls_security_level        = may
 smtpd_tls_cert_file             = /etc/ssl/certs/ssl-cert-snakeoil.pem
 smtpd_tls_key_file              = /etc/ssl/private/ssl-cert-snakeoil.key
 smtpd_tls_CApath                = /etc/ssl/certs/
 smtpd_tls_session_cache_database= btree:$data_directory/smtpd_tls_session_cache
diff --git a/roles/MX/files/etc/postfix/virtual/alias_maps.cf b/roles/MX/templates/etc/postfix/virtual/alias_maps.cf.j2
index 8e3a778..8e3a778 100644
--- a/roles/MX/files/etc/postfix/virtual/alias_maps.cf
+++ b/roles/MX/templates/etc/postfix/virtual/alias_maps.cf.j2
diff --git a/roles/MX/files/etc/postfix/virtual/alias_catchall_maps.cf b/roles/MX/templates/etc/postfix/virtual/catchall_maps.cf.j2
index f8324f6..f8324f6 100644
--- a/roles/MX/files/etc/postfix/virtual/alias_catchall_maps.cf
+++ b/roles/MX/templates/etc/postfix/virtual/catchall_maps.cf.j2
diff --git a/roles/MX/files/etc/postfix/virtual/mailbox_domains.cf b/roles/MX/templates/etc/postfix/virtual/mailbox_domains.cf.j2
index 74304a4..74304a4 100644
--- a/roles/MX/files/etc/postfix/virtual/mailbox_domains.cf
+++ b/roles/MX/templates/etc/postfix/virtual/mailbox_domains.cf.j2
diff --git a/roles/MX/templates/etc/postfix/virtual/reserved_alias_maps.j2 b/roles/MX/templates/etc/postfix/virtual/reserved_alias_maps.j2
new file mode 100644
index 0000000..fe04715
--- /dev/null
+++ b/roles/MX/templates/etc/postfix/virtual/reserved_alias_maps.j2
@@ -0,0 +1,4 @@
+# RFC 822 section 6.3 and RFC 2142 section 4 mandatory aliases are
+# forwarded to the admin team.
+postmaster@fripost.org  admin@fripost.org
+abuse@fripost.org       admin@fripost.org
diff --git a/roles/MX/templates/etc/postfix/virtual/transport_catchall_maps.cf.j2 b/roles/MX/templates/etc/postfix/virtual/transport_catchall_maps.cf.j2
new file mode 100644
index 0000000..cc189cf
--- /dev/null
+++ b/roles/MX/templates/etc/postfix/virtual/transport_catchall_maps.cf.j2
@@ -0,0 +1,8 @@
+server_host      = ldapi://%2Fprivate%2Fldapi/
+version          = 3
+search_base      = fvd=%d,ou=virtual,o=mailHosting,dc=fripost,dc=org
+scope            = base
+bind             = none
+query_filter     = (&(objectClass=FripostVirtualDomain)(fvd=%d)(fripostOptionalMaildrop=*))
+result_attribute = fvd
+result_format    = smtpl:[127.0.0.1]:2599
diff --git a/roles/MX/files/etc/postfix/virtual/transport_lists_maps.cf b/roles/MX/templates/etc/postfix/virtual/transport_lists_maps.cf.j2
index 27c93d1..6a0965f 100644
--- a/roles/MX/files/etc/postfix/virtual/transport_lists_maps.cf
+++ b/roles/MX/templates/etc/postfix/virtual/transport_lists_maps.cf.j2
@@ -1,11 +1,12 @@
 server_host      = ldapi://%2Fprivate%2Fldapi/
 version          = 3
 search_base      = fvl=%u,fvd=%d,ou=virtual,o=mailHosting,dc=fripost,dc=org
 scope            = base
 bind             = none
 query_filter     = (&(|(objectClass=FripostVirtualList)(objectClass=FripostVirtualListCommand))(fvl=%u))
 result_attribute = fvl
-# We use these maps for both virtual mailboxes and transport (in the
-# former case the result is ignored, only the existence of a match
-# matters.)
-result_format    = smtp:[127.0.0.1]:2345
+{% if 'lists' in group_names %}
+result_format    = smtpl:[127.0.0.1]:{{ lists.port }}
+{% else %}
+result_format    = smtps:[{{ lists.host }}]:{{ lists.port }}
+{% endif %}
diff --git a/roles/MX/files/etc/postfix/virtual/mailbox_maps.cf b/roles/MX/templates/etc/postfix/virtual/transport_mailbox_maps.cf.j2
index da1b2cf..3e003db 100644
--- a/roles/MX/files/etc/postfix/virtual/mailbox_maps.cf
+++ b/roles/MX/templates/etc/postfix/virtual/transport_mailbox_maps.cf.j2
@@ -1,8 +1,12 @@
 server_host      = ldapi://%2Fprivate%2Fldapi/
 version          = 3
 search_base      = fvl=%u,fvd=%d,ou=virtual,o=mailHosting,dc=fripost,dc=org
 scope            = base
 bind             = none
 query_filter     = (&(objectClass=FripostVirtualUser)(fvl=%u))
 result_attribute = fvl
-result_format    = OK
+{% if 'LDA' in group_names %}
+result_format    = smtpl:[127.0.0.1]:{{ LDA.port }}
+{% else %}
+result_format    = smtps:[{{ LDA.host }}]:{{ LDA.port }}
+{% endif %}
diff --git a/roles/MX/templates/etc/postfix/virtual/transport_reserved_maps.pcre.j2 b/roles/MX/templates/etc/postfix/virtual/transport_reserved_maps.pcre.j2
new file mode 100644
index 0000000..e240e91
--- /dev/null
+++ b/roles/MX/templates/etc/postfix/virtual/transport_reserved_maps.pcre.j2
@@ -0,0 +1,6 @@
+if !/@fripost\.org$/
+# For other domains, RFC 822 section 6.3 and RFC 2142 section 4
+# mandatory aliases are forwarded to OUR admin team and to the domain
+# owner or postmaster, if there are any.
+/^(?:postmaster|abuse)(?:\+.*)?@/   reserved-alias:
+endif
diff --git a/roles/common/files/etc/postfix/master.cf b/roles/common/files/etc/postfix/master.cf
index 38b2ecb..17613b0 100644
--- a/roles/common/files/etc/postfix/master.cf
+++ b/roles/common/files/etc/postfix/master.cf
@@ -9,37 +9,45 @@
 #               (yes)   (yes)   (yes)   (never) (100)
 # ==========================================================================
 smtp      inet  n       -       -       -       -       smtpd
 16132     inet  n       -       -       -       -       smtpd
 2526      inet  n       -       -       -       -       smtpd
 2580      inet  n       -       -       -       -       smtpd
 submission inet n       -       -       -       -       smtpd
 pickup    fifo  n       -       -       60      1       pickup
 cleanup   unix  n       -       -       -       0       cleanup
 qmgr      fifo  n       -       n       300     1       qmgr
 tlsmgr    unix  -       -       -       1000?   1       tlsmgr
 rewrite   unix  -       -       -       -       -       trivial-rewrite
 bounce    unix  -       -       -       -       0       bounce
 defer     unix  -       -       -       -       0       bounce
 trace     unix  -       -       -       -       0       bounce
 verify    unix  -       -       -       -       1       verify
 flush     unix  n       -       -       1000?   0       flush
 proxymap  unix  -       -       n       -       -       proxymap
 proxywrite unix -       -       n       -       1       proxymap
 smtp      unix  -       -       -       -       -       smtp
+smtpl     unix  -       -       -       -       -       smtp
+  -o smtp_bind_address=127.0.0.1
+smtps     unix  -       -       -       -       -       smtp
+  -o smtp_bind_address=172.16.0.1
 relay     unix  -       -       -       -       -       smtp
 #       -o smtp_helo_timeout=5 -o smtp_connect_timeout=5
 showq     unix  n       -       -       -       -       showq
 error     unix  -       -       -       -       -       error
 retry     unix  -       -       -       -       -       error
 discard   unix  -       -       -       -       -       discard
 local     unix  -       n       n       -       -       local
 virtual   unix  -       n       n       -       -       virtual
 lmtp      unix  -       -       -       -       -       lmtp
 anvil     unix  -       -       -       -       1       anvil
 scache    unix  -       -       -       -       1       scache
+127.0.0.1:2599 inet n   -       -       -       -       smtpd
+  -o cleanup_service_name=cleanup-catchall
+cleanup-catchall unix n -       -       -       0       cleanup
+  -o virtual_alias_maps=cdb:$config_directory/virtual/reserved_alias_maps,ldap:$config_directory/virtual/alias_maps.cf,ldap:/etc/postfix-mx/virtual/catchall_maps.cf
 reserved-alias unix  -  n       n       -       -       pipe
-  flags=Rhu user=mail argv=/usr/local/sbin/reserved-alias.pl ${original_recipient} @fripost.org
+  flags=Rhu user=nobody argv=/usr/local/sbin/reserved-alias.pl ${sender} ${original_recipient} @fripost.org
 amavisfeed unix -       -       n       -       2       lmtp
   -o lmtp_destination_recipient_limit=1000
   -o lmtp_send_xforward_command=yes
   -o lmtp_data_done_timeout=1200s