Decongestion potential bottlenecks on trivial_rewrite(8).

Which might be caused by slow LDAP lookups in transport_maps.  Instead,
we alias each addresses for which we want a custom transport to a
dedicated "dummy" domain, and use a static (CDB) transport_maps to map
said domains to their transport;  the receiver can then use canonical(8)
to restore the original envelope recipient.  Since the alias resolution
is performed by cleanup(8), which can run in parallel with other
instances, it should decongestion bottlenecks under heavy loads.

So far only the MX:es have been decongestioned.  The list manager and
the MDA should be treated as well.

10 files changed, 48 insertions, 33 deletions

diff --git a/roles/IMAP/files/etc/postfix/recipient_canonical.pcre b/roles/IMAP/files/etc/postfix/recipient_canonical.pcre
new file mode 100644
index 0000000..07c5859
--- /dev/null
+++ b/roles/IMAP/files/etc/postfix/recipient_canonical.pcre
@@ -0,0 +1,4 @@
+# Restore the original envelope recipient (drop our internal domain).
+# Extensions are preserved as they are included in $2.
+
+/^([^\/]+)\/(.+)@[^@]+$/    $2@$1
diff --git a/roles/IMAP/tasks/mda.yml b/roles/IMAP/tasks/mda.yml
index 1aac519..0358f12 100644
--- a/roles/IMAP/tasks/mda.yml
+++ b/roles/IMAP/tasks/mda.yml
@@ -29,6 +29,12 @@
     - mailbox.cf
     - transport_content_filter.cf
 
+- name: Copy recipient canonical
+  copy: src=etc/postfix/recipient_canonical.pcre
+        dest=/etc/postfix-{{ postfix_instance[inst].name }}/recipient_canonical.pcre
+        owner=root group=root
+        mode=0644
+
 - name: Start Postfix
   service: name=postfix state=started
   when: not r.changed
diff --git a/roles/IMAP/templates/etc/postfix/main.cf.j2 b/roles/IMAP/templates/etc/postfix/main.cf.j2
index e4c01bd..d0421ce 100644
--- a/roles/IMAP/templates/etc/postfix/main.cf.j2
+++ b/roles/IMAP/templates/etc/postfix/main.cf.j2
@@ -55,6 +55,11 @@ virtual_mailbox_domains = ldap:$config_directory/virtual/mailbox_domains.cf
 virtual_mailbox_maps    = ldap:$config_directory/virtual/mailbox.cf
 transport_maps          = ldap:$config_directory/virtual/transport_content_filter.cf
 
+# Restore the original envelope recipient
+relay_domains               = $myhostname
+recipient_canonical_classes = envelope_recipient
+recipient_canonical_maps    = pcre:$config_directory/recipient_canonical.pcre
+
 # Don't rewrite remote headers
 local_header_rewrite_clients =
 # Tolerate occasional high latency
diff --git a/roles/MX/tasks/main.yml b/roles/MX/tasks/main.yml
index e8dadb1..2670703 100644
--- a/roles/MX/tasks/main.yml
+++ b/roles/MX/tasks/main.yml
@@ -53,12 +53,11 @@
     - list.cf
     - alias_domains.cf
     - catchall.cf
-    - transport_reserved_alias
-    - transport_list.cf
+    - transport
 
 - name: Compile the Reserved Transport Maps
   postmap: instance={{ postfix_instance[inst].name }}
-           src=/etc/postfix-{{ postfix_instance[inst].name }}/virtual/transport_reserved_alias db=cdb
+           src=/etc/postfix-{{ postfix_instance[inst].name }}/virtual/transport db=cdb
            owner=root group=root
            mode=0644
 
diff --git a/roles/MX/templates/etc/postfix/main.cf.j2 b/roles/MX/templates/etc/postfix/main.cf.j2
index 6c2004a..8bed701 100644
--- a/roles/MX/templates/etc/postfix/main.cf.j2
+++ b/roles/MX/templates/etc/postfix/main.cf.j2
@@ -51,25 +51,22 @@ relayhost     = [{{ MTA_out.host }}]:{{ MTA_out.port }}
 relay_domains =
 
 # Virtual transport
-{% if 'LDA' in group_names %}
-virtual_transport = smtpl:[127.0.0.1]:{{ LDA.port }}
-{% else %}
-virtual_transport = smtps:[{{ LDA.host }}]:{{ LDA.port }}
-{% endif %}
-
+# We use a dedicated "virtual" domain to decongestion potential
+# bottlenecks on trivial_rewrite(8) due to slow LDAP lookups in
+# tranport_maps.
+virtual_transport = error:5.1.1 Virtual transport unavailable
 virtual_mailbox_domains = ldap:$config_directory/virtual/mailbox_domains.cf
 virtual_alias_maps      = pcre:$config_directory/virtual/reserved_alias.pcre
+                          # first we do the alias resolution...
                           ldap:$config_directory/virtual/alias.cf
-                          # stop the alias resolution (by making finding
-                          # an A -> A alias) before searching for
-                          # catch-alls and domain aliases
-                          $virtual_mailbox_maps
+                          # ...and unless there is matching mailbox/list...
+                          ldap:$config_directory/virtual/mailbox.cf
+                          ldap:$config_directory/virtual/list.cf
+                          # ...we resolve alias domains and catch alls
                           ldap:$config_directory/virtual/alias_domains.cf
                           ldap:$config_directory/virtual/catchall.cf
-virtual_mailbox_maps    = ldap:$config_directory/virtual/mailbox.cf
-                          ldap:$config_directory/virtual/list.cf
-transport_maps          = cdb:$config_directory/virtual/transport_reserved_alias
-                          ldap:$config_directory/virtual/transport_list.cf
+virtual_mailbox_maps    =
+transport_maps          = cdb:$config_directory/virtual/transport
 
 # Don't rewrite remote headers
 local_header_rewrite_clients     =
@@ -77,6 +74,7 @@ local_header_rewrite_clients     =
 smtp_send_xforward_command       = yes
 # Avoid splitting the envelope and scanning messages multiple times
 smtp_destination_recipient_limit = 1000
+reserved-alias_recipient_limit   = 1
 # Tolerate occasional high latency
 smtp_data_done_timeout           = 1200s
 
diff --git a/roles/MX/templates/etc/postfix/virtual/list.cf.j2 b/roles/MX/templates/etc/postfix/virtual/list.cf.j2
index 8bcd5df..6100c01 100644
--- a/roles/MX/templates/etc/postfix/virtual/list.cf.j2
+++ b/roles/MX/templates/etc/postfix/virtual/list.cf.j2
@@ -6,4 +6,6 @@ scope            = base
 bind             = none
 query_filter     = (&(objectClass=FripostVirtualList)(fvl=%u))
 result_attribute = fvl
-result_format    = %S
+# Use a dedicated "virtual" domain to decongestion potential bottlenecks
+# on trivial_rewrite(8) due to slow LDAP lookups in tranport_maps.
+result_format    = %D/%U@lists.guilhem.org
diff --git a/roles/MX/templates/etc/postfix/virtual/mailbox.cf.j2 b/roles/MX/templates/etc/postfix/virtual/mailbox.cf.j2
index b421e9a..fe27124 100644
--- a/roles/MX/templates/etc/postfix/virtual/mailbox.cf.j2
+++ b/roles/MX/templates/etc/postfix/virtual/mailbox.cf.j2
@@ -6,4 +6,6 @@ scope            = base
 bind             = none
 query_filter     = (&(objectClass=FripostVirtualUser)(fvl=%u))
 result_attribute = fvl
-result_format    = %S
+# Use a dedicated "virtual" domain to decongestion potential bottlenecks
+# on trivial_rewrite(8) due to slow LDAP lookups in tranport_maps.
+result_format    = %D/%U@mda.guilhem.org
diff --git a/roles/MX/templates/etc/postfix/virtual/transport.j2 b/roles/MX/templates/etc/postfix/virtual/transport.j2
new file mode 100644
index 0000000..6d244dc
--- /dev/null
+++ b/roles/MX/templates/etc/postfix/virtual/transport.j2
@@ -0,0 +1,13 @@
+reserved.locahost.localdomain   reserved-alias:
+
+{% if 'LDA' in group_names %}
+mda.guilhem.org smtpl:[127.0.0.1]:{{ LDA.port }}
+{% else %}
+mda.guilhem.org smtps:[{{ LDA.host }}]:{{ LDA.port }}
+{% endif %}
+
+{% if 'lists' in group_names %}
+lists.guilhem.org smtpl:[127.0.0.1]:{{ lists.port }}
+{% else %}
+lists.guilhem.org smtps:[{{ lists.host }}]:{{ lists.port }}
+{% endif %}
diff --git a/roles/MX/templates/etc/postfix/virtual/transport_list.cf.j2 b/roles/MX/templates/etc/postfix/virtual/transport_list.cf.j2
deleted file mode 100644
index eb696db..0000000
--- a/roles/MX/templates/etc/postfix/virtual/transport_list.cf.j2
+++ /dev/null
@@ -1,13 +0,0 @@
-server_host      = ldapi://%2Fprivate%2Fldapi/
-version          = 3
-search_base      = fvl=%u,fvd=%d,ou=virtual,o=mailHosting,dc=fripost,dc=org
-domain           = static:all
-scope            = base
-bind             = none
-query_filter     = (&(objectClass=FripostVirtualList)(fvl=%u))
-result_attribute = fvl
-{% if 'lists' in group_names %}
-result_format    = smtpl:[127.0.0.1]:{{ lists.port }}
-{% else %}
-result_format    = smtps:[{{ lists.host }}]:{{ lists.port }}
-{% endif %}
diff --git a/roles/MX/templates/etc/postfix/virtual/transport_reserved_alias.j2 b/roles/MX/templates/etc/postfix/virtual/transport_reserved_alias.j2
deleted file mode 100644
index 4af5318..0000000
--- a/roles/MX/templates/etc/postfix/virtual/transport_reserved_alias.j2
+++ /dev/null
@@ -1 +0,0 @@
-reserved.locahost.localdomain   reserved-alias: