How to install & configure Mailman.

1 files changed, 405 insertions, 2 deletions

diff --git a/fripost-docs.org b/fripost-docs.org
index 76a807a..00baf00 100644
--- a/fripost-docs.org
+++ b/fripost-docs.org
@@ -1393,8 +1393,8 @@ If everything goes through, it is now time to modify Postfix's main.cf:
 
  :: /etc/postfix/main.cf
     [...]
+    smtpd_sasl_auth_enable          = no
     smtpd_sasl_authenticated_header = yes
-    smtpd_sasl_auth_enable          = yes
     smtpd_sasl_local_domain         = fripost.org
     # TODO:add sasl exceptions for our other clients
     smtpd_sasl_exceptions_networks  = $mynetworks
@@ -1448,7 +1448,7 @@ What the user type is here emphasized and prefixed with a `*'
          Verify return code: 0 (ok)
      ---            
      250 DSN
-  *  EHLO localhost
+  *  EHLO localhost.localdomain
      [...]
      250-ETRN       
      250-AUTH LOGIN PLAIN
@@ -1599,6 +1599,8 @@ responsability to masquerade it I suppose.
     /^Received:\s+from\s+([._[:alnum:]-]+\s+\([._[:alnum:]-]+\s+\[[[:xdigit:].:]{3,39}\]\))(\s+\(using\s+(TLSv1|SSLv[23])\s+with\s+cipher\s+\S+\s+\([\/0-9]+\s+bits\)\)\s+).*(\(Authenticated sender:\s+[^)]+\)\s+).*(by\s+smtp\.fripost\.org\s+\([^)]+\)\s+with\s+E?SMTPS?A?\s+id\s+[[:xdigit:]]+.*)/
         REPLACE Received: from [127.0.0.1] (localhost [127.0.0.1])${2}${4}${5}
 
+    /^X-Originating-IP:/    IGNORE
+
 
   :: /etc/postfix/main.cf
   
@@ -1863,6 +1865,407 @@ mv hooks/post-update.sample hooks/post-update
 
 echo "Mötesprotokoll" > fripost-meetings.git/description
 
+** Configuring the list managers
+Right now, the list managers are hosted on our outgoing SMTP (and Mail
+Submission Agent), namely GNU. However, incoming email that is to be delivered
+to a list, as for regular email, is handled by the MX:s since we do not relay a
+whole domain for lists.
+
+*** Configuring the MTA on the MX:s
+Postfix does not support virtual transport out of the box. Virtual lists need
+to be forwarded to a local alias first (replacing the '@' by '#', hence '#' needs to
+be forbidden in list names), that can in turn be piped into a command
+or transported elswere.
+
+  :: /etc/postfix/main.cf
+     virtual_alias_maps = ..., ldap:$config_directory/ldap/virtual_alias_lists.cf
+     mailbox_transport_maps = ldap:$config_directory/ldap/transport_lists.cf
+
+  :: /etc/postfix/ldap/virtual_alias_maps.cf
+     test-list@fripost.org                   test-list#fripost.org
+     test-list-admin@fripost.org             test-list-admin#fripost.org
+     test-list-bounces@fripost.org           test-list-bounces#fripost.org
+     test-list-confirm@fripost.org           test-list-confirm#fripost.org
+     test-list-join@fripost.org              test-list-join#fripost.org
+     test-list-leave@fripost.org             test-list-leave#fripost.org
+     test-list-owner@fripost.org             test-list-owner#fripost.org
+     test-list-request@fripost.org           test-list-request#fripost.org
+     test-list-subscribe@fripost.org         test-list-subscribe#fripost.org
+     test-list-unsubscribe@fripost.org       test-list-unsubscribe#fripost.org
+
+     test-schleuder@fripost.org              test-schleuder#fripost.org
+     test-schleuder-bounces@fripost.org      test-schleuder-bounces#fripost.org
+     test-schleuder-sendkey@fripost.org      test-schleuder-sendkey#fripost.org
+     TODO: give the LDAP configuration
+
+  :: /etc/postfix/ldap/transport_lists.cf
+     test-list#fripost.org              smtp:[127.0.0.1]:2345
+     test-list-admin#fripost.org        smtp:[127.0.0.1]:2345
+     test-list-bounces#fripost.org      smtp:[127.0.0.1]:2345
+     test-list-confirm#fripost.org      smtp:[127.0.0.1]:2345
+     test-list-join#fripost.org         smtp:[127.0.0.1]:2345
+     test-list-leave#fripost.org        smtp:[127.0.0.1]:2345
+     test-list-owner#fripost.org        smtp:[127.0.0.1]:2345
+     test-list-request#fripost.org      smtp:[127.0.0.1]:2345
+     test-list-subscribe#fripost.org    smtp:[127.0.0.1]:2345
+     test-list-unsubscribe#fripost.org  smtp:[127.0.0.1]:2345
+
+     test-schleuder#fripost.org         smtp:[127.0.0.1]:2345
+     test-schleuder-bounces#fripost.org smtp:[127.0.0.1]:2345
+     test-schleuder-sendkey#fripost.org smtp:[127.0.0.1]:2345
+     TODO: give the LDAP configuration
+
+Note: in 'virtual_alias_maps', 'virtual_alias_lists.cf' should come before the
+catchalls to be effective.
+
+
+So every email that is to be delivered to a list manager is dropped into
+127.0.0.1:2345 using the SMTP protocol.
+
+*** Configuring the MTA on the machine hosting the list managers
+
+In the rest of this section, we assume there is a tunnel from each MX (port 2345)
+to the machine hosting the lists managers (port 2345).
+
+Since this machine is currently also hosting the outgoing SMTP and the Mail
+Submission Agent, we cannot the whole Postfix server to lists. Instead, we create a
+new Postfix instance for this purpose. (We need to because we need custom
+'virtual_alias_maps' that cannot be specified for a particular SMTP server only.)
+
+**** Installation
+
+sudo apt-get install postfix postfix-pcre postfix-cdb
+
+**** Creating a new postfix instance
+
+Reference: http://www.postfix.org/MULTI_INSTANCE_README.html
+
+  sudo postmulti -e init
+  sudo postmulti -I postfix-lists -G mta -e create
+  sudo ln -s ../postfix/dynamicmaps.cf /etc/postfix-lists/
+
+
+/etc/postfix/main.cf should be modified with
+
+  :: /etc/postfix/main.cf
+    ...
+    multi_instance_wrapper = ${command_directory}/postmulti -p --
+    multi_instance_enable = yes
+    multi_instance_directories = /etc/postfix-lists
+    ...
+
+  :: /etc/postfix-lists/main.cf
+    master_service_disable =
+    queue_directory = /var/spool/postfix-lists
+    mail_owner = postfix
+    multi_instance_group = mta      
+    multi_instance_name = postfix-lists
+    multi_instance_enable = yes     
+    
+    readme_directory = no
+    data_directory = /var/lib/postfix-lists
+         
+    smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
+    myorigin     = /etc/mailname    
+    myhostname   = lists.fripost.org
+    
+    mydestination = $myhostname
+    mynetworks    = 127.0.0.0/8
+    
+    default_database_type = cdb
+    
+    recipient_delimiter  = +        
+    alias_database       =          
+    alias_maps           = 
+    local_recipient_maps = $transport_maps
+    
+    virtual_mailbox_domains = pcre:$config_directory/virtual_domains.pcre
+    virtual_alias_maps      = pcre:$config_directory/virtual_aliases.pcre
+    virtual_mailbox_maps    =       
+    
+    virtual_transport = error:5.1.1 Virtual transport unavailable
+    default_transport = smtp:[127.0.0.1]
+    
+    relay_domains                         = $myhostname
+    transport_maps                        = cdb:$config_directory/transport_mailman
+                                            cdb:$config_directory/transport_schleuder
+    mailman_destination_recipient_limit   = 1
+    schleuder_destination_recipient_limit = 1
+
+  :: /etc/postfix-lists/master.cf
+    127.0.0.1:2345 inet  n  -       -       -       -       smtpd
+    ...
+    mailman   unix  -       n       n       -       -       pipe
+        flags=FR user=list:list argv=/usr/lib/mailman/bin/postfix-to-mailman.py 127.0.0.1 ${user}
+        # TODO: put ${nexthop} back (it's lists.fripost.org)
+    schleuder  unix  -       n       n       -       -       pipe
+        flags=FR user=schleuder:schleuder argv=/usr/local/bin/postfix-to-schleuder.sh ${user}
+
+(Don't forget to remove the other 'inet' services in the /etc/postfix-lists/master.cf)
+
+Note: you need to to append the configuration directory to Postfix commands to talk to this
+instance, for instance:
+- sudo postfix -c /etc/postfix-lists reload     # reload (without -c, it reloads both the slave and the master instances)
+- sudo postmap -c /etc/postfix-lists /etc/postfix-lists/transport_mailman   # postmap
+- sudo postfix -c /etc/postfix-lists flush   # flush the mail queue
+- mailq -C /etc/postfix-lists   # dump the mail queue
+- ...
+
+  :: /etc/postfix-lists/virtual_domains.pcre
+    # Accept all domains that are not our destination.
+    # (Only the MX's destinations are required, but...)
+    !/^lists\.fripost\.org$/    OK
+
+  :: /etc/postfix-lists/virtual_aliases.pcre
+    # Keep the local part, but replace the local part by our relay domain.
+    /^([^@]+)@/   ${1}@lists.fripost.org
+
+  :: /etc/postfix-lists/transport_mailman
+    test-mailman#fripost.org@lists.fripost.org                 mailman:
+    test-mailman-admin#fripost.org@lists.fripost.org           mailman:
+    test-mailman-bounces#fripost.org@lists.fripost.org         mailman:
+    test-mailman-confirm#fripost.org@lists.fripost.org         mailman:
+    test-mailman-join#fripost.org@lists.fripost.org            mailman:
+    test-mailman-leave#fripost.org@lists.fripost.org           mailman:
+    test-mailman-owner#fripost.org@lists.fripost.org           mailman:
+    test-mailman-request#fripost.org@lists.fripost.org         mailman:
+    test-mailman-subscribe#fripost.org@lists.fripost.org       mailman:
+    test-mailman-unsubscribe#fripost.org@lists.fripost.org     mailman:
+    
+  :: /etc/postfix-lists/transport_schleuder
+    test-schleuder#fripost.org@lists.fripost.org            schleuder:
+    test-schleuder-bounces#fripost.org@lists.fripost.org    schleuder:
+    test-schleuder-sendkey#fripost.org@lists.fripost.org    schleuder:
+
+Note: we could use LDAP lookups in transport as well, but it is not easy for
+list commands, and we have write access to the disk when adding a new list
+anyway. Also, searching in a CDB table is much more efficent.
+
+After modifying /etc/postfix-lists/transport_mailman, type
+
+  sudo postmap -c /etc/postfix-lists /etc/postfix-lists/transport_mailman
+
+to produce a CDB table. It is not necessary to reload Postfix after that, but
+you may have to wait one minute or two for Postfix to reload the file in memory.
+If you are in a hurry, type
+
+  sudo postfix -c /etc/postfix-lists reload
+
+to reload this instance only.
+
+
+Finally, we need a new set of rules for logcheck:
+
+  :: /etc/logcheck/ignore.d.server/postfix-lists
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/smtpd\[[[:digit:]]+\]: (dis)?connect from [^[:space:]]+$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/smtpd\[[[:digit:]]+\]: [[:alnum:]]+: client=[._[:alnum:]-]+\[[[:xdigit:].:]{3,39}\]$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/cleanup\[[[:digit:]]+\]: [[:alnum:]]+: (resent-|)message-id=<?[^>]+>?( \(added by [^[:space:]]+\))?$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/qmgr\[[[:digit:]]+\]: [[:alnum:]]+: from=<[^[:space:]]*>, size=[[:digit:]]+, nrcpt=[[:digit:]]+ \(queue active\)$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/pipe\[[[:digit:]]+\]: [[:upper:][:digit:]]+: to=<[^[:space:]]+>,( orig_to=<[^[:space:]]+>,)* relay=(mailman|schleuder), delay=[.[:digit:]]+(, delays=([.[:digit:]]+/){3}[.[:digit:]]+)?(, dsn=2(\.[[:digit:]]+){2})?, status=sent \(delivered via (mailman|schleuder) service\)$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/local\[[[:digit:]]+\]: [[:upper:][:digit:]]+: to=<[^[:space:]]+>,( orig_to=<[^[:space:]]+>,)? relay=local, delay=[[:digit:].]+(, delays=([.[:digit:]]+/){3}[.[:digit:]]+)?(, dsn=[45](\.[[:digit:]]+){2})?, status=(deferred|bounced) \(.+\)$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/n?qmgr\[[[:digit:]]+\]: [[:alnum:]]+: from=<.*>, status=expired, returned to sender$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/n?qmgr\[[[:digit:]]+\]: [[:alnum:]]+: message-id=(<?[^[:space:]]+>?)?( \(added by [^[:space:]]+\))?$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/n?qmgr\[[[:digit:]]+\]: [[:alnum:]]+: removed$
+    ^\w{3} [ :[:digit:]]{11} [._[:alnum:]-]+ postfix-lists/n?qmgr\[[[:digit:]]+\]: [[:alnum:]]+: skipped, still being delivered$
+
+(We could use the whole /etc/logcheck/ignore.d.server/postfix, but it's better to
+stick to the smallest rule set.)
+
+*** GNU Mailman
+
+**** Installation
+
+  :: sudo apt-get install mailman
+
+As of Debian 6.0 (Squeeze), saldy only mailman2 is available, and we need to apply
+third party patches for virtual domains to work. Hopefully GNU Mailman 3 will be
+available with Wheezy: it has native support for virtual domains, a LMTP server, a
+much nicer interface and design...
+
+References:
+- http://wiki.list.org/pages/viewpage.action?pageId=4030604
+- http://mail.python.org/pipermail/mailman-users/2010-January/068571.html
+- for Mailman 3: http://wiki.list.org/display/DEV/Mailman+3.0
+
+  
+  cd $HOME && wget http://www.msapiro.net/mm/2.1.13-1_vhost.patch
+  cd /var/lib/mailman
+  sudo patch -p1 < $HOME/2.1.13-1_vhost.patch
+
+Two hunks fail due to Debian specific patches, but it's merely line numbers that changed:
+1 out of 1 hunk FAILED -- saving rejects to file Mailman/Defaults.py.in.rej
+1 out of 1 hunk FAILED -- saving rejects to file Mailman/HTMLFormatter.py.rej
+
+In 'Defaults.py', the DEFAULT_MSG_FOOTER should be kept to
+  ...
+  %(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s
+
+as we want the a fully qualified list here. But we need to patch 'HTMLFormatter.py'.
+
+  sudo patch -p0 << EOF
+--- Mailman/HTMLFormatter.py
++++ Mailman/HTMLFormatter.py
+@@ -382,7 +382,8 @@
+         d = {
+             '<mm-mailman-footer>' : self.GetMailmanFooter(),
+             '<mm-list-name>' : self.real_name,
+-            '<mm-email-user>' : self._internal_name,
++            '<mm-email-user>' : self.local_part,
++            '<mm-complete-name>' : self.internal_name(),
+             '<mm-list-description>' : Utils.websafe(self.description),
+             '<mm-list-info>' : 
+                 '<!---->' + BR.join(self.info.split(NL)) + '<!---->',
+EOF
+
+  
+We need a last patch to keep fully qualified lists in URLs:
+
+  sudo patch -p0 << EOF
+--- Mailman/MailList.py
++++ Mailman/MailList.py
+@@ -253,7 +253,7 @@
+         # Using "local_part" here works for both site wide lists on
+         # the default url host and for vhost lists on the vhost url host.
+         return Utils.ScriptURL(scriptname, self.web_page_url, absolute) + \\ 
+-               '/' + self.local_part
++               '/' + self._internal_name
+ 
+     def GetOptionsURL(self, user, obscure=0, absolute=0):
+         url = self.GetScriptURL('options', absolute)
+EOF
+
+  sudo patch -p0 << EOF
+--- Mailman/Archiver/Archiver.py
++++ Mailman/Archiver/Archiver.py
+@@ -162,7 +162,7 @@
+             if hostname == mm_cfg.DEFAULT_URL_HOST:
+-                fullname = self.local_part
++                fullname = self._internal_name
+             else:
+-                fullname = os.path.join(hostname, self.local_part)
++                fullname = os.path.join(hostname, self._internal_name)
+             url = mm_cfg.PUBLIC_ARCHIVE_URL % {
+                 'listname': fullname,
+                 'hostname': hostname
+EOF
+
+  sudo patch -p0 << EOF
+--- bin/postfix-to-mailman.py
++++ bin/postfix-to-mailman.py
+@@ -111,6 +111,11 @@
+                              'mailman_destination_recipient_limit=1 '
+                              'in main.cf?')
+         sys.exit(EX_USAGE)
++    try:
++      l,d = local.split('#',2)
++      local = '%s@%s' % (l,d)
++    except ValueError:
++      l,d = local, None
+ 
+     # Redirect required addresses to 
+     if local in ('postmaster', 'abuse', 'mailer-daemon'):
+@@ -140,8 +145,9 @@
+                 '-subscribe',
+                 '-unsubscribe',
+                 ):
+-        if local.endswith(ext):
+-            mlist = local[:-len(ext)]
++        if l.endswith(ext):
++            mlist = l[:-len(ext)]
++            if d: mlist = '%s@%s' % (mlist,d)
+             func  = ext[1:]
+             break
+EOF
+
+  sudo find -L /var/lib/mailman -type f -a \( -name '*.orig' -o -name '*.rej' \) -delete
+
+**** Configuration
+
+  :: /etc/mailman/mm_cfg.py
+    DEFAULT_URL_PATTERN = 'http://%s/cgi-bin/mailman/'
+    PRIVATE_ARCHIVE_URL = '/cgi-bin/mailman/private'
+    IMAGE_LOGOS         = '/images/mailman/'
+    DEFAULT_EMAIL_HOST = 'lists.fripost.org'
+    DEFAULT_URL_HOST   = 'smtp.fripost.org' # TODO: change that to lists.fripost.org once the A record is changed
+    MTA = None
+    DEB_LISTMASTER = 'listmaster@lists.fripost.org'
+    ACCEPTABLE_LISTNAME_CHARACTERS = '[-+_.=a-z0-9@]'
+    PUBLIC_ARCHIVE_URL = 'http://%(hostname)s/pipermail/%(listname)s/'
+    DEFAULT_CHARSET = 'UTF-8'
+    add_language('en', 'English', 'utf-8')
+    add_language('sv', 'Swedish', 'utf-8')
+
+TODO: https; use a better URL scheme for the two list managers, perhaps something like
+https://lists.fripost.org/mailman/ and https://lists.fripost.org/schleuder/ .
+
+TODO: what URL format shall we choose (cf. DEFAULT_MSG_FOOTER)?
+%(real_name)s@%(host_name)s vs %(host_name)s/%(host_name)s
+(Right now it is the first choice.)
+
+
+A first list 'mailman' is required:
+
+  sudo -u list ./bin/newlist -q -u smtp.fripost.org mailman listmaster@fripost.org xxxxxxxxxxxxxxxx
+
+The daemon can now be started:
+
+  sudo /etc/init.d/mailman start
+
+
+To create a list:
+
+  sudo -u list ./bin/newlist -q -u smtp.fripost.org test-mailman@fripost.org user@fripost.org xxxxxxxxxxxxxxxx
+
+TODO: switch to '-u lists.fripost.org' when the DEFAULT_URL_HOST is updated.
+
+**** Web server configuration
+
+  sudo apt-get install apache2 libapache2-mod-python
+  ln -s ../mods-available/python.load /etc/apache2/mods-enabled/
+
+A template can be found in '/etc/mailman/apache.conf'.
+
+In our case the archives under /pipermail/ do not have the right forwat, a quick &
+dirty fix is to use a RewriteRule:
+
+    <Directory /var/lib/mailman/archives/public/>
+      RewriteEngine On
+      RewriteBase /
+      RewriteRule ^([^@]+)@([^/]+)/ /pipermail/$2/$1 [L]
+      ...
+    </Directory>
+ 
+TODO: Forbid access to '/create': it is not a proper way to create lists in our setting,
+since one needs to update the LDAP directory first.
+
+Note: when creating a new list with '-u lists.example.org', it is not visible under
+"http://smtp.fripost.org/cgi-bin/mailman/listinfo", but one can access it under
+"http://smtp.fripost.org/cgi-bin/mailman/listinfo/listname@lists.example.org". (TODO:
+check that). As usual the list owner can make the list invisible, though.
+
+*** Schleuder 
+
+**** Installation
+
+**** Patches
+
+**** Web server configuration
+
+*** Create a new list
+
+We need two small scripts to create new lists (one for GNU Mailman, the other
+for Schleuder). Postfix will pipe email into them as 'list' and 'schleuder' user
+respectively, hence the two files transport_mailman and transport_schleuder.
+
+These scripts should:
+- Ensure that the email is signed with the Admin WebPanel GPG key,
+- Create a new list, given for instance in the subject,
+- Append the new commands to transport_mailman or transport_schleuder,
+- Hash the transport file.
+
+In the case of Schleuder we also, create the web.conf file with the provided
+password.
+
 ** Logging
 *** Overview
 We want to limit how much we log for privacy reasons. At the same time we want