custom MailSystemInterface implementation

for mailsystem to work, you need to implement MailSystemInterface class and enable your new mailsystem class as default, for example in module hook_install():

    variable_set('mail_system', array_merge(
       variable_get('mail_system', array('default-system' => 'DefaultMailSystem')), // Previously set mail_system variable
       array('login' => 'MyCustomMailClass')   // My new key(s) which ADD to the previous keys
    );

Note: array('login' => 'MyCustomMailClass') "login" is a module name, which will be used to identify your new mailinterface.

Next you should call your new mailsystem from hook_mail(), like:

drupal_mail('login', 'registration', $form_state['values']['register_email'], language_default(), $params, '[email protected]', true);

And class implementation could look someting like:

class MyCustomMailClass implements MailSystemInterface {

/**
 * Concatenate and wrap the e-mail body for plain-text mails.
 *
 * @param $message
 *   A message array, as described in hook_mail_alter().
 *
 * @return
 *   The formatted $message.
 */
public function format(array $message) {
    $message['body'] = implode("\n\n", $message['body']);

    if ($message['module'] == 'login') {
        $body =  $message['body'];
        $message['body'] = array();

        // Convert any HTML to plain-text.
        $message['body']['plain'] = drupal_html_to_text($body);
        $message['body']['plain'] = drupal_wrap_mail($message['body']['plain']);

        // Wrap the mail body for sending.
        $message['body']['html'] = drupal_wrap_mail($body);
    } else {
        $message['body'] = drupal_wrap_mail($message['body']);
    }

    return $message;
}

/**
 * Send an e-mail message, using Drupal variables and default settings.
 *
 * @see <a href="http://php.net/manual/en/function.mail.php
  " title="http://php.net/manual/en/function.mail.php
  " rel="nofollow">http://php.net/manual/en/function.mail.php
  </a>   * @see drupal_mail()
 *
 * @param $message
 *   A message array, as described in hook_mail_alter().
 * @return
 *   TRUE if the mail was successfully accepted, otherwise FALSE.
 */
public function mail(array $message) {
    $mimeheaders = array();

    if ($message['module'] == 'login') {
        $separator = md5(time());
        // carriage return type (we use a PHP end of line constant)
        $eol = PHP_EOL;

        // main header
        $message['headers']['From'] = $message['from'];
        $message['headers']['Sender'] = $message['from'];
        $message['headers']['Return-Path'] = ASK_US_REPLY_TO_HEADER_MAIL;
        $message['headers']['Reply-to'] = $message['from'];
        $message['headers']['Errors-To'] = ASK_US_REPLY_TO_HEADER_MAIL;
        $message['headers']['Content-Type'] = "multipart/alternative; boundary=\"".$separator."\"";

        // message
        $body = "--".$separator.$eol;
        $body .= "Content-Type: text/plain; charset=\"UTF-8\"; format=\"flowed\"; delsp=\"yes\"".$eol;
        $body .= "Content-Transfer-Encoding: 8bit".$eol.$eol;
        $body .= $message['body']['plain'].$eol;

        // message
        $body .= "--".$separator.$eol;
        $body .= "Content-Type: text/html; charset=\"UTF-8\"; format=\"flowed\"; delsp=\"yes\"".$eol;
        $body .= "Content-Transfer-Encoding: 8bit".$eol.$eol;
        $body .= $message['body']['html'].$eol;
        $body .= "--".$separator."--";

        foreach ($message['headers'] as $name => $value) {
          $mimeheaders[] = $name . ': ' . mime_header_encode($value);
        }

        $line_endings = variable_get('mail_line_endings', MAIL_LINE_ENDINGS);
        $sendM = mail(
          $message['to'], mime_header_encode($message['subject']),
          preg_replace('@\r?\n@', $line_endings, $body),
          join("\n", $mimeheaders)
        );

        return $sendM;
    } else {
        $message['headers']['From'] = $message['from'];
        $message['headers']['Sender'] = $message['from'];
        $message['headers']['Return-Path'] = ASK_US_REPLY_TO_HEADER_MAIL;
        $message['headers']['Reply-to'] = $message['from'];
        $message['headers']['Errors-To'] = ASK_US_REPLY_TO_HEADER_MAIL;
        $message['headers']['Content-Type'] = 'text/html; charset=UTF-8; format=flowed; delsp=yes';

        if(isset($message['customMode']) && $message['customMode'] == 1) {
            $message['headers']['Content-Type'] = 'text/plain; charset=UTF-8; format=flowed; delsp=yes';
        }

        foreach ($message['headers'] as $name => $value) {
            $mimeheaders[] = $name . ': ' . mime_header_encode($value);
        }

        $line_endings = variable_get('mail_line_endings', MAIL_LINE_ENDINGS);
        $heads = join("\n", $mimeheaders);

        $sendM = mail(
          $message['to'], mime_header_encode($message['subject']),
          // Note: e-mail uses CRLF for line-endings. PHP's API requires LF
          // on Unix and CRLF on Windows. Drupal automatically guesses the
          // line-ending format appropriate for your system. If you need to
          // override this, adjust $conf['mail_line_endings'] in settings.php.
          preg_replace('@\r?\n@', $line_endings, $message['body']),
          // For headers, PHP's API suggests that we use CRLF normally,
          // but some MTAs incorrectly replace LF with CRLF. See #234403.
          $heads
        );

       return $sendM;
    }
}}