1#@!#!123s

ï¿½ï¿½ï¿½ï¿½JFIFï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ï¿½ 1#@!#!123s
: / usr / share / web-monitoring-tool /
Filename : reporter.py
back
#!/opt/cloudlinux/venv/bin/python3 -bb
# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2020 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENCE.TXT
#

import json
import os
from datetime import date, timedelta, datetime
from email.header import Header
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import Dict, Any, List, Tuple

from clcommon import clemail
from clcommon.cpapi import get_admin_email as system_admin_email
from clcommon.mail_helper import MailHelper
from clcommon.utils import run_command
from prettytable import PrettyTable, ALL

from sentry import init_wmt_sentry_client, setup_logger

logger = setup_logger('email_reporter')

WMT_DIR = '/usr/share/web-monitoring-tool/'
WMT_API = os.path.join(WMT_DIR, 'wmtbin', 'wmt-api')


class WMTTemplateError(clemail.jinja2.exceptions.TemplateError):
    pass


def to_seconds(microseconds: int) -> str:
    secs = str(microseconds / 1000000.0)
    return secs[:4]


def _get_from_email():
    admin_email = system_admin_email()
    # system_admin_email may return multiple emails divided by commas,
    # example: 'admin1@email.com,admin2@email.com'
    # get the first one if multiple emails returned
    return admin_email.split(',')[0].strip()


def _get_to_email():
    response = run_command([WMT_API, '--email-get'])
    response = json.loads(response)
    if 'email' not in response or response['email'] == '':
        return system_admin_email()
    return response['email']


def _generate_summary_table(summary_data: Dict[str, Any]):
    """Generate summary report table"""
    table_columns = [('All', 'count_all'), ('Successful', 'count_successful'), ('Failed', 'count_failed'),
                     ('Undone', 'count_undone'), ('Average time (sec)', 'average_time')]
    summary_data['average_time'] = to_seconds(summary_data['average_time'])
    return _generate_table_data(table_columns, summary_data, True)


def _generate_error_table(error_reports: List[Dict[str, Any]]):
    """Generate error report table"""
    table_columns = [('URL', 'url'), ('Errors', 'count_errors'), ('Error codes', 'error_codes')]
    for i, report in enumerate(error_reports):
        error_reports[i]['error_codes'] = report['code'].split(',')
    return _generate_table_data(table_columns, error_reports)


def _generate_duration_table(duration_reports: List[Dict[str, Any]]):
    """Generate duration report table"""
    table_columns = [('URL', 'url'), ('Average time (sec)', 'average_time')]
    for i, report in enumerate(duration_reports):
        duration_reports[i]['average_time'] = to_seconds(report['average_time'])
    return _generate_table_data(table_columns, duration_reports)


def _generate_table_data(table_columns, table_data, one_row=False):
    def gen_table_body():
        table_b = list()
        if one_row:
            table_line = []
            for _, user_data_key in table_columns:
                cell = str(table_data.get(user_data_key, '---'))
                table_line.append(cell)
            table_b.append(table_line)
        else:
            for data in table_data:
                table_line = []
                for _, user_data_key in table_columns:
                    cell = data.get(user_data_key, '---')
                    if isinstance(cell, list):
                        cell = ', '.join(map(str, cell))
                    else:
                        cell = str(cell)
                    table_line.append(cell)
                table_b.append(table_line)
        return table_b

    columns = [header for header, _ in table_columns]
    table = PrettyTable(columns)
    table.horizontal_char = '='
    table.junction_char = "="
    list(map(table.add_row, gen_table_body()))
    s_table = table.get_string()
    s_html_table = table.get_html_string(format=True,
                                         border=True,
                                         hrules=ALL,
                                         vrules=ALL)
    return s_table, s_html_table


def _generate_template_data(report):
    summary_text_table, summary_html_table = _generate_summary_table(report['summary_report'])
    error_text_table, error_html_table = _generate_error_table(report['error_report'])
    duration_text_table, duration_html_table = _generate_duration_table(report['duration_report'])
    template_data = {
        'SUMMARY_REPORT': summary_text_table,
        'SUMMARY_HTML_REPORT': summary_html_table,
        'ERROR_REPORT': error_text_table,
        'ERROR_HTML_REPORT': error_html_table,
        'DURATION_REPORT': duration_text_table,
        'DURATION_HTML_REPORT': duration_html_table,
        'FROMMAIL': _get_from_email(),
        'LOCALE': 'en_US',
        'TOMAIL': _get_to_email(),
        'DATE': (date.today() - timedelta(days=1)).strftime("%b %d %Y"),
        'TONAME': 'Administrator'}
    return template_data


def generate_msg_body(templ_data):
    text_templ_path = os.path.join(WMT_DIR, 'templates', 'wmt_notify.txt')
    if not os.path.exists(WMT_DIR) or not os.path.exists(text_templ_path):
        logger.info(
            "Unable to find templates: file '%s' does not exist. ",
            text_templ_path)

    html_templ_path = os.path.join(WMT_DIR, 'templates', 'wmt_notify.html')
    if not os.path.exists(html_templ_path):
        logger.info(
            "Unable to find optional HTML message template '%s'. ",
            html_templ_path)
        html_templ_path = None

    html_body = None
    try:
        yesterday = datetime.now() - timedelta(1)
        yesterday = datetime.strftime(yesterday, '%Y-%m-%d')
        subject, text_body = clemail.ClEmail.generate_mail_jinja2(
            text_templ_path, templ_data=templ_data, subject=f'WMT report - {yesterday}')

        if html_templ_path:
            _, html_body = clemail.ClEmail.generate_mail_jinja2(
                html_templ_path, templ_data=templ_data, subject=f'WMT report - {yesterday}')

    except (clemail.jinja2.exceptions.TemplateError, IOError) as e:
        raise WMTTemplateError(
            'Can not generate message; no templates in "{}". '
            'Jinja2: {}'.format(
                WMT_DIR, str(e)))
    return subject, text_body, html_body


def generate_msg(templ_data):
    subject, text_body, html_body = generate_msg_body(templ_data)

    text_body = text_body \
        .encode('utf-8', 'xmlcharrefreplace') \
        .decode('utf-8')
    html_body = html_body \
        .encode('utf-8', 'xmlcharrefreplace') \
        .decode('utf-8')

    # Attach parts into message container.
    # According to RFC 2046, the last part of a multipart message,
    # in this case
    # the HTML message, is best and preferred.
    msg = MIMEMultipart('alternative')
    msg.attach(MIMEText(text_body, 'plain', 'utf-8'))
    msg.attach(MIMEText(html_body, 'html', 'utf-8'))
    # configure message headers
    msg['Subject'] = Header(subject, 'utf-8').encode()
    msg['From'] = templ_data.get('FROMMAIL')
    msg['To'] = templ_data['TOMAIL']
    return msg


def patched_run_command(cmd_list: List) -> Tuple[int, str, str]:
    """
    use logger to log error and additional info if command executed unsuccessfully,
    otherwise works as run_command(cmd_list)
    """
    ret_code, out, err = run_command(cmd_list, return_full_output=True)
    if ret_code != 0:
        cmd = ' '.join(cmd_list)
        msg = f'Error during command: {cmd}'
        logger.error(msg, extra={'output': out})
    return ret_code, out, err


def main():
    init_wmt_sentry_client()
    try:
        patched_run_command([WMT_API, '--report-generate'])
        ret_code, response, _ = patched_run_command([WMT_API, '--report-get'])
        if ret_code != 0:
            logger.info('Can\'t create email: report is bad or corrupted\nExiting')
            return
        mailhelper = MailHelper()
        _response = response.encode('us-ascii', 'xmlcharrefreplace').decode('us-ascii')
        response = json.loads(_response)
        if 'report' not in response:
            # check that we have correct response from wmt-api --report-get
            # that contains 'report' field
            raise Exception('Key "report" is not found in JSON report')
        template_data = _generate_template_data(response['report'])
        msg = generate_msg(template_data)
        to_addrs = [i.strip() for i in msg['To'].split(',')]
        mailhelper.sendmail(msg['From'], to_addrs, msg)
    except WMTTemplateError as e:
        logger.warning(e)
    except Exception as e:
        logger.exception(e)


if __name__ == "__main__":
    main()