import datetime
import json
import logging
import os
import re
import sys
import time
from collections import OrderedDict
from multiprocessing import Process, Value
from typing import List, Dict, Type, Union

import requests

sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
from examples.CustomParser import PARSER
from examples.config import (
    BOT_TOKEN,
    PROXY,
    MONGO_PORT,
    MONGO_IP,
    MONGO_PASS,
    MONGO_USER,
    CHANNEL_REPORT,
    OWNER,
)
from telegramnews.database import Database, MongoDatabase
from telegramnews.parser import NewsParser
from telegramnews.telegrambot import Telegram


def start_many(
    channels: List[Dict[str, any]],
    custom_parser: Dict[int, Type[NewsParser]],
    token: str,
    filter: int = 0,
    owner: int = 0,
    debug: bool = False,
    proxy: str = None,
    mongo: Dict[str, Union[str, int]] = None,
    error: int = None,
):
    """

    :param channels:
    :param custom_parser: Dictionary with inherited classes of NewsParser
    :param token: Telegram bot token
    :param filter: Can filter
    :param owner:
    :param debug:
    :param proxy:
    :param mongo:
    :param error:
    """
    processes = []
    kwargs = {
        "owner": owner,
        "debug": debug,
        "proxy": proxy,
        "mongo": mongo,
        "error": error,
    }
    for idx, site in enumerate(channels):
        processes.append(
            run_process(
                site, filter if filter else idx + 1, custom_parser, token, kwargs
            )
        )
    start_watchdog(processes, token, kwargs)


def run_process(site, idx, custom_parsers, token, kwargs):
    counter = Value("i", 0)
    sleeper = Value("i", 300)
    args = (site, counter, sleeper, token)
    if idx not in custom_parsers:
        p = Process(target=NewsParser, args=args, kwargs=kwargs)
    else:
        p = Process(target=custom_parsers[idx], args=args, kwargs=kwargs)
    if "channels" not in site and not kwargs["debug"]:
        return
    p.start()
    return {
        "process": p,
        "counter": counter,
        "sleeper": sleeper,
        "warning": 0,
        "old_counter": 0,
        "site": site,
        "id": idx,
    }


def start_watchdog(processes, token, kwargs):
    time.sleep(300)
    i = 0
    while True:
        time.sleep(31)
        i += 1
        check_process(processes, PARSER, i, token, kwargs)


def check_process(processes, custom_parsers, iteration, token, kwargs):
    for idx, p in enumerate(processes):
        # Skip if poll interval high and process still sleeping
        if 30 / p["sleeper"].value * iteration % 1:
            continue
        if p["counter"].value > p["old_counter"]:
            p["old_counter"] = p["counter"].value
            p["warning"] = 0
        else:
            logging.info(p)
            p["warning"] += 1
            if p["warning"] > 3:
                p["process"].terminate()
                if p["warning"] > 5:
                    p["process"].kill()
                processes.append(
                    run_process(p["site"], p["id"], custom_parsers, token, kwargs)
                )
                logging.error("Process restarted: " + p["site"]["name"])
                del processes[idx]


def get_channels(path, filter=0, all=False):
    with open("{}/{}".format(os.path.dirname(sys.argv[0]), path)) as json_file:
        channels = list(json.load(json_file).values())
    if filter:
        return [channels[filter]]
    return channels if all else channels[1:]


def set_channels(channels, path, not_all=True):
    if not_all:
        channels = get_channels(path, all=True)[0] + channels
    with open(
        "{}/{}".format(os.path.dirname(sys.argv[0]), path), "w", newline="\n"
    ) as outfile:
        json.dump(
            {
                "{:03.0f}".format(idx + 1): channel
                for idx, channel in enumerate(channels)
            },
            outfile,
            sort_keys=True,
            indent=4,
        )


def crawl_members(url):
    time.sleep(0.2)
    private = True
    if not url[:4] == "http":
        url = "https://t.me/" + url
        private = False
    try:
        if PROXY:
            html = requests.get(url, proxies={"https": PROXY}).text
        else:
            html = requests.get(url, proxies={"https": PROXY}).text
        members_str = re.findall('tgme_page_extra">([0-9]+) members</div>', html)[0]
        members = int(members_str) - 2
        return {"current_members": members, "private": private}
    except IndexError:
        logging.info(url)
        return None


def count_all_subchannels(channels, serious):
    db = MongoDatabase(
        {"port": MONGO_PORT, "ip": MONGO_IP, "pass": MONGO_PASS, "user": MONGO_USER}
    )
    result = {}
    for channel in channels:
        channel_infos = crawl_members(channel["link"])
        if channel_infos is None:
            continue
        channel_infos["old_members"] = db.get_members(channel["id"])
        if serious:
            db.set_members(channel_infos["current_members"], channel["id"])
        result[channel["id"]] = channel_infos
    return result


def get_counted_list(serious):
    channels = {"sites": {}}
    for site in get_channels("channels.json", all=True):
        sub_channels = count_all_subchannels(site["channels"], serious)
        absolut_members = sum(
            channel["current_members"] for channel in sub_channels.values()
        )
        absolut_old_members = sum(
            channel["old_members"] for channel in sub_channels.values()
        )
        all_private = all(channel["private"] for channel in sub_channels.values())
        channels["sites"][site["name"]] = {
            "channels": sub_channels,
            "absolut_members": absolut_members,
            "absolut_old_members": absolut_old_members,
            "all_private": all_private,
        }
    channels["absolut_members"] = sum(
        channel["absolut_members"] for channel in channels["sites"].values()
    )
    channels["absolut_old_members"] = sum(
        channel["absolut_old_members"] for channel in channels["sites"].values()
    )
    return channels


def render_text(channels):
    sorted_channel_list = OrderedDict(
        sorted(
            (channels["sites"]).items(),
            key=lambda kv: kv[1]["absolut_members"],
            reverse=True,
        )
    )
    l = str(len(str(max([x["absolut_members"] for x in sorted_channel_list.values()]))))
    result = ["" for x in range((int(len(sorted_channel_list) / 98) + 1))]
    result[0] = "*Tagesbericht DerNewsChannel*\n\n"
    for idx, (name, channel) in enumerate(sorted_channel_list.items()):
        diff = calc_diff(channel["absolut_members"], channel["absolut_old_members"])
        result[int((idx + 1) / 98)] += "`{{:2}}. {{}} {{:{}}} {{:2}} {{}}`\n".format(
            l
        ).format(
            idx + 1,
            "p" if channel["all_private"] else " ",
            channel["absolut_members"],
            diff,
            name[:15],
        )
    complete_diff = calc_diff(
        channels["absolut_members"], channels["absolut_old_members"]
    )
    result[-1] += "\nGesamt: *{}* {}".format(
        channels["absolut_members"],
        "({})".format(complete_diff) if complete_diff else "",
    )
    return result


def send_stats(serious=False):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    text = render_text(get_counted_list(serious))
    for t in text:
        tg.send(t, CHANNEL_REPORT, {}, disable_notification=True)


def calc_diff(new, old):
    diff = ""
    idiff = new - old
    if idiff > 0:
        diff = "+{}".format(idiff)
    elif idiff < 0:
        diff = "-{}".format(idiff * -1)
    return diff


def short_list(md=False):
    result = (
        "Perferkt formatierte News auf Telegram mit [@DerNewsChannel](https://t.me/dernewschannel)!\n\nFolgende "
        "Kanäle werden zurzeit angeboten:\n"
    )
    for site in sorted(get_channels("channels.json"), key=lambda k: k["name"].upper()):
        for channel in sorted(
            site["channels"], key=lambda k: k["template"], reverse=True
        ):
            if channel["template"] in [3, 4, 6]:
                if not channel["link"][:4] == "http":
                    if not md:
                        result += "@" + channel["link"].replace("_", "\_")
                    else:
                        result += "* [{}](https://t.me/{})".format(
                            site["name"], channel["link"]
                        )
                else:
                    if md:
                        result += "* "
                    result += "[{}]({})".format(
                        site["name"].replace("_", "\_"), channel["link"]
                    )
            elif channel["template"] == 5:
                result += " ([Instant ⚡]({}))️".format(channel["link"])
        result += "\n"
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    print(result.replace("\_", "_"))
    if not md:
        tg.send(result, OWNER, [])


def send_summary(serious=False, full=False, fast=False):
    db = Database()
    articles = [
        {"name": a[2].replace(".", "․"), "count": a[1], "date": a[0], "added": a[3]}
        for a in db.count_articles()
    ]
    site_list = {}
    date_list = {}
    results = {}
    for a in articles:
        if a["name"] not in site_list:
            site_list[a["name"]] = []
            date_list[a["name"]] = a["added"]
        site_list[a["name"]].append(
            {
                "count": a["count"],
                "date": datetime.datetime.strptime(a["date"], "%Y-%m-%d"),
            }
        )
    for site, data in sorted(site_list.items(), key=lambda s: s[0].lower()):
        try:
            channels = next(
                (
                    item
                    for item in get_channels("channels.json")
                    if item["name"] == site.replace("․", ".")
                ),
                None,
            )
        except TypeError:
            continue
        if not channels:
            continue
        mini = min([x["date"] for x in data])
        days = abs((datetime.datetime.today() - mini).days)
        counter = 0
        for x in data:
            counter += x["count"]
        if channels["category"] not in results:
            results[channels["category"]] = "*{}*\n".format(
                channels["category"].upper()
            )
        if (
            datetime.datetime.today()
            - datetime.datetime.strptime(date_list[site], "%Y-%m-%d")
        ).days > 13:
            results[channels["category"]] += "\n*{}* (Ø{} M/Tag)\n".format(
                site, "{:.1f}".format(counter / days).replace(".", ",")
            )
        else:
            results[channels["category"]] += "\n*{} 🆕* \n".format(site)
        try:
            for channel in channels["channels"]:
                try:
                    members = ""  # "[[{}👥]]".format(db.get_members(channel['id'])[0])
                except IndexError:
                    members = ""
                if not channel["link"][:4] == "http":
                    results[channels["category"]] += "@{} {}\n".format(
                        channel["link"].replace("_", "\_"), members
                    )
                elif channel["template"] == 5:
                    results[
                        channels["category"]
                    ] += "[⚡️ Instant View]({}) {}\n".format(channel["link"], members)
                else:
                    results[channels["category"]] += "[{}]({}) {}\n".format(
                        site, channel["link"], members
                    )
        except TypeError:
            continue
    other = "*SONSTIGES*\n\n"
    tg = Telegram()
    for category, text in sorted(results.items()):
        if text.count("\n") < 6:
            other += text + "\n\n"
            results.pop(category, "")
    results["Sonstiges"] = other
    results["Info"] = INFO
    ORDER = ["News", "Technik"]
    i = 0
    channel_id = CHANNEL_TEST if not serious else CHANNEL_MAIN
    send = (
        ["Info", "Sonstiges"]
        + [c for c in results if c not in ORDER and not c in ["Sonstiges", "Info"]]
        + ORDER[::-1]
    )
    if not full:
        for category in send:
            i = tg.edit(
                i,
                results[category],
                channel_id,
                {},
                disable_notification=True,
                disable_preview=True,
            )
    else:
        tg.unpin(channel_id)
        if not fast:
            time.sleep(300)
        first = tg.send(
            results[send[0]],
            channel_id,
            {},
            disable_notification=True,
            disable_preview=True,
        )
        for i in reversed(range(first)):
            tg.delete(i, channel_id)
        tg.pin(channel_id, first)
        time.sleep(5)
        for category in send[1:]:
            tg.send(
                results[category],
                channel_id,
                {},
                disable_notification=True,
                disable_preview=True,
            )
            time.sleep(1)


def clean_up_db():
    db = Database()
    db.clean_up()


def migrate_news(mongo, news):
    logging.info("Migrate %s news", len(news))
    for idx, d in enumerate(news):
        try:
            if not int(idx % (len(news) / 100)):
                logging.info("Migrate %s%%", int(100 / len(news) * idx) + 1)
            id, s_id, title, text, link, img, date, tags, m_id, site_name = d
            if mongo.quick_check_news(link, link, site_name):
                mongo.check_news(title, text, link, site_name)
                continue
            x = mongo.insert_news(
                title, text, link, img, site_name, date=date, tags=tags
            )
            mongo.update_message_id(x, m_id)
        except Exception as e:
            logging.error(d)
            logging.exception(e)


def migrate_channels(mongo, channels):
    logging.info("Migrate %s channels", len(channels))
    for c in channels:
        c_id, s_id, t_id, link, short, public, pinned, template, added, members, name = (
            c
        )
        mongo.insert_channel(
            name,
            t_id,
            link,
            template,
            public,
            short,
            added=added,
            members=members,
            pinned=pinned,
        )


def migrate_sites(mongo, sites):
    logging.info("Migrate %s sites", len(sites))
    for s in sites:
        s_id, name, rss, added = s
        mongo.insert_site(name, rss, added=added)


def migrate_to_mongo():
    db = Database()
    mongo = MongoDatabase()
    sites, channels, news = db.get_all()
    print(len(sites), len(channels), len(news))
    migrate_sites(mongo, sites)
    migrate_channels(mongo, channels)
    migrate_news(mongo, news)


def update_channel_names(path="channels.json"):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    sites = get_channels(path)
    for key, site in enumerate(sites):
        if "channels" not in site:
            continue
        for idx, channel in enumerate(site["channels"]):
            time.sleep(1)
            try:
                channel_link = tg.get_chat_infos(channel["id"])
            except:
                logging.warning("Timed out " + site["name"])
                channel_link = channel["link"]
            if (
                not (channel_link[:4] == "http" and channel["link"][:4] == "http")
                and not channel_link == channel["link"]
            ):
                print(channel_link, channel["link"])
                sites[key]["channels"][idx]["link"] = channel_link
    set_channels(sites, path)


def init():
    db = MongoDatabase(
        {"port": MONGO_PORT, "ip": MONGO_IP, "pass": MONGO_PASS, "user": MONGO_USER}
    )
    main_channel = get_channels("channels.json", all=True)[0]
    db.insert_main_channel(
        main_channel["channels"][0]["id"], main_channel["channels"][0]["link"], True, 0
    )


def start_tool(arg):
    if arg == "--stats":
        send_stats()
    elif arg == "--stats-live":
        send_stats(serious=True)
    elif arg == "--mstats":
        send_summary()
    elif arg == "--mstats-live":
        send_summary(serious=True)
    elif arg == "--mstats-full":
        send_summary(full=True)
    elif arg == "--mstats-fast":
        send_summary(fast=True, full=True)
    elif arg == "--mstats-live-full":
        send_summary(serious=True, full=True)
    elif arg == "--clean":
        clean_up_db()
    elif arg == "--short-list":
        short_list()
    elif arg == "--migrate":
        migrate_to_mongo()
    elif arg == "--short-list-md":
        short_list(md=True)
    elif arg == "--update-channel-names":
        update_channel_names()
    elif arg == "--init":
        init()

b7411d59fd5910082fdec30d29035877df9b98cf
<mask token>


def run_process(site, idx, custom_parsers, token, kwargs):
    counter = Value('i', 0)
    sleeper = Value('i', 300)
    args = site, counter, sleeper, token
    if idx not in custom_parsers:
        p = Process(target=NewsParser, args=args, kwargs=kwargs)
    else:
        p = Process(target=custom_parsers[idx], args=args, kwargs=kwargs)
    if 'channels' not in site and not kwargs['debug']:
        return
    p.start()
    return {'process': p, 'counter': counter, 'sleeper': sleeper, 'warning':
        0, 'old_counter': 0, 'site': site, 'id': idx}


def start_watchdog(processes, token, kwargs):
    time.sleep(300)
    i = 0
    while True:
        time.sleep(31)
        i += 1
        check_process(processes, PARSER, i, token, kwargs)


def check_process(processes, custom_parsers, iteration, token, kwargs):
    for idx, p in enumerate(processes):
        if 30 / p['sleeper'].value * iteration % 1:
            continue
        if p['counter'].value > p['old_counter']:
            p['old_counter'] = p['counter'].value
            p['warning'] = 0
        else:
            logging.info(p)
            p['warning'] += 1
            if p['warning'] > 3:
                p['process'].terminate()
                if p['warning'] > 5:
                    p['process'].kill()
                processes.append(run_process(p['site'], p['id'],
                    custom_parsers, token, kwargs))
                logging.error('Process restarted: ' + p['site']['name'])
                del processes[idx]


def get_channels(path, filter=0, all=False):
    with open('{}/{}'.format(os.path.dirname(sys.argv[0]), path)) as json_file:
        channels = list(json.load(json_file).values())
    if filter:
        return [channels[filter]]
    return channels if all else channels[1:]


def set_channels(channels, path, not_all=True):
    if not_all:
        channels = get_channels(path, all=True)[0] + channels
    with open('{}/{}'.format(os.path.dirname(sys.argv[0]), path), 'w',
        newline='\n') as outfile:
        json.dump({'{:03.0f}'.format(idx + 1): channel for idx, channel in
            enumerate(channels)}, outfile, sort_keys=True, indent=4)


def crawl_members(url):
    time.sleep(0.2)
    private = True
    if not url[:4] == 'http':
        url = 'https://t.me/' + url
        private = False
    try:
        if PROXY:
            html = requests.get(url, proxies={'https': PROXY}).text
        else:
            html = requests.get(url, proxies={'https': PROXY}).text
        members_str = re.findall('tgme_page_extra">([0-9]+) members</div>',
            html)[0]
        members = int(members_str) - 2
        return {'current_members': members, 'private': private}
    except IndexError:
        logging.info(url)
        return None


def count_all_subchannels(channels, serious):
    db = MongoDatabase({'port': MONGO_PORT, 'ip': MONGO_IP, 'pass':
        MONGO_PASS, 'user': MONGO_USER})
    result = {}
    for channel in channels:
        channel_infos = crawl_members(channel['link'])
        if channel_infos is None:
            continue
        channel_infos['old_members'] = db.get_members(channel['id'])
        if serious:
            db.set_members(channel_infos['current_members'], channel['id'])
        result[channel['id']] = channel_infos
    return result


<mask token>


def render_text(channels):
    sorted_channel_list = OrderedDict(sorted(channels['sites'].items(), key
        =lambda kv: kv[1]['absolut_members'], reverse=True))
    l = str(len(str(max([x['absolut_members'] for x in sorted_channel_list.
        values()]))))
    result = ['' for x in range(int(len(sorted_channel_list) / 98) + 1)]
    result[0] = '*Tagesbericht DerNewsChannel*\n\n'
    for idx, (name, channel) in enumerate(sorted_channel_list.items()):
        diff = calc_diff(channel['absolut_members'], channel[
            'absolut_old_members'])
        result[int((idx + 1) / 98)
            ] += '`{{:2}}. {{}} {{:{}}} {{:2}} {{}}`\n'.format(l).format(
            idx + 1, 'p' if channel['all_private'] else ' ', channel[
            'absolut_members'], diff, name[:15])
    complete_diff = calc_diff(channels['absolut_members'], channels[
        'absolut_old_members'])
    result[-1] += '\nGesamt: *{}* {}'.format(channels['absolut_members'], 
        '({})'.format(complete_diff) if complete_diff else '')
    return result


def send_stats(serious=False):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    text = render_text(get_counted_list(serious))
    for t in text:
        tg.send(t, CHANNEL_REPORT, {}, disable_notification=True)


def calc_diff(new, old):
    diff = ''
    idiff = new - old
    if idiff > 0:
        diff = '+{}'.format(idiff)
    elif idiff < 0:
        diff = '-{}'.format(idiff * -1)
    return diff


def short_list(md=False):
    result = """Perferkt formatierte News auf Telegram mit [@DerNewsChannel](https://t.me/dernewschannel)!

Folgende Kanäle werden zurzeit angeboten:
"""
    for site in sorted(get_channels('channels.json'), key=lambda k: k[
        'name'].upper()):
        for channel in sorted(site['channels'], key=lambda k: k['template'],
            reverse=True):
            if channel['template'] in [3, 4, 6]:
                if not channel['link'][:4] == 'http':
                    if not md:
                        result += '@' + channel['link'].replace('_', '\\_')
                    else:
                        result += '* [{}](https://t.me/{})'.format(site[
                            'name'], channel['link'])
                else:
                    if md:
                        result += '* '
                    result += '[{}]({})'.format(site['name'].replace('_',
                        '\\_'), channel['link'])
            elif channel['template'] == 5:
                result += ' ([Instant ⚡]({}))️'.format(channel['link'])
        result += '\n'
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    print(result.replace('\\_', '_'))
    if not md:
        tg.send(result, OWNER, [])


def send_summary(serious=False, full=False, fast=False):
    db = Database()
    articles = [{'name': a[2].replace('.', '․'), 'count': a[1], 'date': a[0
        ], 'added': a[3]} for a in db.count_articles()]
    site_list = {}
    date_list = {}
    results = {}
    for a in articles:
        if a['name'] not in site_list:
            site_list[a['name']] = []
            date_list[a['name']] = a['added']
        site_list[a['name']].append({'count': a['count'], 'date': datetime.
            datetime.strptime(a['date'], '%Y-%m-%d')})
    for site, data in sorted(site_list.items(), key=lambda s: s[0].lower()):
        try:
            channels = next((item for item in get_channels('channels.json') if
                item['name'] == site.replace('․', '.')), None)
        except TypeError:
            continue
        if not channels:
            continue
        mini = min([x['date'] for x in data])
        days = abs((datetime.datetime.today() - mini).days)
        counter = 0
        for x in data:
            counter += x['count']
        if channels['category'] not in results:
            results[channels['category']] = '*{}*\n'.format(channels[
                'category'].upper())
        if (datetime.datetime.today() - datetime.datetime.strptime(
            date_list[site], '%Y-%m-%d')).days > 13:
            results[channels['category']] += '\n*{}* (Ø{} M/Tag)\n'.format(site
                , '{:.1f}'.format(counter / days).replace('.', ','))
        else:
            results[channels['category']] += '\n*{} 🆕* \n'.format(site)
        try:
            for channel in channels['channels']:
                try:
                    members = ''
                except IndexError:
                    members = ''
                if not channel['link'][:4] == 'http':
                    results[channels['category']] += '@{} {}\n'.format(channel
                        ['link'].replace('_', '\\_'), members)
                elif channel['template'] == 5:
                    results[channels['category']
                        ] += '[⚡️ Instant View]({}) {}\n'.format(channel[
                        'link'], members)
                else:
                    results[channels['category']] += '[{}]({}) {}\n'.format(
                        site, channel['link'], members)
        except TypeError:
            continue
    other = '*SONSTIGES*\n\n'
    tg = Telegram()
    for category, text in sorted(results.items()):
        if text.count('\n') < 6:
            other += text + '\n\n'
            results.pop(category, '')
    results['Sonstiges'] = other
    results['Info'] = INFO
    ORDER = ['News', 'Technik']
    i = 0
    channel_id = CHANNEL_TEST if not serious else CHANNEL_MAIN
    send = ['Info', 'Sonstiges'] + [c for c in results if c not in ORDER and
        not c in ['Sonstiges', 'Info']] + ORDER[::-1]
    if not full:
        for category in send:
            i = tg.edit(i, results[category], channel_id, {},
                disable_notification=True, disable_preview=True)
    else:
        tg.unpin(channel_id)
        if not fast:
            time.sleep(300)
        first = tg.send(results[send[0]], channel_id, {},
            disable_notification=True, disable_preview=True)
        for i in reversed(range(first)):
            tg.delete(i, channel_id)
        tg.pin(channel_id, first)
        time.sleep(5)
        for category in send[1:]:
            tg.send(results[category], channel_id, {}, disable_notification
                =True, disable_preview=True)
            time.sleep(1)


def clean_up_db():
    db = Database()
    db.clean_up()


def migrate_news(mongo, news):
    logging.info('Migrate %s news', len(news))
    for idx, d in enumerate(news):
        try:
            if not int(idx % (len(news) / 100)):
                logging.info('Migrate %s%%', int(100 / len(news) * idx) + 1)
            id, s_id, title, text, link, img, date, tags, m_id, site_name = d
            if mongo.quick_check_news(link, link, site_name):
                mongo.check_news(title, text, link, site_name)
                continue
            x = mongo.insert_news(title, text, link, img, site_name, date=
                date, tags=tags)
            mongo.update_message_id(x, m_id)
        except Exception as e:
            logging.error(d)
            logging.exception(e)


def migrate_channels(mongo, channels):
    logging.info('Migrate %s channels', len(channels))
    for c in channels:
        (c_id, s_id, t_id, link, short, public, pinned, template, added,
            members, name) = c
        mongo.insert_channel(name, t_id, link, template, public, short,
            added=added, members=members, pinned=pinned)


def migrate_sites(mongo, sites):
    logging.info('Migrate %s sites', len(sites))
    for s in sites:
        s_id, name, rss, added = s
        mongo.insert_site(name, rss, added=added)


def migrate_to_mongo():
    db = Database()
    mongo = MongoDatabase()
    sites, channels, news = db.get_all()
    print(len(sites), len(channels), len(news))
    migrate_sites(mongo, sites)
    migrate_channels(mongo, channels)
    migrate_news(mongo, news)


<mask token>


def start_tool(arg):
    if arg == '--stats':
        send_stats()
    elif arg == '--stats-live':
        send_stats(serious=True)
    elif arg == '--mstats':
        send_summary()
    elif arg == '--mstats-live':
        send_summary(serious=True)
    elif arg == '--mstats-full':
        send_summary(full=True)
    elif arg == '--mstats-fast':
        send_summary(fast=True, full=True)
    elif arg == '--mstats-live-full':
        send_summary(serious=True, full=True)
    elif arg == '--clean':
        clean_up_db()
    elif arg == '--short-list':
        short_list()
    elif arg == '--migrate':
        migrate_to_mongo()
    elif arg == '--short-list-md':
        short_list(md=True)
    elif arg == '--update-channel-names':
        update_channel_names()
    elif arg == '--init':
        init()

<mask token>


def start_many(channels: List[Dict[str, any]], custom_parser: Dict[int,
    Type[NewsParser]], token: str, filter: int=0, owner: int=0, debug: bool
    =False, proxy: str=None, mongo: Dict[str, Union[str, int]]=None, error:
    int=None):
    """

    :param channels:
    :param custom_parser: Dictionary with inherited classes of NewsParser
    :param token: Telegram bot token
    :param filter: Can filter
    :param owner:
    :param debug:
    :param proxy:
    :param mongo:
    :param error:
    """
    processes = []
    kwargs = {'owner': owner, 'debug': debug, 'proxy': proxy, 'mongo':
        mongo, 'error': error}
    for idx, site in enumerate(channels):
        processes.append(run_process(site, filter if filter else idx + 1,
            custom_parser, token, kwargs))
    start_watchdog(processes, token, kwargs)


def run_process(site, idx, custom_parsers, token, kwargs):
    counter = Value('i', 0)
    sleeper = Value('i', 300)
    args = site, counter, sleeper, token
    if idx not in custom_parsers:
        p = Process(target=NewsParser, args=args, kwargs=kwargs)
    else:
        p = Process(target=custom_parsers[idx], args=args, kwargs=kwargs)
    if 'channels' not in site and not kwargs['debug']:
        return
    p.start()
    return {'process': p, 'counter': counter, 'sleeper': sleeper, 'warning':
        0, 'old_counter': 0, 'site': site, 'id': idx}


def start_watchdog(processes, token, kwargs):
    time.sleep(300)
    i = 0
    while True:
        time.sleep(31)
        i += 1
        check_process(processes, PARSER, i, token, kwargs)


def check_process(processes, custom_parsers, iteration, token, kwargs):
    for idx, p in enumerate(processes):
        if 30 / p['sleeper'].value * iteration % 1:
            continue
        if p['counter'].value > p['old_counter']:
            p['old_counter'] = p['counter'].value
            p['warning'] = 0
        else:
            logging.info(p)
            p['warning'] += 1
            if p['warning'] > 3:
                p['process'].terminate()
                if p['warning'] > 5:
                    p['process'].kill()
                processes.append(run_process(p['site'], p['id'],
                    custom_parsers, token, kwargs))
                logging.error('Process restarted: ' + p['site']['name'])
                del processes[idx]


def get_channels(path, filter=0, all=False):
    with open('{}/{}'.format(os.path.dirname(sys.argv[0]), path)) as json_file:
        channels = list(json.load(json_file).values())
    if filter:
        return [channels[filter]]
    return channels if all else channels[1:]


def set_channels(channels, path, not_all=True):
    if not_all:
        channels = get_channels(path, all=True)[0] + channels
    with open('{}/{}'.format(os.path.dirname(sys.argv[0]), path), 'w',
        newline='\n') as outfile:
        json.dump({'{:03.0f}'.format(idx + 1): channel for idx, channel in
            enumerate(channels)}, outfile, sort_keys=True, indent=4)


def crawl_members(url):
    time.sleep(0.2)
    private = True
    if not url[:4] == 'http':
        url = 'https://t.me/' + url
        private = False
    try:
        if PROXY:
            html = requests.get(url, proxies={'https': PROXY}).text
        else:
            html = requests.get(url, proxies={'https': PROXY}).text
        members_str = re.findall('tgme_page_extra">([0-9]+) members</div>',
            html)[0]
        members = int(members_str) - 2
        return {'current_members': members, 'private': private}
    except IndexError:
        logging.info(url)
        return None


def count_all_subchannels(channels, serious):
    db = MongoDatabase({'port': MONGO_PORT, 'ip': MONGO_IP, 'pass':
        MONGO_PASS, 'user': MONGO_USER})
    result = {}
    for channel in channels:
        channel_infos = crawl_members(channel['link'])
        if channel_infos is None:
            continue
        channel_infos['old_members'] = db.get_members(channel['id'])
        if serious:
            db.set_members(channel_infos['current_members'], channel['id'])
        result[channel['id']] = channel_infos
    return result


<mask token>


def render_text(channels):
    sorted_channel_list = OrderedDict(sorted(channels['sites'].items(), key
        =lambda kv: kv[1]['absolut_members'], reverse=True))
    l = str(len(str(max([x['absolut_members'] for x in sorted_channel_list.
        values()]))))
    result = ['' for x in range(int(len(sorted_channel_list) / 98) + 1)]
    result[0] = '*Tagesbericht DerNewsChannel*\n\n'
    for idx, (name, channel) in enumerate(sorted_channel_list.items()):
        diff = calc_diff(channel['absolut_members'], channel[
            'absolut_old_members'])
        result[int((idx + 1) / 98)
            ] += '`{{:2}}. {{}} {{:{}}} {{:2}} {{}}`\n'.format(l).format(
            idx + 1, 'p' if channel['all_private'] else ' ', channel[
            'absolut_members'], diff, name[:15])
    complete_diff = calc_diff(channels['absolut_members'], channels[
        'absolut_old_members'])
    result[-1] += '\nGesamt: *{}* {}'.format(channels['absolut_members'], 
        '({})'.format(complete_diff) if complete_diff else '')
    return result


def send_stats(serious=False):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    text = render_text(get_counted_list(serious))
    for t in text:
        tg.send(t, CHANNEL_REPORT, {}, disable_notification=True)


def calc_diff(new, old):
    diff = ''
    idiff = new - old
    if idiff > 0:
        diff = '+{}'.format(idiff)
    elif idiff < 0:
        diff = '-{}'.format(idiff * -1)
    return diff


def short_list(md=False):
    result = """Perferkt formatierte News auf Telegram mit [@DerNewsChannel](https://t.me/dernewschannel)!

Folgende Kanäle werden zurzeit angeboten:
"""
    for site in sorted(get_channels('channels.json'), key=lambda k: k[
        'name'].upper()):
        for channel in sorted(site['channels'], key=lambda k: k['template'],
            reverse=True):
            if channel['template'] in [3, 4, 6]:
                if not channel['link'][:4] == 'http':
                    if not md:
                        result += '@' + channel['link'].replace('_', '\\_')
                    else:
                        result += '* [{}](https://t.me/{})'.format(site[
                            'name'], channel['link'])
                else:
                    if md:
                        result += '* '
                    result += '[{}]({})'.format(site['name'].replace('_',
                        '\\_'), channel['link'])
            elif channel['template'] == 5:
                result += ' ([Instant ⚡]({}))️'.format(channel['link'])
        result += '\n'
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    print(result.replace('\\_', '_'))
    if not md:
        tg.send(result, OWNER, [])


def send_summary(serious=False, full=False, fast=False):
    db = Database()
    articles = [{'name': a[2].replace('.', '․'), 'count': a[1], 'date': a[0
        ], 'added': a[3]} for a in db.count_articles()]
    site_list = {}
    date_list = {}
    results = {}
    for a in articles:
        if a['name'] not in site_list:
            site_list[a['name']] = []
            date_list[a['name']] = a['added']
        site_list[a['name']].append({'count': a['count'], 'date': datetime.
            datetime.strptime(a['date'], '%Y-%m-%d')})
    for site, data in sorted(site_list.items(), key=lambda s: s[0].lower()):
        try:
            channels = next((item for item in get_channels('channels.json') if
                item['name'] == site.replace('․', '.')), None)
        except TypeError:
            continue
        if not channels:
            continue
        mini = min([x['date'] for x in data])
        days = abs((datetime.datetime.today() - mini).days)
        counter = 0
        for x in data:
            counter += x['count']
        if channels['category'] not in results:
            results[channels['category']] = '*{}*\n'.format(channels[
                'category'].upper())
        if (datetime.datetime.today() - datetime.datetime.strptime(
            date_list[site], '%Y-%m-%d')).days > 13:
            results[channels['category']] += '\n*{}* (Ø{} M/Tag)\n'.format(site
                , '{:.1f}'.format(counter / days).replace('.', ','))
        else:
            results[channels['category']] += '\n*{} 🆕* \n'.format(site)
        try:
            for channel in channels['channels']:
                try:
                    members = ''
                except IndexError:
                    members = ''
                if not channel['link'][:4] == 'http':
                    results[channels['category']] += '@{} {}\n'.format(channel
                        ['link'].replace('_', '\\_'), members)
                elif channel['template'] == 5:
                    results[channels['category']
                        ] += '[⚡️ Instant View]({}) {}\n'.format(channel[
                        'link'], members)
                else:
                    results[channels['category']] += '[{}]({}) {}\n'.format(
                        site, channel['link'], members)
        except TypeError:
            continue
    other = '*SONSTIGES*\n\n'
    tg = Telegram()
    for category, text in sorted(results.items()):
        if text.count('\n') < 6:
            other += text + '\n\n'
            results.pop(category, '')
    results['Sonstiges'] = other
    results['Info'] = INFO
    ORDER = ['News', 'Technik']
    i = 0
    channel_id = CHANNEL_TEST if not serious else CHANNEL_MAIN
    send = ['Info', 'Sonstiges'] + [c for c in results if c not in ORDER and
        not c in ['Sonstiges', 'Info']] + ORDER[::-1]
    if not full:
        for category in send:
            i = tg.edit(i, results[category], channel_id, {},
                disable_notification=True, disable_preview=True)
    else:
        tg.unpin(channel_id)
        if not fast:
            time.sleep(300)
        first = tg.send(results[send[0]], channel_id, {},
            disable_notification=True, disable_preview=True)
        for i in reversed(range(first)):
            tg.delete(i, channel_id)
        tg.pin(channel_id, first)
        time.sleep(5)
        for category in send[1:]:
            tg.send(results[category], channel_id, {}, disable_notification
                =True, disable_preview=True)
            time.sleep(1)


def clean_up_db():
    db = Database()
    db.clean_up()


def migrate_news(mongo, news):
    logging.info('Migrate %s news', len(news))
    for idx, d in enumerate(news):
        try:
            if not int(idx % (len(news) / 100)):
                logging.info('Migrate %s%%', int(100 / len(news) * idx) + 1)
            id, s_id, title, text, link, img, date, tags, m_id, site_name = d
            if mongo.quick_check_news(link, link, site_name):
                mongo.check_news(title, text, link, site_name)
                continue
            x = mongo.insert_news(title, text, link, img, site_name, date=
                date, tags=tags)
            mongo.update_message_id(x, m_id)
        except Exception as e:
            logging.error(d)
            logging.exception(e)


def migrate_channels(mongo, channels):
    logging.info('Migrate %s channels', len(channels))
    for c in channels:
        (c_id, s_id, t_id, link, short, public, pinned, template, added,
            members, name) = c
        mongo.insert_channel(name, t_id, link, template, public, short,
            added=added, members=members, pinned=pinned)


def migrate_sites(mongo, sites):
    logging.info('Migrate %s sites', len(sites))
    for s in sites:
        s_id, name, rss, added = s
        mongo.insert_site(name, rss, added=added)


def migrate_to_mongo():
    db = Database()
    mongo = MongoDatabase()
    sites, channels, news = db.get_all()
    print(len(sites), len(channels), len(news))
    migrate_sites(mongo, sites)
    migrate_channels(mongo, channels)
    migrate_news(mongo, news)


<mask token>


def init():
    db = MongoDatabase({'port': MONGO_PORT, 'ip': MONGO_IP, 'pass':
        MONGO_PASS, 'user': MONGO_USER})
    main_channel = get_channels('channels.json', all=True)[0]
    db.insert_main_channel(main_channel['channels'][0]['id'], main_channel[
        'channels'][0]['link'], True, 0)


def start_tool(arg):
    if arg == '--stats':
        send_stats()
    elif arg == '--stats-live':
        send_stats(serious=True)
    elif arg == '--mstats':
        send_summary()
    elif arg == '--mstats-live':
        send_summary(serious=True)
    elif arg == '--mstats-full':
        send_summary(full=True)
    elif arg == '--mstats-fast':
        send_summary(fast=True, full=True)
    elif arg == '--mstats-live-full':
        send_summary(serious=True, full=True)
    elif arg == '--clean':
        clean_up_db()
    elif arg == '--short-list':
        short_list()
    elif arg == '--migrate':
        migrate_to_mongo()
    elif arg == '--short-list-md':
        short_list(md=True)
    elif arg == '--update-channel-names':
        update_channel_names()
    elif arg == '--init':
        init()

<mask token>


def start_many(channels: List[Dict[str, any]], custom_parser: Dict[int,
    Type[NewsParser]], token: str, filter: int=0, owner: int=0, debug: bool
    =False, proxy: str=None, mongo: Dict[str, Union[str, int]]=None, error:
    int=None):
    """

    :param channels:
    :param custom_parser: Dictionary with inherited classes of NewsParser
    :param token: Telegram bot token
    :param filter: Can filter
    :param owner:
    :param debug:
    :param proxy:
    :param mongo:
    :param error:
    """
    processes = []
    kwargs = {'owner': owner, 'debug': debug, 'proxy': proxy, 'mongo':
        mongo, 'error': error}
    for idx, site in enumerate(channels):
        processes.append(run_process(site, filter if filter else idx + 1,
            custom_parser, token, kwargs))
    start_watchdog(processes, token, kwargs)


def run_process(site, idx, custom_parsers, token, kwargs):
    counter = Value('i', 0)
    sleeper = Value('i', 300)
    args = site, counter, sleeper, token
    if idx not in custom_parsers:
        p = Process(target=NewsParser, args=args, kwargs=kwargs)
    else:
        p = Process(target=custom_parsers[idx], args=args, kwargs=kwargs)
    if 'channels' not in site and not kwargs['debug']:
        return
    p.start()
    return {'process': p, 'counter': counter, 'sleeper': sleeper, 'warning':
        0, 'old_counter': 0, 'site': site, 'id': idx}


def start_watchdog(processes, token, kwargs):
    time.sleep(300)
    i = 0
    while True:
        time.sleep(31)
        i += 1
        check_process(processes, PARSER, i, token, kwargs)


def check_process(processes, custom_parsers, iteration, token, kwargs):
    for idx, p in enumerate(processes):
        if 30 / p['sleeper'].value * iteration % 1:
            continue
        if p['counter'].value > p['old_counter']:
            p['old_counter'] = p['counter'].value
            p['warning'] = 0
        else:
            logging.info(p)
            p['warning'] += 1
            if p['warning'] > 3:
                p['process'].terminate()
                if p['warning'] > 5:
                    p['process'].kill()
                processes.append(run_process(p['site'], p['id'],
                    custom_parsers, token, kwargs))
                logging.error('Process restarted: ' + p['site']['name'])
                del processes[idx]


def get_channels(path, filter=0, all=False):
    with open('{}/{}'.format(os.path.dirname(sys.argv[0]), path)) as json_file:
        channels = list(json.load(json_file).values())
    if filter:
        return [channels[filter]]
    return channels if all else channels[1:]


def set_channels(channels, path, not_all=True):
    if not_all:
        channels = get_channels(path, all=True)[0] + channels
    with open('{}/{}'.format(os.path.dirname(sys.argv[0]), path), 'w',
        newline='\n') as outfile:
        json.dump({'{:03.0f}'.format(idx + 1): channel for idx, channel in
            enumerate(channels)}, outfile, sort_keys=True, indent=4)


def crawl_members(url):
    time.sleep(0.2)
    private = True
    if not url[:4] == 'http':
        url = 'https://t.me/' + url
        private = False
    try:
        if PROXY:
            html = requests.get(url, proxies={'https': PROXY}).text
        else:
            html = requests.get(url, proxies={'https': PROXY}).text
        members_str = re.findall('tgme_page_extra">([0-9]+) members</div>',
            html)[0]
        members = int(members_str) - 2
        return {'current_members': members, 'private': private}
    except IndexError:
        logging.info(url)
        return None


def count_all_subchannels(channels, serious):
    db = MongoDatabase({'port': MONGO_PORT, 'ip': MONGO_IP, 'pass':
        MONGO_PASS, 'user': MONGO_USER})
    result = {}
    for channel in channels:
        channel_infos = crawl_members(channel['link'])
        if channel_infos is None:
            continue
        channel_infos['old_members'] = db.get_members(channel['id'])
        if serious:
            db.set_members(channel_infos['current_members'], channel['id'])
        result[channel['id']] = channel_infos
    return result


def get_counted_list(serious):
    channels = {'sites': {}}
    for site in get_channels('channels.json', all=True):
        sub_channels = count_all_subchannels(site['channels'], serious)
        absolut_members = sum(channel['current_members'] for channel in
            sub_channels.values())
        absolut_old_members = sum(channel['old_members'] for channel in
            sub_channels.values())
        all_private = all(channel['private'] for channel in sub_channels.
            values())
        channels['sites'][site['name']] = {'channels': sub_channels,
            'absolut_members': absolut_members, 'absolut_old_members':
            absolut_old_members, 'all_private': all_private}
    channels['absolut_members'] = sum(channel['absolut_members'] for
        channel in channels['sites'].values())
    channels['absolut_old_members'] = sum(channel['absolut_old_members'] for
        channel in channels['sites'].values())
    return channels


def render_text(channels):
    sorted_channel_list = OrderedDict(sorted(channels['sites'].items(), key
        =lambda kv: kv[1]['absolut_members'], reverse=True))
    l = str(len(str(max([x['absolut_members'] for x in sorted_channel_list.
        values()]))))
    result = ['' for x in range(int(len(sorted_channel_list) / 98) + 1)]
    result[0] = '*Tagesbericht DerNewsChannel*\n\n'
    for idx, (name, channel) in enumerate(sorted_channel_list.items()):
        diff = calc_diff(channel['absolut_members'], channel[
            'absolut_old_members'])
        result[int((idx + 1) / 98)
            ] += '`{{:2}}. {{}} {{:{}}} {{:2}} {{}}`\n'.format(l).format(
            idx + 1, 'p' if channel['all_private'] else ' ', channel[
            'absolut_members'], diff, name[:15])
    complete_diff = calc_diff(channels['absolut_members'], channels[
        'absolut_old_members'])
    result[-1] += '\nGesamt: *{}* {}'.format(channels['absolut_members'], 
        '({})'.format(complete_diff) if complete_diff else '')
    return result


def send_stats(serious=False):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    text = render_text(get_counted_list(serious))
    for t in text:
        tg.send(t, CHANNEL_REPORT, {}, disable_notification=True)


def calc_diff(new, old):
    diff = ''
    idiff = new - old
    if idiff > 0:
        diff = '+{}'.format(idiff)
    elif idiff < 0:
        diff = '-{}'.format(idiff * -1)
    return diff


def short_list(md=False):
    result = """Perferkt formatierte News auf Telegram mit [@DerNewsChannel](https://t.me/dernewschannel)!

Folgende Kanäle werden zurzeit angeboten:
"""
    for site in sorted(get_channels('channels.json'), key=lambda k: k[
        'name'].upper()):
        for channel in sorted(site['channels'], key=lambda k: k['template'],
            reverse=True):
            if channel['template'] in [3, 4, 6]:
                if not channel['link'][:4] == 'http':
                    if not md:
                        result += '@' + channel['link'].replace('_', '\\_')
                    else:
                        result += '* [{}](https://t.me/{})'.format(site[
                            'name'], channel['link'])
                else:
                    if md:
                        result += '* '
                    result += '[{}]({})'.format(site['name'].replace('_',
                        '\\_'), channel['link'])
            elif channel['template'] == 5:
                result += ' ([Instant ⚡]({}))️'.format(channel['link'])
        result += '\n'
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    print(result.replace('\\_', '_'))
    if not md:
        tg.send(result, OWNER, [])


def send_summary(serious=False, full=False, fast=False):
    db = Database()
    articles = [{'name': a[2].replace('.', '․'), 'count': a[1], 'date': a[0
        ], 'added': a[3]} for a in db.count_articles()]
    site_list = {}
    date_list = {}
    results = {}
    for a in articles:
        if a['name'] not in site_list:
            site_list[a['name']] = []
            date_list[a['name']] = a['added']
        site_list[a['name']].append({'count': a['count'], 'date': datetime.
            datetime.strptime(a['date'], '%Y-%m-%d')})
    for site, data in sorted(site_list.items(), key=lambda s: s[0].lower()):
        try:
            channels = next((item for item in get_channels('channels.json') if
                item['name'] == site.replace('․', '.')), None)
        except TypeError:
            continue
        if not channels:
            continue
        mini = min([x['date'] for x in data])
        days = abs((datetime.datetime.today() - mini).days)
        counter = 0
        for x in data:
            counter += x['count']
        if channels['category'] not in results:
            results[channels['category']] = '*{}*\n'.format(channels[
                'category'].upper())
        if (datetime.datetime.today() - datetime.datetime.strptime(
            date_list[site], '%Y-%m-%d')).days > 13:
            results[channels['category']] += '\n*{}* (Ø{} M/Tag)\n'.format(site
                , '{:.1f}'.format(counter / days).replace('.', ','))
        else:
            results[channels['category']] += '\n*{} 🆕* \n'.format(site)
        try:
            for channel in channels['channels']:
                try:
                    members = ''
                except IndexError:
                    members = ''
                if not channel['link'][:4] == 'http':
                    results[channels['category']] += '@{} {}\n'.format(channel
                        ['link'].replace('_', '\\_'), members)
                elif channel['template'] == 5:
                    results[channels['category']
                        ] += '[⚡️ Instant View]({}) {}\n'.format(channel[
                        'link'], members)
                else:
                    results[channels['category']] += '[{}]({}) {}\n'.format(
                        site, channel['link'], members)
        except TypeError:
            continue
    other = '*SONSTIGES*\n\n'
    tg = Telegram()
    for category, text in sorted(results.items()):
        if text.count('\n') < 6:
            other += text + '\n\n'
            results.pop(category, '')
    results['Sonstiges'] = other
    results['Info'] = INFO
    ORDER = ['News', 'Technik']
    i = 0
    channel_id = CHANNEL_TEST if not serious else CHANNEL_MAIN
    send = ['Info', 'Sonstiges'] + [c for c in results if c not in ORDER and
        not c in ['Sonstiges', 'Info']] + ORDER[::-1]
    if not full:
        for category in send:
            i = tg.edit(i, results[category], channel_id, {},
                disable_notification=True, disable_preview=True)
    else:
        tg.unpin(channel_id)
        if not fast:
            time.sleep(300)
        first = tg.send(results[send[0]], channel_id, {},
            disable_notification=True, disable_preview=True)
        for i in reversed(range(first)):
            tg.delete(i, channel_id)
        tg.pin(channel_id, first)
        time.sleep(5)
        for category in send[1:]:
            tg.send(results[category], channel_id, {}, disable_notification
                =True, disable_preview=True)
            time.sleep(1)


def clean_up_db():
    db = Database()
    db.clean_up()


def migrate_news(mongo, news):
    logging.info('Migrate %s news', len(news))
    for idx, d in enumerate(news):
        try:
            if not int(idx % (len(news) / 100)):
                logging.info('Migrate %s%%', int(100 / len(news) * idx) + 1)
            id, s_id, title, text, link, img, date, tags, m_id, site_name = d
            if mongo.quick_check_news(link, link, site_name):
                mongo.check_news(title, text, link, site_name)
                continue
            x = mongo.insert_news(title, text, link, img, site_name, date=
                date, tags=tags)
            mongo.update_message_id(x, m_id)
        except Exception as e:
            logging.error(d)
            logging.exception(e)


def migrate_channels(mongo, channels):
    logging.info('Migrate %s channels', len(channels))
    for c in channels:
        (c_id, s_id, t_id, link, short, public, pinned, template, added,
            members, name) = c
        mongo.insert_channel(name, t_id, link, template, public, short,
            added=added, members=members, pinned=pinned)


def migrate_sites(mongo, sites):
    logging.info('Migrate %s sites', len(sites))
    for s in sites:
        s_id, name, rss, added = s
        mongo.insert_site(name, rss, added=added)


def migrate_to_mongo():
    db = Database()
    mongo = MongoDatabase()
    sites, channels, news = db.get_all()
    print(len(sites), len(channels), len(news))
    migrate_sites(mongo, sites)
    migrate_channels(mongo, channels)
    migrate_news(mongo, news)


<mask token>


def init():
    db = MongoDatabase({'port': MONGO_PORT, 'ip': MONGO_IP, 'pass':
        MONGO_PASS, 'user': MONGO_USER})
    main_channel = get_channels('channels.json', all=True)[0]
    db.insert_main_channel(main_channel['channels'][0]['id'], main_channel[
        'channels'][0]['link'], True, 0)


def start_tool(arg):
    if arg == '--stats':
        send_stats()
    elif arg == '--stats-live':
        send_stats(serious=True)
    elif arg == '--mstats':
        send_summary()
    elif arg == '--mstats-live':
        send_summary(serious=True)
    elif arg == '--mstats-full':
        send_summary(full=True)
    elif arg == '--mstats-fast':
        send_summary(fast=True, full=True)
    elif arg == '--mstats-live-full':
        send_summary(serious=True, full=True)
    elif arg == '--clean':
        clean_up_db()
    elif arg == '--short-list':
        short_list()
    elif arg == '--migrate':
        migrate_to_mongo()
    elif arg == '--short-list-md':
        short_list(md=True)
    elif arg == '--update-channel-names':
        update_channel_names()
    elif arg == '--init':
        init()

<mask token>


def start_many(channels: List[Dict[str, any]], custom_parser: Dict[int,
    Type[NewsParser]], token: str, filter: int=0, owner: int=0, debug: bool
    =False, proxy: str=None, mongo: Dict[str, Union[str, int]]=None, error:
    int=None):
    """

    :param channels:
    :param custom_parser: Dictionary with inherited classes of NewsParser
    :param token: Telegram bot token
    :param filter: Can filter
    :param owner:
    :param debug:
    :param proxy:
    :param mongo:
    :param error:
    """
    processes = []
    kwargs = {'owner': owner, 'debug': debug, 'proxy': proxy, 'mongo':
        mongo, 'error': error}
    for idx, site in enumerate(channels):
        processes.append(run_process(site, filter if filter else idx + 1,
            custom_parser, token, kwargs))
    start_watchdog(processes, token, kwargs)


def run_process(site, idx, custom_parsers, token, kwargs):
    counter = Value('i', 0)
    sleeper = Value('i', 300)
    args = site, counter, sleeper, token
    if idx not in custom_parsers:
        p = Process(target=NewsParser, args=args, kwargs=kwargs)
    else:
        p = Process(target=custom_parsers[idx], args=args, kwargs=kwargs)
    if 'channels' not in site and not kwargs['debug']:
        return
    p.start()
    return {'process': p, 'counter': counter, 'sleeper': sleeper, 'warning':
        0, 'old_counter': 0, 'site': site, 'id': idx}


def start_watchdog(processes, token, kwargs):
    time.sleep(300)
    i = 0
    while True:
        time.sleep(31)
        i += 1
        check_process(processes, PARSER, i, token, kwargs)


def check_process(processes, custom_parsers, iteration, token, kwargs):
    for idx, p in enumerate(processes):
        if 30 / p['sleeper'].value * iteration % 1:
            continue
        if p['counter'].value > p['old_counter']:
            p['old_counter'] = p['counter'].value
            p['warning'] = 0
        else:
            logging.info(p)
            p['warning'] += 1
            if p['warning'] > 3:
                p['process'].terminate()
                if p['warning'] > 5:
                    p['process'].kill()
                processes.append(run_process(p['site'], p['id'],
                    custom_parsers, token, kwargs))
                logging.error('Process restarted: ' + p['site']['name'])
                del processes[idx]


def get_channels(path, filter=0, all=False):
    with open('{}/{}'.format(os.path.dirname(sys.argv[0]), path)) as json_file:
        channels = list(json.load(json_file).values())
    if filter:
        return [channels[filter]]
    return channels if all else channels[1:]


def set_channels(channels, path, not_all=True):
    if not_all:
        channels = get_channels(path, all=True)[0] + channels
    with open('{}/{}'.format(os.path.dirname(sys.argv[0]), path), 'w',
        newline='\n') as outfile:
        json.dump({'{:03.0f}'.format(idx + 1): channel for idx, channel in
            enumerate(channels)}, outfile, sort_keys=True, indent=4)


def crawl_members(url):
    time.sleep(0.2)
    private = True
    if not url[:4] == 'http':
        url = 'https://t.me/' + url
        private = False
    try:
        if PROXY:
            html = requests.get(url, proxies={'https': PROXY}).text
        else:
            html = requests.get(url, proxies={'https': PROXY}).text
        members_str = re.findall('tgme_page_extra">([0-9]+) members</div>',
            html)[0]
        members = int(members_str) - 2
        return {'current_members': members, 'private': private}
    except IndexError:
        logging.info(url)
        return None


def count_all_subchannels(channels, serious):
    db = MongoDatabase({'port': MONGO_PORT, 'ip': MONGO_IP, 'pass':
        MONGO_PASS, 'user': MONGO_USER})
    result = {}
    for channel in channels:
        channel_infos = crawl_members(channel['link'])
        if channel_infos is None:
            continue
        channel_infos['old_members'] = db.get_members(channel['id'])
        if serious:
            db.set_members(channel_infos['current_members'], channel['id'])
        result[channel['id']] = channel_infos
    return result


def get_counted_list(serious):
    channels = {'sites': {}}
    for site in get_channels('channels.json', all=True):
        sub_channels = count_all_subchannels(site['channels'], serious)
        absolut_members = sum(channel['current_members'] for channel in
            sub_channels.values())
        absolut_old_members = sum(channel['old_members'] for channel in
            sub_channels.values())
        all_private = all(channel['private'] for channel in sub_channels.
            values())
        channels['sites'][site['name']] = {'channels': sub_channels,
            'absolut_members': absolut_members, 'absolut_old_members':
            absolut_old_members, 'all_private': all_private}
    channels['absolut_members'] = sum(channel['absolut_members'] for
        channel in channels['sites'].values())
    channels['absolut_old_members'] = sum(channel['absolut_old_members'] for
        channel in channels['sites'].values())
    return channels


def render_text(channels):
    sorted_channel_list = OrderedDict(sorted(channels['sites'].items(), key
        =lambda kv: kv[1]['absolut_members'], reverse=True))
    l = str(len(str(max([x['absolut_members'] for x in sorted_channel_list.
        values()]))))
    result = ['' for x in range(int(len(sorted_channel_list) / 98) + 1)]
    result[0] = '*Tagesbericht DerNewsChannel*\n\n'
    for idx, (name, channel) in enumerate(sorted_channel_list.items()):
        diff = calc_diff(channel['absolut_members'], channel[
            'absolut_old_members'])
        result[int((idx + 1) / 98)
            ] += '`{{:2}}. {{}} {{:{}}} {{:2}} {{}}`\n'.format(l).format(
            idx + 1, 'p' if channel['all_private'] else ' ', channel[
            'absolut_members'], diff, name[:15])
    complete_diff = calc_diff(channels['absolut_members'], channels[
        'absolut_old_members'])
    result[-1] += '\nGesamt: *{}* {}'.format(channels['absolut_members'], 
        '({})'.format(complete_diff) if complete_diff else '')
    return result


def send_stats(serious=False):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    text = render_text(get_counted_list(serious))
    for t in text:
        tg.send(t, CHANNEL_REPORT, {}, disable_notification=True)


def calc_diff(new, old):
    diff = ''
    idiff = new - old
    if idiff > 0:
        diff = '+{}'.format(idiff)
    elif idiff < 0:
        diff = '-{}'.format(idiff * -1)
    return diff


def short_list(md=False):
    result = """Perferkt formatierte News auf Telegram mit [@DerNewsChannel](https://t.me/dernewschannel)!

Folgende Kanäle werden zurzeit angeboten:
"""
    for site in sorted(get_channels('channels.json'), key=lambda k: k[
        'name'].upper()):
        for channel in sorted(site['channels'], key=lambda k: k['template'],
            reverse=True):
            if channel['template'] in [3, 4, 6]:
                if not channel['link'][:4] == 'http':
                    if not md:
                        result += '@' + channel['link'].replace('_', '\\_')
                    else:
                        result += '* [{}](https://t.me/{})'.format(site[
                            'name'], channel['link'])
                else:
                    if md:
                        result += '* '
                    result += '[{}]({})'.format(site['name'].replace('_',
                        '\\_'), channel['link'])
            elif channel['template'] == 5:
                result += ' ([Instant ⚡]({}))️'.format(channel['link'])
        result += '\n'
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    print(result.replace('\\_', '_'))
    if not md:
        tg.send(result, OWNER, [])


def send_summary(serious=False, full=False, fast=False):
    db = Database()
    articles = [{'name': a[2].replace('.', '․'), 'count': a[1], 'date': a[0
        ], 'added': a[3]} for a in db.count_articles()]
    site_list = {}
    date_list = {}
    results = {}
    for a in articles:
        if a['name'] not in site_list:
            site_list[a['name']] = []
            date_list[a['name']] = a['added']
        site_list[a['name']].append({'count': a['count'], 'date': datetime.
            datetime.strptime(a['date'], '%Y-%m-%d')})
    for site, data in sorted(site_list.items(), key=lambda s: s[0].lower()):
        try:
            channels = next((item for item in get_channels('channels.json') if
                item['name'] == site.replace('․', '.')), None)
        except TypeError:
            continue
        if not channels:
            continue
        mini = min([x['date'] for x in data])
        days = abs((datetime.datetime.today() - mini).days)
        counter = 0
        for x in data:
            counter += x['count']
        if channels['category'] not in results:
            results[channels['category']] = '*{}*\n'.format(channels[
                'category'].upper())
        if (datetime.datetime.today() - datetime.datetime.strptime(
            date_list[site], '%Y-%m-%d')).days > 13:
            results[channels['category']] += '\n*{}* (Ø{} M/Tag)\n'.format(site
                , '{:.1f}'.format(counter / days).replace('.', ','))
        else:
            results[channels['category']] += '\n*{} 🆕* \n'.format(site)
        try:
            for channel in channels['channels']:
                try:
                    members = ''
                except IndexError:
                    members = ''
                if not channel['link'][:4] == 'http':
                    results[channels['category']] += '@{} {}\n'.format(channel
                        ['link'].replace('_', '\\_'), members)
                elif channel['template'] == 5:
                    results[channels['category']
                        ] += '[⚡️ Instant View]({}) {}\n'.format(channel[
                        'link'], members)
                else:
                    results[channels['category']] += '[{}]({}) {}\n'.format(
                        site, channel['link'], members)
        except TypeError:
            continue
    other = '*SONSTIGES*\n\n'
    tg = Telegram()
    for category, text in sorted(results.items()):
        if text.count('\n') < 6:
            other += text + '\n\n'
            results.pop(category, '')
    results['Sonstiges'] = other
    results['Info'] = INFO
    ORDER = ['News', 'Technik']
    i = 0
    channel_id = CHANNEL_TEST if not serious else CHANNEL_MAIN
    send = ['Info', 'Sonstiges'] + [c for c in results if c not in ORDER and
        not c in ['Sonstiges', 'Info']] + ORDER[::-1]
    if not full:
        for category in send:
            i = tg.edit(i, results[category], channel_id, {},
                disable_notification=True, disable_preview=True)
    else:
        tg.unpin(channel_id)
        if not fast:
            time.sleep(300)
        first = tg.send(results[send[0]], channel_id, {},
            disable_notification=True, disable_preview=True)
        for i in reversed(range(first)):
            tg.delete(i, channel_id)
        tg.pin(channel_id, first)
        time.sleep(5)
        for category in send[1:]:
            tg.send(results[category], channel_id, {}, disable_notification
                =True, disable_preview=True)
            time.sleep(1)


def clean_up_db():
    db = Database()
    db.clean_up()


def migrate_news(mongo, news):
    logging.info('Migrate %s news', len(news))
    for idx, d in enumerate(news):
        try:
            if not int(idx % (len(news) / 100)):
                logging.info('Migrate %s%%', int(100 / len(news) * idx) + 1)
            id, s_id, title, text, link, img, date, tags, m_id, site_name = d
            if mongo.quick_check_news(link, link, site_name):
                mongo.check_news(title, text, link, site_name)
                continue
            x = mongo.insert_news(title, text, link, img, site_name, date=
                date, tags=tags)
            mongo.update_message_id(x, m_id)
        except Exception as e:
            logging.error(d)
            logging.exception(e)


def migrate_channels(mongo, channels):
    logging.info('Migrate %s channels', len(channels))
    for c in channels:
        (c_id, s_id, t_id, link, short, public, pinned, template, added,
            members, name) = c
        mongo.insert_channel(name, t_id, link, template, public, short,
            added=added, members=members, pinned=pinned)


def migrate_sites(mongo, sites):
    logging.info('Migrate %s sites', len(sites))
    for s in sites:
        s_id, name, rss, added = s
        mongo.insert_site(name, rss, added=added)


def migrate_to_mongo():
    db = Database()
    mongo = MongoDatabase()
    sites, channels, news = db.get_all()
    print(len(sites), len(channels), len(news))
    migrate_sites(mongo, sites)
    migrate_channels(mongo, channels)
    migrate_news(mongo, news)


def update_channel_names(path='channels.json'):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    sites = get_channels(path)
    for key, site in enumerate(sites):
        if 'channels' not in site:
            continue
        for idx, channel in enumerate(site['channels']):
            time.sleep(1)
            try:
                channel_link = tg.get_chat_infos(channel['id'])
            except:
                logging.warning('Timed out ' + site['name'])
                channel_link = channel['link']
            if not (channel_link[:4] == 'http' and channel['link'][:4] ==
                'http') and not channel_link == channel['link']:
                print(channel_link, channel['link'])
                sites[key]['channels'][idx]['link'] = channel_link
    set_channels(sites, path)


def init():
    db = MongoDatabase({'port': MONGO_PORT, 'ip': MONGO_IP, 'pass':
        MONGO_PASS, 'user': MONGO_USER})
    main_channel = get_channels('channels.json', all=True)[0]
    db.insert_main_channel(main_channel['channels'][0]['id'], main_channel[
        'channels'][0]['link'], True, 0)


def start_tool(arg):
    if arg == '--stats':
        send_stats()
    elif arg == '--stats-live':
        send_stats(serious=True)
    elif arg == '--mstats':
        send_summary()
    elif arg == '--mstats-live':
        send_summary(serious=True)
    elif arg == '--mstats-full':
        send_summary(full=True)
    elif arg == '--mstats-fast':
        send_summary(fast=True, full=True)
    elif arg == '--mstats-live-full':
        send_summary(serious=True, full=True)
    elif arg == '--clean':
        clean_up_db()
    elif arg == '--short-list':
        short_list()
    elif arg == '--migrate':
        migrate_to_mongo()
    elif arg == '--short-list-md':
        short_list(md=True)
    elif arg == '--update-channel-names':
        update_channel_names()
    elif arg == '--init':
        init()

import datetime
import json
import logging
import os
import re
import sys
import time
from collections import OrderedDict
from multiprocessing import Process, Value
from typing import List, Dict, Type, Union

import requests

sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
from examples.CustomParser import PARSER
from examples.config import (
    BOT_TOKEN,
    PROXY,
    MONGO_PORT,
    MONGO_IP,
    MONGO_PASS,
    MONGO_USER,
    CHANNEL_REPORT,
    OWNER,
)
from telegramnews.database import Database, MongoDatabase
from telegramnews.parser import NewsParser
from telegramnews.telegrambot import Telegram


def start_many(
    channels: List[Dict[str, any]],
    custom_parser: Dict[int, Type[NewsParser]],
    token: str,
    filter: int = 0,
    owner: int = 0,
    debug: bool = False,
    proxy: str = None,
    mongo: Dict[str, Union[str, int]] = None,
    error: int = None,
):
    """

    :param channels:
    :param custom_parser: Dictionary with inherited classes of NewsParser
    :param token: Telegram bot token
    :param filter: Can filter
    :param owner:
    :param debug:
    :param proxy:
    :param mongo:
    :param error:
    """
    processes = []
    kwargs = {
        "owner": owner,
        "debug": debug,
        "proxy": proxy,
        "mongo": mongo,
        "error": error,
    }
    for idx, site in enumerate(channels):
        processes.append(
            run_process(
                site, filter if filter else idx + 1, custom_parser, token, kwargs
            )
        )
    start_watchdog(processes, token, kwargs)


def run_process(site, idx, custom_parsers, token, kwargs):
    counter = Value("i", 0)
    sleeper = Value("i", 300)
    args = (site, counter, sleeper, token)
    if idx not in custom_parsers:
        p = Process(target=NewsParser, args=args, kwargs=kwargs)
    else:
        p = Process(target=custom_parsers[idx], args=args, kwargs=kwargs)
    if "channels" not in site and not kwargs["debug"]:
        return
    p.start()
    return {
        "process": p,
        "counter": counter,
        "sleeper": sleeper,
        "warning": 0,
        "old_counter": 0,
        "site": site,
        "id": idx,
    }


def start_watchdog(processes, token, kwargs):
    time.sleep(300)
    i = 0
    while True:
        time.sleep(31)
        i += 1
        check_process(processes, PARSER, i, token, kwargs)


def check_process(processes, custom_parsers, iteration, token, kwargs):
    for idx, p in enumerate(processes):
        # Skip if poll interval high and process still sleeping
        if 30 / p["sleeper"].value * iteration % 1:
            continue
        if p["counter"].value > p["old_counter"]:
            p["old_counter"] = p["counter"].value
            p["warning"] = 0
        else:
            logging.info(p)
            p["warning"] += 1
            if p["warning"] > 3:
                p["process"].terminate()
                if p["warning"] > 5:
                    p["process"].kill()
                processes.append(
                    run_process(p["site"], p["id"], custom_parsers, token, kwargs)
                )
                logging.error("Process restarted: " + p["site"]["name"])
                del processes[idx]


def get_channels(path, filter=0, all=False):
    with open("{}/{}".format(os.path.dirname(sys.argv[0]), path)) as json_file:
        channels = list(json.load(json_file).values())
    if filter:
        return [channels[filter]]
    return channels if all else channels[1:]


def set_channels(channels, path, not_all=True):
    if not_all:
        channels = get_channels(path, all=True)[0] + channels
    with open(
        "{}/{}".format(os.path.dirname(sys.argv[0]), path), "w", newline="\n"
    ) as outfile:
        json.dump(
            {
                "{:03.0f}".format(idx + 1): channel
                for idx, channel in enumerate(channels)
            },
            outfile,
            sort_keys=True,
            indent=4,
        )


def crawl_members(url):
    time.sleep(0.2)
    private = True
    if not url[:4] == "http":
        url = "https://t.me/" + url
        private = False
    try:
        if PROXY:
            html = requests.get(url, proxies={"https": PROXY}).text
        else:
            html = requests.get(url, proxies={"https": PROXY}).text
        members_str = re.findall('tgme_page_extra">([0-9]+) members</div>', html)[0]
        members = int(members_str) - 2
        return {"current_members": members, "private": private}
    except IndexError:
        logging.info(url)
        return None


def count_all_subchannels(channels, serious):
    db = MongoDatabase(
        {"port": MONGO_PORT, "ip": MONGO_IP, "pass": MONGO_PASS, "user": MONGO_USER}
    )
    result = {}
    for channel in channels:
        channel_infos = crawl_members(channel["link"])
        if channel_infos is None:
            continue
        channel_infos["old_members"] = db.get_members(channel["id"])
        if serious:
            db.set_members(channel_infos["current_members"], channel["id"])
        result[channel["id"]] = channel_infos
    return result


def get_counted_list(serious):
    channels = {"sites": {}}
    for site in get_channels("channels.json", all=True):
        sub_channels = count_all_subchannels(site["channels"], serious)
        absolut_members = sum(
            channel["current_members"] for channel in sub_channels.values()
        )
        absolut_old_members = sum(
            channel["old_members"] for channel in sub_channels.values()
        )
        all_private = all(channel["private"] for channel in sub_channels.values())
        channels["sites"][site["name"]] = {
            "channels": sub_channels,
            "absolut_members": absolut_members,
            "absolut_old_members": absolut_old_members,
            "all_private": all_private,
        }
    channels["absolut_members"] = sum(
        channel["absolut_members"] for channel in channels["sites"].values()
    )
    channels["absolut_old_members"] = sum(
        channel["absolut_old_members"] for channel in channels["sites"].values()
    )
    return channels


def render_text(channels):
    sorted_channel_list = OrderedDict(
        sorted(
            (channels["sites"]).items(),
            key=lambda kv: kv[1]["absolut_members"],
            reverse=True,
        )
    )
    l = str(len(str(max([x["absolut_members"] for x in sorted_channel_list.values()]))))
    result = ["" for x in range((int(len(sorted_channel_list) / 98) + 1))]
    result[0] = "*Tagesbericht DerNewsChannel*\n\n"
    for idx, (name, channel) in enumerate(sorted_channel_list.items()):
        diff = calc_diff(channel["absolut_members"], channel["absolut_old_members"])
        result[int((idx + 1) / 98)] += "`{{:2}}. {{}} {{:{}}} {{:2}} {{}}`\n".format(
            l
        ).format(
            idx + 1,
            "p" if channel["all_private"] else " ",
            channel["absolut_members"],
            diff,
            name[:15],
        )
    complete_diff = calc_diff(
        channels["absolut_members"], channels["absolut_old_members"]
    )
    result[-1] += "\nGesamt: *{}* {}".format(
        channels["absolut_members"],
        "({})".format(complete_diff) if complete_diff else "",
    )
    return result


def send_stats(serious=False):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    text = render_text(get_counted_list(serious))
    for t in text:
        tg.send(t, CHANNEL_REPORT, {}, disable_notification=True)


def calc_diff(new, old):
    diff = ""
    idiff = new - old
    if idiff > 0:
        diff = "+{}".format(idiff)
    elif idiff < 0:
        diff = "-{}".format(idiff * -1)
    return diff


def short_list(md=False):
    result = (
        "Perferkt formatierte News auf Telegram mit [@DerNewsChannel](https://t.me/dernewschannel)!\n\nFolgende "
        "Kanäle werden zurzeit angeboten:\n"
    )
    for site in sorted(get_channels("channels.json"), key=lambda k: k["name"].upper()):
        for channel in sorted(
            site["channels"], key=lambda k: k["template"], reverse=True
        ):
            if channel["template"] in [3, 4, 6]:
                if not channel["link"][:4] == "http":
                    if not md:
                        result += "@" + channel["link"].replace("_", "\_")
                    else:
                        result += "* [{}](https://t.me/{})".format(
                            site["name"], channel["link"]
                        )
                else:
                    if md:
                        result += "* "
                    result += "[{}]({})".format(
                        site["name"].replace("_", "\_"), channel["link"]
                    )
            elif channel["template"] == 5:
                result += " ([Instant ⚡]({}))️".format(channel["link"])
        result += "\n"
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    print(result.replace("\_", "_"))
    if not md:
        tg.send(result, OWNER, [])


def send_summary(serious=False, full=False, fast=False):
    db = Database()
    articles = [
        {"name": a[2].replace(".", "․"), "count": a[1], "date": a[0], "added": a[3]}
        for a in db.count_articles()
    ]
    site_list = {}
    date_list = {}
    results = {}
    for a in articles:
        if a["name"] not in site_list:
            site_list[a["name"]] = []
            date_list[a["name"]] = a["added"]
        site_list[a["name"]].append(
            {
                "count": a["count"],
                "date": datetime.datetime.strptime(a["date"], "%Y-%m-%d"),
            }
        )
    for site, data in sorted(site_list.items(), key=lambda s: s[0].lower()):
        try:
            channels = next(
                (
                    item
                    for item in get_channels("channels.json")
                    if item["name"] == site.replace("․", ".")
                ),
                None,
            )
        except TypeError:
            continue
        if not channels:
            continue
        mini = min([x["date"] for x in data])
        days = abs((datetime.datetime.today() - mini).days)
        counter = 0
        for x in data:
            counter += x["count"]
        if channels["category"] not in results:
            results[channels["category"]] = "*{}*\n".format(
                channels["category"].upper()
            )
        if (
            datetime.datetime.today()
            - datetime.datetime.strptime(date_list[site], "%Y-%m-%d")
        ).days > 13:
            results[channels["category"]] += "\n*{}* (Ø{} M/Tag)\n".format(
                site, "{:.1f}".format(counter / days).replace(".", ",")
            )
        else:
            results[channels["category"]] += "\n*{} 🆕* \n".format(site)
        try:
            for channel in channels["channels"]:
                try:
                    members = ""  # "[[{}👥]]".format(db.get_members(channel['id'])[0])
                except IndexError:
                    members = ""
                if not channel["link"][:4] == "http":
                    results[channels["category"]] += "@{} {}\n".format(
                        channel["link"].replace("_", "\_"), members
                    )
                elif channel["template"] == 5:
                    results[
                        channels["category"]
                    ] += "[⚡️ Instant View]({}) {}\n".format(channel["link"], members)
                else:
                    results[channels["category"]] += "[{}]({}) {}\n".format(
                        site, channel["link"], members
                    )
        except TypeError:
            continue
    other = "*SONSTIGES*\n\n"
    tg = Telegram()
    for category, text in sorted(results.items()):
        if text.count("\n") < 6:
            other += text + "\n\n"
            results.pop(category, "")
    results["Sonstiges"] = other
    results["Info"] = INFO
    ORDER = ["News", "Technik"]
    i = 0
    channel_id = CHANNEL_TEST if not serious else CHANNEL_MAIN
    send = (
        ["Info", "Sonstiges"]
        + [c for c in results if c not in ORDER and not c in ["Sonstiges", "Info"]]
        + ORDER[::-1]
    )
    if not full:
        for category in send:
            i = tg.edit(
                i,
                results[category],
                channel_id,
                {},
                disable_notification=True,
                disable_preview=True,
            )
    else:
        tg.unpin(channel_id)
        if not fast:
            time.sleep(300)
        first = tg.send(
            results[send[0]],
            channel_id,
            {},
            disable_notification=True,
            disable_preview=True,
        )
        for i in reversed(range(first)):
            tg.delete(i, channel_id)
        tg.pin(channel_id, first)
        time.sleep(5)
        for category in send[1:]:
            tg.send(
                results[category],
                channel_id,
                {},
                disable_notification=True,
                disable_preview=True,
            )
            time.sleep(1)


def clean_up_db():
    db = Database()
    db.clean_up()


def migrate_news(mongo, news):
    logging.info("Migrate %s news", len(news))
    for idx, d in enumerate(news):
        try:
            if not int(idx % (len(news) / 100)):
                logging.info("Migrate %s%%", int(100 / len(news) * idx) + 1)
            id, s_id, title, text, link, img, date, tags, m_id, site_name = d
            if mongo.quick_check_news(link, link, site_name):
                mongo.check_news(title, text, link, site_name)
                continue
            x = mongo.insert_news(
                title, text, link, img, site_name, date=date, tags=tags
            )
            mongo.update_message_id(x, m_id)
        except Exception as e:
            logging.error(d)
            logging.exception(e)


def migrate_channels(mongo, channels):
    logging.info("Migrate %s channels", len(channels))
    for c in channels:
        c_id, s_id, t_id, link, short, public, pinned, template, added, members, name = (
            c
        )
        mongo.insert_channel(
            name,
            t_id,
            link,
            template,
            public,
            short,
            added=added,
            members=members,
            pinned=pinned,
        )


def migrate_sites(mongo, sites):
    logging.info("Migrate %s sites", len(sites))
    for s in sites:
        s_id, name, rss, added = s
        mongo.insert_site(name, rss, added=added)


def migrate_to_mongo():
    db = Database()
    mongo = MongoDatabase()
    sites, channels, news = db.get_all()
    print(len(sites), len(channels), len(news))
    migrate_sites(mongo, sites)
    migrate_channels(mongo, channels)
    migrate_news(mongo, news)


def update_channel_names(path="channels.json"):
    tg = Telegram(BOT_TOKEN, proxy=PROXY)
    sites = get_channels(path)
    for key, site in enumerate(sites):
        if "channels" not in site:
            continue
        for idx, channel in enumerate(site["channels"]):
            time.sleep(1)
            try:
                channel_link = tg.get_chat_infos(channel["id"])
            except:
                logging.warning("Timed out " + site["name"])
                channel_link = channel["link"]
            if (
                not (channel_link[:4] == "http" and channel["link"][:4] == "http")
                and not channel_link == channel["link"]
            ):
                print(channel_link, channel["link"])
                sites[key]["channels"][idx]["link"] = channel_link
    set_channels(sites, path)


def init():
    db = MongoDatabase(
        {"port": MONGO_PORT, "ip": MONGO_IP, "pass": MONGO_PASS, "user": MONGO_USER}
    )
    main_channel = get_channels("channels.json", all=True)[0]
    db.insert_main_channel(
        main_channel["channels"][0]["id"], main_channel["channels"][0]["link"], True, 0
    )


def start_tool(arg):
    if arg == "--stats":
        send_stats()
    elif arg == "--stats-live":
        send_stats(serious=True)
    elif arg == "--mstats":
        send_summary()
    elif arg == "--mstats-live":
        send_summary(serious=True)
    elif arg == "--mstats-full":
        send_summary(full=True)
    elif arg == "--mstats-fast":
        send_summary(fast=True, full=True)
    elif arg == "--mstats-live-full":
        send_summary(serious=True, full=True)
    elif arg == "--clean":
        clean_up_db()
    elif arg == "--short-list":
        short_list()
    elif arg == "--migrate":
        migrate_to_mongo()
    elif arg == "--short-list-md":
        short_list(md=True)
    elif arg == "--update-channel-names":
        update_channel_names()
    elif arg == "--init":
        init()
