import sys
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import pandas as pd
import numpy as np
import logging
import json
import re
import hashlib
from Outer import *
from PandasGrapher import *
from PPDFFilterer import *
from PPStator import *
from DFFormatter import *

#--------------------------------------------------------------------------------------
class PPFramor() :

  pd.options.display.float_format = '{:.0f}'.format

  #--------------------------------------------------------------------------------------
  def __init__(self,param,file,tsm=None) :
    self.param=param
    self.tsm=tsm
    self.p=self.param.getAll()
    logging.warning("PPFramor begins")
    self.file=file
    self.decimal='.'
    self.ppregex=self.p['ppregex']
    self.ppregexclude=self.p['ppregexclude']
    self.percentiles=self.p['percentiles']
    self.setRawdatas()
    self.report()
    logging.warning("PPFramor ends")

#--------------------------------------------------------------------------------------
  def getRawdatas(self) :
    return(self.rawdatas)

#--------------------------------------------------------------------------------------
  def getDatas(self) :
    return(self.datas)
#--------------------------------------------------------------------------------------
  def getFile(self) :
    return(self.file)

#--------------------------------------------------------------------------------------
  def setRawdatas(self) :
    self.p['datafile']=self.file
    self.DFF=DFFormatter.getFromFactory(self.p['type'],self.p)

    ppf=PPDFFilterer(self.DFF.getDf(),self.param)
    self.datas=ppf.getDfOK()
    self.ko=ppf.getDfKO()
    self.tsmList=self.datas['ts10m'].unique()
    if self.tsm is not None:
      self.datas=self.datas[ self.datas['ts10m'] == self.tsm ]
    self.rawdatas=self.datas.groupby('PurePath')['ResponseTime'].describe(percentiles=self.percentiles)

#--------------------------------------------------------------------------------------
  def report(self) :
    self.p['out'].h2("PP from " + self.file)
    self.ppStator=PPStator(self.param,self.datas)
    self.ppStatorKO=PPStator(self.param,self.ko)
    with pd.option_context('display.max_rows', None, 'display.max_colwidth', 0, 'display.float_format','{:.2f}'.format) :
      self.p['out'].out("Summary stats",self.ppStator.getXstats(),escape=False)
      self.p['out'].out("Error stats",self.ppStatorKO.getXstats(),escape=False)

#--------------------------------------------------------------------------------------
  def getTsmlist(self,tsm) :
    return(self.tsmList)

#--------------------------------------------------------------------------------------
class PPComparator() :

  #--------------------------------------------------------------------------------------
  def __init__(self,param,f1,f2) :
    self.param=param
    self.p=self.param.getAll()
    self.grapher=PandasGrapher(self.param)
    self.f1=f1
    self.f2=f2
    self.df1=f1.getRawdatas()
    self.df2=f2.getRawdatas()
    self.go()

  @staticmethod
  #--------------------------------------------------------------------------------------
  def deltaCountPe(x) :
    return(PPComparator.deltaPe('count',x))

  @staticmethod
  #--------------------------------------------------------------------------------------
  def deltaStdPe(x) :
    return(PPComparator.deltaPe('std',x))

  @staticmethod
  #--------------------------------------------------------------------------------------
  def deltaMeanPe(x) :
    return(PPComparator.deltaPe('mean',x))

  @staticmethod
  #--------------------------------------------------------------------------------------
  def deltaP50Pe(x) :
    return(PPComparator.deltaPe('50%',x))

  @staticmethod
  #--------------------------------------------------------------------------------------
  def deltaP95Pe(x) :
    return(PPComparator.deltaPe('95%',x))

  @staticmethod
  #--------------------------------------------------------------------------------------
  def deltaPe(field,x) :
    kx=field+"_x"
    ky=field+"_y"
    if x[kx] > 0 :
      return(100*(x[ky]-x[kx])/x[kx])
    return(0)

  #--------------------------------------------------------------------------------------
  def go(self) :
    #print(self.df1)
    #print(self.df2)
    dfm=pd.merge(self.df1,self.df2,on='PurePath',how='outer')
    dfm.reset_index(inplace=True)
    dfm['DMax']=dfm.apply(lambda x: x['max_y'] - x['max_x'],axis=1 )
    dfm['DStd']=dfm.apply(lambda x: x['std_y'] - x['std_x'],axis=1 )
    dfm['DCount']=dfm.apply(lambda x: x['count_y'] - x['count_x'],axis=1 )
    dfm['DMean']=dfm.apply(lambda x: x['mean_y'] - x['mean_x'],axis=1 )
    dfm['DP50']=dfm.apply(lambda x: x['50%_y'] - x['50%_x'],axis=1 )
    dfm['DP95']=dfm.apply(lambda x: x['95%_y'] - x['95%_x'],axis=1 )
    dfm['DStd%']=dfm.apply(PPComparator.deltaStdPe,axis=1 )
    dfm['DCount%']=dfm.apply(PPComparator.deltaCountPe,axis=1 )
    dfm['DMean%']=dfm.apply(PPComparator.deltaMeanPe,axis=1 )
    dfm['DP50%']=dfm.apply(PPComparator.deltaP50Pe,axis=1 )
    dfm['DP95%']=dfm.apply(PPComparator.deltaP95Pe,axis=1 )
    dfm.fillna(value=0,inplace=True)
    filter=True
    if filter :
      #dfm=dfm[(dfm['count_x'] > self.p['autofocuscount']) | (dfm['count_y'] > self.p['autofocuscount']) ]
      #dfm=dfm[(dfm['mean_x'] > self.p['autofocusmean']) | (dfm['mean_y'] > self.p['autofocusmean']) ]
      #dfm=dfm[(abs(dfm['DMeanPe']) > 10)]
      pass
    #logging.warning(dfm)
    if dfm.size == 0 :
      return
    with pd.option_context('display.max_rows', None) :
      self.p['out'].h2("Comparison of requests : x=" + self.f1.getFile() + " y=" + self.f2.getFile())
      self.p['out'].out("Compare",dfm[['PurePath','count_x','count_y','DCount','DCount%',
          'mean_x','mean_y','DMean','DMean%',
          'std_x','std_y','DStd','DStd%',
          '50%_x','50%_y','DP50','DP50%',
          '95%_x','95%_y','DP95','DP95%',
          'max_x','max_y','DMax']],
          escape=False,classes='tablePPcompareProcessor')
      self.graphIt(self.f1.getFile(),self.f1.getDatas())
      self.graphIt(self.f2.getFile(),self.f2.getDatas())

  #--------------------------------------------------------------------------------------
  def graphIt(self,title,df) :
      self.dfall=pd.DataFrame(df.groupby(self.p['timeGroupby'])['StartTime'].count().apply(lambda x: 0))
      self.grapher.setDfall(self.dfall)
      dg=df.groupby(self.p['timeGroupby'])['ResponseTime']
      self.p['out'].h3("Time view for " + title)
      self.grapher.graphBasicsNew("Time view " + title, dg, [
      { 'aggr' : 'Max', 'dgaggr' : dg.max(), 'color' : 'red'},
      { 'aggr' : 'Mean', 'dgaggr' : dg.mean(), 'color' : 'green'},
      { 'aggr' : 'Q50', 'dgaggr' : dg.quantile(0.5), 'color' : 'green'},
      { 'aggr' : 'Q95', 'dgaggr' : dg.quantile(0.95), 'color' : 'green'},
      ])

#--------------------------------------------------------------------------------------
class PPCompareProcessor() :

  #--------------------------------------------------------------------------------------
  def __init__(self,param) :
    self.param=param
    self.p=self.param.getAll()
    self.p['out'].setPPdiv("PPCompareProcessor")
    self.dfs=[]
    logging.warning(self.p)
    self.dfs.append(PPFramor(self.param,self.p['file1']))
    if ('file2' in self.p) & (self.p['file2'] is not None) :
      self.p['out'].h1('PPCompareProcessor compare ' + self.p['file1'] + ' and ' + self.p['file2'])
      #self.dfs.append(PPFramor(self.param,self.p['file1']))
      self.dfs.append(PPFramor(self.param,self.p['file2']))
    else :
      self.dfs.append(PPFramor(self.param,self.p['file1']))
      #self.p['out'].h1('PPCompareProcessor autocompare ' + self.p['file1'])
      #fr=PPFramor(self.param,self.p['file1'])
      #for tsm in fr.getTsmlist('ts10m') :
      #  self.dfs.append(PPFramor(self.param,self.p['file1'],tsm))
      pass

  #--------------------------------------------------------------------------------------
  def setBehavior(self) :
    self.ppregex=self.p['ppregex']
    self.ppregexclude=self.p['ppregexclude']
    pd.options.display.float_format = '{:.0f}'.format

  #--------------------------------------------------------------------------------------
  def go(self) :
    for i in range(0,len(self.dfs)-1) :
      PPComparator(self.param,self.dfs[i],self.dfs[i+1])


"""add BackTestTrade

Revision ID: af41b751b50f
Revises: d194e8be4f48
Create Date: 2022-01-16 03:16:51.583115

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'af41b751b50f'
down_revision = 'd194e8be4f48'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('beck_test_trades',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('symbol', sa.String(), nullable=False),
    sa.Column('buy_time', sa.String(), nullable=False),
    sa.Column('sell_time', sa.String(), nullable=False),
    sa.Column('profit', sa.Float(), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    op.alter_column('trades', 'createdAt', existing_type=sa.Integer(), type_=sa.BigInteger())
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('beck_test_trades')
    op.alter_column('trades', 'createdAt', existing_type=sa.BigInteger(), type_=sa.Integer())
    # ### end Alembic commands ###


import serial
import array

class SerialPort (serial.Serial):

    def __init__(self):
        serial.Serial.__init__(self)

        settings = self.getSettingsDict()
        self.applySettingsDict(settings)
        self.baudrate = 115200
        self.port = "COM9"

        pass

    def calculate_crc(self, data):
        crc = 0
        for x in data:
            crc += x

        return crc

    def transmit_binary(self, data):
        #
        # http://stackoverflow.com/questions/472977/binary-data-with-pyserialpython-serial-port
        #

        transmit = array.array('B', data).tostring()
        print(transmit)
        self.write(transmit)

    def send_ping(self):
        packet = [0x0C, 0x03, 0xF0, 0xAA, 0x00]
        print(packet)
        crc = self.calculate_crc(packet)
        crc = crc.to_bytes(2, byteorder='big')
        print(crc)
        for x in crc:
            print(x)
            packet.append(x)
        packet.append(0x0D)
        print(packet)
        self.transmit_binary(packet)

    def character(self, b):
        return b.decode('latin1')

    def send_help(self):
        packet = [0x68, 0x65, 0x6C, 0x70, 0x0D]
        self.transmit_binary(packet)
        while (1):
            data = self.character(self.read(1))
            print(data)


import json
import numpy as np
import cv2

def test_load_contours():
    with open('data/cilia/predictions/nuclei/RPE1-GFPCep135_NC_48h_100k_mCETN2-AF488_rArl13b-AF568_gCEP164-AF647_1_MMStack_10-Pos_000_001_max_C0.json', 'r') as f:
        contours = json.load(f)
    assert type(contours) == dict
    cnts = [np.array(cnt['contour']) for cnt in contours.values()]
    bg = np.zeros((2048, 2048), dtype='uint8')
    cv2.drawContours(bg, cnts, -1, 255, -1)
    cv2.imwrite('../out/test.png', bg)


from django.db.models import Q, Sum
from django.http import HttpResponseRedirect
from django.views import View
from django.shortcuts import render
from django import forms
from django.urls import reverse_lazy
from stock.models import Stock, RebookOrder, RebookOrderItem, RebookOrderItemDestinationStock
from django.contrib.auth.models import User
from django.views import generic
from django.contrib.auth.mixins import LoginRequiredMixin

from stock.rebook import RebookOnPositionOverviewBase, BookStockOnPositionBase, BookStockOnPositionFormBase

class RebookOrderForm(forms.ModelForm):
    class Meta:
        model = RebookOrder
        fields = ["user"]
        widgets = {
            'user': forms.RadioSelect(),
        }

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.fields['user'].required = True

class RebookOrderItemForm(forms.ModelForm):
    class Meta:
        model = RebookOrderItem
        fields = ["amount"]

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        for visible in self.visible_fields():
            visible.field.widget.attrs["class"] = "form-control"
        self.fields['amount'].required = True

    def clean_amount(self):
        amount = self.cleaned_data['amount']
        print(f"{self.instance.stock}")
        if amount > self.instance.stock.bestand:
            self.add_error("amount", "Umzubuchende Menge kann nicht größer als der Bestand sein")
        if amount <= 0:
            self.add_error("amount", "Umzubuchende Menge muss mindestens 1 sein")
        return amount

class AssignRebookOrderView(LoginRequiredMixin, View):
    template_name = "rebook/order/accept_rebook.html"
    title = "Umbuchauftrag anlegen"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.stocks = None
        self.context = None
        self.form = None
        self.rebookitem_forms = None

    def dispatch(self, request, *args, **kwargs):
        self.stocks = Stock.objects.filter(pk__in=request.GET.getlist("item"))
        self.form = self.get_form()
        self.rebookitem_forms = self.get_rebookitem_forms()
        self.context = self.get_context()
        return super().dispatch(request, *args, **kwargs)

    def get_rebookitem_forms(self):
        forms = []
        index = 0
        for stock in self.stocks:
            data = {}
            if self.request.method == "POST":
                amount = self.request.POST.getlist("amount")[index]
                data["amount"] = amount
            form = RebookOrderItemForm(data=data)
            form.instance.stock = stock
            form.instance.position = stock.lagerplatz
            form.instance.sku = stock.sku_instance
            forms.append((form, stock))
            index += 1
        return forms

    def get_form(self):
        if self.request.method == "POST":
            form = RebookOrderForm(data=self.request.POST)
        else:
            form = RebookOrderForm()
        return form

    def get_context(self):
        context = {"title": self.title, "stocks": self.stocks, "form": self.form,
                   "rebookitem_forms": self.rebookitem_forms}
        return context

    def get(self, request, *args, **kwargs):
        print(f"Hey dead: ")
        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        if self.form.is_valid() is True:
            has_invalid_form = None
            for form, stock in self.rebookitem_forms:
                print(f"3anlash: {request.method}")
                print(f"sam: {form.data}")
                if form.is_valid() is False:
                    has_invalid_form = True
            if has_invalid_form is True:
                return render(request, self.template_name, self.context)
            else:
                self.save_rebook_order()
                return HttpResponseRedirect(reverse_lazy("stock:list"))
        else:
            return render(request, self.template_name, self.context)

    def save_rebook_order(self):
        rebook_order_instance = self.form.save()
        for rebookitem_form, stock in self.rebookitem_forms:
            rebookitem_form_instance = rebookitem_form.save(commit=False)
            rebookitem_form_instance.rebook_order = rebook_order_instance
            rebookitem_form_instance.save()

class RebookOrderListView(LoginRequiredMixin, generic.ListView):
    template_name = "rebook/order/rebook_order_list.html"
    paginate_by = 15
    title = "Umbuchaufträge Übersicht"

    def get_queryset(self):
        queryset = RebookOrder.objects.filter(user=self.request.user)
        return queryset

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context.update(self.get_context())
        return context

    def get_context(self):
        return {"title": self.title}

class RebookOrderView(LoginRequiredMixin, View):
    template_name = "rebook/order/rebook_order.html"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.rebook_order = None
        self.rebook_order_items = None
        self.item = None
        self.context = None

    def dispatch(self, request, *args, **kwargs):
        self.rebook_order = RebookOrder.objects.get(pk=self.kwargs.get("pk"))
        self.rebook_order_items = []
        for item in self.rebook_order.rebookorderitem_set.all():
            rebooked_amount = item.rebookorderitemdestinationstock_set.all().aggregate(
                total=Sum("rebooked_amount")).get("total", 0) or 0
            self.rebook_order_items.append((item, rebooked_amount))

        self.context = self.get_context()
        return super().dispatch(request, *args, **kwargs)

    def get(self, request, *args, **kwargs):
        return render(request, self.template_name, self.context)

    def get_context(self):
        return {"title": f"Umbuchauftrag {self.rebook_order.pk}", "rebook_order": self.rebook_order,
                "rebook_order_items": self.rebook_order_items}

class RebookOrderRebookOnPositionOverview(RebookOnPositionOverviewBase):
    template_name = "rebook/order/rebook_on_position_list.html"

    def get_position_from_GET_or_session(self):
        get_value = self.request.GET.get("position", "") or ""

        if get_value != "":
            self.request.session["rebook_order_position"] = get_value
            return get_value.strip()
        else:
            return (self.request.session.get("rebook_order_position", "") or "").strip()

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.rebook_order = None
        self.item = None

    def pre_dispatch(self):
        self.rebook_order = RebookOrder.objects.get(pk=self.kwargs.get("pk"))
        self.item = RebookOrderItem.objects.get(pk=self.kwargs.get("item_pk"))
        self.stock = Stock.objects.get(pk=self.item.stock.pk)
        self.position = self.get_position_from_GET_or_session()

    def redirect(self):
        if self.item.rebooked is True and self.item.amount == self.item.rebooked_amount:
            return HttpResponseRedirect(reverse_lazy("stock:rebook_order", kwargs={"pk": self.rebook_order.pk}))

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["rebook_order"] = self.rebook_order
        context["rebook_order_item"] = self.item
        rebooked_amount = self.item.rebookorderitemdestinationstock_set.all().aggregate(
            total=Sum("rebooked_amount")).get("total", 0) or 0
        context["rebook_amount"] = self.item.amount-rebooked_amount or 0
        return context

class BookStockOrderOnPositionForm(BookStockOnPositionFormBase):
    def __init__(self, item=None, **kwargs):
        super().__init__(**kwargs)
        self.item = item
        self.stock = self.item.stock

    def get_amount_limit(self):
        rebooked_amount = self.item.rebookorderitemdestinationstock_set.all().aggregate(
            total=Sum("rebooked_amount")).get("total", 0) or 0
        rebooked_amount = self.item.amount-rebooked_amount or 0
        return rebooked_amount

class RebookOrderBookStockOnPosition(BookStockOnPositionBase):
    template_name = "rebook/order/book_stock_on_position.html"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.rebook_order = None
        self.item = None
        self.destination_stock = None
        self.item_destination_stock = None

    def pre_dispatch(self):
        self.rebook_order = RebookOrder.objects.get(pk=self.kwargs.get("pk"))
        self.item = RebookOrderItem.objects.get(pk=self.kwargs.get("item_pk"))
        self.stock = self.item.stock

    def get_context(self):
        context = super().get_context()
        context["rebook_order"] = self.rebook_order
        context["rebook_order_item"] = self.item
        rebooked_amount = self.item.rebookorderitemdestinationstock_set.all().aggregate(
            total=Sum("rebooked_amount")).get("total", 0) or 0
        context["rebook_amount"] = self.item.amount-rebooked_amount or 0
        return context

    def pre_rebook_stock_to_destination(self, destination_stock, destination_sku, destination_position):
        if self.item is not None:
            self.item_destination_stock = RebookOrderItemDestinationStock(
                destination_stock=destination_stock, destination_position=destination_position,
                destination_sku=destination_sku, rebook_order_item=self.item,
                rebooked_amount=self.form.cleaned_data.get("amount"))

    def post(self, request, *args, **kwargs):
        if self.stock.id is None:
            self.item.stock = None
        self.item_destination_stock.save()
        total_amount = self.item.rebookorderitemdestinationstock_set.all().aggregate(
            total=Sum("rebooked_amount")).get("total")
        if self.item.amount == total_amount:
            self.item.rebooked = True
            self.item.save()
        self.finish_rebook_order_if_finished()
        return HttpResponseRedirect(reverse_lazy("stock:rebook_order", kwargs={"pk": self.rebook_order.pk}))

    def finish_rebook_order_if_finished(self):
        items = self.rebook_order.rebookorderitem_set.all()
        for item in items:
            rebooked_amount = 0
            for destination_stock_item in item.rebookorderitemdestinationstock_set.all():
                rebooked_amount += destination_stock_item.rebooked_amount or 0
            if rebooked_amount != item.amount or item.rebooked is not True:
                return

        if items.count() > 0:
            self.rebook_order.completed = True
            self.rebook_order.save()

    def get_form(self):
        if self.request.method == "POST":
            self.form = BookStockOrderOnPositionForm(data=self.request.POST, item=self.item)
        else:
            self.form = BookStockOrderOnPositionForm(item=self.item)
        return self.form


from scipy.stats import kurtosis, entropy

import mne
from mne.utils import _check_preload
import numpy as np
import matplotlib.pyplot as pl

import logging
logger = logging.getLogger(__name__)

def mark_tms_bad(raw, events, tmin=-0.002, tmax=0.01):
    ""
    #eog_events = mne.preprocessing.find_eog_events(raw)
    onsets = events[:, 0] / raw.info['sfreq'] + tmin
    durations = [tmax] * len(events)
    descriptions = ['bad artifact'] * len(events)
    tms_annot = mne.Annotations(onsets, durations, descriptions,
                                orig_time=raw.info['meas_date'])
    raw.set_annotations(tms_annot)

    return raw

def interpolate_tms_pulse(inst, tmin=-0.002, tmax=0.01, order=3, points=1):
    """[summary]

    Parameters
    ----------
    inst : [type]
        [description]
    tmin : float, optional
        [description], by default -0.002
    tmax : float, optional
        [description], by default 0.01
    order : int, optional
        [description], by default 3
    points : int, optional
        [description], by default 1

    Returns
    -------
    [type]
        [description]
    """

    _check_preload(inst, 'interpolate')

    t = inst.times
    mask_artifact = np.logical_and(t > tmin, t < tmax)

    t_interp_min = tmin - points / inst.info['sfreq']
    t_interp_max = tmax + points / inst.info['sfreq']

    mask_interp = np.logical_and(t >= t_interp_min,
                                 t <= t_interp_max)

    mask_interp = np.logical_and(np.logical_not(mask_artifact),
                                 mask_interp)

    epoch_data = inst.get_data()
    inst._data = epoch_data

    for epoch in epoch_data:
        for ch, data in enumerate(epoch):
            coeffs = np.polyfit(t[mask_interp], data[mask_interp], deg=order)
            poly = np.poly1d(coeffs)
            x_fit = poly(t[mask_artifact])

            epoch[ch, mask_artifact] = x_fit

    return inst

def read_bids_events(fname):

    events_file = np.recfromcsv(fname, delimiter='\t')

    events_name = np.unique(events_file['trial_type'])
    events_id = {k: j for j, k in enumerate(events_name)}

    events = list()
    for field in ['sample', 'duration', 'trial_type']:
        if field != 'trial_type':
            events.append(events_file[field])
        else:
            arr = [events_id[k] for k in events_file[field]]
            events.append(np.array(arr))

    events = np.int_(np.vstack(events).T)

    return events, events_id

def reject_electrodes(raw, method='kurtosis', threshold=5):
    """[summary]

    TODO: Insert the possibility to avoid normalization.

    Parameters
    ----------
    raw : [type]
        [description]
    method : str, optional
        [description], by default 'kurtosis'
    threshold : int, optional
        [description], by default 5

    Returns
    -------
    [type]
        [description]
    """

    _check_preload(raw, 'reject_electrodes')

    if method == 'kurtosis':
        metric = kurtosis(raw._data, axis=1)

    elif method == 'probability':
        metric = entropy(raw._data, axis=1)

    tmp_metric = np.sort(metric)
    idx = int(np.round(metric.shape[0] * .1))
    tmp_metric = tmp_metric[idx:-idx]

    metric = (metric - np.mean(tmp_metric)) / np.std(tmp_metric)

    ch_mask = np.abs(metric) > threshold

    # TODO: Check for multiple times that this function is called
    raw.info['bads'] = list(np.array(raw.info['ch_names'])[ch_mask])

    logger.info('Channels removed: '+' '.join(raw.info['bads']))

    return raw

def reject_epochs(epochs, method='jointprob', threshold=5):
    return

def reject_jointprob(epochs, selected=None,
                     local_threshold=5,
                     global_threshold=5):
    """[summary]

    Parameters
    ----------
    epochs : [type]
        [description]
    selected : [type], optional
        [description], by default None
    local_threshold : int, optional
        [description], by default 5
    global_threshold : int, optional
        [description], by default 5

    Returns
    -------
    [type]
        [description]
    """

    _check_preload(epochs, 'reject_jointprob')
    data = epochs._data
    nepochs, nchan, ntimes = data.shape

    if selected is None:
        # Channels from bads must be deleted
        selected = np.arange(nchan)

    channel_data = np.moveaxis(data, 0, -1)

    joint_prob_e, rejected_e = jointprob(channel_data, local_threshold, jp=None, normalize=True)
    _, rejected_tmp = jointprob(data[:, selected], local_threshold, jp=joint_prob_e[selected], normalize=True)

    rejected_e = np.zeros((nchan, nepochs))
    rejected_e[selected, :] = rejected_tmp

    epoch_data = data.reshape(nepochs, -1)
    joint_prob, rejected = jointprob(epoch_data, global_threshold, jp=None, normalize=True)

    rejected = np.logical_or(rejected, np.sum(rejected_e, axis=1) != 0)

    rejected = np.logical_or(rejected, np.sum(rejected_e[selected], axis=1) != 0)

    return rejected

def jointprob(data, threshold=0, jp=None, normalize=True, nbins=1000):

    nchan, ntimes, nevents = data.shape
    if jp is None:
        jp = np.zeros((nchan, nevents))

        for ch in range(nchan):
            data_prob, pdf = real_probability(data[ch], nbins)

            for e in range(nevents):

                data_ = data_prob[e*ntimes:(e+1)*ntimes]
                jp[ch, e] = -1 * np.sum(np.log(data_))

        if normalize:
            jp = (jp - np.mean(jp)) / np.std(jp)

    rejected = np.abs(jp) > threshold

    return jp, rejected

def real_probability(data, nbins):

    if nbins > 0:
        size = data.shape[0] * data.shape[1]
        histogram = np.zeros((nbins))
        max_ = np.max(data)
        min_ = np.min(data)
        data = np.floor((data - min_) / (max_ - min_) * (nbins - 1))
        data = np.int_(data.flatten())

        for i in range(size):
            histogram[data[i]] += 1

        data_prob = histogram[data] / size
        histogram /= size
        pl.plot(histogram)
    else:
        data = (data - data.mean()) / data.std()
        data_prob = np.exp(-.5 * data * data) / (2 * np.pi)
        data_prob /= np.sum(data_prob)
        histogram = data_prob

    return data_prob, histogram

def remove_data(epoch, tmin, tmax, replacement='average'):
    """Removes data between defined timepoints and replaces data with
    different approaches:

    Parameters
    ----------
    epoch : [type]
        [description]
    tmin : [type]
        [description]
    tmax : [type]
        [description]
    replacement : str, optional
        [description], by default 'average'
    """
    return


import pytest

from crossandra.common import *
from crossandra.rule import *

# ESC = ESCAPED
# UC = UPPERCASE
# LC = LOWERCASE
# POS = POSITIVE
# NEG = NEGATIVE
# USCORE = UNDERSCORE

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("'test'", ("'test'", 6)),  # CLASSIC
        ("'''", ("''", 2)),  # STRING CONTAINING QUOTE
        ("test", NOT_APPLIED),  # NO QUOTE
        ("test'", NOT_APPLIED),  # NO STARTING QUOTE
        ("'test", NOT_APPLIED),  # NO ENDING QUOTE
        ("\\'test'", NOT_APPLIED),  # ESC STARTING QUOTE
        ("'test\\'", NOT_APPLIED),  # ESC ENDING QUOTE
        ("''", ("''", 2)),  # EMPTY
    ],
)
def test_SINGLE_QUOTED_STRING(string, result):
    assert SINGLE_QUOTED_STRING.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ('"test"', ('"test"', 6)),  # CLASSIC
        ('"""', ('""', 2)),  # STRING CONTAINING QUOTE
        ("test", NOT_APPLIED),  # NO QUOTE
        ('test"', NOT_APPLIED),  # NO STARTING QUOTE
        ('"test', NOT_APPLIED),  # NO ENDING QUOTE
        ('\\"test"', NOT_APPLIED),  # ESC STARTING QUOTE
        ('"test\\"', NOT_APPLIED),  # ESC ENDING QUOTE
        ('""', ('""', 2)),  # EMPTY
    ],
)
def test_DOUBLE_QUOTED_STRING(string, result):
    assert DOUBLE_QUOTED_STRING.apply(string) == result

# test_STRING <=> test_SINGLE_QUOTED_STRING and test_DOUBLE_QUOTED_STRING

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("'t'", ("'t'", 3)),  # CLASSIC
        ("'''", NOT_APPLIED),  # SINGLE QUOTE CHAR
        ("t", NOT_APPLIED),  # NO QUOTE
        ("t'", NOT_APPLIED),  # NO STARTING QUOTE
        ("'t", NOT_APPLIED),  # NO ENDING QUOTE
        ("\\'t'", NOT_APPLIED),  # ESC STARTING QUOTE
        ("'t\\'", NOT_APPLIED),  # ESC ENDING QUOTE
        ("''", NOT_APPLIED),  # EMPTY
    ],
)
def test_CHAR(string, result):
    assert CHAR.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("A", ("A", 1)),  # UC LETTER
        ("a", ("a", 1)),  # LC LETTER
        ("!", NOT_APPLIED),  # ASCII CHAR BETWEEN UC AND LC
        ("@", NOT_APPLIED),  # ASCII CHAR BEFORE UC
        ("|", NOT_APPLIED),  # ASCII CHAR AFTER LC
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_LETTER(string, result):
    assert LETTER.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("word", ("word", 4)),  # CLASSIC
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_WORD(string, result):
    assert WORD.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("0", (0, 1)),  # CLASSIC
        ("/", NOT_APPLIED),  # CHAR BEFORE DIGITS IN ASCII TABLE
        (":", NOT_APPLIED),  # CHAR AFTER DIGITS IN ASCII TABLE
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_DIGIT(string, result):
    assert DIGIT.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("69", (69, 2)),  # CLASSIC
        ("069", (69, 3)),  # LEADING ZERO
        ("1_000_000", (1000000, 9)),  # USCORE SEP
        ("1__0", (10, 4)),  # TWO USCORES AS SEP
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_INT(string, result):
    assert INT.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("+69", (69, 3)),  # POS INT
        ("-69", (-69, 3)),  # NEG INT
        ("69", NOT_APPLIED),  # NOT SIGNED
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_SIGNED_INT(string, result):
    assert SIGNED_INT.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("3.14", (3.14, 4)),  # CLASSIC
        ("3.0", (3.0, 3)),  # POST-COMMA TRAILING ZERO
        ("69.42", (69.42, 5)),  # N-DIGIT INT BASE
        ("0.92", (0.92, 4)),  # ZERO-DIGIT
        (".92", (0.92, 3)),  # IMPLICIT ZERO-DIGIT
        ("3.", (3.0, 2)),  # IMPLICIT POST-COMMA ZERO
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_DECIMAL(string, result):
    assert DECIMAL.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("3", (3, 1)),  # CLASSIC: DIGIT
        ("D", (13, 1)),  # CLASSIC: UC LETTER
        ("a", (10, 1)),  # CLASSIC: LC LETTER
        ("g", NOT_APPLIED),  # INVALID HEX VALUE
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_HEXDIGIT(string, result):
    assert HEXDIGIT.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("W", ("W", 1)),  # ONE LETTER
        ("_", ("_", 1)),  # ONE USCORE
        ("word", ("word", 4)),  # ONE WORD
        ("two_words", ("two_words", 9)),  # TWO WORDS, USCORE SEP
        ("_word", ("_word", 5)),  # 'PRIVATE' ONE WORD
        ("_two_words", ("_two_words", 10)),  # 'PRIVATE' TWO WORDS, USCORE SEP
        ("0word", NOT_APPLIED),  # LEADING DIGIT
        ("word0", ("word0", 5)),  # TRAILING DIGIT
        ("_0word", ("_0word", 6)),  # 'PRIVATE' LEADING DIGIT
        ("_word0", ("_word0", 6)),  # 'PRIVATE' TRAILING DIGIT
        ("0", NOT_APPLIED),  # SINGLE DIGIT
        ("69420", NOT_APPLIED),  # N-DIGITS
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_C_NAME(string, result):
    assert C_NAME.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("\n", ("\n", 1)),  # LF NEWLINE
        ("\r\n", ("\r\n", 2)),  # CRLF NEWLINE
        ("\r", NOT_APPLIED),  # CR NEWLINE
        ("\\n", NOT_APPLIED),  # ESC LF NEWLINE
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_NEWLINE(string, result):
    assert NEWLINE.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("1e3", (1_000.0, 3)),  # INT, LC E, INT
        ("1e+3", (1_000.0, 4)),  # INT, E, POS INT
        ("1e-3", (0.001, 4)),  # INT, E, NEG INT
        ("1E3", (1_000.0, 3)),  # INT, UC E, INT
        ("1.0e3", (1_000.0, 5)),  # DECIMAL, E, INT
        ("1.0e+3", (1_000.0, 6)),  # DECIMAL, E, POS INT
        ("1.0e-3", (0.001, 6)),  # DECIMAL, E, NEG INT
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_FLOAT(string, result):
    assert FLOAT.apply(string) == result

@pytest.mark.parametrize(
    ["string", "result"],
    [
        ("+1e3", (1_000.0, 4)),  # POS INT, E, INT
        ("-1e3", (-1_000.0, 4)),  # NEG INT, E, INT
        ("+1e+3", (1_000.0, 5)),  # POS INT, E, POS INT
        ("+1e-3", (0.001, 5)),  # POS INT, E, NEG INT
        ("+1.0e3", (1_000.0, 6)),  # POS DECIMAL, E, INT
        ("-1.0e3", (-1_000.0, 6)),  # NEG DECIMAL, E, INT
        ("+1.0e+3", (1_000.0, 7)),  # POS DECIMAL, E, POS INT
        ("+1.0e-3", (0.001, 7)),  # POS DECIMAL, E, NEG INT
        ("", NOT_APPLIED),  # EMPTY
    ],
)
def test_SIGNED_FLOAT(string, result):
    assert SIGNED_FLOAT.apply(string) == result

# test_NUMBER <=> test_INT and test_FLOAT

# test_SIGNED_NUMBER <=> test_SIGNED_INT and test_SIGNED_FLOAT


import datetime
import re

import pandas as pd

import python.imports.utils as utils
from python.imports.init_params import *

_datetime_fmt = '%Y-%m-%d-%H-%M-%S'
_datetime_fmt1 = '%Y-%m-%dT%H:%M:%S'

def instrument_wind_code(code, instrument_id_list):
    for instrument_id in instrument_id_list:
        if instrument_id.find(code.upper() + '.') == 0:
            return instrument_id
    if re.match(r'[A-Z][A-Z]?\d+', code):
        temp_code = re.sub(r'[A-Z][A-Z]?\d', re.search(r'[A-Z][A-Z]?', code).group(), code, 1)
        for instrument_id in instrument_id_list:
            if instrument_id.find(temp_code + '.') == 0:
                return instrument_id
    return code

def get_all_instrument_dic(ip, headers):
    instrument_dic = {}
    try:
        instrument_list = utils.call('mktInstrumentsListPaged', {}, 'market-data-service', ip,
                                     headers)['page']
        for item in instrument_list:
            instrument_dic[item['instrumentId']] = item
        return instrument_dic
    except Exception as e:
        print("failed update market data for: " + ip + ",Exception:" + str(e))

def import_exchange_trades(ip, headers):
    # 获取bct所有标的物
    instruments_dic = get_all_instrument_dic(ip, headers)
    xinhu_exchange_trades = pd.read_csv(import_exchange_trade_excel_file, encoding="gbk").to_dict(orient='records')
    records = []
    for trade in xinhu_exchange_trades:
        underlyer = instrument_wind_code(trade['code'], instruments_dic.keys())
        if trade['code'].find('.') < 0 and underlyer == trade['code']:
            print('BCT不存在合约代码 {underlyer} '.format(underlyer=str(underlyer)))
            continue
        if not instruments_dic.get(underlyer):
            print('BCT不存在合约代码 {underlyer} '.format(underlyer=str(underlyer)))
            continue
        multiplier = instruments_dic.get(underlyer)['multiplier'] if instruments_dic.get(underlyer)[
            'multiplier'] else 1
        direction = "BUYER" if trade['direction'] == 1 else "SELLER"
        open_close = "OPEN" if trade['direction'] == 1 else "CLOSE"  # 开平
        deal_price = trade['price']  # 交易价格
        deal_amount = trade['amount']  # 交易数量
        # 2020 - 06 - 18 - 20 - 59 - 00
        day_str = trade['dayStr']
        deal_time = datetime.datetime.strptime(day_str, _datetime_fmt).strftime(_datetime_fmt1)
        trade_id = str(trade['order_sys_id']) + "-" + str(trade['trade_id'])

        trade_account = '新湖场内交易账号'  # 交易账户
        exchange_trade = {
            "bookId": book_name,
            "instrumentId": underlyer,
            "direction": direction,
            "openClose": open_close,
            "dealPrice": deal_price,
            "dealAmount": deal_amount,
            "dealTime": deal_time,
            "tradeId": trade_id,
            "tradeAccount": trade_account,
            "multiplier": multiplier
        }
        records.append(exchange_trade)

    for record in records:
        try:
            utils.call('exeTradeRecordSave', record, 'exchange-service', ip, headers)
        except Exception as e:
            print("场内流水导入失败 {tradeId} msg:{error}".format(tradeId=record['tradeId'], error=repr(e)))

# exeTradeRecordSave
if __name__ == '__main__':
    headers = utils.login(bct_login_ip, bct_login_body)
    import_exchange_trades(bct_login_ip, headers)


import re
import sys
from collections import defaultdict
from random import random

# Based on https://github.com/veekaybee/markovhn/blob/master/markov.py

lookback = 2

class Markov():

    def __init__(self, filename):
        self.filename = filename
        self.markov_map, self.titles = init(filename)

    def generate_sentence(self):
        sentence = []
        next_word = sample(self.markov_map[''].items())
        while next_word != '':
            sentence.append(next_word)
            next_word = sample(self.markov_map[' '.join(sentence[-lookback:])].items())
        sentence = ' '.join(sentence)
        for title in self.titles: #Prune titles that are substrings of actual titles
            if sentence in title:
                return self.generate_sentence()
        return sentence

    def get_sentences(self, count=10):
        sentences = []
        while len(sentences) < count:
            sentences.append(self.generate_sentence())
        for sentence in sentences:
            print (sentence)

def init(filename):
    archive = open(filename, encoding="utf-8")
    titles = archive.read().split("\n")
    archive.close()
    markov_map = defaultdict(lambda: defaultdict(int))

    #Generate map in the form word1 -> word2 -> occurences of word2 after word1
    for title in titles[:-1]:
        title = title.split()
        if len(title) > lookback:
            for i in range(len(title)+1):
                markov_map[' '.join(title[max(0,i-lookback):i])][' '.join(title[i:i+1])] += 1

    #Convert map to the word1 -> word2 -> probability of word2 after word1
    for _, following in markov_map.items():
        total = float(sum(following.values()))
        for key in following:
            following[key] /= total
    return markov_map, titles

#Typical sampling from a categorical distribution
def sample(items):
    next_word = None
    t = 0.0
    for k, v in items:
        t += v
        if t and random() < v/t:
            next_word = k
    return next_word


import logging
import importlib
import argparse
import os
from .lib.link_extraction.link_extractor import LinkExtractor
from .lib.body_reader.body_reader import BodyReader
from .link_writer import LinkWriter
from scraper.common.comms.rabbit.rabbit_outputter import RabbitOutputter
from scraper.common.comms.rabbit.rabbit_reciever import RabbitReciever
from scraper.common.comms.rabbit.rabbit_helpers import (
    rabbit_blocking_connection_factory,
)

def main():
    logging.basicConfig()
    logging.getLogger().setLevel(logging.INFO)
    parser = argparse.ArgumentParser(description="Distributed Scraper")
    subs = parser.add_subparsers(dest="mode")

    add_modes(subs)

    opts = [
        ("--rabbit_host", "-rh", os.environ.get("RABBIT_HOST")),
        ("--rabbit_port", "-rp", os.environ.get("RABBIT_PORT", "5672")),
        ("--rabbit_user", "-ru", os.environ.get("RABBIT_USER")),
        ("--rabbit_password", "-rpwd", os.environ.get("RABBIT_PASS")),
    ]

    for opt in opts:
        parser.add_argument(opt[0], opt[1], default=opt[2])

    args = parser.parse_args()
    print(args)
    if args.mode == "links":
        run_link_extractor(args)
    elif args.mode == "body":
        run_result_processor(args)

def add_modes(subs):
    link_extractor = subs.add_parser("links")
    link_extractor.add_argument("--extractors", "-ex")

    body_reader = subs.add_parser("body")
    body_reader.add_argument("--body_readers", "-br")

def run_link_extractor(args):
    conn_factory = _get_rabbit_conn_factory(args)

    link_outputter = RabbitOutputter(("links", "direct"), conn_factory)
    link_writer = LinkWriter(link_outputter)

    for clazz in _get_constructors(args.extractors, LinkExtractor):
        link_extractor = clazz(link_writer)
        rec = RabbitReciever(
            conn_factory,
            ("result", "topic"),
            _queue_name_from_class(clazz),
            link_extractor.for_links_pattern(),
        )
        link_extractor.register_receiver(rec)
        link_extractor.run()

def run_result_processor(args):
    conn_factory = _get_rabbit_conn_factory(args)

    for clazz in _get_constructors(args.body_readers, BodyReader):
        body_reader = clazz()
        rec = RabbitReciever(
            conn_factory,
            ("result", "topic"),
            _queue_name_from_class(clazz),
            body_reader.for_links_pattern(),
        )
        body_reader.register_receiver(rec)
        body_reader.run()

def _queue_name_from_class(clazz):
    return f"{clazz.__module__}.{clazz.__name__}"

def _get_rabbit_conn_factory(args):
    return rabbit_blocking_connection_factory(
        args.rabbit_host,
        args.rabbit_port,
        (args.rabbit_user, args.rabbit_password),
    )

def _get_constructors(class_string, super_class):
    classes = class_string.split(",")
    splits = [clazz.split(".") for clazz in classes]
    module_classes = [(split[:-1], split[-1]) for split in splits]

    for (module_str, clazz_str) in module_classes:
        module_str = ".".join(module_str)
        module = importlib.import_module(module_str)
        clazz = getattr(module, clazz_str)

        if not issubclass(clazz, super_class):
            raise Exception(f"Must subclass {super_class.__name__}")

        yield clazz

if __name__ == "__main__":
    main()


# Copyright (c) 2020 Seagate Technology LLC and/or its Affiliates
#
# This program is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>. For any questions
# about this software or licensing, please email opensource@seagate.com or
# cortx-questions@seagate.com.

# -*- coding: utf-8 -*-
from lettuce import *

import json
import os
import psutil
import time

# Add the top level directory to the sys.path to access classes
topdir = os.path.dirname(os.path.dirname(os.path.dirname \
            (os.path.dirname(os.path.abspath(__file__)))))
os.sys.path.insert(0, topdir)

from test.automated.rabbitmq.rabbitmq_ingress_processor_tests import RabbitMQingressProcessorTests
from framework.rabbitmq.rabbitmq_egress_processor import RabbitMQegressProcessor
from dbus import SystemBus, Interface, exceptions as debus_exceptions

@step('Given that the "([^"]*)" service is "([^"]*)" and SSPL_LL is running')
def given_that_the_name_service_is_condition_and_sspl_ll_is_running(step, name, condition):
    # Apply the condition to the service to guarantee a known starting state
    assert condition in ("running", "halted")
    start_stop_service(name, condition)

    # Check that the state for sspl_ll service is active
    found = False

    # Support for python-psutil < 2.1.3
    for proc in psutil.process_iter():
        if proc.name == "sspl_ll_d" and \
           proc.status in (psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING):
               found = True

    # Support for python-psutil 2.1.3+
    if found == False:
        for proc in psutil.process_iter():
            pinfo = proc.as_dict(attrs=['cmdline', 'status'])
            if "sspl_ll_d" in str(pinfo['cmdline']) and \
                pinfo['status'] in (psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING):
                    found = True

    assert found == True

    # Clear the message queue buffer out
    while not world.sspl_modules[RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
        world.sspl_modules[RabbitMQingressProcessorTests.name()]._read_my_msgQ()

@step('When I send in the actuator message to "([^"]*)" the "([^"]*)"')
def when_i_send_in_the_actuator_message_to_action_the_service(step, action, service):
    egressMsg = {
        "title": "SSPL Actuator Request",
        "description": "Seagate Storage Platform Library - Actuator Request",
        "username" : "JohnDoe",
        "signature" : "None",
        "time" : "2015-05-29 14:28:30.974749",
        "expires" : 500,

        "message" : {
            "sspl_ll_msg_header": {
                "schema_version": "1.0.0",
                "sspl_version": "1.0.0",
                "msg_version": "1.0.0"
            },
            "actuator_request_type": {
                "service_controller": {
                    "service_name" : service,
                    "service_request": action
                }
            }
        }
    }
    world.sspl_modules[RabbitMQegressProcessor.name()]._write_internal_msgQ(RabbitMQegressProcessor.name(), egressMsg)

@step('Then SSPL_LL "([^"]*)" the "([^"]*)" and I get the service is "([^"]*)" response')
def then_sspl_ll_action_the_service_and_i_get_the_service_is_condition_response(step, action, service, condition):

    service_name = None
    service_response = None
    time.sleep(10)
    while not world.sspl_modules[RabbitMQingressProcessorTests.name()]._is_my_msgQ_empty():
        ingressMsg = world.sspl_modules[RabbitMQingressProcessorTests.name()]._read_my_msgQ()
        time.sleep(10)
        print("Received: %s" % ingressMsg)

        try:
            # Verify module name and thread response
            msg = ingressMsg.get("actuator_response_type").get("service_controller")
            service_name = msg["service_name"]
            time.sleep(10)
            print("service_name: %s" % service_name)

            msg = ingressMsg.get("actuator_response_type").get("service_controller")
            service_response = msg["service_response"]
            #time.sleep(4)
            print("service_response: %s" % service_response)
            break
        except Exception as exception:
            time.sleep(4)
            print(exception)

    assert service_name == service
    assert service_response == condition

def start_stop_service(service_name, action):
    assert action in ("running", "halted")

    # Obtain an instance of d-bus to communicate with systemd
    bus = SystemBus()

    # Obtain a manager interface to d-bus for communications with systemd
    systemd = bus.get_object('org.freedesktop.systemd1',
                             '/org/freedesktop/systemd1')
    manager = Interface(systemd, dbus_interface='org.freedesktop.systemd1.Manager')

    if action == "running":
        manager.StartUnit(service_name + ".service", 'replace')
    else:
        manager.StopUnit(service_name + ".service", 'replace')


import pytest

import spam

def test_ham():
    seq = ('spam', 'ham', 'eggs')
    value = 'spamspamspam'
    iter_ = iter(seq)
    ins = spam.Ham(iter_, value)
    assert ins.iter is iter_
    assert value is ins.value

    assert next(ins) == seq[0]
    assert next(ins) == seq[1]
    assert next(ins) == seq[2]
    with pytest.raises(StopIteration) as exc_info:
        next(ins)
    assert exc_info.value.value == value

async def spam_await(awaitable):
    return await awaitable

def test_spam():
    seq = ('spam', 'ham', 'eggs')
    value = 'spamspamspam'
    ins = spam.Spam(seq, value)
    assert seq is ins.seq
    assert value is ins.value

    co = spam_await(ins)
    assert co.send(None) == seq[0]
    assert co.send(None) == seq[1]
    assert co.send(None) == seq[2]
    with pytest.raises(StopIteration) as exc_info:
        co.send(None)
    assert exc_info.value.value == value

def test_spam_send_value():
    seq = ('spam', 'ham', 'eggs')
    value = 'spamspamspam'
    ins = spam.Spam(seq, value)

    co = spam_await(ins)
    assert co.send(None) == seq[0]
    assert co.send(2) == seq[1] * 2

def test_spam_throw():
    seq = ('spam', 'ham', 'eggs')
    value = 'spamspamspam'
    ins = spam.Spam(seq, value)
    co = spam_await(ins)
    assert co.send(None) == seq[0]
    assert co.throw(ValueError) == seq[1]
    with pytest.raises(Exception) as exc_info:
        co.throw(Exception, 'err')
    assert exc_info.value.args[0] == 'err'


# -*- coding: utf-8 -*-
"""
Created on Thu June 26 11:00:10 2020

@author: Gerardo Cervantes

Purpose: This will handle having both the generator and discriminator, as well as training the models given the data
"""

import torch
from src.data import data_load
from src.losses.loss_functions import supported_loss_functions, supported_losses
from src.optimizers.optim_functions import supported_optimizer_list, optimizer_factory
from torch_ema import ExponentialMovingAverage
from torchinfo import summary
from src.data.data_load import normalize, unnormalize
from src.utils import os_helper
import os

class GanModel:

    def __init__(self, generator, discriminator, num_classes, accelerator, torch_dtype, model_arch_config, train_config):

        if train_config.getboolean('channels_last'):
            discriminator.set_channels_last()
            generator.set_channels_last()

        self.num_classes = num_classes
        self.device = accelerator.device
        generator = generator.to(self.device)
        discriminator = discriminator.to(self.device)
        self.torch_dtype = torch_dtype
        self.latent_vector_size = int(model_arch_config['latent_vector_size'])
        beta1 = float(train_config['beta1'])
        beta2 = float(train_config['beta2'])
        generator_wd = float(train_config['generator_wd'])
        discriminator_wd = float(train_config['discriminator_wd'])
        generator_lr = float(train_config['generator_lr'])
        discriminator_lr = float(train_config['discriminator_lr'])
        self.orthogonal_value = float(model_arch_config['orthogonal_value'])
        loss_type = model_arch_config['loss_type']
        self.is_omni_loss = 'omni-loss' == loss_type.lower()
        self.accumulation_iterations = int(train_config['accumulation_iterations'])
        self.mixed_precision = train_config.getboolean('mixed_precision')
        self.batch_iterations = 0

        self.criterion, self.fake_label, self.real_label = supported_loss_functions(train_config['loss_function'])
        if self.criterion is None:
            raise ValueError("Loss given not found: " + train_config['loss_function'] +
                             "\nPossible options are: " + str(supported_losses()))
        gan_optimizer = optimizer_factory(train_config['optimizer'])
        if not gan_optimizer:
            raise ValueError("Optimizer given not found: " + train_config['optimizer'] +
                             "\nPossible options are: " + str(supported_optimizer_list()))
        # Setup Adam optimizers for both G and D
        self.optimizerD = gan_optimizer(discriminator.parameters(), lr=discriminator_lr, betas=(beta1, beta2),
                                     weight_decay=discriminator_wd)
        self.optimizerG = gan_optimizer(generator.parameters(), lr=generator_lr, betas=(beta1, beta2),
                                     weight_decay=generator_wd)
        discriminator.zero_grad()
        generator.zero_grad()

        # Set EMA
        ema_enabled = model_arch_config.getboolean('generator_ema')
        ema_decay = model_arch_config['ema_decay']
        self.ema = ExponentialMovingAverage(generator.parameters(),
                                            decay=float(ema_decay)) if ema_enabled else None

        self.netD, self.netG = discriminator, generator
        self.accelerator = accelerator

    def create_real_labels(self, b_size, labels):
        if self.is_omni_loss:
            # Real_label is of size [b_size, num_classes]
            real_label = torch.nn.functional.one_hot(labels, num_classes=self.num_classes)
            # Adds 2 columns, One with all values 1, the second with all values 0. Size [b_size, num_classes + 2]
            real_label = torch.concat((real_label, torch.ones(b_size, 1, device=self.device, dtype=self.torch_dtype),
                                    torch.zeros(b_size, 1, device=self.device, dtype=self.torch_dtype)), dim=1)
            # Replace the 0s and 1s with value of the fake_label, or true_label
            real_label[real_label == 0] = self.fake_label
            real_label[real_label == 1] = self.real_label
        else:
            real_label = torch.full((b_size,), self.real_label, dtype=self.torch_dtype, device=self.device)
        return real_label

    def create_fake_labels(self, b_size):
        if self.is_omni_loss:
            fake_label = torch.full((b_size, self.num_classes + 1), self.fake_label, dtype=self.torch_dtype, device=self.device)
            fake_label = torch.concat((fake_label, torch.ones(b_size, 1, device=self.device, dtype=self.torch_dtypee)), dim=1)
        else:
            fake_label = torch.full((b_size,), self.fake_label, dtype=self.torch_dtype, device=self.device)
        return fake_label

    def train_step(self, batches_accumulated):
        self.netG.train()
        self.netD.train()
        discriminator_loss, accum_labels = self.discriminator_step(batches_accumulated)
        generator_loss = self.generator_step(accum_labels)
        if self.ema:
            self.ema.update()
        self.netD.requires_grad_(requires_grad=True)
        return discriminator_loss, generator_loss

    def discriminator_step(self, batches_accumulated):

        self.netD.requires_grad_(requires_grad=True)
        self.optimizerD.zero_grad(set_to_none=True)
        accum_labels = []
        total_discrim_error = 0
        for batch in batches_accumulated:
            with self.accelerator.no_sync(self.netD):

                real_data, labels = batch
                b_size = real_data.size(dim=0)
                accum_labels.append(labels)

                discrim_output = self.netD(real_data, labels)

                real_label = self.create_real_labels(b_size, labels)
                discrim_on_real_error = self.criterion(discrim_output, real_label)
                discrim_on_real_error = discrim_on_real_error / self.accumulation_iterations

                self.accelerator.backward(discrim_on_real_error)

                # Train with all-fake batch
                noise = torch.randn(b_size, self.latent_vector_size, device=self.device, dtype=real_data.dtype)
                with torch.no_grad():
                    # Generate fake image batch with G
                    fake = self.netG(noise, labels)

                # Classify all fake batch with D
                fake_output = self.netD(fake.detach(), labels)

                # Calculate D's loss on the all-fake batch
                fake_label = self.create_fake_labels(b_size)
                discrim_on_fake_error = self.criterion(fake_output, fake_label.reshape_as(fake_output))
                discrim_on_fake_error = discrim_on_fake_error / self.accumulation_iterations
                self.accelerator.backward(discrim_on_fake_error)

                total_discrim_error += discrim_on_real_error + discrim_on_fake_error
        self.optimizerD.step()
        self.optimizerD.zero_grad(set_to_none=True)
        return total_discrim_error.item(), accum_labels

    def generator_step(self, accum_labels):
        self.netD.requires_grad_(requires_grad=False)
        self.optimizerG.zero_grad(set_to_none=True)

        total_generator_error = 0
        for label in accum_labels:
            with self.accelerator.no_sync(self.netG):
                # with self.accelerator.autocast():
                b_size = label.size(dim=0)
                # Train with all-fake batch
                noise = torch.randn(b_size, self.latent_vector_size, device=self.device, requires_grad=True, dtype=self.torch_dtype)
                # Generate fake image batch with G
                fake = self.netG(noise, label)
                self.netD.requires_grad_(requires_grad=False)
                output_update = self.netD(fake, label)
                self.netD.requires_grad_(requires_grad=True)
                real_label = self.create_real_labels(b_size, label)
                generator_error = self.criterion(output_update, real_label) / self.accumulation_iterations

                self.accelerator.backward(generator_error)
                total_generator_error += generator_error

            if not self.orthogonal_value == 0:
                with self.accelerator.autocast():
                    generator_error = self.apply_orthogonal_regularization(self.netG)
                    total_generator_error += generator_error
                self.accelerator.backward(generator_error)
        self.optimizerG.step()
        self.optimizerG.zero_grad(set_to_none=True)
        return total_generator_error.item()

    # Accelerate Models should be done after loading the models
    def accelerate_models(self):
        self.netD, self.netG, self.optimizerD, self.optimizerG = self.accelerator.prepare(
            self.netD, self.netG, self.optimizerD, self.optimizerG
        )

    def apply_orthogonal_regularization(self, model):
        model_orthogonal_loss = 0.0
        for name, param in model.named_parameters():
            if len(param.size()) >= 2 and param.requires_grad and 'conv' in name:
                param = param.view(param.size(dim=0), -1)
                mult_out = torch.mm(torch.t(param), param)
                orthogonal_matrix = torch.mul(mult_out, 1 - torch.eye(mult_out.size(dim=0), device=self.device))
                # Frobenius norm
                model_orthogonal_loss += self.orthogonal_value * torch.norm(orthogonal_matrix, p='fro')
        return model_orthogonal_loss

    # If class_embeddings is provided, then it will ignore labels to generate the images
    def generate_images(self, noise, labels, class_embeddings=None, unnormalize_img=True):
        self.netG.eval() # Set Generater to Eval mode
        with torch.no_grad():
            if self.ema:
                with self.ema.average_parameters():
                    if class_embeddings:
                        forward_with_class_embeddings = getattr(self.netG, "forward_with_class_embeddings", None)
                        if callable(forward_with_class_embeddings):
                            fake = forward_with_class_embeddings(noise, class_embeddings)
                        else:
                            raise ValueError('Given generator model does not support providing class embeddings '
                                                'directly.')
                    else:
                        fake = self.netG(noise, labels)
            else:
                fake = self.netG(noise, labels)

            fake = fake.detach()
            if unnormalize_img:
                fake = unnormalize(fake)
            return fake

    def save(self, model_dir, step_num, compiled=False):
        generator_path = os.path.join(model_dir, os_helper.ModelType.GENERATOR.value + '_step_' + str(step_num) + '.pt')
        discrim_path = os.path.join(model_dir, os_helper.ModelType.DISCRIMINATOR.value + '_step_' +
                                    str(step_num) + '.pt')

        # Save uncompiled model, so you can go from compile to uncompiled
        if compiled:
            netG = self.netG._orig_mod
            netD = self.netD._orig_mod
        else:
            netG = self.netG
            netD = self.netD
        torch.save(netG.state_dict(), generator_path)
        torch.save(netD.state_dict(), discrim_path)
        if self.ema:
            ema_path = os.path.join(model_dir, os_helper.ModelType.EMA.value + '_step_' + str(step_num) + '.pt')
            torch.save(self.ema.state_dict(), ema_path)

    def load(self, generator_path, discrim_path, ema_path=None):
        self.netG.load_state_dict(torch.load(generator_path))
        self.netD.load_state_dict(torch.load(discrim_path))
        if self.ema and ema_path:
            self.ema.load_state_dict(torch.load(ema_path))

    # Writes text file with information of the generator and discriminator instances
    def save_architecture(self, save_dir, data_config):

        image_height, image_width = data_load.get_image_height_and_width(data_config)
        discriminator_stats = summary(self.netD, input_data=[torch.zeros((1, 3, image_height, image_width)),
                                                             torch.zeros(1, dtype=torch.int64)], verbose=0,
                                      device=self.device)
        generator_stats = summary(self.netG, input_data=[torch.zeros(1, self.latent_vector_size),
                                                         torch.zeros(1, dtype=torch.int64)], verbose=0,
                                  device=self.device)

        with open(os.path.join(save_dir, 'architecture.txt'), 'w', encoding='utf-8') as text_file:
            text_file.write('Generator\n\n')
            text_file.write(str(generator_stats))
            text_file.write(str(self.netG))
            text_file.write('\n\nDiscriminator\n\n')
            text_file.write(str(discriminator_stats))
            text_file.write(str(self.netD))


import random
import time
import os
from os import path

import pytest

import common.cmd as cmd
from common.core import (  # NOQA
    get_dev, read_dev, write_dev,  # NOQA
    random_string, verify_data,
    open_replica,
    get_dev_path, check_dev_existence,
)
from common.frontend import get_socket_path
from common.constants import (
    LONGHORN_DEV_DIR, PAGE_SIZE, SIZE,
    VOLUME_NAME, ENGINE_NAME
)

def test_device_creation(first_available_device,
                         grpc_controller_device_name_test,  # NOQA
                         grpc_replica1, grpc_replica2):  # NOQA
    block_device = get_dev(grpc_replica1, grpc_replica2,
                           grpc_controller_device_name_test)
    assert block_device
    check_dev_existence(VOLUME_NAME)
    longhorn_dev = get_dev_path(VOLUME_NAME)
    dev_info = os.stat(first_available_device)
    assert dev_info.st_rdev == os.stat(os.devnull).st_rdev
    assert dev_info.st_rdev != os.stat(longhorn_dev).st_rdev
    test_basic_rw(block_device)

def test_basic_rw(dev):  # NOQA
    for i in range(0, 10):
        base = random.randint(1, SIZE - PAGE_SIZE)
        offset = (base // PAGE_SIZE) * PAGE_SIZE
        length = base - offset
        data = random_string(length)
        verify_data(dev, offset, data)

def test_metrics(grpc_replica1, grpc_replica2, grpc_controller):  # NOQA
    rw_dev = get_dev(grpc_replica1, grpc_replica2,
                     grpc_controller)
    metrics = grpc_controller.metrics_get()
    metrics = metrics.metrics

    # No r/w IO
    assert metrics.readThroughput == 0
    assert metrics.writeThroughput == 0
    assert metrics.readIOPS == 0
    assert metrics.writeIOPS == 0

    # Create r/w IO
    base = random.randint(1, SIZE - PAGE_SIZE)
    offset = (base // PAGE_SIZE) * PAGE_SIZE
    length = base - offset
    data = random_string(length)
    verify_data(rw_dev, offset, data)

    # metrics update period is 1 second
    max_iters = 10
    for i in range(max_iters):
        try:
            metrics = grpc_controller.metrics_get()
            metrics = metrics.metrics

            # it's hard to confirm the accurate value of metric
            assert metrics.readThroughput != 0
            assert metrics.writeThroughput != 0
            assert metrics.readIOPS != 0
            assert metrics.writeIOPS != 0
            break
        except Exception as e:
            if i == max_iters - 1:
                raise e
            time.sleep(0.1)

def test_beyond_boundary(dev):  # NOQA
    # check write at the boundary
    data = random_string(128)
    verify_data(dev, SIZE - 1 - 128, data)

    # out of bounds
    with pytest.raises(EnvironmentError) as err:
        write_dev(dev, SIZE, "1")
    assert 'No space left' in str(err.value)
    assert len(read_dev(dev, SIZE, 1)) == 0

    # normal writes to verify controller/replica survival
    test_basic_rw(dev)

def test_frontend_show(grpc_engine_manager, grpc_controller,  # NOQA
                       grpc_replica1, grpc_replica2):  # NOQA
    open_replica(grpc_replica1)
    open_replica(grpc_replica2)

    replicas = grpc_controller.replica_list()
    assert len(replicas) == 0

    r1_url = grpc_replica1.url
    r2_url = grpc_replica2.url
    v = grpc_controller.volume_start(
        SIZE, SIZE, replicas=[r1_url, r2_url])

    ft = v.frontend
    if ft == "tgt":
        assert v.endpoint == path.join(LONGHORN_DEV_DIR, VOLUME_NAME)
    elif ft == "socket":
        assert v.endpoint == get_socket_path(VOLUME_NAME)
        engine = grpc_engine_manager.engine_get(ENGINE_NAME)
        assert engine.status.endpoint == path.join(LONGHORN_DEV_DIR,
                                                   VOLUME_NAME)

    info = cmd.info_get(grpc_controller.address)
    assert info["name"] == VOLUME_NAME
    assert info["endpoint"] == v.endpoint

# https://github.com/rancher/longhorn/issues/401
def test_cleanup_leftover_blockdev(grpc_controller,  # NOQA
                                   grpc_replica1, grpc_replica2):  # NOQA
    open_replica(grpc_replica1)
    open_replica(grpc_replica2)

    replicas = grpc_controller.replica_list()
    assert len(replicas) == 0

    blockdev = path.join(LONGHORN_DEV_DIR, VOLUME_NAME)
    assert not path.exists(blockdev)
    open(blockdev, 'a').close()

    r1_url = grpc_replica1.url
    r2_url = grpc_replica2.url
    grpc_controller.volume_start(
        SIZE, SIZE, replicas=[r1_url, r2_url])

    info = cmd.info_get(grpc_controller.address)
    assert info["name"] == VOLUME_NAME


from django.shortcuts import render, redirect
from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.contrib.auth.decorators import login_required
from quote.models import Quote, QuoteForm
from quote.services import get_all_quotes, get_single_quote, create_job_with_quote, create_pdf
from job.services import get_single_job
from django.views.generic import DetailView, CreateView, UpdateView, DeleteView, ListView
from django.http import JsonResponse, HttpResponseRedirect
from django.core.paginator import Paginator
from django.urls import reverse_lazy, reverse
from user.models import User
from django.db.models import Q
from django.views.decorators.csrf import csrf_exempt
from job.models import Job
from django.core.exceptions import ValidationError
from django.core.mail import send_mail, EmailMessage
from smtplib import SMTPException
import os

@csrf_exempt
def create_job(request):
    if request.is_ajax():
        quote_id = request.POST.get('data')
        quote = get_single_quote(quote_id)

        job = create_job_with_quote(quote)

        try:
            job.full_clean()
            job.save()
            quote.associated_job = job._id
            quote.save()

            url = reverse('job-details', kwargs={'obj_id':job._id})
            data = {'success': url}
            name = quote.customer_first_name + ' ' + quote.customer_last_name
            messages.success(request, 'Job for ' + name + ' created')
            return JsonResponse(data)

        except ValidationError as err:
            data = {'error': err.message_dict}
            return JsonResponse(data)

@csrf_exempt
def send_quote(request):
    if request.is_ajax():
        quote_id = request.POST.get('data')
        quote = get_single_quote(quote_id)

        try:
            if quote.customer_email == None:
                raise ValidationError('Must have a customer email')
            elif quote.date_of_job == None:
                raise ValidationError('Must have a date of job')
            elif quote.materials == None:
                raise ValidationError('Must have materials')
            elif quote.price_of_materials == None:
                raise ValidationError('Must have price of materials')
            elif quote.price_of_labour == None:
                raise ValidationError('Must have price of labour')
            elif quote.quote_price == None:
                raise ValidationError('Must have quote price')

        except ValidationError as err:
            return JsonResponse( {'status': 'error'} )

        quote.issued_by_first_name = quote.issued_by.first_name
        quote.issued_by_last_name = quote.issued_by.last_name
        quote.issued_by_address = quote.issued_by.address
        quote.issued_by_number = quote.issued_by.phone_num
        quote_dict = vars(quote)

        pdf = create_pdf(quote_dict)

        email = EmailMessage(
            subject='Frenchay Fencing Quote',
            body='Please find attached your quote.\n\nPlease do not reply to this email. Instead, contact us using the details on your quote.',
            from_email=os.environ.get('EMAIL_USER'),
            to=[quote.customer_email],
            cc=[quote.issued_by.email]
        )
        email.attach('quote.pdf', pdf, 'application/pdf')
        try:
            email.send()
            quote.sent_quote = True
            quote.save()
            return JsonResponse( {'status': 'success'} )
        except SMTPException as err:
            return JsonResponse( {'status': 'error'} )

class QuoteListView(LoginRequiredMixin, ListView):
    model = Quote
    template_name = "quote/view_quotes.html"
    paginate_by = 10

    def get_queryset(self):
        quotes = get_all_quotes()
        q = self.request.GET.get("search", None)

        #if user enters search term
        if q is not None:
            quotes = quotes.filter(
                Q(customer_first_name__icontains=q) |
                Q(customer_last_name__icontains=q) |
                Q(address__icontains=q) |
                Q(customer_phone_num__icontains=q)
                )

        for item in quotes:
            item.id = item._id

        return quotes

class QuoteDetailView(LoginRequiredMixin, DetailView):
    model = Quote

    def get_object(self, queryset=None):
        quote = get_single_quote(self.kwargs.get('obj_id'))
        quote.id = quote._id
        return quote

    def render_to_response(self, context, **response_kwargs):
        # split the materials into an array to display differently on frontend
        context['object'].materials = context['object'].materials.splitlines()

        response_kwargs.setdefault('content_type', self.content_type)
        return self.response_class(
            request=self.request,
            template=self.get_template_names(),
            context=context,
            using=self.template_engine,
            **response_kwargs
        )

class QuoteCreateView(LoginRequiredMixin, CreateView):
    model = Quote
    form_class = QuoteForm

    def form_valid(self, form):
        form.instance.issued_by = self.request.user
        form.instance.address = form.cleaned_data['address']
        name = form.cleaned_data['customer_first_name'] +  ' ' + form.cleaned_data['customer_last_name']
        messages.success(self.request, 'Quote for ' + name + ' created')

        return super().form_valid(form)

class QuoteUpdateView(LoginRequiredMixin, UpdateView):
    model = Quote
    form_class = QuoteForm
    pk_url_kwarg = 'obj_id'

    def get_object(self, queryset=None):
        quote = get_single_quote(self.kwargs.get('obj_id'))
        quote.id = quote._id
        return quote

    def get_initial(self):
        quote = get_single_quote(self.kwargs.get('obj_id'))
        quote = vars(quote)
        address = quote['address']
        split_address = address.split(',')
        quote['street'] = split_address[0].lstrip()
        quote['city'] = split_address[1].lstrip()
        quote['post_code'] = split_address[2].lstrip()
        return quote

    def form_valid(self, form):
        form.instance.address = form.cleaned_data['address']
        name = form.cleaned_data['customer_first_name'] +  ' ' + form.cleaned_data['customer_last_name']
        messages.success(self.request, 'Quote for ' + name + ' updated')

        return super().form_valid(form)

class QuoteDeleteView(LoginRequiredMixin, DeleteView):
    model = Quote
    pk_url_kwarg = 'obj_id'
    success_url = reverse_lazy('view-quotes')

    def get_object(self, queryset=None):
        quote = get_single_quote(self.kwargs.get('obj_id'))
        quote.id = quote._id

        return quote

    def delete(self, request, *args, **kwargs):
        quote = get_single_quote(self.kwargs.get('obj_id'))

        # when quote is deleted also delete associated job
        if quote.associated_job != None:
            job = get_single_job(quote.associated_job)
            job.delete()

        name = quote.customer_first_name +  ' ' + quote.customer_last_name

        quote.delete()

        success_url = reverse_lazy('view-quotes')
        messages.success(self.request, 'Quote and associated job for ' + name + ' deleted')
        return HttpResponseRedirect(success_url)


import os

import pygame
import time

pygame.init()

from Components import AppManager, ShortcutManager, Background

DISPLAY = pygame.display.set_mode((64*20, 64*20*0.5))
pygame.display.set_caption("Epic Fox os")

def prepareApps():
    for folder in os.listdir("apps"):
        AppManager.prepareApp(folder)

def mainLoop():
    prepareApps()
    shortcutManager = ShortcutManager(DISPLAY.get_size())
    shortcutManager.loadShortcuts()

    background = Background(pygame.image.load("assets/Debinyan_Wallpaper.png"))

    OS_RUNNING = True

    prev = time.time()
    dt = 0
    while OS_RUNNING:
        now = time.time()
        dt = time.time() - prev
        prev = now

        # Events
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                OS_RUNNING = False
            if event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE:
                OS_RUNNING = False

        keys, mpos, mrel, mbuttons = pygame.key.get_pressed(), pygame.mouse.get_pos(), pygame.mouse.get_rel(), pygame.mouse.get_pressed(3)
        # Update
        shortcutManager.update()
        for app in AppManager.WORKING_APPS:
            events = app.update(dt, keys, mpos, mrel, mbuttons)
            if any(event.id == 0 for event in events):
                AppManager.killApp(app)
            elif any(event.id == 2 for event in events):
                AppManager.hideApp(app)
        # Draw
        DISPLAY.fill((0, 0, 0))
        background.render(DISPLAY)
        shortcutManager.draw(DISPLAY)
        for app in AppManager.WORKING_APPS:
            app.draw(DISPLAY)
        pygame.display.update()

    pygame.quit()

if __name__ == "__main__":
    mainLoop()


import tkinter as tk
from screens.widgets import ControllPanel, AnotherPanel
from multiprocessing import Pool
from collector.collector import NewsCollector
from collector.processing import DataProcessor
from api.news_service import NewsService
from utils.docker_utils import SplashContainer
import subprocess
import pathlib
import os
from datetime import datetime
from dateutil import parser
import time
import traceback
import sys
import pandas as pd
import requests
import numpy as np

SRC = pathlib.Path(__file__).parent
sys.path.append(SRC)

class MenuBar(tk.Menu):

    def __init__(self, parent):

        tk.Menu.__init__(self, parent)

        menu1 = tk.Menu(self, tearoff=0)
        menu1.add_command(label = 'Control Panel', command = lambda: parent.show_controll_panel())
        menu1.add_command(label = 'Another Panel', command = lambda: parent.show_another_panel())
        menu1.add_command(label = 'Option 3', command = lambda: print("Menu 1 Option 3 clicked"))
        menu1.add_separator()
        menu1.add_command(label = 'Option 4', command = lambda: print("Menu 1 Option 4 clicked"))

        self.add_cascade(label = "Menu 1", menu = menu1)

        menu2 = tk.Menu(self, tearoff=0)
        menu2.add_command(label = 'Option 1', command = lambda: print("Menu 2 Option 1 clicked"))
        menu2.add_command(label = 'Option 2', command = lambda: print("Menu 2 Option 2 clicked"))
        menu2.add_command(label = 'Option 3', command = lambda: print("Menu 2 Option 3 clicked"))
        menu2.add_separator()
        menu2.add_command(label = 'Option 4', command = lambda: print("Menu 2 Option 4 clicked"))

        self.add_cascade(label = "Menu 2", menu = menu2)

class MyApp(tk.Tk):
    CWD = pathlib.Path(__file__).parent.absolute()
    TRENDING_KEYWORDS_FILE = os.path.join(CWD,'collector','output','trending.txt')

    def __init__(self, *args, **kwargs):

        tk.Tk.__init__(self, *args, **kwargs)
        self.main_frame = tk.Frame(self, bg = '#272727', height = 1000, width = 2000)
        self.main_frame.pack_propagate(0)
        self.main_frame.pack(fill='both', expand = 'true')
        self.main_frame.grid_columnconfigure(0, weight = 1)
        self.main_frame.grid_rowconfigure(0, weight = 1)
        self.now = datetime.now()

        menubar = MenuBar(self)
        tk.Tk.config(self, menu = menubar)

        #init news collector
        self.news_collector = NewsCollector()

        #init splash container
        self.splash_container = SplashContainer()

        self.splash_logs = list()

        #Init data processor
        self.data_processor = DataProcessor()

        #Init api
        self.api = NewsService()

        self.show_controll_panel()

        self.scraper_running = None
        self.current_keyword_index = None
        self.is_splash_running = None

        self.load_trending_keywords()

        self.spider_names = ['cnn_spider', 'market_watch_spider', 'reuters_spider']
        self.tasks_done = None

        self.scraping_hours = self._get_scraping_hours()
        self.is_scrapping = False

        self.main_frame.after(1000, self._loop)

    def load_trending_keywords(self):
        try:
            self._add_log_to_controll_panel("Updating trending logs")
            self.trending_keywords = self.data_processor.get_trending_keyword()
            self.control_panel.update_trending_keywords(self.trending_keywords)

        except FileNotFoundError as e:
            print("Trending keyword file is not found")
            traceback.print_exc()

    def show_controll_panel(self):
        print('showing control panel')
        self.control_panel = ControllPanel(self.main_frame, self)
        self._show_widget(self.control_panel)

        #Check health
        try:
            self.api.check_health()
            self._add_log_to_controll_panel("News Service API is healhty")
        except Exception as e:
            self._add_log_to_controll_panel("[ERROR] News Service API is not healthy")

    def show_another_panel(self):
        print("showing another panel")
        w = AnotherPanel(self.main_frame)
        self._show_widget(w)

    def _show_widget(self, widget):
        widget.grid(row = 0, column = 0, sticky = 'nsew')
        widget.tkraise()

    def _add_log_to_controll_panel(self, text):
        self.control_panel.add_log(text)

    def task_done(self, result):
        self.tasks_done.append(result)
        self._add_log_to_controll_panel(f'[ DONE ] {str(result)}')

        if self.check_all_tasks_done():
            self._add_log_to_controll_panel('All tasks done')

    def check_all_tasks_done(self):
        # Simple logic, if tasks done list has equal number of elements with spider_names then it's true
        if len(self.tasks_done) == len(self.spider_names):
            return True
        else:
            return False

    def stop_scraping(self):
        self.is_scrapping = False

    def _start_scraper_async(self, keyword):

        #CAUTION: self.tasks_done must be init to be a list(), not gonna handle error here

        #Clear tasks done

        self.tasks_done.clear()

        data = self.news_collector.get_spider_data(keyword, 'business', False, 'today', 1)
        pool = Pool()

        # init pool
        for spider_name in self.spider_names:
            self._add_log_to_controll_panel(f'[ CNN Spider ],args: {str(data)}')
            pool.apply_async(self.news_collector.start_search_process, (data, spider_name), callback=self.task_done)

        pool.close()
        print("end scraper")

    def _get_splash_container(self):
        return self.splash_container

    def _get_collector(self):
        return self.news_collector

    def start_splash(self):
        if self.splash_container.is_running() == False:
            self.control_panel.add_log("Splash container started by user")
            self.splash_container.start()
        else:
            self.control_panel.add_log("Splash container started by user but it's already running")

    def stop_splash(self):
        if self.splash_container.is_running() == True:
            print("Stopping splash ...")
            self.control_panel.add_log("Splash container stopped by user")
            # Using subprocess because Docker SDK has the following issue:
            # stop command sent but docker container not stopped,
            # it doesn't show up in 'docker ps' but you can
            # 'docker logs -f splash' and see it's running
            # self.splash_container.stop()
            # this is a temporary solution
            subprocess.run('docker rm -f splash'.split(" "))

    def _loop(self):
        is_running = self.splash_container.is_running()

        if is_running != self.is_splash_running:
            self.is_splash_running = is_running
            self.control_panel.toggle_docker_status(self.is_splash_running)

        #Prevent errors by skipping loop if docker splash container is not running
        if self.is_splash_running == False:
            self._add_log_to_controll_panel("Docker is not running, all scraping process will be hold")
            self.main_frame.after(1000, self._loop)
            return

        # scraping hours:
        # 6 A.M
        # 8 A.M
        # 10 A.M
        # 12 A.M
        # 2 P.M
        # 4 P.M
        # 6 P.M
        # 8 P.M
        # 10 P.M

        # now = datetime.now()

        # current_hour = now.hour

        # if current_hour in self.scraping_hours:
        #     self.is_scrapping = True

        if self.is_scrapping == True:
            #Initiate scraping process if is_scraping = True
            tick = time.time()
            if self.tasks_done == None:
                print("Starting from index 0")
                self.current_keyword_index = 0
                keywords = self.trending_keywords.most_common()
                keyword, rank = keywords[self.current_keyword_index]
                self.tasks_done = list()
                self._start_scraper_async(keyword)
                self.control_panel.update_scraping_keywords_index(self.current_keyword_index)
                self._add_log_to_controll_panel(f"Scraping with keyword: {keyword}")
            else:
                if self.check_all_tasks_done() == True:
                    if self.current_keyword_index < len(self.trending_keywords) - 1:
                        self.current_keyword_index += 1
                        keywords = self.trending_keywords.most_common()
                        keyword, rank = keywords[self.current_keyword_index]
                        self._start_scraper_async(keyword)
                        self.control_panel.update_scraping_keywords_index(self.current_keyword_index)
                        self._add_log_to_controll_panel(f"Scraping with keyword: {keyword}")
                    else:
                        self.tasks_done = None
                        self._add_log_to_controll_panel("Completely scraped all trending keywords")
                        tock = time.time() - tick
                        self._add_log_to_controll_panel(f"Elapsed time {tock} seconds")
                        self.is_scrapping = False

        self.main_frame.after(1000, self._loop)

    def start_scraping_trending_keywords(self):
        self.is_scrapping = True

    def _get_scraping_hours(self):
        return [7,10,13,16,18,21]

    def fetch_headlines(self):
        now = datetime.now()
        self._add_log_to_controll_panel(f"Fetching headlines for today {now}")
        try:
            self.news_collector._get_newsapi_headlines(True)
            self.news_collector.start_get_cnn_headlines(detach = False)
        except Exception as e:
            self._add_log_to_controll_panel(f"Oops! error occured, check below error")
            self._add_log_to_controll_panel(str(e))
            traceback.print_exc()

    def generate_trending_keywords(self):
        self._add_log_to_controll_panel("Generating trending keywords")
        self.data_processor.generate_trending_keywords()
        self._add_log_to_controll_panel("Generated trending keywords. Updated UI")
        self.trending_keywords = self.data_processor.get_trending_keyword()
        self.control_panel.update_trending_keywords(self.trending_keywords)

    def process_today_data(self):
        try:
            today_headlines_file = self.data_processor.get_today_headlines_file()
            self._add_log_to_controll_panel(f'Processing today data from {today_headlines_file} ...')
            self.data_processor.process_data(today_headlines_file)
            self._add_log_to_controll_panel(f"Processed data and saved to {today_headlines_file}")
            self.news_collector.reload_collected_data()
            self._add_log_to_controll_panel(f"Reload collected_data object")
        except FileNotFoundError as e:
            print(f"{today_headlines_file} is not found. Make sure to fetch the data first")
            self._add_log_to_controll_panel(f"Error! Couldn't find {today_headlines_file} to process")
            print(str(e))

    def save_headlines_to_db(self):
        df = self.news_collector.get_fetched_headlines()
        print(f'Saving {self.news_collector.HEADLINE_CSV_PATH} to database')

        for index, row in df.iterrows():
            print(row['title'])

            row['authors'] = str(row['authors'])
            row['keywords'] = str(row['keywords'])

            if row['authors'] == 'nan':
                row['authors'] = ''

            if row['keywords'] == 'nan':
                row['keywords'] = ''

            authors = row['authors'].split(",")
            for author in authors:
                author = author.strip()

            keywords = row['keywords'].split(",")
            for keyword in keywords:
                keyword = keyword.strip()

            try:
                response = self.api.save_news(
                    search_term = str(row['search_term']),
                    title= str(row['title']),
                    text = str(row['text']),
                    authors = authors,
                    source = str(row['source']),
                    url = str(row['url']),
                    image_url = str(row['image_url']),
                    date = parser.isoparse(row['date']),
                    summary = str(row['summary']),
                    keywords = keywords,
                    sentiment = str(row['sentiment'])
                )
                print(response.content)
            except requests.exceptions.HTTPError as e:
                print("###")
                print("Error while saving to db")
                print("data")
                print(row)
                print("###")
                print(e)

if __name__ == '__main__':
    root = MyApp()
    root.title('My app')
    root.mainloop()


import csv
import pytest

import requests
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

from Tests.Home import Homepage, Child_fund_calculator

baseurl = "https://www.sunlife.com.hk/HK?vgnLocale=en_CA"
testdata_filename = "test_data2.csv"

def get_data():
    # Retrieve values from CSV
    with open(testdata_filename) as f:
        data = [(line['expected_breadcrumb'], line['link_locator']) for line in csv.DictReader(f)]
        # print(data)
    return data

def chrome_opt():
    chrome_options = Options()
    # chrome_options.add_argument("--headless")
    chrome_options.add_argument("--disable-notifications")
    chrome_options.add_argument("--start-maximized")
    # chrome_options.add_argument(f"--window-size={1920},{3926}")
    chrome_options.add_argument("--hide-scrollbars")
    return chrome_options

@pytest.fixture(scope='function', autouse=True)
def driver(request):
    opt = chrome_opt()
    driver = webdriver.Chrome(options=opt)
    driver.implicitly_wait(8)
    driver.get(baseurl)

    def driver_teardown():
        print("closing browser")
        driver.close()

    request.addfinalizer(driver_teardown)
    yield driver

@pytest.fixture(scope='function', autouse=True)
def homeObj(driver):
    obj = Homepage(driver)
    yield obj

@pytest.fixture(scope='function', autouse=True)
def fundCalcobj(driver):
    obj = Child_fund_calculator(driver)
    yield obj

def get_links_from_a_webpage():
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(baseurl)
    alllinks = driver.find_elements_by_tag_name("a")
    urls = []
    for link in alllinks:
        if "#" not in str(link.get_attribute("href")) and "javascript" not in str(
                link.get_attribute("href")) and "linkedin" not in str(link.get_attribute("href")):
            if link.text != '':
                urls.append((link.text, link.get_attribute("href")))

    return urls

def get_elements_from_xpath():
    chrome_options = Options()
    chrome_options.add_argument("--headless")
    driver = webdriver.Chrome(options=chrome_options)
    driver.get(baseurl)
    element_list = driver.find_elements_by_xpath("//footer[@id='footer']//li/a")
    element_txt_list = [ele.text for ele in element_list if ele.text != ""]
    text_redundant = ["Follow us on YouTube",
                      "Follow us on Facebook",
                      "Follow us on instagram",
                      "Follow us on Twitter",
                      "Follow us on Linkedin"
                      ]
    result = filter(lambda x: x not in text_redundant, element_txt_list)
    # print(len(list(filtered_ele_texts)))
    return result

@pytest.mark.usefixtures("driver", "homeObj", "fundCalcobj")
class TestExample():
    @pytest.mark.parametrize("link_text, url", get_links_from_a_webpage())
    def test_01_check_for_broken_links(self, link_text, url):
        flag = True
        statusCode = None
        try:
            statusCode = requests.get(url).status_code
            if statusCode == 200:
                print(link_text + ":", url, "PASSED with response code:", statusCode)
            else:
                print(link_text + ":", url, "FAILED with response code:", statusCode)
                flag = False
        except Exception as err:
            print("Something went wrong: ", err)
        if flag == False:
            pytest.fail("Link Maybe Broken...{}".format(statusCode), pytrace=False)

    # def test_02_verify_title(self, driver, homeObj):
    #     print(driver.title)
    #     try:
    #         assert driver.title
    #     except:
    #         pytest.fail("page title not present", pytrace=False)

    def test_02_verify_signin_form_present(self, driver, homeObj):
        homeObj.clickSigninbtn()
        homeObj.verify_signin_form()

    def test_03_verify_get_a_quote_list(self, driver, homeObj):
        homeObj.verify_list_of_prods_in_dropdown()

    def test_04_select_a_product_and_get_a_quote(self, driver, homeObj):
        homeObj.verify_moved_to_get_a_quote()

    def test_05_open_mega_menu_life_moments(self, homeObj):
        homeObj.hover_to_mega_menu()

    def test_06_life_moments_megamenu_links(self, homeObj):
        homeObj.hover_to_mega_menu()
        homeObj.get_megamenu_links("life_moments")

    # def sample_name(self):
    #     picture_page = "https://www.sunlife.co.id/ID/Life+Moments?vgnLocale=en_CA"
    #
    #     name = os.path.splitext(os.path.basename(urlsplit(picture_page).path))
    #     print(name[0])

    def test_11_protection_megamenu_links(self, homeObj):
        homeObj.hover_to_Protection()
        homeObj.get_megamenu_links("Protection")

    def test_12_investment_megamenu_links(self, homeObj):
        homeObj.hover_to_Investment()
        homeObj.get_megamenu_links("Investment")

    def test_13_about_us_megamenu_links(self, homeObj):
        homeObj.hover_to_About_us()
        homeObj.get_megamenu_links("About_us")

    # def test_14_find_an_advisor_dropdown_list_of_cities(self, homeObj):
    #     homeObj.list_of_cities_in_dropdown()

    # def test_15_move_to_find_an_advisor_page_after_selecting_city(self, homeObj):
    #     homeObj.verify_moved_to_find_an_advisor_page()

    #
    # def test_16_Clicking_talk_to_an_advisor_moves_to_find_an_advisor_page(self, homeObj):
    #     homeObj.talk_to_an_advisor_today()

    def test_17_discover_your_needs_dropdown_menu_works(self, homeObj):
        homeObj.discover_your_needs_select_list()

    # def test_18_financial_calculator(self, homeObj, fundCalcobj):
    #     homeObj.click_on_calculate_now()
    #     fundCalcobj.calculate_child_finance()
    #
    # def test_19_contact_advisor(self, homeObj, fundCalcobj):
    #     homeObj.click_on_calculate_now()
    #     fundCalcobj.contact_advisor_form()

    def test_20_verify_search_suggestions(self, homeObj):
        homeObj.search_suggestions()

    def test_21_verify_language_and_region_section_opens_on_clicking_lang(self, homeObj):
        homeObj.click_language_to_open()

    # @pytest.mark.parametrize("expected_breadcrumb, link_locator",
    #                          get_data())
    # def test_22_verify_clicking_on_header_link_moves_to_corresponding_page_and_breadcrumb_is_displayed(self, homeObj,
    #                                                                                expected_breadcrumb,
    #                                                                                link_locator):
    #     homeObj.check_links_move_to_page_and_display_exp_breadcrumb(expected_breadcrumb, link_locator)

    # def test_practice(self):
    #     # print(get_data())
    #     text_redundant = ["Follow us on YouTube",
    #                       "Follow us on Facebook",
    #                       "Follow us on instagram",
    #                       "Follow us on Twitter",
    #                       "Follow us on Linkedin"
    #                       ]
    #     # result = filter(lambda x: x not in text_redundant, get_elements_from_xpath())
    #     result = get_elements_from_xpath()
    #     type(result)
    #     for ele in result:
    #         print(ele)
    #         # print(ele.text)

    @pytest.mark.parametrize("ele_text", get_elements_from_xpath())
    def test_verify_footer_links_are_working_properly(self, homeObj, ele_text):
        homeObj.check_footer_links_moves_to_new_url(ele_text)

if __name__ == '__main__':
    pytest.main()