import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import torch
import torchvision
import torch.nn as nn  # All neural network modules, nn.Linear, nn.Conv2d, BatchNorm, Loss functions
import torchvision.datasets as datasets  # Has standard datasets we can import in a nice way
import torchvision.transforms as transforms  # Transformations we can perform on our dataset
import torch.nn.functional as F  # All functions that don't have any parameters
from torch.utils.data import DataLoader, Dataset  # Gives easier dataset managment and creates mini batches
from torchvision.datasets import ImageFolder
import torch.optim as optim  # For all Optimization algorithms, SGD, Adam, etc.
from PIL import Image
from tqdm import tqdm
from torchvision import models
from sklearn.model_selection import train_test_split

class ImageLoader(Dataset):
    def __init__(self, dataset, transform=None):
        self.dataset = self.checkChannel(dataset)  # some images are CMYK, Grayscale, check only RGB
        self.transform = transform

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, item):
        image = Image.open(self.dataset[item][0])
        classCategory = self.dataset[item][1]
        if self.transform:
            image = self.transform(image)
        return image, classCategory

    def checkChannel(self, dataset):
        datasetRGB = []
        for index in range(len(dataset)):
            if (Image.open(dataset[index][0]).getbands() == ("R", "G", "B")):  # Check Channels
                datasetRGB.append(dataset[index])
        return datasetRGB

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # use gpu or cpu
print(device)

dataset = ImageFolder("D:/mzy/img1_0/train/")
train_data, test_data, train_label, test_label = train_test_split(dataset.imgs, dataset.targets, test_size=0.2,
                                                                  random_state=42)
#
train_transform = transforms.Compose([
    transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize([0.5] * 3, [0.5] * 3)
])  # train transform
test_transform = transforms.Compose([
    transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize([0.5] * 3, [0.5] * 3)
])  # test transform
train_dataset = ImageLoader(train_data, train_transform)
test_dataset = ImageLoader(test_data, test_transform)
train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=True)

# load pretrain model and modify...

# model = models.resnet50(pretrained=True)
model = models.resnet50(pretrained=False)
model.load_state_dict(torch.load('D:/mzy/resnet50-19c8e357.pth'))

# If you want to do finetuning then set requires_grad = False
# Remove these two lines if you want to train entire model,
# and only want to load the pretrain weights.

for param in model.parameters():
    param.requires_grad = False  # 原本是False

num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, 2)
model.to(device)

# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)

# Train and test
def train(num_epoch, model):
    for epoch in range(0, num_epoch):
        #         current_loss = 0.0
        #         current_corrects = 0
        losses = []
        model.train()
        loop = tqdm(enumerate(train_loader), total=len(train_loader))  # create a progress bar
        for batch_idx, (data, targets) in loop:
            data = data.to(device=device)
            targets = targets.to(device=device)
            scores = model(data)

            loss = criterion(scores, targets)
            optimizer.zero_grad()
            losses.append(loss)
            loss.backward()
            optimizer.step()
            _, preds = torch.max(scores, 1)
            #             current_loss += loss.item() * data.size(0)
            #             current_corrects += (preds == targets).sum().item()
            #             accuracy = int(current_corrects / len(train_loader.dataset) * 100)
            loop.set_description(f"Epoch {epoch + 1}/{num_epoch} process: {int((batch_idx / len(train_loader)) * 100)}")
            loop.set_postfix(loss=loss.data.item())

        # save model
        torch.save({
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
        }, 'checpoint_epoch_' + str(epoch) + '.pt')

# model.eval() is a kind of switch for some specific layers/parts of the model that behave differently,
# during training and inference (evaluating) time. For example, Dropouts Layers, BatchNorm Layers etc.
# You need to turn off them during model evaluation, and .eval() will do it for you. In addition,
# the common practice for evaluating/validation is using torch.no_grad() in pair with model.eval()
# to turn off gradients computation:

def test():
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for x, y in test_loader:
            x = x.to(device)
            y = y.to(device)
            output = model(x)
            _, predictions = torch.max(output, 1)
            correct += (predictions == y).sum().item()
            test_loss = criterion(output, y)

    test_loss /= len(test_loader.dataset)
    print("Average Loss: ", test_loss, "  Accuracy: ", correct, " / ",
          len(test_loader.dataset), "  ", int(correct / len(test_loader.dataset) * 100), "%")

# if __name__ == "__main__":
train(6, model)  # train
test()  # test

print("----> Loading checkpoint")
checkpoint = torch.load("./checpoint_epoch_4.pt")  # Try to load last checkpoint
model.load_state_dict(checkpoint["model_state_dict"])
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])

# Check the test set
dataset = ImageFolder("D:/mzy/img1_0/test/",
                      transform=transforms.Compose([
                          transforms.Resize((224, 224)),
                          transforms.ToTensor(),
                          transforms.Normalize([0.5] * 3, [0.5] * 3)
                      ]))
# print(dataset)
dataloader = DataLoader(dataset, batch_size=1, shuffle=False)

# for j, (data, labels) in enumerate(dataloader):
with torch.no_grad():
    model.eval()
    for data, target in dataloader:
        data, target = data.to(device), target.to(device)
        output = model(data)
        _, predicted = torch.max(output, 1)
#         print(f"predicted ----> {predicted[0]}")

dataset = ImageFolder("D:/mzy/mzy/img1_0/120207/")
# print(dataset)
dataloader = ImageLoader(dataset.imgs, train_transform)

train = pd.read_csv('train2.csv', header=None,
                    names=['imgId', 'skc', 'spuId', 'cid1', 'cid2', 'leafCid', 'img', 'imgPath'], keep_default_na=False)
train = train[train['imgPath'] != '']

# train[:5]

train = train.reset_index(drop=True)

def RandomImagePrediction(filepath, i):
    img_array = Image.open(filepath).convert("RGB")
    data_transforms = transforms.Compose([
        transforms.Resize((224, 224)),
        transforms.ToTensor(),
        transforms.Normalize([0.5] * 3, [0.5] * 3)
    ])
    img = data_transforms(img_array).unsqueeze(
        dim=0)  # Returns a new tensor with a dimension of size one inserted at the specified position.
    load = DataLoader(img)

    for x in load:
        x = x.to(device)
        pred = model(x)
        _, preds = torch.max(pred, 1)
        #         print(f"class : {preds}")
        if preds[0] == 0:
            # print(0)
            train1.loc[i, 'imgPath'] = ''

#             return True
#         else:
#             return False

train1 = train[['imgId', 'skc', 'spuId', 'cid1', 'cid2', 'leafCid', 'img', 'imgPath']]
print(len(train1))
print(train1[:5])

for i in range(len(train)):
    RandomImagePrediction('D:/xiangsitu/imgs/' + train1.loc[i]['imgPath'], i)
#     if(not RandomImagePrediction('../input/jiafang-1w/imgs/imgs/' + train.iloc[i]['imgPath'])):
#         train.iloc[i]['imgPath']=''

# print(train1[:15])
train2 = train1[train1['imgPath'] != '']
print(len(train2))
train3 = train1[train1['imgPath'] == '']
print(len(train3))

outputpath1 = '120207/data.csv'
outputpath0 = '120207/test_result_0.csv'
train2.to_csv(outputpath1, sep=',', index=False, header=True)
train3.to_csv(outputpath0, sep=',', index=False, header=True)


#!/usr/bin/env python
"""
AER1415 Computer Optimization - Assignment 2

Author: Atilla Saadat
Submitted: Mar 31, 2021
Email: asaadat@utias-sfl.net

Question 1

"""

from numpy import *
import matplotlib
from matplotlib import pyplot as plt
from mpl_toolkits import mplot3d
from matplotlib import cm
import matplotlib.colors as colors

from rosen import *

def newtons_method(plot=False,verbose=False):
	#initialize parameters
	x0 = array([-1., -2.])
	x = [x0]
	vals = [rosen(x0)]
	#calculate first 5 iterates
	for iterNum in range(5):
		x_k = x[-1]
		g_k = rosen_gk(x_k)
		H_k = rosen_Hk_n2(x_k)

		#inverse hessian search direction calculation
		p_k = -dot(linalg.inv(H_k),g_k)
		x_k1 = x_k + p_k
		val_k1 = rosen(x_k1)

		x.append(x_k1)
		vals.append(val_k1)

	x = array(x)
	vals = array(vals)

	if verbose:
		for idx,x_k in enumerate(x):
			print('Iter: {} - x*: {}, val:: {}'.format(idx,x_k,vals[idx]))
	#plot and save contour
	if plot:
		print('Creating Contour Plot - Please Wait')
		X, Y = meshgrid(linspace(-1.5, 2, 500), linspace(-3, 3, 500))
		Z = array([rosen([j,i]) for i,j in zip(X.flatten(),Y.flatten())]).reshape(X.shape)
		fig, ax = plt.subplots()
		fig.set_size_inches(18.5, 10.5)
		pcm = ax.pcolormesh(X, Y, Z,norm=colors.LogNorm(vmin=Z.min(), vmax=Z.max()),cmap='RdBu_r',rasterized=True)
		ax.plot(*x.T,'-o',color='#16F41B',markersize=2)
		ax.set_xlabel(r'$x_1$')
		ax.set_ylabel(r'$x_2$')
		ax.set_title('Newtons method - Rosenbrock Test Function - n=2')
		fig.colorbar(pcm, ax=ax, extend='max',label=r'$f(x)$')
		print('Contour Plot Done!')
		plt.savefig('Q1_NewtonsMethod.svg', format = 'svg', bbox_inches = 'tight')
		plt.show()

	return {'x*': x[-1], 'val': vals[-1]}

newtons_method(plot=True,verbose=True)


import logging
import logging.config

date = "0816"
filePath1 = '../input/' + date + '/research.csv'
filePath2 = '../input/' + date + '/gasUsed.csv'
outputPath = '../input/' + date + '/output2.csv'
logOutPath = '../input/' + date + '/logOut.log'

# 1.logging格式的配置文件
def getLogging(confName="applog"):
    logging.config.fileConfig("logging.conf")
    return logging.getLogger(confName)

logger = getLogging()

if __name__ == '__main__':
    logger = getLogging()
    logger.info("wushibniadsadas")
    logger.info("Start print log")
    logger.debug("Do something")
    logger.warning("Something maybe fail.")
    logger.info("Finish")

# # 2.编程式使用
# if __name__ == '__main__':
#     # 创建记录器
#     logger = logging.getLogger("LOGOUT")
#     logger.setLevel(logging.DEBUG)  # 默认是WARING
#     # 关于setLevel的优先级，是一个层层过滤的关系好比用网来过滤，WARNING就是网眼更大的网
#     # 先用logger的网过滤，再用handler的网过滤，最后剩下的就是输出的
#
#     # 创建处理器handler
#     consoleHandler = logging.StreamHandler()
#     # consoleHandler.setLevel(logging.DEBUG)
#
#     fileHandler = logging.FileHandler(filename=logOutPath)  # 默认使用logger的日志输出级别，可以不指定
#     # fileHandler.setLevel(logging.DEBUG)
#
#     # 创建格式化器Formatters
#     formatter = logging.Formatter("%(asctime)s | %(levelname)8s | %(filename)s [line:%(lineno)d] => %(message)s",
#                                   "%Y-%m-%d %H:%M:%S")
#
#     # 给处理器设置格式
#     consoleHandler.setFormatter(formatter)
#     fileHandler.setFormatter(formatter)
#
#     # 记录器要设置处理器
#     logger.addHandler(consoleHandler)
#     logger.addHandler(fileHandler)
#
#     # # 定义一个过滤器
#     # flt = logging.Filter("cn.cccb")  # cn.cccb开头的logger为合法
#     # # 关联过滤器
#     # logger.addFilter(flt)
#     # fileHandler.addFilter(flt)
#
#     logger.info("Start print log")
#     logger.debug("Do something")
#     logger.warning("Something maybe fail.")
#     logger.info("Finish")
# 3.字典式配置


# -*- coding: utf-8 -*-

"""Notebook reader and writer."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import json

try:
    import nbformat as nbf
    from nbformat.v4.nbbase import validate
except ImportError:
    import IPython.nbformat as nbf
    from IPython.nbformat.v4.nbbase import validate

from ..lib.markdown import MarkdownFilter
from ..lib.python import PythonFilter
from ..ext.six import string_types
from ..utils.utils import _ensure_string

#------------------------------------------------------------------------------
# Utility functions
#------------------------------------------------------------------------------

def _cell_input(cell):
    """Return the input of an ipynb cell."""
    return _ensure_string(cell.get('source', []))

def _cell_output(cell):
    """Return the output of an ipynb cell."""
    outputs = cell.get('outputs', [])
    # Add stdout.
    stdout = ('\n'.join(_ensure_string(output.get('text', ''))
                        for output in outputs)).rstrip()
    # Add text output.
    text_outputs = []
    for output in outputs:
        out = output.get('data', {}).get('text/plain', [])
        out = _ensure_string(out)
        # HACK: skip <matplotlib ...> outputs.
        if out.startswith('<matplotlib'):
            continue
        text_outputs.append(out)
    return stdout + '\n'.join(text_outputs).rstrip()

def _compare_notebook_cells(cell_0, cell_1):
    return all((cell_0['cell_type'] == cell_1['cell_type'],
                _cell_input(cell_0) == _cell_input(cell_1),
                _cell_output(cell_0) == _cell_output(cell_1)))

def _compare_notebooks(nb_0, nb_1):
    return all(_compare_notebook_cells(cell_0, cell_1)
               for cell_0, cell_1 in zip(nb_0['cells'], nb_1['cells']))

#------------------------------------------------------------------------------
# Notebook reader
#------------------------------------------------------------------------------

class NotebookReader(object):
    """Reader for notebook cells.

    nbformat v4 only."""

    # Metadata that is basically never important enough to appear in text
    # formats.
    # TODO: expose this as configurable?
    ignore_meta = ["collapsed", "trusted", "celltoolbar"]

    def __init__(self):
        self._notebook_metadata = {}

    def read(self, nb):
        assert nb['nbformat'] >= 4

        yield {
            'cell_type': 'notebook_metadata',
            "metadata": nb['metadata']
        }

        for cell in nb['cells']:
            ipymd_cell = {}
            metadata = self.clean_meta(cell)
            if metadata:
                ipymd_cell['metadata'] = metadata
            ctype = cell['cell_type']
            ipymd_cell['cell_type'] = ctype
            if ctype == 'code':
                ipymd_cell['input'] = _cell_input(cell)
                ipymd_cell['output'] = _cell_output(cell)
            elif ctype == 'markdown':
                ipymd_cell['source'] = _ensure_string(cell['source'])
            else:
                continue
            yield ipymd_cell

    def clean_meta(self, cell):
        metadata = cell.get('metadata', {})
        for key in self.ignore_meta:
            metadata.pop(key, None)
        return metadata

#------------------------------------------------------------------------------
# Notebook writer
#------------------------------------------------------------------------------

class NotebookWriter(object):
    def __init__(self, keep_markdown=None, ipymd_skip=False):
        self._nb = nbf.v4.new_notebook()
        self._count = 1
        self._markdown_filter = MarkdownFilter(keep_markdown)
        self._code_filter = PythonFilter(ipymd_skip=ipymd_skip)

    def append_markdown(self, source, metadata=None):
        # Filter Markdown contents.
        source = self._markdown_filter(source)
        if not source:
            return
        self._nb['cells'].append(
            nbf.v4.new_markdown_cell(source,
                                     metadata=metadata))

    def append_code(self, input, output=None, image=None, metadata=None):
        input = self._code_filter(input)
        cell = nbf.v4.new_code_cell(input,
                                    execution_count=self._count,
                                    metadata=metadata)
        if output:
            cell.outputs.append(nbf.v4.new_output('execute_result',
                                {'text/plain': output},
                                execution_count=self._count,
                                metadata={},
                                ))
        if image:
            # TODO
            raise NotImplementedError("Output images not implemented yet.")
        self._nb['cells'].append(cell)
        self._count += 1

    def write_notebook_metadata(self, metadata):
        self._nb.metadata.update(metadata)

    def write(self, cell):
        metadata = cell.get("metadata", {})
        if cell['cell_type'] == 'markdown':
            self.append_markdown(cell['source'], metadata=metadata)
        elif cell['cell_type'] == 'code':
            self.append_code(cell['input'], cell['output'], metadata=metadata)

    @property
    def contents(self):
        validate(self._nb)
        return self._nb

NOTEBOOK_FORMAT = dict(
    reader=NotebookReader,
    writer=NotebookWriter,
    file_extension='.ipynb',
    file_type='json',
)


from Polynomial import Polynomial

def derivative(poly):
    """
    Function to calculate the derivative of a polynomial
    """
    newP = []
    for i,c in enumerate(poly.list):
        if i!=0:
            newP.append(i*c)

    return Polynomial(newP)

Polynomial.derivative = derivative

def area(poly, a, b):
    """
    Function to calculate the definite integral of a polynomial in the given range.
    """
    newP = [0]
    for i,c in enumerate(poly.list):
        newP.append(c/(i+1))

    newP = Polynomial(newP)
    return newP[b] - newP[a]

Polynomial.area = area

if __name__ == '__main__':
    # Test cases
    p = Polynomial([1, 2, 3])
    pd = p.derivative()
    print(pd)

    p = Polynomial([1, 2, 3])
    print(p.area(1,2))


import numpy as np
import random
import matplotlib
import matplotlib.pyplot as plt

def Var(array):
    mean = 0
    n = len(array)**3
    for k in range(0,len(array)):
        for i in range(0,len(array)):
            for j in range(0,len(array)):
                mean = mean + array[k,i,j]
    mean = mean/n

    summation = 0
    for k in range(0,len(array)):
        for i in range(0,len(array)):
            for j in range(0,len(array)):
                summation = summation + (x[k,i,j]-mean)**2

    var = summation/n
    return var

# test here

def Contour_plot(array,subarray):
    pixel_n = len(array)
    xlabel = np.arange(0,pixel_n)
    ylabel = np.arange(0,pixel_n)
    X, Y = np.meshgrid(xlabel,ylabel)
    Z = array[subarray]
    plt.contourf(X,Y,Z,20,cmap='RdGy')
    plt.xlabel('Pixel x location')
    plt.ylabel('Pixel y location')
    plt.xticks(np.arange(0,pixel_n,2))
    plt.yticks(np.arange(0,pixel_n,2))
    cb = plt.colorbar()
    cb.set_label('Field value')
    plt.show()

def Weight(array):
    n = len(array)
    for k in range(0,n):
        for i in range(0,n):
            for j in range(0,n):
                array[k,i,j] = (np.e**k) * array[k,i,j]

    return array


bl_info = {
    "name": "Swap Vertex Group Data",
    "description": "Swaps vertex group data between two vertex groups.",
    "author": "LQR471814",
    "blender": (2, 90, 0),
    "category": "Mesh",
}

import bpy

class OPERATOR_OT_SwapVertexGroups(bpy.types.Operator):
    bl_idname = "object.swap_vertex_groups"
    bl_label = "Swap Vertex Groups"
    bl_description = "Swaps vertex group data between two vertex groups."
    bl_options = {'REGISTER', 'UNDO'}

    g1 : bpy.props.StringProperty(name = "First Vertex Group", default = "")
    g2 : bpy.props.StringProperty(name = "Second Vertex Group", default = "")

    @classmethod
    def poll(cls, context):
        return context.active_object is not None

    def execute(self, context):
        selected = context.selected_objects

        for obj in selected:
            vg1 = obj.vertex_groups.get(self.g1)
            vg2 = obj.vertex_groups.get(self.g2)
            if vg1 == None:
                self.report({'ERROR'}, "First vertex group does not exist in object!")
                return {"CANCELLED"}
            elif vg2 == None:
                self.report({'ERROR'}, "Second vertex group does not exist in object!")
                return {"CANCELLED"}

            vg_idx1 = vg1.index
            vs1 = getVertexAndWeights(obj, vg_idx1)

            vg_idx2 = vg2.index
            vs2 = getVertexAndWeights(obj, vg_idx2)

            try:
                vg1.remove(list(vs1.keys()))
                vg2.remove(list(vs2.keys()))
            except RuntimeError:
                self.report({'ERROR'}, "This action cannot be performed in edit mode!")
                return {"CANCELLED"}

            for key in list(vs1.keys()):
                vg2.add([key], vs1[key], 'REPLACE')

            for key in list(vs2.keys()):
                vg1.add([key], vs2[key], 'REPLACE')

        return {"FINISHED"}

    def invoke(self, context, event):
        return context.window_manager.invoke_props_dialog(self)

def getVertexAndWeights(obj, grp_idx):
    vs = {}
    for v in obj.data.vertices:
        try:
            if grp_idx in [vg.group for vg in v.groups]:
                vs[v.index] = v.groups[0].weight
        except RuntimeError:
            vs[v.index] = 0
        except IndexError:
            vs[v.index] = 0
    return vs

def menu_func(self, context):
    self.layout.operator(OPERATOR_OT_SwapVertexGroups.bl_idname)

def register():
    bpy.utils.register_class(OPERATOR_OT_SwapVertexGroups)
    bpy.types.TOPBAR_MT_app_system.append(menu_func)

def unregister():
    bpy.utils.unregister_class(OPERATOR_OT_SwapVertexGroups)
    bpy.types.TOPBAR_MT_app_system.remove(menu_func)

if __name__ == "__main__":
    register()


#-*- coding: utf-8 -*-

from testlib import testcase
import sys

class TestCase(testcase.TestObject):
	def __init__(self, config='tests.conf'):
		testcase.TestObject.__init__(self, config)

		self.links = self.make_json_list('json_lists/deposit-update/links.json')
		self.edits = self.make_json_list('json_lists/deposit-update/edits.json')
		self.errors = self.make_json_list('json_lists/deposit-update/errors.json')

	def execute(self):
		if not testcase.TestObject.execute(self):
			self.log.write('error', 'login failed')
			return False

		for link in self.links:
			if not self.visit_dlink(link):
				self.log.write('error', 'error visiting %s, see above' % link['url'])
				return False

		if not self.click_btn('wire-transfer-title', by='id'):
			self.log.write('error', 'update link unclickable, see above')
			return False

		num_of_results = len(self.driver.find_elements_by_xpath('//tr')) - 1

		for edit in self.edits:
			if not self.edit_control(edit['name'], edit['value'], by='id', clear=bool(int(edit['clear']))):
				self.log.write('error', 'error editing %s, see above' % edit['name'])
				return False

			if not self.click_btn(u'Выписать счет', by='text'):
				self.log.write('error', 'submit not clicked, see above')
				return False

			error = self.errors[self.edits.index(edit)]
			if not (error['name'] == error['value'] == ''):
				if not self.check_error(error['name'], error['value'], bool(int(error['ok']))):
					self.log.write('error', 'error-text NOK, see above')
					return False
			else:
				if num_of_results >= len(self.driver.find_elements_by_xpath('//tr')) - 1:
					self.log.write('error', 'no new field appeared')
					return False

		self.log.write('info', '%s PASSED' % sys.argv[0])
		return True

if __name__ == '__main__':
	tc = TestCase()
	sys.exit(int(not tc.execute()))


from flask import Flask,request,render_template,url_for,redirect,make_response
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime
from flask_httpauth import HTTPBasicAuth

app = Flask(__name__,template_folder='template')
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///todo.db'
db = SQLAlchemy(app)
auth = HTTPBasicAuth()

users = {
    'Vivek':'123vivek',
    'Manas':'123manas',
    'Payas':'123payas'
}

@auth.verify_password                                             #authentication
def verify_password(username, password):
    if username in users and users[username] == password:
        return username

#DATABASE MODEL
class Todo(db.Model):
    task_id = db.Column(db.Integer, primary_key = True)
    name = db.Column(db.String(100),nullable = False)
    task_detail = db.Column(db.String(200), nullable = False)
    time = db.Column(db.DateTime, default = datetime.utcnow)

    #from assignment.py import db
    #db.create_all()

    def __repr__(self):
        return f"Task: {self.task_id} created by {self.name}"

#ADD TASK POST REQUEST
@app.route('/',methods = ['POST','GET'])
def test():
    if(request.method == 'POST'):
        request_name = request.form['task_name']
        request_details = request.form['details']
        new = Todo(task_detail = request_details, name = request_name)

        try:
            db.session.add(new)
            db.session.commit()
            return redirect('/')

        except:
            return f"Something is wrong!"

    else:
        return render_template('index.html', tasks = Todo.query.all())

#DELETE TASK
@app.route('/delete/<int:task_id>')
@auth.login_required
def del_func(task_id):
    _id = Todo.query.get_or_404(task_id)

    try:
        db.session.delete(_id)
        db.session.commit()
        return redirect('/')

    except:
        return f"There some problem with the deletion."

#UPDATE TASK
@app.route('/update/<int:task_id>',methods = ['POST','GET'])
@auth.login_required
def update(task_id):
    Task = Todo.query.get_or_404(task_id)
    if request.method == 'POST':
        Task.task_detail = request.form['details']
        Task.name = request.form['task_name']

        try:
            db.session.commit()
            return redirect('/')

        except:
            return f"There's some error with the updation!"
    else:
        return render_template('update.html',task = Task)

if __name__ == '__main__':
    app.run(debug=True)


import zipfile

search_string = 'verifying'

zip = zipfile.ZipFile('news.zip')

for filename in zip.namelist():

   file = zip.open(filename)

   for line_no, line in enumerate(file, 1):

       if search_string.lower() in line.lower():
           print( filename + ' : ' + str(line_no) + ' : ' + line)

   file.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys

def kangaroo(x1, v1, x2, v2):
    # Complete this function

    k1 = x1
    k2 = x2

    if k1 == k2:
        return "YES"

    if k1 != k2 and v1 == v2:
        return "NO"

    if (k1 < k2 and v1 < v2) or (k2 < k1 and v2 < v1):
        return "NO"

    prevDist = k2 - k1

    if prevDist > 0:
        flag = True
    else:
        flag = False

    while True:
        k1 = k1 + v1
        k2 = k2 + v2
        diff = k2 - k1

        if prevDist < 0 and flag == True:
            return "NO"

        if prevDist < diff + 1:
            return "NO"

        if k1 == k2:
            return "YES"

        prevDist = diff

x1, v1, x2, v2 = input().strip().split(' ')
x1, v1, x2, v2 = [int(x1), int(v1), int(x2), int(v2)]
result = kangaroo(x1, v1, x2, v2)
print(result)


from datetime import datetime

from kfp.v2 import compiler
from kfp.v2.google.client import AIPlatformClient

from pipelines.src.config_loader import load_pipeline_config
from pipelines.src.tutorial import inference_pipeline as inference
from pipelines.src.tutorial import training_pipeline as simple
from pipelines.src.tutorial import training_pipeline_parallelized as parallel
from pipelines.src.tutorial import training_pipeline_with_conditiion as conditional
from pipelines.src.utils import upload_file_to_gcs

def compile_and_run_pipeline(
    sync: bool = False,
    enable_caching: bool = False,
    name: str = None,
    **kwargs,
) -> None:
    """
    Compile the pipeline and run it just after, using the generated JSON file.

    :param sync: defaults to True
    :param enable_caching: defaults to False
    """
    package_path = compile_pipeline(name=name, local=True)
    run_pipeline(
        package_path,
        sync=sync,
        enable_caching=enable_caching,
    )

def compile_pipeline(local: bool = True, **kwargs) -> str:
    """
    Compile the pipeline.

    - Compile the pipeline to a .json file spec saved locally

    :param local: if set to False, store the JSON pipeline file in Cloud Storage.

    :return: The package path created by the compilation. This can be used by the `run_pipeline` function
    """
    # load config
    config = load_pipeline_config()
    pipeline_name = f"{config['uc_name']}-{config['experiment_name']}".replace("_", "-")

    # compile pipeline
    package_path_no_timestamp = f"{config['PACKAGE_PATH']}/{pipeline_name}.json"
    package_path_timestamp = (
        f"{config['PACKAGE_PATH']}/{pipeline_name}-{_get_timestamp_str()}.json"
    )

    def pipeline_func():
        if "simple" in config["experiment_name"]:
            return simple()
        elif "conditional" in config["experiment_name"]:
            return conditional()
        elif "parallel" in config["experiment_name"]:
            return parallel()
        elif "inference" in config["experiment_name"]:
            return inference()

    for package_path in [package_path_no_timestamp, package_path_timestamp]:
        compiler.Compiler().compile(
            pipeline_func=pipeline_func,
            package_path=package_path,
            pipeline_name=pipeline_name,
            type_check=True,
        )
        if not local:
            upload_file_to_gcs(package_path, config["BUCKET_PIPELINE_CONFIGS"])
    return package_path

def run_pipeline(
    package_path: str,
    sync: bool = False,
    enable_caching: bool = False,
) -> None:
    """
    Submit the pipeline to Vertex Pipelines.

    :param package_path: The path to the JSON file used to run the pipeline: either local or GCS URI
    :param sync: if set to True, this function will wait for the the pipeline job to finish before continuing
    :param enable_caching: if set to True, the vertex pipeline will only run from the last component that has
        been changed. Components before this will not be run and artifacts from the previous vertex pipeline
        job will be used. This is very practical during development to test new changes in a specific component.
    """
    config = load_pipeline_config()
    if sync:
        raise NotImplementedError(
            "The syncronous run of the vertex pipeline job is no longer implemented."
        )
    api_client = AIPlatformClient(
        project_id=config["PROJECT_ID"], region=config["REGION"]
    )
    service_account = f"{config['SERVICE_ACCOUNT_NAME']}@{config['PROJECT_ID']}.iam.gserviceaccount.com"

    api_client.create_run_from_job_spec(
        job_spec_path=package_path,
        pipeline_root=config["PIPELINE_ROOT"],
        enable_caching=enable_caching,
        service_account=service_account,
        labels=None,
    )

def run_latest_pipeline(
    sync: bool = False,
    enable_caching: bool = False,
    local: bool = False,
    **kwargs,
) -> None:

    package_path = get_last_pipeline_config_path(local=local)
    run_pipeline(
        package_path,
        sync=sync,
        enable_caching=enable_caching,
    )

def get_last_pipeline_config_path(local: bool = False) -> str:
    """Find the last config of compiled pipeline, either in local or on GCS.

    :param local: Whether to find for the last config in local or on the GCS bucket, defaults to False

    :return: The path to the last config JSON file
    """
    config = load_pipeline_config()

    if local:
        path_prefix = f'{config["PACKAGE_PATH"]}/'
    else:
        path_prefix = f"gs://{config['BUCKET_PIPELINE_CONFIGS']}/"

    valid_name = f"{config['uc_name']}-{config['experiment_name']}".replace("_", "-")

    return f"{path_prefix}{valid_name}.json"

def _get_timestamp_str():
    timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M")
    return timestamp


import time

class Noeud:
    def __init__(self, nom, voisins, id, coords, station):
        """Constructeur de la class Noeud.

        Args:
            nom (str): nom du noeud
            voisins (dict): liste des noeud voisins
            id (int): identifien du noeud
            coords (tuple): Les coordonnées du noeud
            station (bool): Variable boolean pour savoir si le noeud est une station
        """
        self.nom = nom
        self.voisins = voisins
        self.id = id
        self.coords = coords
        self.station = station

    def __str__(self):
        return self.nom

    def __repr__(self):
        return self.nom

class Graphe:
    """_summary_
    """
    def __init__(self, noeuds):
        self.noeuds = noeuds

    def __str__(self):
        return str(self.noeuds)

    def __repr__(self):
        return str(self.noeuds)

    def ajouter_noeud(self, noeud):
        """ajoute un noeud à la liste

        Args:
            noeud (Node): noeud a ajouter
        """
        self.noeuds.append(noeud)

    def ajouter_arete(self, noeud1, noeud2, poids_d, poids_e, couleur):
        """Ajoute une arête entre deux noeuds

        Args:
            noeud1 (Node): Noeud de départ
            noeud2 (Node): Noeud d'arrivée
            poids_d (int): Poids du noeud de départ
            poids_e (int): Poids du noeuds d'arrivé
            couleur (str): couleur de la piste
        """
        self.noeuds[self.noeuds.index(noeud1)].voisins[noeud2] = (
            poids_d, poids_e, couleur)

    def supprimer_noeud(self, noeud):
        """Supprime un noeud de la liste

        Args:
            noeud (Node): Noeud à supprimer
        """
        self.noeuds.remove(noeud)

    def supprimer_arete(self, noeud1, noeud2):
        """Supprime une arête de la liste

        Args:
            noeud1 (Node): Noeud de départ de l'arête
            noeud2 (Node): Noeud d'arrivé de la liste
        """
        del self.noeuds[noeud1.id].voisins[noeud2]

    def get_noeud(self, nom):
        """Retourne un noeud selon son nom

        Args:
            nom (str): Nom du noeud

        Returns:
            Node: Le noeud correspondant au nom
        """
        for noeud in self.noeuds:
            if noeud.nom == nom:
                return noeud

    def get_noeuds(self):
        """Renvoie la liste des noeuds

        Returns:
            lst: liste des noeuds
        """
        return self.noeuds

    def get_voisins(self, noeud):
        """Renvoie un dictionnaire des noeuds voisins à un noeuds spécifique

        Args:
            noeud (Node): Noeud sélectionné

        Returns:
            dict: Voisins du noeud sélectionné
        """
        return self.noeuds[self.noeuds.index(noeud)].voisins
        # return self.noeuds[noeud.id].voisins

    def get_couleur(self, noeud):
        """_summary_

        Args:
            noeud (Node): Noeud sélectionné

        Returns:
            str: Renvoie la couleur du noeud
        """
        return self.noeuds[noeud.id].couleur

    def get_noeuds_couleur(self, couleur):
        """_summary_

        Args:
            couleur (str): couelur du noeud

        Returns:
            lst: liste de noeud
        """
        noeuds = []
        for noeud in self.noeuds:
            if noeud.couleur == couleur:
                noeuds.append(noeud)
        return noeuds

def dijkstra(graphe, depart, arrivee, niveau_skieur):
    """Algorithme permettant de trouver le plus court chemin entre deux noeuds d'un graphe en fonction du niveau du skieur

    Args:
        graphe (Graphe): Graphe où l'algorithme est appliqué
        depart (Node): Noeud de départ
        arrivee (Node): Noeud d'arrivé
        niveau_skieur (str): Le niveau du skieur

    Returns:
        lst*int: chemin*cout du chemin
    """
    t = time.time()

    # Initialisation
    noeuds = graphe.get_noeuds()
    initialisation(noeuds, depart, "dijkstra")

    # Boucle principale
    noeuds_a_traiter = noeuds.copy()
    while len(noeuds_a_traiter) > 0:
        # On prend le noeud avec le cout le plus faible
        noeud = min(noeuds_a_traiter, key=lambda x: x.cout)
        # On le retire de la liste des noeuds à traiter
        noeuds_a_traiter.remove(noeud)
        # On regarde les voisins du noeud
        if niveau_skieur == "debutant":
            for voisin, (poids_voisin, _, couleur) in graphe.get_voisins(noeud).items():
                # On calcule le cout du chemin
                cout_chemin = plus_court_chemin(
                    noeud, poids_voisin, couleur, "d")
                # On regarde si le chemin est plus court
                verif_plus_court_chemin(
                    noeud, voisin, cout_chemin, arrivee, "dijkstra")
        elif niveau_skieur == "expert":
            for voisin, (_, poids_voisin, couleur) in graphe.get_voisins(noeud).items():
                # On calcule le cout du chemin
                cout_chemin = plus_court_chemin(
                    noeud, poids_voisin, couleur, "d")
                # On regarde si le chemin est plus court
                verif_plus_court_chemin(
                    noeud, voisin, cout_chemin, arrivee, "dijkstra")

    # On reconstruit le chemin
    chemin = reconstruction_chemin(arrivee)
    print("Chemin le plus court de {} à {} :".format(depart.nom, arrivee.nom))
    print(chemin, '---------------')
    print("Temps d'execution : {} secondes".format(time.time() - t))
    return chemin, arrivee.cout

def astar(graphe, depart, arrivee, niveau_skieur):
    t = time.time()
    # Initialisation
    noeuds = graphe.get_noeuds()
    initialisation(noeuds, depart, "astar")

    # Boucle principale
    noeuds_a_traiter = noeuds.copy()
    while len(noeuds_a_traiter) > 0:
        # On prend le noeud avec le cout le plus faible
        noeud = min(noeuds_a_traiter, key=lambda x: x.cout_estime)
        # On le retire de la liste des noeuds à traiter
        noeuds_a_traiter.remove(noeud)
        # On regarde les voisins du noeud
        if niveau_skieur == "debutant":
            for voisin, (poids_voisin, _, couleur) in graphe.get_voisins(noeud).items():
                # On calcule le cout du chemin
                cout_chemin = plus_court_chemin(
                    noeud, poids_voisin, couleur, "d")
                # On regarde si le chemin est plus court
                verif_plus_court_chemin(
                    noeud, voisin, cout_chemin, arrivee, "astar")
        elif niveau_skieur == "expert":
            for voisin, (_, poids_voisin, couleur) in graphe.get_voisins(noeud).items():
                # On calcule le cout du chemin
                cout_chemin = plus_court_chemin(
                    noeud, poids_voisin, couleur, "e")
                # On regarde si le chemin est plus court
                verif_plus_court_chemin(
                    noeud, voisin, cout_chemin, arrivee, "astar")

    # On reconstruit le chemin
    chemin = reconstruction_chemin(arrivee)
    print("Chemin le plus court de {} à {} :".format(depart.nom, arrivee.nom))
    print(chemin, '---------------')
    print("Temps d'execution : {} secondes".format(time.time() - t))
    return chemin, arrivee.cout

def initialisation(noeuds, depart, algo):
    """Initialisation pour l'algorithme d'astar

    Args:
        noeuds (node): Noeud actuel
        depart (Node): Noeud de départ
        algo (str): algorrithme
    """
    for noeud in noeuds:
        noeud.cout = float("inf")
        if algo == "astar":
            noeud.cout_estime = float("inf")
        noeud.precedent = None
    depart.cout = 0
    if algo == "astar":
        depart.cout_estime = 0

def plus_court_chemin(noeud, poids_voisin, couleur, niveau_skieur):
    """Permet de choisir le niveau du skieur

    Args:
        noeud (node): Noeud actuel
        poids_voisin (int): Poids entre le noeud actuel et son noeud voisin
        couleur (str): type de piste de ski
        niveau_skieur (str): Niveau du skieur

    Returns:
        int: Poids du chemin
    """
    if niveau_skieur == "d":
        if couleur == "bleu":
            cout_chemin = noeud.cout + poids_voisin*2
        elif couleur == "rouge":
            cout_chemin = noeud.cout + poids_voisin*5
        elif couleur == "noir":
            cout_chemin = noeud.cout + poids_voisin*8
        elif couleur == "téléphérique":
            cout_chemin = noeud.cout + poids_voisin*3
        elif couleur == "téléski":
            cout_chemin = noeud.cout + poids_voisin*4
        elif couleur == "télésiège":
            cout_chemin = noeud.cout + poids_voisin*5
        else:
            cout_chemin = noeud.cout + poids_voisin
    else:
        if couleur == "bleu":
            cout_chemin = noeud.cout + poids_voisin*1.5
        elif couleur == "rouge":
            cout_chemin = noeud.cout + poids_voisin*2
        elif couleur == "noir":
            cout_chemin = noeud.cout + poids_voisin*3
        elif couleur == "téléphérique":
            cout_chemin = noeud.cout + poids_voisin*3
        elif couleur == "téléski":
            cout_chemin = noeud.cout + poids_voisin*4
        elif couleur == "télésiège":
            cout_chemin = noeud.cout + poids_voisin*5
        else:
            cout_chemin = noeud.cout + poids_voisin
    return cout_chemin

def verif_plus_court_chemin(noeud, voisin, cout_chemin, arrivee, algo):
    """Vérifie que le chemin choisie est le plus court

    Args:
        noeud (Node): Noeud choisie
        voisin (Node): Noeud voisin
        cout_chemin (int): Poids du noeud
        arrivee (Node): Noeud d'arrivé
        algo (str): algorithme chosit
    """
    if cout_chemin < voisin.cout:
        voisin.cout = cout_chemin
        voisin.precedent = noeud
        if algo == "astar":
            voisin.cout_estime = cout_chemin + \
                abs(voisin.coords[0] - arrivee.coords[0]) + \
                abs(voisin.coords[1] - arrivee.coords[1])

def reconstruction_chemin(arrivee):
    """_summary_

    Args:
        arrivee (Node): Noeud d'arrivé

    Returns:
        lst: Liste de noeud correspondant au chemin
    """
    chemin = []
    noeud = arrivee
    while noeud is not None:
        chemin.append(noeud)
        noeud = noeud.precedent
    chemin.reverse()
    return chemin


"""ファイル仲介モジュール"""

import re
import os

class FileInterface:
    @staticmethod
    def read(path):
        """指定したファイルのテキストを返す関数"""
        try:
            with open(path, 'r') as f:
                text = f.read().strip()
        except FileNotFoundError:
            text = ""
        return text

    @staticmethod
    def write(path, text):
        """指定したファイルに指定されたテキストを書き込む関数"""
        try:
            with open(path, 'w') as f:
                f.write(text)
        except FileNotFoundError:
            with open(path, 'x') as f:
                f.write(text)

    @staticmethod
    def search(target_name, orgin_dir):
        """指定した名前のファイルを指定したディレクトリから探索し、そのパスを返す関数"""
        for root, dirs, files in os.walk(orgin_dir):
            if target_name in files:
                return os.path.join(root, target_name)
        return None  # ファイルが見つからなかった場合はNoneを返す

    @staticmethod
    def compress(text):
        """文字列を短くする関数"""
        # 連続した記号の排除
        pattern = re.compile(r'([^a-zA-Z0-9\s])\1+')
        non_double_symbol = pattern.sub(r'\1', text)
        # 空行の排除
        lines = non_double_symbol.split("\n")
        non_blank_lines = [line for line in lines if line.strip() != ""]
        return "\n".join(non_blank_lines)

    @staticmethod
    def head_slice(text, max_chars):
        """指定した文字数以上の場合、先頭を優先して返す関数"""
        return text if len(text) < max_chars else text[:max_chars]

    @staticmethod
    def tail_slice(text, max_chars):
        """指定した文字数以上の場合、末尾を優先して返す関数"""
        return text if len(text) < max_chars else text[-max_chars:]


from os import path
import os
from pathlib import Path
import re
from .sysx import *

is_abs = path.isabs # 判断是否是绝对路径
is_file = path.isfile # 判断是否是文件
is_dir = path.isdir # 判断是否是文件夹
is_link = is_symlink = path.islink # 重命名判断软链接的方法
is_exist = path.exists # 判断路径是否存在

__all__ = (
    "is_abs",
    "is_file",
    "is_dir",
    "is_link",
    "is_exist",
    "not_exist",
    "is_rel",
    "cwd",
    "dirname",
    "basename",
    "to_abs",
    "is_symlink"
)

def not_exist(*args, **kwargs):
    """
    判断路径是否不存在
    """
    return not is_exist(*args, **kwargs)

def is_rel(p):
    """
    判断路径是否是相对路径
    p 参数是一个 路径字符串或者 Path 对象
    """
    return not is_abs(p)

def cwd(path_str=None):
    """
    获取或者设置当前的工作目录
    如果给出了 path_str 那么就是设置 没有给出就是获取当前的工作目录
    """
    if path_str is None:
        return os.getcwd()
    else:
        if is_rel(path_str):
            path_str = Path(cwd()) / path_str
        return os.chdir(path_str)

def dirname(path_str):
    """
    传入一个路径返回这个目标路径的上一个文件夹路径
    """
    # 判断路径是否是以 / 或者 \ 结尾的如果是去掉在返回 path.dirname 的处理结果

    if path_str.endswith(r"/") or path_str.endswith("\\"):
        path_str = path_str[0:len(path_str)-1]
    return path.dirname(path_str)

def basename(path_str):
    """
    传入一个路径返回这个目录路径指向的那个文件或者文件夹
    """
    if path_str.endswith(r"/") or path_str.endswith("\\"):
        path_str = path_str[0:len(path_str)-1]
    return path.basename(path_str)

def to_abs(path_arg):
    """
    传入一个路径字符串或者一个Path将一个路径转换成绝对路径, 返回路径字符串
    """
    if is_abs(path_arg):
        return path_arg

    path_str = str(Path(cwd()) / path_arg)

    path_list = re.split(r"[\\/]",path_str)
    path_list = [get_env(i) if i.startswith("%") and i.endswith("%") else i
                 for i in path_list]

    path_str = str(Path("/".join(path_list)))
    return path_str


from flask import Flask, request, render_template
import datetime, os

app = Flask(__name__)

@app.route("/01-form", methods=["GET","POST"])
def form_views():
    if request.method == "GET":
        return render_template("01-form.html")
    else:
        uname = request.form.get('uname')
        hobby = request.form.getlist("hobby")
        country = request.form.getlist("country")
        print(uname)
        print(hobby)
        print(country)
        return "data recv OK"

@app.route("/02-file", methods=["GET", "POST"])
def file_views():
    if request.method == "GET":
        return render_template("02-file.html")
    else:
        uname = request.form["uname"]
        uimg = request.files["uimg"]
        now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")
        new_name = now_time + uimg.filename
        try:
            basedir = os.path.dirname(__file__)
            name = os.path.join(basedir,"static",new_name)
            uimg.save(name)
        except Exception as e:
            print("update fail...")
            print(e)
        return "update OK..."

@app.route("/03-release", methods=["GET","POST"])
def release_views():
    if request.method == "GET":
        return render_template("03-release.html")
    else:
        utitle = request.form["utitle"]
        utype = request.form["utype"]
        uwrite = request.form["uwrite"]
        print("utitle:",utitle)
        print("utype:", utype)
        print("uwrite:",uwrite)
        if request.files["uimg"]:
            filename = request.files["uimg"]
            generate_filename(filename)
            return "Img saved ok."
        return "No img."

def generate_filename(filename):
    basedir = os.path.dirname(__file__)
    now_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")
    new_name = now_time + filename.filename
    img_url = os.path.join(basedir,"static",new_name)
    filename.save(img_url)

if __name__ == "__main__":
    app.run(debug=True)


def isolate_cries(sDir, iResample = 8000, iSigmaSecs=0.05):
    """
    expected peak corrolation by convolution, then edge detection
    iResample = 8000 Hz resampling rate
        Hard coded variables obtained by trial:
            linerar smoothing coefficient:  iLinearSmooth = 20 frames
            minima threshhold factor:  iThreshFact = 10 times
            horozontal cluster threshhold:  iThreshCluster = 0.01 seconds * iSampleRate
            keep sections threshhold factor:  iThreshKeepFact = 5
    can save each cut to separate file
    returns list of cuts, each cut is peak data (correlation with gaussian)
    """
    import numpy as np
    from scipy import signal
    import common_fxns

    iLinearSmooth = 50

    (iSampleRate,
        aTime,
        aAudio,
        aCorr,
        aOrigAudio) = common_fxns.process_dir(sDir, iSmooth=iLinearSmooth, iResample=iResample, iSigmaSecs=iSigmaSecs)

    iThreshFact = 5.0
    iThreshClusterSecs = 0.01
    iThreshCluster = iThreshClusterSecs * iSampleRate
    iThreshKeepFact = 4.0

    from scipy import ndimage
    aFirstDeriv = ndimage.sobel(aCorr)

    aFirstDeriv = common_fxns.smooth(aFirstDeriv, iLinearSmooth)
    aSecondDeriv = ndimage.sobel(aFirstDeriv)

    #---Get minima and filter
    #intersecion of derivative with zero line (get maxima and minima)
    aZero = np.zeros(np.shape(aSecondDeriv))
    iTolerance = 10.0        #tolerance to zero-intersect
    aZeroIntersect = np.argwhere(np.isclose(aZero, aFirstDeriv, atol=iTolerance)).reshape(-1)

    #positive second derivitave (get minima)
    aMinimaIndeces = np.array([ i for i in aZeroIntersect if aSecondDeriv[i] > 0 ])

    #apply threshold filter for minima (y axis filter)
    iMax = np.max(aCorr)
    iMin = np.min(aCorr)
    iAmplThresh = (iMax - iMin)/iThreshFact      #we only want to keep low minima
    aMinimaIndeces = np.array( [i for i in aMinimaIndeces if aCorr[i] < iAmplThresh] )

    #apply proximity filter to single out consecutive groups of minima
    lMinimaIndeces = []
    for i in range(len(aMinimaIndeces) - 1):
        if aMinimaIndeces[i+1] - aMinimaIndeces[i] > iThreshCluster:
            lMinimaIndeces.append(aMinimaIndeces[i])

    if len(aMinimaIndeces) >0:
        lMinimaIndeces.append(aMinimaIndeces[-1])

    lMinimaIndecesFilt = [int(i) for i in lMinimaIndeces]

    #add the beginning and end of the track
    lMinimaIndecesFilt.insert(0, 0)
    lMinimaIndecesFilt.insert(len(lMinimaIndecesFilt), int(len(aCorr)-1))

    #get time values from indeces
    aTimeIntersects = np.array(aTime)[lMinimaIndecesFilt]

    #cut up audio according to minima indeces
    iMin = 1
    iCut = 1

    laAudios = []
    laCorrs = []

    while iMin < len(lMinimaIndecesFilt):
        iStart = lMinimaIndecesFilt[iMin -1]
        iEnd = lMinimaIndecesFilt[iMin]

        aAudioCut = np.array(aAudio[iStart : iEnd])
        aCorrAudioCut = np.array(aCorr[iStart : iEnd])
        aOrigAudioCut = np.array(aOrigAudio[iStart : iEnd])

        if aCorrAudioCut.any():
            #keep only peaks with a minimum amplitude
            bKeep = (np.max(aCorrAudioCut) - np.min(aCorrAudioCut)) > np.max(aCorr)/iThreshKeepFact
        else:
            bKeep = False

        #throw away sections with max amplitude less than max/iThreshKeepFact
        if bKeep:
            laAudios.append(aAudioCut)
            laCorrs.append(aCorrAudioCut)

            iCut += 1

        iMin += 1

    return iSampleRate, laAudios, laCorrs


# way too long but it works

def swap_case(s):
    words = s.split()
    new_words = ""
    # for each space seperated string in dataset
    for word in words:
        # for each letter in words, swapcase
        new_word = ""
        for letter in word:
            if letter.isupper():
                new_word += letter.lower()
            else:
                new_word += letter.upper()
        new_words += new_word + " "
    new_words = new_words[:-1]
    return new_words


#!/usr/bin/env python3

#ENPM673 Spring 2022
#Section 0101
#Jerry Pittman, Jr. UID: 117707120
#jpittma1@umd.edu
#Project #2 Functions

import numpy as np
import cv2
import scipy
from scipy import fft, ifft
from numpy import histogram_bin_edges, linalg as LA
import matplotlib.pyplot as plt
import sys
import math
import os
from os.path import isfile, join

'''Problem 1 Functions'''
def convertImagesToMovie(folder):
    fps =   3
    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    videoname=('night_drive')
    frames = []

    files = [f for f in os.listdir(folder) if isfile(join(folder, f))]

    #pictures were not in order, sadly
    files.sort(key = lambda x: x[5:-4])
    files.sort()

    for i in range(len(files)):
        filename=folder + files[i]
        img = cv2.imread(filename)
        height, width, layers = img.shape
        size = (width,height)
        # cv2.imshow('image', img)
        # print("size is ", size)
        if img is not None:
            frames.append(img)
        else:
            print("Failed to read")

    video = cv2.VideoWriter(str(videoname)+".avi",fourcc, fps, size)

    for i in range(len(frames)):
        # writing to a image array
        video.write(frames[i])

    #convert heightxwidth from 370,1224 to 480x640so divisible by 8
    cap=cv2.VideoCapture(str(videoname)+".avi")
    size = (640, 480)
    # size = (1280, 720)
    video_new = cv2.VideoWriter(str(videoname)+".avi",fourcc, fps, size)

    while True:
        ret,frame_new=cap.read()
        if ret==True:
            b=cv2.resize(frame_new,size,fx=0,fy=0, interpolation = cv2.INTER_CUBIC)
            video_new.write(b)
        else:
            break
    video_new.release()
    video.release()
    cap.release()
    cv2.destroyAllWindows()

    return video_new

def createHistogramFigure(img, name):

    plt.hist(img.ravel(),256,[0,256])
    # plt.show()
    plt.savefig(name)
    print("Histogram figure saved as: ", name)

def createHistogram(input):
    hist_vals = list()

    for i in range(256):
        c = np.where(input == i)
        hist_vals.append( [ i , len(c[0]) ] )

    return hist_vals

def conductHistogramEqualization(img):
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)

    #taking the V channel of the hsv image
    vlist = hsv[:,:,2]

    height,width = vlist.shape
    # height,width, _ = img.shape

    hist = createHistogram(vlist)

    '''Cumulative Distibution Function
    Number of pixels with intensity less than or equal to "i" intensity
    normalized by N pixels'''
    cdf = list()
    z = 0
    for i in range(len(hist)):
        z = z + (hist[i][1]/(height * width))
        cdf.append(round(z*255))

    hnew = np.asarray(cdf)

    #Equalize Image based on CDF results
    #----Modifiy Images of video----
    hsv[:,:,2] = hnew[hsv[:,:,2]]

    color = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)

    # cv2.imshow('HSV Equalized',color)

    return hist, color

def conductAdaptiveHistogramEqualization(image):
    # img = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    # image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    # print("img.shape: ",img.shape)
    # img=image.copy

    '''Define Clip value and search window'''
    clipLimit=0.001
    nrBins=256

    kernel_size = tuple([max(s // 8, 1) for s in image.shape])
    # kernel_size = tuple([max(s // 8, 1) for s in img.shape])
    # print("kernel size: ", kernel_size) #60,80, 1

    kernel_size = [int(k) for k in kernel_size]
    # print("kernel size: ", kernel_size) #60,80, 1

    # grayscaleLevels=7800   #as many grayscale levels as possible; this gave me best results
    grayscaleLevels=2**14   #as many grayscale levels as possible; this gave me best results
    # grayscaleLevels=height*width
    ndim = image.ndim
    dtype = image.dtype

    # pad the image such that the shape in each dimension
    # - is a multiple of the kernel_size and
    # - is preceded by half a kernel size
    pad_start_per_dim = [k // 2 for k in kernel_size]

    pad_end_per_dim = [(k - s % k) % k + int(np.ceil(k / 2.))
                       for k, s in zip(kernel_size, image.shape)]

    image = np.pad(image, [[p_i, p_f] for p_i, p_f in
                           zip(pad_start_per_dim, pad_end_per_dim)],
                   mode='reflect')

    # determine gray value bins
    bin_size = 1 + grayscaleLevels // nrBins
    lut = np.arange(grayscaleLevels, dtype=np.min_scalar_type(grayscaleLevels))
    lut //= bin_size

    image = lut[image]

    # calculate graylevel mappings for each contextual region
    # rearrange image into flattened contextual regions
    ns_hist = [int(s / k) - 1 for s, k in zip(image.shape, kernel_size)]
    hist_blocks_shape = np.array([ns_hist, kernel_size]).T.flatten()
    hist_blocks_axis_order = np.array([np.arange(0, ndim * 2, 2),
                                       np.arange(1, ndim * 2, 2)]).flatten()

    hist_slices = [slice(k // 2, k // 2 + n * k)
                   for k, n in zip(kernel_size, ns_hist)]

    hist_blocks = image[tuple(hist_slices)].reshape(hist_blocks_shape)
    hist_blocks = np.transpose(hist_blocks, axes=hist_blocks_axis_order)
    hist_block_assembled_shape = hist_blocks.shape
    hist_blocks = hist_blocks.reshape((np.product(ns_hist), -1))

    # Calculate actual clip limit
    if clipLimit > 0.0:
        clim = int(np.clip(clipLimit * np.product(kernel_size), 1, None))
    else:
        # largest possible value, i.e., do not clip (AHE)
        clim = np.product(kernel_size)

    # smallestClip=np.product(kernel_size)
    # print("smallest clip, ", smallestClip)
    # print("current clip is ", int(np.clip(clipLimit * np.product(kernel_size), 1, None)))
    # clim=40 #openCV is 40 by default, supposedly

    hist = np.apply_along_axis(np.bincount, -1, hist_blocks, minlength=nrBins)

    #Clip_histogram
    clipped_hist = np.apply_along_axis(clipHistogram, -1, hist, clip_limit=clim)

    '''Create look-up table (LUT) is used to convert the dynamic range
    of the input image into the desired output dynamic range.'''
    #Map Histogram
    hist_map = createLUT(clipped_hist, 0, grayscaleLevels - 1, np.product(kernel_size))
    hist_map = hist_map.reshape(hist_block_assembled_shape[:ndim] + (-1,))

    # duplicate leading mappings in each dimension
    map_array = np.pad(hist_map,
                       [[1, 1] for _ in range(ndim)] + [[0, 0]],
                       mode='edge')

    '''Perform multilinear interpolation of graylevel mappings'''
    '''To remove artifacts after Adaptive Histogram Equalization'''
    # rearrange image into blocks for vectorized processing
    ns_proc = [int(s / k) for s, k in zip(image.shape, kernel_size)]
    blocks_shape = np.array([ns_proc, kernel_size]).T.flatten()
    blocks_axis_order = np.array([np.arange(0, ndim * 2, 2),
                                  np.arange(1, ndim * 2, 2)]).flatten()
    blocks = image.reshape(blocks_shape)
    blocks = np.transpose(blocks, axes=blocks_axis_order)
    blocks_flattened_shape = blocks.shape
    blocks = np.reshape(blocks, (np.product(ns_proc),
                                 np.product(blocks.shape[ndim:])))

    # calculate interpolation coefficients
    coeffs = np.meshgrid(*tuple([np.arange(k) / k
                                 for k in kernel_size[::-1]]), indexing='ij')
    coeffs = [np.transpose(c).flatten() for c in coeffs]
    inv_coeffs = [1 - c for dim, c in enumerate(coeffs)]

    # sum over contributions of neighboring contextual
    # regions in each direction
    result = np.zeros(blocks.shape, dtype=np.float32)
    for iedge, edge in enumerate(np.ndindex(*([2] * ndim))):

        edge_maps = map_array[tuple([slice(e, e + n)
                                     for e, n in zip(edge, ns_proc)])]
        edge_maps = edge_maps.reshape((np.product(ns_proc), -1))

        # apply map
        edge_mapped = np.take_along_axis(edge_maps, blocks, axis=-1)

        # interpolate
        edge_coeffs = np.product([[inv_coeffs, coeffs][e][d]
                                  for d, e in enumerate(edge[::-1])], 0)

        result += (edge_mapped * edge_coeffs).astype(result.dtype)

    result = result.astype(dtype)

    # rebuild result image from blocks
    result = result.reshape(blocks_flattened_shape)
    blocks_axis_rebuild_order =\
        np.array([np.arange(0, ndim),
                  np.arange(ndim, ndim * 2)]).T.flatten()
    result = np.transpose(result, axes=blocks_axis_rebuild_order)
    result = result.reshape(image.shape)

    # undo padding
    unpad_slices = tuple([slice(p_i, s - p_f) for p_i, p_f, s in
                          zip(pad_start_per_dim, pad_end_per_dim,
                              image.shape)])
    result = result[unpad_slices]

    # result=(img_CLAHE).astype(np.uint8)
    # color = cv2.cvtColor(result, cv2.COLOR_HSV2BGR)

    return hist, clipped_hist, result
    # return hist, clipped_hist, color
    # return result

def clipHistogram(hist, clip_limit):
    """Perform clipping of the histogram and redistribution of bins.
    The histogram is clipped and the number of excess pixels is counted.
    Afterwards the excess pixels are equally redistributed across the
    whole histogram
    """
    # calculate total number of excess pixels
    excess_mask = hist > clip_limit
    excess = hist[excess_mask]
    n_excess = excess.sum() - excess.size * clip_limit
    hist[excess_mask] = clip_limit

    # Second part: clip histogram and redistribute excess pixels in each bin
    bin_incr = n_excess // hist.size  # average binincrement
    upper = clip_limit - bin_incr  # Bins larger than upper set to cliplimit

    low_mask = hist < upper
    n_excess -= hist[low_mask].size * bin_incr
    hist[low_mask] += bin_incr

    mid_mask = np.logical_and(hist >= upper, hist < clip_limit)
    mid = hist[mid_mask]
    n_excess += mid.sum() - mid.size * clip_limit
    hist[mid_mask] = clip_limit

    while n_excess > 0:  # Redistribute remaining excess
        prev_n_excess = n_excess
        for index in range(hist.size):
            under_mask = hist < clip_limit
            step_size = max(1, np.count_nonzero(under_mask) // n_excess)
            under_mask = under_mask[index::step_size]
            hist[index::step_size][under_mask] += 1
            n_excess -= np.count_nonzero(under_mask)
            if n_excess <= 0:
                break
        if prev_n_excess == n_excess:
            break

    return hist

def createLUT(histogram, min_value, max_value, pixels):
    '''Cumulutative sum of histogram bins'''
    out = np.cumsum(histogram, axis=-1).astype(float)
    out *= (max_value - min_value) / pixels
    out += min_value
    np.clip(out, a_min=None, a_max=max_value, out=out)

    return out.astype(int)

'''Problem 3 Functions'''
def solveHomographyAndWarp(img):
    source = np.array([[560, 50], [145, 220],[1180, 220], [725, 50]])#main

    dest = np.array([[0, 0], [0, 500], [200, 500],[200, 0]])

    H , status = cv2.findHomography(source, dest)

    new_img = cv2.warpPerspective(img, H, (200,500))

    return new_img

#Uses inverse homography to unwarp
def unwarpImage(img, s0, s1):

    source = np.array([[560, 50], [145, 220],[1180, 220], [725, 50]])#main

    dest = np.array([[0, 0], [0, 500], [200, 500],[200, 0]])

    H , status = cv2.findHomography(source, dest)

    new_img = cv2.warpPerspective(img, np.linalg.inv(H), (s1,s0))

    return new_img

###--Solves for histogram and max of each column/lane---###
def histogram(img):

    hist_vals = list()
    index = list()

    for i in range(img.shape[1]):
        z = np.where(img[:,i] > 0)
        hist_vals.append(len(z[0]))
        index.append(i)

    tmp_1 = hist_vals[:120]
    tmp_2 = hist_vals[120:]
    max1 = max(tmp_1)
    max2 = max(tmp_2)
    col1 = hist_vals.index(max1)
    col2 = hist_vals.index(max2)

    # bins=200
    # plt.plot(index,hist_vals, bins)
    # plt.show()
    # plt.savefig("prob3_histogram.png")
    return col1, col2
    # return col1, col2, index, hist_vals, bins

def solvePolygon(left, right):
    x = np.arange(500)
    x_sqr = np.square(x)
    ones = np.ones(500)

    x_pts = np.stack((x_sqr,x,ones))

    #calculating y = dot product of the coefficients and x
    points_left = np.dot(left,x_pts).astype(np.uint8)
    points_right = np.dot(right,x_pts).astype(np.uint8)

    pnts_left = np.vstack((points_left,x)).T
    pnts_right = np.vstack((points_right,x)).T

    #reversing the right lane y points in order to plot using polylines
    pnts_right[:,0] = pnts_right[::-1,0]
    pnts_right[:,1] = pnts_right[::-1,1]

    points = np.concatenate((pnts_left,pnts_right), axis = 0)

    return points, pnts_left, pnts_right

def addCurveDirectionText(center, left, right, img, curve):
    # def direction(img_center,left_max,right_max,res_img,radius_dir):

    l_center = left + (right - left)/2
    dev = l_center - center

    #Straight
    if ( -14 < dev < -9.5):
        img = cv2.putText(img,'Straight',(50,50),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,255),2,cv2.LINE_AA)
        img = cv2.putText(img,"ROC {}".format(curve),(50,80),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,255),2,cv2.LINE_AA)
    #Right
    elif ( -9.5 < dev < -3):
        img = cv2.putText(img,'Right',(50,50),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,255),2,cv2.LINE_AA)
        img = cv2.putText(img,"ROC {}".format(curve),(50,80),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,255),2,cv2.LINE_AA)
    #Left
    else:#( 6 < dev )
        img = cv2.putText(img,'Left',(50,50),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,255),2,cv2.LINE_AA)
        img = cv2.putText(img,"ROC {}".format(curve),(50,80),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,255),2,cv2.LINE_AA)


import matplotlib.pyplot as plt
import numpy as np

f40 = open("/data/Lei_student/Hussain/ML/dm6/peakerror/40_mcc_llocal.csv", "r")
f30 = open("/data/Lei_student/Hussain/ML/dm6/peakerror/30_mcc_llocal.csv", "r")
f20 = open("/data/Lei_student/Hussain/ML/dm6/peakerror/20_mcc_llocal.csv", "r")
f10 = open("/data/Lei_student/Hussain/ML/dm6/peakerror/10_mcc_llocal.csv", "r")

distance_to_default = []
labels = []

count = 0
for i in f40.readlines():
    if count > 0:
        distance_to_default.append(float(i.split(",")[1]) - .7477454791039003)
        labels.append(40)
    count += 1

count = 0
for i in f30.readlines():
    if count > 0:
        distance_to_default.append(float(i.split(",")[1]) - .7023926834240052)
        labels.append(30)
    count += 1

count = 0
for i in f20.readlines():
    if count > 0:
        distance_to_default.append(float(i.split(",")[1]) - .734564190240499)
        labels.append(20)
    count += 1

count = 0
for i in f10.readlines():
    if count > 0:
        distance_to_default.append(float(i.split(",")[1]) - .8746081504702194)
        labels.append(10)
    count += 1

fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111)
ax.set_title("Distance to default of each parameter combination")
plt.scatter(labels, distance_to_default)
ax.set_xlabel("Number of labels")
ax.set_ylabel("Parameter MCC - Default MCC")
fig.savefig("/data/Lei_student/Hussain/ML/dm6/peakerror/distance_to_default.png")


from sklearn.metrics.pairwise import linear_kernel
from data_cleaning import prepareData, DataSets
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
import numpy as np
from scipy.stats.stats import pearsonr

class ContentCompare:
    def __init__(self, user_v):
        self.item_df, self.user_df = prepareData(load_stored_data=True, reduce=True, min_user_reviews=100,
                                                 min_movie_raings=50)
        self.item_df.clean_items = self.item_df.clean_items[
            self.item_df.clean_items.index.isin(self.user_df.ratings["movieId"].unique())]

        self.user_v = user_v
        self.cos_sim = None
        self.pearson_sim = None
        self.euclidian_sim = None

        self.indexToId = pd.DataFrame(self.item_df.clean_items.index)
        self.indexToId["index"] = self.indexToId.index

    def pearson_compare(self, id1, id2=None, item_df=None):
        index1 = int(list(self.indexToId[self.indexToId["movieId"] == id1]["index"].values)[0])
        if id2 is not None:
            index2 = int(list(self.indexToId[self.indexToId["movieId"] == id2]["index"].values)[0])

        if item_df is None:
            item_df = self.item_df.clean_items
        self.pearson_sim = pd.DataFrame(np.corrcoef(item_df.to_numpy()))

        if id2 is not None:
            return self.pearson_sim[index1][index2]
        return self.pearson_sim[index1]

    def cosine_compare(self, id_to_compare_to, id_to_compare_to_exclusively=None, item_df=None):
        index1 = int(list(self.indexToId[self.indexToId["movieId"] == id_to_compare_to]["index"].values)[0])

        if id_to_compare_to_exclusively is not None:
            index2 = int(list(self.indexToId[self.indexToId["movieId"] == id_to_compare_to_exclusively]["index"].values)[0])

        if item_df is None:
            item_df = self.item_df.clean_items
        self.cos_sim = pd.DataFrame(cosine_similarity(item_df.to_numpy()))

        if id_to_compare_to_exclusively is not None:
            return self.cos_sim[index1][index2]
        return self.cos_sim[index1]

    def euclidian_compare(self, id1, id2):
        index1 = int(list(self.indexToId[self.indexToId["movieId"] == id1]["index"].values)[0])
        if id2 is not None:
            index2 = int(list(self.indexToId[self.indexToId["movieId"] == id2]["index"].values)[0])

        v1 = self.item_df.clean_items[index1]
        v2 = self.item_df.clean_items[index2]

        return np.linalg.norm(v1 - v2)[0]

    def compareVectorToItems(self, distance_measure, test_vector):
        movie_df_cpy = self.item_df.clean_items.copy()

        user_vec_index = max(np.array(self.item_df.clean_items.index)) + 1
        user_vec = test_vector
        movie_df_cpy.loc[user_vec_index] = user_vec

        self.indexToId.loc[max(np.array(self.indexToId.index)) + 1] = [user_vec_index,
                                                                       max(np.array(self.indexToId.index)) + 1]

        if distance_measure == "cosine":
            movie_similarity_predictions = self.cosine_compare(user_vec_index, item_df=movie_df_cpy)

        if distance_measure == "pearson":
            movie_similarity_predictions = self.pearson_compare(user_vec_index, item_df=movie_df_cpy)

        userprofile_to_movie_similarities = pd.DataFrame(
            {"prediction_collaborative": movie_similarity_predictions,
             "itemId": self.indexToId["movieId"]}).sort_values(by=['prediction_collaborative'], ascending=False)

        userprofile_to_movie_similarities = userprofile_to_movie_similarities[
            userprofile_to_movie_similarities["itemId"] != user_vec_index]

        return userprofile_to_movie_similarities

    def queryUser(self, profile_vector_type="weighted", distance_measure="cosine"):
        """
        gets a list of the movies that the users user vector
        can do two types,  a single weitted vecetor or 5 vectors of the different rating classes

        :param profile_vector_type: "weighted", "classed"
        :param distance_measure: "cosine", "euclidian", "pearson"
        :return: list of movies with their probs of liking
        """

        movie_df_cpy = self.item_df.clean_items.copy()

        if profile_vector_type == "weighted":
            user_vec = self.user_v
            return self.compareVectorToItems(distance_measure=distance_measure, test_vector=user_vec)

        if profile_vector_type == "classed":
            user_vec_classes, user_weight_classes = self.class_user_v
            class_ratings_df = pd.DataFrame({"movieId": movie_df_cpy.index})

            rating_val = 1
            for rating_vector in user_vec_classes:
                cls_rating = self.compareVectorToItems(distance_measure=distance_measure,
                                                       test_vector=rating_vector).rename(
                    {"comparison_val": f"rating_{rating_val}_similarity"}, axis='columns')
                class_ratings_df = pd.merge(class_ratings_df, cls_rating, on='movieId')
                rating_val += 1

            def mostSimilarRating(row):
                final_scores = [row[f"rating_{i + 1}_similarity"] for i in range(5)]
                top_voted_rating = final_scores.index(max(final_scores)) + 1
                return top_voted_rating / 5

            class_ratings_df["voted_score"] = class_ratings_df.apply(lambda row: mostSimilarRating(row), axis=1)

            return class_ratings_df

        # add user vecs to the movie vecs

if __name__ == '__main__':
    items, users = prepareData(load_stored_data=True)

    set_split = DataSets(items, users, 5)


# -*- coding: utf-8 -*-
"""
Created on Mon Sep  5 22:51:10 2022

@author: CCD
"""

import pandas as pd
import os
import datetime
import numpy as np
from sklearn.cluster import AgglomerativeClustering as HAC

ruta_output = 'Datos/Output/'
fechas = [f.name for f in os.scandir(ruta_output) if f.is_dir()]
fechas.remove('20210726')

df_votos_agregado = pd.DataFrame({'fecha':[], 'n_votacion':[],
                                  'nombre':[], 'partido':[],
                                  'voto':[]})

i= 0
for f in fechas:
    date = datetime.datetime.strptime(f, '%Y%m%d')
    ruta_fecha = os.path.join(ruta_output, f)
    votos = [v.name for v in os.scandir(ruta_fecha) if v.is_dir()]
    for v in votos:
        voto_num = int(v.split('_')[1])
        ruta_voto = os.path.join(ruta_fecha, v)
        df_votos = pd.read_json(ruta_voto + '/votos_final.json')
        df_votos['fecha'] = date
        df_votos['n_votacion'] = voto_num

        df_votos_agregado = pd.concat([df_votos_agregado, df_votos])
        i += 1
print(i)
df_votos_agregado = df_votos_agregado.pivot(index=['fecha', 'n_votacion'], columns=['nombre'], values='voto').sort_index()
df_votos_agregado.to_csv('Datos/Analisis/Votos.csv')
#Limpiando votos
def EtiquetaVoto(x):
    map_votos = {'SI +++': 1, 'NO ---': -1, 'ABST.': 0}
    if x not in map_votos:
        return np.nan
    else:
        return map_votos[x]

df_votos_agregado = df_votos_agregado.applymap(EtiquetaVoto)

#Limpiando declarativos
def LimpiaDeclarativo(row):
    favor = row[row==1].count()
    contra = row[row==-1].count()
    if favor >= 104 and contra < 10:
        return row.apply(lambda x: np.nan)
    else:
        return row

df_votos_agregado = df_votos_agregado.apply(LimpiaDeclarativo, axis = 1)

#1: Calculando % de veces que votaron igual
def CalculaSimilitud(df):
    print('Evaluando similitud de voto')
    congresistas = list(df.columns)
    # similitud = {}
    matriz_similitud = np.zeros((len(congresistas), len(congresistas)))
    for i, c1 in enumerate(congresistas):
        # similitud[c1] = {}
        print('\t' +'(' + str(i+1) + '/' + str(len(congresistas)) + ') Calculando similitud de ' + c1 )
        for j, c2 in enumerate(congresistas[i+1:]):
            congs = [c1, c2]
            df_comp = df.loc[:,congs]
            df_comp['check'] = df_comp[c1] == df_comp[c2]
            df_comp['sum'] = df_comp[c1] + df_comp[c2]
            votos_validos = len(df_comp) - df_comp['sum'].isna().sum()

            thresh = 80
            if votos_validos < thresh:
                simil = np.nan
            else:
                simil = df_comp['check'].sum()/votos_validos
            # similitud[c1][c2] = df_comp['check'].sum()/len(df_comp)
            matriz_similitud[i, i + j + 1] = simil

    matriz_similitud = matriz_similitud + matriz_similitud.T
    np.fill_diagonal(matriz_similitud, 1)
    df_similitud = pd.DataFrame(data = matriz_similitud, index = congresistas, columns = congresistas)
    return df_similitud

matriz_similitud = CalculaSimilitud(df_votos_agregado)
matriz_distancias = 1 - matriz_similitud

#2 Limpia Congresistas con insuficiente info
def LimpiaCongresista(df):
    df_final = df.copy(deep=True)
    for cong in df:
        if np.sum(np.isnan(df[cong]))== len(df)-1:
            print('Congresista ' + cong + ' se elimina por falta de datos')
            df_final = df_final.drop(index=cong, columns=cong)
    return df_final
matriz_distancias = LimpiaCongresista(matriz_distancias)

#3 Corriendo modelo de agrupamiento
Agglom = HAC(linkage = 'complete', affinity='precomputed', n_clusters=10)
# Agglom = HAC(linkage = 'complete', affinity='precomputed', distance_threshold=0.25, n_clusters = None)

Clustering = Agglom.fit(matriz_distancias)

def SortMatriz(df, model):
    cong = df.index
    labels = model.labels_
    dict_clusters = dict(zip(cong, labels))
    df_clusters = pd.DataFrame.from_dict(dict_clusters, orient='index', columns=['Cluster']).reset_index()
    df_clusters.sort_values(by=['Cluster', 'index'], inplace=True)
    ordered_cong = list(df_clusters['index'])
    return df.reindex(index = ordered_cong, columns = ordered_cong), df_clusters
matriz_distancias_ordered, df_clusters = SortMatriz(matriz_distancias, Clustering)

matriz_distancias_ordered.to_excel('Datos/Analisis/Distancias Sorted.xlsx')
df_clusters.to_excel('Datos/Analisis/Clusters.xlsx')

import matplotlib.pyplot as plt
import seaborn as sns
plt.rcParams['axes.facecolor'] = (1,1,1)
plt.rcParams['figure.facecolor'] = (1,1,1)
plt.rcParams['figure.figsize'] = 6, 5
plt.rcParams['figure.dpi'] = 120

plt.figure()
plt.title('Matriz de distancias')
ax = sns.heatmap(matriz_distancias_ordered,cmap="rocket_r", yticklabels=False, xticklabels=False)
plt.show()

plt.figure()
plt.title('Matriz de distancias')
ax = sns.heatmap(matriz_distancias,cmap="rocket_r", yticklabels=False, xticklabels=False)
plt.show()

# from matplotlib import pyplot as plt
# from scipy.cluster.hierarchy import dendrogram

# def plot_dendrogram(model, **kwargs):
#     # Create linkage matrix and then plot the dendrogram

#     # create the counts of samples under each node
#     counts = np.zeros(model.children_.shape[0])
#     n_samples = len(model.labels_)
#     for i, merge in enumerate(model.children_):
#         current_count = 0
#         for child_idx in merge:
#             if child_idx < n_samples:
#                 current_count += 1  # leaf node
#             else:
#                 current_count += counts[child_idx - n_samples]
#         counts[i] = current_count

#     linkage_matrix = np.column_stack(
#         [model.children_, model.distances_, counts]
#     ).astype(float)

#     # Plot the corresponding dendrogram
#     dendrogram(linkage_matrix, **kwargs)

# plt.title("Hierarchical Clustering Dendrogram")
# # plot the top three levels of the dendrogram
# plot_dendrogram(Clustering, truncate_mode="level", p=3)
# plt.xlabel("Number of points in node (or index of point if no parenthesis).")
# plt.show()

# hierarchy.dendrogram(Z, orientation="right", labels=df.index)

matriz_similitud.to_excel('Datos/Analisis/similitud.xlsx')

Votaciones Congreso - copia/HCA Matriz Similitud.py


import json
import logging
import os
import re
from typing import List, Dict
from logrec.util.files import get_two_levels_subdirs

logger = logging.getLogger(__name__)

home_dir = os.environ["HOME"]

path_to_dup_jsons = os.path.join(home_dir, 'log-recommender-master', 'java-github-duplicates.json')
base_raw_projects_dir = os.path.join(home_dir, 'raw_datasets/allamanis/nodup_all')

def build_name_to_name_in_split(base_raw_projects_dir: str):
    res = {}
    for dir, subdir, subsubdir in get_two_levels_subdirs(base_raw_projects_dir):
        repl = re.sub('^[0-9]*_', '', subsubdir)
        res[repl] = os.path.join(dir, subdir, subsubdir)
    return res

NON_EXISTING_PATH=os.path.join(home_dir, 'this_path_will_never_exist')

def convert_to_real_split_path(path: str, name_to_name_in_split: Dict[str, str]):
    return re.sub('^(.*?)/', lambda name: (name_to_name_in_split[name.group(1)] if name.group(1) in name_to_name_in_split else NON_EXISTING_PATH) + '/', path)

def convert_to_real_path(path: str, name_to_name_in_split: Dict[str, str]):
    return os.path.join(base_raw_projects_dir, path)

def process_dup_group(duplicate_group: List[str], name_to_real_name: Dict[str, str]):
    existing_real_paths = []
    for duplicate in duplicate_group:
        real_path = convert_to_real_path(duplicate, name_to_real_name)
        if os.path.exists(real_path):
            existing_real_paths.append(real_path)
    for file in existing_real_paths[1:]:
        logger.info(f"Removing: {file}...")
        os.remove(file)
    return len(existing_real_paths) - 1 if len(existing_real_paths) > 0 else 0

def run():
    logger.info("Loading json ...")
    with open(path_to_dup_jsons, 'r') as f:
        dup_json = json.load(f)

    #logger.info("Matching json names to our names ...")
    #name_to_real_name = build_name_to_name_in_split(base_raw_projects_dir)
    #for i, (name, name_in_split) in enumerate(name_to_real_name.items()):
    #    logger.info(f'{name} --> {name_in_split}')
    #    if i == 5: break
    #logger.info("...")
    name_to_real_name=None

    n_files_removed_total = 0
    n_duplicated_files_total = 0
    total_groups = len(dup_json)
    for i, duplicate_group in enumerate(dup_json):
        logger.info(f"Processing duplication group {i} out of {total_groups} ...")
        n_duplicated_files = len(duplicate_group) - 1
        n_files_removed = process_dup_group(duplicate_group, name_to_real_name)
        logger.info(f'Duplicated files removed: {n_files_removed} out of {n_duplicated_files}')
        n_duplicated_files_total += n_duplicated_files
        n_files_removed_total += n_files_removed
    logger.info(f"Total duplicates removed: {n_files_removed_total} out of {n_duplicated_files_total}")

if __name__ == '__main__':
    run()


import pandas as pd
import  numpy as np
df = pd.read_csv(
    "C:\\Sayed\\Python\\py-master\\pandas\\2_dataframe_basics\\weather_data.csv",
     parse_dates=["day"])
print(df.head(10))
rows, colms = df.shape
print(rows, colms)
print(df[2:3])
print(df[:])
print(df[1:])
print(df.columns)
print(df['event'])
print(df[['day','event']])
print(df.describe())
print(df[df['temperature']>=32])

data_frame={
    'day':['12','23'],
    'score':[12,45.6],
    'name':['omi',12.5]
}
df_new= pd.DataFrame(data_frame)
print(df_new)
print(df)
print(type(df['day'][0]))
# new_df=df.fillna(0)
# print(new_df)
new_df = df.fillna({
    'day':max(df['day']),
    'temperature': df['temperature'].mean(),
    'windspeed': df['windspeed'].max(),
    'event':'Rain'
})
new_df = new_df.replace({
    0:np.nan
})
new_df= new_df.fillna({
    'windspeed':new_df['windspeed'].mean()
})

new_df = new_df.replace({
    'temperature': '[A-Za-z]',
    'windspeed': '[A-Za-z]'
}, '', regex=True)

new_df =new_df.replace(['Rain','Sunny','Snow'],[1,2,3])

print(new_df)