user_exec_yes=False
if __name__ =="__main__":
    user_exec_yes=input("exec?") in ["y","Y",""]
if user_exec_yes: print("Initializing Program.")

import numpy as np
import matplotlib as mpl
mpl.use("agg")
mpl.rcParams["animation.ffmpeg_path"]="/cygdrive/c/Users/awaw6/ffmpeg/bin/ffmpeg.exe"
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3
from mpl_toolkits.mplot3d.art3d import Poly3DCollection as p3c
import matplotlib.animation as ani
import matplotlib.colors as colors
import intermediateAxis as ia
import os
import track_data as td
import planarProjections as pp

class Rot_Box(object):

    mag=lambda self,vec:sum([el**2 for el in vec])**.5

    def __init__(self,scale=(1,1,1)):
        rnd=lambda x:int(x+.5*(1-2*(x<0)))
        mrnd=lambda m: tuple(rnd(el) for el in m)

        strc=lambda vx,scale:tuple(ela*elb for ela,elb in zip(vx,scale))
        mmstrc=lambda verts,scale:[[strc(vx,scale) for vx in vxs] for vxs in verts]

        mrotx=lambda t:np.array([[1,0,0],
                                [0,np.cos(t),-np.sin(t)],
                                [0,np.sin(t),np.cos(t)]
                               ])
        mroty=lambda t:np.array([[np.cos(t),0,np.sin(t)],
                                [0,1,0],
                                [-np.sin(t),0,np.cos(t)]
                               ])
        mrotz=lambda t:np.array([[np.cos(t),-np.sin(t),0],
                                [np.sin(t),np.cos(t),0],
                                [0,0,1]
                               ])
        rotx=lambda t,vx:np.matmul(mrotx(t),vx)
        roty=lambda t,vx:np.matmul(mroty(t),vx)
        rotz=lambda t,vx:np.matmul(mrotz(t),vx)
        rotmcc=lambda mt,vx:rotx(mt[0],roty(mt[1],rotz(mt[2],vx)))
        self.rotmcc=rotmcc

        x=[-1,-1, 1, 1]
        y=[-1,-1,-1,-1]#normal axis
        z=[-1, 1, 1,-1]
        self.scale=scale

        verts=[list(zip(x,y,z))]
        for i in range(3):
            verts+=[[mrnd(tuple(rotx(np.pi/2,vx))) for vx in verts[-1]]]
        for i in (1,-1):
            verts+=[[mrnd(tuple(rotz(np.pi/2*i,vx))) for vx in verts[0]]]
        self.verts=mmstrc(verts,scale)

    def read_data(self,do_PsC=False,**kwargs):

        file_name=ia.model(do_PsC,**kwargs)
        dataf=open(file_name,"r")
        data=eval(dataf.read())
        dataf.close()
        Ts_to_mangs=lambda Ts:list(zip(*[[tf-ti for ti,tf in zip(el[:-1],el[1:])] for el in Ts]))
        Os_to_mangs=lambda Os:list(zip(*[[omg*dt for omg in omgs] for omgs in Os]))
        data["mangs"]=Ts_to_mangs(data["Ts"])
        #data.update(mangsC=Os_to_mangs(data["OsC"])
        return data

    def axes_to_verts(axes_list,ascale=(1,1,1)):
        axes=[[ax*sc for ax,sc in zip(axes,ascale)] for axes in axes_list]
        def mkvert(axes,pos=np.zeroes(3),vxs=[]):
            if axes:
                for i in [1,-1]:
                    mkvert(axes[1:],pos+i*np.array(axes[0]),vxs)
            else:
                print("yielding:",pos)
                vxs.append(tuple(pos))
            return vxs
        return [mkvert(axes) for axes in axes_list]

    def datrot(self,verts,mangs):
        data=[]
        for mang in mangs:
            data.append(verts)
            verts=[[tuple(self.rotmcc(mang,vx)) for vx in vxs] for vxs in verts]
        return np.array(data)#returns list of verts in the form of cartesian tuples

    def mkanimation(self,time=20,data=None,Vs=None,mangs=None,track_data=None,
            draw_artists=["axes"],animate=True,fps=None,save="",**kwargs):
        if "dt" not in kwargs: kwargs.update(dt=.01)
        dt=kwargs["dt"]
        save_loc="./Videos"
        verts_to_xyz=lambda verts:tuple([[vx[i] for vx in vxs] for vxs in verts] for i in range(3))
        def create_lines(verts):
            lines=[]
            xs,ys,zs=verts_to_xyz(verts)
            for i in range(len(verts)): lines.append(ax.plot(xs[i],ys[i],zs[i],"o")[0])
            return lines
        def animate_lines(num,lines):
            xs,ys,zs=verts_to_xyz(data[num])
            for i,line in enumerate(lines):
                line.set_data(xs[i],ys[i])
                line.set_3d_properties(zs[i])

        def create_polys(verts):
            create_polys=[]
            for i,el in enumerate(verts):
                poly=p3c([el],alpha=.5,zorder=0)
                poly.set_color(colors.rgb2hex([0]+[i/len(verts)*.5+.35]*2))
                create_polys.append(poly)
            return create_polys
        def animate_polys(num,polys):
            verts=data[num]
            for vxs,poly in zip(verts,polys):
                poly.set_verts([vxs])
                #poly.set_sort_zpos(0)

        def genVec(verts):#returns (xyzs=_,xyzs_=(xs_,ys_,zs_)) of prince axes
            xyzs=verts_to_xyz([verts[i] for i in [4,2,3]])
            xyzs_=[[sum(ws)/2 for ws in wss] for wss in xyzs]
            return xyzs_
        """
        def create_axes(verts):
            xyzs,xyzs_=genVec(verts)
            Vs_=zip(*[el[0] for el in Vs])
            axes=[ax.quiver(*[[0]*3]*3,*xyzs_,colors=["r"]*3,linewidth=2,arrow_length_ratio=.3,zorder=10)[0],
                  ax.quiver(*[[0]*3]*3,*Vs_,colors=["b"]*3,linewidth=2,arrow_length_ratio=.3,zorder=10)[0],
                 ]
            return axes
        def animate_axes(num,axes):
            xyzs,xyzs_=genVec(data[num])
            origin,coords_=zip(*[[0]*3]*3),zip(*xyzs_)
            Vs_=[el[num] for el in Vs]
            for axis,pos_ in zip(axes[::-1],[coords_,Vs_]):
                axis.set(segments=list(zip(origin,pos_)))
            #for axe in axes: axe.set_sort_zpos(1)
        """
        def create_axes(verts):
            xyzs_=np.array(genVec(verts))
            Vs_=np.array(list(zip(*[el[0] for el in Vs])))
            axes=[]
            if track_data: iterate=zip([Vs_],["rgb"])
            else: iterate=zip([xyzs_,Vs_],["ccc","rgb"])
            for pos_,col in iterate:
                for i in range(3):
                    pt=np.array(pos_[:,i])
                    axes.append(ax.plot(*zip(-pt,pt),col[i],linewidth=2,zorder=1,)[0])
            return axes
        def animate_axes(num,axes):
            xyzs_=np.array(genVec(data[num]))
            Vs_=np.array(list(zip(*[el[num] for el in Vs])))
            if track_data: iterate=enumerate([Vs_])
            else: iterate=enumerate([xyzs_,Vs_])
            for i,pos_ in iterate:
                for j in range(3):
                    pt=np.array(pos_[:,j])
                    axes[3*i+j].set_data(*zip(-pt[:2],pt[:2]))
                    axes[3*i+j].set_3d_properties([-pt[2],pt[2]])

        def create_diags(verts):
            diags=[]
            xyzs=np.array(verts_to_xyz([verts[i] for i in [0,2]]))
            for i in range(2):
                for j in range(4):
                    diags.append(ax.plot(*zip([0]*3,xyzs[:,i,j]))[0])
            return diags
        def animate_diags(num,diags):
            xyzs=np.array(verts_to_xyz([verts[i] for i in [0,2]]))
            for i in range(2):
                for j in range(4):
                    diags[4*i+j].set_data(*zip([0]*2,xyzs[:2,i,j]))
                    diags[4*i+j].set_3d_properties([0,xyzs[2,i,j]])

        def graph_track():
            data_td=td.track_tv_data
            for i,name in enumerate(ax_p2d):
                for j in range(3):
                    ax_p2d[name].plot(data_td[j][0],[el[i] for el in data_td[j][1]],label=name+str(j)+"m")

        def create_p2d(data_d):
            p2d={}
            for name in list(data_d)[1:]:
                for j,col in zip(range(len(data_d[name])),"rgb"):
                    ax_p2d[name].plot(data_d["time"][j],data_d[name][j],col,label=name+str(j))
                    p2d.update({name+str(j):ax_p2d[name].plot([0,0],[0,data_d[name][j][0]],col)[0]})
                ax_p2d[name].legend()
                ax_p2d[name].set_xlabel("Time")
                ax_p2d[name].set_ylabel(name)
                ax_p2d[name].set_title("Time vs "+name)
                ax_p2d[name].grid()
            return p2d
        def animate_p2d(num,p2d):
            for name in list(data_d)[1:]:
                for j in range(len(data_d[name])):
                    p2d[name+str(j)].set_data([data_d["time"][j][num]]*2,[0,data_d[name][j][num]])

        animators=[animate_lines,animate_polys,animate_axes,animate_diags,animate_p2d]
        creators=[create_lines,create_polys,create_axes,create_diags,create_p2d]
        def animate(num,artists):
            for anifunc in animators:
                name=anifunc.__name__
                name=name[name.index("_")+1:]
                if name in list(artists): anifunc(num,artists[name])

        def init_art(verts,artists):
            arts={}
            for crefunc in creators:
                name=crefunc.__name__
                name=name[name.index("_")+1:]
                if name in list(artists):
                    if name!="p2d": arts.update({name:crefunc(verts)})
                    else: arts.update({name:crefunc(data_d)})
            if "polys" in artists:
                for poly in arts["polys"]: ax.add_collection(poly)
            return arts

        draw_p2d="p2d" in draw_artists
        if mangs==None or Vs==None or draw_p2d:
            data_model=self.read_data(do_PsC=draw_p2d,**kwargs)

        if draw_p2d:
            fig=plt.figure(figsize=plt.figaspect(1/3))
            ax=fig.add_subplot(1,3,1,projection="3d")
            var_names=["Psx","Psy"]
            ax_p2d={name:fig.add_subplot(1,3,i+2) for i,name in enumerate(var_names)}
            if track_data:
                data_d=track_data
            else:
                data_d={"time":data_model["time"]*3,"Psx":data_model["PsCx"],"Psy":data_model["PsCy"]}
        else:
            fig=plt.figure()
            ax=p3.Axes3D(fig)
            data_d=None

        verts=self.verts
        if mangs==None:
            mangs=data_model["mangs"]
        if Vs==None:
            Vs=(np.array(data_model["Vs"])*.5).tolist()

        ax.set_xlim3d([-2,2]);ax.invert_xaxis()
        ax.set_ylim3d([-2,2]);ax.invert_yaxis()
        ax.set_zlim3d([-2,2])

        ax.set_xlabel("X")
        ax.set_ylabel("Y")
        ax.set_zlabel("Z")

        ax.legend()
        ax.set_title("Rotation of Principal Axes in 3D")

        artists=init_art(verts,draw_artists)

        plt.tight_layout()

        print("Data reading complete. Initializing animation...")

        if not os.path.exists(save_loc):
            os.makedirs(save_loc)
        fig.savefig(save_loc+"/rotbox.png")

        if animate:
            if data==None: data=self.datrot(verts,mangs)
            data_len=len(data)
            if not fps: fps=data_len/time
            else:
                if data_len//(time*fps)==0:
                    print("fps too high.")
                    return
                def mod_data(data):
                    interval=len(data)//(time*fps)
                    if interval:
                        return [data[i] for i in range(0,len(data),interval)]
                    else: return data
                data=mod_data(data)
                data_len=len(data)
                if draw_p2d:
                    for name in list(data_d)[1:]:
                        for j in range(3):
                            data_d[name][j]=mod_data(data_d[name][j])
                    data_d["time"]=mod_data(data_d["time"])
                    for name in data_d:
                        data_d[name]=[vec+[0]*(data_len-len(vec)) for vec in data_d[name]]

                Vs=[mod_data(vec) for vec in Vs]
                Vs=[vec+[(0,0,0)]*(data_len-len(vec)) for vec in Vs]

            print("Animating...")

            poly_ani=ani.FuncAnimation(fig,animate,data_len,fargs=(artists,),interval=200)

            print("Animation complete. Saving...")

            myWriter=ani.FFMpegWriter(fps=fps)
            if save and input("Are you sure? (y/n)") not in ["y","Y"]:
                return
            poly_ani.save(save_loc+"/rotatingBox({}){}.mp4".format(dt,save),writer=myWriter)

            print("Save complete. End of program.")

def __main__():
    scale=tuple(.5/(el/max(ia.Is)+.1) for el in ia.Is)
    scale=tuple(scale[i] for i in [0,1,2])
    fps=10
    video_time,cond_time=10,1
    kwargs=dict(cond=("time",cond_time),dt=.001,Os=(.1,3,.5)) #tblock
    #kwargs=dict(cond=("time",cond_time),dt=.001,Os=(.1,16,.1))  #nasablock

    rb=Rot_Box(scale=scale)

    def proj_test():
        data=rb.read_data(**kwargs)
        Vs=data["Vs"]
        Vs2d=[[(v[:2],v[2]>=0) for v in vec] for vec in Vs]
        Vs3d_calc=[[(pp.proj_to_v3(v[0],1),v[1]) for v in vec] for vec in Vs2d]
        Vs3d_calc=[[(*v[0][:2],v[0][2]*(-1+2*v[1])) for v in vec] for vec in Vs3d_calc]
        rb.mkanimation(time=10,draw_artists=["axes"],fps=fps,**kwargs,Vs=Vs3d_calc)
        quit()
    #proj_test()

    track_times,Vs_exp=td.track_data_fps(fps,video_time/cond_time)
    #Vs_exp=[[tuple(v[i] for i in (1,0,2)) for v in V] for V in Vs_exp] #for nasa
    track_Psx=[[v[0] for v in V] for V in Vs_exp]
    track_Psy=[[v[1] for v in V] for V in Vs_exp]
    track_data={"time":track_times,"Psx":track_Psx,"Psy":track_Psy}
    #rb.mkanimation(time=video_time,draw_artists=["axes","p2d"],fps=fps,**kwargs)
    rb.mkanimation(time=video_time,draw_artists=["axes","p2d"],fps=fps,**kwargs,Vs=Vs_exp,track_data=track_data)

if user_exec_yes:
    __main__()


# encoding: utf-8
from random import Random

from django.conf import settings
from django.core.mail import EmailMessage

from app_api.models import EmailVerifyRecord

def random_str(random_length=8):
    str_base = ''
    chars = 'AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz0123456789'
    length = len(chars) - 1
    random = Random()
    for i in range(random_length):
        str_base += chars[random.randint(0, length)]
    return str_base

def send_email(email, send_type="login_auth"):
    email_record = EmailVerifyRecord()
    if send_type == "login_auth":
        code = random_str(4)
    else:
        code = random_str(16)
    email_record.code = code
    email_record.email = email
    email_record.send_type = send_type
    email_record.save()

    if send_type == "login_auth":
        email_title = "OnlineMooc 平台 邮箱登录验证码"
        email_body = f"验证码为: {code}, 请在五分钟内填写！"
        msg = EmailMessage(email_title, email_body, settings.EMAIL_FROM, [email])
        msg.content_subtype = "html"
        send_status = msg.send()
        if send_status:
            pass


import os
import argparse
import numpy as np

import torch
import torch.nn as nn
from torch.optim import Adam

from torch.utils.data import TensorDataset, DataLoader, SubsetRandomSampler
from sklearn.model_selection import KFold

import funcs
from models import AlexNet

torch.manual_seed(42)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# root_path = 'D:/Shelly\'s/Spectograms_77_24_Xethrue/'
# subfolders = ['activity_spectogram_77GHz', 'Spectrograms_24GHz', 'spectogram_Xethru']

parser = argparse.ArgumentParser(description='PyTorch ImageNet Training')
parser.add_argument('--root', default='D:/Shelly\'s/Spectograms_77_24_Xethrue/', type=str,
                    help='root folder of data')
parser.add_argument('--subfolder', default='activity_spectogram_77GHz', type=str,
                    help='Images folder')
parser.add_argument('--epoch', default=10, type=int,
                    help='number of training epoches')
parser.add_argument('--batch_size', default=8, type=int,
                    help='number of batch size')
parser.add_argument('--kfolds', default=5, type=int,
                    help='number of K folders')
parser.add_argument('--lr', default=1e-3, type=float,
                    help='learning rate')
parser.add_argument('--save_name', default='k_cross_model.pt', type=str,
                    help='the name of saved model')
parser.add_argument('--spatial', default='True', type=str,
                    help='True is using SpatialGate')
parser.add_argument('--pretrain', default='True', type=str,
                    help='True is loading pretrained model')

def main():
    args = parser.parse_args()
    print ("args", args)

    if not os.path.isdir(args.root+args.subfolder):
        raise NameError("The directory does not exist. Please check inputs of --root and --subfolder args.")

    num_epochs=args.epoch
    batch_size=args.batch_size
    k=args.kfolds
    splits=KFold(n_splits=k,shuffle=True,random_state=42)
    foldperf={}

    classes = [x[0].replace('\\', '/').split('/')[-1] for x in os.walk(args.root+args.subfolder)][1:]

    images, labels = funcs.load_data(args.root, args.subfolder, classes)

    indices = torch.randperm(images.size()[0])
    images=images[indices]
    labels=labels[indices]

    dataset = TensorDataset(images, labels)

    for fold, (train_idx,val_idx) in enumerate(splits.split(np.arange(len(dataset)))):

        print('Fold {}'.format(fold + 1))

        train_sampler = SubsetRandomSampler(train_idx)
        test_sampler = SubsetRandomSampler(val_idx)
        train_loader = DataLoader(dataset, batch_size=batch_size, sampler=train_sampler)
        test_loader = DataLoader(dataset, batch_size=batch_size, sampler=test_sampler)

        model = AlexNet(num_classes=len(classes), spatial=args.spatial, pre_trained=args.pretrain)
        model.to(device)
        optimizer = Adam(model.parameters(), lr=args.lr)
        criterion = nn.CrossEntropyLoss()

        history = {'train_loss': [], 'test_loss': [],'train_acc':[],'test_acc':[], 'cms': []}

        for epoch in range(num_epochs):
            train_loss, train_correct = funcs.train_epoch(model,device,train_loader,criterion,optimizer)
            test_loss, test_correct, cm = funcs.valid_epoch(model,device,test_loader,criterion)

            train_loss = train_loss / len(train_loader.sampler)
            train_acc = train_correct / len(train_loader.sampler) * 100
            test_loss = test_loss / len(test_loader.sampler)
            test_acc = test_correct / len(test_loader.sampler) * 100

            print("Epoch:{}/{} AVG Training Loss:{:.3f} AVG Test Loss:{:.3f} AVG Training Acc {:.2f} % AVG Test Acc {:.2f} %".format(epoch + 1,
                                                                                num_epochs,
                                                                                train_loss,
                                                                                test_loss,
                                                                                train_acc,
                                                                                test_acc))
            history['train_loss'].append(train_loss)
            history['test_loss'].append(test_loss)
            history['train_acc'].append(train_acc)
            history['test_acc'].append(test_acc)
            history['cms'].append(cm)

        foldperf['fold{}'.format(fold+1)] = history

    torch.save(model, args.save_name)

    testl_f,tl_f,testa_f,ta_f=[],[],[],[]
    cms = []
    k=args.kfolds
    for f in range(1,k+1):

        best_idx = foldperf['fold{}'.format(f)]['test_acc'].index((max(foldperf['fold{}'.format(f)]['test_acc'])))

        tl_f.append(foldperf['fold{}'.format(f)]['train_loss'][best_idx])
        testl_f.append(foldperf['fold{}'.format(f)]['test_loss'][best_idx])

        ta_f.append(foldperf['fold{}'.format(f)]['train_acc'][best_idx])
        testa_f.append(foldperf['fold{}'.format(f)]['test_acc'][best_idx])

        cms.append(foldperf['fold{}'.format(f)]['cms'][best_idx])

    print('Performance of {} fold cross validation'.format(k))
    print("Minmum Training Loss: {:.3f} Minmum Test Loss: {:.3f} Average Best Training Acc: {:.2f} Average Best Test Acc: {:.2f}".format(np.mean(tl_f),np.mean(testl_f),np.mean(ta_f),np.mean(testa_f)))
    print('Confusion Matrix:')
    np.set_printoptions(precision=3)
    print(sum(cms)/len(cms))

    return foldperf

if __name__ == '__main__':
    foldperf = main()


from torch import nn
import torch
import torch.functional as F
import torch.nn.functional as F2
import time
import math
from torch.autograd import Variable
import numpy as np

class ResidualBlock(nn.Module):

    def __init__(self, in_channel, out_channel, kernel_size=3, dilation=None):
        super(ResidualBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_channel, out_channel, kernel_size=(1, kernel_size), padding=0, dilation=dilation)
        # self.conv1.weight = self.truncated_normal_(self.conv1.weight, 0, 0.02)
        # self.conv1.bias.data.zero_()

        self.ln1 = nn.LayerNorm(out_channel, eps=1e-8)
        self.conv2 = nn.Conv2d(out_channel, out_channel, kernel_size=(1, kernel_size), padding=0, dilation=dilation*2)
        # self.conv1.weight = self.truncated_normal_(self.conv1.weight, 0, 0.02)
        # self.conv1.bias.data.zero_()

        self.ln2 = nn.LayerNorm(out_channel, eps=1e-8)
        self.dilation = dilation
        self.kernel_size = kernel_size

    def forward(self, x): # x: [batch_size, seq_len, embed_size]
        x_pad = self.conv_pad(x, self.dilation)
        out =  self.conv1(x_pad).squeeze(2).permute(0, 2, 1)
        out = F2.relu(self.ln1(out))
        out_pad = self.conv_pad(out, self.dilation*2)
        out = self.conv2(out_pad).squeeze(2).permute(0, 2, 1)
        out = F2.relu(self.ln2(out))
        out = out + x
        return out

    def conv_pad(self, x, dilation):
        inputs_pad = x.permute(0, 2, 1)  # [batch_size, embed_size, seq_len]
        inputs_pad = inputs_pad.unsqueeze(2)  # [batch_size, embed_size, 1, seq_len]
        pad = nn.ZeroPad2d(((self.kernel_size - 1) * dilation, 0, 0, 0))
        inputs_pad = pad(inputs_pad)  # [batch_size, embed_size, 1, seq_len+(self.kernel_size-1)*dilations]
        return inputs_pad

    def truncated_normal_(self, tensor, mean=0, std=0.09):
        size = tensor.shape
        tmp = tensor.new_empty(size + (4,)).normal_()
        valid = (tmp < 2) & (tmp > -2)
        ind = valid.max(-1, keepdim=True)[1]
        tensor.data.copy_(tmp.gather(-1, ind).squeeze(-1))
        tensor.data.mul_(std).add_(mean)
        return tensor

class NextItNet_Decoder(nn.Module):

    def __init__(self, model_para):
        super(NextItNet_Decoder, self).__init__()
        self.model_para = model_para
        self.item_size = model_para['item_size']
        self.embed_size = model_para['dilated_channels']
        self.embeding = nn.Embedding(self.item_size, self.embed_size)
        stdv = np.sqrt(1. / self.item_size)
        self.embeding.weight.data.uniform_(-stdv, stdv) # important initializer
        # nn.init.uniform_(self.in_embed.weight, -1.0, 1.0)

        self.dilations = model_para['dilations']
        self.residual_channels = model_para['dilated_channels']
        self.kernel_size = model_para['kernel_size']
        rb = [ResidualBlock(self.residual_channels, self.residual_channels, kernel_size=self.kernel_size,
                            dilation=dilation) for dilation in self.dilations]
        self.residual_blocks = nn.Sequential(*rb)

        self.final_layer = nn.Linear(self.residual_channels, self.item_size)
        self.final_layer.weight.data.normal_(0.0, 0.01)  # initializer
        self.final_layer.bias.data.fill_(0.1)

    def forward(self, x, onecall=False): # inputs: [batch_size, seq_len]
        inputs = self.embeding(x) # [batch_size, seq_len, embed_size]

        dilate_outputs = self.residual_blocks(inputs)

        if onecall:
            hidden = dilate_outputs[:, -1, :].view(-1, self.residual_channels) # [batch_size, embed_size]
        else:
            hidden = dilate_outputs.view(-1, self.residual_channels) # [batch_size*seq_len, embed_size]
        out = self.final_layer(hidden)

        return out

# class NextItNet_Decoder(nn.Module):
#
#     def __init__(self, model_para):
#         super(NextItNet_Decoder, self).__init__()
#         self.model_para = model_para
#         self.item_size = model_para['item_size']
#         self.embed_size = model_para['dilated_channels']
#         self.embeding = nn.Embedding(self.item_size, self.embed_size)
#
#         self.dilations = model_para['dilations']
#         self.residual_channels = model_para['dilated_channels']
#         self.kernel_size = model_para['kernel_size']
#         residual_block = [nn.ModuleList([nn.Conv2d(self.residual_channels, self.residual_channels,
#                                                   kernel_size=(1, model_para['kernel_size']), padding=0, dilation=dilation),
#                                          nn.LayerNorm(self.residual_channels),
#                                          # Layer_norm(self.residual_channels),
#                                          # nn.ReLU(),
#                                          nn.Conv2d(self.residual_channels, self.residual_channels,
#                                                    kernel_size=(1, model_para['kernel_size']), padding=0, dilation=2*dilation),
#                                          nn.LayerNorm(self.residual_channels),
#                                          # Layer_norm(self.residual_channels),
#                                          # nn.ReLU()
#                                 ]) for dilation in self.dilations]
#         self.residual_blocks = nn.ModuleList(residual_block)
#
#         self.softmax_layer = nn.Linear(self.residual_channels, self.item_size)
#
#     def forward(self, x, onecall=False): # inputs: [batch_size, seq_len]
#         inputs = self.embeding(x) # [batch_size, seq_len, embed_size]
#
#         for i, block in enumerate(self.residual_blocks):
#             ori = inputs
#
#             inputs_pad = self.conv_pad(inputs, self.dilations[i])
#             # print(inputs_pad.size())
#             dilated_conv = block[0](inputs_pad).squeeze(2) # [batch_size, embed_size, seq_len]
#             dilated_conv = dilated_conv.permute(0, 2, 1)
#             relu1 = F2.relu(block[1](dilated_conv)) # [batch_size, seq_len, embed_size]
#
#             inputs_pad = self.conv_pad(relu1, self.dilations[i]*2)
#             # print(inputs_pad.size())
#             dilated_conv = block[2](inputs_pad).squeeze(2)  # [batch_size, embed_size, seq_len]
#             dilated_conv = dilated_conv.permute(0, 2, 1)
#             relu1 = F2.relu(block[3](dilated_conv))  # [batch_size, seq_len, embed_size]
#             inputs = ori + relu1
#
#         if onecall:
#             hidden = inputs[:, -1, :].view(-1, self.residual_channels) # [batch_size, embed_size]
#         else:
#             hidden = inputs.view(-1, self.residual_channels) # [batch_size*seq_len, embed_size]
#         out = self.softmax_layer(hidden)
#
#         return out
#
#     def conv_pad(self, inputs, dila_):
#         inputs_pad = inputs.permute(0, 2, 1)  # [batch_size, embed_size, seq_len]
#         inputs_pad = inputs_pad.unsqueeze(2)  # [batch_size, embed_size, 1, seq_len]
#         pad = nn.ZeroPad2d(((self.kernel_size - 1) * dila_, 0, 0, 0))
#         inputs_pad = pad(inputs_pad)  # [batch_size, embed_size, 1, seq_len+(self.kernel_size-1)*self.dilations[i]]
#         return inputs_pad

class Layer_norm(nn.Module):
    def __init__(self, size):
        super(Layer_norm, self).__init__()
        # self.beta = torch.zeros(size, requires_grad=True)
        # self.gamma = torch.ones(size, requires_grad=True)
        self.beta = nn.Parameter(torch.zeros(size))
        # nn.init.zeros_(self.beta)
        self.gamma = nn.Parameter(torch.ones(size))
        # nn.init.ones_(self.gamma)
        self.size = size
        self.epsilon = 1e-8

    def forward(self, x):
        shape = x.size()
        # print(shape)
        # print(x.mean(dim=2).size())
        # print(x.std(dim=2, unbiased=False).size())
        x = (x - x.mean(dim=2).view(shape[0], shape[1], 1)) / (x.std(dim=2, unbiased=False).view(shape[0], shape[1], 1) + self.epsilon)
        return self.gamma * x + self.beta

class SamePad2d(nn.Module):
    """Mimics tensorflow's 'SAME' padding.
    """

    def __init__(self, kernel_size, stride):
        super(SamePad2d, self).__init__()
        self.kernel_size = torch.nn.modules.utils._pair(kernel_size)
        self.stride = torch.nn.modules.utils._pair(stride)

    def forward(self, input):
        in_width = input.size()[2]
        in_height = input.size()[3]
        out_width = math.ceil(float(in_width) / float(self.stride[0]))
        out_height = math.ceil(float(in_height) / float(self.stride[1]))
        pad_along_width = ((out_width - 1) * self.stride[0] +
                           self.kernel_size[0] - in_width)
        pad_along_height = ((out_height - 1) * self.stride[1] +
                            self.kernel_size[1] - in_height)
        pad_left = math.floor(pad_along_width / 2)
        pad_top = math.floor(pad_along_height / 2)
        pad_right = pad_along_width - pad_left
        pad_bottom = pad_along_height - pad_top
        return F2.pad(input, (pad_left, pad_right, pad_top, pad_bottom), 'constant', 0)

    def __repr__(self):
        return self.__class__.__name__


import keras as k
from keras import backend as K
from keras.layers import Input, Dense, Dropout, Embedding, LSTM

class SimpleModel(k.models.Model):

    def __init__(self, input_dim, output_dim, layer_dims):
        input0, output0 = self._create_model(input_dim, output_dim, layer_dims)
        super(SimpleModel, self).__init__(input0, output0)

    def _create_model(self, input_dim, output_dim, layer_dims):
        input0 = Input(input_dim)

        x = Dense(layer_dims[0], activation="relu")(input0)
        x = Dense(layer_dims[1], activation="relu")(x)

        output0 = Dense(output_dim, activation="softmax")(x)

        return input0, output0

    def reset(self):
        for layer in self.layers:
            for k, initializer in layer.__dict__.items():
                if "initializer" not in k:
                    continue
                # find the corresponding variable
                var = getattr(layer, k.replace("_initializer", ""))
                var.assign(initializer(var.shape, var.dtype))

class LSTMTextModel(k.models.Model):

    def __init__(self, input_dim, output_dim):
        input0, output0 = self._create_model(input_dim, output_dim)
        super(LSTMTextModel, self).__init__(input0, output0)

    def _create_model(self, input_dim, output_dim):
        input0 = Input(shape=input_dim, dtype='int32')

        embedding_layer = Embedding(20000, 50, input_length=input_dim[0], trainable=True)

        embedded_sequences = embedding_layer(input0)

        x = LSTM(128, dropout=0.2, recurrent_dropout=0.2)(embedded_sequences)
        output0 = Dense(output_dim, activation='softmax')(x)

        return input0, output0

    def reset(self):
        for layer in self.layers:
            for k, initializer in layer.__dict__.items():
                if "initializer" not in k:
                    continue
                # find the corresponding variable
                var = getattr(layer, k.replace("_initializer", ""))
                var.assign(initializer(var.shape, var.dtype))

    # for x2: optimizer = Adam(lr=0.001)
    # for x1: optimizer = Adam(lr=0.0005)

if __name__ == "__main__":

    model = SimpleModel((50, ), 2)
    model.summary()


from django.shortcuts import render,redirect
from django.http import HttpResponse
from django.template import loader
from .models import *
from django.db.models import Subquery
# Create your views here.
def abc(req):
    u=User1.objects.all()
    #req.session["name"]="roshabh"
    return render(req,'site1/myimg.html',{"mydata"})

def login(request):
    mes=""
    if request.method == 'POST':
        if(request.POST.get("psw")==request.POST.get("psw_repeat")):
            u=User1()
            u.email=request.POST.get("email")
            u.pws=request.POST.get("psw")
            u.contact=request.POST.get("contact")
            u.address=request.POST.get("address")
            u.save()
            mes="sucessful"
            request.session["name"]=request.POST.get("email")

        else:

            mes="error"

    print(mes)
    return render(request,'site1/login.html',{"m":mes})

def signin(request):
    res=""
    if request.method=='POST':
        name=request.POST.get("email1")
        passw=request.POST.get("psw1")
        print(name)
        print(passw)

        try:
            u=User1.objects.get(email=name,pws=passw)
            print("match")
            res="successful"
            request.session["name"]=request.POST.get("email1")
            return redirect ("http://127.0.0.1:8000/site1/home")
        except User1.DoesNotExist:
            print("not match")
            res="failed"

    return render(request,'site1/signin.html',{"a":res})

def createpost(request):
        if request.method == 'POST':
            if request.POST.get('title') and request.POST.get('content'):
                post=Post()
                post.title= request.POST.get('title')
                post.content= request.POST.get('content')
                post.save()

                return render(request, 'site1/login1.html')

        else:
                return render(request,'site1/login1.html')

from .forms import *

def hotel_image_view(request):

    if request.method == 'POST':
        form = HotelForm(request.POST, request.FILES)

        if form.is_valid():
            form.save()
            return redirect('success')
    else:
        form = HotelForm()
    return render(request, 'site1/hotel_image_form.html', {'form' : form})

def success(request):
    return HttpResponse('successfuly uploaded')

# Python program to view
# for displaying images

def display_hotel_images(request):

	if request.method == 'GET':

		# getting all the objects of hotel.
		Hotels = Hotel.objects.all()
		return render(request, 'site1/display_images.html',{'hotel_images' : Hotels})

def cam(request):
    prod=Products.objects.all()
    return render(request, 'site1/cam.html',{'cam' : prod})
def  home(request):

    try:

        name=request.session["name"]
        if request.method=="POST":
            s=request.POST.get("search2")
            p=Products.objects.filter(product=s)
            print(p)
            return render(request,'site1/home.html',{'data':p})
    except KeyError:
        return redirect('http://127.0.0.1:8000/site1/signin/')
    return render(request,'site1/home.html',{"name":name})
def logout(request):
    del request.session["name"]
    return redirect('http://127.0.0.1:8000/site1/signin/')

def myview(request,question_id):
    return HttpResponse(str(question_id))

def disp(req,id):
    if req.method=='POST':
        id1=req.POST.get("id")
        if req.session.get('name'):
            user=req.session["name"]
        else:
            return redirect('http://127.0.0.1:8000/site1/signin/')
        c=Cart(user_id=user,productid=id1)
        c.save()
    prod=Products.objects.get(id=id)
    return render(req, 'site1/disp.html',{'cam' : prod})

def cart(req):

    prod=Cart.objects.all().filter(user_id=req.session["name"])
    p=Products.objects.filter(id__in=Subquery(prod.values('productid')))
    if req.method=='POST':
        prod=Cart.objects.all().filter(user_id=req.session["name"])
        for i in prod:
            a=AddtoCart(user_id=i.user_id,productid=i.productid)
            a.save()
        Cart.objects.all().filter(user_id=req.session["name"]).delete()
    return render(req,'site1/cart.html',{'cam' : p})
def deletecart(request,id):
    Cart.objects.all().filter(user_id=request.session["name"],productid=id).delete()
    return redirect("http://127.0.0.1:8000/site1/cart/")


import numpy as np
import pandas as pd
import pickle
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from sklearn.model_selection import GridSearchCV
from sklearn.neural_network import MLPClassifier
from imblearn.over_sampling import SMOTE
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.feature_selection import SelectFromModel
from sklearn import metrics

def data_proc(df,cat_fields,cont_fields,response):
    df = df
    cat_fields = cat_fields
    cont_fields = cont_fields
    response = response
    # Split data into train and test set
    _Xtrain, _Xtest, ytrain, ytest=train_test_split(
        df[cat_fields+cont_fields],df[response], test_size=.3, random_state=42)

    # Impute missing categorical or continuous values
    catimp = SimpleImputer(missing_values="", strategy="most_frequent")
    contimp = SimpleImputer(missing_values=np.NaN, strategy="most_frequent")
    # Apply imputers to training data
    _cont_df, _cat_df = _Xtrain[cont_fields], _Xtrain[cat_fields]
    cont_df_imp = contimp.fit_transform(_cont_df)
    cat_df_imp = catimp.fit_transform(_cat_df.astype(str))
    cat_df_imp_ = pd.DataFrame(cat_df_imp, columns=_cat_df.columns)
    # Scale continuous features to eliminate magnitude bias - training
    std_scaler = StandardScaler()
    cont_df_scl = pd.DataFrame(
        std_scaler.fit_transform(cont_df_imp), columns=_cont_df.columns)
    # Recombine cat and continuous fields - train
    Xtrain = pd.concat([cont_df_scl, cat_df_imp_], axis=1)
    # Apply imputers to test data
    _cont_df2, _cat_df2 = _Xtest[cont_fields], _Xtest[cat_fields]
    cont_df_imp2 = contimp.transform(_cont_df2)
    cat_df_imp2 = catimp.transform(_cat_df2.astype(str))
    cat_df_imp2 = pd.DataFrame(cat_df_imp2, columns=_cat_df2.columns)
    # Scale continuous features to eliminate magnitude bias. - test
    cont_df_scl2 = pd.DataFrame(
        std_scaler.transform(cont_df_imp2), columns=_cont_df.columns)
    # Recombine cat and continuous fields - test
    Xtest = pd.concat([cont_df_scl2, cat_df_imp2], axis=1)
    # Save scaler for model deployment else comment out
    pickle.dump(std_scaler, open('scaler.sav', 'wb'))

    # One hot encode categorical variables as combined test and train dataset
    # Define variable to identify test and train data
    Xtrain["train"] = 1
    Xtest["train"] = 0
    # Combine test and train data
    combined = pd.concat([Xtrain, Xtest])
    # isolate categorical variables
    cat_df = combined[cat_fields]
    # create encoder and fit; ignore unknown values that come up in predict data set
    enc = OneHotEncoder(handle_unknown='ignore')
    enc.fit(cat_df)
    # transform data set using encoder
    onehotlabels = enc.transform(cat_df).toarray()
    # create df of one-hot array with original labels from df
    cat_df_enc = pd.DataFrame(onehotlabels, columns=enc.get_feature_names(input_features=cat_df.columns))
    # Combine cat/cont variables
    combined.reset_index(inplace=True, drop=True)
    combined_2 = pd.concat([cat_df_enc, combined[cont_fields], combined['train']], axis=1)
    # split combined data set back to train and test
    Xtrain = combined_2[combined_2["train"] == 1]
    Xtest = combined_2[combined_2["train"] == 0]
    Xtrain.drop(["train"], axis=1, inplace=True)
    Xtest.drop(["train"], axis=1, inplace=True)
    # save encoder for predict data set
    pickle.dump(enc, open('encoder.sav', 'wb'))

    # Deploy SMOTE algorithm in case data set is unbalanced
    sm = SMOTE(sampling_strategy='auto')
    X_resampled, y_resampled = sm.fit_sample(Xtrain, ytrain)
    Xtrain = pd.DataFrame(X_resampled, columns=Xtrain.columns)
    ytrain = pd.Series(y_resampled)

    # Use feature selection function to calculate the optimal number of features. Only applicable classification
    # problems
    def feature_selection(xtrain,ytrain,xtest,ytest):
        init_model = XGBClassifier(random_state = 42)
        init_model.fit(xtrain,ytrain)
        # make predictions for test data and evaluate
        thresholds = np.sort(np.unique(init_model.feature_importances_))
        thresh_df = pd.DataFrame()
        for thresh in thresholds:
            # select features using threshold
            selection = SelectFromModel(init_model, threshold=thresh, prefit=True)
            select_xtrain = selection.transform(xtrain)
            # train model
            selection_model = LogisticRegression(max_iter = 1000)
            selection_model.fit(select_xtrain, ytrain)
            # eval model
            select_xtest = selection.transform(xtest)
            y_pred = selection_model.predict(select_xtest)
            predictions = [round(value) for value in y_pred]
            recall = metrics.recall_score(ytest, predictions)
            thresh_df = thresh_df.append({'thresh':thresh, 'num':select_xtrain.shape[1], 'recall':recall},ignore_index=True)
        thresh_df=thresh_df[thresh_df['num']!=1.0]
        val = thresh_df['num'].values[thresh_df['recall'] == thresh_df['recall'].max()].max().astype(int)
        feature_matrix = pd.DataFrame({'field':Xtest.columns,'value':init_model.feature_importances_}).sort_values(by = 'value',ascending = False)
        cols = feature_matrix['field'][:val].values.tolist()
        xtrain = xtrain[cols]
        xtest = xtest[cols]
        return(xtrain,xtest)

    Xtrain,Xtest = feature_selection(xtrain=Xtrain,ytrain=ytrain,xtest=Xtest,ytest=ytest)
    return(Xtrain,ytrain,Xtest,ytest)


import math
import random
from collections import namedtuple, deque
from itertools import count

import gym
import matplotlib.pyplot as plt
import torch

BATCH_SIZE = 128
GAMMA = 0.99
EPS_START = 0.9
EPS_END = 0.05
EPS_DECAY = 30
TAU = 0.005
LR = 1e-4

env = gym.make("CartPole-v1")

Transition = namedtuple('Transition', ('state', 'action', 'next_state', 'reward'))

class ReplayMemory:

    def __init__(self, capacity):
        self.memory = deque([], maxlen=capacity)

    def push(self, *args):
        """Save a transition"""
        self.memory.append(Transition(*args))

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def clean(self):
        self.memory.clear()

    def __len__(self):
        return len(self.memory)

class AgentNN(torch.nn.Module):  # DQL module
    def __init__(self, n_observations, n_actions):
        super(AgentNN, self).__init__()
        self.module = torch.nn.Sequential(
            torch.nn.Linear(n_observations + n_actions, 128),
            torch.nn.ReLU(),
            torch.nn.Linear(128, 128),
            torch.nn.ReLU(),
            torch.nn.Linear(128, 1),
        )

    def forward(self, observation: torch.Tensor, action: torch.Tensor) -> torch.Tensor:
        """
        :param observation: a ndarray representing the current state.
        :param action: an int.
        :return: expected return.
        """
        inin = torch.cat((observation, action), dim=1)

        return self.module(inin)

episode_durations = []

def plot_durations(show_result=False):
    plt.figure(1)
    durations_t = torch.tensor(episode_durations, dtype=torch.float)
    if show_result:
        plt.title('Result')
    else:
        plt.clf()
        plt.title('Training...')
    plt.xlabel('Episode')
    plt.ylabel('Duration')
    plt.plot(durations_t.numpy())

    plt.pause(0.001)  # pause a bit so that plots are updated

def peek_action(steps_done: int, state: torch.Tensor) -> torch.Tensor:
    sample = random.random()
    # eps_threshold = EPS_END + (EPS_START - EPS_END) * math.exp(-1. * steps_done / EPS_DECAY)
    # eps_threshold = min(1.0, EPS_END + (EPS_START - EPS_END) / pow(math.log(steps_done+2), 3))
    eps_threshold = min(1.0, (EPS_START - EPS_END) / pow(math.log(steps_done+2), 3))
    if sample > eps_threshold:
        with torch.no_grad():
            left = torch.Tensor([[1, 0]])
            right = torch.Tensor([[0, 1]])

            out = policy_net.forward(
                torch.cat((state, state), dim=0),
                torch.cat((left, right), dim=0)
            ).argmax()

            # if out is 0 then them going left is better and if out is 1 then going right is better
            return_out = torch.zeros((1, 2), dtype=torch.float32)
            return_out[0][out] = 1

            return return_out
    else:
        return_out = torch.zeros((1, 2), dtype=torch.float32)
        return_out[0][env.action_space.sample()] = 1

        return return_out

def optimize_model():
    if len(memory) < BATCH_SIZE:
        return
    transitions = memory.sample(BATCH_SIZE)
    # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for
    # detailed explanation). This converts batch-array of Transitions
    # to Transition of batch-arrays.
    batch = Transition(*zip(*transitions))

    # Compute a mask of non-final states and concatenate the batch elements
    # (a final state would've been the one after which simulation ended)
    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,
                                            batch.next_state)), dtype=torch.bool)
    non_final_next_states = torch.cat([s for s in batch.next_state
                                       if s is not None])
    state_batch = torch.cat(batch.state)
    action_batch = torch.cat(batch.action)
    reward_batch = torch.cat(batch.reward).unsqueeze(1)

    state_action_values = policy_net(state_batch, action_batch)

    go_to_the_left_actions = torch.cat((torch.ones(non_final_next_states.shape[0], 1), torch.zeros(non_final_next_states.shape[0], 1)), dim=1)
    go_to_the_right_actions = torch.cat((torch.zeros(non_final_next_states.shape[0], 1), torch.ones(non_final_next_states.shape[0], 1)), dim=1)

    next_state_values = torch.zeros(BATCH_SIZE, 1)
    with torch.no_grad():
        value_for_left = target_net(non_final_next_states, go_to_the_left_actions)
        value_for_right = target_net(non_final_next_states, go_to_the_right_actions)

        my_max = torch.max(value_for_left, value_for_right)
        next_state_values[non_final_mask] = my_max

    expected_state_action_values = (next_state_values * GAMMA) + reward_batch

    # Compute Huber loss
    criterion = torch.nn.SmoothL1Loss()
    loss = criterion(state_action_values, expected_state_action_values)

    # Optimize the model
    optimizer.zero_grad()
    loss.backward()
    # In-place gradient clipping
    torch.nn.utils.clip_grad_value_(policy_net.parameters(), 100)
    optimizer.step()

# Get number of actions from gym action space
n_actions = env.action_space.n
# Get the number of state observations
state, info = env.reset()
n_observations = len(state)

policy_net = AgentNN(n_observations, n_actions)
target_net = AgentNN(n_observations, n_actions)
target_net.load_state_dict(policy_net.state_dict())

optimizer = torch.optim.AdamW(policy_net.parameters(), lr=LR, amsgrad=True)
memory = ReplayMemory(10000)

if torch.cuda.is_available():
    num_episodes = 600
else:
    num_episodes = 600

for i_episode in range(num_episodes):
    # Initialize the environment and get it's state
    state, info = env.reset()
    state = torch.tensor(state, dtype=torch.float32).unsqueeze(0)
    step_done = 0

    for t in count():
        action = peek_action(step_done, state)
        step_done += 1
        observation, reward, terminated, truncated, _ = env.step((action == 1).squeeze(0).nonzero().item())
        reward = torch.tensor([reward])
        done = terminated or truncated

        if terminated:
            next_state = None
        else:
            next_state = torch.tensor(observation, dtype=torch.float32).unsqueeze(0)

        # Store the transition in memory
        memory.push(state, action, next_state, reward)

        # Move to the next state
        state = next_state

        # Perform one step of the optimization (on the policy network)
        optimize_model()

        # Soft update of the target network's weights
        # θ′ ← τ θ + (1 −τ )θ′
        target_net_state_dict = target_net.state_dict()
        policy_net_state_dict = policy_net.state_dict()
        for key in policy_net_state_dict:
            target_net_state_dict[key] = policy_net_state_dict[key] * TAU + target_net_state_dict[key] * (1 - TAU)
        target_net.load_state_dict(target_net_state_dict)

        if len(memory) > 9000:
            memory.clean()

        if len(episode_durations) > 1000:
            episode_durations.clear()

        if done:
            episode_durations.append(t + 1)
            plot_durations()
            break

print('Complete')
plot_durations(show_result=True)
plt.ioff()
plt.show()

torch.save({
    "my_agent": policy_net.state_dict()
}, "my_agent_file.pt")

env = gym.make("CartPole-v1", render_mode="human")

for i_episode in range(num_episodes):
    # Initialize the environment and get it's state
    state, info = env.reset()
    state = torch.tensor(state, dtype=torch.float32).unsqueeze(0)
    step_done = 0

    for t in count():
        action = peek_action(step_done, state)
        step_done += 1
        observation, reward, terminated, truncated, _ = env.step((action == 1).squeeze(0).nonzero().item())

        done = terminated or truncated

        if terminated:
            next_state = None
        else:
            next_state = torch.tensor(observation, dtype=torch.float32).unsqueeze(0)

        # Move to the next state
        state = next_state

        if done:
            break


#---------------------------------------------------------------------------
#
# Modeling Atlantic Forest regenerability
#
# 3. Combining maps to define a final raster map of natural vegetation
#    in the extended limit of the Atlantic Forest
# Including FBDS map
#
# Bernardo BS Niebuhr - bernardo_brandaum@yahoo.com.br
# Mauricio Humberto Vancine
#
# 02/06/2017
# Feel free to modify and share this code
#---------------------------------------------------------------------------

###--------------------------------------------------------------------###
## Import python modules
import os # operational system commands
import grass.script as grass # GRASS GIS commands
###--------------------------------------------------------------------###

# 1. Gather FBDS maps for all regions and all uses and export

# Region of study - for gathering FBDS maps regardless of AF limits

# # First trial - set it manually - NO!
#grass.run_command('g.region', n=3028995, s=138585, w=-74890, e=2742860, res = 30, flags = 'p')
#grass.run_command('g.region', vector = 'limite_leima_ribeiroetal2009_albers_sad69_dissolve_shp', res = 30, flags = 'ap')

# List of maps
maps_fbds = grass.list_grouped('rast', pattern = '*v2*')['PERMANENT']

# Region of study
# # Second trial - align it to the resolution - problem - the pixels may displace a little!!
grass.run_command('g.region', rast = maps_fbds, res = 30, flags = 'ap')

# # Third trial - use base map and increase an extesion to the west (dividible by the resolution 30m) to include all states
grass.run_command('g.region', rast = 'MA_extendido_rast', flags = 'p') # This extension already has resolution = 30m
grass.run_command('g.region', w = 'w-300000', flags = 'p') # reduce the west limit in 300km, which inclues all FBDS state maps and is divisible per 30
# this is ok! we kept only that.

# Combine maps
grass.run_command('r.patch', input = maps_fbds, output = 'FBDS_map_all_states', overwrite = True)

# Export FBDS map for all uses
folder = r'G:\regenerabilidade_MA\02_saidas\FBDS_combined'
os.chdir(folder)
grass.run_command('r.out.gdal', input = 'FBDS_map_all_states', output = 'FBDS_map_all_states.tif')
# With compression
grass.run_command('r.out.gdal', input = 'FBDS_map_all_states', output = 'FBDS_map_all_states_compressed.tif', createopt="COMPRESS=DEFLATE")

# 2. Gather FBDS maps for all regions and each use separately, and export

out_dir = r'G:\regenerabilidade_MA\02_saidas\FBDS_combined\separated_classes'
os.chdir(out_dir)

# 1 = water
expression1 = 'FBDS_map_all_states_water = if(FBDS_map_all_states == 1, 1, 0)'
grass.mapcalc(expression1, overwrite = True)

grass.run_command('r.out.gdal', input = 'FBDS_map_all_states_water', output = 'FBDS_map_all_states_water.tif')

# 2 = antropic areas
expression2 = 'FBDS_map_all_states_antropic_areas = if(FBDS_map_all_states == 2, 1, 0)'
grass.mapcalc(expression2, overwrite = True)

grass.run_command('r.out.gdal', input = 'FBDS_map_all_states_antropic_areas', output = 'FBDS_map_all_states_antropic_areas.tif')

# 3 = building area
expression3 = 'FBDS_map_all_states_building_areas = if(FBDS_map_all_states == 3, 1, 0)'
grass.mapcalc(expression3, overwrite = True)

grass.run_command('r.out.gdal', input = 'FBDS_map_all_states_building_areas', output = 'FBDS_map_all_states_building_areas.tif')

# 4 = forest
expression4 = 'FBDS_map_all_states_forest = if(FBDS_map_all_states == 4, 1, 0)'
grass.mapcalc(expression4, overwrite = True)

grass.run_command('r.out.gdal', input = 'FBDS_map_all_states_forest', output = 'FBDS_map_all_states_forest.tif')

# 5 = non forest natural areas
expression5 = 'FBDS_map_all_states_non_forest_natural_areas = if(FBDS_map_all_states == 5, 1, 0)'
grass.mapcalc(expression5, overwrite = True)

grass.run_command('r.out.gdal', input = 'FBDS_map_all_states_non_forest_natural_areas', output = 'FBDS_map_all_states_non_forest_natural_areas.tif')

# 6 = silviculture
expression6 = 'FBDS_map_all_states_silviculture = if(FBDS_map_all_states == 6, 1, 0)'
grass.mapcalc(expression6, overwrite = True)

grass.run_command('r.out.gdal', input = 'FBDS_map_all_states_silviculture', output = 'FBDS_map_all_states_silviculture.tif')

# better to do 1/null??

# Exporting compressed
per_class_maps_fbds = grass.list_grouped('rast', pattern = 'FBDS_map_all_states_*') ['PERMANENT']
out_dir = r'G:\regenerabilidade_MA\02_saidas\FBDS_combined\separated_classes'
os.chdir(out_dir)

for i in per_class_maps_fbds:
    # With compression
    grass.run_command('r.out.gdal', input = i, output = i+'_compressed.tif', createopt="COMPRESS=DEFLATE")

# 3. Cross sugarcane and pasture with antropic areas of FBDS map

# Region
grass.run_command('g.region', raster = 'FBDS_map_all_states', flags = 'p')
grass.run_command('g.region', zoom = 'FBDS_map_all_states', flags = 'p') # This extension already has resolution = 30m

# Cross maps
# First overlap sugarcane
expression7 = 'FBDS_antropic_areas1_sugarcane7 = if(isnull(canasat_2013_albers_sad69_rast), FBDS_map_all_states_antropic_areas, 7)'
grass.mapcalc(expression7, overwrite = True)
# Then, overlap pasture
expression8 = 'FBDS_antropic_areas1_pasture8 = if(pastagens_br_2015_albers_sad69_tif == 1 &&& FBDS_antropic_areas1_sugarcane7 != 7, 8, FBDS_antropic_areas1_sugarcane7)'
grass.mapcalc(expression8, overwrite = True)
# Consider sugarcane and pasture only where antropic areas == 1
expression9 = 'FBDS_antropic_areas1_sugarcane7_pasture8 = if(FBDS_map_all_states_antropic_areas == 0, 0, FBDS_antropic_areas1_pasture8)'
grass.mapcalc(expression9, overwrite = True)

# Remove intermediate maps
grass.run_command('g.remove', type = 'raster', pattern = 'FBDS_antropic_areas1_sugarcane7', flags = 'f')
grass.run_command('g.remove', type = 'raster', pattern = 'FBDS_antropic_areas1_pasture8', flags = 'f')

# Export
out_dir = r'G:\regenerabilidade_MA\02_saidas\FBDS_combined\separated_classes'
os.chdir(out_dir)

grass.run_command('r.out.gdal', input = 'FBDS_antropic_areas1_sugarcane7_pasture8', output = 'FBDS_antropic_areas1_sugarcane7_pasture8.tif')
grass.run_command('r.out.gdal', input = 'FBDS_antropic_areas1_sugarcane7_pasture8', output = 'FBDS_antropic_areas1_sugarcane7_pasture8_compressed.tif', createopt="COMPRESS=DEFLATE")

#------------------------------------------
# 3. Maps for the expanded AF (the consensus limit) - resolution 30m

out_dir_ext_AF = r'G:\regenerabilidade_MA\02_saidas\extended_AF_consensus_30m'
os.chdir(out_dir_ext_AF)

maps_export = []

# Region of study - for the AF
grass.run_command('g.region', vector = 'limit_af_consensus_maa_ribeiroetal2009_lawaf2006_wwf_albers_sad69', res = 30, flags = 'p',
                  align = 'MA_extendido_rast') # nao da a resolucao exata 30m - vamos adicionar um pouco na extensao

# Mask for the AF limit
grass.run_command('r.mask', vector = 'limit_af_consensus_maa_ribeiroetal2009_lawaf2006_wwf_albers_sad69') # Mask for the extended AF

# a. FBDS map for this delimitation

# FBDS map
grass.mapcalc('FBDS_map_extended_AF = FBDS_map_all_states', overwrite = True) # Filling SP with a binary map of forest patches
maps_export = maps_export + ['FBDS_map_extended_AF']

# b. Get only FBDS forest
# With 1/0
grass.mapcalc('FBDS_map_extended_AF_forest = if(FBDS_map_all_states == 4, 1, 0)', overwrite = True)
# With 1/null
grass.mapcalc('FBDS_map_extended_AF_forest_1null = if(FBDS_map_all_states == 4, 1, null())', overwrite = True)

maps_export = maps_export + ['FBDS_map_extended_AF_forest_1null']

# c. SOS MA forest map - 2014

# already calculated for 30m
maps_export = maps_export + ['atlas_ma_sos_ma_2014_mata_restinga_mangue_albers_sad69_rast']

# SOS MA forest map - 2005? export ou mosaic? We won't use it
# mapbiomas all? mapbiomas only forest? mapbiomas only forest < 10ha? export ou mosaic? which biome? We won't use it

# d. map of cities - FBDS + cities(SOS+IBGE) + Argentina

#### CHANGE HERE LATER WHEN PY CITIES IS UPDATED

# Mosaic
maps_cities = ['FBDS_map_all_states_building_areas', 'urban_areas_composition_ibge_SOS_limiteMAestendido_albers_sad69_rast', 'urban_areas_AR_albers_sad69_rast']
grass.run_command('r.patch', input = maps_cities, output = 'cities_FBDS_SOS_IBGE_AR', overwrite = True)

# Version 1/null
expression10 = 'cities_FBDS_SOS_IBGE_AR_1null = if(cities_FBDS_SOS_IBGE_AR == 1, 1, null())'
grass.mapcalc(expression10, overwrite = True)

# Version 1/0
expression11 = 'cities_FBDS_SOS_IBGE_AR_1_0 = if(isnull(cities_FBDS_SOS_IBGE_AR_1null), 0, 1)'
grass.mapcalc(expression11, overwrite = True)

# Remove temp map
grass.run_command('g.remove', type = 'raster', pattern = 'cities_FBDS_SOS_IBGE_AR', flags = 'f')

maps_export = maps_export + ['cities_FBDS_SOS_IBGE_AR_1_0']

# e. forest map Paraguay and Argentina (without considering forest plantation GFW, cities, roads, and water)
#### CHANGE HERE LATER WHEN PY CITIES IS UPDATED

expression12 = 'hansen_2000_treecover_threshold95_no_forestry_no_cities = if(isnull(gfw_tree_plantations_albers_sad69_rast) &&& \
isnull(cities_FBDS_SOS_IBGE_AR_1null) &&& \
isnull(water_AR_albers_sad69_rast) &&& \
isnull(roads_AR_albers_sad69_rast) &&& \
isnull(roads_PY_albers_sad69_rast) &&& \
isnull(rivers_PY_albers_sad69_rast) &&& \
hansen_2000_treecover_threshold95_albers_sad69 > 0, 1, null())'
grass.mapcalc(expression12, overwrite = True)

maps_export = maps_export + ['hansen_2000_treecover_threshold95_no_forestry_no_cities']

# f. water map FBDS + ANA + Argentina

#### CHANGE HERE LATER WHEN PY AND AR WATER MAPS ARE UPDATED

# mosaic ANA + FBDS + AR
maps_water = ['geoft_bho_massa_dagua_albers_sad69_rast', 'FBDS_map_all_states_water', 'water_AR_albers_sad69_rast', 'rivers_PY_albers_sad69_rast']
grass.run_command('r.patch', input = maps_water, output = 'water_FBDS_ANA_AR', overwrite = True)

# transform in 1/null
expression13 = 'water_FBDS_ANA_AR_1null = if(water_FBDS_ANA_AR == 1, 1, null())'
grass.mapcalc(expression13, overwrite = True)

# transform in 1/0
expression14 = 'water_FBDS_ANA_AR_1_0 = if(isnull(water_FBDS_ANA_AR_1null), 0, 1)'
grass.mapcalc(expression14, overwrite = True)

# remove aux map
grass.run_command('g.remove', type = 'raster', pattern = 'water_FBDS_ANA_AR', flags = 'f')

maps_export = maps_export + ['water_FBDS_ANA_AR_1_0']

# g. Roads DNIT + Py + Ar

# Removing roads AR which overlap with Brazilian territory
expression15 = 'roads_AR_albers_sad69_rast_no_BR = if(isnull(FBDS_map_extended_AF), roads_AR_albers_sad69_rast, null())'
grass.mapcalc(expression15, overwrite = True)

# Mosaic
maps_road = ['dnit_pavimentada_albers_sad69_rast', 'roads_PY_albers_sad69_rast', 'roads_AR_albers_sad69_rast_no_BR']
grass.run_command('r.patch', input = maps_road, output = 'roads_DNIT_PY_AR', overwrite = True)

# transform in 1/0
expression16 = 'roads_DNIT_PY_AR_1_0 = if(isnull(roads_DNIT_PY_AR), 0, 1)'
grass.mapcalc(expression16, overwrite = True)

# remove aux map
grass.run_command('g.remove', type = 'raster', pattern = 'roads_AR_albers_sad69_rast_no_BR', flags = 'f')

maps_export = maps_export + ['roads_DNIT_PY_AR_1_0']

# h. FBDS antropic use + sugarcane and pasture

maps_export = maps_export + ['FBDS_antropic_areas1_sugarcane7_pasture8']

# i. pasture LAPIG 2015 (which are inside antropic areas nor crossed by water or roads)

# within FBDS limit - pasture inside antropic areas
expression17 = 'pasture_br_FBDS_limit_AF = if(FBDS_antropic_areas1_sugarcane7_pasture8 == 8, 1, 0)'
grass.mapcalc(expression17, overwrite = True)

# for outside FBDS area - pasture which is not forest SOS 2014 nor Hansen 2000
expression18 = 'pasture_br_non_SOSMA_2014_Hansen = if(pastagens_br_2015_albers_sad69_tif == 1 &&& \
isnull(atlas_ma_sos_ma_2014_mata_restinga_mangue_albers_sad69_rast) &&& \
isnull(hansen_2000_treecover_threshold95_no_forestry_no_cities), 1, 0)'
grass.mapcalc(expression18, overwrite = True)

# mosic pasture inside FBDS limit + outside
grass.run_command('r.patch', input = 'pasture_br_FBDS_limit_AF,pasture_br_non_SOSMA_2014_Hansen', output = 'pasture_non_forest_BR_2015', overwrite = True)

# exclude areas in which there is water and roads
expression19 = 'pasture_non_forest_water_roads_cities_BR_2015 = if(pasture_non_forest_BR_2015 >= 1 &&& \
!isnull(pasture_non_forest_BR_2015) &&& \
isnull(roads_DNIT_PY_AR) &&& \
isnull(water_FBDS_ANA_AR_1null) &&& \
isnull(cities_FBDS_SOS_IBGE_AR_1null), 1, 0)'
grass.mapcalc(expression19, overwrite = True)

# remove aux map
grass.run_command('g.remove', type = 'raster', pattern = 'pasture_br_FBDS_limit_AF', flags = 'f')
grass.run_command('g.remove', type = 'raster', pattern = 'pasture_non_forest_BR_2015', flags = 'f')
grass.run_command('g.remove', type = 'raster', pattern = 'pasture_br_non_SOSMA_2014_Hansen', flags = 'f')

maps_export = maps_export + ['pasture_non_forest_water_roads_cities_BR_2015']

# j. Combine vegetation maps

# SOS MA forest fragment without cities
expression20 = 'forest_sos_no_cities = if(!isnull(atlas_ma_sos_ma_2014_mata_restinga_mangue_albers_sad69_rast) &&& isnull(cities_FBDS_SOS_IBGE_AR_1null), 1, null())'
grass.mapcalc(expression20, overwrite = True)

# combine FBDS forest with SOSMA no cities and Hansen 2000 (without cities and forestry)
maps_forest = ['FBDS_map_extended_AF_forest', 'forest_sos_no_cities', 'hansen_2000_treecover_threshold95_no_forestry_no_cities']
grass.run_command('r.patch', input = maps_forest, output = 'FBDS_AF_forest_sos_2014_Hansen_no_cities', overwrite = True)

#and subtract water and roads
expression21 = 'FBDS_AF_forest_sos_2014_Hansen_no_cities_water_roads = if(FBDS_AF_forest_sos_2014_Hansen_no_cities > 0 &&& \
!isnull(FBDS_AF_forest_sos_2014_Hansen_no_cities) &&& \
isnull(roads_DNIT_PY_AR) &&& \
isnull(water_FBDS_ANA_AR_1null), FBDS_AF_forest_sos_2014_Hansen_no_cities, null())'
grass.mapcalc(expression21, overwrite = True)

maps_export = maps_export + ['FBDS_AF_forest_sos_2014_Hansen_no_cities_water_roads']

# Transform into 1/0
expression22 = 'FBDS_AF_forest_sos_2014_Hansen_no_cities_water_roads_1_0 = if(isnull(FBDS_AF_forest_sos_2014_Hansen_no_cities_water_roads), 0, 1)'
grass.mapcalc(expression22, overwrite = True)

maps_export = maps_export + ['FBDS_AF_forest_sos_2014_Hansen_no_cities_water_roads_1_0']

# Remove intermediate maps
grass.run_command('g.remove', type = 'raster', pattern = 'FBDS_AF_forest_sos_2014_Hansen_no_cities', flags = 'f')

# Exporting maps 30m
out_dir_ext_AF = r'G:\regenerabilidade_MA\02_saidas\extended_AF_consensus_30m'
os.chdir(out_dir_ext_AF)

for i in maps_export:
    # Without compression
    grass.run_command('r.out.gdal', input = i, output = i+'_30m.tif')
    # With compression
    grass.run_command('r.out.gdal', input = i, output = i+'_30m_compressed.tif', createopt="COMPRESS=DEFLATE")

grass.run_command('r.mask', flags = 'r') # Remove mask

#------------------------------------------
# 3. Maps of vegetation and pasture for the AF (limit AF law + Ribeiro et al 2009) - resolution 30m
# For inferring AF regenerability

# Region of study - for the AF
grass.run_command('g.region', vector = 'limite_leima_ribeiroetal2009_albers_sad69_shp', res = 30, flags = 'p',
                  align = 'MA_extendido_rast')

# Mask for the AF limit
grass.run_command('r.mask', vector = 'limite_leima_ribeiroetal2009_albers_sad69_shp') # Mask for the extended AF

# Export vegetation map and pasture map

mapas_export_regen = ['FBDS_AF_forest_sos_2014_Hansen_no_cities_water_roads_1_0', 'pasture_non_forest_water_roads_cities_BR_2015']

# Exporting maps 30m
out_dir_ext_AF = r'G:\regenerabilidade_MA\02_saidas\regenerability_leima_Ribeiro_30m'
os.chdir(out_dir_ext_AF)

for i in mapas_export_regen:
    # Without compression
    grass.run_command('r.out.gdal', input = i, output = i+'_30m.tif')
    # With compression
    grass.run_command('r.out.gdal', input = i, output = i+'_30m_compressed.tif', createopt="COMPRESS=DEFLATE")

grass.run_command('r.mask', flags = 'r') # Remove mask

#------------------------------------------
# 4. Maps for the expanded AF (the consensus limit) - resolution 1km

maps_export = []

# Region of study - for the AF
grass.run_command('g.region', vector = 'limit_af_consensus_maa_ribeiroetal2009_lawaf2006_wwf_albers_sad69', res = 1000, flags = 'ap')

# Mask
grass.run_command('r.mask', vector = 'limit_af_consensus_maa_ribeiroetal2009_lawaf2006_wwf_albers_sad69') # Mask for the extended AF

# a. FBDS map for this delimitation

# FBDS map
maps_export = maps_export + ['FBDS_map_extended_AF']

# b. FBDS only forest
maps_export = maps_export + ['FBDS_map_extended_AF_forest_1null']

# c. SOS MA forest map - 2014

# already calculated for 30m
maps_export = maps_export + ['atlas_ma_sos_ma_2014_mata_restinga_mangue_albers_sad69_rast']

# d. map of cities - FBDS + cities(SOS+IBGE) + Argentina

#### CHANGE HERE LATER WHEN PY CITIES IS UPDATED

maps_export = maps_export + ['cities_FBDS_SOS_IBGE_AR_1_0']

# e. forest map Paraguay and Argentina (without considering forest plantation GFW, cities, roads, and water)
#### CHANGE HERE LATER WHEN PY CITIES IS UPDATED

maps_export = maps_export + ['hansen_2000_treecover_threshold95_no_forestry_no_cities']

# f. water map FBDS + ANA + Argentina

#### CHANGE HERE LATER WHEN PY AND AR WATER MAPS ARE UPDATED

maps_export = maps_export + ['water_FBDS_ANA_AR_1_0']

# g. Roads DNIT + Py + Ar

maps_export = maps_export + ['roads_DNIT_PY_AR_1_0']

# h. FBDS antropic use + sugarcane and pasture

maps_export = maps_export + ['FBDS_antropic_areas1_sugarcane7_pasture8']

# i. pasture LAPIG 2015 (which are inside antropic areas nor crossed by water or roads)

maps_export = maps_export + ['pasture_non_forest_water_roads_cities_BR_2015']

# j. Combine vegetation maps

maps_export = maps_export + ['FBDS_AF_forest_sos_2014_Hansen_no_cities_water_roads']

maps_export = maps_export + ['FBDS_AF_forest_sos_2014_Hansen_no_cities_water_roads_1_0']

# Exporting maps 1km
out_dir_ext_AF = r'G:\regenerabilidade_MA\02_saidas\extended_AF_consensus_1km'
os.chdir(out_dir_ext_AF)

for i in maps_export:
    # Without compression
    grass.run_command('r.out.gdal', input = i, output = i+'_1km.tif')
    # With compression
    grass.run_command('r.out.gdal', input = i, output = i+'_1km_compressed.tif', createopt="COMPRESS=DEFLATE")

grass.run_command('r.mask', flags = 'r') # Remove mask


"""
Contains some statistics.
"""

import datetime

class Stats:
  def __init__(self):
    self.started = str(datetime.datetime.utcnow())
    self.requests = 0 # total requests (not just image route)
    self.waiting = 0 # waiting in queue
    self.timeout = 0 # timed out for waiting too long
    self.skipped = 0 # skipped waiting when couldn't find element
    self.malformed = 0 # request malformed count
    self.success = 0 # success image request
    self.failed = 0 # image failed request, and hopefully logs to discord.log unless repl.it files fail
    self.toobig = 0 # failed because image too big
    self.pinged = 0 # ping count
    self.usecache = 0 # used from cache count
    # self.unknown for non-discord/google


import streamlit as st
import numpy as np
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from utils import *

# set default page config to wide
st.set_page_config(layout="wide")

# Title text of the app
title_txt = "Plotting intersection of curves using gradient descent"
st.markdown(f"
{title_txt}

", unsafe_allow_html=True)

# Next row of app will have 4 columns
#   column-1 : Input for function-1
#   column-2 : Input for function-2
#   column-3 : Input for no. of points to plot
#   column-4 : Display equations of functions
col1,col2,col3,col4 = st.columns([1,2,2,4])
with col1:
    func1 = st.radio("Function-1", ('DoubleCone1', 'Sphere1', 'Cylinder1'))
    func1 = str(func1)
with col2:
    func2 = st.radio("Function-2", ('DoubleCone2', 'Sphere2', 'Cylinder2'))
    func2 = str(func2)
with col3:
    N = st.slider('Number of points to plot. More the slower', 200, 2000, 500, step=100)
with col4:
    st.write(F1_curve(X, func1))
    st.write(F2_curve(X, func2))

# Next row of app will have 2 columns
#   column-1 : Plot for input functions
#   column-2 : Plot for intersection of the functions
col1,col2 = st.columns(2)

with col1:
    data1 = generate_F1_sample(N, func1)
    data2 = generate_F2_sample(N, func2)

    x, y, z = data1[:, 0], data1[:, 1], data1[:, 2]
    trace1 = go.Scatter3d(x=x, y=y, z=z,mode='markers', name="F1", marker=dict(size=5,sizemode='diameter'))
    x, y, z = data2[:, 0], data2[:, 1], data2[:, 2]
    trace2 = go.Scatter3d(x=x, y=y, z=z,mode='markers', name="F2", marker=dict(size=5,sizemode='diameter'))

    fig = make_subplots()
    fig.add_trace(trace1)
    fig.add_trace(trace2)
    fig.update_layout(margin=dict(l=0, r=0, b=0, t=0), height=600, width=1000)
    st.plotly_chart(fig, use_container_width=True, height=600, width=1000)

with col2:
    ## Plot for Intersection of Curves
    data = generate_intersection_sample(N, func1,func2)
    data = np.array(data)
    x, y, z = data[:, 0], data[:, 1], data[:, 2]

    fig = go.Figure(data=[
            go.Scatter3d(x=x, y=y, z=z,mode='markers',marker=dict(size=5,sizemode='diameter'))]
        )
    fig.update_layout(margin=dict(l=0, r=0, b=0, t=0), height=600, width=800)
    st.plotly_chart(fig, use_container_width=True, height=600, width=800)

readmeLink="#### [Mathematical details and approach](https://github.com/vdivakar/curves-intersection-with-gradient-descent/blob/main/README.md)"
st.markdown(readmeLink, unsafe_allow_html=True)

repoLink="#### [Github Repository](https://github.com/vdivakar/curves-intersection-with-gradient-descent)"
st.markdown(repoLink, unsafe_allow_html=True)


def extractSunflowerpenguinWordpressCom(item):
	'''
	Parser for 'sunflowerpenguin.wordpress.com'
	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return None

	tagmap = [
		('heroine yuri flags',       'When I Reincarnated as the Villainess, I Raised Yuri Flags with the Heroine!?',                      'translated'),
		('black lotus heroine',      'The Female Lead is a Black Lotus',                      'translated'),
		('PRC',       'PRC',                      'translated'),
		('Loiterous', 'Loiterous',                'oel'),
	]

	for tagname, name, tl_type in tagmap:
		if tagname in item['tags']:
			return buildReleaseMessageWithType(item, name, vol, chp, frag=frag, postfix=postfix, tl_type=tl_type)

	return False


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import os
import re
import hickle as hkl
from scipy import *

# reshape list
def reshape(seq, rows, cols):
    return [list(u) for u in zip(*[iter(seq)] * cols)]

# helper function from pykitti: https://github.com/utiasSTARS/pykitti
def rotx(t):
    """Rotation about the x-axis."""
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[1,  0,  0],
                     [0,  c, -s],
                     [0,  s,  c]])

# helper function from pykitti: https://github.com/utiasSTARS/pykitti
def roty(t):
    """Rotation about the y-axis."""
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[c,  0,  s],
                     [0,  1,  0],
                     [-s, 0,  c]])

# helper function from pykitti: https://github.com/utiasSTARS/pykitti
def rotz(t):
    """Rotation about the z-axis."""
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[c, -s,  0],
                     [s,  c,  0],
                     [0,  0,  1]])

# helper function from pykitti: https://github.com/utiasSTARS/pykitti
def pose_from_oxts_packet(lat,lon,alt,roll,pitch,yaw,scale):
    """Helper method to compute a SE(3) pose matrix from an OXTS packet.
    """
    er = 6378137.  # earth radius (approx.) in meters

    # Use a Mercator projection to get the translation vector
    tx = scale * lon * np.pi * er / 180.
    ty = scale * er * \
        np.log(np.tan((90. + lat) * np.pi / 360.))
    tz = alt
    t = np.array([tx, ty, tz])

    # Use the Euler angles to get the rotation matrix
    Rx = rotx(roll)
    Ry = roty(pitch)
    Rz = rotz(yaw)
    R = Rz.dot(Ry.dot(Rx))

    # Combine the translation and rotation into a homogeneous transform
    return R, t

# position of vertex relative to global origin from pykitti: https://github.com/utiasSTARS/pykitti
def pose_from_GIS(lat,lon,scale,origin):
    """Helper method to compute a SE(3) pose matrix from an OXTS packet.
    """
    er = 6378137.  # earth radius (approx.) in meters

    # Use a Mercator projection to get the translation vector
    tx = scale * lon * np.pi * er / 180.
    ty = scale * er * \
        np.log(np.tan((90. + lat) * np.pi / 360.))
    # 2D position
    t = np.array([tx, ty])

    return (t-origin[0:2])

# helper function from pykitti: https://github.com/utiasSTARS/pykitti
def transform_from_rot_trans(R, t):
    """Homogeneous transformation matrix from rotation matrix and translation vector."""
    R = R.reshape(3, 3)
    t = t.reshape(3, 1)
    return np.vstack((np.hstack([R, t]), [0, 0, 0, 1]))

# functions to check if point is in polygon (http://stackoverflow.com/questions/16625507/python-checking-if-point-is-inside-a-polygon)

def ccw(A,B,C):
    return (C[1]-A[1]) * (B[0]-A[0]) >= (B[1]-A[1]) * (C[0]-A[0])

# Return true if line segments AB and CD intersect
def intersect(A,B,C,D):
    return ccw(A,C,D) != ccw(B,C,D) and ccw(A,B,C) != ccw(A,B,D)

def point_in_polygon(pt, poly, inf):
    result = False
    for i in range(poly.shape[0]-1):
        if intersect((poly[i,0], poly[i,1]), ( poly[i+1,0], poly[i+1,1]), (pt[0], pt[1]), (inf, pt[1])):
            result = not result
    if intersect((poly[-1,0], poly[-1,1]), (poly[0,0], poly[0,1]), (pt[0], pt[1]), (inf, pt[1])):
        result = not result
    return result

# create a grid in the form of a numpy array with coordinates representing
# the middle of the cell
# cell resolution: 0.33 cm

def global_grid(origin,endpoint,res):
    # create a grid of x and y values that have an associated x,y centre
    # coordinate in the global space (42.7 m buffer around the edge points)
    xmin = min(origin[0],endpoint[0])-(128./3.)
    xmax = max(origin[0],endpoint[0])+(128./3.)+res
    ymin = min(origin[1],endpoint[1])-(128./3.)
    ymax = max(origin[1],endpoint[1])+(128./3.)+res
    x_coords = np.arange(xmin,xmax,res)
    y_coords = np.arange(ymin,ymax,res)
    gridx,gridy = np.meshgrid(x_coords,y_coords)

    return gridx.T,gridy.T

def find_nearest(n,v,v0,vn,res):
    "Element in nd array `a` closest to the scalar value `a0`"
    if type(v) is np.ndarray:
        idx = np.int16(np.floor(n*(v-v0+res/2.)/(vn-v0+res)))
    else:
        idx = int(np.floor( n*(v-v0+res/2.)/(vn-v0+res) ))

    return idx

# generate the y indices along a line
def linefunction(velx,vely,indx,indy,xrange):
    m = (indy-vely)/(indx-velx)
    b = vely-m*velx
    return m*xrange + b

def run_ransac(data, estimate, is_inlier, sample_size, goal_inliers, max_iterations, stop_at_goal=True, random_seed=None):
    best_ic = 0
    best_model = None
    random.seed(random_seed)
    for i in range(max_iterations):
        s = data[np.random.choice(data.shape[0], 3, replace=False), :]
        m = estimate(s)
        ic = 0
        for j in range(data.shape[0]):
            if is_inlier(m, data[j,:]):
                ic += 1
        if ic > best_ic:
            best_ic = ic
            best_model = m
            if ic > goal_inliers and stop_at_goal:
                break
    print ('took iterations:', i+1, 'best model:', best_model, 'explains:', best_ic)
    return best_model, best_ic

def augment(xyzs):
	axyz = np.ones((len(xyzs), 4))
	axyz[:, :3] = xyzs
	return axyz

def estimate(xyzs):
	axyz = augment(xyzs[:3])
	return np.linalg.svd(axyz)[-1][-1, :]

def is_inlier(coeffs, xyz, threshold):
	return np.abs(coeffs.dot(augment([xyz]).T)) < threshold

def is_close(a,b,c,d,point,distance=0.1):
    D = (a*point[:,0]+b*point[:,1]+c*point[:,2]+d)/np.sqrt(a**2+b**2+c**2)
    return D


import numpy as np
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func
import datetime as dt
import pandas as pd
from flask import Flask
from flask.json import jsonify

engine = create_engine("sqlite:///Resources/hawaii.sqlite")
conn= engine.connect()

#data= pd.read_sql("SELECT * FROM measurement",conn)
app = Flask(__name__)

Base=automap_base()
Base.prepare(engine,reflect=True)
Base.classes.keys()

measurement=Base.classes.measurement
station=Base.classes.station

@app.route("/")
def home():
    return (f"Available Routes:
"
            f"/api/v1.0/precipitation
"
            f"/api/v1.0/stations
"
            f"/api/v1.0/tobs
"
            f"/api/v1.0/start
"
            f"/api/v1.0/start/end"

    )

@app.route("/api/v1.0/precipitation")
def precipitation():

    session=Session(engine)
    precipitation=session.query(measurement.date,measurement.prcp).all()
    session.close()

    prcp_values=[]
    for date,prcp in precipitation:
        measurement_dict={date:prcp}
        prcp_values.append(measurement_dict)

    return jsonify(prcp_values)

@app.route("/api/v1.0/stations")
def stations():
    session= Session(engine)
    stations= session.query(station.station).all()
    session.close()
    station_names=list(np.ravel(stations))
    return jsonify(station_names)

@app.route("/api/v1.0/tobs")
def tobs():
    session= Session(engine)
    temp_results= session.query(measurement.date,measurement.tobs).filter(measurement.date>'2016-8-23').filter(measurement.station=='USC00519281').order_by(measurement.date).all()
    session.close()
    temps=list(np.ravel(temp_results))
    return jsonify(temps)

@app.route("/api/v1.0/<start>")
def tobs_date(start):
    session=Session(engine)
    results=session.query(measurement.date,func.min(measurement.tobs),func.max(measurement.tobs),func.avg(measurement.tobs))\
        .filter(measurement.date==start).all()

    session.close()

    start_date_temps=[]
    for minimum, maximum, average in results:
        start_temp_dict={}
        start_temp_dict["min"]= minimum
        start_temp_dict["max"]= maximum
        start_temp_dict["avg"]= average
        start_date_temps.append(start_temp_dict)

    return jsonify(start_date_temps)

#@app.route("/api/v1.0/<start>/<end>")
#def tobs_se_date()

if __name__ == "__main__":
    app.run(debug=True)


from django.shortcuts import get_object_or_404

import graphene
import graphene_django

from authors.models import Book, Author, Bio

class BookType(graphene_django.DjangoObjectType):
    class Meta:
        model = Book
        fields = '__all__'

class BookMutation(graphene.Mutation):
    class Arguments:
        id = graphene.Int(required=True)
        name = graphene.String(required=False)

    book = graphene.Field(BookType)

    @classmethod
    def mutate(cls, root, info, id, name=None):
        book = get_object_or_404(Book, pk=id)
        if name:
            book.name = name
            book.save()
        return BookMutation(book=book)

class AuthorType(graphene_django.DjangoObjectType):
    class Meta:
        model = Author
        fields = '__all__'

class BioType(graphene_django.DjangoObjectType):
    class Meta:
        model = Bio
        fields = '__all__'

class Query(graphene.ObjectType):
    hello = graphene.String(name=graphene.String(default_value='stranger'))
    goodbye = graphene.String()

    all_books = graphene.List(BookType)
    book_by_id = graphene.Field(BookType, id=graphene.Int(required=True))
    all_authors = graphene.List(AuthorType)
    author_by_id = graphene.Field(AuthorType, id=graphene.Int(required=True))
    all_bios = graphene.List(BioType)
    bio_by_id = graphene.Field(BioType, id=graphene.Int(required=True))
    bios_by_author_name = graphene.List(BioType, name=graphene.String(required=False))

    def resolve_hello(root, info, name):
        return f'hi {name}!'

    def resolve_goodbye(root, info):
        return 'bye-bye!'

    def resolve_all_books(root, info):
        return Book.objects.all()

    def resolve_book_by_id(root, info, id):
        # return get_object_or_404(Book, pk=id)
        # return Book.objects.get(pk=id)
        try:
            obj = Book.objects.get(pk=id)
        except Book.DoesNotExist:
            return None
        return obj

    def resolve_all_authors(root, info):
        return Author.objects.all()

    def resolve_all_bios(root, info):
        return Bio.objects.all()

    def resolve_bios_by_author_name(root, info, name=None):
        all = Bio.objects.all()
        if name:
            return all.filter(author__first_name=name)
        return all

class Mutation(graphene.ObjectType):
    update_book = BookMutation.Field()

schema = graphene.Schema(query=Query, mutation=Mutation)


from rest_framework.viewsets import ViewSet
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework import status

from ....api.serializer.certificate.certificate_serializer import InsertCertificateSerializer, \
    CertificateImageSerializer, GetCertificateSerializer
from ....logic.certificate.certificate_logic import CertificateLogic
from ....shared.based_response.common_response import CreateResponse, ErrorResponse, UpdateResponse
from drf_spectacular.utils import extend_schema

class CertificateController(ViewSet):

    def __init__(self):
        super().__init__()
        self.certificate_logic = CertificateLogic()

    @extend_schema(
        request=InsertCertificateSerializer,
        tags=['certificate'],
        responses={201: None},
    )
    def post(self, request: Request):
        serializer = InsertCertificateSerializer(data=request.data)
        if serializer.is_valid():

            title = serializer.validated_data.get("title", None)
            description = serializer.validated_data.get("description", None)
            earned_date = serializer.validated_data.get("earned_date", None)
            link = serializer.validated_data.get("link", None)
            try:
                self.certificate_logic.create_certificate(title=title, description=description, earned_date=earned_date,
                                                          link=link, user=request.user)
                return CreateResponse()

            except Exception as e:
                return Response(data=str(e), status=status.HTTP_404_NOT_FOUND)

        else:
            ErrorResponse(message=serializer.errors, status_code=status.HTTP_400_BAD_REQUEST)

    @extend_schema(
        request={
            'multipart/form-data': {
                'type': 'object',
                'properties': {
                    'image': {
                        'type': 'string',
                        'format': 'binary'
                    }
                }
            }},
        tags=['certificate'],
        responses={200: str},
    )
    def patch(self, request: Request):
        serializer = CertificateImageSerializer(data=request.data)
        if serializer.is_valid():
            certificate_image = serializer.validated_data.get("image", None)
            self.certificate_logic.set_certificate_image(certificate_image=certificate_image, user=request.user)

            return UpdateResponse()

        else:
            return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    @extend_schema(
        tags=['certificate'],
        responses={200: GetCertificateSerializer},
    )
    def get(self, request: Request):
        user_certificates = self.certificate_logic.get_certificate_by_user(user=request.user)

        serializer = GetCertificateSerializer(user_certificates, many=True)

        return Response(data=serializer.data, status=status.HTTP_200_OK)


def uredi(seznam):
    najmanjsa=min(seznam)
    index=seznam.index(najmanjsa)
    nov_seznam=[]
    for element in seznam:
        if seznam.index(element)>=index:
            nov_seznam.append(element)
    for element in seznam:
        if seznam.index(element)<index:
            nov_seznam.append(element)
    return nov_seznam

def preberi(ime_datoteke):
    zemljevid=open(ime_datoteke,"r")
    n=1
    d={}
    l=[]
    for vrstica in zemljevid:
        vrstica=vrstica.split("\n")[0]
        vrstica=vrstica.split(" ")
        for stevilka in vrstica:
            if stevilka.isdigit():
                l.append(int(stevilka))
        d[n]=uredi(l)
        n=n+1
        l=[]
    return d

def koncna_povezava(zemljevid):
    n=1
    seznam=[]
    pojavi_enkrat=[]
    while n in zemljevid:
        if len(zemljevid[n])==1:
            seznam.append(n)
        n=n+1
    return seznam

def nima_dvojna_krozisca(pot):
    for element in pot:
        n=0
        c=0
        while n<len(pot)-1:
            if pot[n]==element:
                c=c+1
            if c>=2:
                return False
            n=n+1
    return True

zemljevid=preberi("zemljevid.txt")
def mozna_pot(pot,zemljevid):
    krajisca=koncna_povezava(zemljevid)
    if pot[0] not in krajisca or pot[len(pot)-1] not in krajisca:
        return False
    n=1
    while n<len(pot)-2:
        if pot[n] in krajisca:
            return False
        if pot[n]==pot[n+1]:
            return False
        n=n+1
    n=0
    while n<len(pot)-1:
        if pot[n] not in zemljevid[pot[n+1]]:
            return False
        n=n+1
    return True

def uporabljeno_vsako_krozisce(pot,zemljevid):
    vsa_krozisca=[]
    krajisca=koncna_povezava(zemljevid)
    n=1
    while n in zemljevid:
        vsa_krozisca.append(zemljevid[n])
        n=n+1
    if len(pot)==len(vsa_krozisca)-len(krajisca)+2:
        return True
    else:
        return False

def hamiltonova(pot,zemljevid):
    return mozna_pot(pot,zemljevid) and nima_dvojna_krozisca(pot) and uporabljeno_vsako_krozisce(pot,zemljevid)


"""
Purpose: Advent of Code challenge
Day: 17
Date created: 2020-12-17
URL: https://adventofcode.com

Contributor(s):
    Mark M.
"""

import itertools as ittr
from utils import current_file, day_number, get_lines, read_data

# If testing, set DEBUG to True for a smaller data set.
DEBUG: bool = True

# Import data
if DEBUG:
    raw_data = """.#.
..#
###"""

else:
    # Current file filepath
    thisfile = current_file(__file__)

    # AOC day number
    DAY_NO: int = day_number(thisfile.stem)

    raw_data = read_data(f"day-{DAY_NO}-input.txt")

# raw_data = read_data(f"day-17-input.txt")

# Split data into lines for further processing.  Skip any missing or blank lines.
try:
    data = list(map(int, get_lines(raw_data)))
except ValueError:
    pass
finally:
    data = get_lines(raw_data)

grid = [list(i) for i in data]

# -- objects and functions --- #

def create_cubes(matrix):
    cube_set = set()
    for r in range(len(matrix)):
        for c in range(len(matrix[0])):
            if matrix[r][c] == "#":
                cube_set.add((c, r, 0, 0))
    return cube_set

####################################
######### --- Part 1 --- ###########
####################################

"""
If a cube is active and exactly 2 or 3 of its neighbors are also active, the cube
remains active. Otherwise, the cube becomes inactive.

If a cube is inactive but exactly 3 of its neighbors are active, the cube becomes active.
Otherwise, the cube remains inactive.
"""

# gen_positions = ittr.product(range(-1, 2), repeat=3)
# positions =  tuple((pos[0], pos[1], pos[2], 0) for pos in gen_positions if pos != (0, 0, 0))
# n_pos = len(positions)

# tf_data = [[False] * len(data) for x in range(len(data))]
# for r in range(len(data)):
#     for c in range(len(data[r])):
#         if data[r][c] == "#":
#             tf_data[r][c] = True

# directions = []
# neighbor_rng = range(-1, 2)
# for x in neighbor_rng:
#     for y in neighbor_rng:
#         for z in neighbor_rng:
#             directions.append(Cube(x, y, z))
#             print(x, y, z)

def main(matrix, N=3):

    gen_positions = ittr.product(range(-1, 2), repeat=N)
    if N == 3:
        positions =  tuple((pos[0], pos[1], pos[2], 0) for pos in gen_positions if pos != (0, 0, 0))
    elif N == 4:
        positions =  tuple(pos for pos in gen_positions if pos != (0, 0, 0, 0))

    cubes = create_cubes(matrix)

    for _ in range(6):

        next_cubes = set()
        neighbors = set()

        for cube in cubes:
            counter = 0
            for p in positions:
                pos = tuple((cube[i] + p[i] for i in range(4)))
                if pos in cubes:
                    counter += 1
                else:
                    neighbors.add(pos)
            if counter >= 2 and counter <= 3:
                next_cubes.add(cube)

        for cube in neighbors:
            counter = 0
            for p in positions:
                pos = tuple(cube[i] + p[i] for i in range(4))
                if pos in cubes:
                    counter += 1
            if counter == 3:
                next_cubes.add(cube)

        cubes = next_cubes

    return len(cubes)

result = main(grid, 3)
print(f"Part 1 - Number of remaining cubes: {result}")

####################################
######### --- Part 2 --- ###########
####################################

# Switch the main function up to handle 4 dimensions, now.
result = main(grid, 4)
print(f"Part 1 - Number of remaining cubes: {result}")


#Sirve para ver el la posición del mouse en tiempo real

import pyautogui
import time
while True:
    x, y = pyautogui.position()
    color = pyautogui.screenshot().getpixel((x, y))
    print(f"Posición del mouse: ({x}, {y})")
    print(f"Valor RGB en la posición: {color}")
    time.sleep(1)


import dateutil.parser
import json
import time
import unittest

def mysql_mapper(e):
    """
    The WMF EventLogging Analytics MySQL log database has a lot of curious
    legacy compatibility problems.  This function converts an event
    to a format that the MySQL database expects.  If an event comes from
    a non-Mediawiki bot, it will be mapped to 'None' and thus excluded from the stream.
    """
    if 'userAgent' in e and isinstance(e['userAgent'], dict):
        # Get rid of unwanted bots. T67508
        is_bot = e['userAgent'].get('is_bot', False)
        is_mediawiki = e['userAgent'].get('is_mediawiki', False)
        # Don't insert events generated by bots unless they are mediawiki bots.
        if is_bot and not is_mediawiki:
            # Returning None will cause map://
            # reader to exclude this event.
            return None

        # MySQL expects that userAgent is a string, so we
        # convert it to JSON string now.  T153207
        e['userAgent'] = json.dumps(e['userAgent'])

    # jrm.py expects an integer `timestamp` field to convert into
    #  Mediawiki timestamp. Inject it into the event.
    if 'dt' in e:
        # Use the time from `dt`
        e['timestamp'] = int(dateutil.parser.parse(e['dt']).strftime("%s"))
        # Historicaly, EventCapsule did not have `dt` so we remove it from
        # insertion into MySQL.
        del e['dt']
    else:
        # Else just use current time.
        e['timestamp'] = int(time.time())

    return e

# ##### Tests ######
# To run:
#   python -m unittest -v plugins.py
# Or:
#   python plugins.py
#
class TestEventLoggingPlugins(unittest.TestCase):

    def test_mysql_mapper(self):
        e1 = {
            'dt': '2017-11-01T11:00:00',
            'userAgent': {'browser_family': 'Chrome'}
        }
        should_be1 = {'timestamp': 1509548400, 'userAgent': '{"browser_family": "Chrome"}'}
        self.assertEqual(mysql_mapper(e1), should_be1)

        e2 = {
            'dt': '2017-11-01T11:00:00',
            'userAgent': {'is_bot': True}
        }
        self.assertEqual(mysql_mapper(e2), None)

        e3 = {
            'dt': '2017-11-01T11:00:00',
            'userAgent': {'is_bot': True, 'is_mediawiki': True}
        }
        should_be3 = {'timestamp': 1509548400, 'userAgent': json.dumps(e3['userAgent'])}
        self.assertEqual(mysql_mapper(e3), should_be3)

if __name__ == '__main__':
    unittest.main(verbosity=2)


#external
import numpy as np
import pytest
import os
import filecmp

#local
import alphasynchro.ms.peaks.precursors
import alphasynchro.ms.peaks.fragments
import alphasynchro.ms.transitions.frame_transitions
import alphasynchro.io.writing.hdf
import alphasynchro.io.writing.mgf
import alphasynchro.io.writing.prolucid

@pytest.fixture(scope="module")
def precursors():
    test_peaks = alphasynchro.ms.peaks.precursors.Precursors.from_clusters_hdf(
        "./unit_tests/test_clusters.hdf",
    )
    return test_peaks

@pytest.fixture(scope="module")
def fragments():
    test_peaks = alphasynchro.ms.peaks.fragments.Fragments.from_clusters_hdf(
        "./unit_tests/test_clusters.hdf",
    )
    return test_peaks

@pytest.fixture(scope="module")
def transitions():
    test_transitions = alphasynchro.ms.transitions.frame_transitions.Transitions(
        indptr=np.array([0, 2]),
        values=np.array([0, 1]),
        weights=np.array([.5, .7]),
        precursor_indices=np.arange(1),
    )
    return test_transitions

def test_prolucid(precursors, fragments, transitions):
    file_name = "./unit_tests/sandbox_output.prolucid"
    writer = alphasynchro.io.writing.prolucid.Writer(
        file_name=file_name,
        precursors=precursors,
        fragments=fragments,
        transitions=transitions,
    )
    writer.write_to_file()
    expected = read_file_lines("./unit_tests/test_output.prolucid", skip_lines=1)
    output = read_file_lines(file_name, skip_lines=1)
    assert str(output) == str(expected)
    os.remove(file_name)

def test_mgf(precursors, fragments, transitions):
    file_name = "./unit_tests/sandbox_output.mgf"
    writer = alphasynchro.io.writing.mgf.Writer(
        file_name=file_name,
        precursors=precursors,
        fragments=fragments,
        transitions=transitions,
    )
    writer.write_to_file()
    assert filecmp.cmp(file_name, "./unit_tests/test_output.mgf")
    os.remove(file_name)

def test_hdf(precursors, fragments, transitions):
    file_name = "./unit_tests/sandbox_output.hdf"
    writer = alphasynchro.io.writing.hdf.Writer(
        file_name=file_name,
        precursors=precursors,
        fragments=fragments,
        transitions=transitions,
    )
    writer.write_to_file()
    assert filecmp.cmp(file_name, "./unit_tests/test_output.hdf")
    os.remove(file_name)

def read_file_lines(file_name, skip_lines=0):
    with open(file_name) as raw_file:
        lines = raw_file.readlines()
    return lines[skip_lines:]


"""
Library of functions to process images and write out the image files.

Created by DMK on 10/14/2023

Last updated by DMK on 10/14/2023
"""

##### Imports
import numpy as np

from astropy.io import fits
from astroscrappy import detect_cosmics
from datetime import datetime

import os

##### Functions

##### Main wrapper function

def build_images( file_indices, super_bias, flat_field, bad_pixel_mask, header_df, config ):

    for i_file in file_indices:

        # The raw file name to read in
        file_name = header_df['file_name'].values[i_file]

        input_file = fits.open( file_name )

        # Assign the frame values, that can then be reassigned if cosmic subtraction is performed
        frame_values = input_file[0].data

        # Perform cosmic subtraction if flag is turned on in the config, and also only for object frames (not ThAr!)
        if config['image_process']['cosmic_subtract'] and input_file[0].header['imagetyp'] == 'object':

            cosmics_mask, frame_values = detect_cosmics( input_file[0].data, sigclip = 5.0, sigfrac = 0.3, objlim = 5.0, niter = config['image_process']['cosmic_subtract_niter'], sepmed = True, satlevel = np.inf,
                                                        cleantype = 'medmask', gain = header_df['gain'].values[i_file], readnoise = header_df['read_noise'].values[i_file] )

        # Calculate flux errors (in ADU)
        frame_errors = np.sqrt( frame_values + ( header_df['read_noise'].values[i_file] / header_df['gain'].values[i_file] ) ** 2.0 )

        # Now do basic image processing -- bias subtraction and flat fielding

        frame_values = frame_values - super_bias['bias flux'].data
        frame_values = frame_values / flat_field['flat flux'].data

        # Error
        frame_errors = np.sqrt( frame_errors ** 2.0 + super_bias['bias error'].data ** 2.0 + ( frame_values * flat_field['flat error'].data ) ** 2.0 ) / flat_field['flat flux'].data

        # Bad pixel mask
        bpm = np.where( bad_pixel_mask[0].data == 0 )

        # Set places where there is a bad pixel to nan. (Previous pipeline replaced these with values of very low S/N, but nans are now handled)
        frame_values[bpm] = np.nan
        frame_errors[bpm] = np.nan

        ### Set up the output image

        frame_value_hdu = fits.PrimaryHDU( frame_values, header = input_file[0].header )
        frame_error_hdu = fits.ImageHDU( frame_errors, name = 'error' )

        output_file = fits.HDUList( [ frame_value_hdu, frame_error_hdu ] )

        # Additions to the header

        output_file[0].header['FILENAME'] = 'tullcoude_{}.fits'.format( header_df['file_token'].values[i_file] )

        output_file[0].header['HISTORY'] = 'Image processed on {}'.format( datetime.strftime( datetime.now(), '%Y/%m/%d' ) )
        output_file[0].header['HISTORY'] = 'Bias subtracted and flat fielded'

        if config['image_process']['cosmic_subtract'] and input_file[0].header['imagetyp'] == 'object':
            output_file[0].header['HISTORY'] = 'Cosmic ray subtracted'

        output_file.writeto( os.path.join( config['general']['reduction_dir'], 'object_files', 'tullcoude_{}.fits'.format( header_df['file_token'].values[i_file] ) ), overwrite = True )

    return None