#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar  3 23:12:47 2021

@author: tim
Simulation algorthims, with parallel edits for 2+ processor case
"""
# from mpi4py import MPI
import numpy as np
import itertools as it
from comparison_fns import compare, inter_bdry, dist, which_seg, mt, mt_to_l
from parameters import v_s, L, xdomain, ydomain, tip_scaled
from event_fns import valid_events, check_region
from zippering import zip_cat
import random as rnd
from time import time
from plotting import plot_snap

def nuc_t(t):
    '''
    Next nucleation time using rate k, from Gillespie

    Parameters
    ----------
    t : Current time
    Returns
    -------
    Next nucleation time and nucleation point

    '''
    lx = xdomain[1]-xdomain[0]
    ly = ydomain[1]-ydomain[0]
    k = lx*ly*L**3*0.1/8#nucleation rate
    # k=1
    r, x, y = rnd.uniform(0,1), rnd.uniform(xdomain[0],xdomain[1]), rnd.uniform(ydomain[0],ydomain[1])
    p = [x,y] #point
    dt = np.log(1/r)/k #new time
    T = t+dt
    return(T, p)

def update_event_list(rank, MT_list,event_list,t,pevent_list,del_mt,last_result,policy,cross_data, rewind):
    '''
    Parameters
    ----------
    MT_list : List of MT objects
    pair_list : List of tuples of pairwise MT indices. Each tuple represents MTs to be compared
    t: current time
    pevet_list: list of previous events which occured
    del_mt:
    cross_data: for when MT crosses into processor
    rewind: if true, need to recalculate event_list from existing MTs

    Returns
    -------
    Collision policy
    BE CAREFUL: THE DISTANCE RETURNED IN BDRY COLLISION IS DISTANCE OF PRESENT TIP TO BDRY, NOT
    SEGMENT LENGTH!!!
    '''
    r = tip_scaled*0.5 #radius
    mt_index = [mt.number for mt in MT_list if mt.exist] #indices of active mts
    n = len(mt_index)
    l = None
    pair_list = None
    if t==0 or rewind: #calculate all possible pairs
        if rewind and t!= 0:
            event_list[:] = [x for x in event_list if x[2]=='nucleate'] #find all nucleation events
        pair_list = list(it.combinations(mt_index,2)) #list of combinations of indices of MTs
        l = len(pair_list)

        for j in range(n): #check for bdry collisions
            MT = MT_list[j]
            if MT.grow == True: #no bdry collision
                bdry_res = inter_bdry(rank,MT, MT_list) #find intersection info
                next_time = bdry_res[0] + MT.update_t[-1] #time of collision
                event_list.append([False,MT.number,bdry_res[2],next_time,bdry_res[1]])
                #regions
                event_list.append([False,MT.number,'new_region',t+r,None])#new when there's a new region
            elif MT.grow ==False and MT.hit_bdry==False: #disappearing events need to be recalculated
                mt_l = np.sum(MT.seg_dist)#total length of shrinking mt, cannot got lower than this
                disap_t = mt_l/v_s+MT.update_t[-1] #collision distances from 1 to 2, growing can only collide w/ shrinking
                event_list.append([False,MT.number,'disap',disap_t,None]) #disappearing event
        if t== 0: #only time we need to new nucleation
            th = rnd.uniform(0,2*np.pi)
            nucleate  = nuc_t(t) #nucleation event
            event_list.append([False,None,'nucleate',nucleate[0],[nucleate[1],th]])
        if policy == 'comm': #if there's been a communication REDUNDANT, ONLY IN CASE REWIND GOES TO 0
            new_policy = cross_data[0] #did an MT shrink or appear?
            mt_cross = cross_data[1] #MT of interest
            if new_policy =='disap': #shrank
                event_list.append([False,None,'shrink_in',mt_cross.disap_t,None]) #shrinking in the future
            else:
                event_list.append([False,None,'cross_in',mt_cross.update_t[-1],None]) #crossing in
    else:
        if policy == 'comm': #if there's been a communication
            new_policy = cross_data[0] #did an MT shrink or appear?
            mt_cross = cross_data[1] #MT of interest
            if new_policy =='disap': #shrank
                event_list.append([False,None,'shrink_in',mt_cross.disap_t,None]) #shrinking in the future
            else:
                event_list.append([False,None,'cross_in',mt_cross.update_t[-1],None]) #crossing in
        else:
            l_idx = mt_to_l(MT_list, last_result)
            if policy =='cross_in': #crossing in from left or right
                event_list.pop(0)
                new_idx = [last_result]
                mt_index= [p for p in mt_index if ((p != last_result) \
                and check_region(last_result,p,MT_list,t,r))] #for comparing new MT
                pair_list = list(it.product(mt_index,new_idx)) #generate new pairs
                l = len(pair_list)
                MT = MT_list[l_idx]
                bdry_res = inter_bdry(rank,MT, MT_list) #find intersection info
                next_time = bdry_res[0] + MT.update_t[-1] #time of collision
                event_list.append([False,last_result,bdry_res[2],next_time,bdry_res[1]])
                #Find region event
                event_list.append([False,last_result,'new_region',t+r,None])
            elif policy in ['left','right', 'top', 'bottom']: #these mts have entered a new processor
                '''PARALLEL EDIT'''
                new_idx = [last_result]
                mt_index= [p for p in mt_index if ((p != last_result) \
                and check_region(last_result,p,MT_list,t,r))] #for comparing new MT
                pair_list = list(it.product(mt_index,new_idx)) #generate new pairs
                l = len(pair_list)
                event_list[:] = [x for x in event_list if valid_events(x,last_result)] #discard all invalid events
                '''PARALLEL EDIT'''
            elif policy == 'deflect': #deflection
                new_idx = [last_result] #index of updated MTs
                mt_index= [p for p in mt_index if p != last_result and check_region(last_result,p,MT_list,t,r)] #indices of new pairs to check over
                n = len(mt_index)
                pair_list = list(it.product(mt_index,new_idx)) #generate new pairs
                l = len(pair_list)
                assert(last_result == event_list[0][1])
                assert(valid_events(event_list[0],last_result)==False)
                event_list[:] = [x for x in event_list if valid_events(x,last_result)] #discard all invalid events
                MT = MT_list[l_idx]
                #bdry events
                bdry_res = inter_bdry(rank,MT,MT_list) #find intersection info
                next_time = bdry_res[0] + MT.update_t[-1] #time of collision
                event_list.append([False,new_idx[0],bdry_res[2],next_time,bdry_res[1]])
                #Find region event
                event_list.append([False,last_result,'new_region',t+r,None])
            elif policy == 'disap': #if last one disappeared
                if MT_list[l_idx].from_bdry and MT_list[l_idx].origin == rank: #and it's an extension from the same rank
                    prev_idx = MT_list[l_idx].prev_mt #index of newly shrinking MT
                    event_list[:] = [x for x in event_list if valid_events(x,prev_idx)] #discard newly shrinking one
                    event_list[:] = [x for x in event_list if valid_events(x,last_result)] #discard all invalid events
                    new_idx = [prev_idx] #new list of indices
                    mt_index= [p for p in mt_index if p != prev_idx and check_region(last_result,p,MT_list,t,r)]
                    pair_list = list(it.product(mt_index,new_idx)) #new pairs
                    l = len(pair_list)
                    assert MT_list[l_idx].exist==False
                    #bdry events
                    l_idxp = mt_to_l(MT_list, prev_idx) #find microtuble
                    mt_l = np.sum(MT_list[l_idxp].seg_dist)#total length of shrinking mt, cannot got lower than this
                    disap_t = mt_l/v_s+t #collision distances from 1 to 2, growing can only collide w/ shrinking
                    event_list.append([False,prev_idx,'disap',disap_t,None])

                    del_mt.append(MT_list[l_idx]) #add deleted mt to list
                    MT_list.pop(l_idx) #remove from existent mt list
                else: #if last one disappeared, no continuation
                    event_list[:] = [x for x in event_list if valid_events(x,last_result)] #discard all invalid events
                    l = 0 #no new comparisons to be made with non-existant MT
                    assert MT_list[l_idx].exist==False
                    del_mt.append(MT_list[l_idx])
                    MT_list.pop(l_idx)
            elif policy == 'shrink_in': #shrinking case
                event_list.pop(0)
                event_list[:] = [x for x in event_list if valid_events(x,last_result)] #discard all invalid events
                new_idx = [last_result] #new list of indices
                mt_index= [p for p in mt_index if p != last_result and check_region(last_result,p,MT_list,t,r)]
                pair_list = list(it.product(mt_index,new_idx)) #new pairs
                l = len(pair_list)
                #bdry events
                mt_l = np.sum(MT_list[l_idx].seg_dist)#total length of shrinking mt, cannot got lower than this
                disap_t = mt_l/v_s+t #collision distances from 1 to 2, growing can only collide w/ shrinking
                event_list.append([False,last_result,'disap',disap_t,None])
            elif policy in ['1hit2','2hit1']:
                if MT_list[l_idx].grow is False: #if updated MT is due to cat. and it didn't disap
                    new_idx = [last_result] #index of updated MTs
                    mt_index= [p for p in mt_index if p != last_result] #indices of new pairs to check over
                    n = len(mt_index)
                    pair_list = list(it.product(mt_index,new_idx)) #generate new pairs
                    l = len(pair_list)
                    event_list[:] = [x for x in event_list if valid_events(x,last_result)] #discard all invalid events

                    mt_l = np.sum(MT_list[l_idx].seg_dist)#total length of shrinking mt, cannot got lower than this
                    disap_t = mt_l/v_s+t #collision distances from 1 to 2, growing can only collide w/ shrinking
                    event_list.append([False,last_result,'disap',disap_t,None]) #disappearing event
                else: #if two dynamics MTs collide, must be growing
                    new_idx = [last_result] #index of updated MTs
                    mt_index= [p for p in mt_index if p != last_result and check_region(last_result,p,MT_list,t,r)] #indices of new pairs to check over
                    n = len(mt_index)
                    pair_list = list(it.product(mt_index,new_idx)) #generate new pairs
                    l = len(pair_list)
                    event_list[:] = [x for x in event_list if valid_events(x,last_result)] #discard all invalid events

                    l_idx = mt_to_l(MT_list, new_idx[0])
                    MT = MT_list[l_idx]
                    #bdry event
                    bdry_res = inter_bdry(rank,MT,MT_list) #find intersection info
                    next_time = bdry_res[0] + MT.update_t[-1] #time of collision
                    event_list.append([False,new_idx[0],bdry_res[2],next_time,bdry_res[1]])
                    #Find region event
                    event_list.append([False,last_result,'new_region',t+r,None])
            elif policy == 'nucleate':#nucleation occurs
                event_list.pop(0)
                new_idx = [last_result] #index of updated MTs
                mt_index= [p for p in mt_index if check_region(last_result,p,MT_list,t,r)] #indices of new pairs to check over
                n = len(mt_index)
                pair_list = list(it.product(mt_index,new_idx)) #generate new pairs
                l = len(pair_list)
                #bdry events
                MT = MT_list[l_idx]
                bdry_res = inter_bdry(rank,MT, MT_list) #find intersection info
                next_time = bdry_res[0] + MT.update_t[-1] #time of collision
                event_list.append([False,new_idx[0],bdry_res[2],next_time,bdry_res[1]])
                #new nucleation event
                nuc_list = [x for x in event_list if x[2]=='nucleate']#check current nucleation queue
                if len(nuc_list) == 0: #if there are no mpre, calculate
                    th = rnd.uniform(0,2*np.pi)
                    nucleate  = nuc_t(t) #nucleation event
                    event_list.append([False,None,'nucleate',nucleate[0],[nucleate[1],th]])
                #Find region event
                event_list.append([False,last_result,'new_region',t+r,None])
            elif policy == 'new_region': #need to check new region
                new_idx = [last_result] #index of updated MTs
                mt_index= [p for p in mt_index if p!= last_result and check_region(last_result,p,MT_list,t,r)] #indices of new pairs to check over
                n = len(mt_index)
                pair_list = list(it.product(mt_index,new_idx)) #generate new pairs
                l = len(pair_list)
                event_list[:] = [x for x in event_list if valid_events(x,last_result)] #discard all invalid events
                #bdry events
                MT = MT_list[l_idx]
                bdry_res = inter_bdry(0,MT, MT_list) #find intersection info
                next_time = bdry_res[0] + MT.update_t[-1] #time of collision
                event_list.append([False,new_idx[0],bdry_res[2],next_time,bdry_res[1]])
                #Find region event
                event_list.append([False,last_result,'new_region',t+r,None])
            for i in range(l): #compare all pairs
                mt1_i = pair_list[i][0] #numbers of mts to compare
                mt2_i = pair_list[i][1]
                l_mt = mt_to_l(MT_list,mt1_i,mt2_i) #convert to list index
                l_mt1, l_mt2 = l_mt[0],l_mt[1]
                result = compare(MT_list[l_mt1], MT_list[l_mt2],t) #retrieve output object
                if result.policy not in ['no_collision']: #if there is collision
                    assert result.dist >=0 and MT_list[l_mt1].exist and MT_list[l_mt2].exist
                    next_time = result.dist+t
                    event_list.append([True,pair_list[i],result.policy,next_time,result.point]) #apend event details
    event_list.sort(key=lambda x: x[3]) #sort according to time
    pevent_list.append(event_list[0])#record event occurance
    return(event_list[0])

def update(rank,N,mt_list,collided,idxs,policy,dists,pt,t,cross_data):
    '''
    Updates MT based on next event
    Parameters
    ----------
    rank: rank of process
    N: number of current+past MTs in the system
    mt_list : list of MTs
    collided : bool of whether MTs collided (True) or it was a bdry event
    policy : what type of collision event
    dists : event time
    pt : point of intersection
    idx : tuple of idices if two MTs collide, single index if bdry event
    t : current time
    cross_data: for when an MT crosses into the processor

    Returns
    -------
    Index of updated MT
    Whether event is a crossing

    '''
    crossing = False #whether there has been a crossing
    if policy =='1hit2' or policy=='2hit1':
        if policy == '1hit2':
            mt_idx1,mt_idx2 = idxs[0],idxs[1] #index of MT to be updated
        else:
            mt_idx1,mt_idx2 = idxs[1],idxs[0]

        l_idx = mt_to_l(mt_list, mt_idx1,mt_idx2) #convert to list indices
        l_idx1, l_idx2 = l_idx[0],l_idx[1]

        mt1 =mt_list[l_idx1]
        mt2 =mt_list[l_idx2]

        seg_idx = which_seg(mt1,mt2,t) #find segment of intersection
        r = rnd.randint(0, 1)
        zip_res = zip_cat(mt1.angle[-1],mt2.angle[seg_idx],pt,r) #zipper or not
        new_pt,new_angle = zip_res[1], zip_res[0]
        mt1.seg.append(new_pt) #update point
        mt1.update_t.append(dists) #update time
        actual_dist = dist(mt1.seg[-2],mt1.seg[-1])
        mt1.seg_dist.append(actual_dist) #append seg distance
        if not zip_res[2]: #no catastrophe
            mt1.angle.append(new_angle) #TODO: CHANGE ANGLE UPDATE
            mt1.region_t.append(dists) #region change
        else: #catastrophe
            mt1.grow = False
        return(mt_idx1,crossing)
    # if collided is False: #if collision w/ bdry or deflection
    elif policy in ['disap']: #one has disappeared
        mt_idx1 = idxs
        l_idx1 = mt_to_l(mt_list, mt_idx1) #need to convert to indices in the list of existing mts
        assert l_idx1 >= 0
        mt1 =mt_list[l_idx1]
        mt1.exist = False #has disappeared
        mt1.disap_t = dists
        mt1.update_t.append(dists)
        '''PARALLEL EDIT START'''
        if mt1.origin != rank: #it's a continuation of another processor's mt
            assert mt1.from_bdry
            crossing = True #will need to update with crossing_update
            '''PARALLEL EDIT END'''
        elif mt1.from_bdry: #if it's an extension, must update previous MT
            mt_prev_idx = mt1.prev_mt
            l_prev_idx = mt_to_l(mt_list, mt_prev_idx) #convert to list indices

            mt_list[l_prev_idx].grow = False #it must shrink now
            mt_list[l_prev_idx].hit_bdry = False #no longer hitting bdry
            mt_list[l_prev_idx].update_t.append(dists) #update time of new bdry point now shrinking

            # print('previous index', prev_idx)
        return(mt_idx1,crossing)
    elif policy == 'shrink_in':
        mt1 = cross_data[1]
        l_prev_idx  = mt_to_l(mt_list,mt1.prev_mt)
        mt_list[l_prev_idx].grow = False #it must shrink now
        mt_list[l_prev_idx].hit_bdry = False #no longer hitting bdry
        mt_list[l_prev_idx].update_t.append(dists) #update time of new bdry point now shrinking
        return(mt1.prev_mt,crossing)
    elif policy == 'nucleate':
        mt_idx = N
        mt_list.append(mt(mt_idx,rank)) #introduce new MT
        mt1 = mt_list[-1]
        mt1.update_t[-1] = dists
        mt1.region_t.append(dists) #region change
        mt1.seg = [pt[0]] #assign points and angles
        mt1.angle = [pt[1]]
        return(mt_idx,crossing)
    elif policy == 'cross_in':
        mt1 = cross_data[1] #corresponding microtuble
        mt1.number = N #give a number
        mt_list.append(mt1)
        return(N, crossing)
    elif policy =='deflect':
        l_idx = mt_to_l(mt_list,idxs)#convert to list index

        mt1 = mt_list[l_idx] #mt to be updated
        assert l_idx>=0
        mt1.seg.append(pt) #update point
        power = rnd.randint(0,1) #to be used to determine deflection +/-
        dth  =np.pi/16 #change in angle
        th_new = mt1.angle[-1] + (-1)**power*dth #deflection
        if th_new> 2*np.pi:
            th_new -= 2*np.pi
        elif th_new <0:
            th_new += 2*np.pi
        mt1.angle.append(th_new)

        segment_dist = dist(mt1.seg[-2],mt1.seg[-1])
        mt1.seg_dist.append(segment_dist) #segment length

        mt1.update_t.append(dists) #update time
        mt1.region_t.append(dists) #region change
        return(mt1.number,crossing)
    elif policy =='new_region': #new region, no MT change
        l_idx = mt_to_l(mt_list,idxs)
        mt_list[l_idx].region_t.append(dists) #new region calculated at this time
        return(idxs,crossing)
    else: #hit bdry, nucleates new MT
        assert policy in ['top','bottom','left','right']
        l_idx = mt_to_l(mt_list, idxs) #convert to list index
        mt1 = mt_list[l_idx] #mt to be updated
        mt1.seg.append(pt) #update point
        th_og = mt1.angle[-1]
        mt1.angle.append(th_og)
        mt1.update_t.append(dists) #update time
        mt1.grow = False #no longer considered growing
        mt1.hit_bdry = True #hit bdry
        mt_idx = None
        segment_dist = dist(mt1.seg[-2],mt1.seg[-1])
        mt1.seg_dist.append(segment_dist) #segment length
        assert mt1.checkd()
        mt_idx = idxs #original MT
        crossing = True
        '''PARALLEL EDIT END'''
        return(mt_idx,crossing)

def undo_update(rank, mt_list, pevent_list, event_list, del_mt):
    '''
    Undoes updates
    Parameters
    ----------
    mt_list : list of MTs
    collided : bool of whether MTs collided (True) or it was a bdry event
    policy : what type of collision event
    dists : event time
    pt : point of intersection
    idx : tuple of idices if two MTs collide, single index if bdry event
    t : current time

    Returns
    -------
    Time of rewinded step

    '''
    event = pevent_list[-1] #previous event
    idxs, policy = event[1], event[2] #event details

    # if collided is True: #if dynamic ends collided
    assert policy not in ['shrink_in','comm','cross_in']
    if policy =='1hit2' or policy=='2hit1':
        if policy == '1hit2':
            mt_idx1,mt_idx2 = idxs[0],idxs[1] #index of MT to be updated
        else:
            mt_idx1,mt_idx2 = idxs[1],idxs[0]
        l_idx = mt_to_l(mt_list, mt_idx1,mt_idx2) #convert to list indices
        l_idx1, l_idx2 = l_idx[0],l_idx[1]
        assert l_idx1>=0 and l_idx2>=0
        mt1 =mt_list[l_idx1]

        mt1.seg.pop(-1) #un-update point
        mt1.update_t.pop(-1) #un-update time
        mt1.seg_dist.pop(-1) #append seg distance
        if mt1.grow: #no catastrophe
            mt1.angle.pop(-1) #undo angle change
            mt1.region_t.pop(-1)
        else: #catastrophe
            mt1.grow = True #undo cat
    elif policy =='disap': #one has disappeared
        mt_idx1 = idxs
        l_idx1 = mt_to_l(mt_list, mt_idx1) #need to convert to indices in the list of existing mts
        ldel_idx = mt_to_l(del_mt, mt_idx1) #might not exist, check non-existing mts
        mt1 = None
        if l_idx1 >=0: #if in this list
            mt1 =mt_list[l_idx1]
        else: #if in deleted mts
            assert ldel_idx >= 0
            mt1 = del_mt[ldel_idx]
            del_mt.pop(ldel_idx) #delete from deleted mt list
            mt_list.append(mt1) #re-introduce to mtlist
        assert mt1.number == idxs
        mt1.exist = True #has un-dissapeared
        mt1.disap_t = None #undo this
        mt1.update_t.pop(-1)
        if mt1.from_bdry and mt1.origin==rank: #if it's an extension, must un-update previous MT
            mt_prev_idx = mt1.prev_mt
            l_prev_idx = mt_to_l(mt_list, mt_prev_idx) #convert to list indices
            mt_list[l_prev_idx].grow = False #it must shrink now
            mt_list[l_prev_idx].hit_bdry = True #hitting bdry
            mt_list[l_prev_idx].update_t.pop(-1) #un-update time of new bdry point
    elif policy == 'nucleate':
        t = event[3] #in a nucleation event, we do not know the MT index ahead of time
        mt_list2 = [mt for mt in mt_list if mt.update_t[-1]==t] #must find the MT this way
        if len(mt_list2) == 0:
            print([mt.number for mt in mt_list])
            print('CANNOT FIND MT IN LIST', rank,t)
            print(event)
            assert False
        mt_idx = mt_list2[0].number #new mt number
        l_idx = mt_to_l(mt_list, mt_idx) #index in list
        assert l_idx >= 0
        mt_list.pop(l_idx) #un-introduce new MT

        event_list.append(event) #reinstate nucleation event
    elif policy =='new_region': #new region, no MT change
        l_idx = mt_to_l(mt_list,idxs)
        mt_list[l_idx].region_t.pop(-1) #new region calculated at this time
    elif policy =='deflect':
        l_idx = mt_to_l(mt_list,idxs)#convert to list index
        mt1 = mt_list[l_idx] #mt to be updated
        mt1.seg.pop(-1) #un-update point
        mt1.angle.pop(-1)
        mt1.seg_dist.pop(-1) #segment length
        mt1.update_t.pop(-1) #un-update time
        mt1.region_t.pop(-1)
    else: #hit bdry, nucleates new MT
        l_idx = mt_to_l(mt_list, idxs) #convert to list index
        mt1 = mt_list[l_idx] #mt to be updated
        mt1.seg.pop(-1) #update point
        mt1.angle.pop(-1)
        mt1.update_t.pop(-1) #un-update time
        mt1.grow = True #un-no longer considered growing
        mt1.hit_bdry = False #un-hit bdry
        mt1.seg_dist.pop(-1) #segment length
    pevent_list.pop(-1) #un-updated the latest event
    if not pevent_list:
        t_new = 0
    else:
        t_new = pevent_list[-1][3]
    return(t_new)

def exit_data(rank, policy, idx, mt_list,t):
    '''
    Given that MT w/ number idx in rank is crossing outside, prepare data to be
    sent to other process

    Parameters
    ----------
    t: current time
    rank : rank of process
    policy: left/right or disap
    idx : ID of MT in this rank
    mt_list: list of MTs

    Returns
    -------
    Data

    '''
    data = [] #data to be sent
    if policy == 'disap':
        data.append(policy)
        l_idx = mt_to_l(mt_list, idx) #convert to list index
        mt1 = mt_list[l_idx] #mt that disappered
        data.append(mt1) #index in the other processor
    else:
        assert policy in['right','left', 'top', 'bottom']
        if policy=='right': #new event in other process
            data.append('new_left')
        elif policy=='left':
            data.append('new_right')
        elif policy=='top':
            data.append('new_bottom')
        else:
            assert policy=='bottom'
            data.append('new_top')
        l_idx = mt_to_l(mt_list, idx) #convert to list index
        mt1 = mt_list[l_idx] #mt to be updated
        pt = mt1.seg[-1][:] #previous point
        # if rank == 0: #if rank 0 (left), find identified edge
        if policy == 'right':
            pt[0] = xdomain[0]
        elif policy == 'left':
            pt[0] = xdomain[1]
        elif policy == 'top':
            pt[1] = ydomain[0]
        else:
            assert policy == 'bottom'
            pt[1] = ydomain[1]
        th = mt1.angle[-1] #angle
        mt2 = mt(None,rank) #continuation mt
        mt2.angle = [th]
        mt2.seg = [pt]
        mt2.from_bdry = True
        mt2.prev_mt = idx
        mt2.update_t[-1] = t#give update time
        mt2.region_t = [t]
        if len(mt1.seg) > 2: #it changed trajectory before hitting bdry
            mt2.prev_t = mt1.update_t[-2]
        else:
            assert len(mt1.seg)==2
            if mt1.origin == rank: #can search native list for other info
                if mt1.from_bdry:
                    l_idx2 = mt_to_l(mt_list,mt1.prev_mt)#may need to consider ext of ext
                    mtp2 = mt_list[l_idx2]
                    assert len(mtp2.update_t) >= 2 #implicit assumption here that previous mt will be more segs
                        # plot_snap([mt1,mtp2],t,i,'./plots2/',False)
                    mt2.prev_t = mtp2.update_t[-2] #time before it hit wall
                else:
                    mt2.prev_t = mt1.update_t[0]
            else:
                mt2.prev_t = mt1.prev_t #last physical update same as parent MT
        data.append(mt2)
    return(data)

#testing below
if __name__ == '__main__':
    rank = 0
    rnd.seed(0)
    N = 3#number of MTs
    mt_list = []
    for i in range(N):
        mt_list.append(mt(i,rank))
        x,y = rnd.uniform(xdomain[0],xdomain[1]), rnd.uniform(ydomain[0],ydomain[1])
        th = rnd.uniform(0,2*np.pi)
        mt_list[i].seg = [[x,y]]
        mt_list[i].angle = [th]
        mt_list[i].region_t.append(0)

    t = 0
    T = 100
    start = time()
    event_list = []
    pevent_list = []#past events
    del_mt = [] #deleted MTs
    cross_data = None
    last_result = None
    policy = None
    rewind = False
    i=0
    # up_t = 0
    # mt_t = 0
    conv = L/0.08 #time conversion factor
    cross_data = None
    # while t*conv< 100:
    j = 0
    start = time()
    ti = 0
    tf = 0
    i1,i2 = 0,0
    print('Simulating on', xdomain,'x',ydomain)
    print('Simulate to time', T)
    print('Length of', L, 'microm \n')
    while t*conv< T:
        i2 = np.floor(t*conv/10)
        if i2 != i1:
            tf = time()
            print('Sim time (s): ', t*conv)
            print('Wall time elapsed (s): ',tf-ti)
            print('Length of previous event list: ', len(pevent_list))
            print('Length of event list: ', len(event_list),'\n')
            # plot_snap(rank,mt_list,t,i,'./plots2/',False)
            ti = time()
            i1 = i2
        # if i >0:
        #     plot_snap(rank,mt_list,t,i,'./plots5/',False)
        Result = update_event_list(rank, mt_list, event_list,t,pevent_list,del_mt, last_result, policy,cross_data,rewind)
        rewind= False
        policy = Result[2]
        if Result[3]*conv > 90:
            plot_snap(0, mt_list, t,i,dest='./', save=True)
        update_return = update(rank,N,mt_list,Result[0],Result[1],Result[2], Result[3], Result[4],t,cross_data)
        last_result = update_return[0]
        t1 = Result[3]
        t = Result[3]
        if update_return[1]: #if there's a crossing
            cross_data = exit_data(rank, policy, last_result, mt_list,t) #package data to be transferred
            update_event_list(rank, mt_list, event_list,t,pevent_list,del_mt, last_result, policy,cross_data,rewind) #update list with previous changed MT
            policy = 'comm' #policy will be used to add the new event to list
            j+=1
        if policy in ['nucleate', 'top','bottom','cross_in']:
            N+=1
        i+=1
    end = time()
    print('Total steps: ', i)
    print('Percent of crossings: ', j)
    print('Total time: ',end-start)


from core.preprocessor import processor
import dill as dpickle
import numpy as np

class TextToNumber:

    @staticmethod
    def create_number_vector(raw_text_array, append_indicators=True, max_uniqie_words_size=8000, padding_maxlen=70, padding_position='post', output_file='nummerical_text'):
        body_pp = processor(append_indicators=append_indicators, keep_n=max_uniqie_words_size, padding_maxlen=padding_maxlen, padding=padding_position)
        vecs = body_pp.fit_transform(raw_text_array)

        if output_file is not None:
            # Save the preprocessor
            with open(output_file + '_pp.dpkl', 'wb') as f:
                dpickle.dump(body_pp, f)

            # Save the processed data
            np.save(output_file + '_vecs.npy', vecs)

        return vecs

    @staticmethod
    def load_text_processor(fname='title_pp.dpkl'):
        """
        Load preprocessors from disk.
        Parameters
        ----------
        fname: str
            file name of ktext.proccessor object
        Returns
        -------
        num_tokens : int
            size of vocabulary loaded into ktext.processor
        pp : ktext.processor
            the processor you are trying to load
        Typical Usage:
        -------------
        num_decoder_tokens, title_pp = load_text_processor(fname='title_pp.dpkl')
        num_encoder_tokens, body_pp = load_text_processor(fname='body_pp.dpkl')
        """
        # Load files from disk
        with open(fname, 'rb') as f:
            pp = dpickle.load(f)

        num_tokens = max(pp.id2token.keys()) + 1
        # print(f'Size of vocabulary for {fname}: {num_tokens:,}')
        return num_tokens, pp

    @staticmethod
    def load_decoder_inputs(decoder_np_vecs='train_title_vecs.npy'):
        """
        Load decoder inputs.
        Parameters
        ----------
        decoder_np_vecs : str
            filename of serialized numpy.array of decoder input (issue title)
        Returns
        -------
        decoder_input_data : numpy.array
            The data fed to the decoder as input during training for teacher forcing.
            This is the same as `decoder_np_vecs` except the last position.
        decoder_target_data : numpy.array
            The data that the decoder data is trained to generate (issue title).
            Calculated by sliding `decoder_np_vecs` one position forward.
        """
        vectorized_text = np.load(decoder_np_vecs)
        # For Decoder Input, you don't need the last word as that is only for prediction
        # when we are training using Teacher Forcing.
        decoder_input_data = vectorized_text[:, :-1]

        # Decoder Target Data Is Ahead By 1 Time Step From Decoder Input Data (Teacher Forcing)
        decoder_target_data = vectorized_text[:, 1:]

        # print(f'Shape of decoder input: {decoder_input_data.shape}')
        # print(f'Shape of decoder target: {decoder_target_data.shape}')
        return decoder_input_data, decoder_target_data

    @staticmethod
    def load_encoder_inputs(encoder_np_vecs='train_body_vecs.npy'):
        """
        Load variables & data that are inputs to encoder.
        Parameters
        ----------
        encoder_np_vecs : str
            filename of serialized numpy.array of encoder input (issue title)
        Returns
        -------
        encoder_input_data : numpy.array
            The issue body
        doc_length : int
            The standard document length of the input for the encoder after padding
            the shape of this array will be (num_examples, doc_length)
        """
        vectorized_body = np.load(encoder_np_vecs)
        # Encoder input is simply the body of the issue text
        encoder_input_data = vectorized_body
        doc_length = encoder_input_data.shape[1]
        # print(f'Shape of encoder input: {encoder_input_data.shape}')
        return encoder_input_data, doc_length


#!/usr/bin/python

# This script gets interface info such as bytes, packets and drops
# The use case this was developed for was gigamon testing with security onion but it could be used in a variety of ways
# Run example:
# ./check_int.py eno6 eno7
# List interfaces after filename with spaces
# it then writes out the result to a file initial.txt with bytes first then packets then drops.
# this will be analyzed in the check_int_dif script

import sys
import subprocess
import time
import calendar
import datetime

interfaces = sys.argv[1:]
prev_info = []
new_info = []
diff_info = []

def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def interface_diff(count):
        global interfaces
        global prev_info
        global new_info
        global diff_info

        headers = ["Interface", "Bytes", "Packets", "Drops"]
        new_info = []
        diff_info = [headers]
        line = []
        numInt=1
        # Run command and get interface information
        for interface in interfaces:
            # Testing with file
            #command = "cat test_check_rx.txt | grep -A 3 " + interface + " | tail -n 1"
            command = "ip -s link | grep -A 3 " + interface + " | tail -n 1"
            process = subprocess.Popen(command, shell = True, stdout = subprocess.PIPE)
            output, error = process.communicate()

            formatted = []

            # Remove empty space junk to standardize data locations
            for item in output.split(" "):
                if is_number(item):
                    formatted.append(item)

            line = [interface, formatted[0], formatted[1], formatted[3]]
            new_line = [formatted[0], formatted[1], formatted[3]]
            new_info.append(line)
            # write files to use for analysis
            if numInt==1:
                operation="w"
            else:
                operation="a"
            with open("initial.txt", operation) as initialOutput:
                initialOutput.write("%s %s %s\n" % (formatted[0], formatted[1], formatted[3]))
            numInt+=1

        # write out info to stdout
        print("")
        print("Run: " + str(count))
        print(datetime.datetime.fromtimestamp(calendar.timegm(time.gmtime())).isoformat())

        # Print interface information in data table
        if count == 1:
            initial = new_info[:]
            initial.insert(0, headers)
        print("Initial run:")
        print("-------------------------------------------------------------------")
        for i, d in enumerate(initial):
            line = '|'.join(str(x).ljust(16) for x in d)
            print(line)
            if i == 0:
                print('-' * len(line))
        print("-------------------------------------------------------------------")
       # ignore this
       # with open("initial.txt", "w") as initialOutput:
            #initialOutput.write("%s %s %s\n" % (formatted[0], formatted[1], formatted[3]))

interface_diff(1)


# get string from file
lines = []
with open('data.txt', 'r') as file:
    for line in file:
        if line[-1:] == '\n':
            lines.append(int(line[:-1]))
        else:
            lines.append(int(line))
print(lines)

# adds start
lines.append(0)
num_sorted = sorted(lines)
# adds end
num_sorted.append(num_sorted[-1] + 3)
print(num_sorted)

# choose parts with number that have difference == 1
len_parts_dif_1 = []
len_num = len(num_sorted)
index2 = 0
while index2 < len_num - 1:
    if num_sorted[index2 + 1] - num_sorted[index2] == 1:
        count_dif_1 = 1
        for index in range(index2, len_num - 1):
            if num_sorted[index + 1] - num_sorted[index] == 1:
                count_dif_1 += 1
            else:
                index2 = index
                len_parts_dif_1.append(count_dif_1)
                break
    index2 += 1
print('len_parts_dif_1', len_parts_dif_1)

# choose number that have diference == 2
count_dif_2 = 0
for index in range(len(num_sorted)-1):
    if num_sorted[index + 1] - num_sorted[index] == 2:
        count_dif_2 += 1
print('count_dif_2', count_dif_2)

# calculate amount the distinct sets
magic_dict = {2: 1, 3: 2, 4: 4, 5: 7, 6: 11}
product = 1
for item in len_parts_dif_1:
    product *= magic_dict[item]
print('distinct sets', product)


import datetime
import math

def countdown(due_date='2000/01/01', days_only=False):
    """
    :param due_date: year/month/day/hour   hour is optional
    :return: string denoting time left
    """
    # create countdown timer
    future = [int(s) for s in due_date.split('/')]
    f = future
    future = datetime.datetime(year=f[0], month=f[1], day=f[2])
    current = datetime.datetime.now()

    time_left = future - current
    time_left = time_left.days

    if days_only:
        print('time left:', time_left, 'days')
        return time_left

    def year(days):
        if days % 365 != days:
            return int(days / 365)
        else: return 0

    def month(days):
        years = year(days)
        if years >= 1:
            days = days - (years * 365)
        if days % 30 != days:
            return int(days / 30)
        else: return 0

    def day(days):
        months = month(days)
        years = year(days)
        if months >= 1:
            days = days - (years * 365) - (months * 30)
        return days

    t = time_left
    return '{0}yr {1}mth {2}dy left.'.format(year(t), month(t), day(t))

print(countdown('2019/05/22'))


import datetime
import itertools
import math
import os
import random
import glob
import json
from typing import *

import scipy
import imageio
import scipy.misc
import scipy.spatial
import scipy.ndimage
import numpy as np
import torch
import torch.nn as nn
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import umap
from torch.utils.data import Dataset, DataLoader, Sampler
from torch.utils.tensorboard import SummaryWriter

from PIL import Image
import skimage.transform
import matplotlib.pyplot as plt

import elpips
from PythonExtras.distance_matrix import render_distance_matrix, DistanceMatrixConfig

import models
from dcgan import Discriminator, Generator

class CatDataset(Dataset):

    def __init__(self, imageSubdirPath: str, transform: Callable):
        self.rootPath = imageSubdirPath
        self.pathList = glob.glob(os.path.join(self.rootPath, 'cat*.jpg'))

        self.transform = transform

    def __getitem__(self, index):
        path = self.pathList[index]
        image = Image.open(path)

        if self.transform is not None:
            image = self.transform(image)

        return image, os.path.basename(path)  # Pass image name as metadata. (Useful for export.)

    def __len__(self):
        return len(self.pathList)

class AuthorDataset(Dataset):

    def __init__(self, jsonPath: str, normalize=True):
        self.jsonPath = jsonPath

        with open(self.jsonPath, 'r') as file:
            self.data = json.load(file)

        self.names = self.data['names']
        self.vectors = np.asarray(self.data['vectors'])
        # Normalize the vectors.
        self.vectors = self.vectors / np.linalg.norm(self.vectors, axis=-1, keepdims=True)

    def __getitem__(self, index):
        return self.vectors[index], self.names[index]

    def __len__(self):
        return len(self.names)

class InfiniteSampler(Sampler):
    """
    From https://discuss.pytorch.org/t/implementing-an-infinite-loop-dataset-dataloader-combo/35567/3
    """
    def __init__(self, data_source: Sized):
        super().__init__(data_source)
        self.dataset_size = len(data_source)

    def __iter__(self):
        # This wrapper is only needed if we want to use a sample size other than one.
        # Otherwise, iteration over the random permutation tensors is already yielding single indices.
        yield from itertools.islice(self._infinite(), 0, None, 1)  # Infinite iterator

    def _infinite(self):
        g = torch.Generator()
        while True:
            yield from torch.randperm(self.dataset_size, generator=g)

def plot_image_scatter(ax, data, images, downscaleRatio: Optional[int] = None):
    from matplotlib.offsetbox import AnnotationBbox, OffsetImage

    if downscaleRatio:
        ratioInv = 1 / downscaleRatio
        images = [scipy.ndimage.interpolation.zoom(i, [ratioInv, ratioInv, 1], order=1) for i in images]

    ax.scatter(data[:, 0], data[:, 1])

    for i in range(len(images)):
        x, y = tuple(data[i])
        ab = AnnotationBbox(OffsetImage(images[i]), (x, y), frameon=False)
        ax.add_artist(ab)

def l2_sqr_dist_matrix(x: torch.Tensor) -> torch.Tensor:
    gram = torch.mm(x, x.T)
    diag = gram.diagonal()

    return diag.unsqueeze(1) + diag.unsqueeze(0) - 2 * gram

def main():

    dataSize = 32
    batchSize = 8
    elpipsBatchSize = 1
    # imageSize = 32
    imageSize = 64
    nz = 100

    # discCheckpointPath = r'E:\projects\visus\PyTorch-GAN\implementations\dcgan\checkpoints\2020_07_10_15_53_34\disc_step4800.pth'
    discCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netD_epoch_24.pth'
    genCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netG_epoch_24.pth'

    gpu = torch.device('cuda')

    # For now we normalize the vectors to have norm 1, but don't make sure
    # that the data has certain mean/std.
    pointDataset = AuthorDataset(
        jsonPath=r'E:\out\scripts\metaphor-vis\authors-all.json'
    )

    # Take top N points.
    points = np.asarray([pointDataset[i][0] for i in range(dataSize)])
    distPointsCpu = l2_sqr_dist_matrix(torch.tensor(points)).numpy()

    latents = torch.tensor(np.random.normal(0.0, 1.0, (dataSize, nz)),
                           requires_grad=True, dtype=torch.float32, device=gpu)

    scale = torch.tensor(2.7, requires_grad=True, dtype=torch.float32, device=gpu)  # todo Re-check!
    bias = torch.tensor(0.0, requires_grad=True, dtype=torch.float32, device=gpu)  # todo Re-check!

    lpips = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=True).to(gpu)
    # lossModel = lpips
    config = elpips.Config()
    config.batch_size = elpipsBatchSize  # Ensemble size for ELPIPS.
    config.set_scale_levels_by_image_size(imageSize, imageSize)
    lossModel = elpips.ElpipsMetric(config, lpips).to(gpu)

    discriminator = Discriminator(3, 64, 1)
    if discCheckpointPath:
        discriminator.load_state_dict(torch.load(discCheckpointPath))
    else:
        discriminator.init_params()
    discriminator = discriminator.to(gpu)

    generator = Generator(nz=nz, ngf=64)
    if genCheckpointPath:
        generator.load_state_dict(torch.load(genCheckpointPath))
    else:
        generator.init_params()
    generator = generator.to(gpu)

    # optimizerImages = torch.optim.Adam([images, scale], lr=1e-2, betas=(0.9, 0.999))
    optimizerScale = torch.optim.Adam([scale, bias], lr=0.001)
    # optimizerGen = torch.optim.Adam(generator.parameters(), lr=0.0002, betas=(0.5, 0.999))
    # optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.9, 0.999))
    # optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999))
    optimizerLatents = torch.optim.Adam([latents], lr=5e-3, betas=(0.9, 0.999))

    fig, axes = plt.subplots(nrows=2, ncols=batchSize // 2)

    fig2 = plt.figure()
    ax2 = fig2.add_subplot(1, 1, 1)

    outPath = os.path.join('runs', datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S'))
    os.makedirs(outPath)

    summaryWriter = SummaryWriter(outPath)

    for batchIndex in range(10000):

        # noinspection PyTypeChecker
        randomIndices = np.random.randint(0, dataSize, batchSize).tolist()  # type: List[int]
        # # randomIndices = list(range(dataSize))  # type: List[int]
        distTarget = torch.tensor(distPointsCpu[randomIndices, :][:, randomIndices], dtype=torch.float32, device=gpu)
        latentsBatch = latents[randomIndices]

        imageBatchFake = generator(latentsBatch[:, :, None, None].float())

        # todo It's possible to compute this more efficiently, but would require re-implementing lpips.
        # For now, compute the full BSxBS matrix row-by-row to avoid memory issues.
        lossDistTotal = torch.tensor(0.0, device=gpu)
        distanceRows = []
        for iRow in range(batchSize):
            distPredFlat = lossModel(imageBatchFake[iRow].repeat(repeats=(batchSize, 1, 1, 1)).contiguous(),
                                     imageBatchFake, normalize=True)
            distPred = distPredFlat.reshape((1, batchSize))
            distanceRows.append(distPred)
            lossDist = torch.sum((distTarget[iRow] - (distPred * scale + bias)) ** 2)  # MSE
            lossDistTotal += lossDist

        lossDistTotal /= batchSize * batchSize  # Compute the mean.

        distPredFull = torch.cat(distanceRows, dim=0)

        # print('{} - {} || {} - {}'.format(
        #     torch.min(distPred).item(),
        #     torch.max(distPred).item(),
        #     torch.min(distTarget).item(),
        #     torch.max(distTarget).item()
        # ))

        # discPred = discriminator(imageBatchFake)
        # lossRealness = bceLoss(discPred, torch.ones(imageBatchFake.shape[0], device=gpu))
        # lossGen = lossDist + 1.0 * lossRealness
        lossLatents = lossDistTotal

        # optimizerGen.zero_grad()
        # optimizerScale.zero_grad()
        # lossGen.backward()
        # optimizerGen.step()
        # optimizerScale.step()

        optimizerLatents.zero_grad()
        # optimizerScale.zero_grad()
        lossLatents.backward()
        optimizerLatents.step()
        # optimizerScale.step()

        # with torch.no_grad():
        #     # todo  We're clamping all the images every batch, can we clamp only the ones updated?
        #     # images = torch.clamp(images, 0, 1)  # For some reason this was making the training worse.
        #     images.data = torch.clamp(images.data, 0, 1)

        if batchIndex % 100 == 0:
            msg = 'iter {} loss dist {:.3f} scale: {:.3f} bias: {:.3f}'.format(batchIndex, lossDistTotal.item(), scale.item(), bias.item())
            print(msg)

            summaryWriter.add_scalar('loss-dist', lossDistTotal.item(), global_step=batchIndex)

            def gpu_images_to_numpy(images):
                imagesNumpy = images.cpu().data.numpy().transpose(0, 2, 3, 1)
                imagesNumpy = (imagesNumpy + 1) / 2

                return imagesNumpy

            # print(discPred.tolist())
            imageBatchFakeCpu = gpu_images_to_numpy(imageBatchFake)
            # imageBatchRealCpu = gpu_images_to_numpy(imageBatchReal)
            for iCol, ax in enumerate(axes.flatten()[:batchSize]):
                ax.imshow(imageBatchFakeCpu[iCol])
            fig.suptitle(msg)

            with torch.no_grad():
                images = gpu_images_to_numpy(generator(latents[..., None, None]))

                authorVectorsProj = umap.UMAP(n_neighbors=min(5, dataSize), random_state=1337).fit_transform(points)
                plot_image_scatter(ax2, authorVectorsProj, images, downscaleRatio=2)

            fig.savefig(os.path.join(outPath, f'batch_{batchIndex}.png'))
            fig2.savefig(os.path.join(outPath, f'scatter_{batchIndex}.png'))
            plt.close(fig)
            plt.close(fig2)

            with torch.no_grad():
                imagesGpu = generator(latents[..., None, None])
                imageNumber = imagesGpu.shape[0]

                # Compute LPIPS distances, batch to avoid memory issues.
                bs = min(imageNumber, 8)
                assert imageNumber % bs == 0
                distPredEval = np.zeros((imagesGpu.shape[0], imagesGpu.shape[0]))
                for iCol in range(imageNumber // bs):
                    startA, endA = iCol * bs, (iCol + 1) * bs
                    imagesA = imagesGpu[startA:endA]
                    for j in range(imageNumber // bs):
                        startB, endB = j * bs, (j + 1) * bs
                        imagesB = imagesGpu[startB:endB]

                        distBatchEval = lossModel(imagesA.repeat(repeats=(bs, 1, 1, 1)).contiguous(),
                                                  imagesB.repeat_interleave(repeats=bs, dim=0).contiguous(),
                                                  normalize=True).cpu().numpy()

                        distPredEval[startA:endA, startB:endB] = distBatchEval.reshape((bs, bs))

                distPredEval = (distPredEval * scale.item() + bias.item())

                # Move to the CPU and append an alpha channel for rendering.
                images = gpu_images_to_numpy(imagesGpu)
                images = [np.concatenate([im, np.ones(im.shape[:-1] + (1,))], axis=-1) for im in images]

                distPoints = distPointsCpu
                assert np.abs(distPoints - distPoints.T).max() < 1e-5
                distPoints = np.minimum(distPoints, distPoints.T)  # Remove rounding errors, guarantee symmetry.
                config = DistanceMatrixConfig()
                config.dataRange = (0., 4.)
                _, pointIndicesSorted = render_distance_matrix(
                    os.path.join(outPath, f'dist_point_{batchIndex}.png'),
                    distPoints,
                    images,
                    config=config
                )

                # print(np.abs(distPredFlat - distPredFlat.T).max())
                # assert np.abs(distPredFlat - distPredFlat.T).max() < 1e-5
                # todo The symmetry doesn't hold for E-LPIPS, since it's stochastic.
                distPredEval = np.minimum(distPredEval, distPredEval.T)  # Remove rounding errors, guarantee symmetry.
                config = DistanceMatrixConfig()
                config.dataRange = (0., 4.)
                render_distance_matrix(
                    os.path.join(outPath, f'dist_images_{batchIndex}.png'),
                    distPredEval,
                    images,
                    config=config
                )

                config = DistanceMatrixConfig()
                config.dataRange = (0., 4.)
                render_distance_matrix(
                    os.path.join(outPath, f'dist_images_aligned_{batchIndex}.png'),
                    distPredEval,
                    images,
                    predefinedOrder=pointIndicesSorted,
                    config=config
                )

                fig, axes = plt.subplots(ncols=2)
                axes[0].matshow(distTarget.cpu().numpy(), vmin=0, vmax=4)
                axes[1].matshow(distPredFull.cpu().numpy() * scale.item(), vmin=0, vmax=4)
                fig.savefig(os.path.join(outPath, f'batch_dist_{batchIndex}.png'))
                plt.close(fig)

                surveySize = 30
                fig, axes = plt.subplots(nrows=3, ncols=surveySize, figsize=(surveySize, 3))
                assert len(images) == dataSize
                allIndices = list(range(dataSize))
                with open(os.path.join(outPath, f'survey_{batchIndex}.txt'), 'w') as file:
                    for iCol in range(surveySize):
                        randomIndices = random.sample(allIndices, k=3)
                        leftToMid = distPointsCpu[randomIndices[0], randomIndices[1]]
                        rightToMid = distPointsCpu[randomIndices[2], randomIndices[1]]

                        correctAnswer = 'left' if leftToMid < rightToMid else 'right'
                        file.write("{}\t{}\t{}\t{}\t{}\n".format(iCol, correctAnswer, leftToMid, rightToMid,
                                                                 str(tuple(randomIndices))))

                        for iRow in (0, 1, 2):
                            axes[iRow][iCol].imshow(images[randomIndices[iRow]])

                fig.savefig(os.path.join(outPath, f'survey_{batchIndex}.png'))
                plt.close(fig)

            torch.save(generator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex)))
            torch.save(discriminator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex)))

    summaryWriter.close()

if __name__ == '__main__':
    main()


from django.shortcuts import render
from django.contrib.auth.decorators import login_required

from .utils import get_metrics_alerts

# Application Views

# Home

@login_required
def home_page(request):
	context = {
		'page_title': "Homepage"
	}
	return render(request, 'website/home_page.html', context)

@login_required
def rms_homepage(request):
	context = {
		'page_title': "RMS Homepage"
	}
	return render(request, 'website/rms_employee_homepage.html', context)

@login_required
def item_lookup_homepage(request):
	context = {
		'page_title': "Item Lookup Homepage"
	}
	return render(request, 'website/item_lookup_homepage.html', context)

@login_required
def metrics_homepage(request):
	context = {
		'page_title': "Metrics Homepage"
	}
	return render(request, 'website/metrics_homepage.html', context)

@login_required
def notifications_page(request):
	context = {
		'page_title': "Notifications Page",
		'metrics_alerts': get_metrics_alerts()
	}
	return render(request, 'website/notifications_page.html', context)

@login_required
def item_management_homepage(request):
	context = {
		'page_title': "Items Management Homepage",
	}
	return render(request, 'website/item_management_homepage.html', context)


from string import Template
# Retrieve HTML template.

def insert_for_testing():
    with open("Visualization.html", "r") as f:
        html_source = f.read()
        html_template = Template(html_source)

    viz_data_filename = "vis_data.csv"
    article_filename = "article.txt"
    original_url = "https://pravda.ru/"
    context = {
        'DATA_CSV_URL': viz_data_filename,
        'ARTICLE_TEXT_URL': article_filename,
        'ORIGINAL_URL': original_url
    }
    # Merge template URLs into HTML file
    html_output = html_template.substitute(context)
    with open("Visualization.html", mode="w") as html_file:
        html_file.write(html_output)

def revert_for_pushing(data, article, originalurl):
    with open("Visualization.html", "r") as f:
        html_source = f.read()
        html_template = Template(html_source)

    viz_data_filename = "$DATA_CSV_URL";
    article_filename = "$ARTICLE_TEXT_URL";
    original_url = "$ORIGINAL_URL";
    context = {
        data: viz_data_filename,
        article: article_filename,
        originalurl: original_url
    }
    # Merge template URLs into HTML file
    html_output = html_template.substitute(context)
    with open("Visualization.html", mode="w") as html_file:
        html_file.write(html_output)


import numpy as np
import matplotlib.pyplot as plt

class Image:
    """Class describing an n-dimensional object and its Fourier transform.

    For initialization, you may only provide one of the 'density', 'density_fct',
    'ft', 'ft_fct' arguments.

    Attributes:
        density : array_like (ndarray)
            d-D array representing density of an object.
        density_fct : function
            takes a d-dimensional point, evaluates density at this point.
        ft : array_like (ndarray)
            d-D array representing density Fourier transform.
        ft_fct : function
            takes a d-dimensional point, evaluates density Fourier transform.
        x_axes : list of ndarrays
            list of 1-D arrays, list contains d elements, representing
            coordinate axis in the object space.
        k_axes : list of ndarrays
            list of 1-D arrays, contains d elements, representing
            coordinate axis in the Fourier space.

    Examples:
    >>> im = Image();
    >>> im = Image(density_fct=sin, x_axes=np.linspace(0.,4*pi,200));
    >>> im = Image(ft_fct=abs, k_axes=np.linspace(-5.,5.,100));
    """
    def __init__(self,
                 density=None, density_fct=None,
                 ft=None, ft_fct=None,
                 x_axes=None, k_axes=None):

        self.set_default(x_axes=x_axes, k_axes=k_axes)

        # Check input args
        if [density, density_fct, ft, ft_fct].count(None) != 3:
            raise InputError('You need to provide one (and only one) of the following '
                             + "instantiation arguments: 'density', 'density_fct', "
                             + "'ft', 'ft_fct'. Provided, respectively ('None' is chosen "
                             + "by default): " + str(density) + ", " + str(density_fct) + ", "
                             + str(ft) + ", " + str(ft_fct) + ".")

        # Initialization from array
        if density != None:     # Density in the object space
            self.density = density
            self.ft = np.fft.fftn(self.density) # TODO: optimize to rfftn

        # Initialization from density function
        if density_fct!=None:
            # Fast evaluation of the density. For explanation, see, e.g.,
            # http://stackoverflow.com/questions/22774726/numpy-evaluate-function-on-a-grid-of-points
            if (self.d == 1):
                self.density = np.asarray(density_fct(self.x_axes[0]))
            elif (self.d == 2):
                self.density = np.asarray(density_fct(self.x_axes[0][:,None], self.x_axes[1][None,:]))
            elif (self.d == 3):
                self.density = np.asarray(density_fct(self.x_axes[0][:,None,None],
                                                      self.x_axes[1][None,:,None],
                                                      self.x_axes[2][None,None,:]))
            else:
                raise Error("Initialization from function is only realized for dimensions" +
                            " d=1,2,3. Input dimension: self.d=" + str(self.d) + ".");
            self.ft = np.fft.fftn(self.density)

        # Initialization from array
        if ft != None:     # Density in the object space
            self.ft = ft
            self.density = np.fft.ifftn(self.density)

        # Initialization from density function
        if ft_fct!=None:
            if (self.d == 1):
                self.ft = np.asarray(ft_fct(self.k_axes[0]))
            elif (self.d == 2):
                self.ft = np.asarray(ft_fct(self.k_axes[0][:,None], self.k_axes[1][None,:]))
            elif (self.d == 3):
                self.ft = np.asarray(ft_fct(self.k_axes[0][:,None,None],
                                                      self.k_axes[1][None,:,None],
                                                      self.k_axes[2][None,None,:]))
            else:
                raise Error("Initialization from function is only realized for dimensions" +
                            " d=1,2,3. Input dimension: self.d=" + str(self.d) + ".");
            self.density = np.fft.ifftn(self.ft)

    def set_default(self, d=2, N=(200,200),
                    x_min=(0.0,0.0), x_max=(2.*np.pi,2.*np.pi), x_axes=None, k_axes=None):
        """Creates a default axes setting for an image.

        Creates either a box set by default values or a box defined by my_axes.

        d : integer
            Dimension of the image.
        N : (N1, N2,..., Nd)
            Vector of integers. Describes number of points on the grid
            in the corresponding dimension.
        x_min : (x_min1, x_min2,..., x_mind)
            Vector of floats. Describes minimal coordinate value on the
            grid in the corresponding dimension.
        x_max : (x_max1, x_max2,..., x_maxd)
            Vector of floats. Describes maximal coordinate value on the
            grid in the corresponding dimension.
        x_axes : list of ndarrays
            list of 1-D arrays, contains d elements, representing
            coordinate axis in the object space.
        k_axes : list of ndarrays
            list of 1-D arrays, contains d elements, representing
            coordinate axis in the Fourier space.
        """
        if (x_axes==None and k_axes==None):
            self.d = d
            self.N = N
            self.x_min = x_min
            self.x_max = x_max
            self.k_min = [0 for i in range(self.d)]
            self.k_max = [self.N[i] - 1 for i in range(self.d)]
            # Create linear grid from x_min to x_max
            self.x_axes = [np.linspace(self.x_min[i], self.x_max[i], self.N[i]) for i in range(self.d)]
            # Create k-grid from 0 to N
            self.k_axes = [np.linspace(self.k_min[i], self.k_max[i], self.N[i]) for i in range(self.d)]

        elif (x_axes!=None and k_axes==None):
            self.d = len(x_axes)
            self.N = [len(x_axes[i]) for i in range(self.d)]
            self.x_min = [x_axes[i].min() for i in range(self.d)]
            self.x_max = [x_axes[i].max() for i in range(self.d)]
            self.k_min = [0 for i in range(self.d)]
            self.k_max = [self.N[i] - 1 for i in range(self.d)]
            self.x_axes = x_axes
            self.k_axes = [np.linspace(self.k_min[i], self.k_max[i], self.N[i]) for i in range(self.d)]

        elif (x_axes==None and k_axes!=None):
            self.d = len(k_axes)
            self.N = len(k_axes[0])
            self.x_min = [x_min[0] for i in range(self.d)]
            self.x_max = [x_max[0] for i in range(self.d)]
            self.k_min = [k_axes[i].min() for i in range(self.d)]
            self.k_max = [k_axes[i].max() for i in range(self.d)]
            self.x_axes = [np.linspace(self.x_min[i], self.x_max[i], self.N[i]) for i in range(self.d)]
            self.k_axes = k_axes

        elif (x_axes!=None and k_axes!=None):
            if len(k_axes) != len(x_axes):
                raise IOError("Argument size mismatch: grid in x-space ('x_axes') is "
                              + str(len(x_axes)) + "-dimensional, grid in k-space ('k_axes') is "
                              + str(len(k_axes)) + "-dimensional.")
            if len(k_axes[0]) != len(x_axes[0]):
                raise IOError("Argument size mismatch: grid in x-space ('x_axes') has "
                              + str(len(x_axes[0])) + " points in each dimension,"
                              + "grid in k-space ('k_axes') has "
                              + str(len(k_axes)) + " points in each dimension.")
            self.d = len(k_axes)
            self.N = [len(k_axes[i])  for i in range(self.d)]
            self.x_min = [x_axes[i].min() for i in range(self.d)]
            self.x_max = [x_axes[i].max() for i in range(self.d)]
            self.k_min = [k_axes[i].min() for i in range(self.d)]
            self.k_max = [k_axes[i].max() for i in range(self.d)]
            self.x_axes = x_axes
            self.k_axes = k_axes

        else:
            raise Error("Something went wrong during the initialization. Debug 'set_default()' in the class Image.")

    def plot(self):

        ft_ = np.fft.fftshift(self.ft)
        #k_axes_ = np.fft.fftshift(self.k_axes)

        if (self.d == 1):
            f, ax = plt.subplots(2, figsize = (12,9))
            f.subplots_adjust(hspace=0.35)
            ax2 = ax[1].twinx() # Phase will be plotted at the same place as modulus

            # Fonts for title and labels
            title_font = {'fontname':'TUM Neue Helvetica'}
            label_font = {'fontname':'TUM Neue Helvetica'}

            # Remove the plot frame lines. They are unnecessary chartjunk.
            for ax_i in ax:
                ax_i.spines["top"].set_visible(False)
                ax_i.spines["bottom"].set_visible(False)
                ax_i.spines["right"].set_visible(False)
                ax_i.spines["left"].set_visible(False)

                # Ensure that the axis ticks only show up on the bottom and left of the plot.
                # Ticks on the right and top of the plot are generally unnecessary chartjunk.
                ax_i.get_xaxis().tick_bottom()
                ax_i.get_yaxis().tick_left()

            # Limit the range of the plot to only where the data is.
            # Avoid unnecessary whitespace.
            wf = 1.02 # Whitespace factor
            # Density
            ax[0].set_xlim(min(self.x_min), min(self.x_max))
            ax[0].set_ylim(wf * min(self.density), wf * max(self.density))

            # FT modulus
            ax[1].set_xlim(min(self.k_min), max(self.k_max))
            ax[1].set_ylim(wf * min(abs(ft_)), wf * max(abs(ft_)))

            # FT phase
            ax2.set_xlim(min(self.k_min), max(self.k_max))
            ax2.set_ylim( - wf * np.pi, wf * np.pi)

            # Set ticks
            num_ticks = 7

            # Density
            # number of digits to round ticks
            n_rd = 2 - int(round(np.log10(max(self.density) - min(self.density))))
            ax[0].set_yticks(np.linspace(min(self.density),max(self.density),num_ticks))
            ax[0].set_yticklabels([str(round(x, n_rd)) for x in np.linspace(min(self.density),max(self.density),num_ticks)], color='b')

            # FT modulus
            n_rd = 2 - int(round(np.log10(max(np.abs(ft_)) - min(np.abs(ft_)))))
            if n_rd >= 1:
                ax[1].set_yticks(np.linspace(min(np.abs(ft_)),max(np.abs(ft_)),num_ticks))
                ax[1].set_yticklabels([str(round(x, n_rd)) for x in np.linspace(min(np.abs(ft_)),max(np.abs(ft_)),num_ticks)], **label_font)
            else:
                ax[1].set_yticks(np.linspace(min(np.abs(ft_)),max(np.abs(ft_)),num_ticks))
                ax[1].set_yticklabels([str(int(round(x, n_rd))) for x in np.linspace(min(np.abs(ft_)),max(np.abs(ft_)),num_ticks)], **label_font)

            # FT phase
            pi_str = r'$\pi$'
            ax2.set_yticks(np.linspace(-np.pi,np.pi,num_ticks))
            ax2.set_yticklabels(['-' + pi_str,
                                 '-2' + pi_str + '/3',
                                 #'-' + pi_str + '/2',
                                 '-' + pi_str + '/3',
                                 0,
                                 pi_str + '/3',
                                 #pi_str + '/2',
                                 '2' + pi_str + '/3',
                                 pi_str], **label_font)

            # Make sure your axis ticks are large enough to be easily read.
            # You don't want your viewers squinting to read your plot.
            my_fontsize = 14
            my_titlesize = 22
            for i in range(2):
                for tick in ax[i].xaxis.get_major_ticks():
                    tick.label.set_fontsize(my_fontsize)

                for tick in ax[i].yaxis.get_major_ticks():
                    tick.label.set_fontsize(my_fontsize)

            for tick in ax2.xaxis.get_major_ticks():
                tick.label.set_fontsize(my_fontsize)

            for tick in ax2.get_yticklabels():
                tick.set_color('g')
                tick.set_fontsize(my_fontsize)

            # Provide tick lines across the plot to help your viewers trace along
            # the axis ticks. Make sure that the lines are light and small so they
            # don't obscure the primary data lines.
            # Density
            for y in np.linspace(min(self.density), max(self.density), num_ticks):
                ax[0].plot(self.x_axes[0],
                           [y] * len(self.x_axes[0]),
                           "--", lw=0.5, color="black", alpha=0.3)

            # FT modulus
            for y in np.linspace(min(abs(ft_)), max(abs(ft_)), num_ticks):
                ax[1].plot(self.k_axes[0],
                           [y] * len(self.k_axes[0]),
                           "--", lw=0.5, color="black", alpha=0.3)

            # Remove the tick marks; they are unnecessary with the tick lines we just plotted.
            for i in range(2):
                ax[i].tick_params(axis="both", which="both", bottom="off", top="off",
                                  labelbottom="on", left="off", right="off", labelleft="on")

            title_shift = 1.07
            ax[0].plot(self.x_axes[0], self.density)
            ax[0].set_title('Object density', y=title_shift, fontsize=my_titlesize, **title_font)

            ax[1].plot(self.k_axes[0], np.abs(ft_))
            ax[1].set_title('Fourier transform', y=title_shift, fontsize=my_titlesize, **title_font)
            ax[1].set_ylabel('FT modulus', color='b', fontsize=my_fontsize+3, **label_font)
            for t1 in ax[1].get_yticklabels():
                t1.set_color('b')

            ax2.plot(self.k_axes[0], np.angle(ft_), color='g')
            ax2.set_ylabel('FT phase', color='g', fontsize=my_fontsize+3, **label_font)

class Approximation:
    """Class describing an approximation to an n-dimensional object. """

if __name__ == "__main__":
    import doctest
    doctest.testmod(extraglobs={'my_image': Image()})


# Time:  O(n)
# Space: O(1)
# 198
# You are a professional robber planning to rob houses along a street.
# Each house has a certain amount of money stashed, the only constraint stopping you
# from robbing each of them is that adjacent houses have security system connected
# and it will automatically contact the police if two adjacent houses were broken into on the same night.
#
# Given a list of non-negative integers representing the amount of money of each house,
# determine the maximum amount of money you can rob tonight without alerting the police.
#
class Solution:
    # @param num, a list of integer
    # @return an integer
    '''
    Storing both take and skip for each i is the same to solution 2. Proof: assume dp[i] = max(skip[i], take[i]).
    State transfer function:
       skip[i] = max(take[i-1], skip[i-1])   = dp[i-1]
       take[i] = skip[i-1]+n                 = dp[i-2]+n
    => dp[i] = max(dp[i-1], dp[i-2]+n)

    In fact, for each i, only the max from [take[i], skip[i]] is used to eval the later decision,
    so no need to store both take and skip for each i, only need to store the max for each i.
    '''
    def rob(self, nums): # USE THIS
        take, skip = 0, 0
        for n in nums:
            take, skip = skip + n, max(take, skip)
        return max(take, skip)

    # now is max money today can get, last is max money the previous day can get
    def rob2(self, nums):
        last, now = 0, 0
        for i in nums:
            last, now = now, max(last + i, now)
        return now

if __name__ == '__main__':
        print(Solution().rob([8,4,8,5,9,6,5,4,4,10])) # 40


# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

"""
Commands for storage file share operations
"""

#pylint: disable=too-many-arguments

import os.path
from azure.cli.core._logging import get_az_logger

def storage_file_upload_batch(client, destination, source, pattern=None, dryrun=False,
                              validate_content=False, content_settings=None, max_connections=1,
                              metadata=None):
    """
    Upload local files to Azure Storage File Share in batch
    """

    from .files_helpers import glob_files_locally
    source_files = [c for c in glob_files_locally(source, pattern)]

    if dryrun:
        logger = get_az_logger(__name__)
        logger.warning('upload files to file share')
        logger.warning('    account %s', client.account_name)
        logger.warning('      share %s', destination)
        logger.warning('      total %d', len(source_files or []))
        logger.warning(' operations')
        for f in source_files or []:
            logger.warning('  - %s => %s', *f)

        return []

    # TODO: Performance improvement
    # 1. Avoid create directory repeatedly
    # 2. Upload files in parallel
    def _upload_action(source_pair):
        dir_name = os.path.dirname(source_pair[1])
        file_name = os.path.basename(source_pair[1])
        if dir_name and len(dir_name) > 0:
            client.create_directory(share_name=destination,
                                    directory_name=dir_name,
                                    fail_on_exist=False)

        client.create_file_from_path(share_name=destination,
                                     directory_name=dir_name,
                                     file_name=file_name,
                                     local_file_path=source_pair[0],
                                     content_settings=content_settings,
                                     metadata=metadata,
                                     max_connections=max_connections,
                                     validate_content=validate_content)

        return client.make_file_url(destination, dir_name, file_name)

    return list(_upload_action(f) for f in source_files)

def storage_file_download_batch(client, source, destination, pattern=None, dryrun=False,
                                validate_content=False, max_connections=1):
    """
    Download files from file share to local directory in batch
    """

    from .files_helpers import glob_files_remotely, mkdir_p

    source_files = glob_files_remotely(client, source, pattern)

    if dryrun:
        source_files_list = list(source_files)

        logger = get_az_logger(__name__)
        logger.warning('upload files to file share')
        logger.warning('    account %s', client.account_name)
        logger.warning('      share %s', source)
        logger.warning('destination %s', destination)
        logger.warning('    pattern %s', pattern)
        logger.warning('      total %d', len(source_files_list))
        logger.warning(' operations')
        for f in source_files_list:
            logger.warning('  - %s/%s => %s', f[0], f[1], os.path.join(destination, *f))

        return []

    def _download_action(pair):
        destination_dir = os.path.join(destination, pair[0])
        mkdir_p(destination_dir)
        client.get_file_to_path(source,
                                directory_name=pair[0],
                                file_name=pair[1],
                                file_path=os.path.join(destination, *pair),
                                validate_content=validate_content,
                                max_connections=max_connections)
        return client.make_file_url(source, *pair)

    return list(_download_action(f) for f in source_files)


from sonni_tup_kupaytuvchilarga_ajratish import stka

def a2(d):
    sum11 = 1
    for i1 in range(0, len(d), 2):
        h = i1 + 1
        sum11 *= (d[i1]**((d[h]) + 1) - 1) / (d[i1] - 1)
    return sum11

def f7(n):
    list1 = range(3, (n + 1))
    list2 = range(3, (n + 1))
    output_list1 = []
    for i in range(len(list1)):
        q = list1[i]
        index_q = i
        e1 = stka(i)
        y1 = a2(e1)

        for j in range(len(list2)):
            w = list2[j]
            index_w = j
            e2 = stka(j)
            y2 = a2(e2)

            if y1 == y2:
                output_list1.append(q)
                output_list1.append(w)

    return output_list1

C++ dasturlash asoslari/42. Tanlangan masalalar yechimi/f7.py


from django.shortcuts import render_to_response, redirect
from django.urls import reverse
import os
import urllib.request
import urllib.parse
from urllib.parse import urlencode
from app.settings import BUS_STATION_CSV
import csv
from django.core.paginator import Paginator

def index(request):
    return redirect(reverse(bus_stations))

def bus_stations(request):
    page = request.GET.get('page')
    bus_station = []
    x = 0
    y = 0
    if page==None:
        page = 1
    with open(BUS_STATION_CSV, newline='') as csvfile:
        reader = csv.DictReader(csvfile)
        for line in reader:
            bus_station.append({'Name' : line['Name'], 'Street' : line['Street'], 'District' : line['District']})
    p = Paginator(bus_station, 10)
    object = p.get_page(page)
    if object.has_next():
        query_arg_next = {'page': object.next_page_number()}
        query_arg_next = f'bus_stations?{urllib.parse.urlencode(query_arg_next)}'
    else:
        query_arg_next = None
    if object.has_previous():
        query_arg_prev = {'page': object.previous_page_number()}
        query_arg_prev = f'bus_stations?{urllib.parse.urlencode(query_arg_prev)}'
    else:
        query_arg_prev = None

    return render_to_response('index.html', context={
        'bus_stations': object,
        'current_page': page,
        'prev_page_url': query_arg_prev,
        'next_page_url': query_arg_next,
    })


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 11 16:11:59 2019
networks.py
a selection of networks to use in the model
@author: nessa
"""

"""
Copyright 2019 Vanessa Martina Boehm

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import tensorflow as tf
import tensorflow_hub as hub

def conv_encoder(params, is_training=True):

    activation = params['activation']
    latent_size= params['latent_size']
    n_filt = params['n_filt']
    bias   = params['bias']
    dataset = params['data_set']

    def encoder(x):
        with tf.variable_scope('model/encoder',['x'], reuse=tf.AUTO_REUSE):

            net = tf.layers.conv2d(x,n_filt,5,strides=2,activation=None, padding='SAME', use_bias=bias) #64x64 -> 32x32/ 28x28 -> 14x14
            net = tf.layers.batch_normalization(net, training=is_training)
            net = activation(net)

            net = tf.layers.conv2d(net, n_filt*2, 5, 2, activation=None, padding='SAME', use_bias=bias) #32x32 -> 16x16 / 14x14 -> 7*7
            net = tf.layers.batch_normalization(net, training=is_training)
            net = activation(net)

            if dataset in ['celeba']:
                net = tf.layers.conv2d(net, n_filt*4, 5, 2, activation=None, padding='SAME', use_bias=bias) #16x16 -> 8x8
                net = tf.layers.batch_normalization(net, training=is_training)
                net = activation(net)

            net = tf.layers.flatten(net) #8x8*n_filt*4
            net = tf.layers.dense(net, latent_size*2, activation=None)
            return net

    return encoder

def conv_decoder(params, is_training=True):

    activation = params['activation']
    latent_size= params['latent_size']
    n_filt = params['n_filt']
    bias   = params['bias']
    dataset = params['data_set']

    def decoder(z):
        with tf.variable_scope('model/decoder',['z'], reuse=tf.AUTO_REUSE):

            if dataset in ['celeba','cifar10']:
                NN = 8
            elif dataset in ['mnist','fmnist']:
                NN = 7
            net = tf.layers.dense(z,n_filt*4*NN*NN,activation=activation, use_bias=bias)
            net = tf.reshape(net, [-1, NN, NN,n_filt*4])

            if dataset in ['celeba']:
                net = tf.layers.conv2d_transpose(net,n_filt*4, 5, strides=2, padding='SAME', use_bias=bias) # output_size 16x16/14x14
                net = tf.layers.batch_normalization(net, training=is_training)
                net = activation(net)

            net = tf.layers.conv2d_transpose(net,n_filt*2, 5, strides=2, padding='SAME', use_bias=bias) # output_size 16x16/14x14
            net = tf.layers.batch_normalization(net, training=is_training)
            net = activation(net)

            net = tf.layers.conv2d_transpose(net, n_filt, 5, strides=2, padding='SAME', use_bias=bias) # output_size 32x32/28x28
            net = tf.layers.batch_normalization(net, training=is_training)
            net = activation(net)

            net = tf.layers.conv2d_transpose(net, params['output_size'][-1], kernel_size=4, strides=1, activation=None, padding='same', name='output_layer')# bring to correct number of channels
        return net

    return decoder

def fully_connected_encoder(params,is_training):

    activation = params['activation']
    latent_size = params['latent_size']

    def encoder(x):
        with tf.variable_scope('model/encoder', reuse=tf.AUTO_REUSE):
            net = tf.layers.dense(x, 512, name='dense_1', activation=activation)
            net = tf.layers.dense(net, 256, name='dense_2', activation=activation)
            if params['dropout']:
                if is_training:
                    net = tf.nn.dropout(net,rate=params['rate'])
            net = tf.layers.dense(net, 128, name='dense_3', activation=activation)
            net = tf.layers.dense(net, 2*latent_size, name='dense_4', activation=None)
        return net
    return encoder

def fully_connected_decoder(params,is_training):

    activation = params['activation']
    latent_size= params['latent_size']

    def decoder(z):
        with tf.variable_scope('model/decoder', reuse=tf.AUTO_REUSE):
            net = tf.layers.dense(z, 128, name ='dense_1', activation=activation)
            net = tf.layers.dense(net, 256, name='dense_2', activation=activation)
            if params['dropout']:
                if is_training:
                    net = tf.nn.dropout(net, rate=params['rate'])
            net = tf.layers.dense(net, 512, name='dense_3', activation=activation)
            net = tf.layers.dense(net, params['output_size'] , name='dense_4', activation=None)
        return net
    return decoder

def make_encoder(params, is_training):

    network_type = params['network_type']

    if network_type=='fully_connected':
        encoder_ = fully_connected_encoder(params,is_training)
    elif network_type=='conv':
        encoder_ = conv_encoder(params,is_training)
    else:
        raise NotImplementedError("Network type not implemented.")

    def encoder_spec():
        x = tf.placeholder(tf.float32, shape=params['full_size'])
        z = encoder_(x)
        hub.add_signature(inputs={'x':x},outputs={'z':z})

    enc_spec  = hub.create_module_spec(encoder_spec)

    encoder   = hub.Module(enc_spec, name='encoder',trainable=True)

    hub.register_module_for_export(encoder, "encoder")

    return encoder

def make_decoder(params,is_training):

    network_type = params['network_type']

    if network_type=='fully_connected':
        decoder_ = fully_connected_decoder(params,is_training)
    elif network_type=='conv':
        decoder_ = conv_decoder(params, is_training)
    else:
        raise NotImplementedError("Network type not implemented.")

    def decoder_spec():
        z = tf.placeholder(tf.float32, shape=[None,params['latent_size']])
        x = decoder_(z)
        hub.add_signature(inputs={'z':z},outputs={'x':x})

    dec_spec  = hub.create_module_spec(decoder_spec)

    decoder   = hub.Module(dec_spec, name='decoder',trainable=True)

    hub.register_module_for_export(decoder, "decoder")

    return decoder


from src.data.meta import Meta
from src.filtering.scene import SceneHeuristic

import collections
import bisect
import json
import ast

class Content(Meta):
  def __init__(self, id, connector):
    super().__init__(id, connector)
    raw = self.run(f"""SELECT media_id, title, date_added , release_date
                    FROM media
                    WHERE media_id = {self.id};
                    """)[0]

    self.title = raw[1]
    self.date_added = raw[2]
    self.release_date = raw[3]

  def getResults(self, resultType = 'EMOJI', topScenes=False):
    res = self.run(f"""SELECT user_id, media_time_secs, challenge_key
                    FROM results
                    WHERE code='{resultType}' AND media_item_id = {self.id};""")

    data = []
    for i in res:
        data.append([i[0], self.getSceneIdx(i[1]), i[2]])

    if topScenes:
      new = []
      top = SceneHeuristic(data).getTopScenes(topScenes)
      for i in data:
        if i[1] in top:
          new.append(i)
      data = new

    self.results = data
    return self.results

  def getScenes(self):
    scenes = ast.literal_eval(self.run(f"""SELECT scenes FROM media_items_meta WHERE media_id = {self.id} ;""")[0][0])
    temp = collections.OrderedDict()
    for idx, i in enumerate(scenes):
      temp[i['end']] = idx
    self.scenes = temp
    return temp

  def getSceneIdx(self, timestamp):
    return bisect.bisect_left(list(self.scenes.keys()), timestamp)


import os
import torch
from torch.utils.data import Dataset, DataLoader
from skimage import io, transform
import skimage.morphology
from skimage.morphology import square
import numpy as np
import random
import pandas as pd
random.seed(0)
from scipy.ndimage.measurements import label as fun_label
import cv2

'''
    load data for fine-tuning on real data
'''

class loadReal(Dataset):
    '''
        load real data
    '''
    def __init__(self, SUNCG_data, IIW_data, NYU_data, DA_type, transform=None):
        '''
            data contains three dataset
            SUNCG synthetic data, contains ground truth albedo, shading, and normal
            IIW_data: intrinsic image in the wild, contains ground truth WHDR, shaing information
            NYU_data: NYU dataset, contains groundtruth normal, shading information
        '''
        self.SUNCG_list = []
        self.IIW_list = []
        self.NYU_list = []
        with open(SUNCG_data['fileNameList']) as f:
            for line in f:
                self.SUNCG_list.append(line.strip())
        with open(IIW_data['fileNameList']) as f:
            for line in f:
                self.IIW_list.append(line.strip())
        with open(NYU_data['fileNameList']) as f:
            for line in f:
                # NOTE: we have two file names for each NYU data,
                # One in original NYU dataset, used to load image and normal
                # The other in SAW dataset, used to load shading
                self.NYU_list.append(line.strip().split())
        self.SUNCG_dic = SUNCG_data
        self.IIW_dic = IIW_data
        self.NYU_dic = NYU_data
        self.transform = transform
        self.DA_type = DA_type
        self.lenNYU = len(self.NYU_list)
        self.numCompare = 100

    def __len__(self):
        # since the three datasets have different number of samples
        # we use number of IIW images as the length of the dataset
        return len(self.IIW_list)

    def __getitem__(self, idx):
        '''
            load one sample for each of the data set
        '''
        # load data for synthetic images
        label_SUNCG = {}
        image_SUNCG = np.array([])
        if self.SUNCG_dic:
            image_SUNCG, label_SUNCG['albedo'], label_SUNCG['shading'], \
                    label_SUNCG['normal'], label_SUNCG['mask'], label_SUNCG['label'] \
                    = self.load_synthetic(idx)
            if self.transform:
                image_SUNCG, label_SUNCG = self.transform(image_SUNCG, label_SUNCG, self.DA_type)

        # load data for IIW
        label_IIW = {}
        image_IIW = np.array([])
        if self.IIW_dic:
            label_IIW = {}
            image_IIW, label_IIW['saw_mask_1'], label_IIW['saw_mask_2'], \
                    label_IIW['label'] = self.load_IIW(idx)
            if self.transform:
                image_IIW, label_IIW = self.transform(image_IIW, label_IIW, self.DA_type)

        # load data for NYU
        label_NYU = {}
        image_NYU = np.array([])
        if self.NYU_dic:
            label_NYU = {}
            image_NYU, label_NYU['normal'], label_NYU['mask'],\
                    label_NYU['saw_mask_1'], label_NYU['saw_mask_2'] = self.load_NYU(idx)
            if self.transform:
                image_NYU, label_NYU = self.transform(image_NYU, label_NYU, self.DA_type)

        return image_SUNCG, image_IIW, image_NYU, label_SUNCG, label_IIW, label_NYU, self.IIW_list[idx]

    def loadLabel(self, labelName, dataName):
        '''
            load csv file
        '''
        data_csv = pd.read_csv(labelName).values
        data_csv = data_csv[:,2:]

        if dataName == 'SUNCG':
            # for suncg dataset, select 3000 comparisons
            tmpLabel = data_csv[:,-2]
            ind_equal = np.where(tmpLabel == 0)[0]
            ind_darker = np.where(tmpLabel == 1)[0]
            ind_lighter = np.where(tmpLabel == 2)[0]
            if ind_equal.shape[0] > self.numCompare:
                np.random.shuffle(ind_equal)
                ind_equal = ind_equal[0:self.numCompare]
            if ind_darker.shape[0] > self.numCompare:
                np.random.shuffle(ind_darker)
                ind_darker = ind_darker[0:self.numCompare]
            if ind_lighter.shape[0] > self.numCompare:
                np.random.shuffle(ind_lighter)
                ind_lighter = ind_lighter[0:self.numCompare]
            data_csv = np.concatenate((data_csv[ind_equal], data_csv[ind_darker],
                data_csv[ind_lighter]), axis=0)

        return data_csv

    def load_synthetic(self, idx):
        '''
            load one data for SUNCG
            load data for our synthesized data
            including: image, albedo, shading, normal, mask, comparison
        '''
        fileName = self.SUNCG_list[idx]
        # load image, albedo, shading, normal and mask
        imgName = os.path.join(self.SUNCG_dic['dataFolder'], fileName)
        image = io.imread(imgName)
        if len(image.shape)==2:
            image = np.tile(image[...,None], (1, 3))
        albedoName = os.path.join(self.SUNCG_dic['albedoFolder'], fileName)
        albedo = io.imread(albedoName)
        if len(albedo.shape)==2:
            albedo = np.tile(albedo[...,None], (1, 3))
        shadingName = os.path.join(self.SUNCG_dic['shadingFolder'], fileName)
        shading = io.imread(shadingName)
        if len(shading.shape)==2:
            shading = np.tile(shading[...,None], (1, 3))
        normalName = os.path.join(self.SUNCG_dic['normalFolder'], fileName[0:-8] + '_norm_camera.png')
        normal = io.imread(normalName)

        normalMaskName = os.path.join(self.SUNCG_dic['normalFolder'], fileName[0:-8] + '_valid.png')
        normalMask = io.imread(normalMaskName)
        normalMask = np.expand_dims(normalMask, axis=-1)

        maskName = os.path.join(self.SUNCG_dic['maskFolder'], fileName)
        mask = io.imread(maskName)
        mask = cv2.resize(mask, tuple(normalMask.shape[1::-1]), interpolation=cv2.INTER_CUBIC)
        mask = np.expand_dims(mask, axis=-1)

        labelName = os.path.join(self.SUNCG_dic['labelFolder'], fileName[0:-4] + '.csv')

        # pre-process data
        image = 1.0*image/255.0
        albedo = 1.0*albedo/255.0
        shading = 1.0*shading/255.0
        normal = normal.astype(np.float)
        normal = (normal/255.0-0.5)*2
        normal = normal/(np.linalg.norm(normal, axis=-1, keepdims=True) + 1e-6)
        mask = 1.0*mask/255.0
        normalMask = 1.0*normalMask/255.0
        mask = mask*normalMask
        return image, albedo, shading, normal, mask, labelName
        #return image, albedo, shading, normal, mask, label

    def load_IIW(self, idx):
        '''
            load images and labels for IIW  (and labels for SAW)
        '''
        fileName = self.IIW_list[idx]
        imgName = os.path.join(self.IIW_dic['dataFolder'], fileName + '.png')
        image = io.imread(imgName)
        if len(image.shape)==2:
            image = np.tile(image[...,None], (1, 3))

        # prepare shading mask
        shadingName = os.path.join(self.IIW_dic['SAW'], fileName + '.npy')
        sawShading = np.load(shadingName)
        saw_mask_1, saw_mask_2 = self.processShadingMask(sawShading)

        labelName = os.path.join(self.IIW_dic['labelFolder'], fileName + '.csv')
        #label = self.loadLabel(labelName, 'IIW')

        image = 1.0*image/255.0
        return image, saw_mask_1, saw_mask_2, labelName
        #return image, saw_mask_1, saw_mask_2, label

    def load_NYU(self, idx):
        '''
            load images, normals and mask for NYU dataset
        '''
        idx = idx % self.lenNYU
        fileName_SAW = self.NYU_list[idx][0]
        fileName_NYU = self.NYU_list[idx][1]
        imgName = os.path.join(self.NYU_dic['dataFolder'], fileName_NYU + '.png')
        image = io.imread(imgName)
        if len(image.shape)==2:
            image = np.tile(image[...,None], (1,3))
        normalName = os.path.join(self.NYU_dic['normalFolder'], fileName_NYU + '_normal.png')
        normal = io.imread(normalName)
        maskName = os.path.join(self.NYU_dic['normalMask'], fileName_NYU + '_mask.png')
        mask = io.imread(maskName)
        mask = np.expand_dims(mask, axis=-1)
        shadingName = os.path.join(self.NYU_dic['SAW'], fileName_SAW + '.npy')
        shading = np.load(shadingName)
        saw_mask_1, saw_mask_2 = self.processShadingMask(shading)

        image = 1.0*image/255.0
        mask = 1.0*mask/255.0
        normal = normal.astype(np.float)
        normal = (normal/255.0-0.5)*2
        normal = normal/(np.linalg.norm(normal, axis=-1, keepdims=True) + 1e-6)

        return image, normal, mask, saw_mask_1, saw_mask_2

    def processShadingMask(self, sawShading):
        '''
            filtering sawShading
        '''
        # NOTE: saw_mask_1 is the boundary of the shadow
        #       saw_mask_2 is the smooth region of the shadow
        saw_mask_1 = (sawShading == 1)
        saw_mask_1 = skimage.morphology.binary_dilation(saw_mask_1, square(9)).astype(np.float32)

        saw_mask_2 = (sawShading == 2).astype(np.float32)
        return saw_mask_1, saw_mask_2

class dataAugmentation(object):
    '''
        define data augmentation for non-comparison purpose
    '''
    def __init__(self, output_size, expand=10, ind_rotate=True):
        self.size = output_size
        self.expand = expand
        self.ind_rotate = ind_rotate

    def processImage(self, image, sH, sW, H, W, rotate_M):
        # rotate
        imgHeight, imgWidth = image.shape[0:2]
        image = cv2.warpAffine(image, rotate_M, (imgWidth, imgHeight))
        # resize and random crop
        image = cv2.resize(image, (W, H), interpolation=cv2.INTER_CUBIC)
        if len(image.shape) < 3:
            image = image[sH:sH+self.size, sW:sW+self.size]
        else:
            image = image[sH:sH+self.size, sW:sW+self.size, :]
        return image

    def __call__(self, image, label, DA_type):
        '''
            random crop, in-plane rotaion for images
            and apply same data augmentation for the labels
        '''
        imgHeight, imgWidth,_ = image.shape
        if not DA_type == 'compare':
            # first rotate
            if self.ind_rotate:
                if random.random() > 0.5:
                    # random rotate -10 to 10 degrees
                    angle = -10*random.random()
                    rotate_M = cv2.getRotationMatrix2D((imgHeight/2.0, imgWidth/2.0), angle, 1)
                else:
                    rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 0, 1)
            # random crop
            H = self.size + self.expand
            W = self.size + self.expand
            maxH = H - self.size
            maxW = W - self.size
            sH = random.randint(0, maxH)
            sW = random.randint(0, maxW)
        else:
            # rotate 0 degree
            rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 0, 1)

            # no resize and crop
            H = self.size
            W = self.size
            sH = 0
            sW = 0

        # preprocess image
        image = self.processImage(image, sH, sW, H, W, rotate_M)
        image = np.transpose(image, (2,0,1))
        ind = image < 0
        image[ind] = 0
        if 'albedo' in label.keys():
            tmp = self.processImage(label['albedo'], sH, sW, H, W, rotate_M)
            ind = tmp < 0
            tmp[ind] = 0
            label['albedo'] = np.transpose(tmp, (2,0,1))
        if 'shading' in label.keys():
            tmp = self.processImage(label['shading'], sH, sW, H, W, rotate_M)
            ind = tmp < 0
            tmp[ind] = 0
            label['shading'] = np.transpose(tmp, (2,0,1))
        if 'saw_mask_1' in label.keys():
            tmp_mask = self.processImage(label['saw_mask_1'], sH, sW, H, W, rotate_M)
            tmp_mask, numLabel = fun_label(tmp_mask)
            label['saw_mask_1'] = np.expand_dims(tmp_mask, axis=0)
            label['num_saw_mask_1'] = np.array([numLabel])
        if 'saw_mask_2' in label.keys():
            tmp_mask = self.processImage(label['saw_mask_2'], sH, sW, H, W, rotate_M)
            tmp_mask, numLabel = fun_label(tmp_mask)
            label['saw_mask_2'] = np.expand_dims(tmp_mask, axis=0)
            label['num_saw_mask_2'] = np.array([numLabel])
        if 'mask' in label.keys():
            tmp_mask = self.processImage(label['mask'], sH, sW, H, W, rotate_M)
            tmp_mask = (tmp_mask==1.0).astype(np.float)
            label['mask'] = np.expand_dims(tmp_mask, axis=0)
        if 'normal' in label.keys():
            # in-plane rotation
            #rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 0, 1)
            tmpNormal = self.processImage(label['normal'], sH, sW, H, W, rotate_M)
            maskInd = np.sum(np.abs(tmpNormal), axis=2) == 0
            maskInd = np.tile(np.expand_dims(maskInd, -1), (1,1,3))
            # we need to rotate normal
            # NOTE: 0 and 2 coordinate represent normal along x and y plane
            #if DA_type=='compare':
            #    rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 0, 1)
            #else:
            #    rotate_M = cv2.getRotationMatrix2D((self.size/2, self.size/2), 90, 1)
            tmpNormal_xy = np.reshape(tmpNormal[:,:,0:3:2], (-1,2))
            tmpNormal_xy = np.dot(tmpNormal_xy, np.transpose(rotate_M[0:2, 0:2], (1,0)))
            tmpNormal_xy = np.reshape(tmpNormal_xy, (self.size, self.size, 2))
            tmpNormal[:,:,0:3:2] = tmpNormal_xy
            tmpNormal[maskInd] = 0
            tmpNormal = tmpNormal / (np.tile(np.sqrt(np.sum(tmpNormal**2, axis=2, keepdims=True)), (1,1,3)) + 1e-6)
            label['normal'] = np.transpose(tmpNormal, (2,0,1))
        return image, label