import pygame
from pygame.locals import *
from OpenGL.GL import *
from OpenGL.GLU import *
from figure import *
from tkinter import *
from tkinter import ttk
import time
import serial
import threading
from dataPlotter import live_plotter

ApplicationGL = False

class PortSettings:
    Name = "COM3"
    Speed = 115200
    Timeout = 2

class IMU:
    Roll = 0
    Pitch = 0
    Yaw = 0

myport = PortSettings()
myimu  = IMU()

def RunAppliction():
    global ApplicationGL
    myport.Name = Port_entry.get()
    myport.Speed = Baud_entry.get()
    ApplicationGL = True
    ConfWindw.destroy()

ConfWindw = Tk()
ConfWindw.title("Configure Serial Port")
ConfWindw.configure(bg = "#2E2D40")
ConfWindw.geometry('300x150')
ConfWindw.resizable(width=False, height=False)
positionRight = int(ConfWindw.winfo_screenwidth()/2 - 300/2)
positionDown = int(ConfWindw.winfo_screenheight()/2 - 150/2)
ConfWindw.geometry("+{}+{}".format(positionRight, positionDown))

Port_label = Label(text = "Port:", font =("",12), justify= "right", bg = "#2E2D40", fg = "#FFFFFF")
Port_label.place(x = 50, y =30, anchor = "center")
Port_entry = Entry(width = 20, bg = "#37364D", fg = "#FFFFFF", justify = "center")
Port_entry.insert(INSERT,myport.Name)
Port_entry.place(x = 180, y = 30,anchor = "center")

Baud_label = Label(text = "Speed:", font =("",12), justify= "right", bg = "#2E2D40", fg = "#FFFFFF")
Baud_label.place(x = 50, y =80, anchor = "center")
Baud_entry = Entry(width = 20,bg = "#37364D", fg = "#FFFFFF", justify = "center")
Baud_entry.insert(INSERT,str(myport.Speed))
Baud_entry.place(x = 180, y = 80, anchor = "center")

ok_button = Button(text = "Ok",width = 8,command = RunAppliction,bg="#135EF2",fg ="#FFFFFF")
ok_button.place(x = 150, y = 120, anchor="center")

def InitPygame():
    global display
    #global gameDisplay
    pygame.init()
    display = (1280,960)
    #gameDisplay =  pygame.display.set_mode(display, DOUBLEBUF|OPENGL)
    pygame.display.set_mode(display, DOUBLEBUF|OPENGL)
    pygame.display.set_caption('IMU visualizer   (Press Esc to exit)')

def InitGL():
    glClearColor((1.0/255*46),(1.0/255*45),(1.0/255*64),1)
    glEnable(GL_DEPTH_TEST)
    glDepthFunc(GL_LEQUAL)
    glHint(GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST)

    gluPerspective(100, (display[0]/display[1]), 0.1, 50.0)
    glTranslatef(0.0,0.0, -5)

def DrawText(textString):
    font = pygame.font.SysFont ("Courier New",25, True)
    textSurface = font.render(textString, True, (255,255,0), (46,45,64,255))
    textData = pygame.image.tostring(textSurface, "RGBA", True)
    glDrawPixels(textSurface.get_width(), textSurface.get_height(), GL_RGBA, GL_UNSIGNED_BYTE, textData)

def DrawBoard():

    glBegin(GL_QUADS)
    x = 0

    for surface in surfaces:

        for vertex in surface:
            glColor3fv((colors[x]))
            glVertex3fv(vertices[vertex])
        x += 1
    glEnd()

def DrawGL():

    glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)
    glLoadIdentity()
    gluPerspective(90, (display[0]/display[1]), 0.1, 50.0)
    glTranslatef(0.0,0.0, -5)

    glRotatef(round(myimu.Pitch,1), 0, 0, 1)
    glRotatef(round(myimu.Roll,1), -1, 0, 0)
    glRotatef(round(myimu.Yaw,1), 0, 1, 0)

    DrawText(" Roll: {}°     Pitch: {}°      Yaw: {}°".format(round(myimu.Roll,1),round(myimu.Pitch,1),round(myimu.Yaw,1)))
    DrawBoard()

    # plotImg = pygame.image.load('plot.png')

    # imgData = pygame.image.tostring(plotImg, "RGBA", True)

    # glDrawPixels(plotImg.get_width(), plotImg.get_height(), GL_RGBA, GL_UNSIGNED_BYTE, imgData)

    #gameDisplay.blit(plotImg, (300,300))

    pygame.display.flip()

def SerialConnection ():
    global serial_object
    serial_object = serial.Serial( myport.Name, baudrate= myport.Speed, timeout = myport.Timeout)

class ReadLine:
    def __init__(self, s):
        self.buf = bytearray()
        self.s = s

    def readline(self):
        i = self.buf.find(b"\n")
        if i >= 0:
            r = self.buf[:i+1]
            self.buf = self.buf[i+1:]
            return r
        while True:
            i = max(1, min(2048, self.s.in_waiting))
            data = self.s.read(i)
            i = data.find(b"\n")
            if i >= 0:
                r = self.buf + data[:i+1]
                self.buf[0:] = data[i+1:]
                return r
            else:
                self.buf.extend(data)

def ReadData():

    rl = ReadLine(serial_object)

    while True:

        data = rl.readline().decode('unicode_escape')
        #data = serial_object.read(256)
        #data.splitlines()

        # Received dataframe example:
        # Orientation: 347.66, 1.26, 1.38 (YAW, PITCH, ROLL)
        # Quaternion: 0.1076, -0.0097, 0.0132, 0.9941

        if data[0] == 'O': #Orientation
            data = data[13:].rstrip('\n')
            orientationData = data.split(', ')
            yaw = orientationData[0]
            pitch = orientationData[1]
            roll = orientationData[2]

            print(yaw, pitch, roll)

            #Convert string values to float
            myimu.Roll = float(roll)
            myimu.Pitch = float(pitch)
            myimu.Yaw = float(yaw)

def main():
    ConfWindw.mainloop()
    if ApplicationGL == True:
        InitPygame()
        InitGL()

        try:
            SerialConnection()
            line1 = []
            line2 = []
            line3 = []
            list = [0] * 100
            list2 = [0] * 100
            list3 = [0] * 100

            pygame.time.wait(10)
            myThread1 = threading.Thread(target = ReadData)
            myThread1.daemon = True
            myThread1.start()
            while True:
                event = pygame.event.poll()
                line1, line2, line3, list, list2, list3 = live_plotter(myimu.Yaw, myimu.Pitch, myimu.Roll, line1, line2, line3, list, list2, list3)
                if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
                    pygame.quit()
                    break

                DrawGL()
                pygame.time.wait(10)

        except:
            glClear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT)
            DrawText("Sorry, something is wrong :c")
            pygame.display.flip()
            time.sleep(5)

if __name__ == '__main__': main()


import torch
from model import Net
from load_data import (
    data,
    train_dl,
    valid_dl,
    auc_dl,
    all_dl,
    test_dl
)
from utils import (
    display_data,
    init_weights,
    train,
    plot_history,
    plot_auc_curve,
    save_results
)
from parameters import (
    LABELS_MAP,
    DEVICE,
    OPTIMIZER,
    LEARNING_RATE,
    N_EPOCHS,
    LOSS_FN,
    RESULTS_PATH
)

display_data(data, classes=LABELS_MAP)

model = Net().to(DEVICE)
model.apply(init_weights)

print('\nTraining... (with validation)\n')

optimizer = OPTIMIZER(model.parameters(),
                      lr=LEARNING_RATE)
history = train(model,
                train_dl,
                valid_dl,
                optimizer=optimizer,
                lr=LEARNING_RATE,
                epochs=N_EPOCHS,
                loss_fn=LOSS_FN)

plot_history(history, validation=True)

plot_auc_curve(model, auc_dl)

model_ = Net().to(DEVICE)
model_.apply(init_weights)

print('\nTraining... (without validation)\n')

optimizer = OPTIMIZER(model_.parameters(),
                      lr=LEARNING_RATE)
history = train(model_,
                all_dl,
                optimizer=optimizer,
                lr=LEARNING_RATE,
                epochs=N_EPOCHS,
                loss_fn=LOSS_FN)

plot_history(history)

torch.save(model_, './model.pth')

print('\nSaving results...')

save_results(RESULTS_PATH, test_dl, model_)


from flask import render_template, request, redirect, flash, url_for
from app import app, db, models, mail
from flask_mail import Message

@app.route('/')
@app.route('/index')
def index():
    return render_template('index.html',
                           title='Welcome')

@app.route('/chapOnly')
def cFom():
    return render_template('conly.html',
                           title='Welcome')

@app.route('/list')
def list():
    users = models.User.query.all()
    return render_template('list.html',
                           title='List',
                           users=users)

@app.route('/chaplist')
def chaplist():
    users = models.Chap.query.all();
    return render_template('chaperones.html',
                           title='List',
                           users=users)

@app.route('/register', methods=['POST'])
def login():
    error = None
    n=1
    chapText = ""
    UserList=[]
    chapList=[]
    if request.method == 'POST':
        while(request.form['name'+str(n)]!=''):
            print(n)
            print(request.form['name'+str(n)])
            if(models.User.query.filter(models.User.name == request.form['name'+str(n)], models.User.parent == request.form['parent']).count()==0):
                UserList.append(models.User(name=request.form['name'+str(n)], age = int(request.form['age'+str(n)]), parent = request.form['parent'], parent_email = request.form['parent_email'], lunch = (request.form['lunch']!="false")
                , chaperone = (request.form['chap']!="false"), chapName = request.form['chapn'], notes = request.form['notes'+str(n)]))
                db.session.add(UserList[n-1])
                chapText+=request.form['name'+str(n)]+" Age:"+request.form['age'+str(n)]+", "
            else:
                return 'Error, '+request.form['name'+str(n)]+' is already registered.  Please email Shivam at smevawala@lgsinnovations.com for assistance'
            n+=1
        if(request.form['chap']!="false"):
            if(models.Chap.query.filter(models.Chap.name == request.form['chapn'], models.Chap.parent == request.form['parent']).count()==0):
                chap= models.Chap(name  = request.form['chapn'], parent = request.form['parent'], parent_email = request.form['parent_email'], kids = chapText)
                db.session.add(chap)
                chapList.append(chap)
            else:
                return 'Error, '+request.form['chapn']+' is already registered as a chaperone.  Please email Shivam at smevawala@lgsinnovations.com for assistance'

        db.session.commit()
        flash('You registered')
        msg=Message("Thanks for registering for LGS's Bring Your Child to Work Day!", sender="smevawala@lgsinnovations.com", recipients=[UserList[0].parent_email])
        #mail.send(msg)
        return render_template('thanks.html',
                               title='Thank You For Registering',
                               users=UserList, chaps=chapList)
    return redirect(url_for('index'))

@app.route('/registerC', methods=['POST'])
def regC():
    error = None
    UserList=[]
    chapList=[]
    if request.method == 'POST':
        if(models.Chap.query.filter(models.Chap.name == request.form['chapn'], models.Chap.parent == request.form['chapn']).count()==0):
            print("got here")
            chap= models.Chap(name  = request.form['chapn'], parent = request.form['chapn'], parent_email = "" ,kids = '')
            db.session.add(chap)
            chapList.append(chap)
            db.session.commit()
            flash('You registered')
            return render_template('thanks.html',
                                   title='Thank You For Registering',
                                   users=UserList, chaps=chapList)
        else:
            print("got here else")
            return 'Error, '+request.form['chapn']+' is already registered as a chaperone.  Please email Shivam at smevawala@lgsinnovations.com for assistance'
    return redirect(url_for('index'))


from PIL import Image

# difference of getcolors, getpixel, getdata

def main():
    filename = "d:/YID_git_repos/DLCoding/naruto1.jpg"
    image = Image.open(filename).crop((0,0,3,3))
    size = width, height = image.size

    print(image.getcolors( width*height))
    print(image.getpixel((2,2))         )
    print(list(image.getdata())         )

if __name__ == '__main__':
    main()


audio='AudioClip.wav' #location

import librosa
import librosa.display
import matplotlib.pyplot as plt

# Load an audio file
y, sr = librosa.load(audio)

# Compute harmonic component of the audio signal
y_harmonic = librosa.effects.harmonic(y)

# Compute tonnetz
tonnetz = librosa.feature.tonnetz(y=y_harmonic, sr=sr)

# Display the tonnetz
plt.figure(figsize=(10, 4))
librosa.display.specshow(tonnetz, x_axis='time', y_axis='tonnetz')
plt.colorbar()
plt.title('Tonal Centroids (Tonnetz)')
plt.tight_layout()
plt.show()


from lxml.cssselect import CSSSelector
from lxml.html import fromstring
from lxml.etree import tostring
from urllib.request import urlopen
import os
from tqdm import tqdm

def getURLs(url):
	#otwiera stronę
	h = fromstring(urlopen(url+"testy.html").read())

	dict={}
	#pobiera komórki tabel (#oligonukleotydów, długość oligonukleotydu, #błędów)
	sel = CSSSelector('tr td')
	for e in sel(h):
		if e.sourceline not in dict:
			dict[e.sourceline]=[]
		if e.text is not None:
			dict[e.sourceline].append(int(e.text))#.decode('iso-8859-2')

	#pobiera url zbiorów testowych
	sel = CSSSelector('tr td a')
	for e in sel(h):
		dict[e.sourceline].append(url+e.get('href'))#.decode('iso-8859-2')

	#pobiera typy zbiorów
	# dict2={}
	# sel = CSSSelector('h4')
	# for e in sel(h):
		# if e.sourceline not in dict:
			# dict2[e.sourceline]=[]
		# if e.text is not None:
			# dict2[e.sourceline].append(e.text)#.decode('iso-8859-2')

	#dodaje typy zbiorów
	# for x in dict2:
		# for y in dict:
			# if len(dict[y])==4 and x<y:
				# dict[y].append(dict2[x][0])
				# dict[y].append(x)
			# elif len(dict[y])==6 and x>dict[y][5] and x<y:
				# dict[y][4]=dict2[x][0]
				# dict[y][5]=x

	#usuwa nadmiarowe dane
	# for x in dict:
		# dict[x]=dict[x][:-1]

	return dict;

def doFile(dir,nazwa,tab):
	if not os.path.exists(dir):
		os.makedirs(dir)
	f = open(dir+"/"+nazwa,'w')
	f.write(str(tab[0]) + " " + str(tab[1]) + " " + "nie pamiętam co tu powinno być\n")
	for line in urlopen(tab[3]).readlines():
		f.write(line.decode('iso-8859-2'))
	f.close()

if __name__=="__main__":
	dict = getURLs("http://www.cs.put.poznan.pl/mkasprzak/bio/");
	for i,d in enumerate(tqdm(dict)):
		doFile("datasets","plik"+str(i)+".txt",dict[d])


from typing import Tuple
import random
import pygame
from IslandPy.Render.ARenderObject import ARenderObject
from IslandPy.Scenes.AScene import AScene

class Obstacle(ARenderObject):

    speed = 8

    def __init__(self, scene: AScene, size: Tuple[int, int] = (0, 0)):
        super().__init__(scene, size)
        self.images = []
        for i in range(1, 6):
            self.images.append(pygame.image.load(f"res/cactus{i}.png").convert_alpha())

        self._body = self.images[0]
        self.rect = self._body.get_rect()
        self.__start_pos_y = 0
        self.is_started = False
        self.hide()

    def update(self, dt) -> None:
        if self.is_started:
            self.rect.x -= Obstacle.speed

    def change_image(self) -> None:
        b = self.rect.bottom
        self._body = random.choice(self.images)
        self.rect.w, self.rect.h = self._body.get_rect().w, self._body.get_rect().h
        self.rect.bottom = b

    def start(self) -> None:
        self.show()
        self.is_started = True

    def draw(self, surface: pygame.Surface) -> None:
        if self.is_draw:
            surface.blit(self._body, self.rect)


import torch

import torch.nn as nn
import torch.nn.functional as F
import numpy as np

from .embedding import Optcodes
from hydra.utils import instantiate

from core.utils.ray_utils import kp_to_valid_rays
from core.utils.skeleton_utils import (
    Skeleton,
    SMPLSkeleton,
    get_skel_profile_from_rest_pose,
    calculate_kinematic,
)

from einops import rearrange
from core.networks.camcal import CamCal
from typing import Optional, Callable, Mapping, Any, List, Union
from omegaconf import DictConfig

def batchify_fn(
    fn: Callable,
    fn_inputs: Mapping[str, Any],
    N_total: int,
    render_normal: bool = False,
    chunk: int = 4096
):
    """ Break evaluation into batches to avoid OOM
    """
    all_ret = {}

    for i in range(0, N_total, chunk):
        batch_inputs = {k: fn_inputs[k][i:i+chunk] if torch.is_tensor(fn_inputs[k]) else fn_inputs[k]
                        for k in fn_inputs}
        ret = fn(batch_inputs, render_normal=render_normal)
        if ret is None:
            continue

        for k in ret:
            if k not in all_ret:
                all_ret[k] = []
            all_ret[k].append(ret[k])

    all_ret = {k : torch.cat(all_ret[k], 0) for k in all_ret}
    return all_ret

def merge_encodings(
    encoded: Mapping[str, torch.Tensor],
    encoded_is: Mapping[str, torch.Tensor],
    sorted_idxs: torch.Tensor,
    N_rays: int,
    N_total_samples: int,
    inplace: bool = True
):
    """
    merge coarse and fine encodings.
    Parameters
    ----------
    encoded: dictionary of coarse encodings
    encoded_is: dictionary of fine encodings
    sorted_idxs: define how the [encoded, encoded_is] are sorted
    """
    if encoded_is is None:
        return encoded
    if encoded is None:
        return encoded_is
    gather_idxs = torch.arange(N_rays * (N_total_samples)).view(N_rays, -1)
    gather_idxs = torch.gather(gather_idxs, 1, sorted_idxs)

    merged = encoded if inplace else {}

    for k in encoded.keys():
        #if not k.startswith(('pts', 'blend')) and k not in ['g', 'graph_feat', 'bone_logit']:
        if k in ['valid_idxs', 'rw'] or encoded[k] is None or isinstance(encoded[k], dict):
            continue
        if k in encoded and k not in encoded_is:
            merged[k] = encoded[k]
            continue
        merged[k] = merge_samples(encoded[k], encoded_is[k], gather_idxs, N_total_samples)

    # need special treatment here to preserve the computation graph.
    # (otherwise we can just re-encode everything again, but that takes extra computes)
    if 'pts' in encoded and encoded['pts'] is not None:
        if not inplace:
            merged['pts'] = encoded['pts']

        merged['pts_is'] = encoded_is['pts']
        merged['gather_idxs'] = gather_idxs

        merged['pts_sorted'] = merge_samples(encoded['pts'], encoded_is['pts'],
                                                gather_idxs, N_total_samples)

    return merged

def merge_samples(
    x: torch.Tensor,
    x_is: torch.Tensor,
    gather_idxs: torch.Tensor,
    N_total_samples: int
):
    """
    merge coarse and fine samples.
    Parameteters
    ------------
    x: coarse samples of shape (N_rays, N_coarse, -1)
    x_is: importance samples of shape (N_rays, N_fine, -1)
    gather_idx: define how the [x, x_is] are sorted
    """
    if x is None or x.shape[-1] == 0:
        return None
    N_rays = x.shape[0]
    if x.shape[0] != x_is.shape[0]:
        return torch.cat([x, x_is], dim=0)
    x_is = torch.cat([x, x_is], dim=1)
    sh = x_is.shape
    feat_size = np.prod(sh[2:])
    if x_is.shape[1] != N_total_samples:
        # extra signal: may not have the same shape as the standard ray samples
        return x_is
    x_is = x_is.view(-1, feat_size)[gather_idxs, :]
    x_is = x_is.view(N_rays, N_total_samples, *sh[2:])

    return x_is

class ANeRF(nn.Module):

    def __init__(
        self,
        D: int,
        W: int,
        view_W: int,
        pts_embedder: DictConfig,
        pts_posi_enc: DictConfig,
        view_embedder: DictConfig,
        view_posi_enc: DictConfig,
        raycaster: DictConfig,
        skel_type: Skeleton,
        rest_pose: np.ndarray,
        skips: List[int] = [4],
        pred_sdf: bool = False,
        use_framecodes: bool = False,
        framecode_ch: int = 16,
        n_framecodes: int = 0,
        density_noise_std: float = 1.0,
        bkgd_net: Optional[DictConfig] = None,
        cam_cal: Optional[DictConfig] = None,
        pose_opt: Optional[DictConfig] = None,
        **kwargs,
    ):
        '''
        Parameters
        ----------
        D: int, depth of the MLP
        W: int, width of the MLP
        view_W: int, width of the view MLP
        pts_embedder: embedder module config, to encode 3d points w.r.t. body keypoints and poses
        pts_posi_enc: positional encoding config for the pts embedding
        view_embedder: embedder config, to encode view vectors w.r.t. body info
        view_posi_enc: positional encoding config for the view embedding
        raycast: ray casting module config
        skel_type: skeleton, define the details of the skeleton
        skips: list, layers to do skip connection
        pred_sdf: Bool, pred_sdf=True then predicts SDF value
        use_framecodes: Bool, to use framecodes for each frame
        framecode_ch: int, size of the framecode
        n_framecodes: int, number of framecodes
        density_noise_std: float, noise to apply on density during training time
        '''
        super(ANeRF, self).__init__()

        self.D = D
        self.W = W
        self.view_W = view_W
        self.skips = skips
        self.pred_sdf = pred_sdf
        self.use_framecodes = use_framecodes
        self.framecode_ch = framecode_ch
        self.n_framecodes = n_framecodes
        self.skel_type = skel_type
        self.density_noise_std = density_noise_std

        # initialize skeleton settings
        self.init_skeleton(skel_type, rest_pose)
        # instantiate embedder and network
        self.init_embedder(
            pts_embedder=pts_embedder,
            pts_posi_enc=pts_posi_enc,
            view_embedder=view_embedder,
            view_posi_enc=view_posi_enc,
            **kwargs
        )
        self.init_raycast(raycaster, **kwargs)
        self.init_density_net()
        self.init_radiance_net()
        self.init_bkgd_net(bkgd_net)

        self.cam_cal = None
        if cam_cal is not None:
            self.cam_cal = CamCal(**cam_cal)

        self.pose_opt = None
        if pose_opt is not None:
            self.pose_opt = instantiate(
                pose_opt,
                rest_pose=self.rest_pose,
                skel_type=self.skel_type,
                **kwargs
            )

    @property
    def dnet_input(self):
        return self.input_ch

    @property
    def vnet_input(self):
        if self.use_framecodes:
            return self.input_ch_view + self.framecode_ch + self.W
        return self.input_ch_view + self.W

    def init_skeleton(self, skel_type: Skeleton, rest_pose: np.ndarray):
        self.register_buffer('rest_pose', torch.tensor(rest_pose))
        self.skel_profile = get_skel_profile_from_rest_pose(rest_pose, skel_type)

    def init_embedder(
        self,
        pts_embedder: DictConfig,
        pts_posi_enc: DictConfig,
        view_embedder: DictConfig,
        view_posi_enc: DictConfig,
        **kwargs,
    ):
        N_joints = len(self.skel_type.joint_names)
        # initialize points transformation
        self.pts_embedder = instantiate(pts_embedder, N_joints=N_joints, N_dims=3)
        pts_dims, pose_dims = self.pts_embedder.dims

        # initialize positional encoding for points
        self.pts_dims = pts_dims
        self.pose_dims = pose_dims
        self.pts_posi_enc = instantiate(pts_posi_enc, input_dims=pts_dims)

        # initialize view transformation
        self.view_embedder = instantiate(view_embedder, N_joints=N_joints, N_dims=3)

        # initialize positional encoding for views (rays)
        self.view_dims = view_dims= self.view_embedder.dims
        self.view_posi_enc = instantiate(view_posi_enc, input_dims=view_dims, dist_inputs=True)

        self.input_ch = self.pts_posi_enc.dims + pose_dims
        self.input_ch_view = self.view_posi_enc.dims

    def init_raycast(
        self,
        raycast: DictConfig,
        **kwargs,
    ):
        self.raycast = instantiate(raycast, **kwargs, _recursive_=False)

    def init_density_net(self):

        W, D = self.W, self.D

        layers = [nn.Linear(self.dnet_input, W)]

        for i in range(D-1):
            if i not in self.skips:
                layers += [nn.Linear(W, W)]
            else:
                layers += [nn.Linear(W + self.dnet_input, W)]

        self.pts_linears = nn.ModuleList(layers)
        self.sigma_linear = nn.Linear(W, 1)

        if self.pred_sdf:
            self.register_parameter('B',
                nn.Parameter(torch.tensor(0.1), requires_grad=True)
            )

    def init_radiance_net(self):

        W, view_W = self.W, self.view_W

        # Note: legacy code, don't really need nn.ModuleList
        self.views_linears = nn.ModuleList([nn.Linear(self.vnet_input, view_W)])
        self.feature_linear = nn.Linear(W, view_W * 2)
        self.rgb_linear = nn.Linear(view_W, 3)

        if self.use_framecodes:
            self.framecodes = Optcodes(self.n_framecodes, self.framecode_ch)

    def init_bkgd_net(self, bkgd_net: DictConfig):
        if bkgd_net is None:
            self.bkgd_net = None
            return
        self.bkgd_net = instantiate(
            bkgd_net,
            n_framecodes=self.n_framecodes,
            framecode_ch=self.framecode_ch
        )

    def forward(self, *args, forward_type: str = 'rays', **kwargs):
        """ Forward function.

        Args:
            forward_type (str): ['rays', 'render', 'geometry']

        Returns:
            Dictionary of output tensors.
        """
        if forward_type == 'rays':
            return self.forward_rays(*args, **kwargs)
        elif forward_type == 'render':
            return self.forward_render(*args, **kwargs)
        elif forward_type =='geometry':
            return self.forward_geometry(*args, **kwargs)
        else:
            raise NotImplementedError(f'Unknown forward type {forward_type}')

    def forward_rays(self, batch: Mapping[str, Any], pose_opt: bool = False, **kwargs):

        if 'rays_o' not in batch and 'rays_d' not in batch:
            raise NotImplementedError(
                'Rays are not provided as input. '
                'For rendering image with automatically detected rays, set forward_type="render"'
            )

        # Step 1. cast ray
        bgs = batch.get('bgs', None) # background color
        sample_info = self.raycast(batch)
        pts, z_vals = sample_info['pts'], sample_info['z_vals']

        # Step 2. model evaluation
        # TODO: do we need a get_nerf_inputs function?
        network_inputs = self.get_network_inputs(batch, pts, z_vals)
        if pose_opt and self.pose_opt is not None:
            # TODO: is this a good way?
            network_inputs = self.pose_opt(
                network_inputs=network_inputs,
                kp3d=network_inputs['kp3d'],
                bones=network_inputs['bones'],
                kp_idxs=network_inputs['real_kp_idx'],
                N_unique=network_inputs['N_unique'],
            )

        raw, encoded = self.evaluate_pts(network_inputs, coarse=True)

        if raw is None:
            return self._empty_outputs(batch)

        bgnet_ret = None
        if self.training and self.bkgd_net is not None:
            bgnet_ret = self.bkgd_net(batch)
            bgs = bgnet_ret['bgs']

        # Step 3. coarse rendering
        rendered = self.raw2outputs(
            raw,
            z_vals,
            batch['rays_d'],
            bgs=bgs,
            encoded=encoded,
            batch=batch,
        )

        if self.raycast.N_importance == 0:
            return {'rendered': rendered, 'encoded': encoded}

        # Step 4. importance sampling (if importance sampling enabled)
        rendered_coarse = rendered
        encoded_coarse = encoded

        weights = rendered_coarse['weights']
        is_sample_info = self.raycast(
            batch,
            pts=pts,
            z_vals=z_vals,
            weights=weights,
            importance=True,
        )
        pts_is = is_sample_info['pts'] # only importance samples
        z_vals = is_sample_info['z_vals'] # include both coarse and importance samples

        # Step 5. model evaluation (if importance sampling enabled)
        network_inputs = self.get_network_inputs(batch, pts_is, is_sample_info['z_vals_is'])
        raw_is, encoded_is = self.evaluate_pts(network_inputs, encoded_coarse=encoded_coarse)

        # Step 6. merge coarse and importance prediction for rendering
        sorted_idxs = is_sample_info['sorted_idxs']
        N_rays = len(batch['rays_o'])
        N_total_samples = pts.shape[1] + pts_is.shape[1]
        encoded_is = merge_encodings(encoded_coarse, encoded_is, sorted_idxs, N_rays, N_total_samples)

        # Step 7. fine rendering
        if raw_is is not None:
            raw = merge_encodings({'raw': raw}, {'raw': raw_is}, sorted_idxs, N_rays, N_total_samples)['raw']
            rendered = self.raw2outputs(
                raw,
                z_vals,
                batch['rays_d'],
                bgs=bgs,
                encoded=encoded_is,
                batch=batch,
            )
        else:
            rendered = rendered_coarse

        if bgnet_ret is not None:
            rendered.update(**bgnet_ret)

        return self.collect_outputs(rendered, rendered_coarse, encoded_is, encoded_coarse)

    def forward_render(
        self,
        render_data: Mapping[str, Any],
        render_factor: float = 0,
        raychunk: int = 1024*10,
        render_normal: bool = False,
        **kwargs
    ):
        """ Render images with automatically detected rays from camera parameters.

        Args:
            render_data (Mapping[str, Any]): dictionary of rendering data, including skeleton structure and camera info
            render_factor (float): downsample factor for rendering
            raychunk (int): chunk size for ray casting
            render_normal (bool): whether to render surface normal

        Returns:
            Dictionary of rendered outputs.
        """

        assert 'bones' in render_data, 'needs know the pose/bones (bones) parameter during rendering'
        if 'kp3d' not in render_data:
            kp3d, skts = calculate_kinematic(
                self.rest_pose,
                render_data['bones'],
                render_data.get('root_locs', None),
            )
            render_data['kp3d'] = kp3d
            render_data['skts'] = skts

        H, W, focals = render_data['hwf']
        kp3d = render_data['kp3d']
        cam_poses = render_data['c2ws'] # camera-to-world
        centers = render_data.get('center', None)

        if render_factor != 0:
            # change camera setting
            H, W = H // render_factor, W // render_factor
            focals = focals / render_factor
            if centers is not None:
                centers = centers / render_factor
            bgs = render_data['bgs']
            N, _H, _W, C = bgs.shape

            bgs = rearrange(bgs, 'n h w c -> n c h w')
            bgs = F.interpolate(bgs, scale_factor=1 / render_factor, mode='bilinear', align_corners=False)
            bgs = rearrange(bgs, 'n c h w -> n h w c')
            render_data.update(bgs=bgs)

        if len(cam_poses) != len(kp3d):
            assert len(kp3d) == 1 or len(cam_poses) == 1, \
                   f'Number of body poses should either match or can be broadcasted to number of camera poses. ' \
                   f'Got {len(kp3d)} and {len(cam_poses)}'

        rgb_imgs = []
        disp_imgs = []
        alpha_imgs = []

        for i in range(len(cam_poses)):

            # Step 1. find valid rays for this camera
            center = centers[i:i+1] if centers is not None else None
            rays, valid_idxs, cyls, _ = kp_to_valid_rays(
                cam_poses[i:i+1],
                H[i:i+1].cpu().numpy(),
                W[i:i+1].cpu().numpy(),
                focals[i:i+1].cpu().numpy(),
                kps=kp3d[i:i+1],
                centers=center,
            )
            rays_o, rays_d = rays[0]
            valid_idxs = valid_idxs[0]

            # initialize images
            bg = render_data['bgs'][i].cpu()

            if valid_idxs is None or (len(valid_idxs) == 0):
                rgb_imgs.append(bg.clone())
                disp_imgs.append(torch.zeros_like(bg[..., 0]).cpu())
                continue

            # flatten to (H * W, 3)
            rgb_img = bg.clone().flatten(end_dim=1)
            disp_img = torch.zeros_like(bg[..., 0]).flatten(end_dim=1)

            # Step 2. turn them into the format that forward_rays takes
            render_data.update(cyls=cyls)
            batch = self.to_ray_inputs(rays_o, rays_d, render_data, valid_idxs, index=i)

            # Step 3. forward
            if not self.pred_sdf:
                with torch.no_grad():
                    preds = batchify_fn(self.forward_rays, batch, N_total=len(rays_o), chunk=raychunk)
            else:
                preds = batchify_fn(self.forward_rays, batch, N_total=len(rays_o),
                                    chunk=raychunk, render_normal=render_normal)
                preds = {k: v.detach() for k, v in preds.items()}

            # put the rendered values into images
            pred_rgb = preds['rgb_map'].detach()
            rgb_img[valid_idxs] = pred_rgb.cpu()
            rgb_img = rgb_img.reshape(H[i], W[i], 3)

            #pred_disp = preds['disp_map'].detach()
            # TODO: hack, fix this
            pred_disp = preds['acc_map']
            disp_img[valid_idxs] = pred_disp.cpu()
            disp_img = disp_img.reshape(H[i], W[i], 1)

            rgb_imgs.append(rgb_img)
            disp_imgs.append(disp_img)

        rgb_imgs = torch.stack(rgb_imgs, dim=0)
        disp_imgs = torch.stack(disp_imgs, dim=0)

        return {
            'rgb_imgs': rgb_imgs,
            'disp_imgs': disp_imgs
        }

    def get_network_inputs(
        self,
        batch: Mapping[str, Any],
        pts: torch.Tensor,
        z_vals: torch.Tensor,
        keys_from_batch: List[str] = [
            'kp3d', 'skts', 'bones', 'cam_idxs',
            'rays_o', 'rays_d', 'temp_kp', 'temp_bone',
            'temp_skt', 'real_kp_idx', 'real_cam_idx'],
        **kwargs
    ):
        """ Collect network inputs from the batch.

        Args:
            batch (Mapping[str, Any]): batch data
            pts (torch.Tensor): 3D points
            z_vals (torch.Tensor): z values of the points
            keys_from_batch (List[str]): specifying which keys to collect from the batch

        Returns:
            Dictionary containing network inputs.
        """
        ret = {k: batch[k] for k in keys_from_batch if k in batch}
        ret['pts'] = pts
        ret['N_unique'] = batch.get('N_unique', 1)

        if self.cam_cal is not None:
            ret = self.cam_cal(ret, z_vals)

        return ret

    def to_ray_inputs(
        self,
        rays_o: torch.Tensor,
        rays_d: torch.Tensor,
        render_data: Mapping[str, Any],
        valid_idxs: torch.Tensor,
        index: int,
        keys_for_batch: List[str] = ['kp3d', 'skts', 'bones', 'cyls', 'cam_idxs', 'bgs'],
        pixel_data: List[str] = ['bgs'],
    ):
        """ Turning ray data into the format that forward_rays takes.

        Args:
            rays_o (torch.Tensor): ray origins
            rays_d (torch.Tensor): ray directions
            render_data (Mapping[str, Any]): dictionary of rendering data
            valid_idxs (torch.Tensor): valid ray indices
            index (int): index of the current camera
            keys_for_batch (List[str]): specifying which keys to collect from the batch
            pixel_data (List[str]): list of keys that have data in (H, W, C) format

        Returns:
            Dictionary containing inputs for the forward_ray function
        """

        device = rays_o.device
        ray_inputs = {
            'rays_o': rays_o,
            'rays_d': rays_d,
            'N_unique': 1, # always one pose at a time
        }

        N_rays = len(rays_o)
        # TODO: hacky to define it here..

        for k in keys_for_batch:
            v = render_data[k]

            if len(v) > 1:
                v = v[index:index+1]

            if k not in pixel_data:
                sh = v.shape[1:]
                v = v.expand(N_rays, *sh)
            else:
                v = rearrange(v, 'b h w c -> (b h w) c')[valid_idxs.to(v.device)].to(device)
            ray_inputs[k] = v
        return ray_inputs

    def evaluate_pts(self, inputs: Mapping[str, Any], geometry_only: bool = False, geom_extra_rets: List[str] = [], **kwargs):
        """ Evaluate the NeRF function at the given inputs/points.
        """

        if self.pred_sdf:
            inputs['pts'].requires_grad = True

        # Step 1. encode all pts feature
        density_inputs, encoded_pts = self.encode_pts(inputs)
        if density_inputs is None:
            # terminate because not valid points
            return None, None

        # Step 2. density prediction
        sigma, density_feature = self.inference_sigma(density_inputs)

        if self.pred_sdf:
            surface_gradient = torch.autograd.grad(
                outputs=sigma,
                inputs=inputs['pts'],
                grad_outputs=torch.ones_like(sigma),
                create_graph=self.training,
            )[0]
            encoded_pts.update(surface_gradient=surface_gradient)

        density = self.to_density(sigma)

        if geometry_only:
            geom_ret = {'density': density, 'sigma': sigma}
            if 'valid_idxs' in encoded_pts:
                geom_ret['valid_idxs'] = encoded_pts['valid_idxs']
            if 'surface_gradient' in encoded_pts:
                geom_ret['surface_gradient'] = encoded_pts['surface_gradient']
            for k in geom_extra_rets:
                if k in encoded_pts:
                    geom_ret[k] = encoded_pts[k]
                if k == 'density_feature':
                    geom_ret[k] = density_feature
            return None, geom_ret

        # Step 3. encode all ray feature
        view_inputs, encoded_views = self.encode_views(
            inputs,
            refs=encoded_pts['pts_t'],
            encoded_pts=encoded_pts
        )

        # Step 4. rgb prediction
        rgb = self.inference_rgb(view_inputs, density_feature)

        # Step 5: create final outputs
        shape = inputs['pts'].shape[:2] # (N_rays, N_samples)
        outputs = self.fill_prediction(torch.cat([rgb, density], dim=-1), shape, encoded_pts)

        return outputs, self.collect_encoded(encoded_pts, encoded_views)

    def encode_pts(self, inputs: Mapping[str, Any]):
        """ Encode input points into encoding needed for the NeRF model.

        Args:
            inputs (Mapping[str, Any]): dictionary of inputs

        Returns:
            density_inputs (torch.Tensor): density network inputs
            encoded (Mapping[str, torch.Tensor]): detail encoding of the points
        """

        N_rays, N_samples = inputs['pts'].shape[:2]

        encoded = self.pts_embedder(**inputs)
        v_pe = self.pts_posi_enc(encoded['v'])[0]
        r = encoded['r'].reshape(N_rays, N_samples, self.pose_dims)

        # apply positional encoding (PE)
        # pe_fn returns a tuple: (encoded outputs, cutoff weights)
        density_inputs = torch.cat([v_pe, r], dim=-1).flatten(end_dim=1)

        return density_inputs, encoded

    def encode_views(self, inputs: Mapping[str, Any], refs: torch.Tensor, encoded_pts: Mapping[str, torch.Tensor]):
        """ Encode input views into encoding needed for the NeRF model.

        Args:
            inputs (Mapping[str, Any]): dictionary of inputs
            refs (torch.Tensor): reference tensor for expanding rays
            encoded_pts (Mapping[str, Any]): point encoding that could be useful for encoding view

        Returns:
            view_inputs (torch.Tensor): view network inputs
            encoded (Mapping[str, Any]): detail encoding of the views
        """

        N_rays, N_samples = refs.shape[:2]
        encoded = self.view_embedder(**inputs, refs=refs)
        d = encoded['d'].reshape(N_rays, N_samples, self.view_dims)

        # apply positional encoding (PE)
        d_pe = self.view_posi_enc(d, dists=encoded_pts.get('v', None))[0]

        view_inputs = d_pe
        if self.use_framecodes:
            N_rays, N_samples = refs.shape[:2]
            # expand from (N_rays, ...) to (N_rays, N_samples, ...)
            cam_idxs = inputs['cam_idxs']
            cam_idxs = cam_idxs.reshape(N_rays, 1, -1).expand(-1, N_samples, -1)
            framecodes = self.framecodes(cam_idxs.reshape(N_rays * N_samples, -1))
            framecodes = framecodes.reshape(N_rays, N_samples, -1)
            view_inputs = torch.cat([view_inputs, framecodes], dim=-1)

        if 'normal' in encoded_pts and self.pred_sdf:
            normal = encoded_pts['normal']
            view_inputs = torch.cat([view_inputs, normal], dim=-1)

        view_inputs = view_inputs.flatten(end_dim=1)
        if 'valid_idxs' in encoded_pts:
            view_inputs = view_inputs[encoded_pts['valid_idxs']]

        return view_inputs, encoded

    def fill_prediction(self, preds: torch.Tensor, shape: Union[List, tuple], valid_info: Mapping[str, Any]):
        """ Create a full tensor from valid prediction.
        In evaluate_pts, we may avoid computation on some points that do not require prediction.
        The preds tensor shape is thus varying. This function turn the prediction into fixed size
        so that the prediction can be processed with batch operation.

        Args:
            preds (torch.Tensor): prediction tensor
            shape (Union[List, tuple]): shape of the first dimension in preds
            valid_info (Mapping[str, Any]): information that requires to map preds back to a tensor of shape [*shape, pred_size]

        Returns:
            outputs (torch.Tensor): tensor of shape [*shape, pred_size]
        """
        output_dim = preds.shape[-1]

        if valid_info is not None and 'valid_idxs' in valid_info:
            valid_idxs = valid_info['valid_idxs']
            outputs = torch.zeros(np.prod(shape), output_dim)
            # by default force nothing there in the empty space

            # outputs[invalid_idxs, -1] = 0.
            outputs[valid_idxs] += preds
            outputs = outputs.reshape(*shape, output_dim)
        else:
            outputs = preds.reshape(*shape, output_dim)

        return outputs

    def inference_sigma(self, density_inputs: torch.Tensor):
        """ Inference the geometry representation sigma (density or sdf) values
        """
        h = self.forward_density(density_inputs)
        sigma = self.sigma_linear(h)
        return sigma, h

    def inference_rgb(self, view_inputs: torch.Tensor, density_feature: torch.Tensor, rgb_eps: float = 0.001, **kwargs):
        """ Inference the color for the given input points
        """
        rgb = self.forward_view(view_inputs, density_feature)
        rgb = torch.sigmoid(rgb) * (1 + 2 * rgb_eps) - rgb_eps
        return rgb

    def forward_density(self, density_inputs: torch.Tensor):
        h = density_inputs

        for i, l in enumerate(self.pts_linears):
            h = self.pts_linears[i](h)
            h = F.relu(h, inplace=True)
            if i in self.skips:
                h = torch.cat([density_inputs, h], -1)
        return h

    def forward_view(self, view_inputs: torch.Tensor, density_feature: torch.Tensor):
        # produce features for color/radiance
        feature = self.feature_linear(density_feature)
        h = torch.cat([feature, view_inputs], -1)

        for i, l in enumerate(self.views_linears):
            h = self.views_linears[i](h)
            h = F.relu(h, inplace=True)

        return self.rgb_linear(h)

    def forward_geometry(
        self,
        render_data: Mapping[str, Any],
        chunk: int = 1024*64*5,
        return_gradient: bool = False,
        geom_extra_rets: List[str] = [],
        **kwargs
    ):
        """ Forward function for geometry only.

        Args:
            render_data (Mapping[str, Any]): dictionary of rendering data
            chunk (int): chunk size for forward to avoid OOM
            return_gradient (bool): whether to return gradient
            geom_extra_rets (List[str]): list of keys of extra tensors to return

        Returns:
            Dictionary of outputs.
        """

        assert 'pts' in render_data, f'Query locations need to be provided for forward_geometry'
        if 'kp3d' not in render_data:
            assert self.skel_type == SMPLSkeleton, f'Only SMPLSkeleton is supported for kinematic forward!'
            kp3d, skts = calculate_kinematic(
                self.rest_pose,
                render_data['bones'],
                render_data.get('root_locs', None),
            )
            render_data['skts'] = skts
            render_data['kp3d'] = kp3d

        pts = render_data['pts']
        kp3d = render_data['kp3d']
        skts = render_data['skts']
        bones = render_data['bones']

        # only deals with one pose at a time
        assert len(kp3d) == 1
        assert len(skts) == 1
        assert len(bones) == 1

        pts_shape = pts.shape
        pts = pts.reshape(1, -1, 3)
        N_samples = pts.shape[1]

        # pre-allocate
        density = torch.zeros(N_samples, 1)
        sigma = torch.zeros(N_samples, 1)
        extra_dict = {k: None for k in geom_extra_rets}
        if self.pred_sdf:
            sigma += 10.
        gradient = None
        if return_gradient:
            gradient = torch.zeros(N_samples, 3)
        for i in range(0, N_samples, chunk):
            chunk_pts = pts[:, i:i+chunk]
            geom_inputs = {
                'kp3d': kp3d,
                'skts': skts,
                'bones': bones,
                'N_unique': 1,
                'pts': chunk_pts,
            }
            _, preds = self.evaluate_pts(
                geom_inputs,
                geometry_only=True,
                geom_extra_rets=geom_extra_rets,
            )
            if preds is None:
                continue
            preds = {k: v.detach() for k, v in preds.items()}
            if 'valid_idxs' in preds:
                density[i + preds['valid_idxs']] = preds['density']
                sigma[i + preds['valid_idxs']] = preds['sigma']
                if gradient is not None:

                    pred_gradient = preds['surface_gradient'].flatten(end_dim=1)
                    pred_gradient = pred_gradient[preds['valid_idxs']]
                    gradient[i + preds['valid_idxs']] = pred_gradient

                for k in geom_extra_rets:
                    extra_tensor = preds[k]
                    if extra_dict[k] is None:
                        extra_dict[k] = torch.zeros(N_samples, extra_tensor.shape[-1])
                    extra_dict[k][i + preds['valid_idxs']] = extra_tensor
            else:
                density[i:i+chunk] = preds['density']
                sigma[i:i+chunk] = preds['sigma']
                if gradient is not None:
                    gradient[i:i+chunk] = preds['surface_gradient'].flatten(end_dim=1)

                for k in geom_extra_rets:
                    extra_tensor = preds[k]
                    if extra_dict[k] is None:
                        extra_dict[k] = torch.zeros(N_samples, extra_tensor.shape[-1])
                    extra_dict[k][i:i+chunk] = extra_tensor

        outputs = {
            'density': density.reshape(*pts_shape[:-1], 1),
            'sigma': sigma.reshape(*pts_shape[:-1], 1),
            **extra_dict,
        }

        if gradient is not None:
            outputs.update(gradient=gradient.reshape(*pts_shape[:-1], 3))
        return outputs

    def collect_encoded(self, encoded_pts: Mapping[str, torch.Tensor], encoded_views: Mapping[str, torch.Tensor]):
        """ Collect encodings that are useful for loss computations or rendering
        """
        ret = {}
        if self.pred_sdf and 'surface_gradient' in encoded_pts:
            ret['surface_gradient'] = encoded_pts['surface_gradient']
        return ret

    def raw2outputs(
        self,
        raw: torch.Tensor,
        z_vals: torch.Tensor,
        rays_d: torch.Tensor,
        bgs: Optional[torch.Tensor] = None,
        **kwargs
    ):
        # TODO: update
        """Transforms model's predictions to semantically meaningful values.
        Args:
            raw: [num_rays, num_samples along ray, 4]. Prediction from model.
            z_vals: [num_rays, num_samples along ray]. Integration time.
            rays_d: [num_rays, 3]. Direction of each ray.
            bgs: [num_rays, 3]. Background color
            act_fn: activation function for the density
        Returns:
            rgb_map: [num_rays, 3]. Estimated RGB color of a ray.
            disp_map: [num_rays]. Disparity map. Inverse of depth map.
            acc_map: [num_rays]. Sum of weights along each ray.
            weights: [num_rays, num_samples]. Weights assigned to each sampled color.
            depth_map: [num_rays]. Estimated distance to object.
        """
        if self.pred_sdf:
            return self.raw2sdfoutputs(raw, z_vals, rays_d, bgs=bgs, **kwargs)

        #raw2alpha = lambda raw, dists, noise, act_fn=act_fn: 1.-torch.exp(-(act_fn(raw + noise))*dists)

        dists = z_vals[...,1:] - z_vals[...,:-1]
        dists = torch.cat([dists, torch.Tensor([1e10]).expand(dists[...,:1].shape)], -1)  # [N_rays, N_samples]

        # Multiply each distance by the norm of its corresponding direction ray
        # to convert to real world distance (accounts for non-unit directions).
        dists = dists * torch.norm(rays_d[...,None,:], dim=-1)

        rgb = raw[..., :3]

        alpha = 1. - torch.exp(-raw[..., 3] * dists)

        # weights = alpha * tf.math.cumprod(1.-alpha + 1e-10, -1, exclusive=True)
        # sum_{i=1 to N samples} prob_of_already_hit_particles * alpha_for_i * color_for_i
        # C(r) = sum [T_i * (1 - exp(-sigma_i * delta_i)) * c_i] = sum [T_i * alpha_i * c_i]
        # alpha_i = 1 - exp(-sigma_i * delta_i)
        # T_i = exp(sum_{j=1 to i-1} -sigma_j * delta_j) = torch.cumprod(1 - alpha_i)
        # standard NeRF
        weights = alpha * torch.cumprod(torch.cat([torch.ones((alpha.shape[0], 1)), 1.-alpha + 1e-10], -1), -1)[:, :-1]
        rgb_map = torch.sum(weights[...,None] * rgb, -2)  # [N_rays, 3]

        depth_map = torch.sum(weights * z_vals, -1)
        disp_map = 1./torch.max(1e-10 * torch.ones_like(depth_map), depth_map / (torch.sum(weights, -1)  + 1e-10))

        invalid_mask = torch.ones_like(disp_map)
        invalid_mask[torch.isclose(weights.sum(-1), torch.tensor(0.))] = 0.
        disp_map = disp_map * invalid_mask

        acc_map = torch.minimum(torch.sum(weights, -1), torch.tensor(1.))

        if bgs is not None:
            rgb_map = rgb_map + (1. - acc_map)[..., None] * bgs

        return {'rgb_map': rgb_map, 'disp_map': disp_map, 'acc_map': acc_map,
                'weights': weights, 'alpha': alpha}

    def raw2sdfoutputs(
        self,
        raw: torch.Tensor,
        z_vals: torch.Tensor,
        rays_d: torch.Tensor,
        bgs: Optional[torch.Tensor] = None,
        **kwargs
    ):
        """ Sligtly different from usual case
        """

        dists = z_vals[...,1:] - z_vals[...,:-1]
        dists = torch.cat([dists, torch.Tensor([1e10]).expand(dists[...,:1].shape)], -1)  # [N_rays, N_samples]

        # Multiply each distance by the norm of its corresponding direction ray
        # to convert to real world distance (accounts for non-unit directions).
        dists = dists * torch.norm(rays_d[...,None,:], dim=-1)

        # following Lior's paper
        density = raw[..., 3]
        # note: this is in logspace, originally we do exp(-density*dists)
        free_energy = density * dists
        shifted_free_energy = torch.cat([torch.zeros(dists.shape[0], 1), free_energy[:, :-1]], dim=-1)  # shift one step

        # weights = alpha * tf.math.cumprod(1.-alpha + 1e-10, -1, exclusive=True)
        # sum_{i=1 to N samples} prob_of_already_hit_particles * alpha_for_i * color_for_i
        # C(r) = sum [T_i * (1 - exp(-sigma_i * delta_i)) * c_i] = sum [T_i * alpha_i * c_i]
        # alpha_i = 1 - exp(-sigma_i * delta_i)
        # T_i = exp(sum_{j=1 to i-1} -sigma_j * delta_j) = torch.cumprod(1 - alpha_i)
        # standard NeRF
        alpha = 1 - torch.exp(-free_energy)  # probability of it is not empty here
        T = torch.exp(-torch.cumsum(shifted_free_energy, dim=-1))  # probability of everything is empty up to now
        weights = alpha * T # probability of the ray hits something here
        #weights = alpha * torch.cumprod(torch.cat([torch.ones((alpha.shape[0], 1)), 1.-alpha + 1e-10], -1), -1)[:, :-1]

        # get rgb (as usual)
        rgb = raw[..., :3] # [N_rays, N_samples, 3]
        rgb_map = torch.sum(weights[...,None] * rgb, -2)  # [N_rays, 3]

        eps = 1e-10
        inv_eps = 1. / eps
        depth_map = torch.sum(weights * z_vals, -1)
        disp_map = (weights.sum(dim=-1) / depth_map).double()
        disp_map = torch.where(
            (disp_map > 0) & (disp_map < inv_eps) & (weights.sum(dim=-1) > eps), disp_map, inv_eps
        ).float()
        encoded = kwargs['encoded']
        """
        disp_map = 1./torch.max(eps * torch.ones_like(depth_map), depth_map / (torch.sum(weights, -1)  + eps))

        invalid_mask = torch.ones_like(disp_map)
        invalid_mask[torch.isclose(weights.sum(-1), torch.tensor(0.))] = 0.
        disp_map = disp_map * invalid_mask
        """

        acc_map = torch.minimum(torch.sum(weights, -1), torch.tensor(1.))

        if bgs is not None:
            rgb_map = rgb_map + (1. - acc_map)[..., None] * bgs

        return {'rgb_map': rgb_map, 'disp_map': disp_map, 'acc_map': acc_map,
                'weights': weights, 'alpha': alpha}

    def collect_outputs(
        self,
        ret: Mapping[str, torch.Tensor],
        ret0: Optional[Mapping[str, torch.Tensor]] = None,
        encoded: Optional[Mapping[str, torch.Tensor]] = None,
        encoded0: Optional[Mapping[str, torch.Tensor]] = None
    ):
        """ Collect outputs into a dictionary for loss computation/rendering

        Parameter
        ---------
        ret: dictionary of the fine-level rendering
        ret0: dictionary of the coarse-level rendering
        encoded: dictionary of the fine-level model info
        encoded0: dictionary of the corase-level model info

        """

        collected = {'rgb_map': ret['rgb_map'], 'disp_map': ret['disp_map'],
                     'acc_map': ret['acc_map'],}
        if not self.training:
            return collected

        collected['T_i'] = ret['weights']
        collected['alpha'] = ret['alpha']

        if ret0 is not None:
            collected['rgb0'] = ret0['rgb_map']
            collected['disp0'] = ret0['disp_map']
            collected['acc0'] = ret0['acc_map']
            collected['alpha0'] = ret0['alpha']

        if 'j_dists' in ret and self.training:
            collected['j_dists'] = ret['j_dists']
            if ret0 is not None:
                collected['j_dists0'] = ret0['j_dists']

        if encoded is not None and 'surface_gradient' in encoded and self.training:
            collected['surface_gradient'] = encoded['surface_gradient']

            if encoded0 is not None:
                collected['surface_gradient0'] = encoded0['surface_gradient']

        if self.training and 'bgs' in ret:
            collected['bgs'] = ret['bgs']
            collected['bg_preds'] = ret['bg_preds']

        return collected

    def _empty_outputs(self, batch: Mapping[str, Any]):
        N_rays = len(batch['rays_o'])
        empty = torch.zeros(N_rays, 3)
        rgb_empty = batch.get('bgs', empty)
        return {'rgb_map': rgb_empty, 'disp_map': empty[:, 0], 'acc_map': empty[:, 0]}

    def to_density(self, sigma: torch.Tensor):
        """ Turn sigma from raw prediction (value unbounded) to density (value >= 0.)
        """
        if self.pred_sdf:
            return self.sdf2density(sigma)

        noise = 0.
        if self.training and self.density_noise_std > 0.:
            noise = torch.randn_like(sigma) * self.density_noise_std

        sigma = sigma + noise

        return F.relu(sigma, inplace=True)

    def sdf2density(self, sdf: torch.Tensor):
        # use laplace CDF to represent density, with sdf being pdf
        B = (self.B.abs() + 0.0001)
        inv_B = 1 / B
        return inv_B * (0.5 + 0.5 * sdf.sign() * torch.expm1(-sdf.abs() / B))


from django.contrib.auth.decorators import login_required
from django.urls import path, include
from django.views.generic import TemplateView

from travelling_companion import views

app_name = 'travelling_companion'
urlpatterns = [
    path('', views.TripListView.as_view(), name='home'),

    path('trip/<int:pk>', views.TripDetailView.as_view(), name='trip_detail'),
    path('trip/delete/<int:pk>', views.TripDeleteView.as_view(), name='trip_delete'),
    path('trip/create', views.TripCreateView.as_view(), name='trip_create'),
    path('trip/update/<int:pk>', views.TripUpdateView.as_view(), name='trip_update'),

    path('trip/<int:trip_id>/cost', views.CostView.as_view(), name='cost_form'),
    path('trip/<int:trip_id>/cost/<int:pk>', views.CostEditView.as_view(), name='cost_edit_form'),
    path('trip/cost/delete/<int:pk>', views.CostDelete.as_view(), name='cost_delete'),

    path('trip/<int:trip_id>/flight', views.FlightView.as_view(), name='flight_form'),
    path('trip/<int:trip_id>/flight/<int:pk>', views.FlightEditView.as_view(), name='flight_edit_form'),
    path('trip/flight/delete/<int:pk>', views.FlightDelete.as_view(), name='flight_delete'),

    path('trip/<int:trip_id>/accommodation', views.AccommodationView.as_view(), name='accommodation_form'),
    path('trip/<int:trip_id>/accommodation/<int:pk>', views.AccommodationEditView.as_view(), name='accommodation_edit'),
    path('trip/accommodation/delete/<int:pk>', views.AccommodationDelete.as_view(), name='accommodation_delete'),

    path('trip/accept/<int:pk>', views.AccepteTrip.as_view(), name='trip_accept'),
    path('trip/approve/<int:pk>', views.ApproveTrip.as_view(), name='trip_approve'),
    path('trip/<int:pk>/add_person/<int:person_id>', views.AddPersonTrip.as_view(), name='trip_add_person'),
    path('trip/remove_person/<int:pk>', views.RemovePersonTrip.as_view(), name='trip_remove_person'),

    path('location/', views.LocationView.as_view(), name='location_form'),
    path('location/<int:pk>', views.LocationEditView.as_view(), name='location_edit_form'),
    path('location/delete/<int:pk>', views.LocationDelete.as_view(), name='location_delete'),

    path('person/', views.PersonListView.as_view(), name='person_list'),
    path('person/delete/<int:pk>', views.PersonDeleteView.as_view(), name='person_delete'),
    path('person/create', views.PersonCreateView.as_view(), name='person_create'),
    path('person/update/<int:pk>', views.PersonUpdateView.as_view(), name='person_update'),

    path('passport/create', views.PassportCreateView.as_view(), name='passport_create'),

]


# Import packages
import numpy as np

import json
import torch
from torch import nn, optim
from torchvision import datasets, transforms, models

from workspace_utils import active_session
from helper_functions import train_parser, load_dataset, define_transforms
from model_functions import DeepNetworkClassifier, train, test

# Parse arguments from command line
arg_dict = train_parser()

# Load the data
data_directory = arg_dict['data_dir']
train_dir, valid_dir, test_dir = load_dataset(data_directory)

# Define transforms
train_transforms = define_transforms(train=True)
validate_transforms = define_transforms(train=False)
test_transforms = define_transforms(train=False)

# Load the datasets with ImageFolder
train_dataset = datasets.ImageFolder(train_dir, transform=train_transforms)
validate_dataset = datasets.ImageFolder(valid_dir, transform=validate_transforms)
test_dataset = datasets.ImageFolder(test_dir, transform=test_transforms)

# Using the image datasets and the transforms, define the dataloaders
trainloader = torch.utils.data.DataLoader(train_dataset, batch_size = 32, shuffle=True)
validloader = torch.utils.data.DataLoader(validate_dataset, batch_size = 32, shuffle=True)
testloader = torch.utils.data.DataLoader(test_dataset, batch_size = 32, shuffle=True)

# Create a dictionary of torchvision models to choose from
print('Downloading models...')
arch_dict = {'vgg19_bn': models.vgg19_bn(pretrained=True),
             'alexnet': models.alexnet(pretrained=True)}

# Create a dictionary of input sizes for each of the available torchvision models
arch_input_dict = {'vgg19_bn': 25088,
                   'alexnet': 9216}

arch=arg_dict['arch']

# Create an instance of the user-specified pretrained network
model = arch_dict[arch]

# Identify the device on which the model will be trained
GPU = arg_dict['gpu']
if GPU:
    device = torch.device('cuda')
else:
    device = torch.device('cpu')

# Turn off gradient calculations for model parameters
for param in model.parameters():
    param.requires_grad = False

# Build the model classifier
hidden_units = arg_dict['hidden_units']
classifier = DeepNetworkClassifier(arch_input_dict[arch], 102, hidden_units)

model.classifier = classifier

# Add the NLLLoss criterion (since LogSoftmax is called in the classifier)
criterion = nn.NLLLoss()

# Define the optimizer
learning_rate = arg_dict['learning_rate']
optimizer = optim.Adam(model.classifier.parameters(), lr=learning_rate)

# Send the model to the device
model.to(device);

# Train the network
epochs = arg_dict['epochs']

if device == torch.device('cuda'):
    # Keep the workspace active
    with active_session():
        print('Begin training...')
        train(model, trainloader, validloader, criterion, optimizer, epochs, device)
else:
    print('Begin training...')
    train(model, trainloader, validloader, criterion, optimizer, epochs, device)

# Test the network
test(model, testloader, device)

# Save the checkpoint
checkpoint = {'arch':arch,
              'input_size': arch_input_dict[arch],
              'output_size': 102,
              'hidden_layers': hidden_units,
              'state_dict': model.state_dict(),
              'epochs': epochs,
              'optimizer_state_dict': optimizer.state_dict(),
              'class_to_idx':train_dataset.class_to_idx}

save_dir=arg_dict['save_dir']
torch.save(checkpoint, save_dir)
print('Model saved to', save_dir)


import json
import time
import traceback

import requests
from pymongo import ReadPreference

from actions.applet.utils import deal_param
from db import STATUS_REDPACKET_AWARDED, STATUS_READPACKET_ISSUE_SUCCESS, CATEGORY_NOTICE_AWARD, STATUS_NOTICE_UNREAD, \
    STATUS_NOTICE_READ, STATUS_REDPACKET_AWARD_FAILED, STATUS_READPACKET_ISSUE_FAIL
from db.models import RedPacketBox, Race, Member, MemberNotice
from logger import log_utils
from settings import REDPACKET_PLATFORM_QUERY_RESULT_URL, REDPACKET_PLATFORM_HOST
from tasks import app

logger = log_utils.get_logging('lottery_result')

@app.task(bind=True, queue='lottery_result')
def get_lottery_result(self, member: Member, reward_cid, rid):
    """
    获取到小程序抽奖结果
    :param self:
    :param member:
    :param reward_cid:
    :param rid:
    :return:
    """
    logger.info('START(%s): lottery result query, member_cid=%s, reward_cid=%s, rid=%s ' % (
        self.request.id, member.cid, reward_cid, rid))
    try:
        redpacket = RedPacketBox.sync_find_one({'member_cid': member.cid, 'cid': reward_cid, 'record_flag': 1},
                                               read_preference=ReadPreference.PRIMARY)

        race = Race.sync_get_by_cid(redpacket.race_cid)
        time_start = time.time()
        while time.time() - time_start < 30:
            time.sleep(1)

            json_data = json.dumps(
                deal_param(REDPACKET_PLATFORM_QUERY_RESULT_URL, redpkt_account=race.redpkt_account, rid=rid))
            res = requests.get(REDPACKET_PLATFORM_HOST + REDPACKET_PLATFORM_QUERY_RESULT_URL, data=json_data)
            res = res.json()
            status = res.get('data')[0].get('status')

            if status == 2:
                logger.warning('[RedPacket Drawing] the redpacket info: '
                               'member_cid: %s, checkpoint_cid: %s.' % (member.cid, redpacket.checkpoint_cid))

            if status == 3:
                redpacket.draw_status = STATUS_REDPACKET_AWARDED
                redpacket.issue_status = STATUS_READPACKET_ISSUE_SUCCESS

                redpacket.sync_save()

                notice_list = MemberNotice.sync_find(
                    {'member_cid': member.cid, 'category': CATEGORY_NOTICE_AWARD, 'status': STATUS_NOTICE_UNREAD,
                     'checkpoint_cid': redpacket.checkpoint_cid, 'record_flag': 1}).to_list(None)

                for notice in notice_list:
                    notice.status = STATUS_NOTICE_READ
                    notice.sync_save()
                logger.info(
                    '[RedPacket Drew] the redpacket info: member_cid: %s, checkpoint_cid: %s.' % (member.cid,
                                                                                redpacket.checkpoint_cid))
                break

            if status == 5:
                redpacket.status = STATUS_REDPACKET_AWARD_FAILED
                redpacket.issue_status = STATUS_READPACKET_ISSUE_FAIL
                redpacket.sync_save()
                break

            if status == 6:
                logger.warning(
                    '[RedPacket Waiting] the redpacket info: member_cid: %s, checkpoint_cid: %s.' % (member.cid,
                                                                                redpacket.checkpoint_cid))

            if status == 8:
                pass

    except Exception:
        logger.error(str(res.content))
        logger.error(traceback.format_exc())
    logger.info(' END (%s): lottery result query, member_cid=%s, reward_cid=%s, rid=%s ' % (
        self.request.id, member.cid, reward_cid, rid))


# 处理raw_data文件，得到可以用于后续训练的文件
# 目前是第一阶段，只预测罪名
# 导入使用的库
from helper import *
import os, json, time, jieba, re
import matplotlib.pyplot as plt
import multiprocessing
import numpy as np
from sklearn.datasets.base import Bunch
from sklearn.externals import joblib
from sklearn.preprocessing import MultiLabelBinarizer

def process(raw_data_file, trainbunch_file, testbunch_file, split_value):
    '''
    此函数为控制整个处理过程的函数
    输入为原始文件，以及处理后要储存的bunch文件，还有训练集和测试集的分割比例
    函数没有返回值，最终将处理后的bunch写入文件为止
    '''
    databunch = Bunch(contents=[], accu=[])  # databunch 储存对raw_data_file处理后的数据，然后再shuffle成训练集和测试集。
    # contents储存的是处理后的fact， accu储存的是汉字的犯罪标签，new_accu储存的二值化的犯罪标签。
    print("正在写入数据》》》》"); st = time.time()
    with open(raw_data_file, "r", encoding="utf-8") as rawdata_file:
        file_lenth = len(rawdata_file.readlines()); cnt = 0    #  用于记录数据处理进度
    rawdata_file = open(raw_data_file, "r", encoding="utf-8")
    for line_json in rawdata_file.readlines():   # line_json中包含着我们所需的所有信息
        line = json.loads(line_json)             # line是一个字典
        fact = line["fact"]
        accusation = line["meta"]["accusation"]
        new_fact = process_fact(fact)   # new_fact是处理后的犯罪事实，保留了中文, 然后用jieba分词得到最终的结果。
        databunch.contents.append(new_fact)
        databunch.accu.append(accusation)
        cnt += 1
        if cnt%1000 == 0:
            print("写入数据进度为：{:.3f}%".format(cnt/file_lenth*100))
    rawdata_file.close(); et = time.time()
    print("写入数据完毕！！！！用时：{:.3f}s".format(et-st))
    print("正在处理数据》》》》"); st = time.time()
    # new_fact和accu已经写入bunch, 将databunch打乱顺序, 分为trainbunch和testbunch
    random_seed = np.random.randint(0,100)  # 保证databunch中各个内容的shuffle次序是一样的。
    for i in databunch.keys():
        np.random.seed(random_seed)
        np.random.shuffle(databunch[i])
    trainbunch, testbunch = split_train_test(databunch, split_value)
    # 对accu二值化
    multilabelbinarizer = MultiLabelBinarizer(accu_list)
    trainbunch.new_accu = multilabelbinarizer.fit_transform(trainbunch.accu)
    testbunch.new_accu = multilabelbinarizer.transform(testbunch.accu)
    # 将trainbunch, testbunch写入trainbunch_file, testbunch_file
    joblib.dump(trainbunch, trainbunch_file)
    joblib.dump(testbunch, testbunch_file)
    et = time.time()
    print("处理数据完毕！！！！用时：{:.3f}s".format(et - st))

def process_fact(fact):
    new_fact = []
    pattern = re.compile(u"[\u4e00-\u9fa5]+")
    seg_fact = re.findall(pattern=pattern, string=fact)  # 此时的fact为去掉非汉字的列表，['昌宁县人民检察院指控', '年', '月', '日下午', '时许', '被告人段某驾拖车经过鸡飞乡澡塘街子', '时逢堵车', '段某将车停在', '冰凉一夏', '冷饮店门口', '被害人王某的侄子王', '某示意段某靠边未果', '后上前敲打车门让段某离开', '段某遂驾车离开', '但对此心生怨愤', '同年', '月', '日', '时许', '被告人段某酒后与其妻子王', '某一起准备回家', '走到鸡飞乡澡塘街富达通讯手机店门口时停下', '段某进入手机店内对被害人王某进行吼骂', '紧接着从手机店出来拿得一个石头又冲进手机店内朝王某头部打去', '致王某右额部粉碎性骨折', '右眼眶骨骨折', '经鉴定', '被害人王某此次损伤程度为轻伤一级']
    for sub_sentence in seg_fact:
        new_fact.extend([word for word in jieba.cut(sub_sentence)])
    return new_fact

def shuffle_bunch(databunch):
    new_databunch = Bunch()
    for i in databunch.keys():
        np.random.shuffle(databunch[i])
    return new_databunch

def split_train_test(new_databunch, split_value):
    lenth = len(new_databunch.contents) # 数据总量
    split_point = int((1-split_value)*lenth)
    trainbunch = Bunch(); testbunch = Bunch()
    trainbunch.contents = new_databunch.contents[:split_point]
    testbunch.contents = new_databunch.contents[split_point:]
    trainbunch.accu = new_databunch.accu[:split_point]
    testbunch.accu = new_databunch.accu[split_point:]
    return trainbunch, testbunch

if __name__ == '__main__':
    process(RAW_SMALL_DATA, SMALL_TRAINBUNCH_FILE, SMALL_TESTBUNCH_FILE, split_value=0.2)

'''
        for word in line:
            if word not in stopwordslist and len(word)>1 and "县" not in word and "市" not in word and "区" not in word \
                    and "省" not in word and "自治区" not in word and "村" not in word and "镇" not in word \
                    and "营" not in word:
                 newline += word + " "
        if len(newline) > 10:
            newline = newline + "\n"
            o.writelines(newline)
    # 生成相关参数
    accudict, stopwordslist, train_data_list, test_data_list = gen_parameters()

    # 读取JSON文件
    st = time.time()
    writeData(TRAIN_DATA_JSON, TRAIN_DATA, accudict)
    writeData(TEST_DATA_JSON, TEST_DATA, accudict)
    et = time.time()
    time0 = et - st
    print("写入文件用时： {}s".format(str(time0)))

    # 生成分词后文件
    st = time.time()
    pool = multiprocessing.Pool(processes=12)
    for i in train_data_list:
        pool.apply_async(segData, (i,))
    pool.close()
    pool.join()
    pool = multiprocessing.Pool(processes=12)
    for i in test_data_list:
        pool.apply_async(segData, (i,))
    pool.close()
    pool.join()
    et = time.time()
    time0 = et - st
    print("分词用时： {}s".format(str(time0)))

    # 生成process文件,去掉停用词并且过滤掉长度为1的词, 去掉句子长度小于10的句子
    st = time.time()
    pool = multiprocessing.Pool(processes=12)
    for i in train_data_list:
        pool.apply_async(processData, (i,))
    pool.close()
    pool.join()
    pool = multiprocessing.Pool(processes=12)
    for i in test_data_list:
        pool.apply_async(processData, (i,))
    pool.close()
    pool.join()
    et = time.time()
    time0 = et - st
    print("去停用词用时： {}s".format(str(time0)))
    '''


from collections import defaultdict

class Solution:
    """
    This is a variation of longest common substring, we can solve it similar to LCS but space used will
    be massive.

    Also can solve this using Trie but is not most efficient

    1. Binary search on (1, len(s)) to find the longest repeating substring. If match found then try to find longer else find shorter. This will be our window

    2. Using Rabin-Karp hashing algo, find pattern of length = window

    3. Keep going until longest is found
    """

    def pattern_match(self, s: str, length: int, computed_array: "list[int]") -> bool:
        """
        use rabin karp algo to find patterns of len length and return true / false depending on
        whether there was a match or not

        'baa' = ( ( ( 2 * 26**2 ) % p + ( 1 * 26**1 ) ) % p + ( 1 * 26 ** 0 ) ) % p
        """

        hash_map = defaultdict(list)

        mod = 10000001

        hash_ = 0

        # calculate hash for the first window
        for i in range(length):
            hash_ = (hash_ * 26 + (ord(s[i]) - ord("a"))) % mod

        hash_map[hash_].append(0)

        # computing rolling hash here, so no need to keep adding hash
        for i in range(length, len(s)):
            hash_ = (
                (hash_ - computed_array[length - 1] * (ord(s[i - length]) - ord("a")))
                % mod
                + mod
            ) % mod

            # left shifting the hash
            hash_ = (hash_ * 26 + (ord(s[i]) - ord("a")) * 26 ** 0) % mod

            if hash_ in hash_map:
                for index in hash_map[hash_]:
                    string1 = s[index : index + length]
                    string2 = s[i - length + 1 : i + 1]

                    if string1 == string2:
                        return (True, string1)

            hash_map[hash_].append(i - length + 1)

        print(hash_map)

        return (False, "")

    def longestDupSubstring(self, s: str) -> str:
        if len(s) == 0:
            return ""

        low = 1
        high = len(s)
        longest_str = ""

        # {pattern of length n: [ indicies where the patterns start ]}
        computed_array = [26 ** i for i in range(len(s))]

        """
        use binary search so that if string is of len 10, then we try to find longest repeating substring of length
        5, and if that exists then we try to find LRS of length > 5 and < 10 and so on
        """

        while low <= high:  # needs to be low <= high
            middle = (low + high) // 2

            match, string = self.pattern_match(s, middle, computed_array)

            if match:
                low = middle + 1
                longest_str = max(longest_str, string, key=lambda x: len(x))
            else:
                high = middle - 1

        return longest_str

def rabin_karp():
    s = "aaaaaa"

    h = 0

    for (index, char) in enumerate(s[:3]):
        h += 26 ** (2 - index) + (ord(s[index]) - ord("a") + 1)
        print(h)

    print(h)

print(
    Solution().longestDupSubstring(
        "nyvzwttxsshphczjjklqniaztccdrawueylaelkqtjtxdvutsewhghcmoxlvqjktgawwgpytuvoepnyfbdywpmmfukoslqvdrkuokxcexwugogcwvsuhcziwuwzfktjlhbiuhkxcreqrdbj"
    )
)

# rabin_karp()


"""
Created on 9 Oct 2015

@author: Filip Lindau
"""

import numpy as np
import Raytracer.Ray as Ray
import matplotlib.pyplot as mpl
import logging

class RaySource(object):
    def __init__(self, num_rays=1):
        """ Defines a ray source with a set of rays that can be traced.
        Contains a list of rays.

        Input
        numRays: Number of rays in the list
        """
        self.num_rays = num_rays
        self.rays = None
        self.ray_store_list = list()
        self.logger = logging.getLogger("RaySource.")
        self.logger.setLevel(logging.INFO)

    def generate_rays(self):
        """ Reimplement in subclass
        """
        pass

    def draw_rays(self, fig):
        for ray in self.rays:
            ray.drawRay(fig)

    def get_ray_points(self):
        """ Get list of points including color to draw
        """
        points = list()
        for ray_store in self.ray_store_list:
            points.append((ray_store.getArrayPoints(), ray_store.color))
        return points

    def get_rays_pos_array(self):
        """ Get array with ray coordinates. Used for computing.
        """
        return np.array([rl.x for rl in self.ray_store_list])

    def get_rays_time_on_surface(self, surface_number):
        """ Get array of group arrival time for the selected surface.
        Use -1 for last surface.
        """
        return np.array([rl.time[surface_number] for rl in self.ray_store_list])

class Collimated1DSource(RaySource):
    def __init__(self, num_rays=1000, x_dim=1e-3, l=567e-9, W=1, color=(0, 0, 0.85)):
        """ Defines an 1D ray source with a set of rays that can be traced. The rays
        are collimated in the z direction and initialized along the x axis with
        an extent of xDim.
        Contains a list of rays.

        Input
        numRays: Number of rays in the list
        xDim: The rays are evenly spaced around -xDim/2, +xDim/2
        l: Wavelength
        W: Energy
        color: Drawing color of rays
        """
        super(Collimated1DSource, self).__init__(num_rays)
        self.color = color
        self.generate_rays(x_dim, l, W, color)

    def generate_rays(self, x_dim=5e-3, l=567e-9, W=1, color=(0, 0, 0.85)):
        """ Generate rays matrix and a list of RayStores. A RayStore keeps track
        of a ray as it moves through the optical system.

        rays is 3d matrix:
        index0: ray number
        index1: data type (0=x, 1=xp, 2=optical data)
        index2: 4 element data vector

        The optical data is encoded as (wavelength, n, ng, W)

        Examples:
        rays[10, 0, :]... position vector of ray 10
        rays[ :, 1, :]... direction vectors of all rays
        rays[15, 2, 1]... current refractive index for ray 15
        """
        xx = np.linspace(-x_dim / 2, x_dim / 2, self.num_rays)
        x = np.column_stack((xx, np.zeros(self.num_rays), np.zeros(self.num_rays), np.ones(self.num_rays)))
        xp = np.column_stack((np.zeros(self.num_rays), np.zeros(self.num_rays), np.ones(self.num_rays), np.zeros(self.num_rays)))
        data = np.column_stack((l * np.ones(self.num_rays), 1.0 * np.ones(self.num_rays), 1.0 * np.ones(self.num_rays), W * np.ones(self.num_rays)))
        self.rays = np.dstack((x, xp, data)).swapaxes(1, 2)
        self.ray_store_list = list()
        for rn in range(self.num_rays):
            x = self.rays[rn, 0, :]
            xp = self.rays[rn, 1, :]
            l = self.rays[rn, 2, 0]
            n = self.rays[rn, 2, 1]
            W = self.rays[rn, 2, 3]
            r = Ray.RayStore(x=x, xp=xp, n=n, l=l, W=W, color=color)
            self.ray_store_list.append(r)

    def update_rays(self, new_rays):
        """ Update ray data with new rays (in global coordinates)
        Sets rays matrix to newRays matrix and updates the RayStoresList
        """
        d = np.sqrt(np.sum((new_rays[:, 0, 0:3] - self.rays[:, 0, 0:3]) ** 2, 1))
        self.rays = new_rays
        for rn in range(self.num_rays):
            x = self.rays[rn, 0, :]
            xp = self.rays[rn, 1, :]
            l = self.rays[rn, 2, 0]
            n = self.rays[rn, 2, 1]
            ng = self.rays[rn, 2, 2]
            W = self.rays[rn, 2, 3]
            self.ray_store_list[rn].addPos(newX=x, newXp=xp, newN=n, newNg=ng, W=W, distance=d[rn])


from __future__ import unicode_literals

import json
import yaml
from os.path import basename
from functools import partial
from collections import OrderedDict

from .fs import Node, DocumentIndexer
from .engine import Markment
from .views import TemplateContext
from .events import before, after

class Project(object):
    metadata_filename = '.markment.yml'

    def __init__(self, path):
        self.path = path

        self.node = Node(path)
        self.tree = DocumentIndexer(path)
        self.name = basename(path)
        self.version = ''
        self.description = ''

        documentation_index_fallback = 'README.md'

        found = self.node.grep(r'\.(md|markdown)')

        if found:
            documentation_index_fallback = found[0].basename

        self.meta = {
            'project': {
                'name': self.name,
            },
            'documentation': {
                'index': documentation_index_fallback
            }
        }
        self._found_files = []
        self.load(path)

    def load(self, path):
        metadata = self.parse_metadata(path)

        self.meta.update(metadata)

        p = self.meta['project']
        self.name = p.get('name', self.name)
        self.version = p.get('version', self.version)
        self.description = p.get('description', self.description)
        self.github_url = p.get('github_url', '')

        tarball_fallback = zipball_fallback = ''

        if self.github_url:
            tarball_fallback = '{0}/archive/master.tar.gz'.format(self.github_url)
            zipball_fallback = '{0}/archive/master.zip'.format(self.github_url)

        self.zipball_download_url = p.get('zipball_download_url', zipball_fallback)
        self.tarball_download_url = p.get('tarball_download_url', tarball_fallback)

    def parse_metadata(self, path):
        if not self.node.contains(self.metadata_filename):
            return {}

        with self.node.open(self.metadata_filename) as f:
            data = f.read()

        return yaml.load(data.decode('utf-8'))

    def find_markdown_files(self):
        if self._found_files:
            return self._found_files

        for_blobs = lambda info: info['type'] == 'blob'
        blobs = filter(for_blobs, self.tree.find_all_markdown_files())

        for info in blobs:
            name = info['relative_path']
            self._found_files.append(info)

        return self._found_files

    def get_master_index(self):
        """returns the master index, respecting any existing toc"""
        original_master_index = self.find_markdown_files()
        master_index = []
        table_of_contents = self.meta.get('toc', [])
        for toc_item in table_of_contents:
            for item in original_master_index:
                if toc_item == item['relative_path']:
                    master_index.append(item)

        for item in original_master_index:
            if item not in master_index:
                master_index.append(item)

        return master_index

    def pre_render(self, theme, static_url_cb, link_cb, **kw):
        master_index = self.get_master_index()
        total_indexes = len(master_index)
        for position, info in enumerate(master_index, start=1):
            partial_link_cb = partial(link_cb, current_document_info=info)
            before.rendering_markdown.shout(
                info, theme, kw, position, total_indexes)

            info['extra'] = kw
            info['markment'] = md = self.load_markment(
                info, partial_link_cb, **kw)
            info['markdown'] = md.raw
            info['indexes'] = md.index()
            info['documentation'] = md.rendered

            info.update(self.render_html_from_markdown_info(
                        md, info, theme,
                partial(static_url_cb, current_document_info=info),
                partial_link_cb, [], **kw))

            after.partially_rendering_markdown.shout(
                info, theme, kw, position, total_indexes, master_index)

    def post_render(self, theme, static_url_cb, link_cb, **kw):
        master_index = self.get_master_index()
        total_indexes = len(master_index)
        for position, info in enumerate(master_index, start=1):
            info.update(self.render_html_from_markdown_info(
                        info['markment'], info, theme,
                partial(static_url_cb, current_document_info=info),
                partial(link_cb, current_document_info=info), master_index, **kw))
            after.rendering_markdown.shout(
                info, theme, kw, position, total_indexes, master_index)

        return master_index
    def generate(self, theme, static_url_cb=None, link_cb=None, **kw):
        if not static_url_cb:
            static_url_cb = lambda link, *a, **kw: link

        if not link_cb:
            link_cb = lambda link, *a, **kw: link

        self.pre_render(theme, static_url_cb, link_cb, **kw)

        # rendering again, now with the full master_index
        return self.post_render(theme, static_url_cb, link_cb, **kw)

    def load_markment(self, info, link_cb, **kw):
        with self.node.open(info['path']) as f:
            data = f.read()

        try:
            decoded = data.decode('utf-8')
        except UnicodeEncodeError:
            decoded = data

        return Markment(decoded, url_prefix=link_cb)

    def render_html_from_markdown_info(
            self, md, info, theme, static_url_cb, link_cb, master_index, **kw):

        Context = TemplateContext(
            project=self.meta['project'],
            meta=self.meta,
            documentation=md.rendered,
            index=md.index(),
            master_index=list(master_index),
            json=json,
            static_url_cb=static_url_cb,
            link_cb=link_cb,
            info=info,
            current=info,
            **kw)

        ctx = Context.ready_to_render()
        before.rendering_html.shout(info, ctx)
        info['html'] = theme.render(**ctx).encode('utf-8')
        after.rendering_html.shout(info, ctx)
        return info

    @classmethod
    def discover(cls, path, *args, **kw):
        return cls(path, *args, **kw)

    def __repr__(self):
        return '<Project({0})>'.format(repr(self.path))


import json
import logging

from tg import config
from webob.exc import HTTPRequestEntityTooLarge
from paste.deploy.converters import asbool
import pysolr
from pysolr import SolrError
import six

from allura.lib.helpers import shlex_split

log = logging.getLogger(__name__)

escape_rules = {'+': r'\+',
                '-': r'\-',
                '&': r'\&',
                '|': r'\|',
                '!': r'\!',
                '(': r'\(',
                ')': r'\)',
                '{': r'\{',
                '}': r'\}',
                '[': r'\[',
                ']': r'\]',
                '^': r'\^',
                '~': r'\~',
                '*': r'\*',
                '?': r'\?',
                ':': r'\:',
                '"': r'\"',
                ';': r'\;'}

def escape_solr_arg(term):
    """ Apply escaping to the passed in query terms
        escaping special characters like : , etc"""
    term = term.replace('\\', r'\\')   # escape \ first
    for char, escaped_char in escape_rules.items():
        term = term.replace(char, escaped_char)

    return term

def make_solr_from_config(push_servers, query_server=None, **kwargs):
    """
    Make a :class:`Solr <Solr>` instance from config defaults.  Use
    `**kwargs` to override any value
    """
    solr_kwargs = dict(
        commit=asbool(config.get('solr.commit', True)),
        commitWithin=config.get('solr.commitWithin'),
        timeout=int(config.get('solr.long_timeout', 60)),
    )
    solr_kwargs.update(kwargs)
    return Solr(push_servers, query_server, **solr_kwargs)

class Solr:

    """Solr interface that pushes updates to multiple solr instances.

    `push_servers`: list of servers to push to.
    `query_server`: server to read from. Uses `push_servers[0]` if not specified.

    Also, accepts default values for `commit` and `commitWithin`
    and passes those values through to each `add` and `delete` call,
    unless explicitly overridden.
    """

    def __init__(self, push_servers, query_server=None,
                 commit=True, commitWithin=None, **kw):
        self.push_pool = [pysolr.Solr(s, **kw) for s in push_servers]
        if query_server:
            self.query_server = pysolr.Solr(query_server, **kw)
        else:
            self.query_server = self.push_pool[0]
        self._commit = commit
        self.commitWithin = commitWithin

    def add(self, *args, **kw):
        if 'commit' not in kw:
            kw['commit'] = self._commit
        if self.commitWithin and 'commitWithin' not in kw:
            kw['commitWithin'] = self.commitWithin
        responses = []
        for solr in self.push_pool:
            try:
                responses.append(solr.add(*args, **kw))
            except SolrError as e:
                if '(HTTP 413)' in str(e):
                    raise HTTPRequestEntityTooLarge() from e
                else:
                    raise
        return responses

    def delete(self, *args, **kw):
        if 'commit' not in kw:
            kw['commit'] = self._commit
        responses = []
        for solr in self.push_pool:
            responses.append(solr.delete(*args, **kw))
        return responses

    def commit(self, *args, **kw):
        responses = []
        for solr in self.push_pool:
            responses.append(solr.commit(*args, **kw))
        return responses

    def search(self, *args, **kw):
        return self.query_server.search(*args, **kw)

class MockSOLR:

    class MockHits(list):

        @property
        def hits(self):
            return len(self)

        @property
        def docs(self):
            return self

        @property
        def facets(self):
            return {'facet_fields': {}}

    def __init__(self):
        self.db = {}

    def add(self, objects):
        for o in objects:
            o['text'] = ''.join(o['text'])
            json.dumps(o)  # ensure no errors (since pysolr 3.9+ uses json API to solr)
            self.db[o['id']] = o

    def commit(self):
        pass

    def search(self, q, fq=None, **kw):
        if q is None:
            q = ''  # shlex will hang on None
        # Parse query
        preds = []
        q_parts = shlex_split(q)
        if fq:
            q_parts += fq
        for part in q_parts:
            if part in ('&&', 'AND'):
                continue
            if part in ('||', 'OR'):
                log.warning(f"MockSOLR doesn't implement OR yet; treating as AND. q={q} fq={fq}")
                continue
            if ':' in part:
                field, value = part.split(':', 1)
                if value.startswith('(') and value.endswith(')'):
                    value = value[1:-1]
                preds.append((field, value))
            else:
                preds.append(('text', part))
        result = self.MockHits()
        for obj in self.db.values():
            for field, value in preds:
                neg = False
                if field[0] in ('!', '-'):
                    neg = True
                    field = field[1:]
                if field == 'text' or field.endswith('_t'):
                    if (value not in str(obj.get(field, ''))) ^ neg:
                        break
                elif field.endswith('_b'):
                    if (asbool(value) != obj.get(field, False)) ^ neg:
                        break
                else:
                    if (value != str(obj.get(field, ''))) ^ neg:
                        break
            else:
                result.append(obj)

        if asbool(kw.get('hl')):
            result.highlighting = {}
        return result

    def delete(self, *args, **kwargs):
        if kwargs.get('q', None) == '*:*':
            self.db = {}
        elif kwargs.get('id', None):
            del self.db[kwargs['id']]
        elif kwargs.get('q', None):
            for doc in self.search(kwargs['q']):
                self.delete(id=doc['id'])


import firebase_admin
import hashlib
from firebase_admin import credentials
from firebase_admin import firestore

# Use the application default credentials
cred = credentials.Certificate("airflow/dags/cred.json")
firebase_admin.initialize_app(cred)

db = firestore.client()

from random import randint
from datetime import datetime

from airflow import DAG
from airflow.operators.python import PythonOperator, BranchPythonOperator
from airflow.operators.bash import BashOperator

def _parseHttpGet():
    import datetime
    from os import remove

    from sqlalchemy import null

    numberOfLines = 0
    numberOfGet = 0

    f = open("airflow/dags/http.log", "r")
    w = open("airflow/dags/resultGet.txt", "w")

    def returnMonthNumber(arg):
        switcher = {
            "Jan": 1,
            "Feb": 2,
            "Mar": 3,
            "Apr": 4,
            "May": 5,
            "Jun": 6,
            "Jul": 7,
            "Aug": 8,
            "Sep": 9,
            "Oct": 10,
            "Nov": 11,
            "Dec": 12
        }
        return switcher.get(arg)

    getRequests=[]

    for x in f:
        for index, word in enumerate(x.split()):
            if word == '200':
                # print(x)
                if(x.split()[index-3]) == '"GET':

                    tarih = x.split()[index-5]
                    buyukluk = x.split()[index+1]
                    year = 1000*int(tarih[8])+100*int(tarih[9]) + 10*int(tarih[10])+int(tarih[11])
                    month = returnMonthNumber(tarih[4] + tarih[5] + tarih[6])
                    day = 10*int(tarih[1])+int(tarih[2])
                    hour = 10*int(tarih[13])+int(tarih[14])
                    minute = 10*int(tarih[16])+int(tarih[17])
                    second = 10*int(tarih[19])+int(tarih[20])
                    date = int(datetime.datetime(year, month, day, hour, minute, second).timestamp())
                    getRequests.append((date,'GET',int(buyukluk)))
                    numberOfGet += 1

                numberOfLines += 1

    # sortedGet = sorted(getRequests, key=lambda x: x[0])

    Gets = {}

    seenGet = set()

    for x in getRequests:
        if x[0] not in seenGet:
            seenGet.add(x[0])

    for x in seenGet:
        Gets[x]=[]

    for x in seenGet:
        for y in getRequests:
            if y[0] == x:
                Gets[x].append(y[2])

    w.write('Total 200 requests={}\n'.format(numberOfLines))
    w.write('GET={}\n'.format(numberOfGet))

    for x in Gets:
        w.write(str(x)+':'+str(sum(Gets[x]))+'\n')

    f.close()
    w.close()
    return Gets

def _parseHttpPost():
    import datetime
    from os import remove

    from sqlalchemy import null

    numberOfLines = 0
    numberOfPost = 0
    f = open("airflow/dags/http.log", "r")
    w = open("airflow/dags/resultPost.txt", "w")

    def returnMonthNumber(arg):
        switcher = {
            "Jan": 1,
            "Feb": 2,
            "Mar": 3,
            "Apr": 4,
            "May": 5,
            "Jun": 6,
            "Jul": 7,
            "Aug": 8,
            "Sep": 9,
            "Oct": 10,
            "Nov": 11,
            "Dec": 12
        }
        return switcher.get(arg)

    postRequests=[]

    for x in f:
        for index, word in enumerate(x.split()):
            if word == '200':
                if(x.split()[index-3]) == '"POST':
                    tarih = x.split()[index-5]
                    buyukluk = x.split()[index+1]
                    year = 1000*int(tarih[8])+100*int(tarih[9]) + 10*int(tarih[10])+int(tarih[11])
                    month = returnMonthNumber(tarih[4] + tarih[5] + tarih[6])
                    day = 10*int(tarih[1])+int(tarih[2])
                    hour = 10*int(tarih[13])+int(tarih[14])
                    minute = 10*int(tarih[16])+int(tarih[17])
                    second = 10*int(tarih[19])+int(tarih[20])
                    date = int(datetime.datetime(year, month, day, hour, minute, second).timestamp())# print(date)
                    postRequests.append((date,'POST',int(buyukluk)))
                    numberOfPost += 1
                numberOfLines += 1

    # sortedPost = sorted(postRequests, key=lambda x: x[0])

    Posts = {}

    seenPost = set()

    for x in postRequests:
        if x[0] not in seenPost:
            seenPost.add(x[0])

    for x in seenPost:
        Posts[x]=[]

    for x in seenPost:
        for y in postRequests:
            if y[0] == x:
                Posts[x].append(y[2])

    w.write('Total 200 requests={}\n'.format(numberOfLines))
    w.write('POST={}\n'.format(numberOfPost))

    for x in Posts:
        w.write(str(x)+':'+str(sum(Posts[x]))+'\n')

    f.close()
    w.close()
    return Posts

def _addToDatabase(ti):
    Requests = ti.xcom_pull(task_ids=[
        'parseHttpGet',
        'parseHttpPost',
    ])
    for x in Requests[0]:
        data = {
            u'timeStamp':x,
            u'size':sum(Requests[0][x]),
            u'type':'GET'
        }
        db.collection(u'GetRequests').document(hashlib.sha256(str(x).encode('utf-8')).hexdigest()).set(data)

    for x in Requests[1]:
        data = {
            u'timeStamp':x,
            u'size':sum(Requests[1][x]),
            u'type':'POST'
        }
        db.collection(u'PostRequests').document(hashlib.sha256(str(x).encode('utf-8')).hexdigest()).set(data)

with DAG("httpChartDag", start_date=datetime(2022, 1, 1), schedule_interval='@daily', catchup=False) as dag:
    parseHttpGet = PythonOperator(
        task_id="parseHttpGet",
        python_callable=_parseHttpGet
    )
    parseHttpPost = PythonOperator(
        task_id="parseHttpPost",
        python_callable=_parseHttpPost
    )
    addToDatabase = PythonOperator(
        task_id="addToDatabase",
        python_callable=_addToDatabase
    )

[parseHttpGet,parseHttpPost] >> addToDatabase