# from __future__ import print_function

import os
import torch.nn.parallel
import torch.utils.data
import argparse
import numpy as np
from tqdm import tqdm
from utils.saver import Saver
from utils.metrics import Evaluator
from utils.util import *
from PIL import Image
import math
from math import log10
from torchvision import transforms
from dataset import *
from models.model_lib.InterDn import *
import cv2
import torch.nn.functional as F

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

parser = argparse.ArgumentParser(description='N2V_oneframe')

# Data specifications

parser.add_argument('--dataset', type=str, default='CBSD68')
parser.add_argument('--train_dir', type=str, default=r'blend_gaussian10-50')
parser.add_argument('--train_dir_pred', type=str, default=r'blend_gaussian10-50')
parser.add_argument('--valid_dir', type=str, default=r'IL_test')
parser.add_argument('--workers', type=int, default=0,
                    metavar='N', help='dataloader threads')
# training hyper params
parser.add_argument('--epochs', type=int, default=10000, metavar='N',
                    help='number of epochs to train (default: auto)')
parser.add_argument('--start_epoch', type=int, default=0,
                    metavar='N', help='start epochs (default:0)')
parser.add_argument('--batch_size', type=int, default=8)
parser.add_argument('--patch_size', type=int, default=128)

# optimizer params
parser.add_argument('--lr_G', type=float, default=4e-4, metavar='LR',
                    help='learning rate (default: auto)')
# checking point
parser.add_argument('--resume', type=str, default='model.pth.tar',
                    help='put the path to resuming file if needed')
parser.add_argument('--checkname', type=str, default='Unet',
                    help='set the checkpoint name')
# evaluation option
parser.add_argument('--eval_interval', type=int, default=1,
                    help='evaluation interval (default: 1)')
parser.add_argument('--output_path', type=str, default='./pred0_1',
                    help='evaluuation interval (default: 1)')
args = parser.parse_args()

class Trainer(object):
    def __init__(self, args):
        self.args = args
        # Define Saver
        self.saver = Saver(args)
        self.writer = self.saver.create_summary()

        # Define Dataloader
        self.transform_train = transforms.Compose(
            [ RandomFlip(), ToTensor()])
        self.transform_val = transforms.Compose(
            [ ToTensor()])

        self.transform_inv = transforms.Compose([ToNumpy()])

        self.dataset_train = Dataset(args.train_dir,args.train_dir_pred, data_type='float32', transform=self.transform_train, sgm=25, ratio=0.9,randomcrop=(self.args.patch_size,self.args.patch_size),
                             size_window=(5, 5))
        self.dataset_val = Dataset_train_val(args.valid_dir, data_type='float32', transform=self.transform_val, sgm=25, ratio=0.9,
                             size_window=(5,5))

        self.loader_train = torch.utils.data.DataLoader(self.dataset_train, batch_size=args.batch_size, shuffle=True,drop_last=True, num_workers=self.args.workers)
        self.loader_val = torch.utils.data.DataLoader(self.dataset_val, batch_size=1, shuffle=False, num_workers=self.args.workers)

        self.num_train = len(self.dataset_train)
        self.num_val = len(self.dataset_val)

        self.num_batch_train = int((self.num_train / args.batch_size) + ((self.num_train % args.batch_size) != 0))
        self.num_batch_val = int((self.num_val / args.batch_size) + ((self.num_val % args.batch_size) != 0))

        # Define network
        self.netG = InterDnNet(3,3)

        init_net(self.netG, init_type='normal', init_gain=0.02, gpu_ids=[0])

        self.fn_REG = nn.L1Loss().to(device)  # Regression loss: L1

        enc_dict = ['enc1_1.cbr.0.conv.weight', 'enc1_1.cbr.0.conv.bias', 'enc1_2.cbr.0.conv.weight',
                         'enc1_2.cbr.0.conv.bias', 'enc2_1.cbr.0.conv.weight', 'enc2_1.cbr.0.conv.bias',
                         'enc2_2.cbr.0.conv.weight', 'enc2_2.cbr.0.conv.bias', 'enc3_1.cbr.0.conv.weight',
                         'enc3_1.cbr.0.conv.bias', 'dec3_1.decbr.0.deconv.weight', 'dec3_1.decbr.0.deconv.bias',
                         'dec2_2.decbr.0.deconv.weight', 'dec2_2.decbr.0.deconv.bias', 'dec2_1.decbr.0.deconv.weight',
                         'dec2_1.decbr.0.deconv.bias', 'dec1_2.decbr.0.deconv.weight', 'dec1_2.decbr.0.deconv.bias',
                         'dec1_1.decbr.0.deconv.weight', 'tail.0.cbr.0.conv.weight', 'tail.0.cbr.0.conv.bias',
                         'tail.1.cbr.0.conv.weight', 'tail.1.cbr.0.conv.bias']
        # Define Optimizer
        for name, p in self.netG.named_parameters():
            if name in enc_dict:
                p.requires_grad = False

        self.paramsG = self.netG.parameters()
        self.optimG = torch.optim.Adam(filter(lambda x: x.requires_grad is not False, self.netG.parameters()),
                                       lr=args.lr_G, betas=(0.5, 0.999))

        # Clear start epoch if fine-tuning
        self.saver.print_log(args)
        self.saver.print_log('Starting Epoch: {}'.format(args.start_epoch))
        self.saver.print_log('Total Epoches: {}'.format(args.epochs))

        enc_dict=[]
        model_now_dict = self.netG.state_dict()
        pretrained_dict = torch.load(os.path.join(args.resume))
        pretrained_dict = pretrained_dict['state_dict']
        new_state_dict = {k: v for k, v in pretrained_dict.items() if
                          k not in enc_dict}
        model_now_dict.update(new_state_dict)
        self.netG.load_state_dict(model_now_dict)

        self.best_psnr = 0
    def training(self, epoch):

        ## training phase
        self.netG.train()

        loss_l1_train =[]
        loss_G_train = []

        for iter, data in enumerate(tqdm(self.loader_train, ncols=45)):
            label = data['label'].to(device)
            input = data['input'].to(device)
            mask1 = data['mask'].to(device)
            data_pred = data['clean'].to(device)

            # alpha = torch.from_numpy((np.random.rand(1) * 0.2).astype(np.float32)).to(device)
            # lambd = np.random.choice(a=[1,2], size=1, replace=False, p=None)[0]
            # if lambd ==1:
            #     alpha=0.8
            # else:
            #     alpha = 1
            # # forward netG
            # alpha=0.1
            # input = alpha * data_pred + (1-alpha) * input
            output1 = self.netG(input*mask1, 1)

            # backward netG
            self.optimG.zero_grad()

            loss_G1 = self.fn_REG(output1 * (1 - mask1), label * (1 - mask1))

            loss_G = loss_G1

            loss_G.backward()
            self.optimG.step()

            # get losses
            loss_l1_train += [loss_G1.item()]
            loss_G_train += [loss_G.item()]

            self.saver.print_log('TRAIN: EPOCH {}: ITER {:0>4d}/{:0>4d}: LOSS: {:.4f}'
                  .format(epoch, iter, self.num_batch_train, np.mean(loss_G_train)))

        # if epoch % self.args.eval_interval == 0 or epoch == args.epochs and epoch!=0:
        self.writer.add_scalar('train/l1_loss',  np.mean(loss_l1_train), epoch)
        self.writer.add_scalar('train/total_loss',  np.mean(loss_G_train),  epoch )

    def valid(self, epoch):
        ## validation phase
        np.random.seed(0)

        with torch.no_grad():
            self.netG.eval()
            psnr_list = []

            for iter, data in enumerate(tqdm(self.loader_val, ncols=45)):

                input = data['input'].to(device)
                clean = data['clean'].to(device)
                # data_pred = data['data_pred'].to(device)

                # forward netG
                # alpha = 0.8
                # input = alpha * input + (1-alpha) * data_pred
                # for lambda_c in np.arange(1.1,2.0,0.1):

                # for alpha in np.arange(0,0.11,0.01):
                # alpha=0.1
                #     output1 = alpha * input + (1-alpha) * output
                for lambda0 in [0]:
                    output = self.netG(input, lambda0)

                    output1_grad = output.data.cpu().numpy()[0,:,:,:].transpose(1,2,0)
                    output1_grad = np.clip(output1_grad*255,0,255)
                    Image.fromarray(output1_grad.astype(np.uint8)).save(args.output_path+'/test%03dlambda%03d'%(iter+1, lambda0*100)+'.png')

            #     rmse = np.sqrt(torch.mean(((clean)-(output)).cpu()**2))
            #     psnr = 20*math.log10(1.0/rmse)
            #     psnr_list += [psnr]
            # print(np.mean(psnr_list))
            # if epoch%200==0 and epoch != 0:
            #     self.saver.save_checkpoint(
            #         {'epoch': epoch,
            #     'state_dict': self.netG.state_dict(),
            #     # 'optimizer': self.optimizer.state_dict(),
            #     }, filename = 'checkpoint_%03d.pth.tar'%epoch)
            #     self.saver.print_log('PSNR!!! EPOCH:{:0>3d} PSNR {}'.format(epoch, np.mean(psnr_list)))
            #
            # if epoch>3 and np.mean(psnr_list)> self.best_psnr:
            #     self.best_psnr = np.mean(psnr_list)
            #     self.saver.save_checkpoint(
            #         {'epoch': epoch,
            #     'state_dict': self.netG.state_dict(),
            #     # 'optimizer': self.optimizer.state_dict(),
            #     }, filename = 'checkpoint_best.pth.tar')
            #     self.saver.print_log('BETTER PSNR!!! EPOCH:{:0>3d} PSNR {}'.format(epoch, self.best_psnr))
        np.random.seed(None)

def main():

    trainer=Trainer(args)
    trainer.valid(0)

    # for epoch in tqdm(range(args.start_epoch, args.epochs), ncols=45):
    #     # if epoch % args.eval_interval == 0 or epoch == args.epochs:
    #     trainer.valid(epoch)
    #     trainer.training(epoch)
    trainer.writer.close()

if __name__ == '__main__':
    main()


import aiohttp

class AioHttpClient:
    def __init__(self, base_url):
        self.base_url = base_url

    async def __request(self, method, endpoint, params=None, data=None, headers=None):
        url = f"{self.base_url}{endpoint}"

        async with aiohttp.ClientSession() as session:
            async with session.request(
                method, url, params=params, data=data, headers=headers
            ) as response:
                response_data = await response.text()
                return response.status, response_data

    async def _get(self, endpoint, params=None, headers=None):
        return await self.__request("GET", endpoint, params=params, headers=headers)

    async def _post(self, endpoint, data=None, headers=None):
        return await self.__request("POST", endpoint, data=data, headers=headers)

    async def _put(self, endpoint, data=None, headers=None):
        return await self.__request("PUT", endpoint, data=data, headers=headers)

    async def _delete(self, endpoint, headers=None):
        return await self.__request("DELETE", endpoint, headers=headers)


class Statechachine:
    def _init_(self):
        self.Handlers={}
        self.StartState=None #cero
        self.endStates=[]

    def Add_States(self,name, Handler,end_State=0): #self sig. que add_states
        name=name.upper()
        self.handler[name]=Handler #guarda las manzanas
        if end_State:
            self.end_States.append(name) #guarda la basura

    def set_start(self,name,):
        self.StartState=name.upper()

    def run(self,cargo):
        try:
            handler=self.Handlers[self.StartState]#bolsa para guardar bolsa
        except:
            raise Initialization("must call .set_start() before .run")
        if not self.endStates: #0
            raise InitializationError("at least one state must be an end_state")

        while True:
            newState=handler(cargo)
            cargo=handler(cargo)
            if newState.uppee() in self.endStates:
                print("si esta en la bolsa de basura la bolsa de manzanas con cargo", newState)
                break
            else:
                handler=self.handlers[newState.upper()] #guarda nuevas cosas en la bolsa de basura


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
import pickle

messages = pd.read_csv('https://drive.google.com/uc?id=17bpIlFsn1tZE-7OHzcy9HfzqnQ_YtEMD', sep = "\t", header = None)

messages.columns = ["Label", "Message"] #Labeling the columns as Label and Message

messages["Label"].value_counts().plot.bar(rot = 30)
plt.xlabel("Message Label")
plt.ylabel("Frequency")

messages["Label"].value_counts(normalize = True)

# random data
random_data = messages.sample(frac = 1, random_state = 1)

# lengths of the training and testing data, that will be used as future indexes
len_train = round(len(random_data) * .8)
len_test = len(random_data) - len_train

# creating the sets using slicing
training = random_data[:len_train].reset_index(drop = True)
testing = random_data[-len_test:].reset_index(drop = True)

training["Label"].value_counts(normalize = True)

testing["Label"].value_counts(normalize = True)

# "\W" is a regex command that matches character that are not a-z, A-Z, 0-9 and _
training["Message"] = training["Message"].str.replace("\W", " ", regex  = True).str.lower()

# creating the vocabulary
vocabulary = []
training["Message"] = training["Message"].str.split()
for message in training["Message"]:
    for word in message:
        if word not in vocabulary:
            vocabulary.append(word)

d = len(vocabulary)

# creating the dictionary that will be converted to a dataframe
word_freq_per_message = {word:[0]*len(training["Message"]) for word in vocabulary}

# adding the frequencies to word_freq_per_message
for i, message in enumerate(training["Message"]):
    for word in message: # recall that 'message' is a list of words, saved as strings
        word_freq_per_message[word][i] += 1

words_freq_per_message = pd.DataFrame(word_freq_per_message)
# words_freq_per_message

training_final = pd.concat([training, words_freq_per_message], axis = 1)

# training_final

prob_spam = len(training_final[training_final["Label"] == "spam"]) / len(training_final["Label"])
prob_nonspam = 1 - prob_spam

# calculating P(word_i|C_k)
spam_messages = training_final[training_final["Label"] == "spam"]
nonspam_messages = training_final[training_final["Label"] == "ham"]

alpha = 1
n_spam = spam_messages["Message"].apply(len).sum()
n_nonspam = nonspam_messages["Message"].apply(len).sum()

prob_word_given_spam = {}
prob_word_given_nonspam = {}

for word in vocabulary:
    prob_word_given_spam[word] = (spam_messages[word].sum() + alpha) / (n_spam + alpha * d)
    prob_word_given_nonspam[word] = (nonspam_messages[word].sum() + alpha) / (n_nonspam + alpha * d)

class MyModel:

    def classify(message):
        if not isinstance(message, str):
            raise Exception("Argument must be a string")

        message = re.sub("\W", " ", message)
        message = message.lower().split()

        prob_spam_given_message = prob_spam
        prob_nonspam_given_message = prob_nonspam
        for word in message:
            if word in prob_word_given_spam:
                prob_spam_given_message = prob_word_given_spam[word]
            if word in prob_word_given_nonspam:
                prob_nonspam_given_message = prob_word_given_nonspam[word]
        # we added these if clauses to avoid issues when a word of a message is not present in our list (see README for more)

        if prob_spam_given_message > prob_nonspam_given_message:
            res = "spam"
        elif prob_spam_given_message < prob_nonspam_given_message:
            res = "ham"
        else: # if there is equality. It is unlikely to occur, since we're comparing float numbers
            res = "Classification failed"

        return prob_spam_given_message, prob_nonspam_given_message, res
model=MyModel

# checking boundary case

print(model.classify('3'))

print(model.classify("What you doing?how are you?"))

print(model.classify("Ok lar... Joking wif u oni..."))

print(model.classify("dun say so early hor... U c already then say..."))

print(model.classify("MY NO. IN LUTON 0125698789 RING ME IF UR AROUND! H*"))

print(model.classify("Siva is in hostel aha:-."))

print(model.classify("Cos i was out shopping wif darren jus now n i called him 2 ask wat present he wan lor. Then he started guessing who i was wif n he finally guessed darren lor."))

print(model.classify("FreeMsg: Txt: CALL to No: 86888 & claim your reward of 3 hours talk time to use from your phone now! ubscribe6GBP/ mnth inc 3hrs 16 stop?txtStop"))

print(model.classify("Sunshine Quiz! Win a super Sony DVD recorder if you canname the capital of Australia? Text MQUIZ to 82277. B"))

print(model.classify("URGENT! Your Mobile No 07808726822 was awarded a L2,000 Bonus Caller Prize on 02/09/03! This is our 2nd attempt to contact YOU! Call 0871-872-9758 BOX95QU"))

testing["classification"] = testing["Message"].apply(lambda message: model.classify(message)[2])

correct_class_len = 0
total_messages = testing.shape[0] #1114

for row in testing.iterrows():
    if row[1]["Label"] == row[1]["classification"]: #we use row[1] because iterrows() returns (index,row)
        correct_class_len += 1

accuracy = correct_class_len / total_messages
print("Accuracy is {}".format(accuracy))
pickle.dump(model,open('model.pkl','wb'))


# -*- coding: utf-8 -*-
"""
.. codeauthor:: Marius Engelhardt <marius.engelhardt@tu-ilmenau.de>
.. codeauthor:: Daniel Seichter <daniel.seichter@tu-ilmenau.de>
.. codeauthor:: Soehnke Fischedick <soehnke-benedikt.fischedick@tu-ilmenau.de>
"""
import argparse as ap
import datetime
import json
import os

import cv2
import h5py
import re
import numpy as np
import pandas as pd
import functools
from scipy.spatial.transform import Rotation
from tqdm import tqdm
from tqdm.contrib.concurrent import process_map

from ...utils.io import create_or_update_creation_metafile
from ...utils.io import download_file
from ...utils.img import save_indexed_png as save_indexed_png_
from .hypersim import HypersimMeta

SPLITS_CSV_URL = 'https://raw.githubusercontent.com/apple/ml-hypersim/6cbaa80207f44a312654e288cf445016c84658a1/' \
                 'evermotion_dataset/analysis/metadata_images_split_scene_v1.csv'

SCENE_CLASS_URL = 'https://raw.githubusercontent.com/apple/ml-hypersim/6cbaa80207f44a312654e288cf445016c84658a1/' \
                  'evermotion_dataset/analysis/metadata_camera_trajectories.csv'

CAMERA_PARAMETER_CSV_URL = 'https://raw.githubusercontent.com/apple/ml-hypersim/76dbac9b2ee15faf2d677e87db2c0203fc3cc024/' \
                           'contrib/mikeroberts3000/metadata_camera_parameters.csv'

CV_WRITE_FLAGS = (cv2.IMWRITE_PNG_COMPRESSION, 9)

def _parse_args(args=None):
    parser = ap.ArgumentParser(
        formatter_class=ap.ArgumentDefaultsHelpFormatter,
        description='Prepare Hypersim dataset.'
    )
    parser.add_argument(
        'output_path',
        type=str,
        help="Path where to store dataset."
    )
    parser.add_argument(
        'hypersim_path',
        type=str,
        help="Path to downloaded (and uncompressed) Hypersim files."
    )
    parser.add_argument(
        '--additional-subsamples',
        type=int,
        nargs='*',
        default=[10],
        help="Create additional versions with every N samples of the the "
             "Hypersim dataset as well. Note that the subsample is applied "
             "across all samples and NOT per scene or camera. However, as each "
             "trajectory comprises 100 samples, a lot of factors are possible "
             "to subsample each trajectory independently."
    )
    parser.add_argument(
        '--n-processes',
        type=int,
        default=1,
        help='Number of worker processes to spawn.'
    )
    parser.add_argument(
        '--no-tilt-shift-conversion',
        action='store_true',
        default=False,
        help="Disable projecting the data/annotations back to a standard "
             "camera ignoring tilt-shift parameters. Use this argument to "
             "prepare a dataset compatible with < v050)."
    )

    return parser.parse_args(args)

def color_semgentation(label, with_void=True):
    class_colors = HypersimMeta.CLASS_COLORS
    if with_void:
        colors = class_colors
    else:
        colors = class_colors[1:]
    cmap = np.asarray(colors, dtype='uint8')

    return cmap[label], cmap

def save_indexed_png(filepath, label, colormap):
    assert label.min() >= 0 and label.max() <= 40, f'Label should be in [0, 40]' \
                                                   f' but is in [{label.min()}, {label.max()}]'
    return save_indexed_png_(filepath, label, colormap)

def save_pc(filepath, pc):
    height, width = pc.shape[:2]

    file_extension = os.path.splitext(filepath)[1]

    if file_extension == '.npz':
        np.savez_compressed(filepath, pc)

    elif file_extension == '.pcd':

        with open(filepath, 'w') as f:
            # write header
            f.write('VERSION 0.7\n')
            f.write('FIELDS x y z\n')
            f.write('SIZE 4 4 4\n')
            f.write('TYPE f f f\n')
            f.write('COUNT 1 1 1\n')
            f.write(f'WIDTH {width}\n')
            f.write(f'HEIGHT {height}\n')
            f.write('VIEWPOINT 0 0 0 1 0 0 0\n')
            f.write(f'POINTS {height * width}\n')
            f.write('DATA ascii\n')

            # write point data
            for point in pc.reshape(-1, 3):
                f.write(f'{point[0]} {point[1]} {point[2]}\n')
    else:
        raise ValueError('Filetype for point cloud not understood')

def load_pc(scene_path, cam='cam_00', frame=0):
    pc_path = os.path.join(scene_path, 'images', f'scene_{cam}_geometry_hdf5',
                           f'frame.{frame:04d}.position.hdf5')

    assets_to_meters_path = os.path.join(scene_path, '_detail',
                                         'metadata_scene.csv')
    assets_to_meters_csv = pd.read_csv(assets_to_meters_path)
    assets_to_meters = assets_to_meters_csv['parameter_value'][0]

    with h5py.File(pc_path, 'r') as f:
        pc = f['dataset'][:].astype(np.float32) * assets_to_meters

    height, width = pc.shape[:2]

    # load extrinsic as the point cloud is given in world coordinate frame
    extrinsics = load_extrinsics(scene_path, cam, frame)

    rotation = Rotation.from_quat(
        [extrinsics[q] for q in ['quat_x', 'quat_y', 'quat_z', 'quat_w']]
    )
    transform = np.eye(4)
    transform[:3, :3] = rotation.as_matrix()
    transform[:3, 3] = [extrinsics[t] for t in ['x', 'y', 'z']]

    # flatten point cloud for transformation
    pc = pc.reshape(-1, 3)

    # change to homogenous coordinates
    pc = np.concatenate([pc, np.ones((len(pc), 1))], axis=-1)

    # transform points into camera coordinate frame
    pc = np.linalg.inv(transform) @ pc.T
    pc = pc.T[:, :3]

    pc = pc.reshape(height, width, 3)

    return pc

def compute_point_cloud(scene_path, cam, frame, camera_params):
    # code adapted from:
    # https://github.com/apple/ml-hypersim/blob/main/contrib/mikeroberts3000/jupyter/01_casting_rays_that_match_hypersim_images.ipynb

    # note, we do not use the precomputed point clouds as they are of type
    # float16 and, thus, not precise enough for our purposes

    # distance image (note its the distance along the ray, not the distance to
    # the camera)
    filepath = os.path.join(scene_path, 'images', f'scene_{cam}_geometry_hdf5',
                            f'frame.{frame:04d}.depth_meters.hdf5')
    with h5py.File(filepath, 'r') as f:
        distance_img_meters = f['dataset'][:]

    # camera parameters
    width_pixels = int(camera_params["settings_output_img_width"])
    height_pixels = int(camera_params["settings_output_img_height"])

    # camera matrix
    cp = camera_params
    M_cam_from_uv = np.array(
        [[cp["M_cam_from_uv_00"], cp["M_cam_from_uv_01"], cp["M_cam_from_uv_02"]],
         [cp["M_cam_from_uv_10"], cp["M_cam_from_uv_11"], cp["M_cam_from_uv_12"]],
         [cp["M_cam_from_uv_20"], cp["M_cam_from_uv_21"], cp["M_cam_from_uv_22"]]]
    )

    # compute rays
    u_min = -1.0
    u_max = 1.0
    v_min = -1.0
    v_max = 1.0
    half_du = 0.5 * (u_max - u_min) / width_pixels
    half_dv = 0.5 * (v_max - v_min) / height_pixels
    u, v = np.meshgrid(
        np.linspace(u_min+half_du, u_max-half_du, width_pixels),
        np.linspace(v_min+half_dv, v_max-half_dv, height_pixels)[::-1]
    )
    uvs_2d = np.dstack((u, v, np.ones_like(u)))
    # rays = np.dot(M_cam_from_uv, uvs_2d.reshape(-1, 3).T).T
    rays = np.dot(uvs_2d.reshape(-1, 3), M_cam_from_uv.T)   # (h*w, 3)

    # compute point cloud
    normed_rays = rays / np.linalg.norm(rays, ord=2, axis=-1, keepdims=True)
    points_cam = normed_rays * distance_img_meters.reshape(-1, 1)   # (h*w, 3)

    # flip axes to fix projection side
    points_cam *= np.array([1, -1, -1]).reshape(-1, 3)

    # reshape back to image
    points_cam = points_cam.reshape(height_pixels, width_pixels, 3)

    return points_cam

def load_depth_old(scene_path, cam, frame, correct_focal=True, in_mm=True):
    # do not use this code anymore

    depth_path = os.path.join(scene_path,
                              'images',
                              f'scene_{cam}_geometry_hdf5',
                              f'frame.{frame:04d}.depth_meters.hdf5')

    with h5py.File(depth_path, 'r') as f:
        depth = f['dataset'][:].astype(np.float32)

    if correct_focal:
        # code from https://github.com/apple/ml-hypersim/issues/9#issuecomment-754935697
        int_width = depth.shape[1]
        int_height = depth.shape[0]
        flt_focal = 886.81

        npy_imageplane_x = np.linspace((-0.5 * int_width) + 0.5, (0.5 * int_width) - 0.5, int_width)
        npy_imageplane_x = npy_imageplane_x.reshape(1, int_width).repeat(int_height, 0).astype(np.float32)[:, :, None]
        npy_imageplane_y = np.linspace((-0.5 * int_height) + 0.5, (0.5 * int_height) - 0.5, int_height)
        npy_imageplane_y = npy_imageplane_y.reshape(int_height, 1).repeat(int_width, 1).astype(np.float32)[:, :, None]
        npy_imageplane_z = np.full([int_height, int_width, 1], flt_focal, np.float32)
        npy_imageplane = np.concatenate([npy_imageplane_x, npy_imageplane_y, npy_imageplane_z], 2)

        depth = depth / np.linalg.norm(npy_imageplane, 2, 2) * flt_focal

    if in_mm:
        depth = depth * 1000
        depth[depth >= 2 ** 16] = 0

    return depth.astype('uint16')

def load_normal(scene_path, cam, frame):
    normal_path = os.path.join(scene_path,
                               'images',
                               f'scene_{cam}_geometry_hdf5',
                               f'frame.{frame:04d}.normal_bump_cam.hdf5')

    with h5py.File(normal_path, 'r') as f:
        normal = f['dataset'][:].astype(np.float32)

    # note: every valid normal vector in the dataset is already normalized to
    # have unit length

    # convert back to uint8, clipping is necessary to avoid over/-underflows
    normal = ((normal + 1)*127).clip(0, 255).astype('uint8')

    # format is xyz
    return normal

def load_semantic(scene_path, cam, frame, orig_mapping=False):
    # note, by default, we replace -1 (void in original dataset) with 0
    # (void in other datasets).
    gt_path = os.path.join(scene_path,
                           'images',
                           f'scene_{cam}_geometry_hdf5',
                           f'frame.{frame:04d}.semantic.hdf5')

    with h5py.File(gt_path, 'r') as f:
        gt = f['dataset'][:].astype(np.int32)

    assert 0 == np.sum(gt == 0)

    if not orig_mapping:
        gt[gt == -1] = 0

    return gt

def load_instance(scene_path, cam, frame):
    gt_path = os.path.join(scene_path,
                           'images',
                           f'scene_{cam}_geometry_hdf5',
                           f'frame.{frame:04d}.semantic_instance.hdf5')

    with h5py.File(gt_path, 'r') as f:
        gt = f['dataset'][:]

    # Min value is -1 and max value is 4166, so we shift it to store it in an
    # uint16
    assert 0 == np.sum(gt == 0)
    gt[gt == -1] = 0    # note there is no instance with id 0
    gt = gt.astype(np.uint16)

    return gt

def load_box_3d(scene_path, cam, frame):
    extents = os.path.join(
        scene_path,
        '_detail',
        'mesh',
        'metadata_semantic_instance_bounding_box_object_aligned_2d_extents.hdf5'
    )

    orientations = os.path.join(
        scene_path,
        '_detail',
        'mesh',
        'metadata_semantic_instance_bounding_box_object_aligned_2d_orientations.hdf5'
    )
    positions = os.path.join(
        scene_path,
        '_detail',
        'mesh',
        'metadata_semantic_instance_bounding_box_object_aligned_2d_positions.hdf5'
    )
    r_dict = {}
    dict_keys = ['extents', 'orientations', 'positions']
    for path, name in zip([extents, orientations, positions], dict_keys):
        with h5py.File(path, 'r') as f:
            gt = f['dataset'][:].astype(np.float32)
            r_dict[name] = gt.tolist()

    # the boxes are stored in assets coordinates, we need to convert them to
    # meters
    assets_to_meters_path = os.path.join(scene_path, '_detail',
                                         'metadata_scene.csv')
    assets_to_meters_csv = pd.read_csv(assets_to_meters_path)
    assets_to_meters = assets_to_meters_csv['parameter_value'][0]

    new_extents = []
    for extent in r_dict['extents']:
        new_extents.append([extent[0] * assets_to_meters,
                            extent[1] * assets_to_meters,
                            extent[2] * assets_to_meters])
    r_dict['extents'] = new_extents

    new_positions = []
    for position in r_dict['positions']:
        new_positions.append([position[0] * assets_to_meters,
                              position[1] * assets_to_meters,
                              position[2] * assets_to_meters])
    r_dict['positions'] = new_positions

    return r_dict

def tonemap_image(scene_path, cam, frame):
    # see: https://github.com/apple/ml-hypersim/blob/master/code/python/tools/scene_generate_images_tonemap.py
    rgb_hdf5_path = os.path.join(
        scene_path,
        'images',
        f'scene_{cam}_final_hdf5',
        f'frame.{frame:04d}.color.hdf5'
    )
    render_id_path = os.path.join(
        scene_path,
        'images',
        f'scene_{cam}_geometry_hdf5',
        f'frame.{frame:04d}.render_entity_id.hdf5'
    )

    with h5py.File(rgb_hdf5_path, 'r') as f:
        rgb_color = f['dataset'][:].astype(np.float32)

    with h5py.File(render_id_path, 'r') as f:
        render_id = f['dataset'][:].astype(np.int32)

    #
    # compute brightness according to "CCIR601 YIQ" method, use CGIntrinsics strategy for tonemapping, see [1,2]
    # [1] https://github.com/snavely/pbrs_tonemapper/blob/master/tonemap_rgbe.py
    # [2] https://landofinterruptions.co.uk/manyshades
    #

    assert np.all(render_id != 0)

    gamma = 1.0 / 2.2  # standard gamma correction exponent
    inv_gamma = 1.0 / gamma
    percentile = 90  # we want this percentile brightness value in the unmodified image...
    brightness_nth_percentile_desired = 0.8  # ...to be this bright after scaling

    valid_mask = render_id != -1

    if np.count_nonzero(valid_mask) == 0:
        scale = 1.0  # if there are no valid pixels, then set scale to 1.0
    else:
        brightness = 0.3 * rgb_color[:, :, 0] + \
                     0.59 * rgb_color[:, :, 1] + \
                     0.11 * rgb_color[:, :, 2]  # "CCIR601 YIQ" method for computing brightness
        brightness_valid = brightness[valid_mask]

        eps = 0.0001  # if the kth percentile brightness value in the unmodified image
        # is less than this, set the scale to 0.0 to avoid divide-by-zero
        brightness_nth_percentile_current = np.percentile(brightness_valid, percentile)

        if brightness_nth_percentile_current < eps:
            scale = 0.0
        else:

            # Snavely uses the following expression in the code at
            # https://github.com/snavely/pbrs_tonemapper/blob/master/tonemap_rgbe.py:
            # scale = np.exp(np.log(brightness_nth_percentile_desired)*inv_gamma -\
            #         np.log(brightness_nth_percentile_current))
            #
            # Our expression below is equivalent, but is more intuitive, because it
            # follows more directly from the expression:
            # (scale*brightness_nth_percentile_current)^gamma = brightness_nth_percentile_desired

            scale = np.power(brightness_nth_percentile_desired, inv_gamma) / brightness_nth_percentile_current

    rgb_color_tm = np.power(np.maximum(scale * rgb_color, 0), gamma)

    # clip image?
    rgb = (np.clip(rgb_color_tm, 0, 1) * 255).astype(np.uint8)

    return rgb

def load_extrinsics(scene_path, cam, frame):
    # changed in v051:
    # apply 180 degree rotation around x-axis, i.e., flipping y-axis and z-axis
    constant_rot = Rotation.from_euler('zyx', [0, 0, 180], degrees=True)

    # translation
    assets_to_meters_path = os.path.join(scene_path, '_detail',
                                         'metadata_scene.csv')
    assets_to_meters_csv = pd.read_csv(assets_to_meters_path)
    assets_to_meters = assets_to_meters_csv['parameter_value'][0]

    translation_path = os.path.join(scene_path, '_detail', cam,
                                    'camera_keyframe_positions.hdf5')
    with h5py.File(translation_path, 'r') as f:
        camera_position = f['dataset'][frame]
    camera_position *= assets_to_meters

    # rotation
    rotation_path = os.path.join(scene_path, '_detail', cam,
                                 'camera_keyframe_orientations.hdf5')
    with h5py.File(rotation_path, 'r') as f:
        camera_orientations = f['dataset'][frame]
    rotation = Rotation.from_matrix(
        np.dot(camera_orientations, constant_rot.as_matrix())
    )
    quat_x, quat_y, quat_z, quat_w = rotation.as_quat()

    return {
        'x': camera_position[0],
        'y': camera_position[1],
        'z': camera_position[2],
        'quat_x': quat_x,
        'quat_y': quat_y,
        'quat_z': quat_z,
        'quat_w': quat_w
    }

def generate_split_txt_files(split_df: pd.DataFrame, destination_path,
                             splits=None, subsample=1, use_blacklist=True):
    assert subsample > 0, "Undefined behavior for subsample < 1"
    if splits is None:
        splits = list(split_df['split_partition_name'].unique())

    f_names = HypersimMeta.get_split_filelist_filenames(subsample)

    if use_blacklist:
        for scene, camera in HypersimMeta.BLACKLIST.items():
            if camera == '*':  # exclude all trajectories of scene
                old_len = len(split_df)
                split_df = split_df[~(split_df['scene_name'] == scene)]
                print(f"Excluded scene {scene}. "
                      f"{old_len - len(split_df)} samples removed.")
            else:  # only exclude certain trajectories in scene
                old_len = len(split_df)
                split_df = split_df[~((split_df['scene_name'] == scene) & (split_df['camera_name'] == camera))]
                print(f"Excluded {camera} from scene {scene}. "
                      f"{old_len - len(split_df)} samples removed.")

    for split in splits:
        f_name = f_names[split]

        # get filepaths
        filepaths = split_df[split_df['split_partition_name'] == split]['path']
        # remove file extension
        filepaths = [fp.replace('.png', '') for fp in filepaths]
        # apply subsample
        filepaths = filepaths[::subsample]

        np.savetxt(os.path.join(destination_path, f_name),
                   filepaths,
                   fmt="%s")
        print(f"Created filelist: '{f_name}'")

def _write_image(filepath, img):
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    if not cv2.imwrite(filepath, img, CV_WRITE_FLAGS):
        raise RuntimeError(f"Could not write image: '{filepath}'")

def process_one_sample(
    index_entry,
    destination_path,
    dataset_path,
    camera_params,
    apply_tilt_shift_conversion=True
):
    _, entry = index_entry

    scene_path = os.path.join(dataset_path, entry['scene_name'])

    # process data -------------------------------------------------------------
    # -> extrinsics ------------------------------------------------------------
    extrinsics = load_extrinsics(
        scene_path,
        cam=entry['camera_name'],
        frame=entry['frame_id']
    )

    # -> rgb image with default tonemapping ------------------------------------
    rgb = tonemap_image(
        scene_path,
        cam=entry['camera_name'],
        frame=entry['frame_id']
    )

    # -> normal ----------------------------------------------------------------
    normal = load_normal(
        scene_path,
        cam=entry['camera_name'],
        frame=entry['frame_id']
    )

    # -> semantic --------------------------------------------------------------
    semantic = load_semantic(
        scene_path,
        cam=entry['camera_name'],
        frame=entry['frame_id']
    )

    # -> instance --------------------------------------------------------------
    instance_raw = load_instance(
        scene_path,
        cam=entry['camera_name'],
        frame=entry['frame_id']
    )
    # for hypersim, instances may not have a single semantic class,
    # we combine semantic and instance to ensure a unique label,
    # max value of the semantic label is 40 (min. 6 bits) and the maximum value
    # for instances is 4166 (min. 13 bits), so we need 19 bits in total, an
    # easy way to store is to just use an image with three channels
    # (3*uint8 = 24 Bit): the first channel can be used for semantic (uint8)
    # and the following two can be used for the instance label (uint16)
    instance = np.empty((*instance_raw.shape, 3), np.uint8)
    instance[:, :, 0] = semantic
    instance[:, :, 1] = (instance_raw >> 8).astype(np.uint8)
    instance[:, :, 2] = instance_raw.astype(np.uint8)

    # -> scene class -----------------------------------------------------------
    scene_class = entry['scene_class']

    #
    #
    #

    # --------------------------------------------------------------------------
    # convert to standard perspective projection with cx, cy, fx, fy

    # note, Hypersim uses non-standard perspective projection matrices (with
    # tilt-shift photography parameters) in most scenes.
    # as common frameworks, such as MIRA or ROS, do not support this
    # projection, we convert the parameters if possible or project the points
    # back to a standard camera ignoring the tilt-shift parameters
    # note that this is not a perfect conversion and introduces artifacts
    # however, rendering full images with a standard perspective projection
    # requires buying the dataset meshes
    # for more details, see https://github.com/apple/ml-hypersim/issues/24
    if apply_tilt_shift_conversion:
        # get camera parameters ------------------------------------------------
        cp = camera_params.loc[entry['scene_name']]
        width_pixels = int(cp["settings_output_img_width"])
        height_pixels = int(cp["settings_output_img_height"])

        M_cam_from_uv = np.array(
            [[cp["M_cam_from_uv_00"], cp["M_cam_from_uv_01"], cp["M_cam_from_uv_02"]],
             [cp["M_cam_from_uv_10"], cp["M_cam_from_uv_11"], cp["M_cam_from_uv_12"]],
             [cp["M_cam_from_uv_20"], cp["M_cam_from_uv_21"], cp["M_cam_from_uv_22"]]]
        )

        # TODO(dase): is this fine? -> double-check this
        # ensure -1 / 1 for last parameter
        # M_cam_from_uv = M_cam_from_uv / np.abs(cp["M_cam_from_uv_22"])

        # convert parameters to standard perspective projection parameters -----
        # u, v in range [0, 1]
        # default camera origin is at the center of the image
        intrinsics = {
            'fx': (1 / M_cam_from_uv[0, 0]) / 2,
            'fy': (1 / M_cam_from_uv[1, 1]) / 2,
            'cx': 0.5 - 0.5*M_cam_from_uv[0, 2]/M_cam_from_uv[0, 0],
            'cy': 0.5 + 0.5*M_cam_from_uv[1, 2]/M_cam_from_uv[1, 1],
            'a': 1/1000.,     # unit of depth images is mm
            'b': -1
        }
        rgb_intrinsics = {k: v
                          for k, v in intrinsics.items()
                          if k not in ('a', 'b')}
        depth_intrinsics = intrinsics

        # get point cloud ------------------------------------------------------
        # note, we do not use the precomputed point cloud as they is of type
        # float16 and, thus, not precise enough for our purposes (i.e,
        # introduces more artifacts in the back-projection)
        points_cam = compute_point_cloud(
            scene_path,
            cam=entry['camera_name'],
            frame=entry['frame_id'],
            camera_params=cp
        )

        # get depth image as z coordinate from point cloud ---------------------
        depth = points_cam[:, :, -1]    # use z as depth
        depth = depth * 1000    # convert to mm
        depth[depth > np.iinfo('uint16').max] = 0    # clip to 65.535m
        depth = depth.astype('uint16')

        # project points back to standard perspective camera -------------------
        height = height_pixels    # render at same resolution
        width = width_pixels    # render at same resolution
        points_cam_flat = points_cam.reshape((-1, 3))
        points_cam_flat_uv = points_cam_flat / points_cam_flat[:, -1:]  # z = 1
        points_cam_flat_uv = points_cam_flat_uv[:, :2]  # remove z
        points_cam_flat_uv *= np.array([[intrinsics['fx'], intrinsics['fy']]])
        points_cam_flat_uv += np.array([[intrinsics['cx'], intrinsics['cy']]])
        points_cam_flat_uv *= np.array([[width-0.5, height-0.5]])

        # scale to pixels and clip points to image
        points_cam_flat_uv = points_cam_flat_uv.astype('int16')
        np.clip(points_cam_flat_uv[:, 0], 0, width-1,
                out=points_cam_flat_uv[:, 0])
        np.clip(points_cam_flat_uv[:, 1], 0, height-1,
                out=points_cam_flat_uv[:, 1])

        # remove non-unique elements (multiple points may project to same pixel)
        unique_uv, indices, counts = np.unique(points_cam_flat_uv,
                                               axis=0,
                                               return_index=True,
                                               return_counts=True)

        unique_uv = unique_uv[counts == 1]
        indices = indices[counts == 1]

        # create mapped images -------------------------------------------------
        # TODO(dase): do not compute mapping for each sample, it is fixed for
        # all samples of a camera trajectory (and an entire scene)
        rgb_mapped = np.zeros((height, width, 3), dtype='uint8')
        depth_mapped = np.zeros((height, width), dtype='uint16')
        normal_mapped = np.zeros((height, width, 3), dtype='uint8')
        semantic_mapped = np.zeros((height, width), dtype='uint8')
        instance_mapped = np.zeros((height, width, 3), dtype='uint8')

        u, v = points_cam_flat_uv[indices, 0], points_cam_flat_uv[indices, 1]
        rgb_mapped[v, u] = rgb.reshape(-1, 3)[indices]
        depth_mapped[v, u] = depth.reshape(-1)[indices]
        normal_mapped[v, u] = normal.reshape(-1, 3)[indices]
        semantic_mapped[v, u] = semantic.reshape(-1)[indices]
        instance_mapped[v, u] = instance.reshape(-1, 3)[indices]

        rgb = rgb_mapped
        depth = depth_mapped
        normal = normal_mapped
        semantic = semantic_mapped
        instance = instance_mapped

    # --------------------------------------------------------------------------
    else:
        # OLD way (< 0.5.0)
        # note that code below do not allow correct projection into 3D, however,
        # training in 2D may be fine
        # see: https://github.com/apple/ml-hypersim/issues/24
        width_pixels = 1024
        height_pixels = 768
        fov_x = np.pi/3.0
        fov_y = fov_x*(height_pixels/width_pixels)
        f_x = (width_pixels/np.tan(fov_x/2))/2
        f_y = (height_pixels/np.tan(fov_y/2))/2
        intrinsics = {
            'fx': f_x/width_pixels,
            'fy': f_y/height_pixels,
            'cx': 0.5,
            'cy': 0.5,
            'a': 1/1000.,     # unit of depth images is mm
            'b': -1
        }
        rgb_intrinsics = {k: v
                          for k, v in intrinsics.items()
                          if k not in ('a', 'b')}
        depth_intrinsics = intrinsics
        depth = load_depth_old(
            scene_path,
            cam=entry['camera_name'],
            frame=entry['frame_id']
        )
    #
    #
    #

    # -> 3D bounding boxes and orientations ------------------------------------
    boxes_3d_raw = load_box_3d(
        scene_path,
        cam=entry['camera_name'],
        frame=entry['frame_id']
    )

    # the loaded box dict has orientations for instances that are part of the
    # scene but not in the current image, we suppress these orientations by
    # only considering the unique ids based on the current image
    # moreover, the data structure is changed (we extract the box parameters
    # for each instance id)
    boxes_3d = {}
    instance_ids_image = instance[:, :, 1].astype('uint16') << 8
    instance_ids_image += instance[:, :, 2].astype('uint16')

    for box_idx in np.unique(instance_ids_image):
        if box_idx == 0:    # void
            continue

        new_value = {}
        for key, value in boxes_3d_raw.items():
            new_value[key] = value[box_idx]

        # also we need to use the changed instance value
        boxes_3d[int(box_idx)] = new_value

    # calculate orientation
    cam_rot = Rotation.from_quat((extrinsics['quat_x'],
                                  extrinsics['quat_y'],
                                  extrinsics['quat_z'],
                                  extrinsics['quat_w']))
    orientations = {}
    for key, box in boxes_3d.items():
        box_orientation = Rotation.from_matrix(box["orientations"])
        # only take rotation around z-axis and convert to rad
        obj_rot_deg = np.rad2deg(box_orientation.as_rotvec()[2])
        cam_rot_deg = np.rad2deg(cam_rot.as_rotvec()[2])
        obj_rot_in_cam = ((obj_rot_deg-cam_rot_deg)-180) % 360
        orientations[key] = np.deg2rad(obj_rot_in_cam)

    # write stuff to disk ------------------------------------------------------
    # -> extrinsics ------------------------------------------------------------
    extrinsic_path = os.path.join(
        destination_path,
        entry['split_partition_name'],
        HypersimMeta.EXTRINSICS_DIR,
        entry['path'].replace('png', 'json')
    )
    os.makedirs(os.path.dirname(extrinsic_path), exist_ok=True)
    with open(extrinsic_path, 'w') as f:
        json.dump(extrinsics, f, indent=4)

    # -> intrinsics ------------------------------------------------------------
    for intrinsics, dirname in zip((rgb_intrinsics, depth_intrinsics),
                                   (HypersimMeta.RGB_INTRINSICS_DIR,
                                    HypersimMeta.DEPTH_INTRINSICS_DIR)):
        intrinsic_path = os.path.join(
            destination_path,
            entry['split_partition_name'],
            dirname,
            entry['scene_name'],
            f'{entry["camera_name"]}.json'
        )
        os.makedirs(os.path.dirname(intrinsic_path), exist_ok=True)
        with open(intrinsic_path, 'w') as f:
            json.dump(intrinsics, f, indent=4)

    # -> rgb image with default tonemapping ------------------------------------
    rgb_path = os.path.join(
        destination_path,
        entry['split_partition_name'],
        HypersimMeta.RGB_DIR,
        entry['path']
    )
    # image gets converted to BGR as we write it using OpenCV
    _write_image(rgb_path, cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR))

    # -> depth -----------------------------------------------------------------
    depth_path = os.path.join(
        destination_path, entry['split_partition_name'],
        HypersimMeta.DEPTH_DIR,
        entry['path']
    )
    _write_image(depth_path, depth)

    # -> normal ----------------------------------------------------------------
    normal_path = os.path.join(
        destination_path, entry['split_partition_name'],
        HypersimMeta.NORMAL_DIR,
        entry['path']
    )
    # convert xyz to zyx as cv2.imwrite will swap it later again
    _write_image(normal_path, cv2.cvtColor(normal, cv2.COLOR_RGB2BGR))

    # -> semantic --------------------------------------------------------------
    semantic_path = os.path.join(
        destination_path,
        entry['split_partition_name'],
        HypersimMeta.SEMANTIC_DIR,
        entry['path']
    )
    semantic_colored_path = os.path.join(
        destination_path,
        entry['split_partition_name'],
        HypersimMeta.SEMANTIC_COLORED_DIR,
        entry['path']
    )
    _write_image(semantic_path, semantic)
    # additionally save a colored version as indexed png
    os.makedirs(os.path.dirname(semantic_colored_path), exist_ok=True)
    save_indexed_png(semantic_colored_path, semantic,
                     colormap=HypersimMeta.CLASS_COLORS)

    # -> instance --------------------------------------------------------------
    instances_path = os.path.join(
        destination_path,
        entry['split_partition_name'],
        HypersimMeta.INSTANCES_DIR,
        entry['path']
    )
    # image gets converted to BGR as we write it using OpenCV
    _write_image(instances_path, cv2.cvtColor(instance, cv2.COLOR_RGB2BGR))
    instance = cv2.cvtColor(instance, cv2.COLOR_RGB2BGR)

    # -> scene class -----------------------------------------------------------
    scene_class_path = os.path.join(
        destination_path,
        entry['split_partition_name'],
        HypersimMeta.SCENE_CLASS_DIR,
        entry['path'].replace('png', 'txt')
    )
    os.makedirs(os.path.dirname(scene_class_path), exist_ok=True)
    with open(scene_class_path, 'w') as f:
        f.write(scene_class + "\n")

    # -> 3D bounding boxes and orientations ------------------------------------
    # write boxes
    boxes_3d_path = os.path.join(
        destination_path,
        entry['split_partition_name'],
        HypersimMeta.BOXES_3D_DIR,
        entry['path'].replace('png', 'json')
    )
    os.makedirs(os.path.dirname(boxes_3d_path), exist_ok=True)
    with open(boxes_3d_path, 'w') as f:
        json.dump(boxes_3d, f, indent=4)

    # write orientations
    orientation_path = os.path.join(
        destination_path,
        entry['split_partition_name'],
        HypersimMeta.ORIENTATIONS_DIR,
        entry['path'].replace('png', 'json')
    )
    os.makedirs(os.path.dirname(orientation_path), exist_ok=True)
    with open(orientation_path, 'w') as f:
        json.dump(orientations, f, indent=4)

def main(args=None):
    # args ---------------------------------------------------------------------
    args = _parse_args(args)
    dataset_path = args.hypersim_path
    destination_path = args.output_path
    additional_subsamples = args.additional_subsamples

    os.makedirs(destination_path, exist_ok=True)

    # write meta file ----------------------------------------------------------
    create_or_update_creation_metafile(
        destination_path,
        additional_subsamples=args.additional_subsamples,
        apply_tilt_shift_conversion=not args.no_tilt_shift_conversion
    )

    # load splits csv file and filter scenes and cameras -----------------------
    splits_csv_path = os.path.join(dataset_path,
                                   'metadata_images_split_scene_v1.csv')
    if not os.path.exists(splits_csv_path):
        print(f"Downloading splits csv file to: `{splits_csv_path}`")
        download_file(SPLITS_CSV_URL, splits_csv_path,
                      display_progressbar=True)

    split_df = pd.read_csv(splits_csv_path)
    # remove excluded images
    split_df = split_df[split_df['included_in_public_release']]
    split_df.reset_index(drop=True, inplace=True)

    splits = HypersimMeta.SPLITS
    split_df['path'] = split_df["scene_name"] + '/' + split_df["camera_name"]
    split_df['path'] += '/' + split_df["frame_id"].map('{:04d}'.format) + '.png'

    # map val to valid (= HypersimBase.SPLITS[1])
    split_df.loc[split_df['split_partition_name'] == 'val',
                 'split_partition_name'] = 'valid'

    generate_split_txt_files(split_df, destination_path, splits)
    for subsample in additional_subsamples:
        generate_split_txt_files(split_df, destination_path, splits,
                                 subsample=subsample)

    # create folders for each split -----------------------------------------
    for split in splits:
        for sub_f in [HypersimMeta.RGB_DIR,
                      HypersimMeta.RGB_INTRINSICS_DIR,
                      HypersimMeta.DEPTH_DIR,
                      HypersimMeta.DEPTH_INTRINSICS_DIR,
                      HypersimMeta.EXTRINSICS_DIR,
                      HypersimMeta.SEMANTIC_DIR,
                      HypersimMeta.SEMANTIC_COLORED_DIR,
                      HypersimMeta.INSTANCES_DIR,
                      HypersimMeta.BOXES_3D_DIR,
                      HypersimMeta.ORIENTATIONS_DIR,
                      HypersimMeta.SCENE_CLASS_DIR]:
            os.makedirs(
                os.path.join(destination_path,
                             HypersimMeta.SPLIT_DIRS[split],
                             sub_f),
                exist_ok=True
            )

    # download and prepare data for scene class --------------------------------
    scene_class_csv_path = os.path.join(dataset_path,
                                        'metadata_camera_trajectories.csv')
    if not os.path.exists(scene_class_csv_path):
        print(f"Downloading scene csv file to: `{scene_class_csv_path}`")
        download_file(SCENE_CLASS_URL, scene_class_csv_path,
                      display_progressbar=True)

    scene_class_df = pd.read_csv(scene_class_csv_path)
    scene_class_animation_list = scene_class_df['Animation'].tolist()
    scene_class_animation_list = [
        re.split("(_cam_[0-9][0-9])", x)[:2]
        for x in scene_class_animation_list
    ]
    # remove leading underscore
    scene_class_animation_list = [
        [a, b[1:]]
        for a, b in scene_class_animation_list
    ]
    scene_class_list = scene_class_df['Scene type'].tolist()

    # match scene class to split_df
    scene_class_dict = {}
    for idx, (scene_name, camera_name) in enumerate(scene_class_animation_list):
        if scene_name not in scene_class_dict:
            scene_class_dict[scene_name] = {camera_name: scene_class_list[idx]}
        else:
            scene_class_dict[scene_name][camera_name] = scene_class_list[idx]

    scene_names = split_df['scene_name']
    camera_names = split_df['camera_name']
    scene_classes = [
        scene_class_dict[scene_names[idx]][camera_names[idx]]
        for idx in range(len(split_df))
    ]

    # add scene class column to existing df
    split_df.insert(3, 'scene_class', scene_classes)

    # load splits csv file and filter scenes and cameras -----------------------
    camera_parameters_csv_path = os.path.join(
        dataset_path,
        'metadata_camera_parameters.csv.csv'
    )
    if not os.path.exists(camera_parameters_csv_path):
        print(f"Downloading splits csv file to: `{camera_parameters_csv_path}`")
        download_file(CAMERA_PARAMETER_CSV_URL, camera_parameters_csv_path,
                      display_progressbar=True)
    camera_parameters_df = pd.read_csv(camera_parameters_csv_path,
                                       index_col='scene_name')

    # for testing
    # split_df = split_df[split_df['scene_name'] == 'ai_021_002']

    # process samples ----------------------------------------------------------
    start_time = datetime.datetime.now()
    if args.n_processes == 1:
        for index, entry in tqdm(split_df.iterrows(), total=len(split_df)):
            process_one_sample(
                index_entry=(index, entry),
                destination_path=destination_path,
                dataset_path=dataset_path,
                camera_params=camera_parameters_df,
                apply_tilt_shift_conversion=not args.no_tilt_shift_conversion
            )
    else:
        fun_call = functools.partial(
            process_one_sample,
            destination_path=destination_path,
            dataset_path=dataset_path,
            camera_params=camera_parameters_df,
            apply_tilt_shift_conversion=not args.no_tilt_shift_conversion
        )

        process_map(fun_call, split_df.iterrows(),
                    max_workers=args.n_processes,
                    total=len(split_df))

    print(f"Preparing Hypersim dataset was successful. "
          f"Dataset path: {destination_path}. "
          f"It took {datetime.datetime.now() - start_time}")

if __name__ == '__main__':
    main()


from ..coefficient_array import PwCoeffs
from scipy.sparse import dia_matrix
import numpy as np

def make_kinetic_precond(kpointset, c0, eps=0.1, asPwCoeffs=True):
    """
    Preconditioner
    P = 1 / (||k|| + ε)

    Keyword Arguments:
    kpointset --
    """

    nk = len(kpointset)
    nc = kpointset.ctx().num_spins()
    if nc == 1 and nk == 1 and not asPwCoeffs:
        # return as np.matrix
        kp = kpointset[0]
        gkvec = kp.gkvec()
        # assert (gkvec.num_gvec() == gkvec.count())
        N = gkvec.count()
        d = np.array([
            1 / (np.sum((np.array(gkvec.gkvec(i)))**2) + eps)
            for i in range(N)
        ])
        return DiagonalPreconditioner(
            D=dia_matrix((d, 0), shape=(N, N)), c0=c0)
    else:
        P = PwCoeffs(dtype=np.float64, ctype=dia_matrix)
        for k in range(nk):
            kp = kpointset[k]
            gkvec = kp.gkvec()
            # assert (gkvec.num_gvec() == gkvec.count())
            N = gkvec.count()
            d = np.array([
                1 / (np.sum(
                    (np.array(gkvec.gkvec_cart(i)))**2) + eps)
                for i in range(N)
            ])
            for ispn in range(nc):
                P[k, ispn] = dia_matrix((d, 0), shape=(N, N))
        return DiagonalPreconditioner(P, c0)

class Preconditioner:
    def __init__(self):
        pass

class DiagonalPreconditioner(Preconditioner):
    """
    Apply diagonal preconditioner and project resulting gradient to satisfy the constraint.
    """

    def __init__(self, D, c0):
        super().__init__()
        self.c0 = c0
        self.D = D

    def __matmul__(self, other):
        """
        """
        from ..coefficient_array import CoefficientArray
        from .ot_transformations import lagrangeMult

        out = type(other)(dtype=other.dtype)
        if isinstance(other, CoefficientArray):
            for key, Dl in self.D.items():
                out[key] = Dl * other[key]
        else:
            raise ValueError('wrong type given')
        ll = lagrangeMult(other, self.c0, self)
        return out + ll

    def __mul__(self, s):
        """

        """
        from ..coefficient_array import CoefficientArray
        import numpy as np

        if np.isscalar(s):
            for key, Dl in self.D.items():
                self.D[key] = s*Dl
        elif isinstance(s, CoefficientArray):
            out = type(s)(dtype=s.dtype)
            for key in s.keys():
                out[key] = self.D[key] * s[key]
            return out

    __lmul__ = __mul__
    __rmul__ = __mul__

    def __neg__(self):
        """
        """
        from ..coefficient_array import CoefficientArray
        if isinstance(self.D, CoefficientArray):
            out_data = type(self.D)(dtype=self.D.dtype, ctype=self.D.ctype)
            out = DiagonalPreconditioner(out_data, self.c0)
            for k, v in self.D.items():
                out.D[k] = -v
            return out
        else:
            out = DiagonalPreconditioner(self.D, self.c0)
            out.D = -self.D
            return out

    def __getitem__(self, key):
        return self.D[key]

class IdentityPreconditioner(Preconditioner):

    def __init__(self, c0, _f=1):
        super().__init__()
        self.c0 = c0
        self._f = _f

    def __matmul__(self, other):
        from .ot_transformations import lagrangeMult

        ll = lagrangeMult(other, self.c0, self)
        return self._f * other + ll

    def __mul__(self, s):
        return self._f * s

    def __neg__(self):
        return IdentityPreconditioner(self.c0, _f=-self._f)

    def __getitem__(self, key):
        return self._f

    __lmul__ = __mul__
    __rmul__ = __mul__


#This line returns a list of unique words in a file.
fname=input("Enter file: ")
fhand=open(fname)
uniquewords=[]
for line in fhand:
    spl=line.rstrip()
    spl=line.split()
    for word in spl:
        if word not in uniquewords:
            uniquewords.append(word)
uniquewords.sort()
print(uniquewords)


import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.colors as colors

debug_mode = False

def main():
    map_dependencies = {}
    subsystems = [ "AUD", "COR", "DEB", "EDI", "FES", "GMP", "HID", "LLR", "OMP", "PHY", "PLA", "RES", "SDK", "SGC", "SKA", "VFX" ]
    ds = pd.read_csv("inputs/matrix_all.csv", sep=",")
    ds = ds.drop(columns=['asubsystem'])
    print(ds)
    sum = np.array(ds.values)

    # do not annotate values 0 and 1
    annot_values = np.array(sum, dtype=str)
    annot_values = np.where(sum < 3, np.array([""], dtype="U1"), annot_values)

    # set figure size
    sns.set(rc={"figure.figsize": (12, 8)})

    # pallette chosen because it is good for colorblind
    colormap = sns.color_palette("bwr")

    cmap_colors = ['#ffeceb', '#fcdad7', '#fa6557', '#9c0f02' ]
    colormap = colors.ListedColormap(cmap_colors)

    hm = sns.heatmap(
        sum.tolist(),
        vmin=0,
        vmax=9,
        annot=annot_values,
        xticklabels=subsystems,
        yticklabels=subsystems,
        cmap=colormap,
        cbar=True,
        fmt="",
    )
    fig = hm.get_figure()
    fig.savefig("outputs/heatmap.pdf")

    # generate csv for column and row counts
    sumRows(sum, subsystems)
    sumCols(sum, subsystems)

# get the sum for each column in the heatmap
# useful to understand how much included a subsystem is
def sumCols(matrix, subsystems):
    vals = []
    sub_count = len(subsystems)
    for i in range(0, sub_count):
        vals.append(matrix[:, i].sum())
    ds_most_inc_by = pd.DataFrame()
    ds_most_inc_by["subsystem"] = subsystems
    ds_most_inc_by["included_by"] = vals
    result = ds_most_inc_by.sort_values(by=["included_by"], ascending=False)
    result.to_csv("outputs/included_by.csv")

# get the sum for each row in the heatmap
# useful to understand how much a subsystem includes
def sumRows(matrix, subsystems):
    count = 0
    vals = []
    for line in matrix:
        vals.append(line.sum())
        count += 1
    ds_most_inc = pd.DataFrame()
    ds_most_inc["subsystem"] = subsystems
    ds_most_inc["include"] = vals
    result = ds_most_inc.sort_values(by=["include"], ascending=False)
    result.to_csv("outputs/include.csv")

main()


from fastapi import APIRouter
from fastapi import status
from typing import List
from fastapi import Path
from fastapi import HTTPException
from models import Pedido
import uuid
from rotas import rota_produtos

pedidos_db = []

rota_pedidos = APIRouter()

@rota_pedidos.post("/",
                    status_code=status.HTTP_201_CREATED,
                    response_model=Pedido,
                    summary="Criar Pedido",
                    description="Cria pedido com os produtos.")
async def criar_pedido(pedido: Pedido):
    pedido_total = 0
    pedido_items = []
    for item in pedido.items:
        produto_id = item.produto_id
        quantity = item.quantity
        matching_produtos = [p for p in rota_produtos.produtos_db if p["id"] == produto_id]
        if not matching_produtos:
            raise HTTPException(status_code=400, detail=f"Produto ID {produto_id} não existe")
        produto = matching_produtos[0]
        produto_preco = produto["preco"]
        pedido_total += produto_preco * quantity
        pedido_items.append(item)
    pedido_id = uuid.uuid4()
    pedido.id = pedido_id
    pedido.items = pedido_items
    pedidos_db.append(pedido.dict())
    return pedido

@rota_pedidos.get("/",
                    response_model=List[Pedido],
                    summary="Consultar Pedidos",
                    description="Retorna uma lista de pedidos disponíveis ou uma lista vazia.")
async def list_pedidos():
    return pedidos_db

@rota_pedidos.get("/{pedido_id}",
                    response_model=Pedido,
                    summary="Consultar Pedido por ID",
                    description="Retorna um pedido por seu ID.")
async def get_pedido(pedido_id: uuid.UUID = Path(
                                                default=None,
                                                title='ID do tipo uui',
                                                description='Deve ser do tipo uuid')):
    matching_pedidos = [o for o in pedidos_db if o["id"] == pedido_id]
    if not matching_pedidos:
        raise HTTPException(status_code=404, detail=f"Pedido com ID {pedido_id} não encontrado")
    return matching_pedidos[0]

@rota_pedidos.delete("/{pedido_id}",
                    response_model=Pedido,
                    summary="Excluir Pedido",
                    description="Exclui um pedido por seu ID.")
async def delete_pedido(pedido_id: uuid.UUID):
    for idx, pedido in enumerate(pedidos_db):
        if pedido["id"] == pedido_id:
            deleted_pedido = pedidos_db.pop(idx)
            return deleted_pedido
    raise HTTPException(status_code=404, detail=f"Pedido com ID {pedido_id} não encontrado")


# ----------------------------------------------------------------------
# |
# |  Binary_BuildVerificationTest.py
# |
# |  David Brownell <db@DavidBrownell.com>
# |      2022-11-22 10:42:13
# |
# ----------------------------------------------------------------------
# |
# |  Copyright David Brownell 2022-23
# |  Distributed under the Boost Software License, Version 1.0. See
# |  accompanying file LICENSE_1_0.txt or copy at
# |  http://www.boost.org/LICENSE_1_0.txt.
# |
# ----------------------------------------------------------------------
"""Build verification test for binaries"""

# Note that this file will be invoked outside of an activated environment and cannot take a dependency
# on anything in this repository or Common_Foundation.

import os
import subprocess
import stat
import sys
import textwrap

from pathlib import Path
from typing import List, Optional, Set

# ----------------------------------------------------------------------
def EntryPoint(
    args: List[str],
) -> int:
    if len(args) != 2:
        sys.stdout.write(
            textwrap.dedent(
                """\
                ERROR: Usage:

                    {} <temp_directory>

                """,
            ).format(
                args[0],
            ),
        )

        return -1

    temp_directory = Path(args[1])
    assert temp_directory.is_dir(), temp_directory

    build_output_dir = Path(temp_directory)

    # Get the Backup binary
    for potential_dir in [
        "artifacts",
        "Scripts",
        "Backup",
        "Build",
    ]:
        potential_build_output_dir = build_output_dir / potential_dir
        if not potential_build_output_dir.is_dir():
            break

        build_output_dir = potential_build_output_dir

    backup_filename = build_output_dir / "Backup"

    if not backup_filename.is_file():
        potential_backup_filename = backup_filename.with_suffix(".exe")

        if potential_backup_filename.is_file():
            backup_filename = potential_backup_filename

    if not backup_filename.is_file():
        raise Exception("The filename '{}' does not exist.\n".format(backup_filename))

    # https://github.com/actions/upload-artifact/issues/38
    # Permissions are not currently being saved when uploading artifacts, so they must be set here.
    # This will eventually be fixed, which is why I am placing the work around here rather than in
    # the artifact upload- or download-code.

    backup_filename.chmod(stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR)

    # Execute Tests
    result = _ValidateMirror(backup_filename, temp_directory)
    if result != 0:
        return result

    return 0

# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
def _ValidateMirror(
    backup_filename: Path,
    temp_directory: Path,
) -> int:
    source_dir = Path(__file__).parent.parent.parent.parent
    destination = source_dir.parent / "destination"

    command_line = '"{}" mirror execute "{}" "{}"'.format(
        backup_filename,
        destination,
        source_dir,
    )

    sys.stdout.write("Command Line: {}\n\n".format(command_line))

    result = subprocess.run(
        command_line,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
    )

    content = result.stdout.decode("utf-8")

    sys.stdout.write(content)

    if result.returncode != 0:
        return result.returncode

    # Compare the source and destination files

    # ----------------------------------------------------------------------
    def DecorateFilename(
        filename: Path,
        root: Path,
        prefix_to_strip: Optional[Path]=None,
    ) -> Path:
        filename = Path(*filename.parts[len(root.parts):])

        if prefix_to_strip:
            len_prefix_to_strip_parts = len(prefix_to_strip.parts)

            assert filename.parts[:len_prefix_to_strip_parts] == prefix_to_strip.parts, (filename.parts, prefix_to_strip.parts)
            filename = Path(*filename.parts[len_prefix_to_strip_parts:])

        return filename

    # ----------------------------------------------------------------------
    def GetFiles(
        root: Path,
        prefix_to_strip: Optional[Path]=None,
    ) -> Set[Path]:
        if prefix_to_strip:
            if prefix_to_strip.parts[0].endswith(":") or prefix_to_strip.parts[0].endswith(":\\"):
                prefix_to_strip = Path(prefix_to_strip.parts[0].replace(":", "_").rstrip("\\")) / Path(*prefix_to_strip.parts[1:])
            else:
                assert prefix_to_strip.parts[0] == "/", prefix_to_strip.parts
                prefix_to_strip = Path(*prefix_to_strip.parts[1:])

        results: Set[Path] = set()

        for this_root, directories, filenames in os.walk(root):
            this_root = Path(this_root)

            if not directories and not filenames:
                results.add(DecorateFilename(this_root, root, prefix_to_strip))

            for filename in filenames:
                results.add(DecorateFilename(this_root / filename, root, prefix_to_strip))

        return results

    # ----------------------------------------------------------------------

    source_files = GetFiles(source_dir)
    destination_files = GetFiles(destination / "Content", source_dir)

    if source_files != destination_files:
        sys.stdout.write("Source Files:\n{}\n".format("".join("  - {}) {}\n".format(index, source_file) for index, source_file in enumerate(source_files))))
        sys.stdout.write("Destination Files:\n{}\n".format("".join("  - {}) {}\n".format(index, destination_file) for index, destination_file in enumerate(destination_files))))

        return -1

    return 0

# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
if __name__ == "__main__":
    sys.exit(EntryPoint(sys.argv))


from fastapi import APIRouter

from models.artist import ArtistData
from schemas.base import BaseSerializer
from services import artist as svc

router = APIRouter(
    prefix="/artist",
    tags=["artist"],
    dependencies=[],
)

class Serializer(BaseSerializer):
    data: list[ArtistData]

@router.get("/", response_model=Serializer)
async def listing():
    data = await svc.listing()
    return Serializer(data=data)

@router.get("/{artist}")
async def detail(artist: int):
    data = await svc.detail(artist)
    return Serializer(data=[data])


#import filetype
from picamera import PiCamera
import picamera
from time import sleep
import datetime as dt
#now = datetime.datetime.now()

camera = PiCamera()

camera.start_preview(alpha=180)
camera.annotate_text_size = 80
camera.annotate_background = picamera.Color('black')
camera.annotate_text=dt.datetime.now().strftime("%H:%M:%S")
camera.start_recording('/home/pi/Desktop/testvid2.h264')
start = dt.datetime.now()
while (dt.datetime.now() - start).seconds < 3:
        camera.annotate_text = dt.datetime.now().strftime("%H:%M:%S")
        camera.wait_recording(0.2)
#sleep(5)
camera.stop_recording()
camera.stop_preview()


import pandas as pd
import os
import re
from Utils import logger
from enum import Enum

excelNameKey = '立项申请表'
moduleColumns = ['文件路径' ,'项目名称', '最终用户', '签约甲方', '商务路线', '项目说明', '申请人（销售）',
'申请人所属部门', '申请日期', '项目预算', '投标保证金', '联系电话', '投标方式', '投标时间', '付款方式']

enum_key_error_text = {
    1 : '请填写要抓取的关键字',
    2 : '有重复的关键字，分别是：{}'
}

class key_error(Enum):
    success = 0
    no_keys = 1
    duplicate_keys = 2

class DataHandler(object):
    def __init__(self):
        super(DataHandler,self).__init__()
        # 明细数据，原始数据
        self.resData = None

        self.files = {}
        self.excelDict = {}
        self.keyWords = []
        # 需要抓取的文件名的关键字
        self.fileNameKeys = []
        self.colValuesDict = {}

    def InitFileKeysArray(self, fileKeys):
        if(len(fileKeys) > 0):
            fileKeysArray = fileKeys.split('#')
            # 强制带上路径信息
            for key in fileKeysArray:
                keyStr = self.GetRemovedSpaceStr(key)
                if keyStr != '':
                    logger.info("需要抓取文件名的关键字：%s", keyStr)
                    self.fileNameKeys.append(keyStr)

    def ReadFileFolder(self, fileFolderPath):
        if(len(fileFolderPath) > 0):
            self.files = os.walk(fileFolderPath)
            for root, dirs, files in self.files:
                for filePath in files:
                    absPath = os.path.join(root, filePath)
                    path, fileName = os.path.split(absPath)
                    # fullPath, ext = os.path.splitext(absPath)
                    if self.IsValidFile(fileName):
                        curExcel = self.ReadFolderExcel(absPath)
                        self.excelDict[absPath] = curExcel

                    # if ext == ".xls":
                    #     curExcel = self.ReadFolderExcel(absPath)
                    #     print("XLS:" + absPath)
                    # elif ext == ".xlsx":
                    #     print("XLSX:" + absPath)

    def IsValidFile(self, fileName):
        if(len(self.fileNameKeys) > 0):
            for fileKey in self.fileNameKeys:
                if fileKey in fileName:
                    return True
        return False

    def ReadFolderExcel(self, fPath):
        if(len(fPath) > 0):
            return pd.read_excel(fPath)

    def SearchContentFromFiles(self, keys):
        self.InitKeyArray(keys)
        if len(self.keyWords) > 0 and len(self.excelDict) > 0:
            for key in self.excelDict.keys():

                value = self.excelDict[key]
                # print(value.columns)
                for col in self.keyWords:
                    listofPos = self.GetIndexes(value, col)
                    finalValue = ""
                    if len(listofPos) > 0:
                        rowIndex = listofPos[0][0]
                        colIndex = listofPos[0][1]
                        finalValue = value.iloc[rowIndex, colIndex + 1]
                    else:
                        if not self.IsForcedCol(col):
                            logger.warning("在文件%s 没找到的列名%d", key, col)

                    if self.colValuesDict.get(col) == None:
                        # print("列名为空，要初始化，列名是：", key, col)
                        colValues = list()
                        if self.IsForcedCol(col):
                            finalValue = key

                        colValues.append(finalValue)
                        self.colValuesDict[col] = colValues
                    else:
                        if self.IsForcedCol(col):
                            finalValue = key
                        self.colValuesDict[col].append(finalValue)

    def InitKeyArray(self, keys):
        self.keyWords.append('文件路径')
        keyArray = keys.split('#')
        # 强制带上路径信息
        for key in keyArray:
            keyStr = self.GetRemovedSpaceStr(key)
            if keyStr != '':
                logger.info("需要抓取关键字：%s", keyStr)
                self.keyWords.append(keyStr)

    def IsForcedCol(self, colStr):
        return colStr == '文件路径'

    def GetRemovedSpaceStr(self, strParam):
        if isinstance(strParam, str):
            pattern = re.compile(r'\s+')
            return re.sub(pattern, '', strParam)
        return ''

    def MatchRemoveSpace(self, x, value):
        if isinstance(x, str):
            tempX = self.GetRemovedSpaceStr(x)
            # 通过正则表达式匹配去除文本里面的空格，提高模糊匹配能力(不能简单只去除首尾空格)
            # x = x.strip()
            return value in tempX
        else:
            return False

    def BuildDataFrame(self, outputFolderPath):
        # for col, colValues in self.colValuesDict.items():
        #     for values in colValues:
        #         print("列名：", col, "列的值：", values)
        #     print("************************")

        logger.info("Find Excel File Count: %d", len(self.excelDict))
        logger.info("BuildDataFrame %d", len(self.colValuesDict))
        if len(self.colValuesDict) > 0:
            df = pd.DataFrame(self.colValuesDict)
            outputPath = os.path.join(outputFolderPath, "output.xlsx")
            df.to_excel(outputPath, index=False, columns=self.keyWords)

    # 通过某个值获取DataFrame中该值对应的行列数组
    def GetIndexes(self, dfObj, value):
        listOfPos = list()
        # 得到值均为布尔值的DataFrame
        # result = dfObj.applymap(lambda x: (value in str(x)))
        result = dfObj.applymap(lambda x: self.MatchRemoveSpace(x, value))

        # 得到包含该值的列的数组
        seriesObj = result.any()

        columnNames = list(seriesObj[seriesObj == True].index)
        # 得到包含该值的行的数组
        for col in columnNames:
            rows = list(result[col][result[col] == True].index)
            colIndex = result.columns.get_loc(col)
            for row in rows:
                listOfPos.append((row, colIndex, col))
        return listOfPos

    def ReadExcel(self, fPath):
        if(len(fPath) > 0):
            # 忽略前两行
            data = pd.read_excel(io=fPath, sheet_name=None, header=0, skiprows=2)
            self.resData = data['明细']


import curses
import time
import random

def replit_warning(stdscr):
    stdscr.clear()
    stdscr.addstr('WARNING: You are using this project on Replit which is known to have issues with how the text is displayed. For the best experience download this project from GitHub.\nTo exit press Escape\nIf you still want to procede, press any other key.', curses.color_pair(4))
    stdscr.refresh()
    key = stdscr.getkey()
    if ord(key) == 27:
        exit('Goodbye! The GitHub repository for this project is at https://github.com/DillonB07/WPM-Test')

def start_screen(stdscr):
    stdscr.clear()
    stdscr.addstr('Welcome to the Speed Typing Test!')
    stdscr.addstr('\nPress any key to start the test.')
    stdscr.refresh()
    stdscr.getkey()

def load_text():
    with open('text.txt', 'r') as f:
        lines = f.readlines()
        return random.choice(lines).strip()

def display_text(stdscr, target, current, wpm=0):
    stdscr.addstr(target)
    stdscr.addstr(1, 0, f'WPM: {wpm}')

    for i, char in enumerate(current):
        correct_char = target[i]
        color = curses.color_pair(1)
        if char != correct_char:
            color = curses.color_pair(2)

        stdscr.addstr(0, i, char, color)

def wpm_test(stdscr):
    target_text = load_text()
    current_text = []
    wpm = 0
    start_time = time.perf_counter()
    stdscr.nodelay(True)

    while True:
        time_elapsed = max(time.perf_counter() - start_time, 1)
        wpm = round((len(current_text) / (time_elapsed / 60)) / 5)

        stdscr.clear()
        display_text(stdscr, target_text, current_text, wpm)
        stdscr.refresh()

        if ''.join(current_text) == target_text:
            stdscr.nodelay(False)
            break

        try:
            key = stdscr.getkey()
        except curses.error:
            continue

        if ord(key) == 27:
            break

        if key in ('KEY_BACKSPACE', '\b', '\x7f'):
            if current_text:
                current_text.pop()
        elif len(current_text) < len(target_text):
            current_text.append(key)

def main(stdscr):
    curses.init_pair(1, curses.COLOR_GREEN, curses.COLOR_BLACK)
    curses.init_pair(2, curses.COLOR_RED, curses.COLOR_BLACK)
    curses.init_pair(3, curses.COLOR_WHITE, curses.COLOR_BLACK)
    curses.init_pair(4, curses.COLOR_RED, curses.COLOR_WHITE)

    # replit_warning(stdscr)
    start_screen(stdscr)
    while True:
        wpm_test(stdscr)
        stdscr.addstr(
            2, 0, 'You completed the text!\nPress Escape to exit.\nPress any other key to continue')
        key = stdscr.getkey()

        if ord(key) == 27:
            break

curses.wrapper(main)


from src.ConstraintLine import ConstraintLine
from src.PackagePin import PackagePin
from src.IOStandard import IOStandard

# Create the PACKAGE_PIN TCL line
def createPackagePinConstraint(busName, pin, idx):
    line = PackagePin(busName=busName, pin=pin, idx=idx)
    return line

# Create the IOSTANDARD TCL line
def createIOStandardConstraint(busName, idx):
    line = IOStandard(busName=busName, idx=idx)
    return line

# Driver code
if __name__ == "__main__":
    # Input bus name
    busName = input("Please enter the name of the constraint bus. ")

    pinList = []
    print("Please enter the pins, starting from index 0.")
    print("When finished, enter 0x0.")
    while True:
        pin = input("> ")
        if pin == "0x0":
            break
        else:
            pinList.append(pin)

    print()  # Whitespace

    for idx, pin in enumerate(pinList):
        print(createPackagePinConstraint(busName, pin, idx))
        print(createIOStandardConstraint(busName, idx))


import logging
import unittest
from io import StringIO

class MyTest(unittest.TestCase):
    def setUp(self):
        self.stream = StringIO()
        self.handler = logging.StreamHandler(self.stream)
        self.log = logging.getLogger("mylogger")
        self.log.setLevel(logging.INFO)
        for handler in self.log.handlers:
            self.log.removeHandler(handler)
        self.log.addHandler(self.handler)

    def testLog(self):
        self.log.info("test message")
        self.handler.flush()
        # print('[', self.stream.getvalue(), ']')
        self.assertTrue(self.stream.getvalue(), "test message")

    def tearDown(self):
        self.log.removeHandler(self.handler)
        self.handler.close()

if __name__ == "__main__":
    unittest.main()


import pandas as pd
from matplotlib import pyplot as plt
from GridCal.Engine.Devices.editable_device import EditableDevice, DeviceType, GCProp
from GridCal.Engine.Devices.enumerations import BuildStatus

class StaticGenerator(EditableDevice):
    """
    Arguments:

        **name** (str, "StaticGen"): Name of the static generator

        **P** (float, 0.0): Active power in MW

        **Q** (float, 0.0): Reactive power in MVAr

        **P_prof** (DataFrame, None): Pandas DataFrame with the active power profile in MW

        **Q_prof** (DataFrame, None): Pandas DataFrame with the reactive power profile in MVAr

        **active** (bool, True): Is the static generator active?

        **mttf** (float, 0.0): Mean time to failure in hours

        **mttr** (float, 0.0): Mean time to recovery in hours

    """

    def __init__(self, name='StaticGen', idtag=None, code='', P=0.0, Q=0.0, P_prof=None, Q_prof=None, active=True,
                 mttf=0.0, mttr=0.0,
                 capex=0, opex=0, build_status: BuildStatus = BuildStatus.Commissioned):

        EditableDevice.__init__(self,
                                name=name,
                                idtag=idtag,
                                code=code,
                                active=active,
                                device_type=DeviceType.StaticGeneratorDevice,
                                editable_headers={'name': GCProp('', str, ''),
                                                  'idtag': GCProp('', str, 'Unique ID', False),
                                                  'code': GCProp('', str, 'Secondary ID'),
                                                  'bus': GCProp('', DeviceType.BusDevice, ''),
                                                  'active': GCProp('', bool, ''),
                                                  'P': GCProp('MW', float, 'Active power'),
                                                  'Q': GCProp('MVAr', float, 'Reactive power'),
                                                  'mttf': GCProp('h', float, 'Mean time to failure'),
                                                  'mttr': GCProp('h', float, 'Mean time to recovery'),
                                                  'capex': GCProp('e/MW', float,
                                                                  'Cost of investment. Used in expansion planning.'),
                                                  'opex': GCProp('e/MWh', float,
                                                                 'Cost of operation. Used in expansion planning.'),
                                                  'build_status': GCProp('', BuildStatus,
                                                                         'Branch build status. Used in expansion planning.'),
                                                  },
                                non_editable_attributes=['bus', 'idtag'],
                                properties_with_profile={'active': 'active_prof',
                                                         'P': 'P_prof',
                                                         'Q': 'Q_prof'})

        self.bus = None

        self.active_prof = None

        self.mttf = mttf

        self.mttr = mttr

        # Power (MW + jMVAr)
        self.P = P
        self.Q = Q

        # power profile for this load
        self.P_prof = P_prof
        self.Q_prof = Q_prof

        self.capex = capex

        self.opex = opex

        self.build_status = build_status

    def copy(self):
        """
        Deep copy of this object
        :return:
        """
        return StaticGenerator(name=self.name,
                               P=self.P,
                               Q=self.Q,
                               P_prof=self.P_prof,
                               Q_prof=self.Q_prof,
                               mttf=self.mttf,
                               mttr=self.mttr)

    def get_properties_dict(self, version=3):
        """
        Get json dictionary
        :return:
        """

        data = {'id': self.idtag,
                'phases': 'ps',
                'name': self.name,
                'name_code': self.code,
                'bus': self.bus.idtag,
                'active': self.active,
                'P': self.P,
                'Q': self.Q,
                'capex': self.capex,
                'opex': self.opex,
                'build_status': str(self.build_status.value).lower(),
                'technology': ""
                }

        return data

    def get_profiles_dict(self, version=3):
        """

        :return:
        """

        if self.active_prof is not None:
            active_profile = self.active_prof.tolist()
            P_prof = self.P_prof.tolist()
            Q_prof = self.Q_prof.tolist()
        else:
            active_profile = list()
            P_prof = list()
            Q_prof = list()

        return {'id': self.idtag,
                'active': active_profile,
                'p': P_prof,
                'q': Q_prof}

    def get_units_dict(self, version=3):
        """
        Get units of the values
        """
        return {'p': 'MW',
                'q': 'MVAr'}

    def plot_profiles(self, time=None, show_fig=True):
        """
        Plot the time series results of this object
        :param time: array of time values
        :param show_fig: Show the figure?
        """

        if time is not None:
            fig = plt.figure(figsize=(12, 8))

            ax_1 = fig.add_subplot(211)
            ax_2 = fig.add_subplot(212, sharex=ax_1)

            # P
            y = self.P_prof
            df = pd.DataFrame(data=y, index=time, columns=[self.name])
            ax_1.set_title('Active power', fontsize=14)
            ax_1.set_ylabel('MW', fontsize=11)
            df.plot(ax=ax_1)

            # Q
            y = self.Q_prof
            df = pd.DataFrame(data=y, index=time, columns=[self.name])
            ax_2.set_title('Reactive power', fontsize=14)
            ax_2.set_ylabel('MVAr', fontsize=11)
            df.plot(ax=ax_2)

            plt.legend()
            fig.suptitle(self.name, fontsize=20)

            if show_fig:
                plt.show()


"""Files manager."""

import logging
import os
from pathlib import Path
import re
from typing import Optional, Type, Union

from .exceptions import InvalidFileError

logger = logging.getLogger(__name__)
DataFile = Type["File"]

class MetaFile(type):  # pylint: disable=missing-param-doc, missing-type-doc
    """Metaclass to store different file types and its istances."""

    def __init__(cls, name, bases, attrs, **kwargs):
        type.__init__(cls, name, bases, attrs, **kwargs)

        if not bases:
            cls.subtypes = {}
            cls.memory = {}
        else:
            bases[0].subtypes[name.lower().replace("file", "")] = cls
            bases[0].memory[name.lower().replace("file", "")] = list()

    def __call__(cls, *args, **kwargs):
        instance = type.__call__(cls, *args, **kwargs)
        if not instance:
            return instance

        classname = instance.__class__.__name__.lower().replace("file", "")

        if instance not in cls.memory[classname]:
            cls.memory[classname].append(instance)

        return instance

class File(metaclass=MetaFile):
    """Representation of a file containing player data.

    Arguments:
        path (str): actual path of the File.
    """

    uuid_pattern = re.compile(
        r"([0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}"
        r"\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12})\.\w+(?<!_old)$"
    )

    # For hinting purposes only (it is declared inside `MetaFile`)
    subtypes = {}
    memory = {}

    def __new__(cls, path: str):
        if cls == File:
            return File.identify(path)

        self = super().__new__(cls)
        self.__init__(path)

        return self

    def __init__(self, path: str):
        self.path = Path(path)

    def __eq__(self, other: DataFile):
        return self.path == other.path

    def __repr__(self):
        return f"{type(self).__name__}({self.as_posix()!r})"

    @property
    def uuid(self) -> str:
        """Returns the uuid of the player which data is in the file.

        Raises:
            InvalidFileError: if no uuid is found in the filename.

        Returns:
            str: uuid.
        """

        uuid = self.get_uuid_from_filepath(self.path)
        if not uuid:
            raise InvalidFileError(f"{self.path} does not contain a uuid")
        return uuid

    @classmethod
    def get_uuid_from_filepath(cls, filepath: Union[str, Path]) -> Optional[str]:
        """Returns the uuid given the filepath.

        Args:
            filepath (Union[str, Path]): filepath.

        Returns:
            Optional[str]: the uuid if success, None otherwise.
        """

        filepath = Path(filepath)

        try:
            return cls.uuid_pattern.search(filepath.as_posix()).group(1)
        except AttributeError:
            return None

    def read_bytes(self) -> bytes:
        """Returns the file content in bytes.

        Returns:
            bytes: file content.
        """

        return self.path.read_bytes()

    def remove(self):
        """Removes the actual file from the file system."""

        self.path.unlink()

    def as_posix(self) -> str:
        """Returns its file path.

        Returns:
            str: file path.
        """

        return self.path.as_posix()

    def change_uuid(self, uuid: str) -> bool:
        """Changes the uuid of the player data file.

        Args:
            uuid (str): new uuid.

        Returns:
            bool: True if the conversion was success, False otherwise.
        """

        logger.debug("Changing uuid to %s of file %s", uuid, self.path.as_posix())
        new_path = self.path.with_name(uuid + self.path.suffix)
        self.path.rename(new_path)
        self.path = new_path
        return True

    @classmethod
    def identify(cls, path: Union[str, Path]) -> Optional[DataFile]:
        """Identifies the type of player data that `path` stores.

        Args:
            path (Union[str, Path]): path of the player data file.

        Returns:
            Optional[DataFile]: If the file is identified, the a Player object is returned.
                If the file is not identified, None is returned.
        """

        path = Path(path).as_posix()
        for subtype in cls.subtypes:
            if subtype in path:
                return cls.subtypes[subtype](path)

        # TODO: raise exception if file is not identified
        return None

    @classmethod
    def gen_files(cls, path: Path):
        """Scans dir `path` recursively looking for files that contain minecraft player data.

        Args:
            path (Path): root dir to start recursive scan.
        """

        logger.debug("generating files for path %s", path.as_posix())
        for root, _, files in os.walk(path):
            for filename in files:
                file = Path(root).joinpath(filename)
                uuid = cls.get_uuid_from_filepath(file)

                if uuid:
                    File.identify(file)
        logger.info("files generated")

class PlayerDataFile(File):
    """Stores the player's inventory and ender chest, as well as other stats,
    like health, hunger, etc.

    """

class StatsFile(File):
    """Stores player's statistics."""

class AdvancementsFile(File):
    """Stores players advancements."""