import argparse
import torch
import torch.nn as nn
from torch.utils import data, model_zoo
import numpy as np
import pickle
from torch.autograd import Variable
import torch.optim as optim
import scipy.misc
import torch.backends.cudnn as cudnn
import torch.nn.functional as F
import sys
import os
import os.path as osp
import random
from tensorboardX import SummaryWriter

from model.deeplab_multi import DeeplabMulti
from model.deeplab import Res_Deeplab
from modeling.deeplab import DeepLab
from model.discriminator import FCDiscriminator
from utils.loss import CrossEntropy2d
from dataset.gta5_dataset import GTA5DataSet
from dataset.cityscapes_dataset import cityscapesDataSet
from utils.LoadDataSeg import data_loader
from collections import OrderedDict
IMG_MEAN = np.array((104.00698793, 116.66876762, 122.67891434), dtype=np.float32)

MODEL = 'DeepLab'
BATCH_SIZE = 1
ITER_SIZE = 1
NUM_WORKERS = 4
DATA_DIRECTORY = './data/GTA5'
DATA_LIST_PATH = './dataset/gta5_list/train.txt'
IGNORE_LABEL = 255
INPUT_SIZE = '1280,720'
DATA_DIRECTORY_TARGET = './data/Cityscapes/data'
DATA_LIST_PATH_TARGET = './dataset/cityscapes_list/train.txt'
INPUT_SIZE_TARGET = '1024,512'
LEARNING_RATE = 2.5e-4
MOMENTUM = 0.9
NUM_CLASSES = 2
NUM_STEPS = 100000
NUM_STEPS_STOP = 100000  # early stopping
POWER = 0.9
RANDOM_SEED = 1234
#RESTORE_FROM = 'http://vllab.ucmerced.edu/ytsai/CVPR18/DeepLab_resnet_pretrained_init-f81d91e8.pth'
#RESTORE_FROM ='https://download.pytorch.org/models/resnet50-19c8e357.pth'
RESTORE_FROM ='./snapshots/voc2012_1_50000.pth'
RESTORE_FROM_D ='./snapshots/voc2012_1_50000_D1.pth'
SAVE_NUM_IMAGES = 2
SAVE_PRED_EVERY = 10000
SNAPSHOT_DIR = './snapshots/'
WEIGHT_DECAY = 0.0005
LOG_DIR = './log'

LEARNING_RATE_D = 1e-4
LAMBDA_SEG = 0.1
LAMBDA_ADV_TARGET1 = 0.001
LAMBDA_ADV_TARGET2 = 0.001
#LAMBDA_ADV_TARGET1 = 0.0002
#LAMBDA_ADV_TARGET2 = 0.001
GAN = 'Vanilla'

TARGET = 'cityscapes'
SET = 'train'

def get_arguments():
    """Parse all the arguments provided from the CLI.

    Returns:
      A list of parsed arguments.
    """
    parser = argparse.ArgumentParser(description="DeepLab-ResNet Network")
    parser.add_argument("--model", type=str, default=MODEL,
                        help="available options : DeepLab")
    parser.add_argument("--target", type=str, default=TARGET,
                        help="available options : cityscapes")
    parser.add_argument("--batch-size", type=int, default=BATCH_SIZE,
                        help="Number of images sent to the network in one step.")
    parser.add_argument("--iter-size", type=int, default=ITER_SIZE,
                        help="Accumulate gradients for ITER_SIZE iterations.")
    parser.add_argument("--num-workers", type=int, default=NUM_WORKERS,
                        help="number of workers for multithread dataloading.")
    parser.add_argument("--data-dir", type=str, default=DATA_DIRECTORY,
                        help="Path to the directory containing the source dataset.")
    parser.add_argument("--data-list", type=str, default=DATA_LIST_PATH,
                        help="Path to the file listing the images in the source dataset.")
    parser.add_argument("--ignore-label", type=int, default=IGNORE_LABEL,
                        help="The index of the label to ignore during the training.")
    parser.add_argument("--input-size", type=str, default=INPUT_SIZE,
                        help="Comma-separated string with height and width of source images.")
    parser.add_argument("--data-dir-target", type=str, default=DATA_DIRECTORY_TARGET,
                        help="Path to the directory containing the target dataset.")
    parser.add_argument("--data-list-target", type=str, default=DATA_LIST_PATH_TARGET,
                        help="Path to the file listing the images in the target dataset.")
    parser.add_argument("--input-size-target", type=str, default=INPUT_SIZE_TARGET,
                        help="Comma-separated string with height and width of target images.")
    parser.add_argument("--is-training", action="store_true",
                        help="Whether to updates the running means and variances during the training.")
    parser.add_argument("--learning-rate", type=float, default=LEARNING_RATE,
                        help="Base learning rate for training with polynomial decay.")
    parser.add_argument("--learning-rate-D", type=float, default=LEARNING_RATE_D,
                        help="Base learning rate for discriminator.")
    parser.add_argument("--lambda-seg", type=float, default=LAMBDA_SEG,
                        help="lambda_seg.")
    parser.add_argument("--lambda-adv-target1", type=float, default=LAMBDA_ADV_TARGET1,
                        help="lambda_adv for adversarial training.")
    parser.add_argument("--lambda-adv-target2", type=float, default=LAMBDA_ADV_TARGET2,
                        help="lambda_adv for adversarial training.")
    parser.add_argument("--momentum", type=float, default=MOMENTUM,
                        help="Momentum component of the optimiser.")
    parser.add_argument("--not-restore-last", action="store_true",
                        help="Whether to not restore last (FC) layers.")
    parser.add_argument("--num-classes", type=int, default=NUM_CLASSES,
                        help="Number of classes to predict (including background).")
    parser.add_argument("--num-steps", type=int, default=NUM_STEPS,
                        help="Number of training steps.")
    parser.add_argument("--num-steps-stop", type=int, default=NUM_STEPS_STOP,
                        help="Number of training steps for early stopping.")
    parser.add_argument("--power", type=float, default=POWER,
                        help="Decay parameter to compute the learning rate.")
    parser.add_argument("--random-mirror", action="store_true",
                        help="Whether to randomly mirror the inputs during the training.")
    parser.add_argument("--random-scale", action="store_true",
                        help="Whether to randomly scale the inputs during the training.")
    parser.add_argument("--random-seed", type=int, default=RANDOM_SEED,
                        help="Random seed to have reproducible results.")

    parser.add_argument("--restore-from", type=str, default=RESTORE_FROM,
                        help="Where restore model parameters from.")
    parser.add_argument("--restore-from-D", type=str, default=RESTORE_FROM_D,
                        help="Where restore model parameters from.")
    parser.add_argument("--save-num-images", type=int, default=SAVE_NUM_IMAGES,
                        help="How many images to save.")
    parser.add_argument("--save-pred-every", type=int, default=SAVE_PRED_EVERY,
                        help="Save summaries and checkpoint every often.")
    parser.add_argument("--snapshot-dir", type=str, default=SNAPSHOT_DIR,
                        help="Where to save snapshots of the model.")
    parser.add_argument("--weight-decay", type=float, default=WEIGHT_DECAY,
                        help="Regularisation parameter for L2-loss.")
    parser.add_argument("--cpu", action='store_true', help="choose to use cpu device.")
    parser.add_argument("--tensorboard", action='store_true', help="choose whether to use tensorboard.")
    parser.add_argument("--log-dir", type=str, default=LOG_DIR,
                        help="Path to the directory of log.")
    parser.add_argument("--set", type=str, default=SET,
                        help="choose adaptation set.")
    parser.add_argument("--gan", type=str, default=GAN,
                        help="choose the GAN objective.")
    parser.add_argument("--arch", type=str, default='onemodel_sg_one')
    parser.add_argument("--max_steps", type=int, default=100001)
    parser.add_argument("--lr", type=float, default=1e-5)
    # parser.add_argument("--decay_points", type=str, default='none')
    parser.add_argument("--disp_interval", type=int, default=100)
    parser.add_argument("--save_interval", type=int, default=5000)
    parser.add_argument("--snapshot_dir", type=str, default=SNAPSHOT_DIR)
    #parser.add_argument("--restore_from", type=str,default='/home/prlab/willow/Oneshot/SG-One-master/snapshots/onemodel_sg_one_scale/group_1_of_4/step_80000.pth.tar')
    parser.add_argument("--resume", action='store_true')
    parser.add_argument("--start_count", type=int, default=50000)

    # parser.add_argument("--split", type=str, default='train')
    parser.add_argument("--split", type=str, default='train')
    parser.add_argument("--group", type=int, default=1)
    parser.add_argument('--num_folds', type=int, default=4)
    return parser.parse_args()

args = get_arguments()

def lr_poly(base_lr, iter, max_iter, power):
    return base_lr * ((1 - float(iter) / max_iter) ** (power))

def adjust_learning_rate(optimizer, i_iter):
    lr = lr_poly(args.learning_rate, i_iter, args.num_steps, args.power)
    optimizer.param_groups[0]['lr'] = lr
    if len(optimizer.param_groups) > 1:
        optimizer.param_groups[1]['lr'] = lr * 10

def adjust_learning_rate_D(optimizer, i_iter):
    lr = lr_poly(args.learning_rate_D, i_iter, args.num_steps, args.power)
    optimizer.param_groups[0]['lr'] = lr
    if len(optimizer.param_groups) > 1:
        optimizer.param_groups[1]['lr'] = lr * 10
def restore(model, pre_model):
    try:
        model.load_state_dict(pre_model)
    except RuntimeError:
        print("KeyError")
        model_dict = model.state_dict()
        new_model_dict = OrderedDict()
        for k_model, k_pretrained in zip(list(model_dict.keys()), list(pre_model.keys())[:len(model_dict.keys())]):#增加了list
            if model_dict[k_model].size() == pre_model[k_pretrained].size():
                print("%s\t<--->\t%s"%(k_model, k_pretrained))
                new_model_dict[k_model] = pre_model[k_pretrained]
            else:
                print('Fail to load %s'%(k_model))

        model_dict.update(new_model_dict)
        model.load_state_dict(model_dict)
    except KeyError:
        print("Loading pre-trained values failed.")
        raise
# else:
#     print("=> no checkpoint found at '{}'".format(snapshot))

def one_hot(label):
    label = label.squeeze().unsqueeze(0).cpu().numpy()
    one_hot = np.zeros((label.shape[0], args.num_classes, label.shape[1], label.shape[2]), dtype=label.dtype)
    for i in range(args.num_classes):
        one_hot[:,i,...] = (label==i)
    #handle ignore labels
    return torch.FloatTensor(one_hot)

def main():
    """Create the model and start the training."""

    device = torch.device("cuda" if not args.cpu else "cpu")

    w, h = map(int, args.input_size.split(','))
    input_size = (w, h)

    w, h = map(int, args.input_size_target.split(','))
    input_size_target = (w, h)

    cudnn.enabled = True

    # Create network
    if args.model == 'DeepLab':
        #model = DeeplabMulti(num_classes=args.num_classes)
        #model = Res_Deeplab(num_classes=args.num_classes)
        model = DeepLab(backbone='resnet', output_stride=16)
        '''
        if args.restore_from[:4] == 'http' :
            saved_state_dict = model_zoo.load_url(args.restore_from)
        else:
            saved_state_dict = torch.load(args.restore_from)
        #restore(model, saved_state_dict)

        new_params = model.state_dict().copy()
        for i in saved_state_dict:
            # Scale.layer5.conv2d_list.3.weight
            i_parts = i.split('.')
            # print i_parts
            if not i_parts[0] == 'layer4' and not i_parts[0] == 'fc':
                #new_params['.'.join(i_parts[1:])] = saved_state_dict[i]
                new_params[i] = saved_state_dict[i]
                # print i_parts
        model.load_state_dict(new_params)
        '''
    else:
        raise NotImplementedError

    model.train()
    model.to(device)

    cudnn.benchmark = True

    # init D
    model_D1 = FCDiscriminator(num_classes=args.num_classes).to(device)

    # if args.restore_from_D[:4] == 'http':
    #     saved_state_dict = model_zoo.load_url(args.restore_from_D)
    # else:
    #     saved_state_dict = torch.load(args.restore_from_D)
    #     ### for running different versions of pytorch
    # model_dict = model_D1.state_dict()
    # saved_state_dict = {k: v for k, v in saved_state_dict.items() if k in model_dict}
    # model_dict.update(saved_state_dict)
    # model_D1.load_state_dict(saved_state_dict)

    model_D1.train()
    model_D1.to(device)

    if not os.path.exists(args.snapshot_dir):
        os.makedirs(args.snapshot_dir)

    train_loader = data_loader(args)

    # implement model.optim_parameters(args) to handle different models' lr setting

    optimizer = optim.SGD(model.optim_parameters(args),
                          lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay)
    optimizer.zero_grad()

    optimizer_D1 = optim.Adam(model_D1.parameters(), lr=args.learning_rate_D, betas=(0.9, 0.99))
    optimizer_D1.zero_grad()

    if args.gan == 'Vanilla':
        bce_loss = torch.nn.BCEWithLogitsLoss()
    elif args.gan == 'LS':
        bce_loss = torch.nn.MSELoss()
    seg_loss = torch.nn.CrossEntropyLoss()

    interp = nn.Upsample(size=(416, 416), mode='bilinear', align_corners=True)
    #interp_target = nn.Upsample(size=(input_size_target[1], input_size_target[0]), mode='bilinear', align_corners=True)

    # labels for adversarial training

    # set up tensorboard
    if args.tensorboard:
        if not os.path.exists(args.log_dir):
            os.makedirs(args.log_dir)

        writer = SummaryWriter(args.log_dir)

    count = args.start_count  # 迭代次数
    for dat in train_loader:
        if count > args.num_steps:
            break

        loss_seg_value1_anchor = 0
        loss_adv_target_value1 = 0
        loss_D_value1 = 0

        optimizer.zero_grad()
        adjust_learning_rate(optimizer, count)

        optimizer_D1.zero_grad()

        adjust_learning_rate_D(optimizer_D1, count)

        for sub_i in range(args.iter_size):

            # train G

            # don't accumulate grads in D
            for param in model_D1.parameters():
                param.requires_grad = False

            # 相当于group=0时，训练样本对应的类有15类为[0,1,2,3,4,5,6,7,8,9,10,....]，验证集有5类，
            # 现在从训练集类中随机选择两类，然后从其中一类中选择两张图片，对应为基准图片和正样本图片，
            # 两者属于同一类，接着从另一类中选择一张图片作为负样本，属于不同类。其中基准图片对应的是查询集图片
            #############################
            anchor_img, anchor_mask, pos_img, pos_mask, neg_img, neg_mask = dat  # 返回的是基准图片以及mask，正样本以及mask（和基准图片属于同一类），负样本以及mask（和基准图片属于不同类）

            anchor_img, anchor_mask, pos_img, pos_mask, \
                = anchor_img.cuda(), anchor_mask.cuda(), pos_img.cuda(), pos_mask.cuda()  # [1, 3, 386, 500],[1, 386, 500],[1, 3, 374, 500],[1, 374, 500]

            anchor_mask = torch.unsqueeze(anchor_mask, dim=1)  # [1, 1, 386, 500]
            pos_mask = torch.unsqueeze(pos_mask, dim=1)  # [1,1, 374, 500]
            samples = torch.cat([pos_img, anchor_img], 0)

            pred = model(samples, pos_mask)  ##[2, 2, 53, 53],#[2, 2, 53, 53]
            pred = interp(pred)

            loss_seg1_anchor = seg_loss(pred, anchor_mask.squeeze().unsqueeze(0).long())
            D_out1 = model_D1(F.softmax(pred))
            loss_adv_target1 = bce_loss(D_out1, torch.FloatTensor(D_out1.data.size()).fill_(1).to(
                device))  # 相当于将源域的标签设置为1,然后判断判别网络得到的目标预测与源域对应的损失
            '''
            s = torch.stack([s, 1-s])
            loss_s = seg_loss()
            '''
            loss = loss_seg1_anchor + args.lambda_adv_target1 * loss_adv_target1

            # proper normalization
            loss = loss / args.iter_size
            loss.backward()

            loss_seg_value1_anchor += loss_seg1_anchor.item() / args.iter_size
            loss_adv_target_value1 += loss_adv_target1.item() / args.iter_size

            # train D# bring back requires_grad
            for param in model_D1.parameters():
                param.requires_grad = True

            # train with anchor
            pred_target1 = pred.detach()
            D_out1 = model_D1(F.softmax(pred_target1))
            loss_D1 = bce_loss(D_out1, torch.FloatTensor(D_out1.data.size()).fill_(0).to(device))
            loss_D1 = loss_D1 / args.iter_size / 2
            loss_D1.backward()
            loss_D_value1 += loss_D1.item()

            # train with GT
            anchor_gt = Variable(one_hot(anchor_mask)).cuda()
            D_out1 = model_D1(anchor_gt)
            loss_D1 = bce_loss(D_out1, torch.FloatTensor(D_out1.data.size()).fill_(1).to(device))
            loss_D1 = loss_D1 / args.iter_size / 2
            loss_D1.backward()
            loss_D_value1 += loss_D1.item()

        optimizer.step()
        optimizer_D1.step()

        count = count + 1
        if args.tensorboard:
            scalar_info = {

                'loss_seg1_anchor': loss_seg_value1_anchor,
                'loss_adv_target1': loss_adv_target_value1,
                'loss_D1': loss_D_value1,

            }

            if count % 10 == 0:
                for key, val in scalar_info.items():
                    writer.add_scalar(key, val, count)

        # print('exp = {}'.format(args.snapshot_dir))
        print(
        'iter = {0:8d}/{1:8d}, loss_seg1 = {2:.3f}, loss_adv1 = {3:.3f}, loss_D1 = {4:.3f}'.format(
            count, args.num_steps, loss_seg_value1_anchor,loss_adv_target_value1,loss_D_value1))

        if count >= args.num_steps_stop - 1:
            print('save model ...')
            torch.save(model.state_dict(), osp.join(args.snapshot_dir, 'voc2012_' + str(args.num_steps_stop) + '.pth'))
            torch.save(model_D1.state_dict(), osp.join(args.snapshot_dir, 'voc2012_' + str(args.num_steps_stop) + '_D1.pth'))
            break

        if count % args.save_pred_every == 0 and count != 0:
            print('taking snapshot ...')
            torch.save(model.state_dict(), osp.join(args.snapshot_dir, 'voc2012_' + str(count) + '.pth'))
            torch.save(model_D1.state_dict(), osp.join(args.snapshot_dir, 'voc2012_' + str(count) + '_D1.pth'))

    if args.tensorboard:
        writer.close()

if __name__ == '__main__':
    main()


#file for SQLite functions for portfolios

from decimal import Decimal
import sqlite3
import traceback

#establish connection and cursor to return rows
con = sqlite3.connect("portfolio.db")
con.execute("PRAGMA foreign_keys = ON;")
con.row_factory = sqlite3.Row
cursor = con.cursor()

def insertPosition(userID:int, ticker:str,avgPrice:Decimal,shareAmt:int) -> int:
    '''inserts a position into the positions table and returns status code based on success'''

    try:
        avgPrice:float = float(avgPrice)
        cursor.execute("INSERT INTO Positions (userID,symbol,purchasePrice,numShares) VALUES(?,?,?,?)",(userID,ticker,avgPrice,shareAmt))
        con.commit()
        return 1
    except Exception as e:
        traceback.print_exc()
        print(e)
        return 0

def getUserPositions(userID:int) -> list[int]:
    '''gets a list of position ID's that are held by a user'''

    #get the position ID's
    userPositions = cursor.execute("SELECT positionID FROM USER_POSITIONS WHERE userID = ?",(userID,)).fetchall()
    posList:list = []

    #convert to a list
    for position in userPositions:
        posList.append(position['positionID'])
    return posList

def getPositionInfo(positionID:int) -> sqlite3.Row:
    '''returns all the information on a single position'''

    position:sqlite3.Row = cursor.execute("SELECT * FROM USER_POSITIONS WHERE positionID = ?",(positionID,)).fetchone()
    return position

def getPositionInfoByStock(userID:int) -> list[sqlite3.Row]:
    '''returns all the positions held by a user grouped by the ticker'''

    position = cursor.execute('''
    SELECT
    symbol,
    currentPrice,
    openPrice,
    priceChange,
    priceHint,
    total(purchasePrice * numShares)/total(numShares) AS "averagePrice",
    SUM(numShares) AS "totalShares",
    (total(purchasePrice * numShares)/total(numShares)) * SUM(numShares) AS "costBasis",
    dayPLPct,
    priceChange * SUM(numShares) AS "dayPL"
    FROM USER_POSITIONS
    WHERE userID=?
    GROUP BY symbol''',
    (userID,)).fetchall()
    return position

def positionBelongsToUser(positionID:int,userID:int) -> bool:
    '''returns True or False based on if the given position belongs to the given user'''
    position = cursor.execute('''
    SELECT
    userID,positionID
    FROM USER_POSITIONS
    WHERE userID=? AND positionID=?''',
    (userID,positionID)).fetchall()

    return bool(position)

def removePosition(positionID:int) -> None:
    '''removes given position from the database'''
    cursor.execute('''
    DELETE FROM Positions
    WHERE positionID=?''',(positionID,))
    con.commit()


import unittest
import os
from lib.experiments.experiment import Experiment

class ExperimentTest(unittest.TestCase):
    def test_parse_experiment_row_simple(self):
        """
        Parse simple experiment description string
        """
        test_row = "A3; TRN = [1,2,4];           TST = [3];"
        exp = Experiment(test_row)
        self.assertSequenceEqual(exp.training_list, ['img1', 'img2', 'img4'])
        self.assertSequenceEqual(exp.testing_list, ['img3'])
        self.assertEquals(exp.name, "A3")

    def test_parse_experiment_row_ranges(self):
        """
        Parse more complex experiment description string with ranges
        """
        test_row = "B4; TRN = [5,6,7:24,31:42];  TST = [25:30];"
        exp = Experiment(test_row)

        expected_training_list = ['img5', 'img6']
        for rng in [(7, 24), (31, 42)]:
            for i in xrange(rng[0], rng[1] + 1):
                expected_training_list.append('img' + str(i))

        expected_testing_list = [
            'img' + str(i)
            for i in xrange(25, 30 + 1)
        ]

        self.assertSequenceEqual(exp.training_list, expected_training_list)
        self.assertSequenceEqual(exp.testing_list, expected_testing_list)
        self.assertEquals(exp.name, "B4")

    def test_read_experiments_from_file(self):
        """
        Read all experiment sets from the main file
        """
        cwd = os.path.dirname(os.path.abspath(__file__))
        filepath = os.path.join(cwd, 'test_files/experiment_list.txt')
        experiments = Experiment.parse_file(filepath)
        self.assertEquals(len(experiments), 5)
        self.assertEquals(len(experiments['HOM 4']), 4)
        self.assertEquals(len(experiments['HOM38']), 6)
        self.assertEquals(len(experiments['HOM56']), 1)
        self.assertEquals(len(experiments['HET36']), 4)
        self.assertEquals(len(experiments['HET5']), 5)


#!/usr/bin/env python3
"""
Meerkat API Tests

Unit tests for the epi week resource of Meerkat Api
"""

import json

import meerkat_api
import meerkat_abacus.util.epi_week
from . import settings

class MeerkatAPIEpiWeekTestCase(meerkat_api.test.TestCase):

    def setUp(self):
        """Setup for testing"""
        self._mock_epi_week_abacus_logic()

    def test_epi_year_start(self):
        """ Test the epi_year_start function """
        rv = self.app.get('/epi_week_start/2015/1', headers=settings.header)
        data = json.loads(rv.data.decode("utf-8"))
        self.assertEqual(rv.status_code, 200)
        self.assertEqual(data["start_date"], meerkat_abacus.util.epi_week.epi_year_start_date_by_year(2015).isoformat())

    def test_epi_week(self):
        """ Test date to epi week"""
        rv = self.app.get('/epi_week/2015-01-05', headers=settings.header)
        data = json.loads(rv.data.decode("utf-8"))
        self.assertEqual(rv.status_code, 200)
        self.assertEqual(data["epi_week"], 1)
        rv = self.app.get('/epi_week/2015-12-02', headers=settings.header)
        data = json.loads(rv.data.decode("utf-8"))
        self.assertEqual(rv.status_code, 200)
        self.assertEqual(data["epi_week"], 48)

    def test_epi_week_for_dd_mm_yyyy_format(self):
        """ Test date to epi week"""
        rv = self.app.get('/epi_week/04-01-2015', headers=settings.header)
        data = json.loads(rv.data.decode("utf-8"))
        self.assertEqual(rv.status_code, 200)
        self.assertEqual(data["epi_week"], 1)

    def test_epi_week_start(self):
        """ test epi-week to date """
        rv = self.app.get('/epi_week_start/2015/49', headers=settings.header)
        data = json.loads(rv.data.decode("utf-8"))
        self.assertEqual(rv.status_code, 200)
        self.assertEqual(data["start_date"], "2015-12-03T00:00:00")
        rv = self.app.get('/epi_week_start/2015/2', headers=settings.header)
        data = json.loads(rv.data.decode("utf-8"))
        self.assertEqual(rv.status_code, 200)
        self.assertEqual(data["start_date"], "2015-01-08T00:00:00")


from openpyxl import load_workbook
from openpyxl import Workbook
import re
import os
import time

start = time.time()

# path_list = gen_pathList("C:\\Users\\Brian.tsai\\Desktop\\Python\\Tool\\LVC\\","xlsx")
# for file in path_list:
#     print(file)

src_file = "C:\\Users\\Brian.tsai\\Desktop\\Python\\Tool\\LVC\\Carpo FTx6x2 final BOM.xlsx"
gen_file = "C:\\Users\\Brian.tsai\\Desktop\\Python\\Tool\\LVC\\Carpo FTx6x2 final BOM_mod.xlsx"

try:
    os.remove(gen_file)
except OSError as e:
    print(e)
else:
    print("================================File is deleted successfully================================")

wb_gen = Workbook()
ws_gen = wb_gen["Sheet"]
ws_gen.title = "Coverage_Report"

ws_gen.cell(row=1, column=1, value="No.")
ws_gen.cell(row=1, column=2, value="Components")
ws_gen.cell(row=1, column=3, value="Component Type")
ws_gen.cell(row=1, column=4, value="Value")
ws_gen.cell(row=1, column=5, value="CAP Bank Name")
ws_gen.cell(row=1, column=6, value="Qty")
ws_gen.cell(row=1, column=7, value="Test Item")
ws_gen.cell(row=1, column=8, value="Test Environment")
ws_gen.cell(row=1, column=9, value="Comment")

wb = load_workbook(src_file)
pattern_BOM = r'BOM'
pattern_No = r'^No'
pattern_Reference = r'^Reference'
pattern_Description = r'Description'
pattern_Quantity = r'Quantity'
pattern_RES = r'RES'
pattern_Resistor = r'Resistor'
pattern_CAP = r'CAP'
pattern_Capacitor = r'Capacitor'
pattern_IC = r'IC'
pattern_CONNECTOR = r'CONNECTOR'
pattern_Inductor = r'Inductor'
pattern_REGULATOR = r'REGULATOR'
pattern_IC_Analog = r'IC_Analog'
pattern_IC_Logic = r'IC_Logic'
pattern_JUMPER = r'JUMPER'
pattern_Other = r'Other'
pattern_STIFFENER = r'STIFFENER'
pattern_COVER = r'COVER'

BOM_sheet = ""
column_Map = {}
column_List = []
# print(wb.sheetnames)
for sheet in wb.sheetnames:
    if re.search(pattern_BOM, sheet) != None:
        BOM_sheet = sheet
print(BOM_sheet)

ws_BOM = wb[BOM_sheet]

for i in range (1, ws_BOM.max_row+1, 1):
    for j in range (1, ws_BOM.max_column+1, 1):
        content = ws_BOM.cell(row=i, column=j).value
        if type(content) == str:
            if re.search(pattern_No, content) != None:
                column_Map[content] = j
                column_List.append(j)
            elif re.search(pattern_Quantity, content) != None:
                column_Map[content] = j
                column_List.append(j)
            elif  re.search(pattern_Reference, content) != None:
                column_Map[content] = j
                column_List.append(j)
            elif  re.search(pattern_Description, content) != None:
                column_Map[content] = j
                column_List.append(j)
                break
print(column_Map)
# print(column_List)

# print(ws_BOM.max_row)
for i in range (3, ws_BOM.max_row+1, 1):
    Number = 0
    repeat_num = 0
    component = []
    Description = []
    Value = ""
    for j in column_List:
        content = ws_BOM.cell(row=i, column=j).value
        if (j == 1):
            Number = content
            if type(Number) != int:
                break
        elif (j == 5):
            repeat_num = content
        elif (j == 6):
            try:
                component = content.split(",")
                # print(component)
            except AttributeError as e:
                print("Error message: " + e)
                print("Row: " + str(i) + ", Column: " + str(j))
        elif ( j == 8):
            if re.search(pattern_RES, content) != None:
                Description = content.split("~")
                Value = Description[4]
                Description = "Resistor"
            elif re.search(pattern_Resistor, content) != None:
                Description = content.split("~")
                Value = Description[4].split("/")[0]
                Description = "Resistor"
            elif re.search(pattern_CAP, content) != None:
                Description = content.split("~")
                Value = Description[5]
                Description = "Capacitor"
            elif re.search(pattern_Capacitor, content) != None:
                Description = content.split("~")
                Value = Description[4].split("/")[0]
                Description = "Capacitor"
            elif re.search(pattern_REGULATOR, content) != None:
                Description = content.split("~")
                Value = Description[2]
                Description = Description[0] + "_" + Description[1]
            elif re.search(pattern_IC_Analog, content) != None:
                Description = content.split("~")
                Value = Description[-1]
                Description = Description[0] + "_" + Description[1]
            elif re.search(pattern_IC_Logic, content) != None:
                Description = content.split("~")
                Value = Description[-1]
                Description = Description[0] + "_" + Description[1]
            elif re.search(pattern_CONNECTOR, content) != None:
                Description = content.split("~")
                Value = Description[-1]
                Description = "Connector"
            elif re.search(pattern_Inductor, content) != None:
                Description = content.split("~")
                Value = Description[4]
                Description = "Inductor"
            elif re.search(pattern_JUMPER, content) != None:
                Description = content.split("~")
                Value = Description[4]
                Description = "JUMPER"
            elif re.search(pattern_Other, content) != None:
                Description = content.split("~")
                Value = Description[-1]
                Description = Description[1]
            elif re.search(pattern_STIFFENER, content) != None:
                Description = content.split("-")
                Value = content
                Description = Description[0]
            elif re.search(pattern_COVER, content) != None:
                Description = content.split("-")
                Value = content
                Description = Description[0]
            else:
                try:
                    Value = content
                    Description = content.split("~")
                    Description = Description[0] + "_Not in my case"
                except AttributeError as e:
                    print("Error message: " + e)
                    print("Row: " + str(i) + ", Column: " + str(j))

    for repeat in range(1,repeat_num+1,1):
        ws_gen.cell(row=ws_gen.max_row+1, column=1, value=Number)
        ws_gen.cell(row=ws_gen.max_row, column=2, value=component[repeat-1])
        ws_gen.cell(row=ws_gen.max_row, column=3, value=Description)
        ws_gen.cell(row=ws_gen.max_row, column=4, value=Value)
        ws_gen.cell(row=ws_gen.max_row, column=6, value=repeat_num)

wb_gen.save(gen_file)

end = time.time()
print("Program time: " + str(end-start) + "(s)")


import cv2
import numpy as np
from time import sleep
from piCam import PiVideoStream
#from motorDriver import Motors
import warnings
import math

class LineFollower:
    def __init__(self):
        global stream
        stream = PiVideoStream()
        stream.start()
        self.grey = None

    def follow(self) -> None:
        frameWidth = 480
        frameHeight = 240
        halfFrameWidth = 240
        halfFrameHeight = 120

        self.frame = stream.read()

        self.frame = cv2.flip(self.frame, 0)

        #roi = self.frame[100:200,0:frameWidth] not being used rn

        self.line = cv2.inRange(self.frame,(0,0,0),(45,45,45))
        self.line = cv2.GaussianBlur(self.line,(5,5),0)
        kernel = np.ones((3,3), np.uint8)

        #self.line = cv2.erode(self.line, kernel, iterations=3)
        #self.line = cv2.dilate(self.line,kernel,iterations=2) idk reilly doesnt have it

        #lineContours,_ = cv2.findContours(self.line, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
        if len(self.line) > 100:
            #largestContour = max(lineContours,key=cv2.contourArea)
            m = cv2.moments(self.line)
            if m['m00'] > 0:
                xPos = int(m['m10']/m['m00'])
                yPos = int(m['m01']/m['m00'])
                cv2.circle(self.frame,(xPos,yPos),5,(255,0,0),-1)
                cv2.circle(self.frame,(halfFrameWidth,270),5,(0,255,0),-1)
                cv2.line(self.frame,(halfFrameWidth,270),(xPos,yPos),(0,255,0),2)
                opposite = halfFrameWidth-xPos
                adjacent = 270-yPos #change 300 for faster turning, less smooth
                angleRad = math.atan2(opposite,adjacent)
                angle = angleRad*(180/3.14159)
                print("angle = ",angle)
            else:
                angle = 0

        else:
            angle = 0

        return angle

    def checkSilver(self):
        gray = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
        ret, silver = cv2.threshold(gray,254,255,cv2.THRESH_BINARY)
        silver = cv2.countNonZero(silver)
        print(silver)

        if silver > 300: #tune
            isSilver = True
        else:
            isSilver = False
        return isSilver

    def lineInFrame(self):
        if len(self.line) > 10000: #tune
            lineInFrame = True
        else:
            lineInFrame = False

        return lineInFrame

if __name__ == '__main__':
    follower = LineFollower()
    cap = cv2.VideoCapture(0)
    follower.follow()


import os
import string
import sassie.analyze.apbs.gui_mimic_apbs as gui_mimic_apbs
#import gui_mimic_apbs as gui_mimic_apbs

import filecmp
from unittest import main
from nose.tools import assert_equals
from mocker import Mocker, MockerTestCase

pdb_data_path = os.path.join(os.path.dirname(os.path.realpath(
    __file__)), '..', '..', 'data', 'pdb_common') + os.path.sep
dcd_data_path = os.path.join(os.path.dirname(os.path.realpath(
    __file__)), '..', '..', 'data', 'dcd_common') + os.path.sep
other_data_path = os.path.join(os.path.dirname(os.path.realpath(
    __file__)), '..', '..', 'data', 'other_common') + os.path.sep
module_data_path = os.path.join(os.path.dirname(os.path.realpath(
    __file__)), '..', '..', 'data', 'interface', 'apbs') + os.path.sep

paths = {'pdb_data_path': pdb_data_path, 'dcd_data_path': dcd_data_path,
         'other_data_path': other_data_path, 'module_data_path': module_data_path}

class Test_Apbs_Filter(MockerTestCase):

    '''
    System integration test for apbs_filter.py / sassie 1.0

    Test to see whether apbs_filter catches improper input.

    Inputs tested:

            runname:        project name
            pdbfile:        reference pdb name
            infile:         input trajectory filename (pdb or dcd)
            ph:             pH value
            temperature:    temperature value (K)
            ion_conc:       concentration of solute ion (M)
            ion_radius:     radius of solute ion (angstroms)

    Inputs not tested:  (Not yet implemented)

            manual_flag:    user input file flag (0=default inputs; 1=user input file)
            manual_file:    user input file name

    Use cases tested:

    1.  check runname
        a.  check for invalid characters in runname
    2.  check input PDB file
        a.  PDB file doesn't exist
        b.  PDB file exists
            i.  PDB file is valid
            ii. PDB file isn't valid
    3.  check input trajectory file
        a.  file doesn't exist
        b.  file exists
            i.  file name doesn't start with "." and must end in ".pdb" or ".dcd"
            ii. file is .pdb (value = 0)
                1.  PDB file is valid
                2.  PDB file isn't valid
            iii.file is .dcd (value = 1)
                1.  DCD file is valid
                2.  DCD file isn't valid
    4.  check if input PDB and trajectory files are compatible
        a.  trajectory file is a DCD file
            i.  input PDB and DCD are compatible
            ii. input PDB and DCD are not compatible
        b.  trajectory file is a PDB file
            i.  input PDB and PDB are compatible
            ii. input PDB and PDB are not compatible
    5.  check if temperature >0
    6.  check if pH >=0
    7.  check if ion concentration >=0
    8.  check if ion radius > 0

    '''

    def setUp(self):

        gui_mimic_apbs.test_variables(self, paths)

    def test_1(self):
        '''
        test if runname has incorrect character
        '''
        self.runname = 'run_&'
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for value error '''
        expected_error = ['file or path : run_& has incorrect character : &']
        assert_equals(return_error, expected_error)

    def test_2(self):
        '''
        test if input PDB file exists
        '''
        self.pdbfile = os.path.join(
            module_data_path, 'does_not_exist.pdb')
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['reference pdb file, ' +
                          self.pdbfile[3:] + ', does not exist']
        assert_equals(return_error, expected_error)

    def test_3(self):
        '''
        test if input PDB file is a valid file
        '''
        self.pdbfile = os.path.join(module_data_path, 'not_valid.pdb')
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['reference pdb file, ' +
                          self.pdbfile[3:] + ', is not a valid pdb file']
        assert_equals(return_error, expected_error)

    def test_4(self):
        '''
        test if input trajectory file exists
        '''
        self.infile = os.path.join(
            module_data_path, 'does_not_exist.dcd')
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['file : ' +
                          self.infile[3:] + ' does not exist']
        assert_equals(return_error, expected_error)

    def test_5(self):
        '''
        test if input trajectory file is a valid dcd file
        '''
        self.infile = os.path.join(
            module_data_path, 'not_valid.dcd')
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['input file, ' +
                          self.infile[3:] + ', is not a valid pdb or dcd file']
        assert_equals(return_error, expected_error)

    def test_6(self):
        '''
        test if input trajectory file is a valid pdb file
        '''
        self.infile = os.path.join(
            module_data_path, 'not_valid.pdb')
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['input file, ' +
                          self.infile[3:] + ', is not a valid pdb or dcd file']
        assert_equals(return_error, expected_error)

    def test_7(self):
        '''
        test if "dcd" trajectory file has the same number of atoms as input PDB
        '''
        self.infile = os.path.join(
            module_data_path, 'non_matching.dcd')
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['input pdb ' + self.pdbfile[3:] + ' and dcd file ' +
                          self.infile[3:] + ' are not compatible']
        assert_equals(return_error, expected_error)

    def test_8(self):
        '''
        test if "pdb" trajectory file has the same number of atoms as input PDB
        '''
        self.infile = os.path.join(
            module_data_path, 'non_matching.pdb')
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['reference pdb file ' + self.pdbfile[3:] + ' and input pdb file ' +
                          self.infile[3:] + ' are not compatible']
        assert_equals(return_error, expected_error)

    def test_9(self):
        '''
        test if temperature is > 0
        '''
        self.temperature = '0'
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['temperature needs to be greater than zero : 0.0']
        assert_equals(return_error, expected_error)

    def test_10(self):
        '''
        test if pH >=0
        '''
        self.ph = '-1'
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['ph needs to be greater than or equal to zero : -1.0']
        assert_equals(return_error, expected_error)

    def test_11(self):
        '''
        test if ion concentration >=0
        '''
        self.ion_conc = '-1'
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['ion concentration needs to be greater than or equal to zero : -1.0']
        assert_equals(return_error, expected_error)

    def test_12(self):
        '''
        test if ion radius >0
        '''
        self.ion_radius = '0'
        return_error = gui_mimic_apbs.run_module(
            self, test_filter=True)

        ''' check for file error '''
        expected_error = ['ion radius needs to be greater than zero : 0.0']
        assert_equals(return_error, expected_error)

    def tearDown(self):
        if os.path.exists(self.runname):
            shutil.rmtree(self.runname)

if __name__ == '__main__':
    main()


from Config.config              import db
from sqlalchemy.orm             import relationship
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()

class Car(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    model = db.Column(db.String(10), nullable=True)

    brand_id = db.Column(db.Integer, db.ForeignKey('brand.id'), nullable=True)
    brand = db.relationship('Brand', foreign_keys=[brand_id])

    subbrand_id = db.Column(db.Integer, db.ForeignKey('subbrand.id'), nullable=True)
    subbrand = db.relationship('Subbrand', foreign_keys=[subbrand_id])

    car_type_id = db.Column(db.Integer, db.ForeignKey('car_type.id'), nullable=True)
    car_type = db.relationship('Car_type', foreign_keys=[car_type_id])

    accident = db.relationship('Accident', uselist=False)

    def __init__(self, model, brand_id, subbrand_id, car_type_id):
        self.model = model
        self.brand_id = brand_id
        self.subbrand_id = subbrand_id
        self.car_type_id = car_type_id

    def __repr__(self):
        return f'<id {self.id}>'

    def serialize(self):
        return {
            'id': self.id,
            'model': self.model,
            'brand_id': self.brand_id,
            'subbrand_id': self.subbrand_id,
            'car_type_id': self.car_type_id
        }


from __future__ import absolute_import

import itertools
import logging
import uuid
from builtins import object
from typing import TYPE_CHECKING
from typing import Dict
from typing import Iterator
from typing import Optional
from typing import Union

from google.protobuf import timestamp_pb2

from apache_beam.portability.api import beam_job_api_pb2
from apache_beam.portability.api import beam_job_api_pb2_grpc
from apache_beam.utils.timestamp import Timestamp

if TYPE_CHECKING:
  from google.protobuf import struct_pb2  # pylint: disable=ungrouped-imports
  from apache_beam.portability.api import beam_runner_api_pb2
  from apache_beam.portability.api import endpoints_pb2

_LOGGER = logging.getLogger(__name__)

def make_state_event(state, timestamp):
  if isinstance(timestamp, Timestamp):
    proto_timestamp = timestamp.to_proto()
  elif isinstance(timestamp, timestamp_pb2.Timestamp):
    proto_timestamp = timestamp
  else:
    raise ValueError("Expected apache_beam.utils.timestamp.Timestamp, "
                     "or google.protobuf.timestamp_pb2.Timestamp. "
                     "Got %s" % type(timestamp))

  return beam_job_api_pb2.JobStateEvent(
      state=state, timestamp=proto_timestamp)

class AbstractJobServiceServicer(beam_job_api_pb2_grpc.JobServiceServicer):
  """Manages one or more pipelines, possibly concurrently.
  Experimental: No backward compatibility guaranteed.
  Servicer for the Beam Job API.
  """
  def __init__(self):
    self._jobs = {}  # type: Dict[str, AbstractBeamJob]

  def create_beam_job(self,
                      preparation_id,  # stype: str
                      job_name,  # type: str
                      pipeline,  # type: beam_runner_api_pb2.Pipeline
                      options  # type: struct_pb2.Struct
                     ):
    # type: (...) -> AbstractBeamJob
    """Returns an instance of AbstractBeamJob specific to this servicer."""
    raise NotImplementedError(type(self))

  def Prepare(self,
              request,  # type: beam_job_api_pb2.PrepareJobRequest
              context=None,
              timeout=None
             ):
    # type: (...) -> beam_job_api_pb2.PrepareJobResponse
    _LOGGER.debug('Got Prepare request.')
    preparation_id = '%s-%s' % (request.job_name, uuid.uuid4())
    self._jobs[preparation_id] = self.create_beam_job(
        preparation_id,
        request.job_name,
        request.pipeline,
        request.pipeline_options)
    self._jobs[preparation_id].prepare()
    _LOGGER.debug("Prepared job '%s' as '%s'", request.job_name, preparation_id)
    return beam_job_api_pb2.PrepareJobResponse(
        preparation_id=preparation_id,
        artifact_staging_endpoint=self._jobs[
            preparation_id].artifact_staging_endpoint(),
        staging_session_token=preparation_id)

  def Run(self,
          request,  # type: beam_job_api_pb2.RunJobRequest
          context=None,
          timeout=None
         ):
    # type: (...) -> beam_job_api_pb2.RunJobResponse
    # For now, just use the preparation id as the job id.
    job_id = request.preparation_id
    _LOGGER.info("Running job '%s'", job_id)
    self._jobs[job_id].run()
    return beam_job_api_pb2.RunJobResponse(job_id=job_id)

  def GetJobs(self,
              request,  # type: beam_job_api_pb2.GetJobsRequest
              context=None,
              timeout=None
             ):
    # type: (...) -> beam_job_api_pb2.GetJobsResponse
    return beam_job_api_pb2.GetJobsResponse(
        job_info=[job.to_runner_api() for job in self._jobs.values()])

  def GetState(self,
               request,  # type: beam_job_api_pb2.GetJobStateRequest
               context=None
              ):
    # type: (...) -> beam_job_api_pb2.GetJobStateResponse
    return beam_job_api_pb2.JobStateEvent(
        state=self._jobs[request.job_id].get_state())

  def GetPipeline(self,
                  request,  # type: beam_job_api_pb2.GetJobPipelineRequest
                  context=None,
                  timeout=None
                 ):
    # type: (...) -> beam_job_api_pb2.GetJobPipelineResponse
    return beam_job_api_pb2.GetJobPipelineResponse(
        pipeline=self._jobs[request.job_id].get_pipeline())

  def Cancel(self,
             request,  # type: beam_job_api_pb2.CancelJobRequest
             context=None,
             timeout=None
            ):
    # type: (...) -> beam_job_api_pb2.CancelJobResponse
    self._jobs[request.job_id].cancel()
    return beam_job_api_pb2.CancelJobResponse(
        state=self._jobs[request.job_id].get_state())

  def GetStateStream(self, request, context=None, timeout=None):
    # type: (...) -> Iterator[beam_job_api_pb2.GetJobStateResponse]
    """Yields state transitions since the stream started.
      """
    if request.job_id not in self._jobs:
      raise LookupError("Job {} does not exist".format(request.job_id))

    job = self._jobs[request.job_id]
    for state, timestamp in job.get_state_stream():
      yield make_state_event(state, timestamp)

  def GetMessageStream(self, request, context=None, timeout=None):
    # type: (...) -> Iterator[beam_job_api_pb2.JobMessagesResponse]
    """Yields messages since the stream started.
      """
    if request.job_id not in self._jobs:
      raise LookupError("Job {} does not exist".format(request.job_id))

    job = self._jobs[request.job_id]
    for msg in job.get_message_stream():
      if isinstance(msg, tuple):
        resp = beam_job_api_pb2.JobMessagesResponse(
            state_response=make_state_event(*msg))
      else:
        resp = beam_job_api_pb2.JobMessagesResponse(message_response=msg)
      yield resp

  def DescribePipelineOptions(self, request, context=None, timeout=None):
    # type: (...) -> beam_job_api_pb2.DescribePipelineOptionsResponse
    return beam_job_api_pb2.DescribePipelineOptionsResponse()

class AbstractBeamJob(object):
  """Abstract baseclass for managing a single Beam job."""

  def __init__(self,
               job_id,  # type: str
               job_name,  # type: str
               pipeline,  # type: beam_runner_api_pb2.Pipeline
               options  # type: struct_pb2.Struct
              ):
    self._job_id = job_id
    self._job_name = job_name
    self._pipeline_proto = pipeline
    self._pipeline_options = options
    self._state_history = [(beam_job_api_pb2.JobState.STOPPED,
                            Timestamp.now())]

  def prepare(self):
    # type: () -> None
    """Called immediately after this class is instantiated"""
    raise NotImplementedError(self)

  def run(self):
    # type: () -> None
    raise NotImplementedError(self)

  def cancel(self):
    # type: () -> Optional[beam_job_api_pb2.JobState.Enum]
    raise NotImplementedError(self)

  def artifact_staging_endpoint(self):
    # type: () -> Optional[endpoints_pb2.ApiServiceDescriptor]
    raise NotImplementedError(self)

  def get_state_stream(self):
    # type: () -> Iterator[Optional[beam_job_api_pb2.JobState.Enum]]
    raise NotImplementedError(self)

  def get_message_stream(self):
    # type: () -> Iterator[Union[int, Optional[beam_job_api_pb2.JobMessage]]]
    raise NotImplementedError(self)

  @property
  def state(self):
    """Get the latest state enum."""
    return self.get_state()[0]

  def get_state(self):
    """Get a tuple of the latest state and its timestamp."""
    # this is safe: initial state is set in __init__
    return self._state_history[-1]

  def set_state(self, new_state):
    """Set the latest state as an int enum and update the state history.

    :param new_state: int
      latest state enum
    :return: Timestamp or None
      the new timestamp if the state has not changed, else None
    """
    if new_state != self._state_history[-1][0]:
      timestamp = Timestamp.now()
      self._state_history.append((new_state, timestamp))
      return timestamp
    else:
      return None

  def with_state_history(self, state_stream):
    """Utility to prepend recorded state history to an active state stream"""
    return itertools.chain(self._state_history[:], state_stream)

  def get_pipeline(self):
    # type: () -> beam_runner_api_pb2.Pipeline
    return self._pipeline_proto

  @staticmethod
  def is_terminal_state(state):
    from apache_beam.runners.portability import portable_runner
    return state in portable_runner.TERMINAL_STATES

  def to_runner_api(self):
    # type: () -> beam_job_api_pb2.JobInfo
    return beam_job_api_pb2.JobInfo(
        job_id=self._job_id,
        job_name=self._job_name,
        pipeline_options=self._pipeline_options,
        state=self.state)


'''
Pands project for Programming and Scripting
analysis.py
Author: Nur Bujang

To write a program called analysis.py that:
    1. Outputs a summary of each variable to a single text file,
    2. Saves a histogram of each variable to png files,
    3. Outputs a scatter plot of each pair of variables.
    4. Perform any other analysis I think is appropriate

Save iris.data file from UCI website and upload into pands-project folder
General Process: Load data, Analyze/visualize dataset, Model training, Model Evaluation, Model Testing,
'''

'''
IMPORT MODULES
'''
import numpy as np # for computational operations
import pandas as pd # for data loading from other sources and processing
import seaborn as sns # for data visualization
import matplotlib.pyplot as plt # for data visualization
from sklearn.model_selection import train_test_split # to split data
from sklearn.neighbors import KNeighborsClassifier # import kNN Classifier from sklearn
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, classification_report, f1_score, accuracy_score # import sklearn module to print confusion matrix and report
from sklearn.linear_model import LogisticRegression # import Logistic Regression from sklearn
from sklearn.tree import DecisionTreeClassifier # import Decision Tree Classifier from sklearn
from sklearn.svm import SVC # import Support Vector Machine Classification from sklearn
from sklearn.ensemble import RandomForestClassifier # import Random Forest Classifier from sklearn
from sklearn.naive_bayes import GaussianNB # import Gaussian Naïve Bayes from sklearn

'''
LOAD DATA AND ADD COLUMN HEADER
'''
columns = ['Sepal length (cm)', 'Sepal width (cm)', 'Petal length (cm)', 'Petal width (cm)', 'Iris species'] # instantiate a list of column names
# instantiate a pandas dataframe object called df, read the csv file with the read_csv funtion and assign a name to each column
df = pd.read_csv('iris.data', names=columns) # df contains iris.data and add column names to the dataframe
print('\nThe first 2 lines of the dataset\n',df.head(2))  # print out first 2 lines to see if the column names were added properly
# Output: Column names were added properly

'''
PRE-PROCESSING - Quick lookover on the dataset, check for missing values, duplicates
'''
# QUICK LOOKOVER OF THE DATASET
df.value_counts("Iris species")  # value_counts funtion to see how many lines for each species
print('\nThe number of rows for each Iris species\n',df.value_counts("Iris species"))  # print how many lines for each species
# Output: 50 lines for each species

# BASIC INFO ABOUT THE DATASET, COLUMN NUMBERS, DATA TYPES, NON-NULL VALUES
# can also be used to see of there are missing values from the number of non-null values
df.info()  # method to print out in terminal output the basic information about the dataframe
# print(df.info()) # commented out because I don't want to print it twice
# Output:
# It is a dataframe
# 150 rows, 0 to 149, 5 data columns and column names
# number of non-null values (data that is not missing) in each column
# 4 float datatypes and 1 object

# GET NUMBER OF MISSING VALUES IN EACH COLUMN
df.isnull().sum() # method to get sum of null values in the dataset
print ('\nThe number of missing value\n',df.isnull().sum()) # print out number of missing values
# OR
# df.isna().sum() # sum of na values
# print(df.isna().sum()) # print out number of missing values

# REMOVE DUPLICATES
df.drop_duplicates() # method to remove duplicates from the dataset
print('\nData shape after duplicate removal is',df.drop_duplicates().shape)  # print out the shape of the dataset after duplicate removal
# output: 147 rows, 5 columns, meaning 3 rows were removed

'''
Question 1. SUMMARY into text file, containing basic statistical analysis
df. describe to get count, mean, standard deviation, min and max values, lower, mid and upper percentile
'''
df.describe()  # describe method to get basic statistical analysis data
# print (df.describe()) # to print out description of data in terminal output, but I do not want this
# What I want is to export data into a .txt file called summary.txt
# use the built-in open() function
text_file = open("summary.txt", "wt") # with open function, open a new file called summary, mode is w, and t is text mode and call it text_file
# then, export into text file called summary, convert pandas dataframe to string
n = text_file.write(df.describe().to_string()) # with write method, write the converted df.describe() string into a text file, and call it n
text_file.close()  # ALWAYS close file when done

'''
Question 2. DATA VISUALISATION: Histogram of each variable into png file
'''
sns.set(style="whitegrid")  # set background of seaborn histogram to whitegrid
fig, axs = plt.subplots(2, 2, figsize=(7, 8)) # with subplots function, create a figure containing subplots with multiple axes
# set subplot arrangement/grid position in 2 directions (2D grid) (2 down by 2 across)
# set size of whole figure (7 width, 8 height))
sns.histplot(data=df, x="Sepal length (cm)", # x-axis is sepal length
             kde=True, color="olive", ax=axs[0, 0]) # ax lists the subplot location at first row, first column
# Where:
# seaborn histplot function was used to plot the histogram
# dataset used was df
# determine x-axis label
# display KDE line : Kernel density estimation (KDE) shows the data using a continuous curve
# KDE creates data 'smoothing' of the density distribution with a Gaussian kernel
# set the histogram bar color for this plot to olive
# ax list contains the coordinate/grid location of each subplot on the figure
sns.histplot(data=df, x="Sepal width (cm)", kde=True, color="green", # x-axis is sepal width, histogram bar color is green
             ax=axs[0, 1])  # list of subplot location at first row, second column
sns.histplot(data=df, x="Petal length (cm)", # x-axis is petal length, histogram bar color is blue
             kde=True, color="blue", ax=axs[1, 0]) # list of subplot location at second row, first column
sns.histplot(data=df, x="Petal width (cm)", # x-axis is petal width, histogram bar color is purple
             kde=True, color="purple", ax=axs[1, 1]) # list of subplot location at second row, second column
fig.suptitle('Histogram of petal and sepal dimensions of three Iris species',
             fontweight='bold', size=14)  # function to customize figure's super title, default font color is black, fontweight is bold, fontsize is 14
fig.tight_layout()  # method to fit all subplots into one figure nicely automatically
plt.savefig('iris.png')  # function to save output into png file
plt.show()  # function to show plot

'''
Question 3. DATA VISUALISATION: Scatter plot of each pair of variables
'''
# df contains 3 classes (setosa, virginicus, versicolor) and 50 replicates each
# within that, the variables are Petal length, Petal width, Sepal length and Sepal width
# Perform a Scatter Plot matrix (aka Pair Plot) to see the relationship between a pair of variables within a combination of multiple variables
# to visualize the whole dataset, using 'Iris species' variable to assign different color to different species
pp = sns.pairplot(df, hue='Iris species', markers=[ # instantiate a seaborn pairplot function called pp
                  "o", "s", "D"], palette='brg', kind='reg', plot_kws={'line_kws': {'color': 'blue'}})
# Where:
# dataset used was df
# seaborn pairplot was used to plot the Scatter Plot
# The diagonal plots represent each of that column variable's data distribution
# hue colors the plot based on the different Iris species value
# marker o is circle, s is square, D is diamond
# palette is set to brg palette
# kind is a chart type, kind='reg' applies a linear regression line to identify the relationship within the scatter plot
# plot_kws and pass in a dictionary object to customize the regression fit and line to blue
plt.suptitle('Scatter Plot for sepal and petal attributes of three Iris species', color='#3D3F38',
             fontweight='bold', size=17) # function to customize figure's super title, font color is hex number 3D3F38, fontweight is bold, fontsize 17
pp.fig.subplots_adjust(top=0.92, bottom=0.08)
# fig.subplots_adjust function shifts the pairplot position, 0.92 of the default 0.9 for the top edge and 0.08 of the default 0.1 for the bottom edge
# this had to be done to show the super title properly
plt.show()  # function to show plot
# Results:
# I. setosa is distinctly different and forms a separate cluster from I. virginica and I. versicolor, which shows some pairwise relationship between these two
# The petal length and width of I. setosa have much narrower distribution compared to the other two species.
# there are overlaps in sepal length and width of all three species.

'''
Question 4. OTHER ANALYSIS:
Data visualization to detect outliers, trends/pattern and Basic Machine Learning analysis
'''

'''
4.1 Pearson correlation analysis by attributes
to determine the degree/strength of linear relationship between 2 continuous variables
correlation efficient closer to 1 indicates a strong +ve relationship, closer to -1 indicates a strong -ve relationship
'''
corr = df.corr(method="pearson")  # instantiate a corr method and call it corr
# default is already Pearson Correlation (for linear), but can be changed to Kendall and Spearman for non-parametric. eg: method="Spearman"
bool_upper_matrix = np.tril(np.ones(corr.shape)).astype(bool)  # eliminate upper triangle for better readibility
# Numpy tril function to extract lower triangle or triu to extract upper triangle
# np.ones returns an array of 1 to create a boolean matrix with the same size as the correlation matrix
# astype converts the upper triangle values to False, while the lower triangle will have the True values
corr = corr.where(bool_upper_matrix) # Pandas where() returns same-sized dataframe, but False is converted to NaN on the upper triangle
print('\nPearson Correlation by attributes\n',corr)  # print as terminal output
# Results:
# High positive correlation between Petal length & Petal width, Petal length & Sepal length and Sepal length and petal width

# OR,

# build a Correlation matrix to visualize the parameters which best correlate with each other easier
# set size of whole figure (9 width, 6 height)
fig, ax = plt.subplots(figsize=(9, 6)) # with subplots function, create 1 figure and a single axes and set the figure size to be width=9 and height=6
fig.suptitle('Correlation matrix for petal and sepal attributes of three Iris species',
             color='#3D3F38', fontweight='bold')  # customize figure's main/super title, color is hex number 3D3F38, fontweight bold
hm = sns.heatmap(corr, annot=True, ax=ax, cmap='coolwarm', square=True, linewidths=0.1, # using heatmap function, create a seaborn heatmap called hm
                linecolor='yellow', cbar_kws={'label': 'Range', 'shrink': 0.9})  # customize heatmap
# Where:
# annot=True: if True, the data value in each cell will be displayed
# coolwarm color palette for the heatmap
# square = True: cell is square-shaped
# linewidth is the width of lines separating each cell and the color is yellow
# I passed arguments into the color bar to show Range as label and shrank it to 0.9 times the original size
# default color bar is vertical, but to move it to the bottom, just add 'orientation': 'horizontal' to cbar argument
hm.set_xticklabels(hm.get_xticklabels(), rotation=0, fontsize=10)
# set the label for x-axis to follow each column name
# rotation sets the xticks "upright" as opposed to sideways in any figure size, just to read easier
hm.set_yticklabels(hm.get_yticklabels(), rotation=0, fontsize=10)
# set the label for y axis to follow each column name
# rotation sets the yticks "upright" in any figure size
plt.show()  # function to show plot
# Results:
# Strong positive correlation between Petal length & Petal width, Petal length & Sepal length, Sepal length & Petal width (same as above)

'''
4.2 Pearson correlation analysis by species
If I group them by species, I will get more insights on which attributes are highly correlated for each Iris species:
'''
df.groupby("Iris species").corr(method="pearson") # Pearson Correlation, but grouped by class
print('\nPearson Correlation by species\n',df.groupby("Iris species").corr(method="pearson")) # print as terminal output
# Results:
# Iris setosa: high correlation between Sepal length & Sepal width
# Iris versicolor: strong correlation between Petal length & Petal width, Petal length & Sepal length
# Iris virginica: high correlation between Petal length & Sepal length

'''
4.3 DATA VISUALIZATION: Box plot
to display data point distribution/spread, skewness, variance and outliers
shows the minimum, first quartile, median, third quartile and maximum
'''
def graph(y):  # define a graph function of y axis
    sns.boxplot(x="Iris species", y=y, data=df) # seaborn boxplot, with Iris species on x-axis
# on x-axis is Iris species, on y-axis is y (attributes) and the data used is the iris dataframe
    sns.stripplot(x="Iris species", y=y, data=df, # function to add a seaborn stripplot/jitter plot over the boxplot
                  jitter=True, linewidth=1, edgecolor="red", alpha=0.35)
# Where:
# on x-axis is Iris species, on y-axis is y (attributes) and the data used is the iris dataframe
# jitter=True will display the dots (jitter)
# linewidth is width of line around the dots, it has to be set to >0 to be visible
# the edgecolor of the jitter/dots is red to make the dots look clearer
# set transparency (alpha) to 0.35 so it is not too opaque
plt.figure(figsize=(10, 10))
plt.subplot(221)  # function to set grid position to top left (2 rows, 2 columns, first top)
graph('Sepal length (cm)') # y-axis is Sepal length (cm)
plt.subplot(222)  # function to set grid position to top right (2 rows, 2 columns, second top)
graph('Sepal width (cm)') # y-axis is Sepal width (cm)
plt.subplot(223)  # function to set grid position to bottom left (2 rows, 2 columns, first bottom)
graph('Petal length (cm)') # y-axis is Petal length (cm)
plt.subplot(224) # function to set grid position to bottom right (2 rows, 2 columns, second bottom)
graph('Petal width (cm)') # y-axis is Petal width (cm)
plt.suptitle('Box Plot for sepal and petal attributes of three Iris species', color='#3D3F38',
             fontweight='bold', size=17) # function to customize the figure's super title, color is hex number 3D3F38, fontweight bold, font size 17
plt.show()  # function to show plot
# Results:
# Iris setosa has the least distributed and smallest petal size
# Iris virginica has the biggest petal size, and Iris versicolor's petal size is between Iris setosa and virginica
# Sepal size may not be a good variable to differentiate species

'''
4.4 DATA VISUALIZATION: Violin Plot
to give more insight into data distribution and density on the y-axis
it contains all data points, unlike the box plot which shows minimum, first quartile, median, third quartile and maximum and error bars
'''
fig, axs = plt.subplots(1, len(columns)-1, figsize=(20,5)) # using subplot function, create a figure containing subplots with multiple axes
# plot the subplots in 1 row of 4 subplots (stacked in 1 direction only, but still use axs because each has their own axes)
# length -1 means all columns (5) -1 = 4 columns
# set the figure size to 20 width and 5 height
for i in range(0,len(columns)-1):
    sns.violinplot(x='Iris species', y=df[columns[i]], data=df, ax=axs[i])
    axs[i].set_ylabel(columns[i])
# Where:
# for i in range can be translated to: for item in columns 1 to 4
# seaborn violinplot, x-axis is Iris species, y is each column name, data is iris dataframe
# ax is the object to draw the plot into, in this case, the columns
# axs[i].set_ylabel(columns[i]) means each column list i is set as the y-axis label
plt.suptitle('Violin Plot for sepal and petal attributes of three Iris species', color='#3D3F38',
             fontweight='bold', size=15) # function to customize the figure's super title, color is hex number 3D3F38, fontweight bold, font size 15
plt.show() # function to show plot

'''
*************DATA PREPARATION FOR BASIC MACHINE LEARNING***************

SPLITTING THE DATA FOR TRAINING AND TESTING
First, split the data into training (80%) and testing (20%) to detect overfitting (=model learned the training data very well but fails on testing)
Later, the testing dataset will be used to check the accuracy of the model.

While most ML examples available uses only 2 variables (sepal length and width only or petal length and width only) to simplify analysis:
X = df.iloc[:,:2] # take everything until the second column (columns 0 and 1) and store the first two columns (Sepal length and Sepal width) into attributes (X)
y = df.iloc[:,4] # store the target variable (Iris species) into labels (y)
I used all 4 variables because the best determinants are still unknown at this point (whether sepal or petal attribute is better than the other).
'''

X = df.iloc[:, :-1].values # X dataset is everything up until the last column but not including the last column (Iris species)
y = df.iloc[:, 4].values # = [:, -1],  y dataset gets all the rows in the last/5th column (target variable (Iris species) into labels (y))
print('\nDataset shape before Train-Test split is',X.shape, y.shape)
# print out the shape (number of rows and columns) AFTER the data split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42) # split the dataset into training (80%) and testing (20%)
# test size is 20%, meaning training size is 80%
# random_state is the seed of randomness to help get the same results everytime. It can be any number, really.
print('\nDataset shape after Train-test split is',X_train.shape, X_test.shape, y_train.shape, y_test.shape)
# print out the shape (number of rows and columns) of test and train data AFTER split in terminal output

'''
4.5 kNN Classifier
used for regression and classification
kNN calculates the distance between the data points and predict the correct species class for the datapoint
where k = number of neighbors/points closest to the test data
'''
for i in np.arange(7, 10): # for i in numpy arange starting at 7 and stopping at 10
# keep k small because there are only 3 species, to prevent overfitting
    knn = KNeighborsClassifier(n_neighbors=i) # instantiate a class and name it knn
    # number of neighbors range from 7-10
    knn.fit(X_train, y_train) # fit method fits the model to the training set
    y_pred_knn = knn.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_knn
    print("\nk-Nearest Neighbor model accuracy for k = %d accuracy is" % i, knn.score(X_test,
          y_test)*100)  # Calculate and print the accuracy of the model using testing dataset ranging (from 7-10)*100
# %d is a placeholder for a number, %s is for string
# % on its own means the values of each i in 7-10 range are then passed in through a tuple using the % operator
    print("k-Nearest Neighbor F1 score for k = %d F1 score is" % i, f1_score(y_test, y_pred_knn, average='micro'))
# Calculate and print the F1 score the model using testing dataset ranging (from 7-10)*100
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive
# Other averaging methods are macro (meaning it is calculated using regular/unweighted mean) and weighted (takes into account each class's support)
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset
# F1 combines precision (% of correct predictions) and recall (proportion of correct predictions over total occurences)

'''
4.6 Logistic Regression
To estimate the relationship between 1 dependent variable and 1 or more independent variables
Used for classification and prediction analysis
uses Sigmoid function (logistic function) instead of linear function in Linear regression
'''
lr = LogisticRegression(C=0.5, random_state=42)  # instantiate a class and name it lr
# C value is a model hyperparameter, which is a model criteria outside of the model and its value cannot be estimated from the data
# default C is 1.0
# High C value means training data is more reliable (reflects real world data), low C value means training data may not reflect real world data
# I picked 0.5 to be somewhere in the middle
# random_state is the seed of randomness to help get the same results everytime. It can be any number
lr.fit(X_train, y_train)  # fit method fits the model to the training set
# compare model’s output (y_pred) with target values that we already have (y_test)
y_pred_lr = lr.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_lr
print('\nLogistic Regression model accuracy is',
      accuracy_score(y_test, y_pred_lr) * 100) # print accuracy score * 100
# Model accuracy score is how many times the model makes correct predictions over the total number of predictions
print('Logistic Regression model F1 score is',
      f1_score(y_test, y_pred_lr, average='micro')) # print out F1 score
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive
'''
4.7 Decision Tree Classifier
to build a classification or regression
branches bifurcate based on Y/N or T/F and breaks the dataset smaller everytime
to eventually form a tree structure with decision nodes and leaf nodes
'''
dtclassifier = DecisionTreeClassifier(random_state=42) # instantiate a class and name it dtclassifier
dtclassifier.fit(X_train, y_train)  # fit method fits the model to the training set
# Predict the response for test dataset
y_pred_dt = dtclassifier.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_dt
# Evaluate the model using testing dataset
dispdt = ConfusionMatrixDisplay.from_estimator(dtclassifier, X_test, y_test) # instantiate a ConfusionMatrixDisplay class and from_estimator method and call it dispdt
# Confusion matrix contains Actual values of Positive(1) and Negative(0) on the x-axis and Predicted values of Positive(1) and Negative(0) on the y-axis
# CM for iris contains Actual setosa, versicolor, virginica on the x-axis and Predicted setosa, versicolor, virginica on the y-axis
# True Positive = the actual and predicted value should be the same = 10
# True Negative = 9+0+0+11 = 20
# False Positive = 0+0 (across) = 0
# False Negative = 0+0 (down) = 0
plt.suptitle('Decision Tree Confusion Matrix for sepal and petal \nattributes of three Iris species',
             color='#3D3F38', fontweight='bold', size=13) # function to customize super title, font color is hex number 3D3F38, fontweight bold, font size 13
plt.grid(False) # function to eliminate white grid within the confusion matrix
plt.show() # function to display Confusion Matrix
print('\nDecision Tree Classification model accuracy is',
      accuracy_score(y_test, y_pred_dt)*100)  # print out accuracy score *100
print('Decision Tree Classification model F1 score is',
      f1_score(y_test, y_pred_dt, average='micro')) # print out F1 score
# Precision = True Positive : (True Positive + False Positive) = 10/(10+0+0 (across)) = 1
# Recall = True positive : (True Positive + False Negative) = 10/(10+0+0 (down)) = 1
# F1 score = 2*((precision*recall)/(precision+recall)) = 2(1/2) = 1
# that is, F1 = how many times it makes a correct prediction in the whole dataset
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive

'''
4.8 Support Vector Machine Classifier
for regression and classification
to map data points into a high dimensional space
and then create the best boundary to separate data into classes by creating a hyperplane line with the most margin from the data point
'''
svclassifier = SVC() # instantiate a class and name it svclassifier
svclassifier.fit(X_train, y_train) # fit method fits the model to the training set
y_pred_svc = svclassifier.predict(X_test)  # instantiate a predict method using the test dataset and call it y_pred_svc
print('\nClassification report for Support Vector Classifier\n',classification_report(y_test, y_pred_svc)) # print out classification report in terminal output
# Classification report results:
# Precision = Ratio of True Positive : (True Positive + False Positive. So, setosa (10/10=1), versicolor (9/9=1), virginica (11/11=1)
# Recall = Ratio of True positive : (True Positive + False Negative)
# F1 score = 2*((precision*recall)/(precision+recall)) = 2(1/2) = 1
# Support = number of actual occurences of that class
# Accuracy = correct predictions for all classes / total number of predictions = 10/10 = 1
print('Confusion matrix for Support Vector Classifier\n',confusion_matrix(y_test, y_pred_svc)) # print out Confusion matrix in terminal output
print('\nSupport Vector Classifier model accuracy is', accuracy_score(
    y_test, y_pred_svc)*100)  # print out accuracy score *100
print('Support Vector Classifier model F1 score is',
      f1_score(y_test, y_pred_svc, average='micro')) # print out F1 score
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive

'''
4.9 Random Forest Classifier
used to perform both regression and classification
to create a cluster of decision trees containing different sub-features from the features, thus forming a 'forest'
each bunch is trained on random subsets from training group (drawn with replacement and can be used again) and features (drawn without replacement and cannot be reused)
each tree picks the features randomly, making it possible to find which features are more important than others
'''
rf = RandomForestClassifier(n_estimators=10, n_jobs=-1) # instantiate a class and name it rf
rf.fit(X_train, y_train)  # fit method fits the model to the training set
# compare model’s output (y_pred) with the target values that we already have (y_test)
y_pred_rf = rf.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_rf
# Random Forest visualization using confusion matrix
disprf= ConfusionMatrixDisplay.from_estimator(rf, X_test, y_test) # instantiate a ConfusionMatrixDisplay class and from_estimator method and call it disprf
plt.suptitle('Random Forest Confusion Matrix for sepal and petal \nattributes of three Iris species',
             color='#3D3F38', fontweight='bold', size=13) # function to customize super title, font color is hex number 3D3F38, fontweight bold, font size 13
plt.grid(False) # function to eliminate white grid within the confusion matrix
plt.show() # function to show plot
print('\nRandom Forest model accuracy score is',
      accuracy_score(y_test, y_pred_rf)*100) # print accuracy score * 100
print('Random Forest model F1 score is', f1_score(
    y_test, y_pred_rf, average='micro')) # print out F1 score
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive

'''
4.10 Gaussian Naïve Bayes Classifier
'Naive' because it assumes that each variable are independent of each other
it predicts the probability of different species based on different attributes
I used Gaussian because data is continuous, and assumed to be normally-distributed
'''
gaussian = GaussianNB() # instantiate a class and name it gaussian
gaussian.fit(X_train, y_train) # fit method fits the model to the training set
y_pred_gs = gaussian.predict(X_test) # instantiate a predict method using the test dataset and call it y_pred_gs
cm = confusion_matrix(y_test, y_pred_gs) # instantiate confusion matrix function and call it cm
accuracy = accuracy_score(y_test,y_pred_gs)*100 # instantiate accuracy score function and call it accuracy
# multiply by 100 here because it is too complicated to do so in the print format
f1 = f1_score(y_test,y_pred_gs,average='micro') # instantiate an f1 score function and name it f1
# Average method setting is set to micro, meaning the average takes into account the sum of True Positive, False Negative and False Positive
print('\nClassification report for Naive-Bayes Classifier\n',classification_report(y_test, y_pred_gs)) # print Classification Report in terminal output
print('\nConfusion matrix for Naive Bayes\n',cm) # print confusion matrix in terminal output
print('\nNaive-Bayes model accuracy score is %.1f' %accuracy) # print out accuracy score, .1f is float with 1 decimal point of the accuracy value
print('Naive-Bayes model F1 score is %.3f' %f1) # # print out F1 score, .3f is float with 3 decimal points of the f1 value
# %.1f and %.3f are format specifiers. They begin with %, then followed by a character that represents the data type, which is a float
# F1 score measures the accuracy of the model, that is how many times it makes a correct prediction in the whole dataset


from fastapi import APIRouter
from app.schema import Loan
from app.models import Products

routes = APIRouter()

@routes.post("/api/v1/loans")
async def create_loan(loan: Loan):
    product = await Products.create_loan(product_name=loan.product_name)
    return {"loan_id": product}

@routes.get("/api/v1/loans/{product_uuid}")
async def get_loan(product_uuid):
    product = await Products.get_loan_by_id(product_uuid=product_uuid)
    return product


from pdfminer.converter import TextConverter
from pdfminer.pdfinterp import PDFPageInterpreter
from pdfminer.pdfinterp import PDFResourceManager
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage
from pdfminer.pdffont import PDFFont
from pdfminer.converter import PDFPageAggregator
import pdfminer
import tabula
import io

def parse_obj(objs, fonts):
    for obj in objs:
        if isinstance(obj, pdfminer.layout.LTTextBox):
            for o in obj._objs:
                if isinstance(o,pdfminer.layout.LTTextLine):
                    text=o.get_text()
                    if text.strip():
                        for c in  o._objs:
                            if isinstance(c, pdfminer.layout.LTChar):
                                if c.fontname not in fonts:
                                    fonts.append(c.fontname)
        # if it's a container, recurse
        elif isinstance(obj, pdfminer.layout.LTFigure):
            parse_obj(obj._objs, fonts)
        else:
            pass

def extract_pdf_info(pdf):
    text = ''
    n_images = 0
    fonts = []
    linkedin = ''
    n_tables = 0

    #no. of tables
    try:
        tables = tabula.read_pdf(pdf, multiple_tables=True, pages='all')
        n_tables = len(tables)
    except:
        pass

    for page in PDFPage.get_pages(pdf, check_extractable=True, caching=True):
        #extract text
        resource_manager = PDFResourceManager()
        fake_file_handle = io.StringIO()
        converter = TextConverter(resource_manager, fake_file_handle)
        page_interpreter = PDFPageInterpreter(resource_manager, converter)
        page_interpreter.process_page(page)
        t = fake_file_handle.getvalue()
        text = text + '\n' + t

        #extract fonts
        device = PDFPageAggregator(resource_manager, laparams=LAParams())
        device_interpretor = PDFPageInterpreter(resource_manager, device)
        device_interpretor.process_page(page)
        layout = device.get_result()
        parse_obj(layout._objs, fonts)

        #no. of images
        try:

            for key in page.resources['XObject'].keys():
                if key.startswith('Im'):
                    n_images += 1
        except KeyError:
            pass

        #linkedIn link
        if page.annots and linkedin == '':
            for annotation in page.annots:
                annotationDict = annotation.resolve()
                if str(annotationDict['Subtype']) == "/'Link'":
                    if str(annotationDict['A']['URI'])[2:-1].startswith('http://www.linkedin.com'):
                        linkedin = str(annotationDict['A']['URI'])[2:-1]

    return {"linkedin": linkedin, "n_tables": n_tables, "fonts": fonts, "n_images": n_images, "text": text}

#file = open('../../../resume-test/Resume_1.pdf', 'rb')
#print(extract_pdf_info(file))


import wave
import pyaudio
import audioop

CHUNK = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 2
RATE = 44100
RECORD_SECONDS = 10

p = pyaudio.PyAudio()

stream = p.open(format=FORMAT,
				channels=CHANNELS,
				rate=RATE,
				input=True,
				frames_per_buffer=CHUNK)

for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
	data = stream.read(CHUNK)
	rms = audioop.rms(data, 2)
	print(f"{rms}           ", end='\r')

stream.stop_stream()
stream.close()
p.terminate()

if __name__ == "__main__":
	pass


"""
            Dertermining Last Friday's Date
            -------------------------------

Problem:
--------

  You want a general solution for finding a date for the last occurrence of
a day of the week. Last Friday, for example:

Solution:
---------

  Python's datetime module has utility functions and classes to help perform
calculations like this. A decent, generic solution to this problem looks like
this:
"""

from datetime import datetime, timedelta

weekdays = ['Monday', "Tuesday", "Wednesday", "Thursday", "Friday",
            "Saturday", "Sunday"]

def get_previous_byday(dayname, start_date=None):
    if start_date is None:
        start_date = datetime.today()

    day_num = start_date.weekday()
    day_num_target = weekdays.index(dayname)
    days_ago = (7 + day_num - day_num_target) % 7

    if days_ago == 0:
        days_ago = 7

    target_date = start_date - timedelta(days= days_ago)
    return target_date

"""
Discussion:
-----------

  This recipe works by mapping the start date and the target date to their
numeric position in the week (with Monday as 0). Modular arithmetic is then
used to figure out how many days ago the target date last occured. From there,
the desired date is calculated from the start date by substracting an
appropriate timedelta instance.

  If you're performing a lot of date calculations like this, you may be
better off installing the python-dateutil package insteas.

"""

if __name__ == "__main__":
    print("Today: ", datetime.today())

    # Go through all the days in the week
    for day in weekdays:
        print(get_previous_byday(day))

    print("\n\nUsing the optional start_date:\n")
    print(get_previous_byday("Sunday", datetime(2012, 12, 21)))


from datetime import datetime, timedelta, date

TODAY = date(2018, 11, 12)

def extract_dates(data):
    """Extract unique dates from DB table representation as shown in Bite"""
    dates = set()
    for element in data.split("|"):
        try:
            valid_date = datetime.strptime(element.strip(), "%Y-%m-%d")
            dates.add(valid_date.date())
        except:
            pass

    return sorted(list(dates), reverse=True)

def calculate_streak(dates):
    """Receives sequence (set) of dates and returns number of days
       on coding streak.

       Note that a coding streak is defined as consecutive days coded
       since yesterday, because today is not over yet, however if today
       was coded, it counts too of course.

       So as today is 12th of Nov, having dates 11th/10th/9th of Nov in
       the table makes for a 3 days coding streak.

       See the tests for more examples that will be used to pass your code.
    """
    highest_streak = 0
    coding_streak = 0
    yesterday = TODAY - timedelta(days=1)

    for i in range(len(dates)):
        if i == 0 and (dates[i] not in [yesterday, TODAY]):
            return coding_streak

        if dates[i] == TODAY:
            coding_streak += 1
            highest_streak += 1

        if dates[i] == yesterday:
            coding_streak += 1
            highest_streak += 1

        previous_date = dates[i - 1]
        current_date = dates[i]

        if current_date not in [TODAY, yesterday] and coding_streak > 0:
            if (previous_date - current_date).days == 1:
                coding_streak += 1
                if coding_streak > highest_streak:
                    highest_streak += 1
            else:
                coding_streak = 0

    return highest_streak

if __name__ == "__main__":
    data = """
    +------------+------------+---------+
    | date       | activity   | count   |
    |------------+------------+---------|
    | 2018-11-11 | pcc        | 1       |
    | 2018-11-07 | pcc        | 1       |
    | 2018-11-09 | 100d       | 2       |
    | 2018-11-10 | 100d       | 1       |
    | 2018-11-08 | pcc        | 1       |
    +------------+------------+---------+
    """
    dates = extract_dates(data)
    print(calculate_streak(dates))


# Importing
from tkinter import *
from PIL import ImageTk, Image
from tkinter import filedialog
from tkinter.messagebox import showinfo
def next_func():
    welcome.destroy()
    root = Tk()
    root.title('Py Image Viewer')
# Functions
    def openfn():
            filename = filedialog.askopenfilename(title='Load Image')
            return filename
    def open_img():
        x = openfn()
        try:
            img = Image.open(x)
            width, height = img.size
            img = img.resize((width, height), Image.ANTIALIAS)
            root.geometry(str(width)+"x"+str(height))
            img = ImageTk.PhotoImage(img)
            panel = Label(root, image=img)
            panel.image = img
            panel.pack()
        except:
            showinfo("Error", "The file you tried to open was not a supported image file.")
            root.destroy()
    open_img()
    root.mainloop()# mainloop

welcome = Tk()
welcome.title('Py Image Viewer')
img2 = ImageTk.PhotoImage(Image.open("Welcome_Screen.png"))
panel2 = Label(welcome, image = img2)
panel2.pack(side = "top", fill = "both", expand = "yes")
intro = Label(welcome, text="This program will help you to load an image", font=("Helevectica",20,'bold')).pack()
next_but = Button(welcome, text="Next", bd=4, padx=14, pady=14, bg='orange', command=next_func, font=("Courier New",16,'bold')).pack()
welcome.mainloop()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import annotations
from typing import Dict, Optional
import os
import click
import simplejson as json
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
import torch
from torchvision import transforms
from models.ImageLoaderCycleGAN import ImageLoaderCycleGAN
from models.CycleGAN import CycleGAN

class DataCycleGAN(object):
    def __init__(
        self: DataCycleGAN,
        pathA: str,
        pathB: str,
        device: torch.Device
    ) -> None:
        self._load_data(pathA, 'A', device)
        self._load_data(pathB, 'B', device)
        return

    def _load_data(
        self: DataCycleGAN,
        path: str,
        side: str,
        device: torch.Device
    ) -> None:
        real = 'real%s' % side
        data = 'data%s' % side
        compose = transforms.Compose([
            transforms.Resize((256, 256)),
            transforms.ToTensor(),
            transforms.Normalize(
                mean=[0.5, 0.5, 0.5],
                std=[0.5, 0.5, 0.5]
            )
        ])
        if path is None:
            setattr(self, real, None)
            setattr(self, data, None)
            return
        reald = Image.open(path).convert('RGB')
        setattr(self, real, reald)
        rdata = compose(reald.copy()).unsqueeze(dim=0).to(device)
        setattr(self, data, rdata)
        return

    def convert_fake(
        self: DataCycleGAN,
        ret: Dict,
        side: str,
        name: str
    ) -> Optional[np.array]:
        setattr(self, 'name%s' % side, name)
        oside = 'A' if side == 'B' else 'B'
        fname = 'fake%s' % oside
        fake = ret.get(fname)
        if fake is None:
            setattr(self, fname, None)
            return
        fdata = fake.cpu().squeeze(dim=0).float().detach().numpy()
        fdata = (
            (np.transpose(fdata, (1, 2, 0)) + 1) / 2.0 * 255.0
        ).astype(np.uint8)
        fimg = Image.fromarray(fdata)
        fimg = fimg.resize((256, 256), Image.BICUBIC)
        setattr(self, fname, fimg)
        return

    def output_each(
        self: DataCycleGAN,
        side: str
    ) -> None:
        oside = 'A' if side == 'B' else 'B'
        fake = getattr(self, 'fake%s' % oside)
        if fake is None:
            return
        fname = 'results/CycleGAN_%s_fake_%s.png' % (
            self.data_name,
            getattr(self, 'name%s' % oside)
        )
        fake.save(fname)
        return

    def output_grid(
        self: DataCycleGAN,
    ) -> None:
        imgs = list()
        titles = list()
        hnum = 0
        wnum = 2
        if self.realA is not None:
            imgs.append(self.realA)
            titles.append('real %s' % self.nameA)
            imgs.append(self.fakeB)
            titles.append('fake %s' % self.nameB)
            hnum += 1
        if self.realB is not None:
            imgs.append(self.realB)
            titles.append('real %s' % self.nameB)
            imgs.append(self.fakeA)
            titles.append('fake %s' % self.nameA)
            hnum += 1
        wnum = 2
        fig = plt.figure(figsize=(wnum * 2.56, hnum * 2.56), facecolor='w')
        for i, (img, title) in enumerate(zip(imgs, titles)):
            ax = fig.add_subplot(hnum, wnum, i + 1)
            ax.imshow(img)
            ax.set_axis_off()
            ax.set_title(title)
        plt.savefig(
            'results/CycleGAN_%s.png' % self.data_name,
            facecolor='w', edgecolor='w',
            bbox_inches='tight', pad_inches=0.1
        )
        return

class ModelCycleGAN(object):
    def __init__(
        self: ModelCycleGAN,
        config: str,
        device: torch.Device
    ) -> None:
        with open(config, 'rt') as rf:
            process_config = json.load(rf)
        prep = ImageLoaderCycleGAN(
            process_config['config_data_json'],
            process_config['config_preprocessor_json']
        )
        takeover_config = {
            'data_name': prep.config.data_name,
            'base_dir': prep.config.base_dir,
        }
        self.model = CycleGAN(
            takeover_config,
            process_config['config_model_json']
        )
        self.model.load()
        self.model.eval()
        self.model.to(device)
        self.nameA = prep.config.name_a
        self.nameB = prep.config.name_b
        self.data_name = prep.config.data_name
        return

    def generate(
        self: ModelCycleGAN,
        data: DataCycleGAN
    ) -> None:
        data.data_name = self.data_name
        with torch.no_grad():
            ret = self.model(dataA=data.dataA, dataB=data.dataB)
        data.convert_fake(ret, 'A', self.nameA)
        data.convert_fake(ret, 'B', self.nameB)
        return

@click.command()
@click.option('--config', '-c', required=True, type=str)
@click.option('--a-path', '-a', type=str)
@click.option('--b-path', '-b', type=str)
@click.option('--grid/--no-grid', is_flag=True, default=True)
def main(config: str, a_path: str, b_path: str, grid: bool) -> None:
    # assertion
    assert(os.path.exists(config))
    if a_path is not None:
        assert(os.path.exists(a_path))
    if b_path is not None:
        assert(os.path.exists(b_path))
    if a_path is None and b_path is None:
        raise(ValueError("specify at least either A or B"))
    # detect device
    if torch.cuda.is_available():
        device = torch.device('cuda')
    else:
        device = torch.device('cpu')
    # load CycleGAN
    model = ModelCycleGAN(config, device)
    # prepare data
    data = DataCycleGAN(a_path, b_path, device)
    # generate fake
    model.generate(data)
    # output
    if grid:
        data.output_grid()
    else:
        data.output_each('A')
        data.output_each('B')
    return

if __name__ == "__main__":
    main()


import torch
import torch.nn as nn  # Gives access to the layers
import torch.nn.functional as F  # Gives access to activation functions
import torch.optim as optim  # Gives access to optimizers

class LinearClassifier(nn.Module):
    def __init__(self, lr, n_classes, input_dims):
        super(LinearClassifier, self).__init__()

        # Define NN Layers
        self.fc1 = nn.Linear(*input_dims, 128) # 3 layer NN with Linear activation function
        self.fc2 = nn.Linear(128, 256)
        self.fc3 = nn.Linear(256, n_classes)

        # Define optimizer and loss function
        self.optimizer = optim.Adam(self.parameters(), lr=lr)  # Optimizer Adam
        self.loss = nn.MSELoss()
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Detect if Cuda GPU is available
        self.to(self.device)  # Send NN to GPU

    def forward(self, data):  # Pytorch handles back propagation, but we need to define forward activation function
        layer1 = F.sigmoid(self.fc1(self.fc1(data)))  # Pass data to the first fully connected conned layer and activate with F.sigmoid
        layer2 = F.sigmoid(self.fc2(layer1))  # Get the output from layer 1, forward to layer 2 and activate, note the order
        layer3 = self.fc3(layer2)  # Not activating last layer, Loss function will handle this for us
        return layer3

    # Define learning loop
    def learn(self, data, labels):
        self.optimizer.zero_grad() # First thing to do at every learning function is to zero gradients between loops
        data = torch.tensor(data).to(self.device)  # Convert data to tensors on device assuming it was not done so
        labels = torch.tensor(data).to(self.device) # Same for labels

        predictions = self.forward(data)  # Get feed forward for data

        cost = self.loss(predictions, labels)  # Get cost associated with how far away are those predictions from the actual labels

        cost.backward()  # Back propagate the cost and optimizer function, this is where dqn learns
        self.optimizer.step()


import nibabel as nib
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image, ImageDraw

png_flag = False
file_list = ['GD.nii.gz', '2wGD.nii.gz']

for i, file in enumerate(file_list):
    if not os.path.exists(file):
        continue
    p = nib.load(file.data).dataobj
    name = str(i)
    print(name, p.shape)
    if png_flag:
        for j in range(p.shape[2]):
            print(i)
            plt.imshow(p[:,:,j])
            plt.savefig(name+'_'+str(j)+'.png')
            plt.close()
            plt.clf()
    else:
        images = []
        for i in range(p.shape[2]):
            im = Image.open(name+'_'+str(i)+'.png')
            images.append(im)
        images[0].save(name+'.gif', save_all=True, append_images=images[1:], optimize=False, duration=40, loop=0)