#!/usr/bin/env python3
from base import create_output_directory
from timetracker import TimeTracker
from base import get_institutions
from base import ask_continue
from shodanfunctions import *
from pathlib import Path
import argparse
import json
import sys
import os

os.chdir(sys.path[0])

parser = argparse.ArgumentParser()
parser.add_argument("input", help="the input file or directory of files to be converted")
parser.add_argument("-y", "--yes", "--assume-yes", help="Automatic yes to prompts; assume \"yes\" as answer to all "
                                                        "prompts and run non-interactively.", action="store_true")
parser.add_argument("-i", "--institutions", help="will add an institution field to every result based on given csv file in config.ini", action="store_true")
args = parser.parse_args()

print('---Shodan converter---')
t = TimeTracker()

institutions = None
if args.institutions:
    institutions = get_institutions()

# A file input
if Path(args.input).is_file():
    convert_file(args.input, 'shodan', institutions)

# A directory input
elif os.path.isdir(args.input):
    input_directory = args.input
    files_to_convert = []
    for file in os.listdir(input_directory):
        if file.endswith(".json"):
            files_to_convert.append(file)
    print('These files will be converted: ' + str(files_to_convert))
    print('Total number of files: ' + str(len(files_to_convert)))
    if not args.yes:
        ask_continue()
    output_directory = create_output_directory(input_directory)
    counter = 0
    for input_file in files_to_convert:
        counter += 1
        str_output_file = output_directory + '/' + input_file[:-5] + '-converted.json'
        print('\r' + 'Converting ' + input_file + '[' + str(counter) + '/' + str(len(files_to_convert)) + ']..', end='')
        with open(str_output_file, 'a') as output_file:
            for str_banner in open(input_directory + '/' + input_file, 'r'):
                if str_banner != '\n':
                    banner = dict_clean_empty(json.loads(str_banner))
                    shodan_to_es_convert(banner, institutions)
                    output_file.write(json.dumps(banner) + '\n')
    print('\nConverted files written in ' + output_directory)
else:
    msg = "{0} is not an existing file or directory".format(args.input)
    raise argparse.ArgumentTypeError(msg)
t.print_statistics()


#!/usr/bin/python3

# ^ ten dziwny komentarz na górze ma znaczenie w systemach UNIX-owych (np. Linux)
# on wskazuje systemowi operacyjnemu gdzie jest interpreter Pythona
# dzięki temu program można uruchamiać ./waluty5.py zamiast python waluty5.py
# a system traktuje ten plik jak program (taki "exec"), a nie plik tekstowy

import requests

def pobierz_dane(data=None):
    if data: # jeśli data jest podana i jest niepusta
        adres=f'http://api.nbp.pl/api/exchangerates/tables/A/{data}?format=json'
    else: # data nie podana - bierzemy najświeższe dane
        adres=f'http://api.nbp.pl/api/exchangerates/tables/A?format=json'

    dane = requests.get(adres).json()
    return dane[0]

def znajdz_walute(tabela, szukana_waluta):
    for waluta in tabela['rates']:
        if waluta['code'] == szukana_waluta:
            return waluta
    return None

def zajmij_sie_jedna_waluta(waluta):
    kod = waluta['code']
    nazwa = waluta['currency']
    kurs = waluta['mid']
    print(f'Waluta {nazwa} (kod {kod}) ma kurs {kurs}')
    kwota = float(input('Podaj kwotę: '))
    kwotaZlote = kwota * kurs
    print(f'{kwota:.2f} {kod} = {kwotaZlote:.2f} PLN')
    kwotaWaluta = kwota / kurs
    print(f'{kwota:.2f} PLN = {kwotaWaluta:.2f} {kod}')

def main():
    try:
        data = input('Podaj datę dla tabeli, lub pusty napis aby pobrać bieżące: ')
        tabela = pobierz_dane(data)
        print('Tabela numer', tabela['no'], 'z dnia', tabela['effectiveDate'])
        while True:
            szukana_waluta = input('Podaj kod waluty (pusty napis aby zakończyć): ').upper()
            if szukana_waluta == '': break
            waluta = znajdz_walute(tabela, szukana_waluta)
            if waluta is None:
                print('Nie ma takiej waluty')
            else:
                zajmij_sie_jedna_waluta(waluta)
    except Exception as e:
        print('Nie udało się pobrać', e)

# Ten if spowoduje, że main (lub cokolwiek pod ifem) wykona się tylko wtedy,
# gdy ten plik jest uruchamiany jako program
# (np. z konsoli: python waluty5.py)

# Ale if będzie nieprawdziwy (i main się nie wykona) jeśli ten plik będzie importowany jako moduł
if __name__ == '__main__':
    main()


# 문제 5.
# 리스트을 선언한 후 정수를 차례로 입력 받다가 -1이 입력되면 입력을 중단하고, -1을 제외한 마지막 세 개의 정수를 출력하는 프로그램을 작성하시오.
# 입력받은 정수가 -1을 제외하고 3개 미만일 경우에는 -1을 제외하고 입력받은 정수를 모두 출력한다.
# 입력 예: 30 20 10 60 80 -1
# 출력 예: 10 60 80

list = []
num = 1
while num != -1 :
    num = int(input("정수를 입력하세요."))
    if num > 0 :
        list.append(num)
print(list[len(list)-3::1])


#!/usr/bin/env python

from SBO import *

if __name__ == '__main__':
    sigma=[1]
    mu=[0]
    n1=1
    n2=1
    l=50
    a=[-3]
    b=[3]
    w=np.array([1])
    muW2=np.array([0])
    sigmaW2=np.array([1])
    M=1000000
    X=np.loadtxt("oldX.txt",ndmin=2)
    W=np.loadtxt("oldW.txt",ndmin=2)
    Y=np.loadtxt("oldY.txt",ndmin=2)
    T=np.loadtxt("hyperparameters.txt",ndmin=2)
    alpha1=T[0,:]
    alpha2=T[1,:]
    sigma0=T[2,:][0]
    muStart=T[3,:][0]
    alg=SBO(f,alpha1,alpha2,sigma0,sigma,mu,muStart,n1,n2,l,a,b,w,muW2,sigmaW2,M,old=True,Xold=X,Wold=W,yold=Y)


#!/usr/bin/python

import i3ipc
import json
import os
import pickle
import subprocess
import sys
from functools import reduce

def setup_displays(depth=0, outputs=("DP1", "HDMI2"), preferred_res="1920x1080"):
    if depth > 1: # Adding modes didn't help
        print("ERROR: Tried adding modes, but xrandr still failed")
        sys.exit(1)

    # Get available display outputs, and try turning on the given outputs that exist.
    modes = parse_xrandr()
    outputs_to_use = [output for output in outputs
                      if output in modes and modes[output]["exists"] and output != "eDP1"]

    if len(outputs_to_use) == 0:
        print("ERROR: Given displays not found.")
        sys.exit(1)

    # Build command to turn on displays.
    outputs_to_use.append("eDP1") # Primary output always comes first
    command = ["xrandr"]
    for i, output in enumerate(outputs_to_use[ : -1]):
        command.extend(
            ["--auto", "--output", output, "--left-of", outputs_to_use[i + 1], "--mode", \
             best_res(modes[output]["res"], preferred_res)])
        if output == "DP1":
            command.extend(["--rotate", "left"])

    with open(os.devnull, "w") as devnull: # Suppress command output
        proc = subprocess.run(command, stdout=devnull, stderr=devnull)

    # Handle xrandr failure.
    if proc.returncode == 1: # Maybe modes weren't present; add them and retry
        add_modes(modes, outputs_to_use, preferred_res)
        setup_displays(depth + 1, outputs, preferred_res)
    elif proc.returncode != 0:
        print("ERROR: xrandr exited with return code %i" % proc.returncode)
        sys.exit(1)

def parse_xrandr():
    xrandr_output = subprocess \
        .check_output("xrandr") \
        .decode("utf-8") \
        .split("\n")[1 : ] # Omit first line, which refers to the screen
    xrandr_output = [line for line in xrandr_output if len(line.strip()) > 0] # Remove empty lines

    outputs_info = {}

    i = 0
    while i < len(xrandr_output):
        line = xrandr_output[i]

        if not line.startswith((" ", "\t")):
            output, status = line.split()[ : 2]
            outputs_info[output] = {"exists": status != "disconnected", "res": []}

            i += 1
            while i < len(xrandr_output) and xrandr_output[i].startswith((" ", "\t")):
                line = xrandr_output[i]

                res = line.split()[0] # Extract resolution
                outputs_info[output]["res"].append(res)

                i += 1

    return outputs_info

def best_res(resolutions, preferred_res=None):
    if preferred_res is not None and preferred_res in resolutions:
        return preferred_res

    best_res, max_res_val = None, - float("inf")
    for res in resolutions:
        if "_" in res:
            res = res[ : res.index("_")]

        res_val = reduce(lambda x, y: x * y, (int(s) for s in res.split("x")))
        if res_val > max_res_val:
            best_res, max_res_val = res, res_val

    return best_res

def add_modes(modes, outputs, mode):
    missing_mode = [output for output in outputs if output in modes and mode not in modes[output]["res"]]

    # Add missing modes, using cvt to obtain the necessary Modeline
    horizontal, vertical = mode.split("x")
    modeline = subprocess \
        .check_output(["cvt", horizontal, vertical]). \
        decode("utf-8") \
        .split("\n")[1][len("Modeline ") :]

    if len(missing_mode) > 0:
        subprocess.run(["xrandr", "--newmode"] + modeline.split())

    for output in missing_mode:
        subprocess.run(["xrandr", "--addmode", output, "%s_60.00" % mode])

# Takes in a dict of mappings for which windows go to which output. Unlisted workspaces default
# to eDP1.
# xrandr output names:
#   eDP1: laptop screen
#   DP1: left monitor
#   DP2: right monitor
def move_workspaces(display_to_ws=None, cache_file=None):
    # Get currently focused workspace.
    i3 = i3ipc.Connection()
    focused_ws = None
    for ws in i3.get_workspaces():
        if ws["focused"]:
            focused_ws = ws["name"]
    assert focused_ws is not None

    assert display_to_ws is not None or cache_file is not None
    if display_to_ws is None:
        with open(cache_file, "rb") as f:
            ws_list = pickle.load(f)

        display_to_ws = {}
        for ws in ws_list:
            display_to_ws[ws["output"]] = ws["name"]

    with open(os.devnull, "w") as devnull:
        command = "i3 workspace {name} && i3 move workspace to output {display}"
        for display, ws_name in display_to_ws.items():
            subprocess.run(command.format(name=ws_name, display=display),
                           shell=True, stdout=devnull, stderr=devnull)

        # Switch focus back to original workspace
        subprocess.run(["i3", "workspace", focused_ws],
                       stdout=devnull, stderr=devnull)

def available_displays():
    return set(subprocess
        .check_output("xrandr | grep -w connected | cut -d \" \" -f 1", shell=True)
        .decode("utf-8")
        .splitlines())

def parse_arguments():
    if len(sys.argv) <= 1:
        return None

    available = available_displays()
    workspace_config = {}
    for i in range(1, len(sys.argv), 2):
        workspace, output = sys.argv[i : i + 2]
        try:
            workspace = int(workspace)
        except:
            print("Non-integer workspace index \"{}\" found".format(workspace))
            return None

        if output not in available:
            print("Invalid display \"{}\" found".format(output))
            return None

        if workspace in workspace_config:
            print("Duplicate workspace {} found".format(workspace))
            return None

        print("Putting workspace {} on display {}".format(workspace, output))
        workspace_config[workspace] = output

    return workspace_config

def main():
    setup_displays()

    cache_file = os.path.expanduser("~/.cache/piyush_i3_cache.pkl")
    workspace_config = parse_arguments()
    if workspace_config is not None:
        print("Moving workspaces according to command line options")
        move_workspaces(workspace_config=workspace_config)
    elif os.path.exists(cache_file):
        print("Restoring from cache file {}".format(cache_file))
        move_workspaces(cache_file=cache_file)

if __name__ == "__main__":
    main()


#!/usr/bin/env python2
import math
import numpy as np
import caffe

class SoftmaxWithLossLayer(caffe.Layer):
    def setup(self, bottom, top):
        if len(bottom) != 2:
            raise Exception("Need two inputs to compute loss.")
        if len(top) != 1 and len(top) != 2:
            raise Exception("Must output \"loss\" blob and optional output \"prob\" blob.")
        params = eval(self.param_str)
        self.softmax_axis = params.get("axis", 1)
        self.ignore_label = params.get("ignore_label", None)
        if "normalization" not in params and "normalize" in params:
            self.normalization = "VALID" if params["normalize"] else "BATCH_SIZE"
        else:
            self.normalization = params.get("normalization", "VALID")

    def reshape(self, bottom, top):
        check_fail = bottom[0].num != bottom[1].num
        if self.axis == 1:
            self.outer_num = bottom[0].num
            self.inner_num = bottom[0].height * bottom[0].width
            check_fail = check_fail or bottom[0].height != bottom[1].channels or bottom[0].width != bottom[1].height
        elif self.axis == 2:
            self.outer_num = bottom[0].num * bottom[0].channels
            self.inner_num = bottom[0].width
            check_fail = check_fail or bottom[0].channels != bottom[1].channels or bottom[0].width != bottom[1].height
        elif self.axis == 3:
            self.outer_num = bottom[0].count / bottom[0].width
            self.inner_num = 1
            check_fail = check_fail or bottom[0].channels != bottom[1].channels or bottom[0].height != bottom[1].height
        else:
            raise Exception("Softmax axis must in (1, 2, 3).")
        if check_fail:
            raise Exception("Number of labels must match number of predictions; \n"
                            "e.g., if softmax axis == 1 and prediction shape is (N, C, H, W), \n"
                            "label count (number of labels) must be N*H*W, \n"
                            "with integer values in {0, 1, ..., C-1}.")
        self.prob = np.zeros_like(bottom[0].data, dtype=np.float32)
        top[0].reshape(1)
        if len(top) == 2:
            top[1].reshape(bottom[0].data.shape)

    def forward(self, bottom, top):
        scale_data = np.exp(bottom[0].data - bottom[0].data.max(axis=self.softmax_axis, keepdims=True))
        self.prob[...] = scale_data / scale_data.sum(axis=self.softmax_axis, keepdims=True)
        prob_data = self.prob.ravel()
        label = bottom[1].data.ravel()
        dim = bottom[0].count / self.outer_num
        count = 0
        loss = 0
        for i in xrange(self.outer_num):
            for j in xrange(self.inner_num):
                label_value = int(label[i * self.inner_num + j])
                if self.ignore_label is None or label_value == self.ignore_label:
                    if label_value < 0 or label_value > self.prob.shape[self.softmax_axis]:
                        raise Exception("Label must be in range(prob.shape[softmax_axis]).")
                    loss -= math.log(max(prob_data[i * dim + label_value * self.inner_num + j], 1.175494351e-38))
                    count += 1
        if self.normalization == "FULL":
            normalizer = self.outer_num * self.inner_num
        elif self.normalization == "VALID":
            if count == -1:
                normalizer = self.outer_num * self.inner_num
            else:
                normalizer = count
        elif self.normalization == "BATCH_SIZE":
            normalizer = self.outer_num
        elif self.normalization == "NONE":
            normalizer = 1
        else:
            raise Exception("Unknown normalization mode: %s" % self.normalization)
        top[0].data[...] = loss / max(1.0, normalizer)
        if len(top) == 2:
            top[1].data[...] = self.prob

    def backward(self, top, propagate_down, bottom):
        if propagate_down[1]:
            raise Exception("SoftmaxWithLoss Layer cannot backpropagate to label inputs.")
        if propagate_down[0]:
            bottom[0].diff[...] = self.prob
            bottom_diff = bottom[0].diff.ravel()
            label = bottom[1].data.ravel()
            dim = bottom[0].count / self.outer_num
            count = 0
            for i in xrange(self.outer_num):
                for j in xrange(self.inner_num):
                    label_value = int(label[i * self.inner_num + j])
                    if self.ignore_label is not None and label_value == self.ignore_label:
                        for c in range(bottom[0].shape[self.softmax_axis]):
                            bottom_diff[i * dim + c * self.inner_num + j] = 0
                    else:
                        bottom_diff[i * dim + label_value * self.inner_num + j] -= 1
                        count += 1
            if self.normalization == "FULL":
                normalizer = self.outer_num * self.inner_num
            elif self.normalization == "VALID":
                if count == -1:
                    normalizer = self.outer_num * self.inner_num
                else:
                    normalizer = count
            elif self.normalization == "BATCH_SIZE":
                normalizer = self.outer_num
            elif self.normalization == "NONE":
                normalizer = 1
            else:
                raise Exception("Unknown normalization mode: %s" % self.normalization)
            bottom[0].diff *= top[0].diff / max(1.0, normalizer)


import contextlib

import torch.nn as nn

@contextlib.contextmanager
def set_mode(net: nn.Module, mode: str):
    """set the model's mode in a context manager"""
    before_train = net.training
    if mode == 'train':
        net.train()
    elif mode == 'eval':
        net.eval()
    else:
        raise NotImplementedError()

    yield net

    if before_train:
        net.train()
    else:
        net.eval()


### Author Name: Manqing Mao
### Luna Lander

import random
import gym
import numpy as np
from collections import deque
from keras.models import Sequential
from keras.layers import Dense
from keras import optimizers
from keras import regularizers
EPISODES = 5000

class DQNAgent:
    def __init__(self, state_size, action_size):

        # get size of state and action
        self.state_size = state_size
        self.action_size = action_size

        # These are hyper parameters for the DQN
        self.discount_factor = 0.99
        self.learning_rate = 0.001
        self.epsilon = 1.0
        self.epsilon_decay = 0.997
        self.epsilon_min = 0.101
        self.sample_size = 32
        self.train_start = 20000
        # create replay memory using deque
        self.memory = deque(maxlen=200000)
        self.train_buffer = deque(maxlen=10240)
        # create main model and target model
        self.model = self.build_model()
        self.target_model = self.build_model()

        # initialize target model
        self.update_target_model()

    # approximate Q function using Neural Network
    # state is input and Q Value of each action is output of network
    def build_model(self):
        model = Sequential()
        model.add(Dense(32, input_dim=self.state_size, activation='relu',kernel_initializer='he_uniform'))
        model.add(Dense(32, activation='relu',kernel_initializer='he_uniform'))
        model.add(Dense(32, activation='relu',kernel_initializer='he_uniform'))
        model.add(Dense(self.action_size, activation='linear',kernel_initializer='he_uniform'))
        model.summary()
        model.compile(loss='mse', optimizer=optimizers.Adam(lr=self.learning_rate))
        return model

    # after some time interval update the target model to be same with model
    def update_target_model(self):
        self.target_model.set_weights(self.model.get_weights())

    # get action from model using epsilon-greedy policy
    def get_action(self, state):
        if np.random.rand() <= self.epsilon:
            return random.randrange(self.action_size)
        else:
            q_value = self.model.predict(state)
            return np.argmax(q_value[0])

    # save sample <s,a,r,s'> to the replay memory
    def append_sample(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))

    # pick samples randomly from replay memory (with batch_size)
    def train_model(self):

        #do training if buffer is full
        if len(self.train_buffer) == self.train_buffer.maxlen:
            batch_size = self.train_buffer.maxlen
            states = np.zeros((batch_size, self.state_size))
            next_states = np.zeros((batch_size, self.state_size))
            action, reward, done = [], [], []

            for i in range(batch_size):
                states[i] = self.train_buffer[i][0]
                action.append(self.train_buffer[i][1])
                reward.append(self.train_buffer[i][2])
                next_states[i] = self.train_buffer[i][3]
                done.append(self.train_buffer[i][4])

            target = self.model.predict(states)
            target_next = self.model.predict(next_states)
            target_val = self.target_model.predict(next_states)

            for i in range(batch_size):
                # Q Learning: get maximum Q value at s' from target model
                if done[i]:
                    target[i][action[i]] = reward[i]
                else:
                    a = np.argmax(target_next[i])
                    target[i][action[i]] = reward[i] + self.discount_factor * (
                        target_val[i][a])

            self.model.fit(states, target, 100, epochs=1, verbose=0)
            self.train_buffer.clear()

env = gym.make('LunarLander-v2')
state_size = env.observation_space.shape[0]
action_size = env.action_space.n
agent = DQNAgent(state_size, action_size)
scores, episodes = [], []
action_count = 0

for e in range(EPISODES):
    done = False
    score = 0
    state = env.reset()
    state = np.reshape(state, [1, state_size])

    while not done:
        action = agent.get_action(state)
        next_state, reward, done, _ = env.step(action)
        next_state = np.reshape(next_state, [1, state_size])

        agent.append_sample(state, action, reward, next_state, done)

        if len(agent.memory) > agent.train_start:
            action_count += 1

            if action_count == 4:
                mini_batch = random.sample(agent.memory, agent.sample_size)
                agent.train_buffer.extend(mini_batch)
                action_count = 0

        agent.train_model()
        score += reward
        state = next_state

        if done:
            agent.update_target_model()
            if agent.epsilon > agent.epsilon_min:
                agent.epsilon *= agent.epsilon_decay
            scores.append(score)
            episodes.append(e)
            print("episode:", e, " score:", score, " epsilon:", agent.epsilon," memory size:", len(agent.memory),"avg last 100:",np.mean(scores[-100:]))

#test model
test_scores = []

for e in range(100):
    done = False
    score = 0
    state = env.reset()
    state = np.reshape(state, [1, state_size])

    while not done:
#         env.render()
        action = np.argmax(agent.model.predict(state)[0])
        next_state, reward, done, _ = env.step(action)
        next_state = np.reshape(next_state, [1, state_size])
        score += reward
        state = next_state

        if done:
            test_scores.append(score)
            print("test episode:", e, " score:", score)

res = []
for i in range(1,len(scores)):
    j = 0 if i < 100 else i-100
    res.append(np.mean(scores[j:i]))


# -*- coding: utf-8 -*-
# author: houzhiwei
# https://grass.osgeo.org/grass74/manuals/r.fill.stats.html

from scrapy import Spider, Request
from GrassModules.items import GrassModulesItem

class GrassSpider(Spider):
	name = 'grass'
	# allowed_domains = ['grass.osgeo.org']
	# start_urls = ['https://grass.osgeo.org/grass74/manuals/full_index.html']
	# base_url = 'https://grass.osgeo.org/grass74/manuals/'

	start_urls = ['file://127.0.0.1/H:/gisdemo/grass-7.5_html_manual/manuals/full_index.html']
	base_url = 'file://127.0.0.1/H:/gisdemo/grass-7.5_html_manual/manuals/'

	def parse(self, response):
		"""处理模块命令索引页"""
		# process requests
		tables = response.xpath("//div[@id='container']//table")
		# print(tables)
		for table in tables:
			trs = table.xpath("//tr")
			for i, tr in enumerate(trs):
				# if i > 1:
				# 	return
				href = tr.xpath("./td[1]/a/@href").extract_first()
				# meta: 将此处获取的数据传递给回调函数（callback）
				yield Request(url=self.base_url + href, callback=self.parse_detail)

	def parse_detail(self, response):
		"""处理模块命令详情页"""
		item = GrassModulesItem()
		item['manual_url'] = response.url
		content = response.css("#container")
		item['name'] = content.css("#name>b::text").extract_first()
		print('name %s' % item['name'])
		item['definition'] = content.xpath('./h2[1][contains(text(),"NAME")]/following-sibling::text()[2]').extract()
		item['keywords'] = self.parse_keywords(content)
		item['synopsis'] = self.parse_synopsis(content)
		item['parameters'] = self.parse_params(content)
		item['description'] = self.parse_des(content)
		item['notes'] = self.parse_notes(content)
		item['see_also'] = self.parse_see_also(content)
		item['authors'] = self.parse_authors(content)
		item['source_code'] = self.parse_source(content)
		return item

	def parse_keywords(self, selector):
		"""解析命令的关键词"""
		# 文本为KEYWORDS的h2之后和文本为SYNOPSIS的h2之前的a标签?
		a_path = './a[preceding-sibling::h2[contains(text(),"KEYWORDS")] and following-sibling::h2[contains(text(),"SYNOPSIS")]]'
		keywords_list = selector.xpath(a_path)
		keywords = []
		for keyword in keywords_list:
			word = keyword.xpath('normalize-space(.//text())').extract_first()
			keywords.append(word)
		# print('keywords %s' % keywords)
		return keywords

	def parse_synopsis(self, selector):
		# help = selector.xpath("./h2[text()='SYNOPSIS']/following::b[1]/text()").extract_first()
		synopsis = selector.css("#synopsis *::text").extract()
		# print('synopsis: %s' % ''.join(synopsis))
		return ''.join(synopsis).strip()

	def parse_params(self, selector):
		dl = selector.xpath('./div[@id="flags"]/dl')
		flags_dt = dl.xpath('//dt')
		parameters = []
		for i in range(0, len(flags_dt)):
			flag = flags_dt[i].xpath('./b/text()').extract_first()
			# print('flag %s' % dt)
			dds = flags_dt[i].xpath('./following-sibling::dd[following-sibling::dt[1]]')
			flag_exp = dds.xpath('.//text()').extract()
			parameters.append(
				{'parameter': str(flag).replace("-", ""), 'flag': flag, 'explanation': ' '.join(flag_exp),
				 'optional': True, })
		dts = selector.css("#parameters>dl>dt")
		input_file = False
		out_file = False
		optional = True
		data_type = 'String'
		for i in range(0, len(dts)):
			dt = dts[i]
			# key
			dt_id = dt.xpath('./b[1]/text()').extract_first().strip()
			if dt_id == 'input':
				input_file = True
			elif dt_id == 'output':
				out_file = True
			# value
			param_type = dt.xpath('string(./em//text())').extract()
			if param_type == 'name':
				data_type = 'String'
			elif param_type == 'value':
				data_type = 'Double'
			elif param_type == 'character':
				data_type = 'String'
			elif param_type == 'float':
				data_type = 'Float'
			# required
			require = dt.xpath('./b[2]/text()').extract_first()
			if require == '[required]':
				optional = False
			# 后面的所有非空节点
			dds_test = dt.xpath('./following-sibling::node()[not(string-length(text())=0)]')
			dd_list = []
			# 遍历所有节点，直到遇到 dt
			for i, item in enumerate(dds_test):
				if len(item.css('dt')) > 0:
					break
				dd_list.append(item)
			# print("dd_list %s" % dd_list)
			# dds = dt.xpath('./following-sibling::dd[following-sibling::dt[1] or not(following-sibling::dt)]')
			# print("dds %s" % dds)
			explanation = None
			alternatives = None
			default = None
			# length = len(dd_list)
			for j, item in enumerate(dd_list):
				if j == 0 and item:
					explanation = item.css('dd::text').extract_first().strip()
					# print("explanation %s" % explanation)
					continue
				alternatives = item.xpath('./em[parent::dd[contains(text(),"Options")]][1]/text()').extract_first()
				if not alternatives:
					alternatives = item.xpath(
						'./text()[parent::dd[contains(text(),"Special characters")]]').extract_first()
				if alternatives:
					alternatives = str(alternatives).replace('Special characters:', '')
					print("alternatives %s" % alternatives)
					continue
				default = item.xpath('./em[parent::dd[contains(text(),"Default")]][1]/text()').extract_first()
				if not default:
					continue
				print("default %s" % default)
			print("explanation %s" % explanation)
			parameters.append(
				{'parameter': dt_id, 'flag': dt_id, 'optional': optional, 'explanation': explanation,
				 'dataType': data_type, 'defaultValue': default, 'isInputFile': input_file, 'isOutputFile': out_file,
				 'alternatives': alternatives})
			print("parameters %s" % parameters)
		return parameters

	def parse_des(self, selector):
		desc_selector = selector.xpath(".//a[@name='description']/parent::h2")
		em = desc_selector.xpath("normalize-space(./following-sibling::em/text())").extract_first()
		# print('em %s' % em)
		des = desc_selector.xpath(
			"normalize-space(./following-sibling::em/following-sibling::text()[1])").extract_first()
		# print('des %s' % des)
		return str(em) + str(des)

	def parse_notes(self, selector):
		notes = selector.xpath(
			"normalize-space(.//text()[preceding-sibling::h2[child::a[@name='notes']] and following-sibling::h2[child::a[@name='see-also']]])").extract()
		# print('notes %s' % notes)
		return ''.join(notes)

	def parse_see_also(self, selector):
		also_selector = selector.xpath(".//a[@name='see-also']/parent::h2")
		alist = also_selector.xpath("./following-sibling::em//a")
		alsos = []
		for a in alist:
			also = a.xpath("normalize-space(.//text())").extract_first()
			alsos.append(also)
		# print('alsos %s' % alsos)
		return alsos

	def parse_authors(self, selector):
		authors_selector = selector.xpath(".//a[@name='authors' or @name='author']/parent::h2")
		authors = authors_selector.xpath("normalize-space(./following-sibling::text()[position()<3])").extract()
		# print('authors %s' % authors)
		return authors

	def parse_source(self, selector):
		source = selector.xpath(
			"./h2[contains(text(),'SOURCE CODE')]/following-sibling::p/a/@href").extract_first()
		return source


#!/usr/bin/env python
# ROS node libs
import rospy
from std_msgs.msg import String
from sensor_msgs.msg import Image,PointCloud2
from std_msgs.msg import Int16, Float32MultiArray
from multiprocessing import Queue, Pool
from cv_bridge import CvBridge, CvBridgeError

# General libs
import numpy as np
import os
import sys
import tensorflow as tf
import cv2
# import PIL
import time
import time
import datetime
# Detector libs
# from object_detection.utils import ops as utils_ops
# from object_detection.utils import label_map_util
# from object_detection.utils import visualization_utils as vis_util

# GPU settings: Select GPUs to use. Coment it to let the system decide
#os.environ["CUDA_VISIBLE_DEVICES"]="0"

class ros_tensorflow_obj():
    def __init__(self):
        # ## Initial msg
        rospy.loginfo('  ## Starting ROS Tensorflow interface ##')
        # ## Load a (frozen) Tensorflow model into memory.
        detection_graph = tf.Graph()
        with detection_graph.as_default():
            od_graph_def = tf.GraphDef()

            # with tf.gfile.GFile("./object_detection/saved_model_frcnn_4000_samples_00/frozen_inference_graph.pb", 'rb') as fid:
            # with tf.gfile.GFile("./object_detection/mayank_pb/frozen_inference_graph_100000.pb", 'rb') as fid:
            # with tf.gfile.GFile("./object_detection/mayank_pb/frozen_inference_graph_14134.pb", 'rb') as fid:
            with tf.gfile.GFile("./object_detection/mayank_pb/frozen_inference_graph_30812.pb", 'rb') as fid:
            # with tf.gfile.GFile("./object_detection/mayank_pb/frozen_inference_graph_new.pb", 'rb') as fid:
                serialized_graph = fid.read()
                od_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(od_graph_def, name='')

        # ## Loading label map
        # Label maps map indices to category names, so that when our convolution network predicts `5`, we know that this corresponds to `airplane`.  Here we use internal utility functions, but anything 			that returns a dictionary mapping integers to appropriate string labels would be fine
        # self.category_index = label_map_util.create_category_index_from_labelmap("./object_detection/Baidu_models/Baidu_ssd_model_0/haval_label_map.pbtxt", use_display_name=True)
        # self.category_index = label_map_util.create_category_index_from_labelmap("./object_detection/mayank_pb/bdd_traffic_label_map_simgle.pbtxt", use_display_name=True)
        # ## Get Tensors to run from model
        self.image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
        # Each box represents a part of the image where a particular object was detected.
        self.boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
        # Each score represent how level of confidence for each of the objects.
        # Score is shown on the result image, together with the class label.
        self.scores = detection_graph.get_tensor_by_name('detection_scores:0')
        self.classes = detection_graph.get_tensor_by_name('detection_classes:0')
        self.num_detections = detection_graph.get_tensor_by_name('num_detections:0')

        # # Tensorflow Session opening: Creates a session with log_device_placement set to True.
        # ## Session configuration
        config=tf.ConfigProto(log_device_placement=True)
        config.log_device_placement = True
        config.gpu_options.allow_growth = True

        # ## Session openning
        try:
            with detection_graph.as_default():
                self.sess = tf.Session(graph=detection_graph, config = config)
                rospy.loginfo('  ## Tensorflow session open: Starting inference... ##')
        except ValueError:
            rospy.logerr('   ## Error when openning session. Please restart the node ##')
            rospy.logerr(ValueError)

        # image_np= cv2.imread("baid_0.jpg",1)
        # image_np_expanded = np.expand_dims(image_np, axis=0)
        # (boxes_out, scores_out, classes_out, num_detections_out) = self.sess.run(
        # [self.boxes, self.scores, self.classes, self.num_detections], feed_dict={self.image_tensor: image_np_expanded})
#############################################33333
        # loading keras graph
        global mymodel
        mymodel = tf.keras.models.load_model("./saved_models/new_model/model_b_lr-4_ep150_ba32_1.h")
        # this is key : save the graph after loading the model
        global graph
        graph = tf.get_default_graph()
        # self.mymodel = tf.keras.models.load_model("./saved_models/new_model/model_b_lr-4_ep150_ba32.h")
        # print("My model is:", mymodel.summary())
        # self.light_co=[]
        #####################################################
        print("ready to process----------------------------------------------------------")
        # [print(n.name) for n in tf.get_default_graph().as_graph_def().node]
        # ROS environment setup
        # ##  Define subscribers
        self.subscribers_def()
        # print("subs")
        # ## Define publishers
        self.publishers_def()
        # print("pub")
        # ## Get cv_bridge: CvBridge is an object that converts between OpenCV Images and ROS Image messages
        self._cv_bridge = CvBridge()
        self.now = rospy.Time.now()

    # Define subscribers
    def subscribers_def(self):
        subs_topic = 'crop_tl_image'
        self._sub = rospy.Subscriber( subs_topic , Image, self.img_callback, queue_size=1, buff_size=2**24)
        # self._sub = rospy.Subscriber( subs_topic , Image, self.img_callback, queue_size=1, buff_size=100)

    # Define publishers
    def publishers_def(self):
        tl_bbox_topic = '/tl_bbox_topic_megs'
        self._pub = rospy.Publisher('tl_bbox_topic', Float32MultiArray, queue_size=1)

    # Camera image callback
    def img_callback(self, image_msg):
        image_msg.encoding = "bgr8"
        first = time.time()
        image_np = self._cv_bridge.imgmsg_to_cv2(image_msg, "bgr8")
        # image_np = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)

        rows, cols, channels = image_np.shape
        image_np_expanded = np.expand_dims(image_np, axis=0)
        second = time.time()
        time_cost = round((second-first)*1000)
        print("post processing Time",time_cost)
        third = time.time()
        (boxes_out, scores_out, classes_out, num_detections_out) = self.sess.run([self.boxes, self.scores, self.classes, self.num_detections],feed_dict={self.image_tensor: image_np_expanded})
        numboxes = np.squeeze(num_detections_out)
        # print("num_detection are",num_detections_out)
        numboxes = 1

        # print("confidence score are",scores_out[:, 0:8])
        tl_bbox = Float32MultiArray()
        if int(numboxes)>=1:
            tmp = -np.ones(7*int(numboxes)+1)

            for i in range(0,int(numboxes)):
                score = float(np.squeeze(scores_out)[i])
                bbox = [float(v) for v in np.squeeze(boxes_out)[i]]
                tmp[0] = numboxes
                if score > 0.3:
                    x_top_left = bbox[1] * cols
                    y_top_left = bbox[0] * rows
                    x_bottom_right = bbox[3] * cols
                    y_bottom_right = bbox[2] * rows
                    width = x_bottom_right - x_top_left
                    height = y_bottom_right - y_top_left
                    tmp[5*i+1] = x_top_left
                    tmp[5*i+2] = y_top_left
                    tmp[5*i+3] = width
                    tmp[5*i+4] = height
                    tmp[5*i+5] = score
                    # tmp[6] = 1
                    # tmp[7] = .999

                    #########################################################333
                    # Adding color model into the tensorflow model
                    # # traffic_light = ["red", "green", "black"]
                    traffic_light =  ["black", "red", "yellow", "green"]

                    # mymodel = tf.keras.models.load_model("./saved_models/new_model/model_b_lr-4_ep150_ba32.h")
                    # frame = img[int(ymin):int(ymax), int(xmin):int(xmax)]
                    frame = image_np[int(y_top_left):int(y_bottom_right), int(x_top_left):int(x_bottom_right)]
                    # introdusing the color model here
                    # test_image = frame.resize((32, 32))
                    test_image = cv2.resize(frame, (32, 32))
                    # test_image = image
                    test_image = tf.keras.preprocessing.image.img_to_array(test_image)
                    test_image = np.expand_dims(test_image, axis=0)
                    test_image /= 255
                    # print("My model is:" ,mymodel.summary())
                    # amodel=self.mymodel
                    with graph.as_default():
                        result = mymodel.predict(test_image)
                        # print("result is ", result)
                    # print("Prediction done")
                    # print(result)
                    #
                    tmp[6] = int(result.argmax())
                    tmp[7 * i + 7] =0.999
                    # tmp[7] = float(result[0, result.argmax()])

                    light_color = traffic_light[result.argmax()]
                    #
                    # light_color = light_color+"\n"
                    # selfight_co.append(light_color)
                    # with open('somefile4.txt', 'a') as the_file:
                    #     the_file.write(light_color)
                    # # '_______________________________________'

                    # cv2.imshow('images', frame)
                    # cv2.waitKey(1)
                    # cv2.destroyAllWindows()
                    # # cv2.waitKey(10)
                    # top = (int(x_top_left), int(y_top_left))
                    # bottom = (int(x_bottom_right), int(y_bottom_right))
                    # cv2.rectangle(image_np, pt1=top, pt2=bottom, color=(0, 255, 0), thickness=2)
                    # cv2.putText(image_np, light_color, (250, 90), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0),lineType=cv2.LINE_AA)
                    ################################################################
        else:
            tmp = [-1.0, 10.0, 10.0, 10.0, 10.0,10.0,10.0]

        # cv2.imshow("abcd", image_np)
        # cv2.waitKey(1)
        # output_path="./image5/"
        # ts = time.time()
        # st = datetime.datetime.fromtimestamp(ts).strftime('%f')
        # cut_frame = output_path
        # cut_frame = cut_frame + "image_" + str(st) + ".jpg"
        # print(cut_frame)
        # cv2.imwrite(cut_frame, image_np)

        tl_bbox.data = tmp
        thorth = time.time()
        time_cost = round((thorth-third)*1000)
        print("Inference Time",time_cost)
        print("bbox ",tl_bbox.data)
        self._pub.publish(tl_bbox)
# Spin once
def spin(self):
    rospy.spin()

def main():
    rospy.init_node('tf_object_detector', anonymous=True)
    tf_ob = ros_tensorflow_obj()
    try:
        rospy.spin()
    except KeyboardInterrupt:
        print("Shutting down")
if __name__ == '__main__':
    main()


import json
import logging

from aiohttp.web import Request, Response

import dataclasses
import ray
import aiohttp.web
import ray.dashboard.optional_utils as optional_utils
import ray.dashboard.utils as dashboard_utils
from ray.dashboard.modules.version import (
    CURRENT_VERSION,
    VersionResponse,
)

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

routes = optional_utils.ClassMethodRouteTable

# NOTE (shrekris-anyscale): This class uses delayed imports for all
# Ray Serve-related modules. That way, users can use the Ray dashboard agent for
# non-Serve purposes without downloading Serve dependencies.
class ServeAgent(dashboard_utils.DashboardAgentModule):
    def __init__(self, dashboard_agent):
        super().__init__(dashboard_agent)

    # TODO: It's better to use `/api/version`.
    # It requires a refactor of ClassMethodRouteTable to differentiate the server.
    @routes.get("/api/ray/version")
    async def get_version(self, req: Request) -> Response:
        # NOTE(edoakes): CURRENT_VERSION should be bumped and checked on the
        # client when we have backwards-incompatible changes.
        resp = VersionResponse(
            version=CURRENT_VERSION,
            ray_version=ray.__version__,
            ray_commit=ray.__commit__,
        )
        return Response(
            text=json.dumps(dataclasses.asdict(resp)),
            content_type="application/json",
            status=aiohttp.web.HTTPOk.status_code,
        )

    @routes.get("/api/serve/deployments/")
    @optional_utils.init_ray_and_catch_exceptions()
    async def get_all_deployments(self, req: Request) -> Response:
        from ray.serve.schema import ServeApplicationSchema

        client = self.get_serve_client()

        if client is None:
            config = ServeApplicationSchema.get_empty_schema_dict()
        else:
            config = client.get_app_config()

        return Response(
            text=json.dumps(config),
            content_type="application/json",
        )

    @routes.get("/api/serve/deployments/status")
    @optional_utils.init_ray_and_catch_exceptions()
    async def get_all_deployment_statuses(self, req: Request) -> Response:
        from ray.serve.schema import serve_status_to_schema, ServeStatusSchema

        client = self.get_serve_client()

        if client is None:
            status_json = ServeStatusSchema.get_empty_schema_dict()
            status_json_str = json.dumps(status_json)
        else:
            status = client.get_serve_status()
            status_json_str = serve_status_to_schema(status).json()

        return Response(
            text=status_json_str,
            content_type="application/json",
        )

    @routes.delete("/api/serve/deployments/")
    @optional_utils.init_ray_and_catch_exceptions()
    async def delete_serve_application(self, req: Request) -> Response:
        from ray import serve

        if self.get_serve_client() is not None:
            serve.shutdown()

        return Response()

    @routes.put("/api/serve/deployments/")
    @optional_utils.init_ray_and_catch_exceptions()
    async def put_all_deployments(self, req: Request) -> Response:
        from ray.serve.schema import ServeApplicationSchema
        from ray.serve._private.api import serve_start

        config = ServeApplicationSchema.parse_obj(await req.json())

        client = serve_start(
            detached=True,
            http_options={
                "host": config.host,
                "port": config.port,
                "location": "EveryNode",
            },
        )

        if client.http_config.host != config.host:
            return Response(
                status=400,
                text=(
                    "Serve is already running on this Ray cluster. Its "
                    f'HTTP host is set to "{client.http_config.host}". '
                    f'However, the requested host is "{config.host}". '
                    f"The requested host must match the running Serve "
                    "application's host. To change the Serve application "
                    "host, shut down Serve on this Ray cluster using the "
                    "`serve shutdown` CLI command or by sending a DELETE "
                    "request to this Ray cluster's "
                    '"/api/serve/deployments/" endpoint. CAUTION: shutting '
                    "down Serve will also shut down all Serve deployments."
                ),
            )

        if client.http_config.port != config.port:
            return Response(
                status=400,
                text=(
                    "Serve is already running on this Ray cluster. Its "
                    f'HTTP port is set to "{client.http_config.port}". '
                    f'However, the requested port is "{config.port}". '
                    f"The requested port must match the running Serve "
                    "application's port. To change the Serve application "
                    "port, shut down Serve on this Ray cluster using the "
                    "`serve shutdown` CLI command or by sending a DELETE "
                    "request to this Ray cluster's "
                    '"/api/serve/deployments/" endpoint. CAUTION: shutting '
                    "down Serve will also shut down all Serve deployments."
                ),
            )

        client.deploy_app(config)

        return Response()

    def get_serve_client(self):
        """Gets the ServeControllerClient to the this cluster's Serve app.

        return: If Serve is running on this Ray cluster, returns a client to
            the Serve controller. If Serve is not running, returns None.
        """

        from ray.serve.context import get_global_client
        from ray.serve.exceptions import RayServeException

        try:
            return get_global_client(_health_check_controller=True)
        except RayServeException:
            logger.debug("There's no Serve app running on this Ray cluster.")
            return None

    async def run(self, server):
        pass

    @staticmethod
    def is_minimal_module():
        return False


#from src.utilities.craft_pytorch.detect import detect_text_per_file as detect_text_per_page
#from src.services.ocr import get_text

import cv2
import config
#import imutils
import pandas as pd
import numpy as np

# Version 1.0 delivery in this week!

class Orientation:

    def __init__(self, image_path, get_text, conf_threshold=50, lang='eng'):

        self.image_path = image_path
        self.image = cv2.imread(image_path)
        self.get_text = get_text

        self.conf_threshold = int(conf_threshold)

        self.timer = {'net': 0, 'restore': 0, 'nms': 0}
        self.text = {}
        self.lang = lang

        # self.re_orient()

    def augment_df(self, line_list):
        dic = []
        for i, box in enumerate(line_list):
            # print(box)
            vertices = box["boundingBox"]["vertices"]
            dic.append({'x1': vertices[0]['x'], 'y1': vertices[0]['y'], 'x2': vertices[1]['x'], 'y2': vertices[1]
                       ['y'], 'x3': vertices[2]['x'], 'y3': vertices[2]['y'], 'x4': vertices[3]['x'], 'y4': vertices[3]['y']})

        df = pd.DataFrame(
            dic, columns=['x1', 'y1', 'x2', 'y2', 'x3', 'y3', 'x4', 'y4'])
        df['height'] = df['y4'] - df['y1']
        df['width'] = df['x2'] - df['x1']
        df['ymid'] = (df['y4'] + df['y3']) * 0.5
        df['area'] = df['width'] * df['height']
        df = df.sort_values(by=['ymid'])
        df['group'] = None
        df['line_change'] = 0
        return df

    def dump_out(self, bbc, rot):
        im = self.image.copy()
        for box in bbc:
            # print(box)
            cv2.polylines(im, [np.array(box).astype(np.int32).reshape((-1, 1, 2))], True, color=(255, 255, 0),
                          thickness=1)
        cv2.imwrite('tmp/' + str(rot) + '.png', im)

    def get_rotaion_angle(self, text_cor_df):

        bbox_df = text_cor_df.copy()
        bbox_df = self.augment_df(bbox_df)

        bbox_df['delta_x'] = bbox_df['x2'] - bbox_df['x1']
        bbox_df['delta_y'] = bbox_df['y2'] - bbox_df['y1']
        box_dir = [bbox_df['delta_x'].mean(), bbox_df['delta_y'].mean()]
        # print(box_dir)
        x_axis = [1, 0]
        try:
            cosine = np.dot(box_dir, x_axis) / \
                (np.linalg.norm(box_dir) * np.linalg.norm(x_axis))
        except Exception as e:
            print('ERROR in finding angle of rotaion!!!')
            print(e)
            cosine = 1
        angle = np.arccos(cosine) * 180 / np.pi
        avrage_height = bbox_df['height'].mean()
        avrage_width = bbox_df['width'].mean()
        # if avrage_height > avrage_width:
        #    angle = 90 - angle

        return angle * np.sign(box_dir[1])

    def rotate_bound(self, image, angle):
        # grab the dimensions of the image and then determine the
        # center
        (h, w) = image.shape[:2]
        (cX, cY) = (w / 2, h / 2)

        # grab the rotation matrix (applying the negative of the
        # angle to rotate clockwise), then grab the sine and cosine
        # (i.e., the rotation components of the matrix)
        M = cv2.getRotationMatrix2D((cX, cY), -angle, 1.0)
        cos = np.abs(M[0, 0])
        sin = np.abs(M[0, 1])

        # compute the new bounding dimensions of the image
        nW = int((h * sin) + (w * cos))
        nH = int((h * cos) + (w * sin))

        # adjust the rotation matrix to take into account translation
        M[0, 2] += (nW / 2) - cX
        M[1, 2] += (nH / 2) - cY

        # perform the actual rotation and return the image

        # return cv2.warpAffine(image, M, (nW, nH),flags=cv2.INTER_LANCZOS4)
        # return cv2.warpAffine(image, M, (nW, nH),flags=cv2.INTER_LANCZOS4, borderMode=cv2.BORDER_REPLICATE)
        return cv2.warpAffine(image, M, (nW, nH), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)

    def re_orient(self, page_dict, font_info):

        page_dict, lines = self.get_text(self.image_path, page_dict, font_info)
        angle = self.get_rotaion_angle(lines)
        print("Angle of tilt detected {} ".format(angle))
        if abs(angle) >= 2.5:
            self.image = self.rotate_bound(self.image, -angle)
            cv2.imwrite(self.image_path, self.image)
            page_dict, lines = self.get_text(
                self.image_path, page_dict, font_info)
            angle = self.get_rotaion_angle(lines)
            print("Angle of tilt after correction {} ".format(angle))

        return page_dict

    #
    # def re_orient(self):
    #     lang = 'hi'
    #     lines = detect_text_per_page([self.image],\
    #                                  network=True, \
    #                                  text_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['text_threshold'], \
    #                                  low_text_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['low_text'],
    #                                  link_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['link_threshold'])[0]
    #
    #     angle = self.get_rotaion_angle(lines)
    #     print("Angle of tilt detected {} ".format(angle) )
    #     rotations = 1
    #     # self.dump_out(lines,rotations)
    #     # Orientation correction
    #     if config.ALIGN_MODE == 'FAST':
    #      tolerance = 0.1
    #     if config.ALIGN_MODE == 'ACCURATE':
    #         tolerance = 0.1
    #
    #     while (abs(angle) > tolerance) and rotations < 3:
    #         self.image = self.rotate_bound(self.image, -angle)
    #
    #         if rotations > 1:
    #             # Remove rotaion artifacts
    #             contours = cv2.findContours(cv2.cvtColor(self.image, cv2.COLOR_BGR2GRAY), cv2.RETR_EXTERNAL,
    #                                         cv2.CHAIN_APPROX_SIMPLE)
    #             contours = contours[0] if len(contours) == 2 else contours[1]
    #             if len(contours) > 0:
    #                 x, y, w, h = cv2.boundingRect(contours[0])
    #                 # print('cropped area reduced ')
    #                 self.image = self.image[y:y + h, x:x + w, :]
    #
    #         lines = detect_text_per_page([self.image], \
    #                                      network=True, \
    #                                      text_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['text_threshold'], \
    #                                      low_text_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['low_text'],
    #                                      link_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['link_threshold'])[0]
    #         angle = self.get_rotaion_angle(lines)
    #         print("Angle of tilt after correction {} ".format(angle))
    #         rotations += 1
    #         # self.dump_out(east_cor,rotations)
    #     cv2.imwrite(self.image_path,self.image)
    #     words = detect_text_per_page([self.image], \
    #                                      network=False, \
    #                                      text_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['text_threshold'], \
    #                                      low_text_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['low_text'],
    #                                      link_threshold=config.LANGUAGE_LINE_THRESOLDS[lang]['link_threshold'])[0]
    #     # if config.ALIGN_MODE not "FAST":
    #     #     upside_down = self.check_orientation(bbox1.gr_cordinates)
    #
    #
    #     return words, lines
    #

  # def check_orientation(self, group_cordinates, margin=5):
    #     upside_down = False
    #     orientation = []
    #     for index, block in enumerate(group_cordinates):
    #         crop = self.image[block[0][1] - margin: block[1][1] + margin,
    #                block[0][0] - margin: block[1][0] + margin]
    #         try:
    #             osd = pytesseract.image_to_osd(crop)
    #             angle = osd.split('\nRotate')[0].split(': ')[-1]
    #             orientation.append(int(angle))
    #         except:
    #             pass
    #     orientation = np.array(orientation)
    #     chk_orientation = orientation > 170
    #
    #     # Taking vote of regions
    #     if chk_orientation.sum() > (len(orientation) * 0.5):
    #         # print ('Image is upside down')
    #         upside_down = True
    #         return upside_down
    #
    #     return upside_down
    #
    #


from django.test import TestCase
from django.urls import reverse
from polls.tests import helpers

class TestQuestionDetailView(TestCase):
    def test_future_question(self):
        future_question = helpers.create_question(question_text='Future question.', days=5)
        url = reverse('polls:detail', args=(future_question.id,))
        response = self.client.get(url)
        self.assertEqual(response.status_code, 404)

    def test_past_question(self):
        past_question = helpers.create_question(question_text='Past Question.', days=-5)
        url = reverse('polls:detail', args=(past_question.id,))
        response = self.client.get(url)
        self.assertContains(response, past_question.question_text)


'''
update-ratestat-tags-from-vote-playlists.py

This script uses the votes playlists (1-10) to add the new votes to and update the ratestat tags of
audio files in the music library.

'''

# import external Python modules
from prettytable import PrettyTable

# Do setup processing so that this script can import all the needed modules from the "mlu" package.
# This is necessary because these scripts are not located in the root directory of the project, but
# instead in the 'scripts' folder.
import envsetup
envsetup.PreparePythonProjectEnvironment()

from mlu.settings import MLUSettings

from com.nwrobel import mypycommons
import com.nwrobel.mypycommons.logger
import com.nwrobel.mypycommons.file
import com.nwrobel.mypycommons.time

mypycommons.logger.initSharedLogger(logDir=MLUSettings.logDir)
logger = mypycommons.logger.getSharedLogger()

# import project-related modules
import mlu.tags.backup
import mlu.ratestats

# --------------------------------------- Helper functions -----------------------------------------
#
def _getRatestatTagUpdatesSummaryFilepath():
    '''
    Returns the filepath for the ratestat tags updates summary log file.
    '''
    backupFilename = "[{}] update-ratestat-tags-from-vote-playlists_tag-updates-summary.txt".format(
        mypycommons.time.getCurrentTimestampForFilename()
    )
    filepath = mypycommons.file.JoinPaths(MLUSettings.logDir, backupFilename)

    return filepath

def _writeRatestatTagUpdatesSummaryFile(audioFileVoteDataList, summaryFilepath):
    '''
    Writes out a log file containing a table in pretty format with the ratestat tags updates.

    Params:
        audioFileVoteDataList: list of AudioFileVoteData objects, the new votes data
        summaryFilepath: path of the log output file to be written
    '''
    tagUpdatesTable = PrettyTable()
    tagUpdatesTable.field_names = ["Title", "Artist", "Votes Added", "New Rating", "New Votes List"]
    tagUpdatesTable.align["Title"] = "l"
    tagUpdatesTable.align["Artist"] = "l"
    tagUpdatesTable.align["Votes Added"] = "r"
    tagUpdatesTable.align["New Rating"] = "r"
    tagUpdatesTable.align["New Votes List"] = "r"

    for audioFileVotesData in audioFileVoteDataList:
        tagHandler = mlu.tags.io.AudioFileTagIOHandler(audioFileVotesData.filepath)
        currentTags = tagHandler.getTags()

        votesAdded = mlu.tags.common.formatValuesListToAudioTag(audioFileVotesData.votes)

        tagUpdatesTable.add_row([
            currentTags.title,
            currentTags.artist,
            votesAdded,
            currentTags.rating,
            currentTags.votes
        ])

    mypycommons.file.writeToFile(filepath=summaryFilepath, content=tagUpdatesTable.get_string())

# ------------------------------- Main script procedure --------------------------------------------
#
if __name__ == "__main__":

    logger.info("Performing full backup (checkpoint) of all music library audio files tags")
    tagsBackupFilepath = mlu.tags.backup.backupMusicLibraryAudioTags()

    logger.info("Copying vote playlist files to temp location in ~cache")
    mlu.ratestats.copyVotePlaylistsToTemp(votePlaylistsSourceDir=MLUSettings.votePlaylistsDir, votePlaylistsTempDir=MLUSettings.tempDir)

    logger.info("Loading audio file votes data from all vote playlists")
    audioFileVoteDataList = mlu.ratestats.getAudioFileVoteDataFromRatePlaylists(votePlaylistsDir=MLUSettings.tempDir)

    logger.info("Vote playlists data loaded successfully")
    logger.info("Writing new ratestats tag data to audio files")

    erroredAudioFilepaths = []
    for audioFileVoteData in audioFileVoteDataList:
        try:
            mlu.ratestats.updateRatestatTagsFromVoteData(audioFileVoteData)
        except:
            logger.exception("updateRatestatTagsFromVoteData operation failed: File='{}'".format(audioFileVoteData.filepath))
            erroredAudioFilepaths.append(audioFileVoteData.filepath)

    logger.info("Votes processing complete")
    logger.info("{} audio files were processed".format(len(audioFileVoteDataList)))
    logger.info("{} audio files failed update".format(len(erroredAudioFilepaths)))

    if (not erroredAudioFilepaths):
        logger.info("Process completed successfully: all ratestat tags updated with new votes")
        logger.info("Writing ratestat tag updates summary file")

        summaryFilepath = _getRatestatTagUpdatesSummaryFilepath()
        _writeRatestatTagUpdatesSummaryFile(audioFileVoteDataList, summaryFilepath)
        logger.info("Summary file written successfully: File='{}'".format(summaryFilepath))

        logger.info('Archiving the old vote playlists which were just processed')
        mlu.ratestats.archiveVotePlaylists(playlistsDir=MLUSettings.tempDir, archiveDir=MLUSettings.votePlaylistsArchiveDir)

        logger.info('Emptying already counted votes from vote playlists')
        mlu.ratestats.resetVotePlaylists(votePlaylistsSourceDir=MLUSettings.votePlaylistsDir, votePlaylistsTempDir=MLUSettings.tempDir)

    else:
        erroredAudioFilepathsFmt = "\n".join(erroredAudioFilepaths)
        logger.info("Failed to update ratestat tag values with new votes for the following files:\n{}".format(erroredAudioFilepathsFmt))

        logger.info("Process completed with failures: undoing all tag changes to the music library (reverting to checkpoint)")
        mlu.tags.backup.restoreMusicLibraryAudioTagsFromBackup(tagsBackupFilepath)

        logger.info("Tags backup restored successfully: all changes were undone - run this script again to retry")

    mypycommons.file.DeleteDirectory(MLUSettings.tempDir)
    logger.info('Script complete')


class TaskSupport(object):
    mixin = 'RecurringTask ScheduledTask SubscribedTask'

    @staticmethod
    def prepare_http_task(task):
        task['type'] = 'http-request'
        for key in ('mimetype', 'data', 'headers', 'timeout'):
            if key in task and not task[key]:
                del task[key]
        return task

    @classmethod
    def queue_http_task(cls, tag, task, **params):
        params.update(tag=tag, task=cls.prepare_http_task(task))
        return cls.create(**params)

    def set_http_task(self, task):
        self.task = self.prepare_http_task(task)
        return self


# ========================================================================
# Copyright 2020 Emory University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========================================================================

from pathlib import Path
from typing import List, Tuple, Dict

import requests

def download_data(path):
    def download(remote_addr: str, local_addr: str):
        r = requests.get(remote_addr)
        with open(local_addr, 'wb') as fin:
            fin.write(r.content)

    url = 'https://raw.githubusercontent.com/emory-courses/cs329/master/dat/pos/wsj-pos.{}.gold.tsv'

    remote = url.format('trn')
    download(remote, path / Path(remote).name)

    remote = url.format('dev')
    download(remote, path / Path(remote).name)

def read_data(filename: str):
    data, sentence = [], []
    fin = open(filename)

    for line in fin:
        l = line.split()
        if l:
            sentence.append((l[0], l[1]))
        else:
            data.append(sentence)
            sentence = []

    return data

def word_count(data: List[List[Tuple[str, str]]]) -> int:
    """
    :param data: a list of tuple list where each inner list represents a sentence and every tuple is a (word, pos) pair.
    :return: the total number of words in the data
    """
    return sum([len(sentence) for sentence in data])

def create_uni_pos_dict(data: List[List[Tuple[str, str]]]) -> Dict[str, List[Tuple[str, float]]]:
    """
    :param data: a list of tuple list where each inner list represents a sentence and every tuple is a (word, pos) pair.
    :return: a dictionary where the key is a word and the value is the list of possible POS tags with probabilities in descending order.
    """
    model = dict()
    # To be updated
    return model

if __name__ == '__main__':
    # create a directory to download pos data
    path = Path.cwd()

    while path.name != 'cs329':
        path = path.parent

    path /= 'dat/pos'
    path.mkdir(parents=True, exist_ok=True)

    # download data
    download_data(path)

    # read data
    trn_data = read_data(path / 'wsj-pos.trn.gold.tsv')
    print(len(trn_data))
    print(trn_data[0])

    # word count
    print(word_count(trn_data))


from code.classes import net
from code.classes import node
from code.classes import grid
from code.algorithms import astar as ast
from code.function import check_constraints
import copy
import numpy as np
import random

def acceptance_probability(cost, new_cost, temperature):
    """
    Calculate probability of accepting new cost
    """
    if new_cost < cost:
        return 1
    else:
        p = np.exp(- (new_cost - cost) / temperature)
        return p

class SimulatedAnnealing():
    """
    Simulated annealing is used to optimize the chip
    """
    def __init__(self, grid_file):
        self.grid_file = copy.deepcopy(grid_file)

    def run(self, cost, iterations):
        """
        Deletes a few lines and places them again using A*
        """
        max_iterations = iterations
        start_temp = 1000
        amount_of_redirects = 3

        all_cost = []
        lowest_cost = []
        all_cost.append(cost)
        lowest_cost.append(cost)

        # Run simulated annealing for every iteration
        for iteration in range(max_iterations):
            backup_grid = copy.deepcopy(self.grid_file)
            counter = 0

            # Progression notifier
            if iteration % 10 == 0:
                print("Nu bij iteratie:", iteration)

            # Get all routes and amount of crosses
            routes = self.grid_file.get_list_of_routes()
            amount_of_crosses = self.grid_file.get_list_of_crosses()
            redirect = {}

            # Remove x amount of routes
            for i in range(amount_of_redirects):
                if counter % 5 == 0:
                    redirect_route = max(amount_of_crosses, key=lambda key: amount_of_crosses[key])
                    route = routes[redirect_route]
                else:
                    redirect_route, route = random.choice(list(routes.items()))

                redirect_route = int(redirect_route)
                start = route[0].get_coordinates_from()
                end = route[-1].get_coordinates_to()

                redirect[redirect_route] = [start, end]
                self.grid_file.remove_route(redirect_route)
                self.grid_file.remove_crosses(redirect_route)

                counter += 1

            self.grid_file.empty_netlists()

            # Formulate the removed routes into a netlist for Astar
            for item in redirect:
                start = redirect[item][0]
                end = redirect[item][1]

                start_x = int(start[0])
                start_y = int(start[1])

                end_x = int(end[0])
                end_y = int(end[1])

                gate_a = self.grid_file.get_current_gate_number(start_x, start_y)
                gate_b = self.grid_file.get_current_gate_number(end_x, end_y)

                self.grid_file.netlists.append((gate_a, gate_b))

            astar = ast.Astar(self.grid_file)
            reset = astar.run()

            # Check if Astar was successful, if yes then calculate probability to accept new state
            # If no then return to state before iteration
            if reset:
                new_cost = self.grid_file.cost_of_route()
                all_cost.append(new_cost)

                current_temp = start_temp - (start_temp/max_iterations) * iteration
                probability = acceptance_probability(cost, new_cost, current_temp)

                # Calculates if new state should be accepted
                if random.uniform(0, 1) < probability:
                    cost = new_cost
                else:
                    self.grid_file = backup_grid
            else:
                self.grid_file = backup_grid

            lowest_cost.append(cost)

        return cost, all_cost, lowest_cost, self.grid_file


from augmentation import *
from data_utils import prepare_data
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_fscore_support, confusion_matrix
from sklearn.utils import shuffle
import pandas as pd
import numpy as np

AUGMENT = 'NO'

"""## Helper Methods"""

def scale_datasets(scaler, train_set, to_scale):
  scaler = scaler().fit(train_set)
  res = []
  for dataset in to_scale:
    res.append(scaler.transform(dataset))
  return res

def display_results(precision_recall_fscore_support):
  disp_df = pd.DataFrame(data=precision_recall_fscore_support, columns=['Pos Class'], index=['Precision', 'Recall', 'F-score', 'support'])
  print(disp_df)

X, y = prepare_data()

"""### Train-Test-Validation Split"""
SEED = 0
DATA_SPLIT_PCT = 0.20

X_train, X_test, y_train, y_test = train_test_split(np.array(X), np.array(y), test_size=DATA_SPLIT_PCT, random_state=SEED)

classifier = LogisticRegression(class_weight='balanced', max_iter = 10000, penalty = 'l1', solver = 'saga', C = 0.01, verbose=1)

"""## No Augmentation"""
train_x_scaled, test_x_scaled = scale_datasets(StandardScaler, X_train, [X_train, X_test])

if AUGMENT == 'NO':
    classifier.fit(train_x_scaled, y_train)
    y_hat_train = classifier.predict(train_x_scaled)
    train_res = precision_recall_fscore_support(y_train, y_hat_train)#, average='binary')

elif AUGMENT == 'GRAD':
    augmented_train_x, augmented_y_train = gradient_augment(train_x_scaled, y_train)
    augmented_train_x_rescaled, augmented_y_train = shuffle(augmented_train_x, augmented_y_train)

    classifier.fit(augmented_train_x_rescaled, augmented_y_train)
    y_hat_train = classifier.predict(augmented_train_x_rescaled)
    train_res = precision_recall_fscore_support(augmented_y_train, y_hat_train, average='binary')
    test_x_scaled, _ = gradient_augment(test_x_scaled, y_test)

elif AUGMENT == 'GRADRESAMP':
    augmented_train_x, augmented_y_train = gradient_augment(train_x_scaled, y_train)
    augmented_train_x, augmented_y_train = resample_augment(augmented_train_x, augmented_y_train)
    augmented_train_x, augmented_y_train = shuffle(augmented_train_x, augmented_y_train)
    augmented_train_x_rescaled = scale_datasets(StandardScaler, augmented_train_x, [augmented_train_x])[0]
    classifier.fit(augmented_train_x_rescaled, augmented_y_train)
    y_hat_train = classifier.predict(augmented_train_x_rescaled)
    train_res = precision_recall_fscore_support(augmented_y_train, y_hat_train, average='binary')
    test_x_scaled, _ = gradient_augment(test_x_scaled, y_test)

else:
    augmented_train_x, augmented_y_train = resample_augment(df_train_x, y_train)

    augmented_train_x_rescaled = scale_datasets(StandardScaler, augmented_train_x, [augmented_train_x])[0]

    augmented_train_x_rescaled, test_x_scaled = scale_datasets(StandardScaler, augmented_train_x, [augmented_train_x, df_test_x])
    classifier.fit(augmented_train_x_rescaled, augmented_y_train)
    y_hat_train = classifier.predict(augmented_train_x_rescaled)
    train_res = precision_recall_fscore_support(augmented_y_train, y_hat_train, average='binary')

#display_results(train_res)
print(train_res)

y_hat_test = classifier.predict(test_x_scaled)
test_res = precision_recall_fscore_support(y_test, y_hat_test)#, average='binary')
#display_results(test_res)
print(test_res)
test_res = confusion_matrix(y_test, y_hat_test, labels=[1, 0])
print(test_res)


#boost:
#Don't forget to add -pthread to your linker options when linking your code.
#You have built the Boost.Python library. You have to add the following
#options when building your own code:
#Compiler options:
#   -I/usr/local/include/python2.4 -I/usr/local/include
#Linker options:
#   -L/usr/local/lib/python2.4 -L/usr/local/lib -lboost_python -lpython2.4

env = Environment(CC = 'g++', CCFLAGS = '-g -O2 -W -Wall -ansi -Wwrite-strings -pedantic')
env.Replace(CPPPATH = ['.', '../pyx12', '/usr/include', '/usr/local/include', \
    '/usr/local/include/python2.4'])
env.Replace(LIBS = ['pyx12', 'boost_unit_test_framework', 'boost_python', 'python2.4']) #, 'libc_r'])
env.Replace(LIBPATH = ['.', '../pyx12', '/usr/lib', '/usr/local/lib', '/usr/local/lib/python2.4'])

env.Program('t1', 't1.cxx', LINKFLAGS='-pthread')
#env.Program('test_csegment', 'test_csegment.cxx') #, LINKFLAGS='-pthread')


#!/usr/bin/python

import os

environment_variables_keys = [
    "IS_PRODUCT_BUILD",
    "BUILD_NAME",
    "JOB_NAME",
    "Variant",
    "Version",
    "IS_RELEASE_BUILD",
    "HAS_REALLY",
    "IS_DAILY_Integrity_BUILD",
    "IS_DAILY_Linux_BUILD",
    "IS_Nightly_Integrity_BUILD",
    "IS_Nightly_Linux_BUILD",
    "IS_DAILY_BUILD",
    "NODE_LABELS"
]

total_environment_variable_keys = len(environment_variables_keys)
counter = 0

with open("environment_vars_for_product.txt", "w") as file:
    for env_var_key in environment_variables_keys:
        counter += 1
        if os.getenv(env_var_key, None) == None:
            print("Missing necessary environment : " + env_var_key)
            continue
        if " " in os.environ[env_var_key]:
            os.environ[env_var_key] = "'" + os.environ[env_var_key] + "'"
        file.write(env_var_key + "=" + os.getenv(env_var_key) +
                   ("\n" if counter < total_environment_variable_keys else ""))


import binwalk
from tkinter import *

# Initial Setup
window = Tk()
window.title("BinwalkGUI")

# Generating labels
label_name_title = Label(window, text="Filename:")
label_name_title.grid(column=0, row=0)

label_name = Label(window, text="", anchor=W)
label_name.grid(sticky=W+E+N+S,column=1, row=0)

label_results_title = Label(window, text="Results:")
label_results_title.grid(column=0, row=2)

label_results_data_header = Label(window, text='Offset        Description', font=("TkFixedFont"), anchor=W)
label_results_data_header.grid(sticky=W+E+N+S,column=1, row=2)

label_results_data = Label(window, text="", justify=LEFT, font=("TkFixedFont"))
label_results_data.grid(column=1, row=3)

# Helper functions
def analyze():
	label_name.configure(text="firmware.zip")

	# binwalk execution

	results_string = ''
	for module in binwalk.scan("firmware.zip", signature=True, quiet=True):
		for result in module.results:
			results_string += "0x%.8X    %s \n" % (result.offset, result.description)

	label_results_data.configure(text=results_string)

# Generating buttons
execute_button = Button(window, text="Analyze File", command=analyze)
execute_button.grid(column=1, row=5)

window.mainloop()


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build\bdist.win-amd64\egg\fragmap\console_color.py
# Compiled at: 2019-08-04 05:05:22
# Size of source mod 2**32: 327 bytes
ANSI_ESC = '\x1b'
ANSI_BG_RED = ANSI_ESC + '[41m'
ANSI_BG_WHITE = ANSI_ESC + '[47m'
ANSI_BG_MAGENTA = ANSI_ESC + '[45m'
ANSI_FG_RED = ANSI_ESC + '[31m'
ANSI_FG_CYAN = ANSI_ESC + '[36m'
ANSI_FG_BLUE = ANSI_ESC + '[34m'
ANSI_RESET = ANSI_ESC + '[0m'
ANSI_UP = ANSI_ESC + '[1A'
ANSI_DOWN = ANSI_ESC + '[1B'


# add modules to Python's search path
import sys
from pathlib import Path
from os.path import dirname, realpath
script_dir = Path(dirname(realpath(__file__)))
root = str(script_dir.parent)
print(root)
sys.path.insert(0, root + '/modules')
sys.path.insert(0, root + '/custom_dists')
# import required modules
import fp_lstm as fp
import numpy as np
import scipy.stats as ss
import tensorflow as tf
import nn_plotter as pltr
import gaussian_circle as gc
import vegas
import fp_solver as fps
import derivative as dr

beta = 200.0
ens_file = 'data/sde_evolve_test_2d_n_001.h5'
cost_file = 'data/sde_evolve_test_2d_n_cost_2_001.h5'

dtype = tf.float64
dimension = 2
num_components = 10
domain = 2.0*np.array([[-1.0, 1.0], [-1.0, 1.0]])

class DiffOp(tf.keras.layers.Layer):
    def __init__(self, f):
        super().__init__(name='DiffOp', dtype=dtype)
        self.f = f

    def call(self, x, y):
        r2 = x*x + y*y
        z = 4.0*(r2 - 1.0)
        with tf.GradientTape() as outer_x, tf.GradientTape() as outer_y:
            outer_x.watch(x)
            outer_y.watch(y)
            with tf.GradientTape() as inner:
                inner.watch([x, y])
                f_ = self.f(x, y)
            grad = inner.gradient(f_, [x, y])
            f_x = grad[0]
            f_y = grad[1]
        f_xx = outer_x.gradient(f_x, x)
        f_yy = outer_y.gradient(f_y, y)
        a = (x*z) * f_x
        b = (y*z) * f_y
        c = 4.0 * (z + 2.0) * f_
        return a + b + c + (f_xx + f_yy) / beta

solver = fp.FPDGM(50, 4, DiffOp, ens_file, domain, None, sinkhorn_iters=10, sinkhorn_epsilon=0.01, dtype=dtype)
solver.summary()

num_components = 10
cov = 0.1*np.identity(dimension)
weights = np.ones(num_components)
rv = gc.GaussianCircle(cov, weights)

class CustomDensity(tf.keras.models.Model):
    def __init__(self, dtype=dtype):
        super().__init__(dtype=dtype)

    def call(self, x):
        return tf.convert_to_tensor(rv.pdf(x), dtype=self.dtype)

    def call_2(self, x, y):
        X = tf.concat([x, y], axis=1)
        return rv.prob(X)

class InitialPDF(tf.keras.layers.Layer):
    def __init__(self, dtype=dtype):
        super().__init__(dtype=dtype)
        #self.dist = tfp.distributions.MultivariateNormalTriL(loc=mean, scale_tril=tf.linalg.cholesky(cov))
        #self.pdf = self.dist.prob
        self.c = tf.cast(tf.math.sqrt((2.0 * np.pi * delta) ** dimension), dtype=dtype)
        self.d = tf.cast(delta**dimension, dtype=dtype)
    def sample(self, size):
        return tf.convert_to_tensor(np.random.multivariate_normal(mean=mean, cov=cov, size=size), dtype=dtype)
    def call(self, x):
        return tf.math.exp(- 0.5 * tf.reduce_sum(x**2, axis=1, keepdims=True) / self.d ) / self.c

real_density = CustomDensity()
partials_rd = dr.FirstPartials(real_density.call_2, 2)
#"""
domain = 2.5 * np.array([[-1.0, 1.0], [-1.0, 1.0]])
plotter = pltr.NNPlotter(funcs=[solver], space=domain, num_pts_per_dim=50)
plotter.plot('images/fp_lstm_before.png')

#"""
ensemble = tf.convert_to_tensor(rv.sample(size=100), dtype=dtype)
weights = tf.convert_to_tensor(np.ones(100), dtype=dtype) #rv.pdf(ensemble)
solver.learn_density(ensemble, weights, domain, epochs=200, initial_rate=0.001)
region = fps.Domain(domain, dtype)
for _ in range(50):
    ensemble = region.sample(1000)
    weights = tf.convert_to_tensor(rv.pdf(ensemble), dtype=dtype)
    solver.learn_function(ensemble, weights, epochs=500, initial_rate=0.001)
    #solver.compute_normalizer(domain)

#"""
ensemble = tf.convert_to_tensor(rv.sample(size=100), dtype=dtype)
a = tf.reshape(solver(ensemble), (-1))
b = rv.pdf(ensemble.numpy())
c = tf.math.abs(a - b)
print(c)
print(tf.reduce_mean(c))
plotter = pltr.NNPlotter(funcs=[solver], space=domain, num_pts_per_dim=50)
plotter.plot('images/fp_lstm_after.png', wireframe=True)
#"""


#!/usr/bin/env python3
# -*- coding=utf-8 -*-

import os, json

def main():
    d = r'D:\My Projects\Python Projects\listening_and_writing'
    f = r'g5s2u8.json'
    p = os.path.join(d, f)
    file = open(p, encoding='utf-8')
    t = file.read()
    file.close()
    j = json.loads(t)
    n = '{'
    for k, v in j.items():
        n += '"' + k + '":{'
        for k1, v1 in v.items():
            if k1 != 'items':
                n += '"' + k1 + '":"' + v1 + '",'
            else:
                n += '"' + k1 + '":['
                for k2, v2 in v1.items():
                    n += '{"en":"' + k2 + '","zh":"' + v2 + '","category":""},'
                n = n[:-1]
                n += '],'
        n = n[:-1]
        n += '},'
    n = n[: -1]
    n += "}"
    print(n)

    f = r'new_' + f
    p = os.path.join(d, f)
    file = open(p, 'w', encoding='utf-8')
    file.write(n)
    file.close()

if __name__ == "__main__":
    main()


# This script is to find the probability of obtaining any one of the 256
# 8-bit binary unmbers which could constitute a changepoint configuration.
# We define \Gamma as the 8 bit number, and \gamma_i as each bimary element
# of \Gamma.
# We can define the probability of \gamma_i as
# P_gamma = (1/(((2*pi)^-0.5)*sigma))*exp((-(D-gamma_i*h)^2)/(2*sigma^2));
# Now we can define some other things. for example, D is fed forward from
# the CW analysis. Sigmma may be given (look into feed forward from CW
# analysis), h fed forward from CW analysis
# We compute the probability for each of the 256 8-bit numbers

from __future__ import division
import numpy as np
import matplotlib as mpl
from argparse import ArgumentParser
import inspect
import os
import datetime
mpl.rcParams['agg.path.chunksize'] = 10000
import matplotlib.pyplot as plt
from binary_structure import binary_structure
from logaddexp import logaddexp
from logaddexpvect import logaddexpvect
import math
import matplotlib as mpl
import plot_functions

def nCr(n,r):
    f = math.factorial
    return f(n) / f(r) / f(n-r)
	#There should exist a scipy nCr function

parser = ArgumentParser()

parser.add_argument("-d", "--data_list", dest = "data",
                  help = "List of data written as [x, x, x ...].", metavar = "LIST", default = [])
parser.add_argument("-i", "--input_file", dest = "infile",
		  help = "Location of the data file to be used as input, as a string.", metavar = "STRING")
parser.add_argument("-o", "--Output_path", dest = "outpath",
                  help = "Location of the data file to be used to print output.", metavar = "STRING")

args = parser.parse_args()

infile = args.infile
if not os.path.exists(infile):
        os.makedirs(infile)
        print('Please double check input file')

data = args.data
#CP_prior = args.CP_prior
output = args.outpath

if output[-1] != '/':
        output += '/'
if not os.path.exists(output):
        os.makedirs(output)
        print(output + ' created.')

data = []
def read_data(infile):
	data = []
	with open(infile, mode='r') as file:
		content = file.read()
	for line in content.split('\n'):
		string_line = line.split(',')
		for item in range(len(string_line)):
			string_line[item] = string_line[item].strip()
			string_line[item] = string_line[item].strip('\t')
			string_line[item] = string_line[item].strip('[')
			string_line[item] = string_line[item].strip(']')
		data_line = []
		for item in string_line:
			if item != '':
				data_line.append(float(item))
		if data_line != []:
			data.append(data_line)
	return data

data = read_data(infile)

all_data = np.asarray(data)
h_sd = 500 # 1e-24
hs = np.linspace(0, h_sd, 1001)
sigma = 50

l_prior = 0.001*np.ones(1000)

h_vals = np.linspace(h_sd/1001.0,h_sd,num=1000)
log_dh = np.log(h_vals[6] - h_vals[5])

#create a list of binary numbers of length len(data)
numlist = np.linspace(0, (2**len(all_data[0]))-1, num=2**len(all_data[0]))
bin_list = []

for itt in range(len(numlist)):
	number = bin(int(numlist[int(itt)]))
	x = []
	for item in number[2:]:
		x.append(int(item))
	while len(x)<len(all_data[0]):
		x = [0] + x
	bin_list.append(x)

bin_array = np.array(bin_list)

l_likelihood =  np.zeros(len(bin_list))

tile_var = [all_data.shape[0], 1]
big_h_vals = np.transpose(np.tile(h_vals, tile_var))
big_prior = np.transpose(np.tile(l_prior, tile_var))
#big_data = np.tile(data, [1000,1])
P_gamma = np.zeros([len(bin_list), len(bin_list[0])])
l_evidence = []
config = 0
n_changepoints = []
l_norm = []

for itt_number in range(len(bin_array)):
	config = itt_number
	binary_number = bin_list[itt_number]
	binary_struc =  binary_structure( binary_number )
	block_length = binary_struc["block_length"]
	block_numbers = binary_struc["block_numbers"]
	n_breaks = binary_struc["n_breaks"]
	n_changepoints.append(binary_struc["n_changepoints"])
	binary_number = np.append(binary_number, 0)
	each_h1 = -1*np.inf * np.ones(big_h_vals.shape)
	prefactor = np.log(np.sqrt(2.0*np.pi)*sigma)
	index = 0
	used_index = []
	data = all_data[itt_number]
	big_data = np.tile(data, [1000,1])
#	print(data)
	print(len(binary_number)-1)
	while index < len(binary_number)-1:
		if binary_number[index] == 1 and binary_number[index+1] ==1:
#			# Sum the data from the start to the end of the block,
			# also marginalise over h define variable end_of_block
			# to define where coherent marginalisation ends
			end_of_block = index + block_length[int(block_numbers[index])-1]
			exponent = -(((big_data[:,index:end_of_block] - big_h_vals[:,index:end_of_block])**2)/(2*sigma*sigma)) # -((D-h)/sqrt(2)*sigma)**2

			each_h1[:,index:end_of_block] = exponent + big_prior[:,index:end_of_block] + log_dh - prefactor
			# P_gamma is the sum of these values, log10(sum(prior * gaussian))
			# Calculate P_gamma chunk by chunk for each chunk in the block

			for row in range(index,end_of_block):
				if row == index:
					P_gamma[config, row] = logaddexpvect(each_h1[:,row])
					#used_index.append(row)
			index = end_of_block - 1 # Progess the index to end of block

		elif binary_number[index] == 1 and binary_number[index + 1] == 0:
			exponent = -(((big_data[:, index] - big_h_vals[:,index])**2) / (2 * sigma * sigma)) # -((D-h)/(sqrt(2)sigma))**2
			each_h1[:,index] = exponent + big_prior[:,index]  + log_dh - prefactor
			P_gamma[config,index] = logaddexpvect(each_h1[:,index]) # There is a numpy logaddexp function

		elif binary_number[index] == 0:
		    	P_gamma[config,index] = (-((data[index])**2)/(2.0*sigma*sigma)) - prefactor

		index = index + 1
		l_likelihood[config] = np.sum(P_gamma[config,:])
#	l_likelihood[config] = np.sum(data)
	l_norm.append(np.log((1/(len(binary_number)-1))*(nCr(len(data)-1,n_changepoints[config]))))
	l_evidence.append(l_likelihood[config] + l_norm[config])
	config += 1

l_odds = l_evidence - l_evidence[1]

sorted_evidence = sorted(l_evidence)
evidence_index = [i[0] for i in sorted(enumerate(l_evidence), key=lambda x:x[1])]
sorted_n_CP = [n_changepoints[i] for i in evidence_index]
print(l_evidence)

# Define the odds of one config vs all other configs
# Denominator is sum of all that are not that index
odds_all = np.zeros(len(l_evidence));
evidence = np.ma.array(np.exp(l_evidence), mask=False)
for index in range(len(l_evidence)):
	evidence.mask[index] = True
	odds_all_denom = np.sum(evidence)
	evidence.mask = False
	odds_all[index] = evidence[index] / odds_all_denom;

sorted_odds_all = sorted(odds_all)
odds_index = [i[0] for i in sorted(enumerate(odds_all), key=lambda x:x[1])]

sorted_odds = sorted(l_odds)
sort_index = [i[0] for i in sorted(enumerate(l_odds), key=lambda x:x[1])]

sorted_binaries = [bin_list[i] for i in odds_index]
sorted_priors = [l_norm[i] for i in odds_index]

sorted_data = [all_data[i] for i in odds_index]

sanity_check = [odds_index[i] - sort_index[i] for i in range(len(sort_index))]

out = output + 'outfile.txt'
o = open(out, 'w+')
o.seek(0)
o.write("Sanity check result: " + str(sum(sanity_check)) + ".\n")

results_dict = []
for i in range(len(sorted_odds_all)):
	results_dict.append({"sorted_binaries":sorted_binaries[i], "sorted_odds_all":sorted_odds_all[i], "sorted_evidence":sorted_evidence[i] , "sorted_priors":sorted_priors[i], "data used":sorted_data[i]})

for i in range(len(results_dict)):
	print(str(results_dict[i]) + "\n")
	o.write(str(results_dict[i]) + "\n")

# Find the right data instance to plot to show data atoms
i = len(all_data[0])
running_total = 0
while i>0:
	running_total += int(2**(i-1))
	i-=2
print(running_total)

plot_functions.barcode_plot(sorted_binaries,  sorted_odds, all_data[running_total],  output)
#for i in sorted_binaries:
#	print(str(i))

# Output key parameters to file.
# Plot barcode plots.

print("Sanity check result: " + str(sum(sanity_check)) + ".\n")


import datetime as dt
from typing import ClassVar, Optional, cast

from cuenca_validations.types import (
    Country,
    CurpValidationRequest,
    Gender,
    State,
)
from cuenca_validations.types.identities import CurpField
from pydantic.dataclasses import dataclass

from ..http import Session, session as global_session
from .base import Creatable, Retrievable

@dataclass
class CurpValidation(Creatable, Retrievable):
    _resource: ClassVar = 'curp_validations'

    created_at: dt.datetime
    names: str
    first_surname: str
    second_surname: Optional[str]
    date_of_birth: dt.date
    country_of_birth: Country
    state_of_birth: State
    gender: Gender
    nationality: Optional[Country]
    manual_curp: Optional[CurpField]
    calculated_curp: CurpField
    validated_curp: Optional[CurpField]
    renapo_curp_match: bool
    renapo_full_match: bool

    @classmethod
    def create(
        cls,
        names: str,
        first_surname: str,
        date_of_birth: dt.date,
        country_of_birth: str,
        state_of_birth: State,
        gender: Gender,
        second_surname: Optional[str] = None,
        manual_curp: Optional[CurpField] = None,
        *,
        session: Session = global_session,
    ) -> 'CurpValidation':
        req = CurpValidationRequest(
            names=names,
            first_surname=first_surname,
            second_surname=second_surname,
            date_of_birth=date_of_birth,
            state_of_birth=state_of_birth,
            country_of_birth=country_of_birth,
            gender=gender,
            manual_curp=manual_curp,
        )
        return cast(
            'CurpValidation',
            cls._create(session=session, **req.dict()),
        )


def single_char(arr, n):
    for i in range(n):
        arr[i][i] = True
    return arr

def double_char(s, arr, n):
    for j in range(1, n):
        if s[j-1] == s[j]:
            arr[j-1][j] = True
    return arr

def all(s, arr, n):
    jump = 2
    while jump < n:
        for i in range(n):
            j = i + jump
            # print(i, jump, j)
            if j < n and s[i] == s[j] and arr[i+1][j-1] == True:
                    arr[i][j] = True
        jump += 1
    return arr

def get_ans(s, arr, n):
    ans = 1
    start = 0
    end = 0
    for i in range(n):
        local_start = i
        local_end = i
        for j in range(i+1, n):
            if arr[i][j] == True:
                local_end = j
        if local_end - local_start > end - start:
            end = local_end
            start = local_start
    ans = s[start: end + 1]
    return ans

if __name__ == "__main__":
    for _ in range(int(input())):
        s = input()
        n = len(s)
        arr = [[0 for i in range(n)] for j in range(n)]
        arr = single_char(arr, n)
        arr = double_char(s, arr, n)
        arr = all(s, arr, n)
        ans = get_ans(s, arr, n)
        print(ans)


class Calibration:
    #   Calibrate.

    #   Determine min max

    #   count when something is out of calibrated range (atypical)

    count_to_calibrate = 100
    deviations_off = 2
    jump_deviation_multiplier = 5

    def __init__(self):
        self.xs = []
        self.x_diff = None
        self.x_avg = None

        self.ys = []
        self.y_diff = None
        self.y_avg = None

        self.zs = []
        self.z_diff = None
        self.z_avg = None

        self.is_calibrated = False
        self.count = 0
        self.count_off_calibration = 0

    def is_jump(self, x, y, z):
        #   Xs
        if x < (self.x_avg - (self.x_diff * self.jump_deviation_multiplier)) or x > (self.x_avg + (self.x_diff * self.jump_deviation_multiplier)):
            return True

        #   Ys
        if y < (self.y_avg - (self.y_diff * self.jump_deviation_multiplier)) or y > (self.y_avg + (self.y_diff * self.jump_deviation_multiplier)):
            return True

        #   Zs
        if z < (self.z_avg - (self.z_diff * self.jump_deviation_multiplier)) or z > (self.z_avg + (self.z_diff * self.jump_deviation_multiplier)):
            return True

        return False

    def is_in_still_range(self, x, y, z):
        #   Xs
        if (self.x_avg - (self.x_diff * self.deviations_off)) <= x <= (self.x_avg + (self.x_diff * self.deviations_off)):
            return True

        #   Ys
        if (self.y_avg - (self.y_diff * self.deviations_off)) <= y <= (self.y_avg + (self.y_diff * self.deviations_off)):
            return True

        #   Zs
        if (self.z_avg - (self.z_diff * self.deviations_off)) <= y <= (self.z_avg + (self.z_diff * self.deviations_off)):
            return True

        return False

    def get_value(self, x, y, z):
        """
        :param x:
        :param y:
        :param z:
        :return: tuple (corrected_x, corrected_y, corrected_z, is_calibrated
        """
        if self.is_calibrated:
            if self.is_jump(x, y, z):
                x = (x - self.x_avg) if self.x_avg > 0 else (x + self.x_avg)
                y = (x - self.y_avg) if self.y_avg > 0 else (x + self.y_avg)
                z = (x - self.z_avg) if self.z_avg > 0 else (x + self.z_avg)
                return (x, y, z, True)

            #   not a jump, so add to count.
            self.count += 1
            self.count_off_calibration += 0 if self.is_in_still_range(x, y, z) else 1

            if self.count == self.count_to_calibrate:
                #   value of calibration is whether it hits 50% of the time.
                self.is_calibrated = (self.count_off_calibration / self.count_to_calibrate) < 0.5
                #   reset stuff.
                self.count_off_calibration = 0
                self.count = 0
                pass

            #   if in calibration range, return zero values
            return 0, 0, 0, True
        else:
            self.xs.append(x)
            self.ys.append(y)
            self.zs.append(z)
            if len(self.xs) == self.count_to_calibrate:
                self.set_range()
            return x, y, z, False

    @staticmethod
    def get_average(value_array):
        return round(sum(value_array) / len(value_array), 3)

    def set_range(self):
        self.x_diff = round((max(self.xs) - min(self.xs)), 3)
        self.x_avg = self.get_average(self.xs)
        self.xs = []

        self.y_diff = round((max(self.ys) - min(self.ys)), 3)
        self.y_avg = self.get_average(self.ys)
        self.ys = []

        self.z_diff = round((max(self.zs) - min(self.zs)), 3)
        self.z_avg = self.get_average(self.zs)
        self.zs = []

        print(self.x_avg, self.x_diff, self.y_avg, self.y_diff, self.z_avg, self.z_diff)
        self.is_calibrated = True

if __name__ == '__main__':
    calibration = Calibration()

    with open('/Users/davidhaverberg/PycharmProjects/wobbleBoard/calibrate.txt') as potentials:
        for i, line in enumerate(potentials):
            line = line.strip().split('|')
            x = round(float(line[0]), 3)
            y = round(float(line[1]), 3)
            z = round(float(line[2]), 3)

            calibration.get_value(x, y, z)