from openmdao.core.driver import Driver, RecordingDebugging
from openmdao.api import SimpleGADriver, Problem, LatinHypercubeGenerator, DOEDriver
from dataclasses import dataclass

from copy import deepcopy
import random
import numpy as np
from itertools import chain
from deap import algorithms, base, tools
from deap.benchmarks import rosenbrock

class DeapDriver(Driver):
    def _declare_options(self):
        self.options.declare("container_class")

    def _get_name(self):
        return "DeapDriver"

    def _setup_driver(self, problem):
        super()._setup_driver(problem)
        self.container = self.options["container_class"](driver=self)

    def run(self):
        final_population = self.container.run_algorithm()
        # Evaluates a point in the middle of the pareto front to have one of
        # the optimal points as the final values in the model
        # self.container.evaluate(pareto_front[len(pareto_front) // 2])
        # print(pareto_front)
        return False

class Individual(list):
    def __init__(self, *args, fitness_class, **kwargs):
        super().__init__(*args, **kwargs)
        self.fitness = fitness_class()

    def __repr__(self):
        return f"Individual({super().__repr__()})"

@dataclass(frozen=True)
class DeapContainer:
    """
    An abstract class for containing the algorithm-specific logic. This is
    instantiated in the Driver's _setup_driver() function with the driver
    itself passed in as an argument.

    This object in itself should be fully stateless.

    The motivation for having this in a dedicated object is mainly that the
    Driver class is already heavily bloated.
    """

    driver: DeapDriver

    def __post_init__(self):
        # FIXME: this API is inflexible
        self.fitness_class = type(
            "Fitness",
            (base.Fitness,),
            {"weights": (-1,) * len(self.problem.model.get_objectives())},
        )

        self.design_var_shapes = {
            name: np.shape(value)
            for (name, value) in self.driver.get_design_var_values().items()
        }
        self.objective_shapes = {
            name: np.shape(value)
            for (name, value) in self.driver.get_objective_values().items()
        }
        self.constraint_shapes = {
            name: np.shape(value)
            for (name, value) in self.driver.get_constraint_values().items()
        }

        self.individual_bounds = self._individual_bounds()

    @property
    def problem(self):
        return self.driver._problem

    def individual_factory(self, *args, **kwargs):
        individual = self.individual_class(fitness_class=self.fitness_class, *args, **kwargs)
        return individual

    def _individual_bounds(self):
        design_vars = self.problem.model.get_design_vars()
        lower, upper = chain.from_iterable(
            (design_vars[key]["lower"].flat, design_vars[key]["upper"].flat)
            for key in self.design_var_shapes.keys()
        )

        return tuple(lower), tuple(upper)

    def convert_design_vars_to_individual(self, design_vars):
        """
        Converts a dict of OpenMDAO design variables into a DEAP individual.
        """
        individual = Individual(
            chain.from_iterable(
                design_vars[key].flat for key in self.design_var_shapes.keys()
            ),
            fitness_class=self.fitness_class,
        )
        return individual

    def convert_individual_to_design_vars(self, individual):
        """
        Converts a DEAP individual into a dict of OpenMDAO design variables.
        """
        ind = deepcopy(individual)

        design_vars = {}
        for name, shape in self.design_var_shapes.items():
            ind_items = np.product(shape)
            design_vars[name] = np.reshape(ind[:ind_items], shape)
            ind = ind[ind_items:]
        return design_vars

    def get_population_generator(self, count):
        return LatinHypercubeGenerator(
            samples=count, criterion="correlation", iterations=count // 10
        )

    def init_population(self, count):
        return [
            self.convert_design_vars_to_individual(dict(case))
            for case in self.get_population_generator(count)(
                self.problem.model.get_design_vars()
            )
        ]

    def evaluate(self, individual):
        pre = id(individual.fitness)
        for (name, value) in self.convert_individual_to_design_vars(individual).items():
            self.driver.set_design_var(name, value)
        assert id(individual.fitness) == pre
        with RecordingDebugging(
            self.driver._get_name(), self.driver.iter_count, self.driver
        ):
            failure_flag, abs_error, rel_error = self.problem.model._solve_nonlinear()

        self.driver.iter_count += 1
        # print(tuple(float(x) for x in self.driver.get_objective_values().values()))
        return tuple(
            chain.from_iterable(
                x.flat for x in self.driver.get_objective_values().values()
            )
        )

    def run_algorithm(self):
        raise NotImplemented("run_algorithm() method not implemented.")


from django.conf.urls import patterns, url

from ventas import viewsInforme,viewsPedido

urlpatterns = patterns('',
    url(r'^$', viewsPedido.venta_desktop, name='venta_desktop'),
    url(r'^fac/', viewsPedido.venta_desktop, name='venta_desktop1'),
    url(r'^mobile/$', viewsPedido.venta_mobile, name='venta_mobile'),
    url(r'^listar/$', viewsInforme.listar, name='listar'),
    url(r'^clientes/', viewsPedido.clientes, name='clientes'),
    url(r'^vendedores/', viewsPedido.vendedores, name='vendedores'),
    url(r'^codproducto/', viewsPedido.codproducto, name='codproducto'),
    url(r'^nomproducto/', viewsPedido.nomproducto, name='nomproducto'),
    url(r'^save/$', viewsPedido.savePedido, name='save'),
    url(r'^save/(?P<anyway>\w+)/$', viewsPedido.savePedido, name='save'),
    url(r'^saveDetalle/$', viewsPedido.saveDetalle, name='saveDetalle'),
    url(r'^deleteDetalle/(?P<id>\d+)/$', viewsPedido.deleteDetalle, name='deleteDetalle'),
    url(r'^pagar/$', viewsPedido.pagarPedido, name='pagarPedido'),
)


import pygame
import numpy as np
from util.helpers import *
from physics.colliding_object import Colliding

class EyeBeam(Colliding):
    def __init__(self, start, end):
        self.start = np.array(start)
        super(EyeBeam, self).__init__(self.start)
        self.end = np.array(end)
        self.collide_type = 'line'

    def unobstructed(self, list_of_game_objects):
        walls_vector = walls_vector_from_game_objects(list_of_game_objects)
        edge_vector = np.array((self.start, self.end))
        return unobstructed_edges(edge_vector, walls_vector)[0]

class Eyes:
    def __init__(self, view_distance=200):
        self.view_distance = view_distance
        self.look_ahead = 10
        self.list_of_game_objects = []

    def update(self, list_of_game_objects):
        self.list_of_game_objects = list_of_game_objects

    def direct_path_to_goal(self, current_position, goal, exclude=[]):
        obstructions = [i for i in self.list_of_game_objects if i not in exclude]
        walls_vector = walls_vector_from_game_objects(obstructions)
        if len(walls_vector) == 0:
            return True
        goal_edge = np.array([[current_position[0], current_position[1], goal[0], goal[1]]])
        return unobstructed_edges(goal_edge, walls_vector)

    def get_mouse_position(self):
        return np.array(pygame.mouse.get_pos()).astype(float)

    def look_for_collisions(self, coords, vector, radius):
        for sign in [1.0, 0.0, -1.0]:
            adjustment = normalise_vector(perpendicular_vector(vector)) * (sign * radius)
            adjusted_coords = coords + adjustment
            ahead_end = adjusted_coords + (vector * self.look_ahead)
            ahead = EyeBeam(adjusted_coords, ahead_end)
            collision = ahead.get_closest_collision(self.list_of_game_objects)
            if collision is not None:
                return collision
        return None

    def look_at_object(self, coords, screen_object):
        if self.direct_path_to_goal(coords, screen_object.coords(), exclude=[screen_object]):
            return screen_object
        else:
            return None

    def visible_objects(self, coords):
        visibles = []
        for screen_object in self.list_of_game_objects:
            eyes_see = self.look_at_object(coords,
                                           screen_object)
            if eyes_see is not None:
                visibles.append(eyes_see)
        return visibles

    def look_for_object(self,
                        coords,
                        object_description):
        matching_objects_in_range = [screen_object for screen_object in \
                            self.list_of_game_objects \
                            if screen_object.image['kind'] != 'wall'
                            and distance_between_points(coords, screen_object.coords()) < self.view_distance \
                            and object_description.viewitems() <= screen_object.image.viewitems()]
        if len(matching_objects_in_range) > 0:
            closest_index = find_closest_point_index(coords, [screen_object.coords() for screen_object in matching_objects_in_range])
            target_object = matching_objects_in_range[closest_index]
            return self.look_at_object(coords, target_object)
        return None


from core.visualization import PlotGraphics
from core.relation_extraction import SpacyRelationExtraction
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn.metrics import f1_score
import numpy as np
import joblib
import os
import sys
import warnings

warnings.filterwarnings('ignore')

class TextPipeline:

    def __init__(self, predictor_name, data_tuple, classifier_name='decision_tree', resources_path=None):
        # root path
        self.resources_folder = os.path.join(os.path.dirname(sys.path[0]), 'resources') \
            if resources_path is None else resources_path
        # initializes the classifier dict
        classifiers = {'decision_tree': DecisionTreeClassifier(random_state=0),
                       'k_neighbors': KNeighborsClassifier(n_neighbors=15)}
        # save the predictor name
        self.predictor_name = predictor_name

        # receives the data
        self.x_train, self.x_test, self.y_train, self.y_test = data_tuple
        # text extraction pipelines
        self.text_extraction_pipes = {}
        # prediction model to use
        self.prediction_model = classifiers[classifier_name]
        # init the visualizer
        self.plt_graphics = PlotGraphics(data_tuple, self.text_extraction_pipes)

    def create_features_pipeline(self, n_features=150, pipeline_obj=None):
        # checks if the transformer
        if pipeline_obj is None:
            # features vectorization
            transformer_obj = TfidfVectorizer(strip_accents='unicode',
                                              stop_words='english',
                                              lowercase=True,
                                              max_features=n_features,
                                              ngram_range=(1, 2),
                                              min_df=0.1, max_df=0.7)
            # creates the pipeline obj
            pipeline_obj = Pipeline([('vectorizer', transformer_obj)])
        # pipeline mapping
        self.text_extraction_pipes['feature'] = pipeline_obj
        # returns the pipeline obj
        return self.text_extraction_pipes['feature']

    def create_label_pipeline(self, relation_extraction=False, n_targets=20, n_jobs=8):
        # target vectorization
        if relation_extraction:
            # uses the spacy relation extraction
            vectorizer = SpacyRelationExtraction(n_relation=n_targets, n_jobs=n_jobs)
        else:
            # otherwise uses a normal vectorizer
            vectorizer = CountVectorizer(strip_accents='unicode',
                                         stop_words='english',
                                         lowercase=True,
                                         max_features=n_targets,
                                         ngram_range=(1, 2),
                                         min_df=0.1, max_df=0.7)
        # pipeline creation
        self.text_extraction_pipes['target'] = Pipeline([('vectorizer', vectorizer)])

    def pickle_predictor(self):
        # save the labels pipeline
        labels_extractor = self.text_extraction_pipes['target']['vectorizer']
        obj_name = os.path.join(self.resources_folder, '_'.join([self.predictor_name, 'labels', 'vectorizer']))
        joblib.dump(labels_extractor, obj_name + '.pkl')

        # saves the model
        obj_name = os.path.join(self.resources_folder, '_'.join([self.predictor_name, 'predictor']))
        joblib.dump(self.prediction_model, obj_name + '.pkl')

    def unpickle_predictor(self):
        # loads the object
        obj_name = os.path.join(self.resources_folder, '_'.join([self.predictor_name, 'labels', 'vectorizer']))
        labels_extractor = joblib.load(obj_name + '.pkl')
        self.text_extraction_pipes['target'] = Pipeline([('vectorizer', labels_extractor)])

        # unpickle the model
        obj_name = os.path.join(self.resources_folder, '_'.join([self.predictor_name, 'predictor']))
        self.prediction_model = joblib.load(obj_name + '.pkl')

    def fit(self, x_vector):
        # fit the feature data
        y_vector = self.text_extraction_pipes['target'].fit_transform(self.y_train).toarray()
        # convert the y_train
        y_vector[y_vector > 1] = 1
        # print some information data
        print('\ninput array, shape:', x_vector.shape)
        print('output array, shape:', y_vector.shape, '\n')

        # fit the model
        self.prediction_model.fit(x_vector, y_vector)

    def predict(self, x_test):
        # convert using the pipeline
        x_test_vector = self.text_extraction_pipes['feature'].transform(x_test)
        # convert the y_test
        predictions = self.prediction_model.predict(x_test_vector)
        # returns the predictions
        return predictions

    def score(self):
        # add the exception treatment
        y_test_vector = self.text_extraction_pipes['target'].transform(self.y_test).toarray()
        # predict the output for the test set
        predictions = self.predict(self.x_test)

        # print some metrics
        class_labels = self.text_extraction_pipes['target']['vectorizer'].get_feature_names()
        class_report = self.calculate_metrics(y_test_vector, predictions, class_labels)
        # plot the data
        self.plt_graphics.plot_bag_words(class_report)

        # return the classification report
        return class_report

    @staticmethod
    def calculate_metrics(y_test, predictions, class_labels):
        # print the results
        y_test[y_test > 1] = 1
        class_report = classification_report(y_test, predictions, target_names=class_labels, output_dict=True)
        print("Classification report: \n", classification_report(y_test, predictions, target_names=class_labels))
        # print("F1 micro averaging:", f1_score(y_test, predictions, average='micro', labels=np.unique(predictions)))
        print("ROC: ", roc_auc_score(y_test, predictions), '\n')
        # return the classification results
        return class_report


from discord.ext import commands

class ErrorHandeler(commands.Cog):
  """A cog for global error handling"""

  def __init__(self, bot: commands.Bot):
   self.bot = bot

  @commands.Cog.listener()
  async def on_command_error(self, ctx: commands.Context, error: commands.CommandError):
    if isinstance(error, commands.MemberNotFound):
      await ctx.send("Please input a valid user")
    if isinstance(error, commands.UnexpectedQuoteError):
      await ctx.send("Your message must be surrounded by quotes.")

def setup(bot: commands.Bot):
    bot.add_cog(ErrorHandeler(bot))


from jinja2 import Environment, PackageLoader
import os
from typing import Dict
import re

class SQLTemplate:
    _templatePath = os.path.join(
        os.path.dirname(os.path.dirname(os.path.relpath(__file__))), "templates"
    )

    _templatePath = os.path.join("templates")
    # raise ValueError(f'templatePath = {_templatePath}')

    def getTemplate(self, sqlAction: str, parameters: Dict, **kwargs) -> str:
        templateName = f"{sqlAction.lower().strip()}.j2"
        templateEnv = Environment(
            loader=PackageLoader(
                package_name="tips", package_path="framework/templates"
            ),
            trim_blocks=True
            # loader=FileSystemLoader(self._templatePath), trim_blocks=True
        )
        cmd = (
            templateEnv.get_template(templateName)
            .render(parameters=parameters, kwargs=kwargs)
            .strip()
            .replace("\n", " ")
        )
        return re.sub("  +", " ", cmd)


# https://www.codewars.com/kata/5b76a34ff71e5de9db0000f2

from itertools import starmap
def solve(arr):
    def conv(t1,t2):
        t1 = tuple(map(int,t1.split(":")))
        t2 = tuple(map(int,t2.split(":")))
        hd1, md = divmod((t2[1] - t1[1] - 1), 60)
        hd2 = (t2[0] - t1[0] + hd1) % 24
        mm = (hd2 % 24) * 60 + md
        return mm, ':'.join(str(c).zfill(2) for c in divmod(mm, 60))
    arr = sorted(set(arr))
    return max(starmap(conv,zip(arr,arr[1:]+arr)))[1]


class IP():
   def __init__(self,ipaddress):
      url='http://m.ip138.com/ip.asp?ip='
      self.IP=ipaddress
      self.site=url+self.IP
      self.header={'User-Agent' :'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'}
   def get_phy(self):
      import requests as RQ
      import re
      try:
         r=RQ.get(self.site)

         r.raise_for_status()
         r.encoding=r.apparent_encoding
         html=r.text[-1000:]
         #print(html)
         answer=re.findall('本站主数据：(.*?)
<p',html,re.S)
         answer=answer[0]
         return '您查询的IP:%s物理地址应该在:%s '%(self.IP,answer)
      except:
         return 'sth wrong'

#素质裤衩分析
'''
您查询的IP：1.1.1.1

本站主数据：澳大利亚

参考数据一：澳大利亚

'''
'''
while True:
   point='.'
   for I in range(7,100,7):
      for j in range(1,100,7):
         for k in range(1,100,70):
            for L in range(1,100,20):
               add=str(I)+point+str(j)+point+str(k)+point+str(L)
               print(add)
               #ip=input()
               i=IP(add)
               ans=i.get_phy()
               print(ans)
'''
#第一个利用接口写的东西

'''num=input()
num_list=list(num)
num_list.remove(' ')
num_list.remove(' ')
new_num_list=[]
print(num_list)
for i in range(6):
   if num_list[i]=='-':
      new_num_list.append(int(num_list[i]+num_list[i+1]))
   '''

'''
sentinel ='' # 遇到这个就结束
lines = []
for line in iter(input, sentinel):
    lines.append(line)

init=list(str(input()))
try:
   init.remove(' ')
except:
   pass
print(int(init[0])+int(init[1]))

string=input()
str_list=list(string)
str_list=str_list.reverse()
new_str=str(str_list)
print(new_str)
'''


import pygame
from flame import Flame

class Firework:
    def __init__(self):
        self.rect = pygame.Rect(640, 720, 25, 50)
        self.image = pygame.Surface( (25, 50) )
        self.image.fill( (255, 255, 255) )
        self.exploded = False
        self.flames = []

    def update(self):
        if not self.exploded:
            self.rect.y -= 2
            if self.rect.y <= 200:
                self.explode()
        else:
            for flame in self.flames:
                flame.update()

    def draw(self, screen):
        if not self.exploded:
            screen.blit(self.image, self.rect)
        else:
            for flame in self.flames:
                flame.draw(screen)

    def explode(self):
        self.exploded = True
        for i in range(1000):
            self.flames.append(Flame())


#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import logging; logger = logging.getLogger("main")
FORMAT = '%(asctime)s - %(levelname)s: %(message)s'
logging.basicConfig(format=FORMAT, level=logging.WARNING)

import time

from flask import Flask, escape, url_for,render_template, g, request, redirect, jsonify, session
from werkzeug import secure_filename

import sys, os
from jinja2 import Environment, PackageLoader
import json

app = Flask(__name__, static_folder='static')

maze = []
width = 0
height = 0

STARTPOS=[1,1]
MAXLIFE = 10
MIN_TIME_BETWEEN_INTERACTIONS=0.2 #seconds

robots = {}

def store_map(rawmaze):

    global maze, width, height

    if is_map_loaded():
        logger.warning("Map already loaded. Ignoring it. Restart the backend if you want to update the map.")
        return

    width = rawmaze["width"]
    height = rawmaze["height"]

    maze = [True if x in [399,431,463,492,493,494,495] else False for x in rawmaze["data"]]

    for j in range(height):
        for i in range(width):
            idx = i + j * width
            if maze[idx]:
                sys.stdout.write('.')
            else:
                sys.stdout.write(' ')
        sys.stdout.write('\n')

    logger.info("Maze successfully loaded!")

def is_map_loaded():
    return width and height

def get_obstacles(x,y):

    # obstacle at centre, north, south, east, west?
    obstacles = [True, True, True, True, True]

    if x >= 0 and y >= 0 and x < width and y < height and maze[x + y * width]:
        obstacles[0] = False

    if x >= 0 and y-1 >= 0 and x < width and y-1 < height and maze[x + (y-1) * width]:
        obstacles[1] = False

    if x >= 0 and y+1 >= 0 and x < width and y+1 < height and maze[x + (y+1) * width]:
        obstacles[2] = False

    if x+1 >= 0 and y >= 0 and x+1 < width and y < height and maze[x+1 + y * width]:
        obstacles[3] = False

    if x-1 >= 0 and y >= 0 and x-1 < width and y < height and maze[x-1 + y * width]:
        obstacles[4] = False

    logger.info(str(obstacles))
    return obstacles

@app.route("/set/<name>/<x>/<y>")
def set_robot(name, x, y):
    logger.info("Placing robot %s to %s,%s" % (name,x,y))

    x = int(x)
    y=int(y)
    c,_,_,_,_ = get_obstacles(x,y)
    if c:
        logger.info("Can not place robot there!")
        return json.dumps(False)

    robots[name]["pos"] = [x,y]
    return json.dumps(True)

def get_robot(name):
    if name not in robots:
        return json.dumps([-1,-1])
    return json.dumps(robots[name]["pos"])

@app.route("/")
def main():
    return render_template('index.html')

@app.route("/live")
def map():
    return render_template('map.html')

@app.route("/get_robots")
def get_robots():

    now = time.time()

    complete_robots = dict(robots)

    for k in list(robots.keys()):
        if robots[k]["life"] <= 0:
            logger.warning("Robot %s has no life left! killing it!" % k)
            del robots[k]
            del complete_robots[k]
            continue

        if now - robots[k]["lastinteraction"] > 60 * 10:
            logger.warning("Robot %s has not being used for 10 min. Removing it." % k)
            del robots[k]
            del complete_robots[k]
            continue

        complete_robots[k]["age"] = now - robots[k]["created"]

    return json.dumps(complete_robots)

def create_new_robot(name):
    logger.info("Placing new robot %s at start position" % name)
    robots[name] = {"pos": STARTPOS,
                    "created": time.time(),
                    "lastinteraction": 0,
                    "life": MAXLIFE
                    }

@app.route('/move/<name>/<direction>')
def move(name, direction):
    if not is_map_loaded():
        logger.error("Map not loaded yet! Reload webpage.")
        return json.dumps([False,[]])

    if name not in robots:
        create_new_robot(name)

    logger.info("Moving robot %s to %s" % (name,direction))

    now = time.time()
    if now - robots[name]["lastinteraction"] < MIN_TIME_BETWEEN_INTERACTIONS:
        logger.error("Too many interactions with %s. Wait a bit." % name)
        return json.dumps([False,[]])

    robots[name]["lastinteraction"] = now

    x,y = robots[name]["pos"]

    if direction == 'N':
        nx, ny = x, y-1
    if direction == 'S':
        nx, ny = x, y+1
    if direction == 'E':
        nx, ny = x+1, y
    if direction == 'W':
        nx, ny = x-1, y

    c,n,s,e,w = get_obstacles(nx,ny)

    if c:
        logger.info("...can not move there!")
        robots[name]["life"] -= 1
        return json.dumps([False,[]])

    else:
        robots[name]["pos"] = [nx,ny]
        return json.dumps([True,[n,s,e,w]])

@app.route("/map", methods=['POST'])
def load_map():
    logger.info("Retrieving the map data...")

    store_map(json.loads([k for k in request.form.keys()][0]))
    return ""

@app.route("/life/<name>")
def life(name):
    return json.dumps(robots[name]["life"] if name in robots else 0)


import logging

from core.connect_db import connect_db
from logger.logger import configLogger
from settings.settings import load_settings

logger = logging.getLogger()

class BaseFetcher(object):

    def __init__(self):
        super(BaseFetcher, self).__init__()

        configLogger()
        self._connect_to_db()

    def run(self):
        running = True
        while running:
            try:
                self._run()
            except Exception as e:
                logger.error('Got error while running : %r' % e)
                running = False
                raise

    def _run(self):
        pass

    def _connect_to_db(self):
        settings = load_settings()
        mongo_config = settings['dbs']['mongo']
        con = connect_db(**mongo_config)


import pandas as pd
import s3fs

def main(event = None, context = None):
    print("Start running LinkedInScraper")
    values = [['Atreish Ramlakhan',
              'New York, New York, United States',
              'Katz School at Yeshiva University',
              'Graduate Teaching Assistant',
              'https://www.linkedin.com/company/16181365/'],
             ['Yuxiao (Henry) Shen',
              'New York, New York, United States',
              'The AAT Project (America’s Amazing Teens, LLC)',
              'Full Stack PHP Web Developer',
              'https://www.linkedin.com/search/results/all/?keywords=The+AAT+Project+%28America%E2%80%99s+Amazing+Teens%2C+LLC%29'],
             ['Shichao Zhou',
              'New York, New York, United States',
              'S&P Global Market Intelligence · Internship',
              'Data Analyst',
              'https://www.linkedin.com/company/162892/'],
             ['Mahlet Melese', 'New York, New York, United States', None, None, None]]
    df = pd.DataFrame(values,columns = [["Full Name", "Location", "Most Recent Company", 'Job Title', 'Company Url']])
    ###LOAD THE FILE INTO S3####
    # prepare csv file name
    pathname = 'ia-final2022-csv/'#specify location of s3:/{my-bucket}/
    filenames = f"{pathname}linkedIn_info.csv" #name of the filepath and csv file

    #encoding must be adjusted to accommodate abnormal characters. Use s3fs to write to S3 bucket
    print("Start adding LinkedIn data to csv")
    byte_encoded_df = df.to_csv(None, index=False).encode() #encodes file as binary
    s3 = s3fs.S3FileSystem(anon=False)
    with s3.open(filenames, 'wb') as file:
        file.write(byte_encoded_df) #writes byte-encoded file to s3 location

    #print success message
    print("Successfull uploaded file to location:"+str(filenames))
    print("Complete running LinkedInScraper")


"""HartPro URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url,include
from django.core.paginator import Paginator
from django.shortcuts import render
from art.models import Tag,Art
import json
from user import helper

import xadmin as admin

def toIndex(request):
    tags1 = Tag.objects.all()
    # locals将当前函数的局部变量转成字典的key-value结构
    #{'request':request,'tags':tags}
    tags = []
    for tag in tags1:
        #判断该类型中是否有文章,如果有就添加进tags
        if Art.objects.filter(tag=tag):
            tags.append(tag)
    #annotate为每个tag对象增加一个字段(Count('art) 统计每种类型下文章数据)
    #
    #读取分类id
    tag_id = request.GET.get('tag')
    if tag_id:
        tag_id = int(tag_id)
        arts = Art.objects.filter(tag_id=tag_id)  #exclude排除条件为真的数据
    else:
        arts = Art.objects.all()

    # #加载所有文章
    # arts = Art.objects.all()

    #将文章进行分页处理
    paginator = Paginator(arts,8)   #分页器

    page = request.GET.get('page')
    page = int(page) if page else 1  # 读取请求参数中page参数，如果没有,默认为1
    pager = paginator.page(page)  # 获取当前页的数据

    #获取登录用户的信息
    login_user= helper.getLoginInfo(request)

    return render(request,'index.html',locals())

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^ueditor/', include('DjangoUeditor.urls')),
    url(r'^user/',include('user.urls')),
    url(r'^art/',include('art.urls')),
    url(r'^$', toIndex),
]


"""
GenT2MF_Trapezoidal.py
Created 3/1/2022
"""
from __future__ import annotations
from typing import List
from juzzyPython.generalType2zSlices.sets.GenT2MF_Prototype import GenT2MF_Prototype
from juzzyPython.intervalType2.sets.IntervalT2MF_Trapezoidal import IntervalT2MF_Trapezoidal
from juzzyPython.type1.sets.T1MF_Trapezoidal import T1MF_Trapezoidal

class GenT2MF_Trapezoidal(GenT2MF_Prototype):
    """
    Class GenT2MF_Trapezoidal
    Creates a new instance of GenT2zMF_Trapezoidal

    Parameters:
        primer
        primer0
        primer1
        primers
        numberOfzLevels

    Functions:
        getZSlice

    """

    def __init__(self, name: str,primer: IntervalT2MF_Trapezoidal = None,primer0: IntervalT2MF_Trapezoidal = None, primer1: IntervalT2MF_Trapezoidal = None,primers: List[IntervalT2MF_Trapezoidal] = None, numberOfzLevels = None) -> None:
        super().__init__(name)
        self.DEBUG = False
        if primer != None:
            stepsize = [0] * 4
            self.numberOfzLevels = numberOfzLevels
            self.support = primer.getSupport()
            self.primer = primer
            slices_fs = [0] * numberOfzLevels
            self.slices_zValues = [0] * numberOfzLevels

            z_stepSize = 1.0/numberOfzLevels
            self.zSlices = [0] * numberOfzLevels
            stepsize[0] = (primer.getLMF().getA() - primer.getUMF().getA())/(numberOfzLevels-1)/2.0
            stepsize[1] = (primer.getLMF().getB() - primer.getUMF().getB())/(numberOfzLevels-1)/2.0
            stepsize[2] = (primer.getUMF().getC() - primer.getLMF().getC())/(numberOfzLevels-1)/2.0
            stepsize[3] = (primer.getUMF().getD() - primer.getLMF().getD())/(numberOfzLevels-1)/2.0

            inner = primer.getLMF().getParameters().copy()
            outer = primer.getUMF().getParameters().copy()

            self.zSlices[0] = IntervalT2MF_Trapezoidal("Slice 0",primer.getUMF(),primer.getLMF())
            self.slices_zValues[0] = z_stepSize
            if self.DEBUG:
                print(self.zSlices[0].toString()+"  Z-Value = "+str(self.slices_zValues[0]))

            for i in range(1,numberOfzLevels):
                self.slices_zValues[i] = self.slices_zValues[i-1]+z_stepSize
                inner[0]-=stepsize[0]
                inner[1]-=stepsize[1]
                inner[2]+=stepsize[2]
                inner[3]+=stepsize[3]
                outer[0]+=stepsize[0]
                outer[1]+=stepsize[1]
                outer[2]-=stepsize[2]
                outer[3]-=stepsize[3]

                if(inner[0]<outer[0]):
                    inner[0] = outer[0]
                if(inner[1]<outer[1]):
                    inner[1] = outer[1]
                if(inner[2]>outer[2]):
                    inner[2] = outer[2]
                if(inner[3]>outer[3]):
                    inner[3] = outer[3]

                self.zSlices[i] = IntervalT2MF_Trapezoidal("Slice "+str(i), T1MF_Trapezoidal("upper_slice "+str(i),outer),T1MF_Trapezoidal("lower_slice "+str(i),inner))

                if self.DEBUG:
                    print(self.zSlices[i].toString()+"  Z-Value = "+str(self.slices_zValues[i]))

        elif primer0 != None and primer1 != None:
            if self.DEBUG:
                print("Number of zLevels: "+str(numberOfzLevels))

            self.numberOfzLevels = numberOfzLevels
            self.support = primer0.getSupport()
            slices_fs = [0] * numberOfzLevels
            self.slices_zValues = [0] * numberOfzLevels
            self.zSlices = [0] * numberOfzLevels

            self.zSlices[0] = primer0
            self.zSlices[0].setName(self.getName()+"_Slice_0")
            self.zSlices[-1] = primer1

            z_stepSize = 1.0/(numberOfzLevels)
            self.slices_zValues[0] = z_stepSize
            self.slices_zValues[-1] = 1.0

            lsu = (primer1.getUMF().getParameters()[0]-primer0.getUMF().getParameters()[0])/(numberOfzLevels-1)
            lsl = (primer0.getLMF().getParameters()[0]-primer1.getLMF().getParameters()[0])/(numberOfzLevels-1)

            rsu = (primer0.getUMF().getParameters()[3]-primer1.getUMF().getParameters()[3])/(numberOfzLevels-1)
            rsl = (primer1.getLMF().getParameters()[3]-primer0.getLMF().getParameters()[3])/(numberOfzLevels-1)

            if self.DEBUG:
                print("lsu = "+str(lsu)+"  lsl = "+str(lsl)+"  rsu = "+str(rsu)+"  rsl = "+str(rsl))

            inner = primer0.getLMF().getParameters().copy()
            outer = primer0.getUMF().getParameters().copy()

            for i in range(1,numberOfzLevels-1):
                self.slices_zValues[i] = self.slices_zValues[i-1]+z_stepSize
                inner[0]-=lsl
                inner[3]+=rsl
                outer[0]+=lsu
                outer[3]-=rsu

                if self.DEBUG:
                    print("Slice "+str(i)+" , inner: "+str(inner[0])+"  "+str(inner[1])+"  "+str(inner[2])+"   outer: "+str(outer[0])+"  "+str(outer[1])+"  "+str(outer[2]))
                self.zSlices[i] = IntervalT2MF_Trapezoidal(self.getName()+"_Slice_"+str(i),T1MF_Trapezoidal("upper_slice "+str(i),outer),T1MF_Trapezoidal("lower_slice "+str(i),inner))

                if self.DEBUG:
                    print(self.zSlices[i].toString()+"  Z-Value = "+str(self.slices_zValues[i]))

        elif primers != None:
            self.numberOfzLevels = len(primers)
            self.support = primers[0].getSupport()

            slices_fs = [0] * self.numberOfzLevels
            self.slices_zValues = [0] * self.numberOfzLevels
            z_stepSize = 1.0/self.numberOfzLevels
            self.slices_zValues[0] = z_stepSize
            self.zSlices = primers.copy()

            for i in range(self.numberOfzLevels):
                self.slices_zValues[i] = z_stepSize*(i+1)
                if self.DEBUG:
                    print(self.zSlices[i].toString()+"  Z-Value = "+str(self.slices_zValues[i]))

    def clone(self) -> GenT2MF_Trapezoidal:
        """Not implemented"""
        print("Not implemented")
        return None

    def getZSlice(self, slice_number: int) -> IntervalT2MF_Trapezoidal:
        """Return the slice number"""
        return self.zSlices[slice_number]

    def getLeftShoulderStart(self) -> float:
        """Not implemented"""
        print("Not implemented")
        return float("Nan")

    def getRightShoulderStart(self) -> float:
        """Not implemented"""
        print("Not implemented")
        return float("Nan")


from backend import credential
import urllib.parse
from google.cloud import storage
import streamlit as st
import os
import json
import fnmatch
import file_io
import utils
import traceback
import io

def init():
    creds_str = credential.google_creds()

    if not os.path.exists('temp'):
        os.makedirs('temp')

    with open('temp/google-credentials.json', 'w') as f:
        json.dump(creds_str, f)

    os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'temp/google-credentials.json'
    storage_client = storage.Client()
    st.session_state['storage_client'] = storage_client

def upload_to_bucket(root_dir, file, uid, name, metadata=None, compress=None):
    dir = f"{root_dir}/{uid}"
    try:
        # get file extension
        extension = os.path.splitext(file.name)[1]
        filename = name + extension

        compressed_file_path = None

        if compress:
            # Compress file
            if compress == 'gzip':
                compressed_file_path = file_io.compress_to_gzip(file)
                filename += '.gz'  # Add '.gz' extension to the filename
            elif compress == 'xz':
                compressed_file_path = file_io.compress_to_xz(file)
                filename += '.xz'  # Add '.xz' extension to the filename
            else:
                raise ValueError(f'Unsupported compression type: {compress}. Supported types are "gzip" and "xz".'
                                 f'if you do not want to compress the file, set compress=None')

        storage_client = st.session_state['storage_client']
        bucket = storage_client.get_bucket(st.secrets['gcp']['bucket_name'])
        blob = bucket.blob(f"{dir}/{filename}")

        if compress:
            # Open the compressed file in read-binary mode for upload
            with open(compressed_file_path, 'rb') as file_obj:
                file_content = file_obj.read()  # read file content once

                default_meta = {
                    'md5_hash': utils.calculate_md5(file_content),
                    'size': utils.calculate_size(file_content),
                    'owner': st.session_state['student_number'],
                    'time': utils.get_current_time()
                }
                # Merge the default metadata with the given metadata
                meta = {**default_meta, **metadata} if metadata else default_meta

                # Set the blob metadata
                blob.metadata = meta

                blob.upload_from_file(io.BytesIO(file_content))
            # Delete the compressed file
            os.remove(compressed_file_path)
        else:
            # If compress is None or False, upload the file as is
            # Convert file_content to a BytesIO object and upload
            file_content = file.read()
            default_meta = {
                'md5_hash': utils.calculate_md5(file_content),
                'size': utils.calculate_size(file_content),
                'owner': st.session_state['student_number'],
                'time': utils.get_current_time()
            }

            # Merge the default metadata with the given metadata
            meta = {**default_meta, **metadata} if metadata else default_meta

            # Set the blob metadata
            blob.metadata = meta

            blob.upload_from_file(io.BytesIO(file_content))

    except Exception as e:
        tb = traceback.format_exc()
        st.error(f'❌Failed to upload to the bucket: **{e}** \n\n **Traceback**:\n ```{tb}```')
        st.stop()

def delete_from_bucket(root_dir, filenames, uid):
    for filename in filenames:
        # Decode the filename to ensure spaces are handled correctly
        decoded_filename = urllib.parse.unquote(filename)
        try:
            storage_client = st.session_state['storage_client']
            bucket = storage_client.get_bucket(st.secrets['gcp']['bucket_name'])
            blob = bucket.blob(f"{root_dir}/{uid}/{decoded_filename}")
            blob.delete()
        except Exception as e:
            st.error(f'failed to delete file ({root_dir}/{uid}/{decoded_filename}) from bucket. **{e}**')
            st.stop()

def download_from_bucket(root_dir, filename, uid):
    try:
        storage_client = st.session_state['storage_client']
        bucket = storage_client.get_bucket(st.secrets['gcp']['bucket_name'])
        blob = bucket.blob(f"{root_dir}/{uid}/{filename}")

        if not os.path.exists('temp'):
            os.makedirs('temp')

        with open(f"temp/{filename}", 'wb') as f:
            storage_client.download_blob_to_file(blob, f)
        return f"temp/{filename}"
    except Exception as e:
        st.error(f'failed to download file from bucket. **{e}**')
        st.stop()

def get_blobs(bucket, dir, name_pattern, extensions):
    blobs = []
    if '*' in name_pattern:
        # If wildcard is present in name_pattern, process as pattern.
        prefix, pattern = name_pattern.split('*', 1)
        # List blobs whose names start with the given prefix
        for blob in bucket.list_blobs(prefix=f"{dir}/{prefix}"):
            for extension in extensions:
                if blob.name.endswith(extension) and fnmatch.fnmatch(blob.name, f"{dir}/{name_pattern}"):
                    blobs.append(blob)
                    # Once a match is found, no need to check other extensions
                    break
    else:
        # If no wildcard is present, process name_pattern as exact file name.
        for extension in extensions:
            blob = bucket.blob(f"{dir}/{name_pattern}{extension}")
            if blob.exists():
                blobs.append(blob)
    return blobs

def get_public_urls_from_blobs(blobs):
    return [blob.public_url for blob in blobs]

def get_blob_md5(blobs):
    return [blob.md5_hash for blob in blobs]

def get_blob_metadata(blobs):
    return [blob.metadata for blob in blobs]

def get_blob_info(root_dir, uid, name_pattern, extensions, infos):
    storage_client = st.session_state['storage_client']
    bucket = storage_client.get_bucket(st.secrets['gcp']['bucket_name'])
    dir = f"{root_dir}/{uid}"

    blobs = get_blobs(bucket, dir, name_pattern, extensions)

    for info in infos:
        if info == 'url':
            return get_public_urls_from_blobs(blobs)
        else:
            metas = get_blob_metadata(blobs)
            return [meta[info] for meta in metas]


# -*- coding: utf-8 -*-
"""
Created on Tue Mar 22 15:37:26 2022

@author: jeros

Hu moments analysis
"""
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.ticker import PercentFormatter

def plotter(huN = 1, bananas = None,oranges = None,lemons = None):

   # if bananas is not None:
   #     plt.hist(bananas[0,:],bins, alpha=0.5, label='b',weights = weights_b )
   # if oranges is not None:
   #     plt.hist(oranges[0,:],bins, alpha=0.5, label='o',weights = weights_o)

   '''Hu moment number histogram'''
   if huN == 0:
       bins = np.linspace(2.85,3.22,100)

   if huN == 1:
       bins = np.linspace(5.5,12.5,100)

   if huN == 2:
       bins = np.linspace(10,16,100)

   if huN == 3:
       bins = np.linspace(9.8,19,100)

   if huN == 4:
       bins = np.linspace(-35,35,100)

   if huN == 5:
        bins = np.linspace(-25,25,100)

   if huN == 6:
        bins = np.linspace(-35,35,100)

   #plt.hist([bananas[huN,:], oranges[huN,:],lemons[huN,:]],label=['B', 'O','L'])
   plt.hist([bananas[huN,:], oranges[huN,:],lemons[huN,:]], bins,label=['B', 'O','L'],density = True)

   plt.title('Hu'+str(huN))

   '''Hu moment number 2 histogram'''
   bins = np.linspace(10,16,100)

   plt.legend(loc='upper right')
   plt.autoscale(enable=True, axis='x', tight=True)
   #plt.gca().yaxis.set_major_formatter(PercentFormatter(1))
   plt.show()


import re

import pandas as pd
import fool
from copy import copy
from starter_code1.NER.ner01 import *

test_data = pd.read_csv('../data/info_extract/test_data.csv', encoding='gb2312', header=0)
# print(test_data.head())

test_data['ner'] = None
ner_id = 1001
ner_dict_new = {}  # 存储所有实体
ner_dict_reverse_new = {}  # 储存所有实体

for i in range(len(test_data)):
    sentence = copy(test_data.iloc[i, 1])
    # TODO: 调用fool积极性实体识别，得到words和ners结果
    words, ners = fool.analysis(sentence)
    # print(words)
    # print(ners)

    ners[0].sort(key=lambda x: x[0], reverse=True)
    for start, end, ner_type, ner_name in ners[0]:
        if ner_type == 'company' or ner_type == 'person':
            # ner_dict_new
            lst = main_extract(ner_name, stop_word, d_4_delete, d_city_province)
            company_main_name = ''.join(lst)  # 对公司名提取主体部分，将包含相同主体部分的公司统一为一个实体
            if company_main_name not in ner_dict_new:
                ner_dict_new[company_main_name] = ner_id
                ner_dict_reverse_new[ner_id] = company_main_name
                ner_id += 1

            sentence = sentence[:start] + ' ner_' + str(ner_dict_new[company_main_name]) + '_ ' + sentence[end:]

    test_data.iloc[i, -1] = sentence

X_test = test_data[['ner']]

# 处理train数据,利用开源工具进行实体识别和并使用实体统一函数储存实体
train_data = pd.read_csv('../data/info_extract/train_data.csv', encoding='gb2312', header=0)
train_data['ner'] = None

for i in range(len(train_data)):
    # 判断正负样本
    if train_data.iloc[i, :]['member1'] == '0' and train_data.iloc[i, :]['member2'] == '0':
        sentence = copy(train_data.iloc[i, 1])
        # TODO:调用fool进行实体识别，得到wods和ners结果
        words, ners = fool.analysis(sentence)
        ners[0].sort(key=lambda x: x[0], reverse=True)
        for start, end, ner_type, ner_name in ners[0]:
            # TODO:调用实体统一函数，储存统一后的实体
            # 并自增ner_id
            if ner_type == 'company' or ner_type == 'person':
                company_main_name = ''.join(
                    main_extract(ner_name, stop_word, d_4_delete, d_city_province))  # 提取公司主体名称
                if company_main_name not in ner_dict_new:
                    ner_dict_new[company_main_name] = ner_id
                    ner_dict_reverse_new[ner_id] = company_main_name
                    ner_id += 1

                # 在句子中用编号替换实体名
                sentence = sentence[:start] + ' ner_' + str(ner_dict_new[company_main_name]) + '_ ' + sentence[end:]

        train_data.iloc[i, -1] = sentence
    else:
        # 将训练集中正样本已经标注的实体也使用编码替换
        sentence = copy(train_data.iloc[i, :])['sentence']
        for company_main_name in [train_data.iloc[i, :]['member1'], train_data.iloc[i, :]['member2']]:
            # TODO:调用实体统一函数，储存统一后的实体
            # 并自增ner_id
            company_main_name = ''.join(
                main_extract(company_main_name, stop_word, d_4_delete, d_city_province))  # 提取公司主体名称
            if company_main_name not in ner_dict_new:
                ner_dict_new[company_main_name] = ner_id
                ner_dict_reverse_new[ner_id] = company_main_name
                ner_id += 1

            # 在句子中用编号替换实体名
            sentence = re.sub(company_main_name, ' ner_%s_ ' % (str(ner_dict_new[company_main_name])), sentence)

        train_data.iloc[i, -1] = sentence

y = train_data.loc[:, ['tag']]
train_num = len(train_data)
X_train = train_data[['ner']]

# 将train和test放在一起提取特征
# X = pd.concat([X_train, X_test])
# X.to_csv('./x.csv', index=False)
# print(X)

from sklearn.ensemble import RandomForestClassifier
from sklearn import preprocessing
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LogisticRegression
import numpy as np

# TODO: 定义需要遍历的参数
paramaeters = {'C': np.logspace(-3, 3, 7)}

# TODO:选择模型
lr = LogisticRegression()

# TODO:利用GridSearchCV
clf = GridSearchCV(lr, paramaeters, cv=5)
clf.fit(X_train, y)

# TODO:对Test_data进行分类
predict =clf.predict(X_test)
predict_prob = clf.predict_proba(X_test)
print(predict)
print(predict_prob)


import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt # for making figures
import os

# read in all the words
current_dir = os.getcwd()
words = open(current_dir+'/makemore/names.txt', 'r').read().splitlines()
# print(f"{words[:8]}")

# build the vocabulary of characters and mappings to/from integers
chars = sorted(list(set(''.join(words))))
stoi = {s:i+1 for i,s in enumerate(chars)}
stoi['.'] = 0
itos = {i:s for s,i in stoi.items()}
# print(itos)

# build the dataset
block_size = 3 # context length: how many characters do we take to predict the next one?

def build_dataset(words):
  X, Y = [], []
  for w in words:
    #print(w)
    context = [0] * block_size
    for ch in w + '.':
      ix = stoi[ch]
      X.append(context)
      Y.append(ix)
      #print(''.join(itos[i] for i in context), '--->', itos[ix])
      context = context[1:] + [ix] # crop and append

  X = torch.tensor(X)
  Y = torch.tensor(Y)
#   print(X.shape, Y.shape)
  return X, Y

import random
random.seed(42)
random.shuffle(words)
n1 = int(0.8*len(words))
n2 = int(0.9*len(words))

Xtr, Ytr = build_dataset(words[:n1])
Xdev, Ydev = build_dataset(words[n1:n2])
Xte, Yte = build_dataset(words[n2:])

g = torch.Generator().manual_seed(42) # for reproducibility
C = torch.randn((27, 10), generator=g)
W1 = torch.randn((30, 200), generator=g)
W2 = torch.randn((200, 27), generator=g)
parameters = [C, W1, W2]

for p in parameters:
  p.requires_grad = True

lri = []
lossi = []
stepi = []

batch = 32

for i in range(100):

  # minibatch construct
  ix = torch.randint(0, Xtr.shape[0], (batch,))

  # forward pass
  emb = C[Xtr[ix]] # (32, 3, 10)
  h = torch.tanh(emb.view(-1, 30) @ W1) # (32, 200)
  logits = h @ W2 # (32, 27)
  loss = F.cross_entropy(logits, Ytr[ix])
  #print(loss.item())

  # backward pass
  for p in parameters:
    p.grad = None
  loss.backward()

  # update
  lr = 0.1
  for p in parameters:
    p.data += -lr * p.grad

  # track stats
  #lri.append(lre[i])
  stepi.append(i)
  lossi.append(loss.item())

#print(loss.item())

plt.plot(stepi, lossi)
plt.show()

# sample from the model
g = torch.Generator().manual_seed(2147483647 + 10)

for _ in range(5):
    out = []
    context = [0] * block_size # initialize with all ...
    while True:
      emb = C[torch.tensor([context])] # (1,block_size,d)
      h = torch.tanh(emb.view(1, -1) @ W1)
      logits = h @ W2
      probs = F.softmax(logits, dim=1)
      ix = torch.multinomial(probs, num_samples=1, generator=g).item()
      context = context[1:] + [ix]
      out.append(ix)
      if ix == 0:
        break

    print(''.join(itos[i] for i in out))


#! /usr/bin/env python

import argparse

parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description='Linearly normalize intensity to between 0 and 255')

parser.add_argument("input_spec", type=str, help="Input specification")
parser.add_argument("out_version", type=str, help="Output image version")
args = parser.parse_args()

import sys
import os

sys.path.append(os.environ['REPO_DIR'] + '/utilities')
from utilities2015 import *
from data_manager import *
from metadata import *
from distributed_utilities import *
from learning_utilities import *

input_spec = load_ini(args.input_spec)
image_name_list = input_spec['image_name_list']
stack = input_spec['stack']
prep_id = input_spec['prep_id']
if prep_id == 'None':
    prep_id = None
resol = input_spec['resol']
version = input_spec['version']
if version == 'None':
    version = None

from scipy.ndimage.interpolation import map_coordinates
from skimage.exposure import rescale_intensity, adjust_gamma
from skimage.transform import rotate

# for section in set(metadata_cache['valid_sections_all'][stack]) - set(metadata_cache['valid_sections'][stack]):
# for section in metadata_cache['valid_sections'][stack]:

for image_name in image_name_list:

#     print "Section", section

    t = time.time()

    img = DataManager.load_image_v2(stack=stack, prep_id=prep_id, fn=image_name, version=version, resol=resol)

    sys.stderr.write('Load image: %.2f seconds.\n' % (time.time() - t))

    t = time.time()
    tb_mask = DataManager.load_thumbnail_mask_v3(stack=stack, prep_id=None, fn=image_name)
#     raw_mask = rescale_by_resampling(tb_mask, new_shape=(img.shape[1], img.shape[0]))
    raw_mask = resize(tb_mask, img.shape) > .5

    save_data(raw_mask,
          DataManager.get_image_filepath_v2(stack=stack, prep_id=prep_id, fn=image_name, version='mask', resol=resol, ext='bp'),
          upload_s3=False)

    sys.stderr.write('Rescale mask: %.2f seconds.\n' % (time.time() - t))

    t = time.time()

    mean_std_all_regions = []
    cx_cy_all_regions = []
    region_size = 5000
    region_spacing = 3000
#     for cx in range(region_size/2, img.shape[1]-region_size/2+1, region_spacing):
#         for cy in range(region_size/2, img.shape[0]-region_size/2+1, region_spacing):
    for cx in range(0, img.shape[1], region_spacing):
        for cy in range(0, img.shape[0], region_spacing):
            region = img[max(cy-region_size/2, 0):min(cy+region_size/2+1, img.shape[0]-1),
                         max(cx-region_size/2, 0):min(cx+region_size/2+1, img.shape[1]-1)]
            region_mask = raw_mask[max(cy-region_size/2, 0):min(cy+region_size/2+1, img.shape[0]-1),
                                   max(cx-region_size/2, 0):min(cx+region_size/2+1, img.shape[1]-1)]
            if np.count_nonzero(region_mask) == 0:
                continue
            mean_std_all_regions.append((region[region_mask].mean(), region[region_mask].std()))
            cx_cy_all_regions.append((cx, cy))

    sys.stderr.write('Compute mean/std for sample regions: %.2f seconds.\n' % (time.time() - t))

    t = time.time()
    mean_map = resample_scoremap(sparse_scores=np.array(mean_std_all_regions)[:,0],
                             sample_locations=cx_cy_all_regions,
                             gridspec=(region_size, region_spacing, img.shape[1], img.shape[0], (0,0)),
                            downscale=4,
                                 interpolation_order=2)

    sys.stderr.write('Interpolate mean map: %.2f seconds.\n' % (time.time() - t)) #10s

    t = time.time()
    mean_map = rescale_by_resampling(mean_map, new_shape=(img.shape[1], img.shape[0]))
    sys.stderr.write('Scale up mean map: %.2f seconds.\n' % (time.time() - t)) #30s

    t = time.time()
    std_map = resample_scoremap(sparse_scores=np.array(mean_std_all_regions)[:,1],
                             sample_locations=cx_cy_all_regions,
                             gridspec=(region_size, region_spacing, img.shape[1], img.shape[0], (0,0)),
                            downscale=4,
                               interpolation_order=2)
    sys.stderr.write('Interpolate std map: %.2f seconds.\n' % (time.time() - t)) #10s

    t = time.time()
    std_map = rescale_by_resampling(std_map, new_shape=(img.shape[1], img.shape[0]))
    sys.stderr.write('Scale up std map: %.2f seconds.\n' % (time.time() - t)) #30s

    # Save mean/std results.

    fp = DataManager.get_intensity_normalization_result_filepath(what='region_centers', stack=stack, fn=image_name)
    create_parent_dir_if_not_exists(fp)
    np.savetxt(fp, cx_cy_all_regions)

    fp = DataManager.get_intensity_normalization_result_filepath(what='mean_std_all_regions', stack=stack, fn=image_name)
    create_parent_dir_if_not_exists(fp)
    np.savetxt(fp, mean_std_all_regions)

    fp = DataManager.get_intensity_normalization_result_filepath(what='mean_map', stack=stack, fn=image_name)
    create_parent_dir_if_not_exists(fp)
    bp.pack_ndarray_file(mean_map.astype(np.float16), fp)

    fp = DataManager.get_intensity_normalization_result_filepath(what='std_map', stack=stack, fn=image_name)
    create_parent_dir_if_not_exists(fp)
    bp.pack_ndarray_file(std_map.astype(np.float16), fp)

    # Export normalized image.

    t = time.time()
    raw_mask = raw_mask & (std_map > 0)
    img_normalized = np.zeros(img.shape, np.float32)
    img_normalized[raw_mask] = (img[raw_mask] - mean_map[raw_mask]) / std_map[raw_mask]
    sys.stderr.write('Normalize: %.2f seconds.\n' % (time.time() - t)) #30s

    t = time.time()
    # FIX THIS! THIS only save uint16, not float16. Need to save as bp instead.
#     img_fp = DataManager.get_image_filepath_v2(stack=stack, prep_id=None, version='NtbNormalizedFloat', resol='down8', section=section, )
#     create_parent_dir_if_not_exists(img_fp)
#     imsave(img_fp, img_normalized[::8, ::8].astype(np.float16))
    save_data(img_normalized.astype(np.float16),
              DataManager.get_intensity_normalization_result_filepath(what='normalized_float_map', stack=stack, fn=image_name),
             upload_s3=False)
    sys.stderr.write('Save float version: %.2f seconds.\n' % (time.time() - t)) #30s

#     t = time.time()
#     img_normalized_uint8 = rescale_intensity_v2(img_normalized, -1, 6)
#     sys.stderr.write('Rescale to uint8: %.2f seconds.\n' % (time.time() - t)) #30s

#     t = time.time()
#     img_fp = DataManager.get_image_filepath_v2(stack=stack, prep_id=None, version='NtbNormalized', resol='raw', section=section)
#     create_parent_dir_if_not_exists(img_fp)
#     imsave(img_fp, img_normalized_uint8)
#     sys.stderr.write('Save uint8 version: %.2f seconds.\n' % (time.time() - t)) #30s

    # Export histogram.

    plt.hist(img_normalized[raw_mask].flatten(), bins=100, log=True);
    fp = DataManager.get_intensity_normalization_result_filepath(what='float_histogram_png', stack=stack, fn=image_name)
    create_parent_dir_if_not_exists(fp)
    plt.savefig(fp)
    plt.close();

#     hist_fp = DataManager.get_intensity_normalization_result_filepath(what='float_histogram', stack=stack, section=section)
#     create_parent_dir_if_not_exists(hist_fp)

#     hist, bin_edges = np.histogram(img_normalized[valid_mask].flatten(), bins=np.arange(0,201,5));

#     plt.bar(bin_edges[:-1], np.log(hist));
#     plt.xticks(np.arange(0, 200, 20), np.arange(0, 200, 20));
#     plt.xlabel('Normalized pixel value (float)');
#     plt.title(metadata_cache['sections_to_filenames'][stack][section])

#     plt.savefig(hist_fp)
#     plt.close();

gamma_map = img_as_ubyte(adjust_gamma(np.arange(0, 256, 1) / 255., 8.))

low = -2.
high = 50.

for image_name in image_name_list:

    img_normalized = load_data(
              DataManager.get_intensity_normalization_result_filepath(what='normalized_float_map', stack=stack, fn=image_name),
             download_s3=False)

    t = time.time()
    img_normalized_uint8 = rescale_intensity_v2(img_normalized, low, high)
    sys.stderr.write('Rescale to uint8: %.2f seconds.\n' % (time.time() - t))

    t = time.time()
    raw_mask = load_data(DataManager.get_image_filepath_v2(stack=stack, prep_id=prep_id, fn=image_name, version='mask', resol=resol, ext='bp'),
                        download_s3=False)
    img_normalized_uint8[~raw_mask] = 0
    sys.stderr.write('Load mask: %.2f seconds.\n' % (time.time() - t))

    img = 255 - img_normalized_uint8
    save_data(gamma_map[img],
              DataManager.get_image_filepath_v2(stack=stack, prep_id=prep_id, fn=image_name, version=args.out_version, resol=resol),
             upload_s3=False)


import argparse
import glob
import multiprocessing as mp
import os
import shutil
import time
import cv2
import tqdm
import numpy as np

from detectron2.config import get_cfg
from partseg import add_partseg_config
from detectron2.data.detection_utils import read_image
from detectron2.utils.logger import setup_logger

from detectron2.engine.defaults import DefaultPredictor
from detectron2.utils.visualizer import ColorMode, Visualizer

# constants
WINDOW_NAME = "COCO detections"

def setup_cfg(args):
    # load config from file and command-line arguments
    cfg = get_cfg()
    add_partseg_config(cfg)
    cfg.merge_from_file(args.config_file)
    cfg.merge_from_list(args.opts)

    # Set score_threshold for builtin models
    cfg.MODEL.RETINANET.SCORE_THRESH_TEST = args.confidence_threshold
    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = args.confidence_threshold
    cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = (
        args.confidence_threshold
    )

    # load weights from the default path
    if not args.custom_weights:
        default_weights = os.path.join(cfg.OUTPUT_DIR, "model_final.pth")
        if os.path.exists(default_weights):
            print("Use the default weights.")
        cfg.MODEL.WEIGHTS = default_weights

    cfg.freeze()
    return cfg

def get_parser():
    parser = argparse.ArgumentParser(description="Detectron2 demo for builtin models")
    parser.add_argument(
        "--config-file",
        default="configs/mask_rcnn_R_50_FPN_chair.yaml",
        metavar="FILE",
        help="path to config file",
    )
    parser.add_argument(
        "--root-dir",
        type=str,
        help="root directory",
        default="datasets/images/test/chair",
    )
    parser.add_argument(
        "--output-dir",
        type=str,
        help="path to output",
        default="datasets/predictions/test/chair",
    )
    parser.add_argument("--shape-list-fn", type=str, help="path to shape list")
    parser.add_argument("--start", type=int, default=0)
    parser.add_argument("--end", type=int, default=None)
    parser.add_argument(
        "--confidence-threshold",
        type=float,
        default=0.5,
        help="Minimum score for instance predictions to be shown",
    )
    parser.add_argument(
        "--custom-weights", action="store_true", help="whether to use custom weights"
    )
    parser.add_argument(
        "--include-image", action="store_true", help="whether to include input images"
    )
    parser.add_argument("--vis", action="store_true")
    parser.add_argument("--with-score", action="store_true")
    parser.add_argument(
        "--opts",
        help="Modify config options using the command-line 'KEY VALUE' pairs",
        default=[],
        nargs=argparse.REMAINDER,
    )
    return parser

if __name__ == "__main__":
    mp.set_start_method("spawn", force=True)
    args = get_parser().parse_args()
    setup_logger(name="fvcore")
    logger = setup_logger()
    logger.info("Arguments: " + str(args))

    cfg = setup_cfg(args)
    predictor = DefaultPredictor(cfg)

    root_dir = args.root_dir
    if args.shape_list_fn:
        with open(args.shape_list_fn, "r") as f:
            image_ids = f.readlines()
        image_ids = [x.strip() for x in image_ids]
    else:
        image_ids = os.listdir(root_dir)
        image_ids = [x for x in image_ids if os.path.isdir(os.path.join(root_dir, x))]
    image_ids = sorted(image_ids)
    image_ids = image_ids[args.start : args.end]

    for image_id in tqdm.tqdm(image_ids[:]):
        file_name = os.path.join(root_dir, image_id, "img.png")
        image = read_image(file_name, format="BGR")
        predictions = predictor(image)
        instances = predictions["instances"].to("cpu")
        pred_masks = instances.pred_masks.numpy()  # [N, H, W]
        pred_masks = (pred_masks * 255).astype(np.uint8)
        # for pred_mask in pred_masks:
        #     cv2.imshow('mask', pred_mask)
        #     if cv2.waitKey(0) == 27:
        #         break  # esc to quit

        output_dir = os.path.join(args.output_dir, image_id)
        if not os.path.isdir(output_dir):
            os.makedirs(output_dir)

        # save
        for idx, pred_mask in enumerate(pred_masks):
            output_file_name = os.path.join(output_dir, f"partmask_{idx}.png")
            cv2.imwrite(output_file_name, pred_mask)

        # Convert image from OpenCV BGR format to Matplotlib RGB format.
        image_rgb = image[:, :, ::-1]
        visualizer = Visualizer(image_rgb, None, instance_mode=ColorMode.IMAGE)
        if not args.with_score:
            # workaround to suppress visualizing scores
            instances.remove("scores")
        vis_output = visualizer.draw_instance_predictions(predictions=instances)

        if args.vis:
            cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
            cv2.imshow(WINDOW_NAME, vis_output.get_image()[:, :, ::-1])
            if cv2.waitKey(0) == 27:
                break  # esc to quit
        else:
            output_file_name = os.path.join(output_dir, f"partmask_all.png")
            vis_output.save(output_file_name)

        if args.include_image:
            shutil.copy(file_name, os.path.join(output_dir, "img.png"))


import os
from flask import Response,Flask, request
from flask_cors import CORS
from insgraph import util, instagram

def create_app(test_config=None):
    """Create and configure an instance of the Flask application."""
    app = Flask(__name__, instance_relative_config=True)
    print("zhuangjb flask start.....:"+__name__)
    CORS(app)
    app.config.from_mapping(
        # a default secret that should be overridden by instance config
        SECRET_KEY='dev',
        # store the database in the instance folder
        DATABASE=os.path.join(app.instance_path, 'insgraph.sqlite'),
    )

    if test_config is None:
        # load the instance config, if it exists, when not testing
        app.config.from_pyfile('config.py', silent=True)
    else:
        # load the test config if passed in
        app.config.update(test_config)

    # ensure the instance folder exists
    try:
        os.makedirs(app.instance_path)
    except OSError:
        pass

    @app.route('/hello')
    def hello():
        return 'Hello, World!'

    @app.before_request
    def option_replay():
        if request.method =='OPTIONS':
            resp = Response('')
            print('xxx')
            resp.headers['Access-Control-Allow-Origin'] = '*'
            resp.headers['Access-Control-Allow-Headers'] = '*'
            resp.headers['Access-Control-Request-Method'] = request.headers['Access-Control-Request-Method']
            return resp

    # @app.after_request
    # def set_allow_origin(resp):
    #     h = resp.headers
    #     if request.method != 'OPTIONS' and 'Origin' in request.headers:
    #         h['Access-Control-Allow-Origin'] = request.headers['Origin']

    # register the database commands
    from insgraph import db
    db.init_app(app)

    # apply the blueprints to the app
    from insgraph import auth, user,case
    app.register_blueprint(auth.bp)
    app.register_blueprint(user.bp)
    app.register_blueprint(case.bp)
    app.register_blueprint(instagram.bp)

    # make url_for('index') == url_for('blog.index')
    # in another app, you might define a separate main index here with
    # app.route, while giving the blog blueprint a url_prefix, but for
    # the tutorial the blog will be the main index
    app.add_url_rule('/', endpoint='index')

    return app


import os

import pytest
import logging
import cocotb

from cocotb.clock import Clock, Timer
from cocotb.binary import BinaryValue
from cocotb.runner import get_runner
from cocotb.triggers import FallingEdge

from cocotbext.uart import UartSource, UartSink

src_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
tests_dir = os.path.dirname(os.path.abspath(__file__))
sim_build = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sim_build", "soc")

@cocotb.test()
async def check_uart_recv(dut):
    """ Test that UART is working """
    clock = Clock(dut.clk, 10, units="ns")  # Create a 10us period clock on port clk
    cocotb.start_soon(clock.start())  # Start the clock

    log = logging.getLogger(f"check_uart_recv")

    dut.RESET.value = BinaryValue('1')
    await FallingEdge(dut.clk)

    dut.RESET.value = BinaryValue('0')
    await FallingEdge(dut.clk)

    rxd = UartSource(dut.RXD, baud=115200, bits=8)
    txd = UartSink(dut.TXD, baud=115200, bits=8)

    await rxd.write(b'ABCDE')

    for i in range(int(1e9/115200/10) * 10):
        await FallingEdge(dut.clk)
    val = await txd.read()
    assert val == b'E'

"""
    LI(gp, 32'h0200_0000);
    ADD(x12,x0,x0);
    ADDI(x2,x0,65);
    Label(L0_);
      LW(x12, gp, 8);
      BNE(x12, x2, LabelRef(L0_));
    SW(x12, gp, 8);
    EBREAK();
"""
@pytest.mark.skip(reason="no way of currently testing this")
def test_runner():
    verilog_sources = [os.path.join(src_dir, "main", "soc.sv")]
    sim = os.getenv("SIM", "icarus")
    runner = get_runner(sim)()
    os.makedirs(os.path.abspath(sim_build), exist_ok=True)
    with open(os.path.abspath(os.path.join(sim_build, "cmd.f")), 'w') as cmd:
        cmd.write('+timescale+1ns/1ps')
    runner.build(
        verilog_sources=verilog_sources,
        toplevel="soc",
        defines=["DEFINE=4", "BENCH=1"],
        includes=[os.path.join(src_dir, "main")],
        extra_args=[
            '-s', 'soc',
            '-f', os.path.abspath(os.path.join(sim_build, "cmd.f"))
        ],
        build_dir=sim_build
    )

    runner.test(
        python_search=[tests_dir],
        toplevel="soc",
        py_module="test_soc",
    )


"""
Given a universal mesh, record the placements of guide nodes as it relative to
universal mesh. And then repoisition guides to that relative position should
the universal mesh change from character to character.

from mgear.shifter import relativeGuidePlacement
reload(relativeGuidePlacement)

Execute the following chunk to record initial placement ----------------------
relativeGuidePlacement.exportGuidePlacement(filepath="Y:/tmp/exampleFile.json",
                                            skip_strings=["hair"])

Load new universal guide mesh with new proportions

Execute the following lines to move the guides to their new position ---------
relativeGuidePlacement.importGuidePlacement(filepath="Y:/tmp/exampleFile.json")

Attributes:
    GUIDE_ROOT (str): name of the root guide node
    SKIP_CONTAINS (list): nodes to skip if they contain the string
    SKIP_CRAWL_NODES (list): nodes to skip crawling hierarchy
    SKIP_NODETYPES (list): skip the query of certain node types
    SKIP_PLACEMENT_NODES (TYPE): nodes to skip updating their positions
    SKIP_SUFFIX (list): skip if node ends with
    UNIVERSAL_MESH_NAME (str): default name of the universal mesh
"""
# python
import json
import math

# dcc
import maya.cmds as mc
import pymel.core as pm
import maya.OpenMaya as om

# mgear
from mgear.core import utils
from mgear.core import vector
from mgear.core import transform
from mgear.core import meshNavigation

# constants -------------------------------------------------------------------

# Designate the root of the hierarchy to crawl
GUIDE_ROOT = "guide"

# Nodes to avoid checking the hierarchy
DEFAULT_SKIP_CRAWL_NODES = ("controllers_org",
                            "spineUI_C0_root",
                            "faceUI_C0_root",
                            "legUI_R0_root",
                            "armUI_L0_root",
                            "legUI_L0_root",
                            "armUI_R0_root")

# nodes that will not have their positions updated
DEFAULT_SKIP_PLACEMENT_NODES = ("controllers_org",
                                "global_C0_root",
                                "spineUI_C0_root",
                                "faceUI_C0_root",
                                "legUI_R0_root",
                                "armUI_L0_root",
                                "legUI_L0_root",
                                "armUI_R0_root")

try:
    SKIP_CRAWL_NODES
    SKIP_PLACEMENT_NODES
except NameError:
    SKIP_CRAWL_NODES = list(DEFAULT_SKIP_CRAWL_NODES)
    SKIP_PLACEMENT_NODES = list(DEFAULT_SKIP_PLACEMENT_NODES)

# skip the node if it even contains the characters in the list
# eg SKIP_CONTAINS = ["hair"]
SKIP_CONTAINS = []

# Avoid nodes of a specified suffix
SKIP_SUFFIX = ["sizeRef", "crv", "crvRef", "blade"]

# Types of nodes to avoid
SKIP_NODETYPES = ["aimConstraint", "pointConstraint", "parentConstraint"]

UNIVERSAL_MESH_NAME = "skin_geo_setup"

# general functions -----------------------------------------------------------

def crawlHierarchy(parentNode,
                   ordered_hierarchy,
                   skip_crawl_nodes,
                   skip_strings=None):
    """recursive function to crawl a hierarchy of nodes to return decendents

    Args:
        parentNode (str): node to query
        ordered_hierarchy (str): list to continuesly pass itself
        skip_crawl_nodes (list): nodes to skip crawl
    """
    if not skip_strings:
        skip_strings = []
    for node in mc.listRelatives(parentNode, type="transform") or []:
        if node in skip_crawl_nodes or node in ordered_hierarchy:
            continue
        if node.endswith(tuple(SKIP_SUFFIX)):
            continue
        if mc.objectType(node) in SKIP_NODETYPES:
            continue
        if [True for skip_str in skip_strings
                if skip_str.lower() in node.lower()]:
            continue
        ordered_hierarchy.append(node)
        crawlHierarchy(node,
                       ordered_hierarchy,
                       skip_crawl_nodes,
                       skip_strings=skip_strings)

def getPostionFromLoop(vertList):
    """Get the center position from the list of edge ids provided

    Args:
        vertList (list): list of edge ids

    Returns:
        list: of translate XYZ, world space
    """
    bb = mc.exactWorldBoundingBox(vertList)
    pos = ((bb[0] + bb[3]) / 2, (bb[1] + bb[4]) / 2, (bb[2] + bb[5]) / 2)
    return pos

def getVertMatrix(closestVert):
    """create a matrix from the closestVert and the normals of the surrounding
    faces for later comparison

    Args:
        node (str): guide node to query
        closestVert (str): closest vert to guide

    Returns:
        list: of matrices
    """
    closestVert = pm.PyNode(closestVert)
    faces = closestVert.connectedFaces()
    normalVector = faces.getNormal("world")
    pm.select(faces)
    faces_str = mc.ls(sl=True, fl=True)
    pm.select(cl=True)
    face_pos = pm.dt.Vector(getPostionFromLoop(faces_str))
    normal_rot = getOrient([normalVector.x, normalVector.y, normalVector.z],
                           [0, 1, 0],
                           ro=0)
    orig_ref_matrix = pm.dt.TransformationMatrix()
    orig_ref_matrix.setTranslation(face_pos, pm.dt.Space.kWorld)
    orig_ref_matrix.setRotation(normal_rot)

    return orig_ref_matrix

def getOrient(normal, tangent, ro=0):
    """convert normal direction into euler rotations

    Args:
        normal (list): of nomel values
        ro (int, optional): rotate order

    Returns:
        list: of euler rotations
    """
    kRotateOrders = [om.MEulerRotation.kXYZ, om.MEulerRotation.kYZX,
                     om.MEulerRotation.kZXY, om.MEulerRotation.kXZY,
                     om.MEulerRotation.kYXZ, om.MEulerRotation.kZYX, ]
    cross = [normal[1] * tangent[2] - normal[2] * tangent[1],
             normal[2] * tangent[0] - normal[0] * tangent[2],
             normal[0] * tangent[1] - normal[1] * tangent[0]]
    tMatrix = normal + [0] + tangent + [0] + cross + [0, 0, 0, 0, 1]
    mMatrix = om.MMatrix()
    om.MScriptUtil.createMatrixFromList(tMatrix, mMatrix)
    tmMatrix = om.MTransformationMatrix(mMatrix)
    rotate = tmMatrix.eulerRotation().reorder(kRotateOrders[ro])
    RAD_to_DEG = (180 / math.pi)
    return [rotate[0] * RAD_to_DEG,
            rotate[1] * RAD_to_DEG,
            rotate[2] * RAD_to_DEG]

def getRepositionMatrix(node_matrix,
                        orig_ref_matrix,
                        mr_orig_ref_matrix,
                        closestVerts):
    """Get the delta matrix from the original position and multiply by the
    new vert position. Add the rotations from the face normals.

    Args:
        node_matrix (pm.dt.Matrix): matrix of the guide
        orig_ref_matrix (pm.dt.Matrix): matrix from the original vert position
        closestVerts (str): name of the closest vert

    Returns:
        mmatrix: matrix of the new offset position, worldSpace
    """
    current_vert = pm.PyNode(closestVerts[0])
    mr_current_vert = pm.PyNode(closestVerts[1])
    current_length = vector.getDistance(current_vert.getPosition("world"),
                                        mr_current_vert.getPosition("world"))

    orig_length = vector.getDistance(orig_ref_matrix.translate,
                                     mr_orig_ref_matrix.translate)
    orig_center = vector.linearlyInterpolate(orig_ref_matrix.translate,
                                             mr_orig_ref_matrix.translate)
    orig_center_matrix = pm.dt.Matrix()
    # orig_center_matrix.setTranslation(orig_center, pm.dt.Space.kWorld)
    orig_center_matrix = transform.setMatrixPosition(
        orig_center_matrix, orig_center)

    current_center = vector.linearlyInterpolate(
        current_vert.getPosition("world"),
        mr_current_vert.getPosition("world"))

    length_percentage = 1
    if current_length != 0 or orig_length != 0:
        length_percentage = current_length / orig_length
    # refPosition_matrix = pm.dt.TransformationMatrix()
    refPosition_matrix = pm.dt.Matrix()
    # refPosition_matrix.setTranslation(current_center, pm.dt.Space.kWorld)
    refPosition_matrix = transform.setMatrixPosition(
        refPosition_matrix, current_center)
    deltaMatrix = node_matrix * orig_center_matrix.inverse()
    deltaMatrix = deltaMatrix * length_percentage
    deltaMatrix = transform.setMatrixScale(deltaMatrix)
    refPosition_matrix = deltaMatrix * refPosition_matrix

    return refPosition_matrix

def getRepositionMatrixSingleRef(node_matrix,
                                 orig_ref_matrix,
                                 mr_orig_ref_matrix,
                                 closestVerts):
    """Get the delta matrix from the original position and multiply by the
    new vert position. Add the rotations from the face normals.

    Args:
        node_matrix (pm.dt.Matrix): matrix of the guide
        orig_ref_matrix (pm.dt.Matrix): matrix from the original vert position
        closestVerts (str): name of the closest vert

    Returns:
        mmatrix: matrix of the new offset position, worldSpace
    """
    closestVerts = pm.PyNode(closestVerts[0])
    faces = closestVerts.connectedFaces()
    normalVector = faces.getNormal("world")
    pm.select(faces)
    faces_str = mc.ls(sl=True, fl=True)
    pm.select(cl=True)
    face_pos = pm.dt.Vector(getPostionFromLoop(faces_str))
    normal_rot = getOrient([normalVector.x, normalVector.y, normalVector.z],
                           [0, 1, 0],
                           ro=0)
    refPosition_matrix = pm.dt.TransformationMatrix()
    refPosition_matrix.setTranslation(face_pos, pm.dt.Space.kWorld)
    refPosition_matrix.setRotation(normal_rot)

    deltaMatrix = node_matrix * orig_ref_matrix.inverse()
    refPosition_matrix = deltaMatrix * refPosition_matrix

    return refPosition_matrix

@utils.viewport_off
@utils.one_undo
def getGuideRelativeDictionaryLegacy(mesh, guideOrder):
    """create a dictionary of guide:[[shape.vtx[int]], relativeMatrix]

    Args:
        mesh (string): name of the mesh
        guideOrder (list): the order to query the guide hierarchy

    Returns:
        dictionary: create a dictionary of guide:[[edgeIDs], relativeMatrix]
    """
    relativeGuide_dict = {}
    mesh = pm.PyNode(mesh)
    for guide in guideOrder:
        guide = pm.PyNode(guide)
        # slow function A
        clst_vert = meshNavigation.getClosestVertexFromTransform(mesh, guide)
        vertexIds = [clst_vert.name()]
        # slow function B
        orig_ref_matrix = getVertMatrix(clst_vert.name())
        #  --------------------------------------------------------------------
        a_mat = guide.getMatrix(worldSpace=True)

        mm = ((orig_ref_matrix - a_mat) * -1) + a_mat
        pos = mm[3][:3]

        mr_vert = meshNavigation.getClosestVertexFromTransform(mesh, pos)
        mr_orig_ref_matrix = getVertMatrix(mr_vert.name())
        vertexIds.append(mr_vert.name())

        node_matrix = guide.getMatrix(worldSpace=True)
        relativeGuide_dict[guide.name()] = [vertexIds,
                                            node_matrix.get(),
                                            orig_ref_matrix.get(),
                                            mr_orig_ref_matrix.get()]
    mc.select(cl=True)
    return relativeGuide_dict

@utils.viewport_off
@utils.one_undo
def yieldGuideRelativeDictionary(mesh, guideOrder, relativeGuide_dict):
    """create a dictionary of guide:[[shape.vtx[int]], relativeMatrix]

    Args:
        mesh (string): name of the mesh
        guideOrder (list): the order to query the guide hierarchy

    Returns:
        dictionary: create a dictionary of guide:[[edgeIDs], relativeMatrix]
    """
    for guide in guideOrder:
        guide = pm.PyNode(guide)
        # slow function A
        clst_vert = meshNavigation.getClosestVertexFromTransform(mesh, guide)
        vertexIds = [clst_vert.name()]
        # slow function B
        orig_ref_matrix = getVertMatrix(clst_vert.name())
        #  --------------------------------------------------------------------
        a_mat = guide.getMatrix(worldSpace=True)

        mm = ((orig_ref_matrix - a_mat) * -1) + a_mat
        pos = mm[3][:3]

        mr_vert = meshNavigation.getClosestVertexFromTransform(mesh, pos)
        mr_orig_ref_matrix = getVertMatrix(mr_vert.name())
        vertexIds.append(mr_vert.name())

        node_matrix = guide.getMatrix(worldSpace=True)
        relativeGuide_dict[guide.name()] = [vertexIds,
                                            node_matrix.get(),
                                            orig_ref_matrix.get(),
                                            mr_orig_ref_matrix.get()]
        yield relativeGuide_dict

@utils.viewport_off
@utils.one_undo
def getGuideRelativeDictionary(mesh, guideOrder):
    """create a dictionary of guide:[[shape.vtx[int]], relativeMatrix]

    Args:
        mesh (string): name of the mesh
        guideOrder (list): the order to query the guide hierarchy

    Returns:
        dictionary: create a dictionary of guide:[[edgeIDs], relativeMatrix]
    """
    relativeGuide_dict = {}
    mesh = pm.PyNode(mesh)
    for result in yieldGuideRelativeDictionary(
            mesh, guideOrder, relativeGuide_dict):
        pass
    return relativeGuide_dict

@utils.viewport_off
@utils.one_undo
def updateGuidePlacementLegacy(guideOrder, guideDictionary):
    """update the guides based on new universal mesh, in the provided order

    Args:
        guideOrder (list): of the hierarchy to crawl
        guideDictionary (dictionary): dict of the guide:edge, matrix position
    """
    for guide in guideOrder:
        if guide not in guideDictionary or not mc.objExists(guide):
            continue
        elif guide in SKIP_PLACEMENT_NODES:
            continue
        (vertexIds,
         node_matrix,
         orig_ref_matrix,
         mr_orig_ref_matrix) = guideDictionary[guide]

        guideNode = pm.PyNode(guide)
        repoMatrix = getRepositionMatrix(pm.dt.Matrix(node_matrix),
                                         pm.dt.Matrix(orig_ref_matrix),
                                         pm.dt.Matrix(mr_orig_ref_matrix),
                                         vertexIds)
        guideNode.setMatrix(repoMatrix, worldSpace=True, preserve=True)

@utils.viewport_off
@utils.one_undo
def yieldUpdateGuidePlacement(guideOrder, guideDictionary):
    """update the guides based on new universal mesh, in the provided order

    Args:
        guideOrder (list): of the hierarchy to crawl
        guideDictionary (dictionary): dict of the guide:edge, matrix position
    """
    for guide in guideOrder:
        if guide not in guideDictionary or not mc.objExists(guide):
            continue
        elif guide in SKIP_PLACEMENT_NODES:
            continue
        (vertexIds,
         node_matrix,
         orig_ref_matrix,
         mr_orig_ref_matrix) = guideDictionary[guide]

        repoMatrix = getRepositionMatrix(pm.dt.Matrix(node_matrix),
                                         pm.dt.Matrix(orig_ref_matrix),
                                         pm.dt.Matrix(mr_orig_ref_matrix),
                                         vertexIds)
        yield repoMatrix

@utils.viewport_off
@utils.one_undo
def updateGuidePlacement(guideOrder, guideDictionary, reset_scale=False):
    """update the guides based on new universal mesh, in the provided order

    Args:
        guideOrder (list): of the hierarchy to crawl
        guideDictionary (dictionary): dict of the guide:edge, matrix position
    """
    updateGen = yieldUpdateGuidePlacement(guideOrder, guideDictionary)
    for guide in guideOrder:
        if guide not in guideDictionary or not mc.objExists(guide):
            continue
        elif guide in SKIP_PLACEMENT_NODES:
            continue
        guideNode = pm.PyNode(guide)
        scl = guideNode.getScale()
        repoMatrix = next(updateGen)
        guideNode.setMatrix(repoMatrix, worldSpace=True, preserve=True)
        if reset_scale:
            guideNode.setScale([1, 1, 1])
        else:
            guideNode.setScale(scl)
        yield True

# ==============================================================================
# Data export, still testing
# ==============================================================================
def _importData(filepath):
    try:
        with open(filepath, 'r') as f:
            data = json.load(f)
            return data
    except Exception as e:
        print(e)

def _exportData(data, filepath):
    try:
        with open(filepath, 'w') as f:
            json.dump(data, f, sort_keys=False, indent=4)
    except Exception as e:
        print(e)

def exportGuidePlacement(filepath=None,
                         reference_mesh=UNIVERSAL_MESH_NAME,
                         root_node=GUIDE_ROOT,
                         skip_crawl_nodes=SKIP_CRAWL_NODES,
                         skip_strings=[]):
    """Export the position of the supplied root node to a file.

    Args:
        filepath (str, optional): path to export too
        reference_mesh (str, optional): mesh to query verts
        root_node (str, optional): name of node to query against
        skip_crawl_nodes (list, optional): of nodes not to crawl
        skip_strings (list, optional): strings to check to skip node

    Returns:
        list: dict, list, str
    """
    if filepath is None:
        filepath = pm.fileDialog2(fileMode=0,
                                  startingDirectory="/",
                                  fileFilter="Export position(*.json)")
        if filepath:
            filepath = filepath[0]
    (relativeGuide_dict,
     ordered_hierarchy) = recordInitialGuidePlacement(
        reference_mesh=reference_mesh,
        root_node=root_node,
        skip_crawl_nodes=skip_crawl_nodes,
        skip_strings=skip_strings)
    data = {}
    data["relativeGuide_dict"] = relativeGuide_dict
    data["ordered_hierarchy"] = ordered_hierarchy
    _exportData(data, filepath)
    print("Guide position exported: {}".format(filepath))
    return relativeGuide_dict, ordered_hierarchy, filepath

@utils.one_undo
def importGuidePlacement(filepath):
    """import the position from the provided file

    Args:
        filepath (str): file to the json
        referenceMesh (str, optional): name of mesh to compare against
    """
    data = _importData(filepath)
    updateGuidePlacement(data["ordered_hierarchy"], data["relativeGuide_dict"])
    return data["relativeGuide_dict"], data["ordered_hierarchy"]

def recordInitialGuidePlacement(reference_mesh=UNIVERSAL_MESH_NAME,
                                root_node=GUIDE_ROOT,
                                skip_crawl_nodes=SKIP_CRAWL_NODES,
                                skip_strings=None):
    """convenience function for retrieving a dict of position

    Args:
        reference_mesh (str, optional): the mesh to query against
        root_node (str, optional): root node to crawl
        skip_crawl_nodes (list, optional): of nodes to avoid
        skip_strings (list, optional): of strings to check if skip

    Returns:
        dict, list: dict of positions, list of ordered nodes
    """
    ordered_hierarchy = []
    relativeGuide_dict = {}
    crawlHierarchy(root_node,
                   ordered_hierarchy,
                   skip_crawl_nodes,
                   skip_strings=skip_strings)
    relativeGuide_dict = getGuideRelativeDictionary(reference_mesh,
                                                    ordered_hierarchy)
    return relativeGuide_dict, ordered_hierarchy


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_pyredatam

Tests for `pyredatam` module.
"""

from __future__ import unicode_literals
import unittest
import nose

import pyredatam
import queries

class RedatamTestCase(unittest.TestCase):

    def test_arealist_query(self):

        # Test case AREALIST1
        area_level = "FRAC"
        variables = "PERSONA.CONDACT"

        area_filter = {"PROV": ["02", "03"]}
        universe_filter = "1 = 1"
        title = "El titulo"

        query = pyredatam.arealist_query(area_level, variables, area_filter,
                                         universe_filter, title)
        self.assertEqual(query, queries.AREALIST1.strip())

        # Test case AREALIST2
        variables = ["PERSONA.CONDACT"]
        query = pyredatam.arealist_query(area_level, variables)
        self.assertEqual(query, queries.AREALIST2.strip())

        # Test case AREALIST3
        area_filter = {"PROV": "02"}
        query = pyredatam.arealist_query(area_level, variables, area_filter)
        self.assertEqual(query, queries.AREALIST3.strip())

    def test_counter_query(self):

        # Test case COUNTER1
        area_level = "RADIO"
        entity_count = "PERSONA"

        area_filter = {"PROV": "02"}
        universe_filter = "1 = 1"
        title = "El titulo"

        query = pyredatam.counter_query(area_level, entity_count, area_filter,
                                        universe_filter, title)
        self.assertEqual(query, queries.COUNTER1.strip())

        # Test case COUNTER2
        area_level = "DPTO"
        entity_count = "FRAC"
        incl_area_name = True
        incl_total = True
        query = pyredatam.counter_query(area_level, entity_count, area_filter,
                                        universe_filter, title, incl_area_name,
                                        incl_total)
        self.assertEqual(query, queries.COUNTER2.strip())

    def test_median_query(self):

        # Test case MEDIAN1
        variable = "PERSONA.P03"
        by_var1 = "PERSONA.CONDACT"
        by_var2 = "PERSONA.P02"
        incl_name = True
        area_break = "PROV"

        area_filter = None
        universe_filter = "1 = 1"
        title = "El titulo"

        query = pyredatam.median_query(variable, by_var1, by_var2, incl_name,
                                       area_break, area_filter,
                                       universe_filter, title)
        self.assertEqual(query, queries.MEDIAN1.strip())

        # Test case MEDIAN2
        variable = "PERSONA.P03"
        incl_name = None
        area_break = None

        universe_filter = None
        title = None
        query = pyredatam.median_query(variable, by_var1, by_var2, incl_name,
                                       area_break, area_filter,
                                       universe_filter, title)
        self.assertEqual(query, queries.MEDIAN2.strip())

        # Test case MEDIAN3
        variable = "PERSONA.P03"
        by_var1 = None
        by_var2 = None
        query = pyredatam.median_query(variable, by_var1, by_var2, incl_name,
                                       area_break, area_filter,
                                       universe_filter, title)
        self.assertEqual(query, queries.MEDIAN3.strip())

if __name__ == '__main__':
    nose.run(defaultTest=__name__)