from __future__ import division
from __future__ import print_function
import time
import copy
import warnings
import six
import pandas as pd
import numpy as np
import seaborn as sns
import deepdish as dd
import matplotlib.pyplot as plt
from .helpers import _get_corrmat, _r2z, _z2r, _rbf, _expand_corrmat_fit, _expand_corrmat_predict, _plot_borderless,\
    _near_neighbor, _timeseries_recon, _count_overlapping, _plot_locs_connectome, _plot_locs_hyp, _gray, _nifti_to_brain
from .brain import Brain
from scipy.spatial.distance import cdist

class Model(object):
    """
    supereeg model and associated locations

    This class holds your supereeg model.  To create an instance, pass a list
    of brain objects and the model will be generated from those brain objects.
    You can also add your own model by passing a numpy array as your matrix and
    the corresponding locations. Alternatively, you can bypass creating a
    new model by passing numerator, denominator, locations, and n_subs
    (see parameters for details).  Additionally, you can include a meta dictionary
    with any other information that you want to save with the model.

    Parameters
    ----------

    data : supereeg.Brain or list supereeg.Brain, supereeg.Nifti or list supereeg.Nifti, or Numpy.ndarray

        A supereeg.Brain object or supereeg.Nifti object,  list of objects, or a Numpy.ndarray of your model.

    locs : pandas.DataFrame or np.ndarray
        MNI coordinate (x,y,z) by number of electrode df containing electrode locations

    template : filepath
        Path to a template nifti file used to set model locations

    numerator : Numpy.ndarray
        (Optional) A locations x locations matrix comprising the sum of the zscored
        correlation matrices over subjects.  If used, must also pass denominator,
        locs and n_subs. Otherwise, numerator will be computed from the brain
        object data.

    denominator : Numpy.ndarray
        (Optional) A locations x locations matrix comprising the sum of the number of
        subjects contributing to each matrix cell. If used, must also pass numerator,
        locs and n_subs. Otherwise, denominator will be computed from the brain
        object data.

    n_subs : int
        The number of subjects used to create the model.  Required if you pass
        numerator/denominator.  Otherwise computed automatically from the data.

    meta : dict
        Optional dict containing whatever you want

    date created : str
        Time created

    Attributes
    ----------
    numerator : Numpy.ndarray
        A locations x locations matrix comprising the sum of the zscored
        correlation matrices over subjects

    denominator : Numpy.ndarray
        A locations x locations matrix comprising the sum of the number of
        subjects contributing to each matrix cell

    n_subs : int
        Number of subject used to create the model

    Returns
    ----------
    model : supereeg.Model instance
        A model that can be used to infer timeseries from unknown locations

    """
    #TODO: __init__ should support data as a brain object, model object, nifti object, or string; if model object, just return data without copying it
    def __init__(self, data=None, locs=None, template=None,
                 numerator=None, denominator=None,
                 n_subs=None, meta=None, date_created=None):

        if all(v is not None for v in [numerator, denominator, locs, n_subs]):
            _handle_superuser(self, numerator, denominator, locs, n_subs)
        else:
            _create_locs(self, locs, template)

            s = self.locs.shape[0]
            self.numerator = np.zeros((s, s))
            self.denominator = np.zeros((s, s))
            self.n_subs = 0

            if type(data) is not list:
                data = [data]

            for d in data:
                d = _format_data(d, self.locs)
                if isinstance(d, Brain):
                    num_corrmat_x, denom_corrmat_x, n_subs = _bo2model(d, self.locs)
                elif isinstance(d, Model):
                    num_corrmat_x, denom_corrmat_x, n_subs = _mo2model(d, self.locs)
                self.numerator += num_corrmat_x
                self.denominator += denom_corrmat_x
                self.n_subs += n_subs

        if not date_created:
            self.date_created = time.strftime("%c")
        else:
            self.date_created = date_created
        self.n_locs = self.locs.shape[0]
        self.meta = meta

    def get_model(self):
        """ Returns a copy the model in the form of a correlation matrix"""
        with np.errstate(invalid='ignore'):
            return _z2r(np.divide(self.numerator, self.denominator))

    def predict(self, bo, nearest_neighbor=True, match_threshold='auto',
                force_update=False, kthreshold=10, preprocess='zscore'):
        """
        Takes a brain object and a 'full' covariance model, fills in all
        electrode timeseries for all missing locations and returns the new brain
        object

        Parameters
        ----------
        bo : supereeg.Brain or a list of brain objects
            The brain data object that you want to predict

        nearest_neighbor : True
            Default finds the nearest voxel for each subject's electrode
            location and uses that as revised electrodes location matrix in the
            prediction.

        match_threshold : 'auto' or int
            auto: if match_threshold auto, ignore all electrodes whose distance
            from the nearest matching voxel is greater than the maximum voxel
            dimension

            If value is greater than 0, inlcudes only electrodes that are within
            that distance of matched voxel

        force_update : False
            If True, will update model with patient's correlation matrix.

        kthreshold : 10 or int
            Kurtosis threshold

        preprocess : 'zscore' or None
            The predict algorithm requires the data to be zscored.  However, if
            your data are already zscored you can bypass this by setting to None.

        Returns
        ----------
        bo_p : supereeg.Brain
            New brain data object with missing electrode locations filled in

        """
        if preprocess not in ('zscore', None,):
            raise ValueError('Please set preprocess to either zscore or None.')

        bo = bo.get_filtered_bo() #TODO: IMPLEMENT THIS in Brain.py -- should return a copy of the brain object with only the electrodes that pass the filtering, and with filter=None

        # if match_threshold auto, ignore all electrodes whose distance from the
        # nearest matching voxel is greater than the maximum voxel dimension
        if nearest_neighbor:
            bo = _near_neighbor(bo, self, match_threshold=match_threshold)

        if self.locs.shape[0] > 1000:
            warnings.warn('Model locations exceed 1000, this may take a while. Good time for a cup of coffee.')

        # if True will update the model with subject's correlation matrix
        if force_update:
            model_corrmat_x = _force_update(self, bo)
        else:
            with np.errstate(invalid='ignore'):
                model_corrmat_x = np.divide(self.numerator, self.denominator)

        bool_mask = _count_overlapping(self, bo)
        case = _which_case(bo, bool_mask)
        if case is 'all_overlap':
            d = cdist(bo.get_locs(), self.locs)
            joint_bo_inds = np.where(np.isclose(d, 0))[0]
            bo.locs = bo.locs.iloc[joint_bo_inds]
            bo.data = bo.data[joint_bo_inds]
            bo.kurtosis = bo.kurtosis[joint_bo_inds]
            bo.label = np.array(bo.label)[joint_bo_inds].tolist()

            return Brain(data=bo.data, locs=bo.locs, sessions=bo.sessions,
                         sample_rate=bo.sample_rate, label=bo.label)
        else:
            # indices of the mask (where there is overlap
            joint_model_inds = np.where(bool_mask)[0]
            if case is 'no_overlap':
                model_corrmat_x, loc_label, perm_locs = _no_overlap(self, bo, model_corrmat_x)
            elif case is 'some_overlap':
                model_corrmat_x, loc_label, perm_locs = _some_overlap(self, bo, model_corrmat_x, joint_model_inds)
            elif case is 'subset':
                model_corrmat_x, loc_label, perm_locs = _subset(self, bo, model_corrmat_x, joint_model_inds)

            model_corrmat_x = _z2r(model_corrmat_x)
            np.fill_diagonal(model_corrmat_x, 0)
            activations = _timeseries_recon(bo, model_corrmat_x, preprocess=preprocess)

            return Brain(data=activations, locs=perm_locs, sessions=bo.sessions,
                        sample_rate=bo.sample_rate, kurtosis=None, label=loc_label)

    def update(self, data, inplace=True,
               locs=None, n=1):
        """
        Update a model with new data.

        Parameters
        ----------
        data : supereeg.Brain, supereeg.Model (or list of either)
            New subject data

        inplace : bool
            Whether to run update in place or return a new model (default True).

        Returns
        ----------
        model : supereeg.Model
            A new updated model object

        """

        if type(data) is not list:
            data = [data]

        if inplace:
            m = self
        else:
            m = copy.deepcopy(self)
        for d in data:
            d = _format_data(d, m.locs, locs, n)
            if isinstance(d, Brain):
                num_corrmat_x, denom_corrmat_x, n_subs = _bo2model(d.get_filtered_bo(), m.locs)
            elif isinstance(d, Model):
                num_corrmat_x, denom_corrmat_x, n_subs = _mo2model(d, m.locs)
                if type(d.meta) == dict:
                    m.update(d.meta)
            m.numerator += num_corrmat_x
            m.denominator += denom_corrmat_x
            m.n_subs += n_subs

        if not inplace:
            return m

    def info(self):
        """
        Print info about the model object

        Prints the number of electrodes, number of subjects, date created,
        and any optional meta data.
        """
        print('Number of locations: ' + str(self.n_locs))
        print('Number of subjects: ' + str(self.n_subs))
        print('Date created: ' + str(self.date_created))
        print('Meta data: ' + str(self.meta))

    def plot_data(self, savefile=None, show=True, **kwargs):
        """
        Plot the supereeg model as a correlation matrix

        This function wraps seaborn's heatmap and accepts any inputs that seaborn
        supports for models less that 2000x2000.  If the model is larger,

        Parameters
        ----------
        show : bool
            If False, image not rendered (default : True)

        Returns
        ----------
        ax : matplotlib.Axes
            An axes object

        """

        with np.errstate(invalid='ignore'):
            corr_mat = _z2r(np.divide(self.numerator, self.denominator))
        np.fill_diagonal(corr_mat, 1)

        if np.shape(corr_mat)[0] < 2000:
            ax = sns.heatmap(corr_mat, cbar_kws = {'label': 'correlation'}, **kwargs)

        else:
            ax = _plot_borderless(corr_mat, savefile=savefile, vmin=-1, vmax=1, cmap='Spectral')
        if show:
            plt.show()

        return ax

    def plot_locs(self, pdfpath=None):
        """
        Plots electrode locations from brain object

        Parameters
        ----------
        pdfpath : str
        A name for the file.  If the file extension (.pdf) is not specified, it will be appended.

        """

        locs = self.locs
        if self.locs .shape[0] <= 10000:
            _plot_locs_connectome(locs, pdfpath)
        else:
            _plot_locs_hyp(locs, pdfpath)

    def save(self, fname, compression='blosc'):
        """
        Save method for the model object

        The data will be saved as a 'mo' file, which is a dictionary containing
        the elements of a model object saved in the hd5 format using
        `deepdish`.

        Parameters
        ----------
        fname : str
            A name for the file.  If the file extension (.mo) is not specified,
            it will be appended.

        compression : str
            The kind of compression to use.  See the deepdish documentation for
            options: http://deepdish.readthedocs.io/en/latest/api_io.html#deepdish.io.save

        """

        mo = {
            'numerator' : self.numerator,
            'denominator' : self.denominator,
            'locs' : self.locs,
            'n_subs' : self.n_subs,
            'meta' : self.meta,
            'date_created' : self.date_created
        }

        if fname[-3:]!='.mo':
            fname+='.mo'

        dd.io.save(fname, mo, compression=compression)

    def get_slice(self, inds, inplace=False):
        """
        Indexes model object data

        Parameters
        ----------
        inds : scalar, list, or numpy array
            locations you wish to index (relative to model.get_locs())

        inplace : bool
            If True, indexes in place; otherwise a new Model object is returned
            (default: False)

        """
        numerator = self.numerator[inds][:, inds]
        denominator = self.denominator[inds][:, inds]
        locs = self.locs.iloc[inds]
        n_subs = self.n_subs
        meta = self.meta
        date_created = time.strftime("%c")

        if inplace:
            self.numerator = numerator
            self.denominator = denominator
            self.locs = locs
            self.n_subs = n_subs
            self.meta = meta
            self.date_created = date_created
        else:
            return Model(numerator=numerator, denominator=denominator, locs=locs,
                         n_subs=n_subs, meta=meta, date_created=date_created)

    def __add__(self, other):
        """
        Add two model objects together. The models must have matching
        locations.  Meta properties are combined across objects, or if properties
        conflict then the values from the first object are preferred.

        Parameters
        ----------
        other: Model object to be added to the current object
        """

        return self.update(other, inplace=False)

    def __sub__(self, other):
        """
        Subtract one model object from another. The models must have matching
        locations.  Meta properties are combined across objects, or if properties
        conflict then the values from the first object are preferred.

        Parameters
        ----------
        other: Model object to be subtracted from the current object
        """

        other.numerator = -other.numerator
        other.denominator = -other.denominator
        result = self.__add__(other)

        #account for n_subs being updated during add operation
        if type(other) == Model:
            result.n_subs -= 2*other.n_subs
        else:
            result.n_subs -= 2

        return result

###################################
# helper functions for init
###################################

def _handle_superuser(self, numerator, denominator, locs, n_subs):
    """Shortcuts model building if these args are passed"""
    self.numerator = numerator
    self.denominator = denominator

    # if locs arent already a df, turn them into df
    if isinstance(locs, pd.DataFrame):
        self.locs = locs
    else:
        self.locs = pd.DataFrame(locs, columns=['x', 'y', 'z'])

    self.n_subs = n_subs

def _create_locs(self, locs, template):
    """get locations from template, or from locs arg"""
    if locs is None:
        if template is None:
            template = _gray(20)
        nii_data, nii_locs, nii_meta = _nifti_to_brain(template)
        self.locs = pd.DataFrame(nii_locs, columns=['x', 'y', 'z'])
    else:
        self.locs = pd.DataFrame(locs, columns=['x', 'y', 'z'])
    if self.locs.shape[0]>1000:
        warnings.warn('Model locations exceed 1000, this may take a while. Go get a cup of coffee or brew some tea!')

def _bo2model(bo, locs):
    """Returns numerator and denominator given a brain object"""
    sub_corrmat = _get_corrmat(bo)
    np.fill_diagonal(sub_corrmat, 0)
    sub_corrmat_z = _r2z(sub_corrmat)
    sub_rbf_weights = _rbf(locs, bo.get_locs())
    n, d = _expand_corrmat_fit(sub_corrmat_z, sub_rbf_weights)
    return n, d, 1

def _mo2model(mo, locs):
    """Returns numerator and denominator for model object"""
    if not isinstance(locs, pd.DataFrame):
        locs = pd.DataFrame(locs, columns=['x', 'y', 'z'])
    if locs.equals(mo.locs):
        return mo.numerator.copy(), mo.denominator.copy(), mo.n_subs
    else:
        # if the locations are not equivalent, map input model into locs space
        with np.errstate(invalid='ignore'):
            sub_corrmat_z = np.divide(mo.numerator, mo.denominator)
        np.fill_diagonal(sub_corrmat_z, 0)
        sub_rbf_weights = _rbf(locs, mo.locs)
        n, d = _expand_corrmat_fit(sub_corrmat_z, sub_rbf_weights)
        return n, d, mo.n_subs

def _format_data(d, model_locs, new_locs=None, n_subs=1):
    """Formats data to generate model object"""
    from .load import load
    from .brain import Brain
    from .nifti import Nifti
    if isinstance(d, six.string_types):
        d = load(d)
    if isinstance(d, np.ndarray):
        if new_locs is None:
            new_locs = model_locs
            if d.shape[0]!=new_locs.shape[0]:
                raise ValueError("Array must have same dimensions as model or"
                                 " you must passed custom locations")
        np.fill_diagonal(d, 0)
        return Model(numerator=_r2z(d), denominator=np.ones_like(d)*n_subs,
                     n_subs=n_subs, locs=new_locs)
    elif isinstance(d, Brain):
        return d
    elif isinstance(d, Nifti):
        return Brain(d)
    elif isinstance(d, Model):
        return d
    else:
        raise TypeError("Did not recognize the type of one of your inputs to the model")

def _force_update(mo, bo):

    # get subject-specific correlation matrix
    sub_corrmat = _get_corrmat(bo)

    # fill diag with zeros
    np.fill_diagonal(sub_corrmat, 0) # <- possible failpoint

    # z-score the corrmat
    sub_corrmat_z = _r2z(sub_corrmat)

    # get _rbf weights
    sub__rbf_weights = _rbf(mo.locs, bo.get_locs())

    #  get subject expanded correlation matrix
    num_corrmat_x, denom_corrmat_x = _expand_corrmat_fit(sub_corrmat_z, sub__rbf_weights)

    # add in new subj data
    with np.errstate(invalid='ignore'):
        model_corrmat_x = np.divide(np.add(mo.numerator, num_corrmat_x), np.add(mo.denominator, denom_corrmat_x))

    return model_corrmat_x

###################################
# helper functions for predict
###################################

def _which_case(bo, bool_mask):
    """Determine which predict scenario we are in"""
    if all(bool_mask):
        return 'all_overlap'
    if not any(bool_mask):
        return 'no_overlap'
    elif sum(bool_mask) == bo.get_locs().shape[0]:
        return 'subset'
    elif sum(bool_mask) != bo.get_locs().shape[0]:
        return 'some_overlap'

def _no_overlap(self, bo, model_corrmat_x):
    """ Compute model when there is no overlap """
    # expanded _rbf weights
    model__rbf_weights = _rbf(pd.concat([self.locs, bo.get_locs()]), self.locs)

    # get model expanded correlation matrix
    num_corrmat_x, denom_corrmat_x = _expand_corrmat_predict(model_corrmat_x, model__rbf_weights)

    # divide the numerator and denominator
    with np.errstate(invalid='ignore'):
        model_corrmat_x = np.divide(num_corrmat_x, denom_corrmat_x)

    # label locations as reconstructed or observed
    loc_label = ['reconstructed'] * len(self.locs) + ['observed'] * len(bo.get_locs())

    # grab the locs
    perm_locs = self.locs.append(bo.get_locs())

    return model_corrmat_x, loc_label, perm_locs

def _subset(self, bo, model_corrmat_x, joint_model_inds):
    """ Compute model when bo is a subset of the model """
    # permute the correlation matrix so that the inds to reconstruct are on the right edge of the matrix
    perm_inds = sorted(set(range(self.locs.shape[0])) - set(joint_model_inds)) + sorted(set(joint_model_inds))
    model_corrmat_x = model_corrmat_x[:, perm_inds][perm_inds, :]

    # label locations as reconstructed or observed
    loc_label = ['reconstructed'] * (len(self.locs)-len(bo.get_locs())) + ['observed'] * len(bo.get_locs())

    # grab permuted locations
    perm_locs = self.locs.iloc[perm_inds]

    return model_corrmat_x, loc_label, perm_locs

def _some_overlap(self, bo, model_corrmat_x, joint_model_inds):
    """ Compute model when there is some overlap """

    # get subject indices where subject locs do not overlap with model locs

    bool_bo_mask= np.sum([(bo.get_locs() == y).all(1) for idy, y in self.locs.iterrows()], 0).astype(bool)
    disjoint_bo_inds = np.where(~bool_bo_mask)[0]
    # d = cdist(bo.get_locs(), self.locs)
    # disjoint_bo_inds = np.where(np.isclose(d, 0))[0]

    # permute the correlation matrix so that the inds to reconstruct are on the right edge of the matrix
    perm_inds = sorted(set(range(self.locs.shape[0])) - set(joint_model_inds)) + sorted(set(joint_model_inds))
    model_permuted = model_corrmat_x[:, perm_inds][perm_inds, :]

    # permute the model locations (important for the _rbf calculation later)
    model_locs_permuted = self.locs.iloc[perm_inds]

    # permute the subject locations arranging them
    bo_perm_inds = sorted(set(range(bo.get_locs().shape[0])) - set(disjoint_bo_inds)) + sorted(set(disjoint_bo_inds))
    sub_bo = bo.get_locs().iloc[disjoint_bo_inds]

    #TODO: would be safer to implement this using bo.get_locs(), bo.get_data()
    bo.locs = bo.locs.iloc[bo_perm_inds]
    bo.data = bo.data[bo_perm_inds]
    bo.kurtosis = bo.kurtosis[bo_perm_inds]

    # permuted indices for unknown model locations
    perm_inds_unknown = sorted(set(range(self.locs.shape[0])) - set(joint_model_inds))
    # expanded _rbf weights
    #model__rbf_weights = _rbf(pd.concat([model_locs_permuted, bo.locs]), model_locs_permuted)
    model__rbf_weights = _rbf(pd.concat([model_locs_permuted, sub_bo]), model_locs_permuted)

    # get model expanded correlation matrix
    num_corrmat_x, denom_corrmat_x = _expand_corrmat_predict(model_permuted, model__rbf_weights)

    # divide the numerator and denominator
    with np.errstate(invalid='ignore'):
        model_corrmat_x = np.divide(num_corrmat_x, denom_corrmat_x)

    # add back the permuted correlation matrix for complete subject prediction
    model_corrmat_x[:model_permuted.shape[0], :model_permuted.shape[0]] = model_permuted

    # label locations as reconstructed or observed
    loc_label = ['reconstructed'] * len(self.locs.iloc[perm_inds_unknown]) + ['observed'] * len(bo.get_locs())

    ## unclear if this will return too many locations
    perm_locs = self.locs.iloc[perm_inds_unknown].append(bo.get_locs())

    return model_corrmat_x, loc_label, perm_locs


import pandas as pd
import numpy as np

dept_df = pd.DataFrame({"deptno": [10, 20, 30, 40],
                        "dname": ["개발", "인사", "관리", "영업"],
                        "loc": ["서울", "부산", "제주", "광주"]})
emp_df = pd.DataFrame({"empno": ["A1", "A2", "A3", "A4", "A5"],
                       "ename": ["홍길동", "이순신", "강감찬", "정조", "영조"],
                       "sal": [1000, 2000, 1500, 4500, 4200],
                       "deptno": [10, 20, 30, 10, 20]})

print(dept_df)
print(emp_df)

# 1. 병합(merge) - 공통컬럼 존재 -> on="공통컬럼"
merge_df = pd.merge(dept_df, emp_df, on="deptno")
print(merge_df)
print(merge_df[["empno", "ename", "sal", "dname"]])

# 2. 병합(merge) - 공통컬럼 X -> left_on="왼쪽 컬럼" right_on="오른쪽 컬럼"
emp_df = pd.DataFrame({"empno": ["A1", "A2", "A3", "A4", "A5"],
                       "ename": ["홍길동", "이순신", "강감찬", "정조", "영조"],
                       "sal": [1000, 2000, 1500, 4500, 4200],
                       "d_no": [10, 20, 30, 10, 20]})
merge_df = pd.merge(dept_df, emp_df, left_on="deptno", right_on="d_no")
print(merge_df)
print(merge_df[["empno", "ename", "sal", "dname"]])

# 3. outer 조인 -> how 속성에 left -> 왼쪽 다, right -> 오른쪽 다, outer -> 싹 다
emp_df = pd.DataFrame({"empno": ["A1", "A2", "A3", "A4", "A5"],
                       "ename": ["홍길동", "이순신", "강감찬", "정조", "영조"],
                       "sal": [1000, 2000, 1500, 4500, 4200],
                       "deptno": [np.nan, 20, 30, 10, 20]})
merge_df = pd.merge(dept_df, emp_df, on="deptno", how="left")
print(merge_df)

# 4. 컬럼과 인덱스 merge
df = pd.DataFrame({"도시": ["서울", "부산", "제주", "제주", "서울", "서울"],
                   "도시번호": [1000, 455, 333, 2222, 666, 7777]})
df2 = pd.DataFrame(["특별시", "광역시", "자치도"], index=["서울", "부산", "제주"], columns=["도시종류"])
merge_df = pd.merge(df, df2, left_on="도시", right_index=True)  # index 속성을 True로 하면 컬럼과 인덱스로 합쳐줌
print(merge_df)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='Usuario',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', primary_key=True, serialize=False)),
                ('nombre', models.CharField(max_length=50)),
                ('apellido', models.CharField(max_length=50)),
                ('dni', models.CharField(max_length=50)),
                ('imagen', models.IntegerField(max_length=10)),
                ('usuario', models.OneToOneField(to=settings.AUTH_USER_MODEL)),
            ],
        ),
    ]


import threading
import queue
import time , random
workers = 2

class worker(threading.Thread):
    def __init__(self,Q):
        self._Q= Q
        threading.Thread.__init__(self)

    def run(self):
        while(1):
            item = self._Q.get()
            if item is None:
                break

            time.sleep(random.randint(10,100)/1000.0)

            print("Tase",item,"finished")

Q= queue.Queue(3)

for i in range(workers):
    cur_thread = worker(Q)
    cur_thread.start()

for item in range(10):
    print("push",item)
    Q.put(item)
Q.put(None)


def get_undecorated(func):

    while hasattr(func, "_Annotation__func"):
        func = func._Annotation__func
    return func

def get_annotations(func, all):
    """
    Get list of annotations for current object
    """
    #There func must be actually annotation object
    if not isinstance(func, Annotation):
        return []

    #Check is it annotation
    if hasattr(func, "_Annotation__func"):
        func_key = get_undecorated(func)
        if  func_key in Annotation.func_to_annotations:
            return Annotation.func_to_annotations[func_key]

    return  []

class Annotation(object):
    """
    Base class for annotations
    """

    annotations_to_funcs = {}
    func_to_annotations = {}

    def __init__(self, func, *args, **kwargs):
        self.__func = func
        cls = self.__class__

        func_key = get_undecorated(func)

        if not cls in Annotation.annotations_to_funcs:
            Annotation.annotations_to_funcs[cls] = []
        Annotation.annotations_to_funcs[cls].append(func_key)

        if not func_key in Annotation.func_to_annotations:
            Annotation.func_to_annotations[func_key] = []
        Annotation.func_to_annotations[func_key].append(cls)

    def __call__(self, *args, **kwargs):
        return self.__func(*args, **kwargs)

    @classmethod
    def get_annotated(cls):
        """
        List of functions with this annotat
        """

        if cls in Annotation.annotations_to_funcs:
            return Annotation.annotations_to_funcs[cls]
        return []


# DEPRECATED MODULE
import requests
from bs4 import BeautifulSoup

mainurl = 'https://www.clevillage.it/5163/'
nexturl = 'https://www.clevillage.it/5163/index.php?week=next'

prog_now = ''
prog_next = ''
dettagli_now = ''
dettagli_next = ''
count = 0

def request_page(url):
    try:
        response = requests.get(url)
    except requests.exceptions.RequestException as e:
        print(e)
        return None

    if(not response.ok):
        return None
    return response.text

def get_dettagli(datifilm, dettaglio):
    div_dettaglio = datifilm.find("div", dettaglio)
    if(div_dettaglio is None):
        return None
    elif(len(div_dettaglio.contents) >= 2):
        return div_dettaglio.contents[1]
    else:
        return div_dettaglio.contents[0]

def get_periodo(soup):
    div_progData = soup.find("div", {"class":"progData"})
    return '_' + div_progData.text + '_\n\n'

def get_orari_films(htmlpage):
    soup = BeautifulSoup(htmlpage, 'html.parser')

    # Periodo
    periodo = get_periodo(soup)

    # Filmcontainer: contengono tutti i dati dei film
    filmcontainers = soup.find_all(lambda tag:
        tag.name=="div" and
        tag.has_attr("class") and
        (tag.attrs=={"class":["filmContainer","oddFilm"]} or tag.attrs=={"class":["filmContainer","evenFilm"]})
    )

    films = ''
    for filmcontainer in filmcontainers:
        datifilms = filmcontainer.find_all("div", attrs = {"class": "datiFilm"})
        for datifilm in datifilms:
            # Titolo
            titolo = get_dettagli(datifilm, "titolo")

            # Durata
            durata = get_dettagli(datifilm, "durata")
            durata = durata[1:]

        # Orari
        orari = [li.get_text(' ', strip=True) for ul in filmcontainer.find_all("ul", attrs={"class": "orari"}) for li in ul.find_all("li")]

        for ul in filmcontainer.find_all("ul", attrs={"class": "orari"}):
            for li in ul.find_all("li"):
                giorno = li.find("div").get_text()
                ore = []
                for ora in li.find_all("span"):
                    ore.append(ora.get_text)

        # Aggiunge titolo in grassetto e durata
        films += '*' + titolo + '* _('+ durata +')_\n'

        # Aggiunge giorno e orari
        for orario in orari:
            if(filmcontainer == filmcontainers[-1] and orario == orari[-1]):
                films += (orario)
            else:
                films += (orario + '\n')

        if(filmcontainer != filmcontainers[-1]):
            films += '\n'
    return periodo + films

def get_dettagli_films(htmlpage):
    soup = BeautifulSoup(htmlpage, 'html.parser')

    # Periodo
    periodo = get_periodo(soup)

    # Filmcontainers: contengono tutti i dati dei film
    filmcontainers = soup.find_all(lambda tag:
        tag.name=="div" and
        tag.has_attr("class") and
        (tag.attrs=={"class":["filmContainer","oddFilm"]} or tag.attrs=={"class":["filmContainer","evenFilm"]})
    )

    films = ''
    for filmcontainer in filmcontainers:
        datifilms = filmcontainer.find_all("div", attrs = {"class": "datiFilm"})
        for datifilm in datifilms:
            # Titolo
            titolo = get_dettagli(datifilm, "titolo")

            # Regia
            regia = get_dettagli(datifilm, "regia")
            regia = regia[1:]

            # Genere
            genere = get_dettagli(datifilm, "genere")
            genere = genere[1:]

            # Durata
            durata = get_dettagli(datifilm, "durata")
            durata = durata[1:]

            # Cast
            cast = get_dettagli(datifilm, "cast")
            cast = cast[1:]

            # Id Scheda
            id_scheda = (datifilm.find(lambda tag: tag.name=="div" and tag.has_attr("idfilm")))['idfilm']

            # Costruisco la stringa di testo
            films += '*' + titolo + '*\n'
            films += 'Regia: ' + regia + '\n'
            films += 'Genere: ' + genere + '\n'
            films += 'Durata: ' + durata + '\n'
            films += 'Cast: ' + cast + '\n'
            films += '[Apri scheda film](' + mainurl + 'scheda.php?id='+ id_scheda + ')\n'

        if(filmcontainer != filmcontainers[-1]):
            films += '\n'
    return periodo + films

def worker():
    global prog_now
    global prog_next
    global dettagli_now
    global dettagli_next
    global count
    print("[Worker]: AVVIO TASK N°", count)
    count += 1

    # Richieste HTTP GET
    print("[Worker]: richiedo la pagina ", mainurl)
    page_now = request_page(mainurl)
    print("[Worker]: richiedo la pagina ", nexturl)
    page_next = request_page(nexturl)
    print("[Worker]: download completato!")

    messaggio_riprova = 'La programmazione per la prossima settimana non è ancora disponibile. Riprovare più tardi.'

    # Genera risposta per /programma
    if(page_now is None):
        print("[Worker]: errore page_now!")
        prog_now = 'Errore. Riprova più tardi.'
    else:
        print("[Worker]: aggiorno la programmazione di oggi")
        prog_now = get_orari_films(page_now)

    # Genera risposta per /dettagli
    if(page_now is None):
        print("[Worker]: errore page_now!")
        dettagli_now = 'Errore. Riprova più tardi.'
    else:
        print("[Worker]: aggiorno dettagli della programmazione di oggi")
        dettagli_now = get_dettagli_films(page_now)

    # Genera risposta per /dettaglinext
    if(page_next is None):
        print("[Worker]: errore page_next!")
        dettagli_next = messaggio_riprova
    else:
        print("[Worker]: aggiorno dettagli della programmazione prossima")
        dettagli_next = get_dettagli_films(page_next)
    if(dettagli_next == ''):
        dettagli_next = messaggio_riprova

    # Genera risposta per /next
    if(page_next is None):
        print("[Worker]: errore page_next!")
        prog_next = messaggio_riprova
    else:
        print("[Worker]: aggiorno la programmazione della prossima settimana")
        prog_next = get_orari_films(page_next)
    if(prog_next == ''):
            prog_next = messaggio_riprova
    print("[Worker]: FINE TASK N°", count-1)


# Copyright 2020 TU Ilmenau. All Rights Reserved.
#
# Code for the Particle regression crop for the
# On the use of a cascaded convolutional neural network for three-dimensional flow measurements using astigmatic PTV task.

# ==============================================================================

import csv
from PIL import Image
import os
import pandas as pd

TBL_IMG_ID='id'
TBL_CLASS='class'
TBL_BBOX_RB_X='xmax'
TBL_BBOX_LT_X='xmin'
TBL_BBOX_RB_Y='ymax'
TBL_BBOX_LT_Y='ymin'
TBL_POS_X='pos_x'
TBL_POS_Y='pos_y'
TBL_POS_Z='pos_z'
TBL_FILENAME='filename'
TBL_CROP_LT_X='cropxmin'
TBL_CROP_LT_Y='cropymin'
TBL_CROP_RB_X='cropxmax'
TBL_CROP_RB_Y='cropymax'

TBL_HEADER=[TBL_IMG_ID,TBL_CLASS,TBL_BBOX_RB_X,TBL_BBOX_LT_X,TBL_BBOX_RB_Y,TBL_BBOX_LT_Y,TBL_FILENAME,TBL_POS_X,TBL_POS_Y,TBL_POS_Z]

path = ''
new_csv = ''

df = pd.read_csv('',
                 sep=',', names=TBL_HEADER)

filenames = [os.path.join(path, file) for file in os.listdir(path) if ".png" in file]

df_new = pd.DataFrame()
fname_list=[]
imageid_list=[]
width_list=[]
height_list=[]
class_list=[]
pos_x_list=[]
pos_y_list=[]
posz_list=[]
xmin_list=[]
ymin_list=[]
xmax_list=[]
ymax_list=[]

for filename in sorted(filenames):

    img_id = int(filename.split('/')[-1].split('.')[0][-4:])
    print(img_id)
    filename_id= filename.split('/')[-1].split('.')[0]
    filename_name=filename.split('/')[-1]
    df_filt = df[df[TBL_IMG_ID] == img_id]

    img = Image.open(filename)
    idx=0
    for _, row in df_filt.iterrows():
        row[TBL_POS_X]=row[TBL_BBOX_LT_X]+(row[TBL_BBOX_RB_X]-row[TBL_BBOX_LT_X])/2
        row[TBL_POS_Y]=row[TBL_BBOX_LT_Y]+(row[TBL_BBOX_RB_Y]-row[TBL_BBOX_LT_Y])/2

        row[TBL_CROP_LT_X]=row[TBL_POS_X]-90
        row[TBL_CROP_LT_Y]=row[TBL_POS_Y]-90
        row[TBL_CROP_RB_X]=row[TBL_POS_X]+90
        row[TBL_CROP_RB_Y]=row[TBL_POS_Y]+90
        area = (row[TBL_CROP_LT_X],
                row[TBL_CROP_LT_Y],
                row[TBL_CROP_RB_X],
                row[TBL_CROP_RB_Y])

        cropped_img = img.crop(area)

        fname = "{0}_{1}_{2}.png".format(filename_id,img_id,idx)
        fname_list.append(fname)
        imageid_list.append(int(img_id))
        width_list.append(210)
        height_list.append(210)
        class_list.append(row[TBL_CLASS])
        posz_list.append(0)
        pos_x_list.append(row[TBL_POS_X])
        pos_y_list.append(row[TBL_POS_Y])
        xmin_list.append(row[TBL_BBOX_LT_X])
        ymin_list.append(row[TBL_BBOX_LT_Y])
        xmax_list.append(row[TBL_BBOX_RB_X])
        ymax_list.append(row[TBL_BBOX_RB_Y])

        cropped_img.save(fname)
        idx += 1

df_new = pd.DataFrame(data={
        'filename':fname_list,
        'imageid':imageid_list,
        'width':width_list,
        'height':height_list,
        'class':class_list,
        'pos_z':posz_list,
        'posx':pos_x_list,
        'posy':pos_y_list,
        'xmin':xmin_list,
        'ymin':ymin_list,
        'xmax':xmax_list,
        'ymax':ymax_list,
        })

df_new.to_csv(new_csv, sep=',', header= True, index=False)


from sklearn import __version__ as sklearn_version
from distutils.version import LooseVersion
from sklearn import datasets
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing   import StandardScaler
from sklearn.linear_model    import Perceptron
from sklearn.metrics         import accuracy_score
from matplotlib.colors       import ListedColormap
import matplotlib.pyplot as plt
from sklearn.linear_model    import LogisticRegression
from sklearn.svm             import SVC
from sklearn.linear_model    import SGDClassifier
from sklearn.tree            import DecisionTreeClassifier
from pydotplus               import graph_from_dot_data
from sklearn.tree            import export_graphviz
from sklearn.ensemble        import RandomForestClassifier
from sklearn.neighbors       import KNeighborsClassifier

def plot_decision_regions(X, y, classifier, test_idx=None, resolution=0.02):

		# setup marker generator and color map
		markers = ('s', 'x', 'o', '^', 'v')
		colors  = ('red', 'blue', 'lightgreen', 'gray', 'cyan')
		cmap    = ListedColormap(colors[:len(np.unique(y))])

		# plot the decision surface
		x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1
		x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1
		xx1, xx2       = np.meshgrid(np.arange(x1_min, x1_max, resolution),
													 np.arange(x2_min, x2_max, resolution))
		Z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)
		Z = Z.reshape(xx1.shape)
		plt.contourf(xx1, xx2, Z, alpha=0.3, cmap=cmap)
		plt.xlim(xx1.min(), xx1.max())
		plt.ylim(xx2.min(), xx2.max())

		# plot the scatter of the actual datas
		for idx, cl in enumerate(np.unique(y)):
				plt.scatter(x=X[y == cl, 0],
										y=X[y == cl, 1],
										alpha=0.8,
										c=colors[idx],
										marker=markers[idx],
										label=cl,
										edgecolor='black')

		# highlight test samples by circle
		if test_idx:
				# plot all samples
				X_test, y_test = X[test_idx, :], y[test_idx]

				plt.scatter(X_test[:, 0],
										X_test[:, 1],
										c='',
										edgecolor='black',
										alpha=1.0,
										linewidth=1,
										marker='o',
										s=100,
										label='test set')

def sigmoid(z):
	return 1.0 / (1.0 + np.exp(-z))

# Logistic Regression Classifier using gradient descent.
class LogisticRegressionGD(object):
		def __init__(self, eta=0.05, n_iter=100, random_state=1):
				self.eta          = eta
				self.n_iter       = n_iter
				self.random_state = random_state

		def fit(self, X, y):
				rgen       = np.random.RandomState(self.random_state)
				self.w_    = rgen.normal(loc=0.0, scale=0.01, size=1 + X.shape[1])
				self.cost_ = []

				for i in range(self.n_iter):
						net_input = self.net_input(X)
						output    = self.activation(net_input)
						errors    = (y - output)
						self.w_[1:] += self.eta * X.T.dot(errors)
						self.w_[0]  += self.eta * errors.sum()

						# note that we compute the logistic `cost` now
						# instead of the sum of squared errors cost
						cost = -y.dot(np.log(output)) - ((1 - y).dot(np.log(1 - output)))
						self.cost_.append(cost)
				return self

		def net_input(self, X):
				"""Calculate net input"""
				return np.dot(X, self.w_[1:]) + self.w_[0]

		def activation(self, z):
				"""Compute logistic sigmoid activation"""
				return 1. / (1. + np.exp(-np.clip(z, -250, 250)))

		def predict(self, X):
				"""Return class label after unit step"""
				return np.where(self.net_input(X) >= 0.0, 1, 0)
				# equivalent to:
				# return np.where(self.activation(self.net_input(X)) >= 0.5, 1, 0)

def gini(p):
		return p * (1 - p) + (1 - p) * (1 - (1 - p))

def entropy(p):
		return - p * np.log2(p) - (1 - p) * np.log2((1 - p))

def error(p):
		return 1 - np.max([p, 1 - p])

iris = datasets.load_iris()
X    = iris.data[:, [2,3]]
y    = iris.target
#-------------------------------------------------------------------------------------------
# split training data and test data
# 全体の30%をテストデータにする。
X_train, X_test, y_train, y_test = train_test_split(
	X, y, test_size=0.3, random_state=1, stratify=y)
#-------------------------------------------------------------------------------------------
# 特徴量のスケーリング
sc = StandardScaler()
sc.fit(X_train) # 平均値と標準偏差の計算
X_train_std = sc.transform(X_train)
X_test_std  = sc.transform(X_test)
#-------------------------------------------------------------------------------------------
# パーセプトロンで学習
ppn = Perceptron(max_iter= 40, eta0=0.1, random_state=0)
ppn.fit(X_train_std, y_train)
#-------------------------------------------------------------------------------------------
# 予測
y_pred = ppn.predict(X_test_std)
print( (y_test!=y_pred).sum() )
#-------------------------------------------------------------------------------------------
# 性能指標の計算
print(accuracy_score(y_test, y_pred))
print(ppn.score(X_test_std, y_test))
#-------------------------------------------------------------------------------------------
# 決定領域のプロット
X_combined_std = np.vstack((X_train_std, X_test_std))
y_combined     = np.hstack((y_train, y_test))
plot_decision_regions(X_combined_std, y_combined, classifier=ppn, test_idx = range(105,150))
plt.tight_layout()
plt.close()
#-------------------------------------------------------------------------------------------
# 決定領域のプロット
lr = LogisticRegression(C=100.0, random_state=1)
lr.fit(X_train_std, y_train)
plot_decision_regions(X_combined_std, y_combined, classifier=lr, test_idx=range(105, 150))
plt.legend()
plt.close()
#-------------------------------------------------------------------------------------------
# 線形SVM
svm = SVC(kernel="linear", C=1.0, random_state=1)
svm.fit(X_train_std, y_train)
plot_decision_regions(X_combined_std, y_combined, classifier=svm,test_idx=range(105, 150))
plt.legend()
plt.close()
#-------------------------------------------------------------------------------------------
# XORテストデータpreparation
np.random.seed(1)
X_xor = np.random.randn(200,2)
y_xor = np.logical_xor(X_xor[:,0]>0, X_xor[:,1]>0)
y_xor = np.where(y_xor, 1, -1)
plt.scatter( X_xor[y_xor==1,  0], X_xor[y_xor==1, 1],  c="b", marker="x", label="1")
plt.scatter( X_xor[y_xor==-1, 0], X_xor[y_xor==-1, 1], c="r", marker="s", label="-1")
plt.tight_layout()
plt.close()
#-------------------------------------------------------------------------------------------
# カーネルSVM
svm = SVC(kernel="rbf", C=10.0, gamma=0.10, random_state=1)
svm.fit(X_xor, y_xor)
plot_decision_regions(X_xor, y_xor, classifier=svm)
plt.legend()
#plt.show()
plt.close()
#-------------------------------------------------------------------------------------------
# 条件
x = np.arange(0.0, 1.0, 0.01)

ent    = [entropy(p) if p != 0 else None for p in x]
sc_ent = [e * 0.5 if e else None for e in ent]
err    = [error(i) for i in x]

fig = plt.figure()
ax  = plt.subplot(111)
for i, lab, ls, c, in zip([ent, sc_ent, gini(x), err],
													['Entropy', 'Entropy (scaled)',
													 'Gini Impurity', 'Misclassification Error'],
													['-', '-', '--', '-.'],
													['black', 'lightgray', 'red', 'green', 'cyan']):
		line = ax.plot(x, i, label=lab, linestyle=ls, lw=2, color=c)

ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.15),
					ncol=5, fancybox=True, shadow=False)

ax.axhline(y=0.5, linewidth=1, color='k', linestyle='--')
ax.axhline(y=1.0, linewidth=1, color='k', linestyle='--')
plt.ylim([0, 1.1])
plt.xlabel('p(i=1)')
plt.ylabel('Impurity Index')
plt.close()
#-------------------------------------------------------------------------------------------
# 決定木
tree = DecisionTreeClassifier(criterion='gini',
															max_depth=4,
															random_state=1)
tree.fit(X_train, y_train)

X_combined = np.vstack((X_train, X_test))
y_combined = np.hstack((y_train, y_test))
plot_decision_regions(X_combined, y_combined,
											classifier=tree, test_idx=range(105, 150))

plt.xlabel('petal length [cm]')
plt.ylabel('petal width [cm]')
plt.legend(loc='upper left')
plt.tight_layout()
plt.close()
#-------------------------------------------------------------------------------------------
# 決定木フローの図示
from PIL import Image
dot_data = export_graphviz(tree,
													 filled=True,
													 rounded=True,
													 class_names=['Setosa',
																				'Versicolor',
																				'Virginica'],
													 feature_names=['petal length',
																					'petal width'],
													 out_file=None)
graph = graph_from_dot_data(dot_data)
#Image(graph.create_png())
graph.write_png('tree.png')
#-------------------------------------------------------------------------------------------
# ランダムフォレスト
forest = RandomForestClassifier(criterion='gini',
																n_estimators=25,
																random_state=1,
																n_jobs=2)
forest.fit(X_train, y_train)

plot_decision_regions(X_combined, y_combined,
											classifier=forest, test_idx=range(105, 150))

plt.xlabel('petal length [cm]')
plt.ylabel('petal width [cm]')
plt.legend(loc='upper left')
plt.tight_layout()
#plt.savefig('images/03_22.png', dpi=300)
plt.show()


#!/usr/bin/python

import sys
import csv
import copy
from string import Template
from generator_helper import camelCase, camelCaseUpper, quoteArray, readCsv
from translation_template import *

languages_csv = "languages.csv"
translations_csv = "translations.csv"
names = ["InfoTag", "SuccessTag", "WarningTag", "DangerTag"]

def formatTemplateLanguage(template, row):
        nameField, isoCodeField = row
        name = nameField.strip()
        isoCode = isoCodeField.strip()
        return Template(template).substitute(
            name=name,
            isoCode=isoCode,
            )

def formatTemplateI18n(template, row, indice, name):
        language = row[0].strip()
        translation = row[indice].strip()
        translation2 = ""
        if ";" in translation:
            a, b = translation.split(";", 2)
            translation = a.strip()
            translation2 = b.strip()
        return Template(template).substitute(
            language=language,
            translation=translation,
            translation2=translation2,
            nameCamelUpper = name
            )

def formatTemplateI18nMethod(template, name, language = ""):
        return Template(template).substitute(
            nameCamelUpper = name,
            language = language

        )

def getLanguageName(row):
    return row[0].strip()

def createIsolanguageType(languageNames):
    return "type IsoLanguage = " + " \n  | ".join(languageNames)

def createIsoLanguage():
    content = readCsv(languages_csv)
    file = open("../src/Bubblegum/PreviewTag/IsoLanguage.elm", "w")
    file.write(headerLanguage)
    languageNames = []
    for row in content:
        if len(row) > 1 :
            file.write(formatTemplateLanguage(templateLanguage, row))
            languageNames.append(getLanguageName(row))
    file.write(footerLanguage)
    file.write(createIsolanguageType(languageNames))
    file.close()

def getAllLanguageNames():
    content = readCsv(languages_csv)
    languageNames = []
    for row in content:
        if len(row) > 1 :
            languageNames.append(getLanguageName(row))
    return languageNames

def createI18n():
    content = readCsv(translations_csv)
    file = open("../src/Bubblegum/PreviewTag/Internationalization.elm", "w")
    file.write(headerI18n)
    def templatesMethod(ii):
        # if ii == 0:
        #     return templateI18nWithPluralMethod
        # else:
            return templateI18nMethod
    def templateslanguage(ii):
        # if ii == 0:
        #     return templateI18nWithPluralLanguage
        # else:
            return templateI18nLanguage
    maxIndice = len(names)
    codeParts = [formatTemplateI18nMethod(templatesMethod(i), names[i]) for i in range(0, maxIndice)]
    translatedLanguages = []
    englishRow = ""
    for row in content:
        if len(row) > 1 :
            if row[0] == "English":
                englishRow = row
            for indice in range(0, maxIndice):
                translatedLanguages.append(getLanguageName(row))
                temp = formatTemplateI18n(templateslanguage(indice), row, indice+1, names[indice])
                codeParts[indice] += temp
    untranslatedLanguages = list(set(getAllLanguageNames()) - set(translatedLanguages))
    for otherlanguage in untranslatedLanguages:
        for indice in range(0, maxIndice):
                row = copy.copy(englishRow)
                row[0] = otherlanguage
                temp = formatTemplateI18n(templateslanguage(indice), row, indice+1, names[indice])
                codeParts[indice] += temp

    file.write("\n\n".join(codeParts))
    file.close()

def main(argv):
    createIsoLanguage()
    createI18n()

if __name__ == "__main__":
   main(sys.argv[1:])


import urllib.request

# Adding information about user agent
opener=urllib.request.build_opener()
opener.addheaders=[('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1941.0 Safari/537.36')]
urllib.request.install_opener(opener)
image_url_prefix = "https://img.erogazou-pinkline.com/wp-content/uploads/2015/10/suzuki_koharu-1278-"
filename_prefix = "/Users/victorzhao/Desktop/test/2015_"
# setting filename and image URL
# image_url = "https://img.erogazou-pinkline.com/wp-content/uploads/2015/10/suzuki_koharu-1278-002.jpg"
# filename = "/Users/victorzhao/Desktop/test/1.jpg"

for i in range(2, 241):
	image_url = image_url_prefix
	filename = filename_prefix + str(i - 1) + ".jpg"
	if i < 10:
		image_url = image_url + "00" + str(i) + ".jpg"
	elif i < 100:
		image_url = image_url + "0" + str(i) + ".jpg"
	else:
		image_url = image_url + str(i) + ".jpg"
	urllib.request.urlretrieve(image_url, filename)


import re
# 正则表达式
# 正则表达式可以判断目字符串是否符合特定要求
# 比如： 判断是否为手机号。身份证号 邮箱号
# 1XXXxxxxxxx
# 地区号+出生日期+编号 X

"""
# digit
\d ：表示任意的一位数字
\d\d:表示任意的两位数字
# word
\w：表示任意的一个字母和数字
# space
\s：表示 空格

. :表示任意的内容  123 a,b,c,!@# 不单独使用 经常放到字符串后面
a.: 在a后边匹配任意内容  ad  ac  af
*:表示*前边的内容出现0次到多次
a.*：a  a1 ab  aaaa a111 abbbb
+ :表示+前边的内容出现一次到多次
a.+  : aa  ab  ax  am  an  abb
? :表示?前面的内容出现0次到一次
a.? : a  a1
^ : （脱字符）  表示以。。。开头
$ : 表示以。。。结尾
{n} :表示内容重复多次（n次） \d\d\d  = \d{3}
{n，m} 表示最少重复n次，最多重复m次
{n,}  : 表示最少重复n次
{,m} ： 表示最多重复m次
"""
# pattern模式
# compile 编译
# 后面写正则表达式的内容
# （）代表从目标字符串中获取的子串
# 每一个（）就是一个group组
pattern = re.compile('(\d)(\w+)')
content = '123helloWorld'
# match 匹配
result = re.match(pattern , content)

if result :
    # 返回的是一个匹配的对象
    print(result)
    # 返回的是符合要求的全部内容
    print(result.group(0))
    print(result.group(1))
    print(result.group(2))
else:
    print('不符合')

pattern = re.compile('my')
result = re.match(pattern , 'myself')
print(result.group(0))
# match 匹配的是 内容的开始部分
# 作用等同于 startwith
result = re.match(pattern,'love myself')
print(result)

pattern = re.compile('lalala')
result = re.match(pattern,' lalala')
print(result)

pattern = re.compile('my')
result = pattern.match('myself')
print(result)

# 贪婪模式与非贪婪模式
# 正则表达式为贪婪模式 尽量找到符合要求的所有内容
# .* 称之为贪婪模式
content = 'aabbababa11b222'
pattern = re.compile('(a.*b)')
result = pattern.match(content)
print(result)
# .*?   : 称之为 非贪婪模式
# 从a 开始 往后 任意字符出现0次或多次 直到遇见第一个b 就结束
pattern = re.compile('(a.*?b)')
result = pattern.match(content)
print(result)
# 匹配任意字符开头 后面找到一个以b开头  以b结尾的内容
pattern = re.compile('.*?(b.*?b)')
result = pattern.match(content)
print(result)

# * + 同为贪婪模式
# *至少0次 至多无限次
# + 至少1次 至多无限次
content ='33bbb2233bbb'
pattern = re.compile('(a.+b)')
result = pattern.match(content)
print(result)

print('hello \n world')
# raw string 会将字符串里面的转义字符输出出来
print(r'hello \n world')

tilte = '123Hello world'
# pattern = re.compile(r'\d\d\d')
# 匹配连续三个数字
pattern = re.compile(r'\d{3}')
result = pattern.match(tilte)
print(result)

# 匹配全国固话  0371-66666666
pattern = re.compile(r'(\d{4})-(\d{8})')
result = pattern.match('0371-68686688')
print(result)
print(result.group(1))
print(result.group(2))

# | 或者 设置用于不同情况的正则
pattern = re.compile('((haha|heihei)balabala)')
result = pattern.match('hahabalabala')
result = pattern.match('heiheibalabala')
print(result)

# 找到字符串当中第一个符合正则的内容
# 注意：只找到第一个
pattern =re.compile(r'http')
# search 查找
result = pattern.search('www.jd.com,http://www.taobao.com')
print(result)

pattern = re.compile(r'you')
result = pattern.search('I love you , I miss you , I  hate you')
print(result)

pattern = re.compile(r'I')
result = pattern.search(' love you I')
print(result)
# findall  找到所有符合的内容
content = '12345,上山打老虎，老虎没打着，打只小松鼠，55555'
pattern = re.compile(r'\d{5}')

result = pattern.findall(content)
print(result)

content = '杨过对战金轮法王,郭靖观战'
# sub : 替换子串（字符串的一部分）
# \s：匹配空白的内容
pattern = re.compile(r'杨\s*过')
result = pattern.sub('吕布',content)
print(result)

pattern = re.compile(r'金轮法王')
result = pattern.sub('服部半藏',result)
print(result)

key_word = [
    (r'杨\s*过','吕布'),
    (r'金轮法王','服部半藏'),
    (r'郭靖','东方不败')
]
print('---------------------------------------')
for pattern , replace in key_word:
    pattern = re.compile(pattern)
    content = pattern.sub(replace,content)
print(content)

# 匹配手机号
pattern = re.compile(r'^((13[0-9])|(14[67])|(15[0-3|5-9])|(18[0|5-9]))\d{8}$')
result = pattern.match('14674239584')
print(result)

content = '567helloworld'
pattern = re.compile('\d{3}')
result = re.match(pattern, content)
print(result)

content = 'adfhhhkkl1229l0'
pattern = re.compile('(a.*?l)')
result = pattern.match(content)
print(result)

content = 'you are the only one I bilieve in'
pattern = re.compile('o')
result = pattern.findall(content)
print(result)

content = '真相永远只有一个，柯南说的'
pattern = re.compile(r'柯\s*南')
result = pattern.sub('新一',content)
print(result)


# Copyright (c) 2011 Mathias Kaerlev.
# See LICENSE for details.

from twisted.internet import protocol

from lacewing.moo.packetloaders.common import OutOfData

from lacewing.multidict import MultikeyDict

class MooProtocol(protocol.Protocol):
    name = None
    id = None

    isAccepted = False

    channels = None
    _packetBuffer = ''

    def __init__(self):
        self.channels = MultikeyDict()

    def sendLoader(self, loader):
        newPacket = self._sendPacket(**self.settings)
        newPacket.loader = loader
        self.transport.write(newPacket.generate())

    def dataReceived(self, data):
        packetBuffer = ''.join([self._packetBuffer, data])
        newPacket = self._receivePacket(**self.settings)
        try:
            while 1:
                bytesRead = newPacket.load(packetBuffer)
                self.loaderReceived(newPacket.loader)
                packetBuffer = packetBuffer[bytesRead:]
                if len(packetBuffer) == 0:
                    break
        except OutOfData:
            pass
        self._packetBuffer = packetBuffer

    def loaderReceived(self, loader):
        """
        Called upon receiving a packet from the
        other end.
        """
        raise NotImplementedError


from whoosh.analysis import Tokenizer,Token
import jieba

class ChineseTokenizer(Tokenizer):
	def __call__(self, value, positions=False, chars=False,keeporiginal=False, removestops=True,start_pos=0, start_char=0, mode='', **kwargs):
		t = Token(positions, chars, removestops=removestops, mode=mode,
            **kwargs)
		# jieba.load_userdict('userdict.txt')
		seglist=jieba.cut(value,cut_all=False)
		for w in seglist:
			t.original = t.text = w
			t.boost = 1.0
			if (positions):
				t.pos=start_pos+value.find(w)
			if (chars):
				t.startchar=start_char+value.find(w)
				t.endchar=start_char+value.find(w)+len(w)
			yield t

def ChineseAnalyzer():
    return ChineseTokenizer()

# analyzer=ChineseAnalyzer()


# Copyright 2017 VMware, Inc.
# All Rights Reserved
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import helpers as log_helpers
from oslo_log import log as logging
from oslo_utils import excutils

from neutron_lib import constants
from neutron_lib import exceptions as n_exc

from vmware_nsx._i18n import _
from vmware_nsx.common import locking
from vmware_nsx.db import nsxv_db
from vmware_nsx.services.lbaas import base_mgr
from vmware_nsx.services.lbaas import lb_const
from vmware_nsx.services.lbaas.nsx_v import lbaas_common as lb_common

LOG = logging.getLogger(__name__)

type_by_compare_type = {
    lb_const.L7_RULE_COMPARE_TYPE_EQUAL_TO: '',
    lb_const.L7_RULE_COMPARE_TYPE_REGEX: '_reg',
    lb_const.L7_RULE_COMPARE_TYPE_STARTS_WITH: '_beg',
    lb_const.L7_RULE_COMPARE_TYPE_ENDS_WITH: '_end',
    lb_const.L7_RULE_COMPARE_TYPE_CONTAINS: '_sub'
}

def policy_to_application_rule(policy):
    condition = ''
    rule_lines = []
    for rule in policy.rules:
        if rule.provisioning_status == constants.PENDING_DELETE:
            # skip this rule as it is being deleted
            continue

        type_by_comp = type_by_compare_type.get(rule.compare_type)
        if type_by_comp is None:
            type_by_comp = ''
            LOG.warnning('Unsupported compare type %(type)s is used in '
                         'policy %(id)s', {'type': rule.compare_type,
                                           'id': policy.id})

        if rule.type == lb_const.L7_RULE_TYPE_COOKIE:
            # Example: acl <id> hdr_sub(cookie) SEEN=1
            hdr_type = 'hdr' + type_by_comp
            rule_line = ('acl %(rule_id)s %(hdr_type)s(cookie) '
                         '%(key)s=%(val)s' % {'rule_id': rule.id,
                                              'hdr_type': hdr_type,
                                              'key': rule.key,
                                              'val': rule.value})
        elif rule.type == lb_const.L7_RULE_TYPE_HEADER:
            # Example: acl <id> hdr(user-agent) -i test
            hdr_type = 'hdr' + type_by_comp
            rule_line = ('acl %(rule_id)s %(hdr_type)s(%(key)s) '
                         '-i %(val)s' % {'rule_id': rule.id,
                                         'hdr_type': hdr_type,
                                         'key': rule.key,
                                         'val': rule.value})
        elif rule.type == lb_const.L7_RULE_TYPE_HOST_NAME:
            # Example: acl <id> hdr_beg(host) -i abcd
            hdr_type = 'hdr' + type_by_comp
            # -i for case insensitive host name
            rule_line = ('acl %(rule_id)s %(hdr_type)s(host) '
                         '-i %(val)s' % {'rule_id': rule.id,
                                         'hdr_type': hdr_type,
                                         'val': rule.value})
        elif rule.type == lb_const.L7_RULE_TYPE_PATH:
            # Example: acl <id> path_beg -i /images
            # -i for case insensitive path
            path_type = 'path' + type_by_comp
            rule_line = ('acl %(rule_id)s %(path_type)s '
                         '-i %(val)s' % {'rule_id': rule.id,
                                         'path_type': path_type,
                                         'val': rule.value})
        elif rule.type == lb_const.L7_RULE_TYPE_FILE_TYPE:
            # Example: acl <id> path_sub -i .jpg
            # Regardless of the compare type, always check contained in path.
            # -i for case insensitive file type
            val = rule.value
            if not val.startswith('.'):
                val = '.' + val
            rule_line = ('acl %(rule_id)s path_sub '
                         '-i %(val)s' % {'rule_id': rule.id,
                                         'val': val})
        else:
            msg = _('Unsupported L7rule type %s') % rule.type
            raise n_exc.BadRequest(resource='edge-lbaas', msg=msg)

        rule_lines.append(rule_line)
        invert_sign = '!' if rule.invert else ''
        condition = condition + invert_sign + rule.id + ' '

    if rule_lines:
        # concatenate all the rules with new lines
        all_rules = '\n'.join(rule_lines + [''])
        # remove he last space from the condition
        condition = condition[:-1]
    else:
        all_rules = ''
        condition = 'TRUE'

    # prepare the action
    if policy.action == lb_const.L7_POLICY_ACTION_REJECT:
        # return HTTP 403 response
        action = 'http-request deny'
    elif policy.action == lb_const.L7_POLICY_ACTION_REDIRECT_TO_POOL:
        action = 'use_backend pool_%s' % policy.redirect_pool_id
    elif policy.action == lb_const.L7_POLICY_ACTION_REDIRECT_TO_URL:
        action = 'redirect location %s' % policy.redirect_url
    else:
        msg = _('Unsupported L7policy action %s') % policy.action
        raise n_exc.BadRequest(resource='edge-lbaas', msg=msg)

    # Build the final script
    script = all_rules + '%(action)s if %(cond)s' % {
        'action': action, 'cond': condition}
    app_rule = {'name': 'pol_' + policy.id, 'script': script}
    return app_rule

def policy_to_edge_and_rule_id(context, policy_id):
    # get the nsx application rule id and edge id
    binding = nsxv_db.get_nsxv_lbaas_l7policy_binding(
        context.session, policy_id)
    if not binding:
        msg = _('No suitable Edge found for policy %s') % policy_id
        raise n_exc.BadRequest(resource='edge-lbaas', msg=msg)
    return binding['edge_id'], binding['edge_app_rule_id']

class EdgeL7PolicyManager(base_mgr.EdgeLoadbalancerBaseManager):
    @log_helpers.log_method_call
    def __init__(self, vcns_driver):
        super(EdgeL7PolicyManager, self).__init__(vcns_driver)

    def _add_app_rule_to_virtual_server(self, edge_id, vse_id, app_rule_id,
                                        policy_position):
        """Add the new nsx application rule to the virtual server"""
        # Get the current virtual server configuration
        vse = self.vcns.get_vip(edge_id, vse_id)[1]
        if 'applicationRuleId' not in vse:
            vse['applicationRuleId'] = []

        # Add the policy (=application rule) in the correct position
        # (position begins at 1)
        if len(vse['applicationRuleId']) < policy_position:
            vse['applicationRuleId'].append(app_rule_id)
        else:
            vse['applicationRuleId'].insert(policy_position - 1, app_rule_id)

        # update the backend with the new configuration
        self.vcns.update_vip(edge_id, vse_id, vse)

    def _del_app_rule_from_virtual_server(self, edge_id, vse_id, app_rule_id):
        """Delete nsx application rule from the virtual server"""
        # Get the current virtual server configuration
        vse = self.vcns.get_vip(edge_id, vse_id)[1]
        if 'applicationRuleId' not in vse:
            vse['applicationRuleId'] = []

        # Remove the rule from the list
        if (app_rule_id in vse['applicationRuleId']):
            vse['applicationRuleId'].remove(app_rule_id)

        # update the backend with the new configuration
        self.vcns.update_vip(edge_id, vse_id, vse)

    def _update_app_rule_possition_in_virtual_server(self, edge_id, vse_id,
                                                     app_rule_id,
                                                     policy_position):
        """Move the new nsx application rule to another position"""
        # Get the current virtual server configuration
        vse = self.vcns.get_vip(edge_id, vse_id)[1]

        # delete the policy (= application rule) from the list
        if app_rule_id in vse['applicationRuleId']:
            vse['applicationRuleId'].remove(app_rule_id)

        # Add the policy (=application rule) in the correct position
        # (position begins at 1)
        if len(vse['applicationRuleId']) < policy_position:
            vse['applicationRuleId'].append(app_rule_id)
        else:
            vse['applicationRuleId'].insert(policy_position - 1, app_rule_id)

        # update the backend with the new configuration
        self.vcns.update_vip(edge_id, vse_id, vse)

    def _get_vse_id(self, context, pol):
        lb_id = pol.listener.loadbalancer_id
        list_id = pol.listener.id
        listener_binding = nsxv_db.get_nsxv_lbaas_listener_binding(
            context.session, lb_id, list_id)
        if listener_binding:
            return listener_binding['vse_id']

    @log_helpers.log_method_call
    def create(self, context, pol):
        # find out the edge to be updated, by the listener of this policy
        lb_id = pol.listener.loadbalancer_id
        lb_binding = nsxv_db.get_nsxv_lbaas_loadbalancer_binding(
            context.session, lb_id)
        if not lb_binding:
            msg = _(
                'No suitable Edge found for listener %s') % pol.listener_id
            raise n_exc.BadRequest(resource='edge-lbaas', msg=msg)
        edge_id = lb_binding['edge_id']
        app_rule = policy_to_application_rule(pol)
        app_rule_id = None
        try:
            with locking.LockManager.get_lock(edge_id):
                # create the backend application rule for this policy
                h = (self.vcns.create_app_rule(edge_id, app_rule))[0]
                app_rule_id = lb_common.extract_resource_id(h['location'])

                # add the nsx application rule (neutron policy) to the nsx
                # virtual server (neutron listener)
                vse_id = self._get_vse_id(context, pol)
                if vse_id:
                    self._add_app_rule_to_virtual_server(
                        edge_id, vse_id, app_rule_id, pol.position)
        except Exception as e:
            with excutils.save_and_reraise_exception():
                self.lbv2_driver.l7policy.failed_completion(context, pol)
                LOG.error('Failed to create L7policy on edge %(edge)s: '
                          '%(err)s',
                          {'edge': edge_id, 'err': e})
                if app_rule_id:
                    # Failed to add the rule to the vip: delete the rule
                    # from the backend.
                    try:
                        self.vcns.delete_app_rule(edge_id, app_rule_id)
                    except Exception:
                        pass

        # save the nsx application rule id in the DB
        nsxv_db.add_nsxv_lbaas_l7policy_binding(context.session, pol.id,
                                                edge_id, app_rule_id)
        # complete the transaction
        self.lbv2_driver.l7policy.successful_completion(context, pol)

    @log_helpers.log_method_call
    def update(self, context, old_pol, new_pol):
        # get the nsx application rule id and edge id from the nsx DB
        edge_id, app_rule_id = policy_to_edge_and_rule_id(context, new_pol.id)
        # create the script for the new policy data
        app_rule = policy_to_application_rule(new_pol)
        try:
            with locking.LockManager.get_lock(edge_id):
                # update the backend application rule for the new policy
                self.vcns.update_app_rule(edge_id, app_rule_id, app_rule)

                # if the position changed - update it too
                if old_pol.position != new_pol.position:
                    vse_id = self._get_vse_id(context, new_pol)
                    if vse_id:
                        self._update_app_rule_possition_in_virtual_server(
                            edge_id, vse_id, app_rule_id, new_pol.position)

        except Exception as e:
            with excutils.save_and_reraise_exception():
                self.lbv2_driver.l7policy.failed_completion(context, new_pol)
                LOG.error('Failed to update L7policy on edge %(edge)s: '
                          '%(err)s',
                          {'edge': edge_id, 'err': e})

        # complete the transaction
        self.lbv2_driver.l7policy.successful_completion(context, new_pol)

    @log_helpers.log_method_call
    def delete(self, context, pol):
        # get the nsx application rule id and edge id from the nsx DB
        try:
            edge_id, app_rule_id = policy_to_edge_and_rule_id(context, pol.id)
        except n_exc.BadRequest:
            # This is probably a policy that we failed to create properly.
            # We should allow deleting it
            self.lbv2_driver.l7policy.successful_completion(context, pol,
                                                            delete=True)
            return

        with locking.LockManager.get_lock(edge_id):
            try:
                # remove the nsx application rule from the virtual server
                vse_id = self._get_vse_id(context, pol)
                if vse_id:
                    self._del_app_rule_from_virtual_server(
                        edge_id, vse_id, app_rule_id)

                # delete the nsx application rule
                self.vcns.delete_app_rule(edge_id, app_rule_id)
            except Exception as e:
                with excutils.save_and_reraise_exception():
                    self.lbv2_driver.l7policy.failed_completion(context, pol)
                    LOG.error('Failed to delete L7policy on edge '
                              '%(edge)s: %(err)s',
                              {'edge': edge_id, 'err': e})

        # delete the nsxv db entry
        nsxv_db.del_nsxv_lbaas_l7policy_binding(context.session, pol.id)

        # complete the transaction
        self.lbv2_driver.l7policy.successful_completion(context, pol,
                                                        delete=True)


import datetime
import calendar
import time
from time import mktime
from dateutil import rrule
from django.contrib.auth import login, logout as auth_logout
from django.http.response import HttpResponse
from django.shortcuts import render, get_object_or_404, get_list_or_404
from django.http import HttpResponseRedirect, Http404
from django.core.urlresolvers import reverse
from django.views.generic import View
from sendemail.forms import EmailSendForm
from vehicles.models import Collection, SentEmailTemplateLog
from django.template.loader import render_to_string
from django.core.mail import send_mail
from gomoto.utils import send_html_mail
from datetime import datetime

class CollectionView(View):
    """Class used for view classes to inherit from that need to grab the collection object."""

    def get_collection(self, request):
        collection_id = request.session.get('collection_id')
        return get_object_or_404(Collection, pk=collection_id)

class SendEmailFormView(CollectionView):
    """View class used to handle the test drive booking form."""
    form_class = EmailSendForm

    def get(self, request, *args, **kwargs):
        """HTTP GET"""
        initial = {}

        template_list = ['Meeting Confirmation', 'Meeting Follow Up']

        form = self.form_class()

        return render(request, 'sendemail/first.html', {
            'form': form,
            'template_list': template_list
        })

    def post(self, request, *args, **kwargs):
        """HTTP POST"""
        form = self.form_class(request.POST)
        if form.is_valid():
            fname = form.cleaned_data['first_name']
            lname = form.cleaned_data['last_name']
            email = form.cleaned_data['email']
            template = request.POST.get(unicode('template'))
            event_date = form.cleaned_data['event_date']
            event_time = form.cleaned_data['event_time']

            #Send email
            send_new_email(request, fname, lname, email, event_date, event_time, template)

            return render(request, 'sendemail/success.html', {
            })

        else:
            #list_of_tuples = [('1', 'Test'),('2', 'Test 2'),]
            #form = EmailSendForm(list_of_tuples)

            return render(request, 'sendemail/first.html', {
                'form': form,
            })

def emailsent(request):
    """Logs a user out."""
    return render(request, 'sendemail/success.html', {
    })

def logout(request):
    """Logs a user out."""
    auth_logout(request)
    return HttpResponseRedirect(reverse('sendemail_index'))

def send_new_email(request, first_name, last_name, email, date, time, template):
    """Sends a test drive booking email to the customer."""

    if template == 'Meeting Confirmation':
        template = 'sendemail/email/meeting_confirmation.html'
        template_short = 'meeting_confirmation'
    elif template == 'Meeting Follow Up':
        template = 'sendemail/email/meeting_follow_up.html'
        template_short = 'meeting_follow_up'

    #format example: May 5th 2016, 2:15 PM EST
    event_date = datetime.combine(date, time)
    print_date = event_date.strftime('%B %-d %Y, %I:%M %p EST')

    html = render_to_string(template, {
        'first_name': first_name,
        'last_name': last_name,
        'email': email,
        'date': print_date,
    })

    send_to = []

    if email:
        send_to.append(email)
        send_html_mail('Meeting Request', html, 'asmith@shopgomoto.com', send_to)

        SentEmailTemplateLog.objects.create(first_name=first_name, last_name=last_name, email=email, event_date=event_date, template=template_short,
                                            sent_by_first_name = 'Amy', sent_by_last_name='Smith', sent_by_email='asmith@shopgomoto.com')


import time
from urllib import parse
import urllib.request
from multiprocessing import Pool

from bs4 import BeautifulSoup
import re
import PostgresqlDBConnent
import MorphAnal
import json
from PgHelper import PgHelper

# text 정제하기

def clean_text(text):
    # content = text.get_text()
    content = text
    cleaned_text = re.sub('[a-zA-Z]', '', content)
    cleaned_text = re.sub('[\{\}\[\]\/?.,;:|\)*~`!^\-_+<>▶▽♡◀━@\#$%&\\\=\(\'\"ⓒ(\n)(\t)]', '', cleaned_text)
    cleaned_text = cleaned_text.replace("🇲\u200b🇮\u200b🇱\u200b🇱\u200b🇮\u200b🇪\u200b", "")
    cleaned_text = cleaned_text.replace("오류를 우회하기 위한 함수 추가 ", "")
    cleaned_text = cleaned_text.replace("동영상 뉴스 오류를 우회하기 위한 함수 추가 ", "")
    cleaned_text = cleaned_text.replace("무단전재 및 재배포 금지", "")
    cleaned_text = cleaned_text.strip('\n')
    return cleaned_text

def blog_crawling(crawling_url):
    print(crawling_url[0])
    # print(crawling_url[1])
    try:
        req = urllib.request.urlopen(crawling_url[0])
        res = req.read()
        soup = BeautifulSoup(res, 'html.parser')

        try:
            # keywords = soup.findAll('div', {'class': 'se_textView'})
            # keywords = soup.findAll('div', {'class': 'se-main-container'})
            # keywords = soup.findAll('div', {'class': 'se-module se-module-text'})
            keywords = soup.find_all('span', {'class': 'se-fs- se-ff- '})
            # print(len(keywords))
            # print(1)
            if len(keywords) > 0:
                for a in keywords:
                    text = clean_text(a.get_text())
                    if len(text) > 0:
                        # print('\t' + text)
                        PostgresqlDBConnent.update_raw_data(text, crawling_url[1])
                    # print(keywords.get_text(strip=True))
                    # MorphAnal.add_data(keywords.get_text(strip=True))
            else:
                keywords = soup.find_all('div', {'id': 'postViewArea'})
                # print('2')
                if len(keywords) > 0:
                    for a in keywords:
                        # text = a.get_text()
                        text = clean_text(a.get_text())
                        if len(text) > 0:
                            # print(text)
                            PostgresqlDBConnent.update_raw_data(text, crawling_url[1])
                else:
                    keywords = soup.findAll('div', {'class': 'se_textView'})
                    # print('3')
                    if len(keywords) > 0:
                        # print(keywords.get_text(strip=True))
                        # print(keywords)
                        for a in keywords:
                            # text = a.get_text()
                            text = clean_text(a.get_text())
                            if len(text) > 0:
                                # print('\t' + text)
                                PostgresqlDBConnent.update_raw_data(text, crawling_url[1])
                    else:
                        print('xxxxxxxxxxxxxxxx')
        except Exception as err:
            print('??????', err)

    except urllib.error.HTTPError as e:
        # err = e.read()
        code = e.getcode()
        print(code)

def search_keyword(pghelper: object, keyword: object) -> object:
    client_id = "7o0ZnBCQzfLEleVV5JDM"
    client_secret = "uHZM6Mq0hE"
    start = 1
    display = 100
    enc_text = parse.quote(keyword)

    while start <= 1000:
        url = "https://openapi.naver.com/v1/search/blog?query=" + enc_text + "&display=" \
                      + str(display) + "&start=" + str(start)  # json 결과
        # url = "https://openapi.naver.com/v1/search/blog.xml?query=" + encText # xml 결과
        request = urllib.request.Request(url)
        request.add_header("X-Naver-Client-Id", client_id)
        request.add_header("X-Naver-Client-Secret", client_secret)
        response = urllib.request.urlopen(request)
        resp_code = response.getcode()

        if resp_code == 200:
            response_body = response.read()
            # print(response_body.decode('utf-8'))
            jsonobject = json.loads(response_body)
            jsonarray = jsonobject.get("items")
            print("\tTotal : ", jsonobject.get("total"))
            print("\tStart : ", jsonobject.get("start"))
            if jsonobject.get("total") == 0:
                break
            else:
                # PostgresqlDBConnent.keyword_add_db(jsonarray)
                if len(jsonarray) > 0:
                    pghelper.keyword_add_db(jsonarray)
                else:
                    print("xxxxx")
        else:
            print("Error Code:" + str(resp_code))

        start = start + display

if __name__ == '__main__':
    print("Search_Keyword call")
    # Search_Keyword("강원도")
    bloglink = []
    rawdata = []
    tour_api_keyword = []

    start_time = time.time()
    # kangwong = ['춘천 맛집', '원주 맛집', '강릉 맛집', '동해 맛집', '태백 맛집', '속초 맛집', '삼척 맛집', '홍천 맛집', '횡성 맛집',
    #             '영월 맛집', '평창 맛집', '정선 맛집', '철원 맛집', '화천 맛집', '양구 맛집', '인제 맛집', '고성 맛집', '양양 맛집']

    # pghelper = PgHelper()

    # pghelper.get_keyword_data(tour_api_keyword)
    # # print(tour_api_keyword.index('[강릉 바우길 11구간] 신사임당길'))
    # for i in tour_api_keyword[3214:]:
    #     print(i)
    #     Search_Keyword(pghelper, i)

    # for i in kangwong:
    #     print(i)
    #     Search_Keyword(pghelper, i)

    # pghelper.get_blog_link(bloglink)
    # pghelper.CloseDB()
    # url1 ="https://blog.naver.com/PostView.nhn?blogId=soapdiary&logNo=222087156559"
    # url2 ="https://blog.naver.com/PostView.nhn?blogId=soapdiary&logNo=222087156559"
    # bloglink.append([url1, url2])
    # for i in bloglink:
    #     Blog_crawling(i)

    # pool = Pool(processes=8)  # 4개의 프로세스를 사용합니다.
    # pool.map(Blog_crawling, bloglink)  # get_contetn 함수를 넣어줍시다.

    print("--- %s seconds ---" % (time.time() - start_time))
    PostgresqlDBConnent.get_raw_data(rawdata)
    #
    # print(rawdata)
    # print(len(rawdata))
    MorphAnal.process_data(rawdata)
    print("--- %s seconds ---" % (time.time() - start_time))
    # PostgresqlDBConnent.update_raw_data('111asdfasdfasdf',
    # 'https://blog.naver.com/eye4y?Redirect=Log&logNo=222091294214')
    # PostgresqlDBConnent.blog_add_db()
    # Blog_crawing("https://blog.naver.com/PostView.nhn?blogId=d2house&1364")
    # Blog_crawing("https://withbeatles.tistory.com/3544")
    print("Search_Keyword return")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 26 00:04:47 2018

@author: peter
"""

import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from keras.models import Model

from keras.layers  import Masking
import random

import math
from datetime import datetime, timedelta
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score
#from MAPE import mean_absolute_percentage_error
from sklearn.metrics import mean_absolute_error

PATH = '/home/peter/Desktop/Diplomovka/'

features = 5

pad = 581

data = pd.read_csv(f'{PATH}'+'call.csv')
data = data[['Date','Expiry Date','Instrument Symbol','Strike Price', 'CTB3','Close','t','std','Last Price']]
data = data.iloc[100:102]
data = data.dropna(0)
scaler = MinMaxScaler(feature_range=(0, 1))
date = data[['Date','Instrument Symbol','Expiry Date']]
date = pd.DataFrame(date)
date.reset_index(inplace=True, drop=True)
dataset = data.drop(['Date','Instrument Symbol','Expiry Date'], axis = 1)
dataset = dataset.astype('float32')
dataset.shape
dataset = scaler.fit_transform(dataset)
dataset = pd.DataFrame(dataset)
dataset = pd.concat([date, dataset],axis=1, ignore_index=True)
dataset.columns = ['Date','Instrument Symbol','Expiry Date','Strike Price', 'CTB3','Close','t','std','Last Price']

train_size = int(len(dataset) * 0.90)
valid_size = int(len(dataset) * 0.05)
test_size = len(dataset) - train_size - valid_size
trainX, validX, testX = dataset.iloc[0:train_size,:], dataset.iloc[train_size:train_size+valid_size,:], dataset.iloc[train_size+valid_size:len(dataset),:]
#testX, testY = data_dist_max_pad_2_gen_for_test(dataset, testX)

model = Sequential()
model.add(Masking(mask_value=0., input_shape=(pad, 5)))
model.add(LSTM(100))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')

def generator_day(dataset, part, batch_size):
 # Create empty arrays to contain batch of features and labels#
 while True:
    for i in range(batch_size):
      # choose random index in features
      index= part.sample(1, replace=False, weights=None, random_state=None, axis=0)
      dataX, dataY = data_dist_day(index, dataset, part)
      dataX = np.reshape(dataX, (len(dataX),pad,5))
      dataY = np.reshape(dataY,(len(dataY),1))

    yield dataX , dataY

def generator_pred_day(dataset, part, batch_size):
 # Create empty arrays to contain batch of features and labels#
 while True:
    for i in range(batch_size):
      # choose random index in features
      index= part.sample(1, replace=False, weights=None, random_state=None, axis=0)
      dataX, dataY = data_dist_day(index, dataset, part)
      dataX = np.reshape(dataX, (len(dataX),pad,5))
      dataY = np.reshape(dataY,(len(dataY),1))

    yield dataX

def data_dist_day(row, dataset, part):
    dataX, dataY = [], []
    dataset = pd.DataFrame(dataset)
    part = pd.DataFrame(part)
    dat = dataset[dataset['Date'] == row.iloc[0]['Date']]
    dat = dat.sort_values(['Strike Price'])
    result = np.zeros((pad,5))
    X = np.array(dat.iloc[:1,3:dat.shape[1]-1])
    result[:X.shape[0],:X.shape[1]] = X
    dataX.append(result)
    dataY.append(np.array(row.iloc[0]['Last Price']))
    print(dataX, dataY)
    return dataX, dataY

neurons = [5, 15, 50, 100, 150, 300]
scores = {}
scor = []

model = Sequential()
model.add(Masking(mask_value=0., input_shape=(pad, 5)))
model.add(LSTM(15))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit_generator(generator_day(dataset, trainX, 128), steps_per_epoch=int(trainX.shape[0]/128), verbose=1, callbacks=None, validation_data=None, validation_steps=None, class_weight=None, max_queue_size=10, workers=1, use_multiprocessing=False, shuffle=True, initial_epoch=0, epochs=1)
yhat = model.predict_generator(generator_pred_day(dataset, testX, 1), steps=testX.shape[0], max_queue_size=10, workers=1, use_multiprocessing=False)
yhat = pd.DataFrame(yhat)
result = pd.DataFrame(testX.iloc[:,:-1])
result = pd.DataFrame(np.zeros((testX.shape[0], 5)))
inv_yhat = pd.DataFrame(np.concatenate((result, yhat), axis=1))
inv_yhat = scaler.inverse_transform(inv_yhat)
pred = pd.DataFrame(inv_yhat[:,-1])
testY = pd.DataFrame(testX.iloc[:,-1])
inv_y = np.concatenate((result, testY), axis=1)
inv_y = scaler.inverse_transform(inv_y)
Y = pd.DataFrame(inv_y[:,-1])
# calculate metrics
rmse = np.sqrt(mean_squared_error(Y, pred))
print('Test RMSE: %.3f' % rmse)
mae = mean_absolute_error(Y, pred)
print('Test MAE: %.3f' % mae)


"""
Имеется алгоритм, описанный словесно.

• Положить в вазочку N конфет.
• Загнуть на левой руке 3 пальца.
• Бросить монетку на стол.
• Если выпал орёл, увеличить количество конфет в вазочке в А раз, иначе добавить в вазочку В конфет.
• Разогнуть один палец на левой руке.
• Если на левой руке остались загнутые пальцы, перейти к п.3.
Вовочка многократно выполнял этот алгоритм при N=11 (ну нравится ему это число!). В результате примерно с равной вероятностью он получал 26, 32, 37, 42, 49, 54, 64 и 88 конфет в вазочке. А какие значения он бы получил при N=5?
Ответ сформулируйте таким образом, как будто объясняете решение задачи двоечнику-восьмикласснику, которому Вы приходитесь репетитором. Ответ оформите в виде файла MS Word, наличие иллюстраций приветствуется. Излагать теорию вероятности не нужно.
"""

import random
result = {}

def candys(N):
    for i in range(3):
        if random.randint(0, 1) == 0:
            N *= 2
        else:
            N += 5
    return N

for i in range(10000):
    var = candys(5)
    if var in result:
        result[var] += 1
    else:
        result[var] = 1

result_keys = list(result.keys())
result_keys.sort()
for i in result_keys:
    print(i, '->', result[i])

Programming/C#/Procedural Programming C#/Exercises/I/4 Candys/Candys.py


#coding:utf8

import json

from flask import Blueprint, render_template
from flask import request,session,redirect

import models

mod = Blueprint('project', __name__,)
# -----------------------------------
#   项目
# -----------------------------------
@mod.route('/project/')
def project_index():
    # 获取项目组进行渲染
    project_group_list = models.get_project_group_list()
    return render_template('/project/project.html', project_group_list=project_group_list)

'''
ajax 获取项目列表
'''

@mod.route('/project/list/')
def project_list():
    if session.get('user') is None: return redirect('/')
    projects = models.get_projects()
    return json.dumps({'data': projects})

'''
项目详情页
'''

@mod.route('/project/details/')
def project_details():
    if session.get('user') is None: return redirect('/')
    project_id = request.args.get('id', '')
    # 获取项目详情
    project_details = models.get_project_id(project_id)
    # 获取项目组名与项目组ID 关联
    project_group_name_id = models.project_group_name(project_id)
    return render_template('/project/project_details.html', value=project_details,
                           project_group_name_id=project_group_name_id)

'''
项目创建
'''

@mod.route('/project/create/', methods=['POST'])
def project_create():
    # 获取form提交数据
    project_name = request.form.get('project_name', '')
    project_details = request.form.get('project_details', '')
    project_env = request.form.get('project_env', '')
    project_group_name_id_list_s = request.values.getlist('project_group_name')

    # 项目保存前检查
    status = models.validata_project_create(project_name, project_details)
    if not status:
        return json.dumps({'code': 400, 'result': '项目创建失败'})

    # 保存项目
    status, result = models.project_save(project_name, project_details, project_env)
    if not status:
        return json.dumps({'code': 400, 'result': result})

    '''
    保存项目和项目组关联关系：
        1、新项目保存 -> 通过项目名获取其ID(项目ID)
        2、将项目ID与项目组ID存入"project_id_project_group_id"关系表中
    '''
    status, result = models.project_id_project_group_id_correlation(project_group_name_id_list_s, project_name)
    if not status:
        return json.dumps({'code': 400, 'result': result})
    return json.dumps({'code': 200, 'result': result})

'''
项目编辑
'''

@mod.route('/project/view/')
def project_view():
    project_id = request.args.get('id', 0)
    project_ = models.get_project_id(project_id)
    # tmp = models.get_project_id_project_group_id(project_id)
    return json.dumps(project_)

'''
项目 编辑 -->保存
'''

@mod.route('/project/update/', methods=['POST'])
def project_update():
    project_id = request.form.get('id', 0)
    project_name = request.form.get('project_name', 0)
    project_details = request.form.get('project_details', 0)
    project_env = request.form.get('project_env', 0)
    project_status = request.form.get('project_status', 0)
    # 获取项目组ID
    project_group_name_id = request.values.getlist('project_group_name')

    # 检查编辑信息
    # print project_id, project_name, project_details, project_env,project_status

    status, message = models.validata_project_update(project_id, project_name, project_details, project_env,
                                                     project_status)
    if status:
        status, message = models.project_update(project_id, project_name, project_details, project_env, project_status,
                                                project_group_name_id)
        if status:
            return json.dumps({'code': 200, 'error': message})
    return json.dumps({'code': 400, 'error': message})

'''
项目删除
'''

@mod.route('/project/delete/')
def project_delete():
    project_id = request.args.get('id', 0)
    status, message = models.project_delete(project_id)
    if status:
        return json.dumps({'code': 200})
    return json.dumps({'code': 400})


import readTXT
from a_star_multigoal import *
import utility
import time
def mainTestSearch(filename):
    row, col = readTXT.get_size_txt(filename)
    maze = readTXT.read_txt(filename, row, col)
    start,goal_list = readTXT.find_multi_goals(maze)
    frontier = PriorityQueue()
    heu_val, visited, path_cost = a_star_multi_init(maze)
    goal_sequence = []
    start_time = time.clock()
    flag, total_cost, node_expanded, goal_sequence = a_star_multigoal(maze, start, goal_list,
                                                                  goal_sequence, frontier,
                                                                  heu_val, visited,
                                                                  path_cost, 0, 0, 0)
    time_used = time.clock() - start_time
    print("Time Used: ", time_used)
    print("Success = ", flag, "\nPath Cost = ", total_cost, "\nNode Expanded = ", node_expanded)
    print("Goal Sequence: ",goal_sequence)
    print("See multiGoalSolution.txt")
    index = 1
    for i, j in goal_sequence:
        if (1 <= index <= 10):
            maze[i][j] = index-1
        if (11 <= index <= 36):
            maze[i][j] = chr(ord('a') + (index-11))
        if (37 <= index <= 62):
            maze[i][j] = chr(ord('A') + (index-37))
        index+= 1

    withPath = open('multiGoalSolution.txt', 'w')
    for i in range(maze.shape[0]):
        for j in range(maze.shape[1]):
            withPath.write("%s" % maze[i][j])
        withPath.write("\n")
    return 1


import unittest
from pymodbus3.pdu import *

class SimplePduTest(unittest.TestCase):
    """
    This is the unittest for the pymodbus3.pdu module
    """

    def setUp(self):
        """ Initializes the test environment """
        self.badRequests = (
            #ModbusPDU(),
            ModbusRequest(),
            ModbusResponse(),
        )
        self.illegal = IllegalFunctionRequest(1)
        self.exception = ExceptionResponse(1, 1)

    def tearDown(self):
        """ Cleans up the test environment """
        del self.badRequests
        del self.illegal
        del self.exception

    def test_not_implemented(self):
        """ Test a base classes for not implemented functions """
        for r in self.badRequests:
            self.assertRaises(NotImplementedError, r.encode)

        for r in self.badRequests:
            self.assertRaises(NotImplementedError, r.decode, None)

    def test_error_methods(self):
        """ Test all error methods """
        self.illegal.decode("12345")
        self.illegal.execute(None)

        result = self.exception.encode()
        self.exception.decode(result)
        self.assertEqual(result, '\x01')
        self.assertEqual(self.exception.exception_code, 1)

    def test_request_exception_factory(self):
        """ Test all error methods """
        request = ModbusRequest()
        request.function_code = 1
        errors = dict((ModbusExceptions.decode(c), c) for c in range(1, 20))
        for error, code in errors.items():
            result = request.do_exception(code)
            self.assertEqual(str(result), "Exception Response(129, 1, {0})".format(error))

    def test_calculate_rtu_frame_size(self):
        """ Test the calculation of Modbus/RTU frame sizes """
        self.assertRaises(
            NotImplementedError,
            ModbusRequest.calculate_rtu_frame_size, ""
        )
        ModbusRequest._rtu_frame_size = 5
        self.assertEqual(ModbusRequest.calculate_rtu_frame_size(""), 5)
        del ModbusRequest._rtu_frame_size

        ModbusRequest._rtu_byte_count_pos = 2
        self.assertEqual(
            ModbusRequest.calculate_rtu_frame_size("\x11\x01\x05\xcd\x6b\xb2\x0e\x1b\x45\xe6"),
            0x05 + 5
        )
        del ModbusRequest._rtu_byte_count_pos

        self.assertRaises(
            NotImplementedError,
            ModbusResponse.calculate_rtu_frame_size, ""
        )
        ModbusResponse._rtu_frame_size = 12
        self.assertEqual(ModbusResponse.calculate_rtu_frame_size(""), 12)
        del ModbusResponse._rtu_frame_size
        ModbusResponse._rtu_byte_count_pos = 2
        self.assertEqual(
            ModbusResponse.calculate_rtu_frame_size("\x11\x01\x05\xcd\x6b\xb2\x0e\x1b\x45\xe6"),
            0x05 + 5
        )
        del ModbusResponse._rtu_byte_count_pos

#---------------------------------------------------------------------------#
# Main
#---------------------------------------------------------------------------#
if __name__ == "__main__":
    unittest.main()


from flask import render_template
from flask import jsonify
from flask import request
from flask import Flask
app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/api/console')
def api_cmd():
    cmd = request.args.get('cmd')
    print(f'CMD: {cmd}')

    if cmd != '':
        response = {'cod': '200'}
    else:
        response = {
            'cod': '400',
            'error': {
                'type': 'Data Error',
                'msg': 'The server has received bad data'
            }
        }

    print(response)
    return jsonify(response)

if __name__ == "__main__":
    app.run(debug=True)


# code from http://kvfrans.com/simple-algoritms-for-solving-cartpole/

import gym
import numpy as np
import matplotlib.pyplot as plt

def run_episode(env, parameters):

    observation = env.reset()
    total_reward = 0
    num_timestamps = 200

    for _ in range(num_timestamps):
        action = 0 if np.matmul(parameters, observation) < 0 else 1
        observation, reward, done, info = env.step(action)
        total_reward += reward

        if done:
            break

    return total_reward

def train():

    env = gym.make('CartPole-v0')

    counter = 0
    num_iterations = 500
    max_reward = 200

    best_reward = 0
    best_params = None

    for _ in range(num_iterations):
        counter += 1
        obs_space_dim = max(env.observation_space.shape)
        # squeeze random parameters between [-1, 1] with dim of obs_space_dim
        parameters = np.random.rand(obs_space_dim)* 2 - 1
        reward = run_episode(env, parameters)
        if reward > best_reward:
            best_reward = reward
            best_params = parameters
            if reward == max_reward:
                break

    return counter

def main():

    results = list()
    num_episodes = 500
    for _ in range(num_episodes):
        results.append(train())

    plt.hist(results, bins=50, density=1, facecolor='g', alpha=0.75)
    plt.xlabel('Episodes required to reach 200')
    plt.ylabel('Frequency')
    plt.title('Histogram of Random Search')
    plt.show()

    print('Average iterations requried to converge : {}'.format(np.sum(results)/ num_episodes))

if __name__ == '__main__':
    main()


import csv

import pandas as pd
df = pd.read_csv(
  "rat.1.4233369827270508-56.71907424926758-1st-April-9am.csv",
  index_col=0,
  parse_dates=True
)
columns = 4
train_data = [tuple(values) for values in df.iloc[:1122, 0:columns].values]

test_data=[tuple(values) for values in df.iloc[1122:, 0:columns].values]
#print(len(test_data))

#print(test_data[1])
scores = pd.DataFrame(index=range(len(test_data)))

for column in range(4):
    scores[column] = None
    with open("histogram{}.csv".format(column + 1)) as csvfile:
        reader = csv.DictReader(
            csvfile, fieldnames=["start","stop","diff","nor"]
        )
        rows = list(reader)[1:]
    for index in range(len(test_data)):
        for row in rows:
            if (test_data[index][column] >= float(row["start"])) and (test_data[index][column] <= float(row["stop"])
            ):
                scores[column][index] = float(row["nor"])

scores = scores.transpose()
average_scores = scores.sum() / len(scores)

with open("result_hbos.csv", "w") as f5:
    writer = csv.writer(f5)
    writer.writerow(["Data","Score","Is_outlier?"])
    for s in range(len(test_data)):
        if average_scores[s]>.6:
            th=0
        else:
            th=1
        writer.writerow((test_data[s],)+(average_scores[s],)+(th,))

#print(average_scores)


from knock30 import conll_read
from knock35 import word_freqs
import matplotlib.pyplot as plt
import operator

texts = conll_read()
counts = word_freqs(texts)
freqs = []

for x, y in sorted(counts.items()):
    freqs.append(int(y))

plt.hist(freqs, bins = 100)
plt.title('Alice Histogram of Word Frequency')
plt.xlabel('Word Frequency')
plt.ylabel('Number of Unique Words')
plt.show()

# c = {}
# for i in freqs:
#     if i in c:
#         c[i] += 1
#     else:
#         c[i] = 1

# for x, y in sorted(c.items(), key=operator.itemgetter(1), reverse=True):
#     print ('%s: %r' % (x, y))