# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import arrow
import logging
import math
from django.utils.timezone import now as today_date
from datetime import date
import datetime
from dateutil import relativedelta

from django.db import models
from main.exceptions import *
from django.contrib.auth.models import User
from djchoices import DjangoChoices, ChoiceItem
from django.core.exceptions import ObjectDoesNotExist
from django.db.models.signals import post_save

#from jsonfield import JSONField

class RetirementTaxChoices(DjangoChoices):
    taxes_taken_out = ChoiceItem('t')
    not_taxed = ChoiceItem('n')

class DebtTypes(DjangoChoices):
    credit_card = ChoiceItem('c')
    loan = ChoiceItem('l')
    student_loan = ChoiceItem('s')
    other = ChoiceItem('o')

class TaxFilingStatuses(DjangoChoices):
    single = ChoiceItem('s', taxee='single')
    married_joint = ChoiceItem('m', taxee='married')
    married_separate = ChoiceItem('e', taxee='married_separately')
    head_of_household = ChoiceItem('h', taxee='head_of_household')

class InterestCompoundingTypes(DjangoChoices):
    continuous = ChoiceItem('c')
    simple = ChoiceItem('s')

class TimeRange(object):
    def __init__(self, start, end):
        self.months_from_start = 0
        self.start = arrow.get(start)
        today = datetime.datetime.utcnow().date()
        self.relative_month = relativedelta.relativedelta(start, today)
        if self.relative_month.days < 0:
            adjustment = -1
        elif self.relative_month.days > 0:
            adjustment = 1
        else:
            adjustment = 0
        self.relative_month = (self.relative_month.years * 12) + self.relative_month.months + adjustment
        self.end = arrow.get(end)
        if self.end and self.end < self.start:
            pass
            # raise ValueError('End cannot be before start')
        self.current = arrow.get(start)

    @classmethod
    def by_start_with_months(cls, start, months):
        start = arrow.get(start)
        return cls(start.date(), start.replace(months=months).date())

    def __iter__(self):
        while self.current.date() < self.end.date():
            yield True
            self.current = self.current.replace(months=1)
            self.relative_month += 1
            self.months_from_start += 1

    @property
    def length(self):
        delta = relativedelta.relativedelta(self.start.date(), self.end.date())
        if delta.days < 0:
            adjustment = -1
        elif delta.days > 0:
            adjustment = 1
        else:
            adjustment = 0
        return (delta.years * 12) + delta.months + adjustment

class Month(object):
    def __init__(self, money=0, interest=0.0, monthly_amount=0, monthly_income_contribution=0, match_amount=0):
        self._money = money
        self._interest = interest
        self._monthly_amount = monthly_amount
        self._monthly_income_contribution = monthly_income_contribution
        self._match_amount = match_amount

    @property
    def to_string(self):
        return 'Money: %s, Interest: %s, Monthly Amount: %s' % (
            self._money, self._interest, self._monthly_amount
        )

    @property
    def balance(self):
        return self._money

    @property
    def interest(self):
        return self._interest

    @property
    def monthly_amount(self):
        return self._monthly_amount

    @property
    def monthly_income_contribution(self):
        return self._monthly_income_contribution

    @property
    def employer_match_amount(self):
        return self._match_amount

class Months(object):
    def __init__(self):
        self.data = {}

    def setMonth(self, month, data):
        self.data[str(month)] = data

    def getBalance(self, month):
        return self.data[str(month)].balance if str(month) in self.data.keys() else 0

    def getInterest(self, month):
        return self.data[str(month)].interest if str(month) in self.data.keys() else 0

    def getMonthlyAmount(self, month):
        return self.data[str(month)].monthly_amount if str(month) in self.data.keys() else 0

    def getMonthlyIncomeContribution(self, month):
        return self.data[str(month)].monthly_income_contribution if str(month) in self.data.keys() else 0

    def getEmployerMatchAmount(self, month):
        return self.data[str(month)].employer_match_amount if str(month) in self.data.keys() else 0

    def __iter__(self):
        for month in sorted(self.data.iterkeys()):
            yield month

class Compound(models.Model):
    start = models.DateField()
    end = models.DateField(null=True)
    initial_balance = models.FloatField(default=0.00)
    apr = models.FloatField(default=0.00)
    apr_type = models.CharField(choices=InterestCompoundingTypes.choices, max_length=1, default='c')

    recurring_bill = models.FloatField(default=0.00)
    monthly_amount_paid_or_received = models.FloatField(default=0.00)
    yearly_expected_increase_of_monthly_amount = models.FloatField(default=0)
    yearly_expected_increase_of_recurring_bill = models.FloatField(default=0)

    company_match = models.FloatField(default=0)
    company_match_limit = models.FloatField(default=-1)

    class AccountTypes(DjangoChoices):
        income = ChoiceItem('i')
        retirement = ChoiceItem('r')
        debt = ChoiceItem('d')
        housing = ChoiceItem('h')
        spending = ChoiceItem('s')
        auto = ChoiceItem('a')
        saving = ChoiceItem('v')
        investment = ChoiceItem('n')

    @property
    def account_type(self):

        # for x, label in self.AccountTypes.labels.iteritems():
            # try:
            #     return eval('self.%s.AccountTypes' % x)
            # except ObjectDoesNotExist:
            #     pass

        for x in self.AccountTypes.labels:
            try:
                return eval('self.AccountTypes.%s' % x)
            except ObjectDoesNotExist:
                pass

    @property
    def user(self):
        for x, label in self.AccountTypes.labels.iteritems():
            try:
                return eval('self.%s.user' % x)
            except ObjectDoesNotExist:
                pass

    def to_date(self, months=0):
        time_range = TimeRange(self.start, arrow.utcnow().replace(months=months).date())
        money = self.initial_balance
        interest_rate = self.apr
        limit = self.company_match_limit if self.company_match_limit > 0 else 999999999
        answer = Months()
        monthly_amount = self.monthly_amount_paid_or_received
        recurring_bill = self.recurring_bill
        firstRun = True

        # if self.account_type == self.AccountTypes.retirement:
        #     self.retirement_income = self.retirement.income.account
        #     self.retirement_income.to_date(months=months)

        for true in time_range:
            deltas = self.compound_deltas.filter(date__gte=time_range.current.floor('month').datetime,
                                                   date__lte=time_range.current.ceil('month').datetime)

            if time_range.current.date() < self.start:
                answer.setMonth(time_range.relative_month, Month())
            elif self.end and time_range.current.date() > self.end:
                answer.setMonth(time_range.relative_month, Month())
            else:
                if deltas.count() > 0:
                    for delta in deltas:
                        money += delta.balance_adjustment
                        interest_rate += delta.apr_adjustment
                        monthly_amount += delta.monthly_amount_adjustment
                        if delta.monthly_amount_adjustment_set is not None:
                            monthly_amount = delta.monthly_amount_adjustment_set

                # Retirement account linked to an income
                amount_from_tied_income = 0
                # if self.account_type == self.AccountTypes.retirement:
                #     amount_from_tied_income = self.retirement_income.data.getMonthlyAmount(month=time_range.relative_month) * self.retirement.percent_from_income
                #     if firstRun:
                #         firstRun = False
                #         previous_amount_from_tied_income = 0
                #     else:
                #         previous_amount_from_tied_income = self.retirement_income.data.getMonthlyAmount(month=time_range.relative_month - 1) * self.retirement.percent_from_income
                #     monthly_amount += (amount_from_tied_income - previous_amount_from_tied_income)

                if self.account_type == self.AccountTypes.retirement:
                    amount_from_tied_income = self.retirement_income.data.getMonthlyAmount(month=time_range.relative_month) * self.retirement.percent_from_income
                    temp = 1000
                    #amount_from_tied_income = temp * self.retirement.percent_from_income
                    amount_from_tied_income = 600
                    if firstRun:
                        firstRun = False
                        previous_amount_from_tied_income = 0
                    else:
                        #previous_amount_from_tied_income = temp * self.retirement.percent_from_income
                        previous_amount_from_tied_income = 100
                    monthly_amount += (amount_from_tied_income - previous_amount_from_tied_income)

                # Yearly raise
                if time_range.months_from_start > 0 and round((time_range.months_from_start-1) / 12) < round(time_range.months_from_start / 12):
                    monthly_amount *= (1 + self.yearly_expected_increase_of_monthly_amount)
                    recurring_bill *= (1 + self.yearly_expected_increase_of_recurring_bill)

                # 401k matching
                match_amount = self.company_match * monthly_amount if self.company_match > 0 else 0
                if round((time_range.months_from_start - 1) / 12) < round(time_range.months_from_start / 12):
                    limit = self.company_match_limit if self.company_match_limit > 0 else 999999999
                if self.company_match != 0:
                    limit -= match_amount
                    if limit < 0:
                        match_amount -= -limit
                        limit = 0

                if self.apr_type == InterestCompoundingTypes.continuous:
                    interest = (money * (math.e ** (interest_rate / 12))) - money
                else:
                    interest = (interest_rate / 12) * money

                money += recurring_bill
                money += interest
                money += monthly_amount
                money += match_amount
                if self.initial_balance < 0 and money > 0:
                    monthly_amount += -money
                    money += -money

                answer.setMonth(time_range.relative_month, Month(money, interest, monthly_amount, amount_from_tied_income, match_amount))
        self.calculated = answer
        return answer

    @property
    def data(self):
        if hasattr(self, 'calculated'):
            return self.calculated
        else:
            raise ValueError('to_date has not been called yet and you are trying to access it')

class CompoundDelta(models.Model):
    name = models.CharField(max_length=120)
    date = models.DateField()
    balance_adjustment = models.FloatField(default=0)
    apr_adjustment = models.FloatField(default=0)
    monthly_amount_adjustment = models.FloatField(default=0)
    monthly_amount_adjustment_set = models.FloatField(null=True)
    compound = models.ForeignKey(Compound, related_name='compound_deltas')

class Income(models.Model):
    name = models.CharField(max_length=120)
    account = models.OneToOneField(Compound)
    user = models.ForeignKey(User, related_name='income_user', null=True)

class Retirement(models.Model):
    name = models.CharField(max_length=120)
    income = models.ForeignKey(Income)
    account = models.OneToOneField(Compound)
    tax = models.CharField(choices=RetirementTaxChoices.choices, max_length=1)
    interest_rate_at_retirement = models.FloatField(default=.04)
    percent_from_income = models.FloatField(default=0)
    user = models.ForeignKey(User, related_name='retirement_user', null=True)

class Debt(models.Model):
    name = models.CharField(max_length=120)
    account_type = models.CharField(max_length=1, choices=DebtTypes.choices, default='o')
    account = models.OneToOneField(Compound)
    user = models.ForeignKey(User, related_name='debt_user', null=True)

class Housing(models.Model):
    name = models.CharField(max_length=120)
    account = models.OneToOneField(Compound)
    property_insurance = models.FloatField(default=100.0)
    renters_insurance_rate = models.FloatField(default=.03)
    pmi_rate = models.FloatField(default=.03) # percent of remaining loan balance, only in effect if loan balance <= 80% of property value
    purchase_price = models.FloatField(default=0)
    o_property_value = models.FloatField(default=0)
    o_property_value_date = models.DateField(default=date.today)
    property_tax_percent = models.FloatField(default=0)
    appreciation_percent = models.FloatField(default=0)
    is_rental = models.BooleanField(default=False)
    user = models.ForeignKey(User, related_name='housing_user', null=True)

    @property
    def renters_insurance(self):
        return self.renters_insurance_rate * self.o_property_value

    def pmi_value(self, month):
        loan_balance = self.account.data.getBalance(month=month)
        rate = (loan_balance * self.pmi_rate) / 12
        return abs(rate) if abs(loan_balance) >= int(.8*self.property_value(month=month)) else 0

    def maintenance_costs(self, month):
        return (.005 / 12) * self.property_value(month=month)

    def calculate_property_value_to_month(self, month):
        #print 'in calculated'
        self.calculated = {}
        if self.o_property_value and self.o_property_value_date:
            time_range = TimeRange(self.o_property_value_date, arrow.utcnow().replace(months=month).date())
            propertyValue = self.o_property_value
            start = self.o_property_value_date
        else:
            time_range = TimeRange(self.account.start, arrow.utcnow().replace(months=month).date())
            propertyValue = self.purchase_price
            start = self.account.start
        x = -16
        for true in time_range:
            if time_range.current.date() >= start:
                propertyValue *= (1 + (self.appreciation_percent / 12))
                self.calculated[time_range.relative_month] = propertyValue
            logging.debug('%s %s: %s' % (x, time_range.current, propertyValue))
            x += 1
        return propertyValue

    def property_value(self, month):
        return 0.05
        # return self.calculated[month]

class Spending(models.Model):
    name = models.CharField(max_length=120)
    account = models.OneToOneField(Compound)
    lifestyle_spending = models.BooleanField(default=False)
    user = models.ForeignKey(User, related_name='spending_user', null=True)

class Auto(models.Model):
    name = models.CharField(max_length=120)
    account = models.OneToOneField(Compound)
    auto_insurance = models.FloatField(default=100)
    maintenance_costs = models.FloatField(default=75)
    salvage_percent = models.FloatField(default=.167)
    purchase_price = models.FloatField(null=True)
    user = models.ForeignKey(User, related_name='auto_user', null=True)

    def to_date(self, months=0):
        months = TimeRange.by_start_with_months(self.account.start, months).length
        purchase_price = self.purchase_price if self.purchase_price else self.account.initial_balance
        salvage_step = (purchase_price - (self.salvage_percent * purchase_price)) / 96
        if months >= 96:
            return self.salvage_percent * purchase_price
        else:
            return purchase_price - months * salvage_step

    @property
    def salvage_value(self):
        return self.account.initial_balance * self.salvage_percent

class Saving(models.Model):
    name = models.CharField(max_length=120)
    account = models.OneToOneField(Compound)
    user = models.ForeignKey(User, related_name='saving_user', null=True)

class Investment(models.Model):
    name = models.CharField(max_length=120)
    account = models.OneToOneField(Compound)
    user = models.ForeignKey(User, related_name='investment_user', null=True)

class Profile(models.Model):
    user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='profile')

    date_of_birth = models.DateField(null=True, default=date.today)
    retirement_age = models.IntegerField(default=65)
    death_age = models.IntegerField(default=85)
    required_income_during_retirement = models.FloatField(default=.8)
    plan_for_social_security = models.BooleanField(default=False)
    exemptions = models.IntegerField(default=2)
    taxable_state = models.CharField(max_length=2, default='TX')
    tax_filing_status = models.CharField(max_length=1, choices=TaxFilingStatuses.choices, null=True)
    cash_on_hand = models.FloatField(default=0)
    cash_on_hand_date = models.DateField(default=date.today)
    opt_in_newsletters = models.BooleanField(default=False)
    first_name = models.CharField(max_length=64, null=True)
    last_name = models.CharField(max_length=64, null=True)
    email = models.EmailField(null=True)
    home_phone_number = models.CharField(max_length=15, null=True)  # Add phone no validator
    cell_phone_number = models.CharField(max_length=15, null=True)
    street_address = models.CharField(max_length=128, null=True)
    city = models.CharField(max_length=64, null=True)
    zip_code = models.IntegerField(null=True)
    state = models.CharField(max_length=64, null=True)
    number_of_dependents = models.CharField(max_length=1, null=True)

    @property
    def retirement_date(self):
        return arrow.get(self.date_of_birth).replace(years=self.retirement_age)

    @property
    def death_date(self):
        return arrow.get(self.date_of_birth).replace(years=self.death_age)

    @property
    def months_between_now_and_retirement(self):
        x = 0
        for true in TimeRange(arrow.utcnow().datetime.date(), self.retirement_date.datetime.date()):
            x += 1
        return x

    @property
    def months_between_now_and_death(self):
        x = 0
        for true in TimeRange(arrow.utcnow().datetime.date(), self.death_date.datetime.date()):
            x += 1
        return x

    def create_profile(sender, instance, created, **kwargs):
        if created:
            Profile.objects.create(user=instance)

    # post_save.connect(create_profile, sender=User)

class Calculating(models.Model):
    #result = models.CharField(max_length=255)
    #result = JSONField()
    result = models.TextField()
    user = models.ForeignKey(User, related_name='calculating_user', null=True)


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import optimize
import math as m

def write_answer_to_file(answer, filename):
    with open(filename, 'w') as f_out:
        f_out.write(str(round(answer, 3)))

data = pd.read_csv('advertising.csv')

print(data.head())
print(data.info())

X = data[['TV','Radio','Newspaper']].values
Y = data[['Sales']].values

meanX = X.mean(axis=0)
stdX = X.std(axis=0)

print("+++++++++++++++++++++++++++++")

#print(meanX)
#print(stdX)

print("+++++++++++++++++++++++++++++")

#print(X)

print("+++++++++++++++++++++++++++++")

X = (X-meanX)/stdX
#print(X)

tmp = np.ones((200, 1))
X = np.hstack((tmp, X))

def mserror(y:np.array, y_pred:np.array)->float:
    tmp = (y-y_pred)**2
    return tmp.sum()/y.size

medianY = np.median(Y)
print("*****************************************")
print(medianY)

tmp = np.full(Y.shape,medianY)
print("*****************************************")
print(tmp[0][0])
print("*****************************************")
print(mserror(tmp, Y))
write_answer_to_file(mserror(Y, tmp), "1.txt")

def normal_equation(X, y):
    return np.linalg.inv(X.transpose().dot(X)).dot(X.transpose()).dot(y)

norm_eq_weights = normal_equation(X, Y)

#write_answer_to_file(norm_eq_weights, "2.txt")

print(norm_eq_weights)

ans2 = norm_eq_weights[0][0]#X.mean(axis=0).dot(norm_eq_weights)
print(ans2)
write_answer_to_file(ans2, "2.txt")

def linear_prediction(X, w):
    return X.dot(w)

y_pred = linear_prediction(X, norm_eq_weights)
#print(Y)
print("=======================================")
#print(y_pred)

ans3 = mserror(Y, y_pred)

print(ans3)

write_answer_to_file(ans3, "3.txt")

def stochastic_gradient_step(X, y, w, train_ind, eta=0.01):
    l=y.size
    grad0 = (2.0/l)*(w[0]+w[1]*X[train_ind][1]+w[2]*X[train_ind][2]+w[3]*X[train_ind][3]-y[train_ind]) # Ваш код здесь
    grad1 = (2.0/l)*X[train_ind][1]*(w[0]+w[1]*X[train_ind][1]+w[2]*X[train_ind][2]+w[3]*X[train_ind][3]-y[train_ind])# Ваш код здесь
    grad2 = (2.0/l)*X[train_ind][2]*(w[0]+w[1]*X[train_ind][1]+w[2]*X[train_ind][2]+w[3]*X[train_ind][3]-y[train_ind])# Ваш код здесь
    grad3 = (2.0/l)*X[train_ind][3]*(w[0]+w[1]*X[train_ind][1]+w[2]*X[train_ind][2]+w[3]*X[train_ind][3]-y[train_ind])# Ваш код здесь
    return  w - eta * np.array([grad0, grad1, grad2, grad3])

def stochastic_gradient_descent(X, y, w_init, eta=1e-2, max_iter=1e4,
                                min_weight_dist=1e-8, seed=42, verbose=False):
    # Инициализируем расстояние между векторами весов на соседних
    # итерациях большим числом.
    weight_dist = np.inf
    # Инициализируем вектор весов
    w = w_init
    # Сюда будем записывать ошибки на каждой итерации
    errors = []
    # Счетчик итераций
    iter_num = 0
    # Будем порождать псевдослучайные числа
    # (номер объекта, который будет менять веса), а для воспроизводимости
    # этой последовательности псевдослучайных чисел используем seed.
    np.random.seed(seed)

    # Основной цикл
    while weight_dist > min_weight_dist and iter_num < max_iter:
        # порождаем псевдослучайный
        # индекс объекта обучающей выборки
        random_ind = np.random.randint(X.shape[0])

        # Ваш код здесь
        w = stochastic_gradient_step(X, y, w, random_ind, eta)

        y_pred = linear_prediction(X, w)
        errors.append(mserror(Y, y_pred))
        iter_num += 1

    return w, errors

stoch_grad_desc_weights, stoch_errors_by_iter = stochastic_gradient_descent(X, Y, np.zeros((4,1)),max_iter=1e5)

plt.plot(range(50), stoch_errors_by_iter[:50])
plt.xlabel('Iteration number')
plt.ylabel('MSE')
#plt.show()

plt.plot(range(len(stoch_errors_by_iter)), stoch_errors_by_iter)
plt.xlabel('Iteration number')
plt.ylabel('MSE')
#plt.show()

print(stoch_grad_desc_weights)

answer4 = mserror(X.dot(stoch_grad_desc_weights), Y)

print(answer4)

write_answer_to_file(answer4, '4.txt')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/12/21 下午4:16
# @Author  : zhaoyu
# @Site    :
# @File    : test_lstm.py
# @Software: PyCharm

import os
os.environ['KMP_DUPLICATE_LIB_OK']='True'
from numpy import array
from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Dense
from keras.utils import plot_model

if __name__ == '__main__':
    length = 5
    seq = array([i / float(length) for i in range(length)])
    X = seq.reshape(len(seq), 1, 1)
    Y = seq.reshape(len(seq), 1)

    n_neurons = length
    n_batch = length
    n_epoch = 1000

    model = Sequential()
    model.add(LSTM(n_neurons, input_shape=(1, 1)))
    model.add(Dense(1))
    model.compile(loss='mean_squared_error', optimizer='adam')
    print(model.summary())

    model.fit(X, Y, epochs=n_epoch, batch_size=n_batch, verbose=2)
    result = model.predict(X, batch_size=n_batch, verbose=0)
    for value in result:
        print('%.1f' % value)

    plot_model(model, to_file='lstm.png')


import markdown
from django.contrib.staticfiles import finders
from wiki.models.urlpath import URLPath
from wiki.models.article import Article, ArticleRevision
from wiki.core.exceptions import NoRootURL

docs_manifest = ["root.md", "develop.md", "use.md", "deploy.md",
                 "use/analyze.md", "use/export.md", "use/input.md",
                 "use/search.md", "use/wiki.md", "develop/django.md",
                 "develop/qiime2.md", "develop/reports.md", "develop/schema.md",
                 "deploy/backup.md", "deploy/install.md", "deploy/secure.md",
                 "deploy/serve.md", "deploy/storage.md", "deploy/upgrade.md"]

class WikiStatic(object):
    def __init__(self, slug, template):
        #Slug is a string, in the case of a static wiki page
        self.slug = slug
        self.template = template
        self.content, self.title = self.process_template()
        self._refresh_root()

    def _refresh_root(self):
        if "/" not in self.slug:
            try:
                self.root = URLPath.get_by_path("/")
            except:
                if self.slug == "root":
                    self.root = self._create_root()
                else:
                    raise ValueError("No root created. Make sure root.md is processed first")
        else:
            try:
                root_name = "/".join(self.slug.split("/")[:-1])
                self.root = URLPath.get_by_path(root_name)
            except:
                self.root = None

    def update_wiki(self):
        try:
            print("Retrieving slug %s" % (self.slug,))
            wiki_page = URLPath.get_by_path(self.slug)
            #Create a new revision and update with the template content
            article = wiki_page.article
            article_revision = ArticleRevision(title=article.current_revision.title,
                                               content=self.content)
            article.add_revision(article_revision)
        except URLPath.DoesNotExist:
            print("Creating a new article")
            if self.root is None:
                self._refresh_root()
            print("Using root %s" % (self.root,))
            print("Pushing to slug %s"% (self.slug,))
            base_slug = self.slug.split("/")[-1]
            wiki_page = URLPath.create_urlpath(self.root, slug=base_slug,
                                         title=self.title,
                                         content=self.content)

    def process_template(self, static_prefix="docs/"):
        #Load self.template and read it
        filepath = finders.find(static_prefix + self.template)
        if filepath is not None:
            with open(filepath, 'r') as md_file:
                md_str = md_file.read()
                md = markdown.Markdown(extensions=['meta'])
                html = md.convert(md_str)
                if 'title' in md.Meta:
                    title = md.Meta['title'][0]
                else:
                    title = self.slug.capitalize()
                content = md_str
        else:
            raise FileNotFoundError("Static document %s not found" % (self.template,))
        return content, title

    def _create_root(self):
        root = URLPath.create_root(title="QUOR'em Wiki",
                                   content=self.content)
        return root

docs_wiki = [WikiStatic(slug=x.split(".md")[0],
                        template=x) for x in docs_manifest]


import scrapy
import re

BASE_URL='https://en.wikipedia.org'

def process_winner_li(winner, country=None):
  wdata={}
  pass
  wdata['link']=BASE_URL + winner.xpath('a/@href').extract()[0]
  text=" ".join(winner.xpath("descendant-or-self::text()").extract())
  wdata['name'] = text.split(',')[0].strip()
  year=re.findall('\d{4}',text)
  if year:
    wdata['year']=year[0]
  else:
    wdata['year']=0
    print('No year in ',text)

  category = re.findall(
    'Physics|Chemistry|Physiology or Medicine|Literature|Peace|Economics',
    text
    )
  if category:
    wdata['category']=category[0]
  else:
    wdata['category']=''
    print('No category in ',text)

  if text.find('*')!=-1:
    wdata['country']=''
    wdata['born_in']=country
  else:
    wdata['country']=country
    wdata['born_in']=''

  wdata['text']=text
  return wdata

class NWinnerItem(scrapy.Item):
  country=scrapy.Field()
  born_in=scrapy.Field()
  name=scrapy.Field()
  link=scrapy.Field()
  text=scrapy.Field()
  year=scrapy.Field()
  category=scrapy.Field()
  place_of_birth=scrapy.Field()
  place_of_death=scrapy.Field()
  date_of_death=scrapy.Field()
  date_of_birth=scrapy.Field()
  gender=scrapy.Field()

class NWinnerSpider(scrapy.Spider):
  name='winners_list'
  allowed_domains=['en.wikipedia.org']
  start_urls=['https://en.wikipedia.org/wiki/List_of_Nobel_laureates_by_country']

  def parse(self, response):
    h2s=response.xpath('//h2')

    for h2 in h2s:
      country=h2.xpath('span[@class="mw-headline"]/text()').extract()
      if country:
        winners=h2.xpath('following-sibling::ol[1]')
        for winner in winners.xpath('./li'):
          wdata=process_winner_li(winner,country[0])
          yield NWinnerItem(
            country=wdata['country'],
            name=wdata['name'],
            link=wdata['link'],
            text=wdata['text'],
            category=wdata['category'],
            year=wdata['year']
            )

class NWinnerSpider(scrapy.Spider):
  def parse_bio(self,response):
    item = response.meta['item']
    href=response.xpath("//li[@id='t-wikibase']/a/@href").extract()
    if href:
      request=scrapy.Request(
        href[0],
        callback=self.parse_wikidata,
        dont_filter=True
        )
      request.meta['item']=item
      yield request

  def parse_wikidata(self, response):
    item= response.meta['item']

    property_codes = [
    {'name':'date_of_birth', 'code':'P569'},
    {'name':'date_of_death', 'code':'P570'},
    {'name':'place_of_birth', 'code':'P19', 'link':True},
    {'name':'place_of_death', 'code':'P20', 'link':True},
    {'name':'gender', 'code':'P21', 'link':True}
    ]

    p_template = '//*[@id="{code}"]/div[2]/div/div/div[2]' \
    '/div[1]/div/div[2]/div[2]{link_html}/text()'

    for prop in property_codes:
      link_html=''
      if prop.get('link'):
        link_html='/a'
      sel=response.xpath(p_template.format(
        code=prop['code'],
        link_html=link_html
        )
      )
      if sel:
        item[prop['name']]=sel[0].extract()

      yield item

  name='winners_full'
  allowed_domains=['en.wikipedia.org']
  start_urls=['https://en.wikipedia.org/wiki/List_of_Nobel_laureates_by_country']

  def parse(self, response):
    h2s=response.xpath('//h2')

    for h2 in h2s:
      country=h2.xpath('span[@class="mw-headline"]/text()').extract()
      if country:
        winners=h2.xpath('following-sibling::ol[1]')
        for winner in winners.xpath('./li'):
          wdata=process_winner_li(winner,country[0])
          request=scrapy.Request(
            wdata['link'],
            callback=self.parse_bio,
            dont_filter=True
            )
          request.meta['item']=NWinnerItem(**wdata)
          yield request


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 26 09:17:19 2018

@author: nsde
"""
#%%
from torch.utils.cpp_extension import load

#%%
square_cuda = load(name = 'square_cuda',
                   sources = ['square_cuda.cpp', 'square_cuda_kernel.cu'],
                   verbose=False,
                   extra_include_paths = ['/usr/local/cuda-8.0/lib64/'])


OSC_template  =  \
"""<device>
    <c_type>       OUT_SIGNAL_CHANGE     </c_type>
    <UDID>         %(udid)s                   </UDID>
    <pin>          %(pin)s                     </pin>
    <port>         %(port)s                     </port>
    <c_version>    %(version)s                   </c_version>
</device>"""

ISC_template =  \
"""<device>
    <c_type>       IN_SIGNAL_CHANGE_POLL </c_type>
    <UDID>         %(udid)s                   </UDID>
    <pin>          %(pin)s                     </pin>
    <port>         %(port)s                     </port>
    <sample_rate>  %(sample_rate)s                     </sample_rate>
    <trigger>      %(trigger)s                  </trigger>
    <c_version>    %(version)s                   </c_version>
</device>"""

ISAP_template = \
"""<device>
    <c_type>       IN_SIGNAL_ADC_POLL </c_type>
    <UDID>         %(udid)s                   </UDID>
    <pin>          %(pin)s                     </pin>
    <sample_rate>  %(sample_rate)s                   </sample_rate>
    <threshold>    %(threshold)s                    </threshold>
    <c_version>    %(version)s                   </c_version>
</device>"""

arc_config_template = \
"""<?xml version="1.0"?>
<data>
    <network>
%(network)s
    </network>
    <node_list>
%(node_l)s
    </node_list>
</data>"""

network_template = \
"""
<boud_rate>      %(boud_rate)s    </boud_rate>
<PANID>          %(PANID)s     </PANID>
<channel>        %(channel)s      </channel>
<encryption_key> 132456  </encryption_key>
"""

node_template = \
"""<node>

    <node_id>      %(node_id)s              </node_id>
    <node_address> %(address)s   </node_address>
    <node_arc>     avr8a            </node_arc>

    <devices>
%(device_l)s
    </devices>
</node>
"""

# ========================================= home config templates =============================

home_device_template = \
"""
<device>
    <type>       %(device_type)s </type>
    <device_id>  %(device_id)s                </device_id>
    <tag>        %(tag)s </tag>
    <zone>      %(zone)s       </zone>
    <c_device>
%(subdevices)s
    </c_device>
</device>
"""

home_config_template = \
"""<?xml version="1.0"?>
<data>
%(devices)s
</data>"""


from urllib.request import urlopen
from urllib.error import HTTPError, URLError
from sys import exit

try:
    u = urlopen( "http://www.python.org" )
except HTTPError as e:
    print( "HTTP Error", e )
    sys.exit()
except URLError as e:
    print( "URL error", e )
    sys.exit()

for i in range( 5 ):
    text = u.readline()
    print( text )

u.close()

Intro to CS with Python/src/homework/listing2703.py


import json
import logging
from collections import OrderedDict
import datetime
from dateutil import parser
from AnalysisEngine.Analysis import Analysis
from AnalysisEngine.TwitterObj import Status, User
from AnalysisEngine import Util
from AnalysisEngine.UserProfiling.UserProfile import UserProfile

class UserProfiling(Analysis):
    _logger = logging.getLogger(__name__)
    _options = ["min", "H", "D", "W"]
    __arguments = [dict(name="user_screen_name", prettyName="User Screen Name", type="string",
                        default="iVoteLeave"),
                   dict(name="Limit", prettyName="Tweet to Process", type="integer",
                         default=1000),
                   dict(name="DisplayLimit", prettyName="Tweet to Display", type="integer",
                        default=100),
                   dict(name="timeInterval", prettyName="Time interval", type="enum", options=_options, default="H"),
                   dict(name="hashtag_grouping", prettyName="Hashtag Groupings", type="dictionary_list", variable=False,
                        default=[
                            dict(name="Leave",
                                 tags=["no2eu", "notoeu", "betteroffout", "voteout", "eureform", "britainout",
                                       "leaveeu", "voteleave", "beleave", "loveeuropeleaveeu"], color=None),
                            dict(name="Remain", tags=["yes2eu", "yestoeu", "betteroffin", "votein", "ukineu", "bremain",
                                                      "strongerin", "leadnotleave", "voteremain"], color=None)])
                   ]

    def __init__(self, analytics_meta):
        super(UserProfiling, self).__init__(analytics_meta)

    @classmethod
    def get_classification(cls):
        return "User"

    @classmethod
    def get_type(cls):
        return "Profile"

    def get_prefered_result(self):
        return "url_html"

    def get_prefered_gdo_app(self):
        return "StaticHTML"

    @classmethod
    def get_args(cls):
        return cls.__arguments + super(UserProfiling, cls).get_args()

    def process(self):

        user_screen_name = self.args["user_screen_name"]
        self.analytics_meta.description += "(" + user_screen_name + ")"
        limit = self.args["Limit"]
        user_name_key = Util.join_keys(Status.SCHEMA_MAP[self.schema]["user"],
                                              User.SCHEMA_MAP[self.schema]["name"])
        time_interval = self.args["timeInterval"]
        display_limit = self.args["DisplayLimit"]
        hashtag_args = self.args["hashtag_grouping"]

        hashtag_groupings = dict([(i, -1) for i in hashtag_args[0]["tags"]] +
                                 [(i, 1) for i in hashtag_args[1]["tags"]])

        query = self.get_time_bounded_query({user_name_key: user_screen_name})
        cursor = self.get_sorted_cursor(query, limit, reverse=True)
        if cursor is None:
            self.analytics_meta.status = "NO DATA IN RANGE"
            self.analytics_meta.save()
            return False

        user_profile = UserProfile(user_screen_name,
                                   self.dataset_meta.description,
                                   str(hashtag_args[0]["name"]),
                                   str(hashtag_args[1]["name"]),
                                   hashtag_groupings,
                                   display_limit)

        n_retweets = 0
        n_original = 0
        first = True
        self._logger.info("Beginning profiling")

        for i,c in enumerate(cursor):
            self._logger.info("Consuming status: " + str(i))
            s = Status(c, self.schema)
            if first:
                user = s.get_user()
                user_profile.no_followers =  user.get_follower_count()
                user_profile.no_friends = user.get_friends_count()
                user_profile.timestamp = s.get_created_at()
                user_profile.createdAt = user.get_profile_creation_date()
                user_profile.name = user.get_real_name()
                user_profile.profileImage = user.get_image_url().replace("_normal","")
                user_profile.timeZone = user.get_time_zone()
                user_profile.location = user.get_location()
                user_profile.description = user.get_description()
                first = False
            if s.get_retweet_status() is None:
                n_original += 1
            else:
                n_retweets += 1
            user_profile.add_tweet(s)

        user_profile.no_retweets = n_retweets
        user_profile.no_original = n_original
        user_profile.process_time_dist(time_interval)
        self._logger.info("Finished profiling")

        self.export_html(user_profile, export_type="userProfile")
        # self.export_json(user_profile.__dict__)

        return True


import os

from django.template import Template, Context

def clamp(val, v_min, v_max):
    return max(v_min, min(val, v_max))

def render_template(path, context):
    f = open(os.path.join('templates', path), 'r')
    raw = f.read()
    f.close()

    t = Template(raw)
    return t.render(Context(context))

# Aggiunge al context le variabili utili a visualizzare il form per l'aggiunta dell'autore
def append2context_authorform(context, request, form, **kwargs):
    context.update(addauthor_form=form,
                   addauthor_success=request.GET.get('addauthor') == 'success',
                   addauthor_error=request.GET.get('addauthor') == 'error',
                   addauthor_success_msg=kwargs.get('success_msg_list'),
                   addauthor_error_msg=kwargs.get('error_msg_list'),
                   )

# Aggiunge al context le variabili utili a visualizzare il form per l'aggiunta/modifica del libro
def append2context_bookform(context, request, form, edit=False, **kwargs):
    key_prefix = 'edit_' if edit else 'add_'
    context.update({
        key_prefix + 'book_action': kwargs.get('action'),
        key_prefix + 'book_form': form,
        key_prefix + 'book_success': request.GET.get(key_prefix + 'book') == 'success',
        key_prefix + 'book_error': request.GET.get(key_prefix + 'book') == 'error',
        key_prefix + 'book_success_msg': kwargs.get('success_msg_list'),
        key_prefix + 'book_error_msg': kwargs.get('error_msg_list'),
    })


# -*- coding: utf-8
from openpyxl.reader.excel import load_workbook
from torcms.model.minforcatalog import MInforCatalog

wb = load_workbook(filename='./database/meta/info_tags.xlsx')
sheet_ranges_arr = [wb['Sheet1'], wb['Sheet2']]
class_arr = ['D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T']
sig_name_arr = []
mappcat = MInforCatalog()

def uu():
    # 在分类中排序
    order_index = 1
    # 父类索引
    papa_index = 1
    # 子类索引
    c_index = 1
    papa_id = 0
    uid = ''
    p_dic = {}
    # 逐行遍历
    for sheet_ranges in sheet_ranges_arr:

        for row_num in range(3, 48):
            # 父类
            p_cell_val = sheet_ranges['A{0}'.format(row_num)].value
            if p_cell_val and p_cell_val != '':
                cell_arr = p_cell_val.split(',')
                p_uid = cell_arr[0].strip().strip('t')
                priv_mask = cell_arr[1].strip().strip('t')
                t_name_arr = sheet_ranges['B{0}'.format(row_num)].value.strip().split(',')
                u_uid = '{0}00'.format(p_uid)

            # 子类
            b_cell_val = sheet_ranges['B{0}'.format(row_num)].value
            c_cell_val = sheet_ranges['C{0}'.format(row_num)].value
            if c_cell_val and c_cell_val != '':
                cell_arr = b_cell_val.split(',')
                c_iud = cell_arr[0].strip().strip('t')
                priv_mask = cell_arr[1].strip().strip('t')
                t_name_arr = c_cell_val.strip().split(',')
                u_uid = '{0}{1}'.format(p_uid, c_iud)
            post_data = {
                'name': [t_name_arr[0]],
                'slug': [t_name_arr[1]],
                'order': [order_index],
                'uid': [u_uid],
                'priv_mask': [priv_mask],
            }
            print(post_data)
            mappcat.insert_data(u_uid, post_data)
            order_index = order_index + 1

if __name__ == '__main__':
    uu()


import pandas as pd
import torch
import numpy as np
import json

config = ''
with open('config.json', 'r') as f:
    config = json.load(f)

tag = pd.read_csv("data/tags.csv")
job = pd.read_csv("data/job_tags.csv")
user = pd.read_csv("data/user_tags.csv")
initdata = None
userkeyword = user.merge(tag, on="tagID", how="left").drop_duplicates()
userkeyword = userkeyword.groupby("userID")["keyword"].apply(list).reset_index(name="tags")
userkeyword.rename(columns = {'userID' : 'ID'}, inplace = True)
userkeyword['Cnt'] = userkeyword['tags'].apply(len)

jobkeyword = job.merge(tag, on="tagID", how="left").drop_duplicates()
jobkeyword = jobkeyword.groupby("jobID")["keyword"].apply(list).reset_index(name="tags")
jobkeyword.rename(columns = {'jobID' : 'ID'}, inplace = True)
jobkeyword['Cnt'] = jobkeyword['tags'].apply(len)
allkeyword = userkeyword.append(jobkeyword).reset_index(drop=True)   # userid, tags 또는 jobid, tags로 구성된 데이터셋

def onehot(keyword):
    find = False
    onehot_vec = np.zeros(tag.shape[0])
    assert (tag.index[ tag["keyword"] == keyword] + 1).any(), keyword
    onehot_vec[tag.index[ tag["keyword"] == keyword] ] = 1
    return onehot_vec

class Word2VecDataSet(torch.utils.data.Dataset):
    def __init__(self, data):  # 데이터에 list 형태의 tags 필드가 존재해야 함
        super().__init__()
        self.dataset = []
        maxc = data.Cnt.max()
        for i in range(len(data)):
            keywords = data.tags[i]
            for j in range(maxc):
                idx = j % len(keywords)
                center = onehot(keywords[idx])
                peripheral = np.zeros(tag.shape[0])
                for k in range(len(keywords)):
                    if j == k:
                        continue
                    else:
                        #                        center = onehot(keywords[j])
                        temp = onehot(keywords[k])
                        peripheral += temp
                #                        print("c: ", center, "p: ", peripheral)
                self.dataset.append((center, peripheral))

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, i):
        return self.dataset[i]

def getAllTagList():
    taglist = tag.keyword
    taglist = taglist.tolist()
    taglist.sort()
    return taglist

def getJobTagList():
    taglist = job.merge(tag, on="tagID", how="left").drop_duplicates()
    taglist = taglist.keyword.drop_duplicates().tolist()
    taglist.sort()
    return taglist

def getUserList():
    return userkeyword

def getJobList():
    return jobkeyword

def getAllList():
    return allkeyword

def getDataset():
    dataset = Word2VecDataSet(jobkeyword)
    return dataset

def getTrainLoader():
    dataset = Word2VecDataSet(jobkeyword)
    return torch.utils.data.DataLoader(dataset, batch_size=config['BATCH_SIZE'], shuffle=True), dataset

if __name__ == "__main__":
    print(getJobIdList())


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import dbus
import dbus.mainloop.glib

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import GLib, Gtk

import os
import os.path
import subprocess
import functools

def device_changed(deviceObjectPath, interface, devChooser):
  """
  Retrieves the device with deviceObjectPath from dbus' system bus and checks
  whether it's an optical disc using the given interface. If it is, calls
  add_device on devChooser, remove_device otherwise.
  """
  systemBus = dbus.SystemBus()

  device = systemBus.get_object("org.freedesktop.UDisks", deviceObjectPath)

  isDisc = device.Get(interface, "DeviceIsOpticalDisc")
  #print("devchange: isDisc: {}, type: {}".format(isDisc, type(isDisc)))

  if device.Get(interface, "DeviceIsOpticalDisc"):
    devChooser.add_device(
        *map(
          functools.partial(device.Get, interface), ["IdLabel", "DeviceFile"]))
  else:
    devChooser.remove_device(device.Get(interface, "DeviceFile"))

def get_devices():
  """
  Returns a list of 2-tuples describing the device paths and medium description
  strings of optical drives on this system.
  """
  systemBus = dbus.SystemBus()

  udisks_proxy = systemBus.get_object(
      "org.freedesktop.UDisks", "/org/freedesktop/UDisks")

  devices = []

  for deviceObjectPath in udisks_proxy.EnumerateDevices():
    device = systemBus.get_object("org.freedesktop.UDisks", deviceObjectPath)
    if device.Get("org.freedesktop.UDisks.Device", "DeviceIsOpticalDisc"):
      devices.append(
          (
            device.Get("org.freedesktop.UDisks.Device", "DeviceFile"),
            device.Get("org.freedesktop.UDisks.Device", "IdLabel")))

  return devices

class DevChooser(Gtk.Window):
  """
  A Gtk main widget allowing to choose from a given list of devices or
  selecting a file.
  """

  def delete_event(self, widget, event, data=None):
    # Returning False will have Gtk emit the "destroy" signal
    return False

  def accept(self, widget, data=None):
    model, tree_iter = self.view.get_selection().get_selected()

    fn = model[tree_iter][1] if not model[tree_iter][2] else self.filename

    if self.bluray_button.get_active():
      fn = "bluray://{}".format(fn)

    env = os.environ
    os.environ["DISPLAY"] = ":0.1"

    subprocess.Popen(
        ["vlc", "--config", "/etc/choose_dev/vlcrc", fn],
        env=env,
        # This works, I don't know why, though.
        stdout = open(os.devnull, "w"),
        stderr = subprocess.STDOUT)

    self.destroy()

  def selection_changed(self, selection):
    model, tree_iter = selection.get_selected()

    self.okButton.set_sensitive(
      tree_iter is not None and             # selection is valid
        (not model[tree_iter][2] or         # not file row
          (not self.filename is None and    # file has been chosen
           os.path.exists(self.filename)))) # file exists

  def choose_file(self, tree_view, path, column):
    # This should always be true for Gtk.ListStore
    assert(len(path.get_indices()) == 1)

    index = path.get_indices()[0]

    # Make sure to only react to the file row
    if not self.store[index][2]:
      return

    dialog = Gtk.FileChooserDialog(
        "Choose file",
        self,
        Gtk.FileChooserAction.OPEN,
        (Gtk.STOCK_CANCEL, Gtk.ResponseType.REJECT,
         Gtk.STOCK_OK    , Gtk.ResponseType.ACCEPT))

    result = dialog.run()

    if Gtk.ResponseType.ACCEPT == result:
      self.filename = dialog.get_filename()
      self.store[index][0] = os.path.basename(self.filename)
      self.update_selection()

    dialog.destroy()

  def add_device(self, label, deviceFile):
    """
    Adds a row to the table using label for the first and deviceFile for the
    second column.

    @param label: label of the device
    @param deviceFile: device file string
    """
    self.store.prepend([label, deviceFile, False])
    self.update_geometry()

  def remove_device(self, deviceFile):
    """
    Removes the row whose second column corresponds to deviceFile.

    @param deviceFile: device file string
    """
    for row in self.store:
      if row[1] == deviceFile and not row[2]:
        self.store.remove(
          self.store.iter_nth_child(parent=None, n=row.path.get_indices()[0]))
        self.update_selection()
        self.update_geometry()
        break

  def update_selection(self):
    self.selection_changed(self.view.get_selection())

  def update_geometry(self):
    # This will make the window as small as possible.
    self.resize(1, 1)

  def __init__(self, devices = []):
    Gtk.Window.__init__(self, type = Gtk.WindowType.TOPLEVEL)

    self.connect("delete_event", self.delete_event)

    self.filename = None

    vBox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=5)
    self.add(vBox)

    # display string, path, is file?
    self.store = Gtk.ListStore(str, str, bool)
    self.store.append(["", "File", True])

    self.view = Gtk.TreeView(self.store)
    self.view.connect("row-activated", self.choose_file)

    renderer = Gtk.CellRendererText()

    # text=0 means the renderer should get its text from that column of the
    # model
    column = Gtk.TreeViewColumn("Name", renderer, text=0)
    self.view.append_column(column)

    column = Gtk.TreeViewColumn("Device", renderer, text=1)
    self.view.append_column(column)

    select = self.view.get_selection()
    select.connect("changed", self.selection_changed)
    select.set_mode(Gtk.SelectionMode.SINGLE)

    vBox.pack_start(self.view, False, False, 0)

    # Optical devices

    for device in devices:
      self.add_device(device[1], device[0])

    # bluray mode

    self.bluray_button = Gtk.CheckButton("Use BluRay protocol")
    self.bluray_button.set_active(False)

    vBox.pack_start(self.bluray_button, False, False, 0)

    # OK / Cancel buttons

    buttonBox = Gtk.ButtonBox(orientation=Gtk.Orientation.HORIZONTAL)
    buttonBox.set_layout(Gtk.ButtonBoxStyle.CENTER)
    buttonBox.set_spacing(2)

    button = Gtk.Button("Cancel", Gtk.STOCK_CANCEL)
    button.connect_object("clicked", Gtk.Widget.destroy, self)
    # child
    # expand = child gains size when parent grows
    # fill = give extra space to child
    # padding = extra space between childs in pixel
    buttonBox.pack_start(button, False, False, 0)

    self.okButton = Gtk.Button("OK", Gtk.STOCK_OK)
    self.okButton.set_sensitive(False)
    self.okButton.connect("clicked", self.accept, None)
    buttonBox.pack_start(self.okButton, False, False, 0)

    vBox.pack_start(buttonBox, False, False, 0)

    self.show_all()

def main():
  dbus.mainloop.glib.DBusGMainLoop(set_as_default = True)

  loop = GLib.MainLoop()

  devChooser = DevChooser(get_devices())
  devChooser.connect_object("destroy", GLib.MainLoop.quit, loop)

  dbus.SystemBus().add_signal_receiver(
      functools.partial(                       # callback function
        device_changed, devChooser=devChooser),
      # For the following, None implies "match all"
      signal_name="DeviceChanged",             # signal name
      dbus_interface="org.freedesktop.UDisks", # dbus interface
      bus_name=None,                           # sender bus name.
      path="/org/freedesktop/UDisks",          # sender object path
      interface_keyword="interface")           # parameter name for interface

  try:
    loop.run()
  except KeyboardInterrupt:
    print("Exiting.")

if __name__ == '__main__':
  main()


from odoo import models, fields

class NameContent(models.TransientModel):
    _name = 'name.conversion'

    first_name = fields.Char('First Name:')
    last_name = fields.Char('Last Name:')

    def concat(self):
        active_id = self.env.context.get('active_id', False)
        active_ids = self.env.context.get('active_ids', False)
        active_model = self.env.context.get('active_model', False)
        concatenate = self.env['order'].browse(active_id)
        joint_name = self.first_name + " " + self.last_name
        concatenate.write({'name': joint_name})


"""
获取中国日报每日播报
利用requests库和正则表达式等方法
@Author:jyang
@Date:5/23/2019
"""
import requests
from requests.exceptions import ConnectionError
import time
import re
import datetime
from bs4 import BeautifulSoup
from urllib import request,parse
from docx import Document
from docx.oxml.ns import qn

# def _save_context_to_debug():
#     # request = requests.get(url)
#     if req.status_code == 200:
#         try:
#             with open('chinadaily.htm', 'w') as f:
#                 f.write(req.text)
#         except IOError as e:
#             print(e.strerror)

def try_request(url):
    page = ''
    while page == '':
        try:
            page = requests.get(url)
            break
        except ConnectionError:
            print("Connection refused by the server.")
            print("Let me sleep for 5 seconds")
            print("Zzzzzz...")
            time.sleep(5)
            print("Was a nice sleep, now let me continue...")
            continue
        except Exception as e:
            print(e)
    return page

def get_hrefs(news):
    """
    get 'href' atti from 'a' tag
    :param news:
    :return: ["//language.chinadaily.com.cn/a/201905/27/WS5ceb90b3a3104842260be04a.html",]
    """
    hrefs = []
    for n in news:
        s = BeautifulSoup(n, 'html.parser')
        href = s.a['href']
        if href not in hrefs:
            hrefs.append(href)
            # print(href)
    return hrefs

def get_news_content(html_doc, filename):
    soup = BeautifulSoup(html_doc, "html.parser")
    cn_title = soup.find(class_='main_title1')
    en_title = soup.find(class_='main_title2')
    content_ps = soup.find(id='Content').find_all('p')
    with open(filename, 'w',encoding='utf-8') as f:
        f.write(cn_title.text + '\n')
        f.write(en_title.text + '\n')
        for p in content_ps:
            f.write(p.text + '\n')

    audio_url = soup.audio['src']
    get_audio(audio_url, filename)

def get_wechat_news_content(html_doc, filename):
    soup = BeautifulSoup(html_doc, "html.parser")
    content_ps = soup.find(id='js_content').find_all('p')
    with open(filename, 'w',encoding='utf-8') as f:
        for p in content_ps:
            f.write(p.text + '\n')

    # audio_url = soup.audio['src']
    # get_audio(audio_url, filename)

def get_audio(url, filename):
    url = "https:" + url
    filename = filename.split('.')[0] + '.' + url.split('.')[-1]
    request.urlretrieve(url, filename)

def transfer_txt_to_doc(transfer_file):
    file = Document()
    file.styles['Normal'].font.name = u'微软雅黑'
    file.styles['Normal']._element.rPr.rFonts.set(qn('w:eastAsia'), u'微软雅黑')

    # transfer_file = 'CNDaily_20190529.txt'
    with open(transfer_file, 'r', encoding='utf-8') as f:
        file.add_paragraph(f.read().strip())

    file.save(transfer_file[:transfer_file.rindex('.') + 1] + 'docx')

def main():
    y = str(datetime.datetime.now().year)
    m = str(datetime.datetime.now().month).zfill(2)
    d = str(datetime.datetime.now().day).zfill(2)
    today = '{0}{1}/{2}'.format(y, m, d)
    url = 'https://language.chinadaily.com.cn/audio_cd'  # https://language.chinadaily.com.cn/
    req = try_request(url)
    # _save_context_to_debug()
    today_news = re.findall(r'<a.*href="//language.chinadaily.com.cn/a/{0}.*</a>'.format(today), req.text) # get all needed tags
    herfs = get_hrefs(today_news)
    print('%s 有%d篇新闻。' % (today, len(herfs)))
    for i, h in enumerate(herfs):
        url = 'https:{0}'.format(h)
        print('新闻播报: %s' % url)
        r = try_request(url)
        filename = 'CNDaily_' + today.replace('/', '') + '.txt'
        get_news_content(r.text, filename)
        transfer_txt_to_doc(filename)

def wechat():
    y = str(datetime.datetime.now().year)
    m = str(datetime.datetime.now().month).zfill(2)
    d = str(datetime.datetime.now().day).zfill(2)
    today = '{0}{1}/{2}'.format(y, m, d)
    url = 'https://mp.weixin.qq.com/s/--cwNHH9juwbjDqeD9R8YQ'
    req = try_request(url)
    filename = 'CNDaily_' + today.replace('/', '') + '.txt'
    get_news_content(req.text, filename)

if __name__ == '__main__':
    main()
    # wechat()


#!/usr/bin/env python
#
# Electrum - Lightweight Bitcoin Client
# Copyright (C) 2015 Thomas Voegtlin
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os
import json
import stat

from datetime import datetime

from enum import IntEnum

from PyQt5.QtGui import QStandardItemModel, QStandardItem
from PyQt5.QtCore import Qt, QItemSelectionModel
from PyQt5.QtWidgets import QPushButton, QVBoxLayout, QLabel, QGridLayout, QLineEdit

from electrum.gui.qt.util import Buttons, CloseButton, OkButton, HelpLabel, WindowModalDialog, MyTreeView, read_QIcon

from electrum.crypto import sha256

from electrum.plugin import BasePlugin, hook
from electrum.i18n import _

from electrum.util import user_dir, bh2u
from electrum.util import PR_TYPE_ONCHAIN, PR_TYPE_LN

def read_schedule_file():
    schedule_path = os.path.join(user_dir(), 'schedule')
    if not os.path.exists(schedule_path):
        return {}
    try:
        with open(schedule_path, "r", encoding='utf-8') as f:
            data = f.read()
        result = json.loads(data)
    except:
        return {}
    if not type(result) is dict:
        return {}
    return result

def write_schedule_file(schedule_dict):
    schedule_path = os.path.join(user_dir(), 'schedule')
    s = json.dumps(schedule_dict, indent=4, sort_keys=True)
    try:
        with open(schedule_path, "w", encoding='utf-8') as f:
            f.write(s)
        os.chmod(schedule_path, stat.S_IREAD | stat.S_IWRITE)
    except FileNotFoundError:
        if os.path.exists(schedule_path):
            raise

ROLE_REQUEST_TYPE = Qt.UserRole
ROLE_REQUEST_ID = Qt.UserRole + 1

class CronPaymentsList(MyTreeView):

    class Columns(IntEnum):
        ADDRESS = 0
        AMOUNT = 1
        DESCRIPTION = 2
        LAST_TIME = 3
        NEXT_TIME = 4

    headers = {
        Columns.ADDRESS: _('Address'),
        Columns.AMOUNT: _('Amount'),
        Columns.DESCRIPTION: _('Description'),
        Columns.LAST_TIME: _('Last payment'),
        Columns.NEXT_TIME: _('Next payment'),
    }
    filter_columns = [Columns.LAST_TIME, Columns.NEXT_TIME, Columns.AMOUNT]

    def __init__(self, parent):
        super().__init__(parent, lambda: None,
                         stretch_column=self.Columns.DESCRIPTION,
                         editable_columns=[])
        self.setMinimumSize(900, 200)
        self.setSortingEnabled(True)
        self.setModel(QStandardItemModel(self))
        self.update()

    def update(self):
        current_schedule = read_schedule_file()
        _list = current_schedule.get('items', [])
        self.model().clear()
        self.update_headers(self.__class__.headers)
        for idx, cron_item in enumerate(_list):
            item = cron_item['invoice']
            schedule = cron_item['schedule']
            invoice_type = item['type']
            if invoice_type == PR_TYPE_LN:
                key = item['rhash']
                icon_name = 'lightning.png'
            elif invoice_type == PR_TYPE_ONCHAIN:
                key = bh2u(sha256(repr(item))[0:16])
                icon_name = 'bitcoin.png'
                if item.get('bip70'):
                    icon_name = 'seal.png'
            else:
                raise Exception('Unsupported type')
            address_str = item['outputs'][0][1]
            message = item['message']
            amount = item['amount']
            amount_str = self.parent.format_amount(amount, whitespaces=True)
            last_str = ''
            next_str = ''
            try:
                from croniter import croniter
                cron = '{minute} {hour} {day} {month} {week}'.format(**schedule)
                item = croniter(cron, datetime.now())
                next_str = item.get_next(datetime).isoformat()
            except:
                pass
            labels = [address_str, amount_str, message, last_str, next_str, ]
            items = [QStandardItem(e) for e in labels]
            self.set_editability(items)
            items[self.Columns.ADDRESS].setIcon(read_QIcon(icon_name))
            items[self.Columns.ADDRESS].setData(key, role=ROLE_REQUEST_ID)
            items[self.Columns.ADDRESS].setData(invoice_type, role=ROLE_REQUEST_TYPE)
            self.model().insertRow(idx, items)
        self.selectionModel().select(self.model().index(0,0), QItemSelectionModel.SelectCurrent)
        self.filter()

class CronPaymentsDialog(WindowModalDialog):

    def __init__(self, parent, title=None):
        WindowModalDialog.__init__(self, parent, title=title)
        vbox = QVBoxLayout(self)
        cron_list = CronPaymentsList(parent)
        vbox.addWidget(cron_list)
        vbox.addStretch()
        vbox.addLayout(Buttons(QPushButton('Edit'), QPushButton('Delete'), ))
        vbox.addStretch()
        vbox.addLayout(Buttons(CloseButton(self), ))

class ScheduleDialog(WindowModalDialog):

    def __init__(self, main_win, outputs):
        WindowModalDialog.__init__(self, main_win, title=_("New scheduled payment"),)
        vbox = QVBoxLayout(self)
        self.setMinimumSize(400, 150)
        minute_label = HelpLabel(_('minute:'), _('''
Defines a specific minute for an hour.
Allowed characters:
"*" any value
"," value list separator
"-" range of values
"/" step values
0-59 allowed values
'''))
        hour_label = HelpLabel(_('hour:'), _('''
Defines a specific hour during the day.
Allowed characters:
"*" any value
"," value list separator
"-" range of values
"/" step values
0-23 allowed values
'''))
        day_label = HelpLabel(_('day of month:'), _('''
Defines the day number for the month.
Allowed characters:
"*" any value
"," value list separator
"-" range of values
"/" step values
0-31 allowed values
'''))
        month_label = HelpLabel(_('month:'), _('''
Defines a month during the year.
Allowed characters:
"*" any value
"," value list separator
"-" range of values
"/" step values
1-12 allowed values
'''))
        week_label = HelpLabel(_('day of week:'), _('''
Specifies the day of the week.
Allowed characters:
"*" any value
"," value list separator
"-" range of values
"/" step values
0-6 allowed values
'''))
        self.minute_edit = QLineEdit('*')
        self.minute_edit.setFixedWidth(30)
        self.minute_edit.textChanged.connect(self.update_schedule_details)
        self.hour_edit = QLineEdit('*')
        self.hour_edit.setFixedWidth(30)
        self.hour_edit.textChanged.connect(self.update_schedule_details)
        self.day_edit = QLineEdit('*')
        self.day_edit.setFixedWidth(30)
        self.day_edit.textChanged.connect(self.update_schedule_details)
        self.month_edit = QLineEdit('*')
        self.month_edit.setFixedWidth(30)
        self.month_edit.textChanged.connect(self.update_schedule_details)
        self.week_edit = QLineEdit('*')
        self.week_edit.setFixedWidth(30)
        self.week_edit.textChanged.connect(self.update_schedule_details)
        grid = QGridLayout()
        grid.addWidget(minute_label, 0, 0, alignment=Qt.AlignCenter)
        grid.addWidget(hour_label, 0, 1, alignment=Qt.AlignCenter)
        grid.addWidget(day_label, 0, 2, alignment=Qt.AlignCenter)
        grid.addWidget(month_label, 0, 3, alignment=Qt.AlignCenter)
        grid.addWidget(week_label, 0, 4, alignment=Qt.AlignCenter)
        grid.addWidget(self.minute_edit, 1, 0, alignment=Qt.AlignCenter)
        grid.addWidget(self.hour_edit, 1, 1, alignment=Qt.AlignCenter)
        grid.addWidget(self.day_edit, 1, 2, alignment=Qt.AlignCenter)
        grid.addWidget(self.month_edit, 1, 3, alignment=Qt.AlignCenter)
        grid.addWidget(self.week_edit, 1, 4, alignment=Qt.AlignCenter)
        every_day = QPushButton('every day')
        every_day.clicked.connect(lambda: self.set_as_text('0 0 * * *'))
        every_week = QPushButton('every week')
        every_week.clicked.connect(lambda: self.set_as_text('0 0 * * 0'))
        every_month = QPushButton('every month')
        every_month.clicked.connect(lambda: self.set_as_text('0 0 1 * *'))
        every_year = QPushButton('every year')
        every_year.clicked.connect(lambda: self.set_as_text('0 0 1 1 *'))
        self.schedule_details = QLabel('')
        vbox.addLayout(grid)
        vbox.addLayout(Buttons(every_day, every_week, every_month, every_year))
        vbox.addWidget(self.schedule_details)
        vbox.addStretch()
        vbox.addLayout(Buttons(CloseButton(self), OkButton(self)))
        self.update_schedule_details()

    def set_as_text(self, value):
        items = value.split(' ')
        self.minute_edit.setText(items[0])
        self.minute_edit.repaint()
        self.hour_edit.setText(items[1])
        self.hour_edit.repaint()
        self.day_edit.setText(items[2])
        self.day_edit.repaint()
        self.month_edit.setText(items[3])
        self.month_edit.repaint()
        self.week_edit.setText(items[4])
        self.week_edit.repaint()
        self.update_schedule_details()

    def update_schedule_details(self):
        try:
            from croniter import croniter
            item = croniter(self.as_text(), datetime.now())
            t = item.get_next(datetime)
        except:
            self.schedule_details.setText('')
            return
        self.schedule_details.setText('new payment expected on:\n%s' % t.strftime("%A, %d %B %Y at %I:%M%p"))

    def as_text(self):
        return '%s %s %s %s %s' % (
            self.minute_edit.text(),
            self.hour_edit.text(),
            self.day_edit.text(),
            self.month_edit.text(),
            self.week_edit.text(),
        )

    def as_dict(self):
        return {
            'minute': self.minute_edit.text(),
            'hour': self.hour_edit.text(),
            'day': self.day_edit.text(),
            'month': self.month_edit.text(),
            'week': self.week_edit.text(),
        }

class ScheduleButton(QPushButton):

    def __init__(self, main_window):
        QPushButton.__init__(self, _("Schedule"))
        self.clicked.connect(lambda: self._on_clicked(main_window))

    def _on_clicked(self, main_win):
        outputs = main_win.read_outputs()
        if main_win.check_send_tab_outputs_and_show_errors(outputs):
            return
        d = ScheduleDialog(main_win, outputs)
        d.show()
        if not d.exec_():
            return
        invoice = main_win.read_invoice()
        if not invoice:
            return
        current_schedule = read_schedule_file()
        if 'items' not in current_schedule:
            current_schedule['items'] = []

        current_schedule['items'].append({
            'invoice': invoice,
            'schedule': d.as_dict(),
        })
        write_schedule_file(current_schedule)
        main_win.do_clear()
        main_win.show_message(_('New automatic payment scheduled successfully'))

class Plugin(BasePlugin):

    def fullname(self):
        return 'Cron Payments'

    def description(self):
        return _("Automated scheduled BitCoin payments")

    def is_available(self):
        return True

    def __init__(self, parent, config, name):
        BasePlugin.__init__(self, parent, config, name)
        self.start_scheduler()

    def on_cron_payments_menu_clicked(self, window, tools_menu):
        d = CronPaymentsDialog(window, _("Scheduled payments"))
        d.show()

    @hook
    def init_menubar_tools(self, main_window, tools_menu):
        tools_menu.addSeparator()
        tools_menu.addAction(_("&Scheduled payments"), lambda: self.on_cron_payments_menu_clicked(main_window, tools_menu))
        buttons = main_window.send_grid.itemAtPosition(6, 1)
        schedule_button = ScheduleButton(main_window)
        buttons.addWidget(schedule_button)

    def execute_payment(self):
        # TODO: to be continue
        pass

    def start_scheduler(self):
        # TODO: to be continue
        pass


import os
from moviepy.editor import VideoFileClip

for file in os.listdir(os.getcwd()):
    if file.endswith(".mp4"):
        print(f'Converting {file}')
        try:
            video = VideoFileClip(file)
            new_file_name = f"{file.split('.webm')[0]}.gif"
            video.write_gif(new_file_name)
            print(f'Successfully converted {file} to {new_file_name}\n')
        except Exception as e:
            print(f'Error converting {file}: {e}')
input('Process finished, press enter to exit')


import random

def agent_Selector(t):
    available_since1 = [lis[1] for lis in t]
    roles1 = [lis[0] for lis in t]
    temp = min(available_since1)

    issue_type = input("Enter the role for issue \n")
    available=[item[2] for item in t if item[0] == issue_type and item[3] == "Yes"]

    Selection_mode = int(input("Please select the mode \n 1: All available mode \n 2: Least busy \n 3: Random \n"))

    if Selection_mode == 1:
        print(available)

    elif Selection_mode == 2:
        Output = [item[2] for item in t
                  if item[0] == "sales" and item[1] == temp and item[3] == "Yes"]
        print(Output)

    elif Selection_mode == 3:
        random_item = random.choice(available)
        print(random_item)

    else :
        print("Enter correct value")

t = list(
        tuple(map(str, input('Enter role, the time since agent is available in mins, agents name and is_available: ').split()))
        for r in
        range(int(input('enter no of agents : '))))

print(t)
agent_Selector(t)


# -*- coding:utf-8 -*-
# Author: yangbin
# Created: 02/02/2018

"""
Reverse a singly linked list.
"""

# Definition for singly-linked list.
# class ListNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution(object):
    def reverseList(self, head):
        """
        :type head: ListNode
        :rtype: ListNode
        """
        pre = None
        cur = head
        while cur:
            tmp = cur.next
            cur.next = pre
            pre = cur
            cur = tmp
        return pre


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import glob

'''
Código que permite organizar y completar la información proveniente de los datos extraidos del DHIME

Created on thu Jun 30 05:3:40 2020
@author: Ing. Sergio Torres

'''

def intervalo_fecha(fecha_inicial,fecha_final):
    return np.array(pd.date_range(fecha_inicial,fecha_final))

def organizar_datos(ruta_archivos_dhime,fecha_inicial,fecha_final):
    filenames=glob.glob(ruta_archivos_dhime+'\*.xls')
    Estaciones=[]
    for file in filenames:
        df=pd.read_excel(file,header=0)
        Estaciones.append(pd.read_excel(file))
    variable=pd.concat(Estaciones,sort=False)
    variable=variable.loc[:,["NombreEstacion","Valor","Fecha"]]
    variable['Fecha'] = pd.to_datetime(df.Fecha)
    variable.set_index("Fecha",inplace=True)
    nombres_est = np.unique(variable['NombreEstacion'])

    # Se define un ciclo para extraer todas las estaciones
    for i in range(0, len(nombres_est)):
        vars()[str(nombres_est[i])] = variable.loc[variable['NombreEstacion'] == nombres_est[i]]

    fecha = intervalo_fecha(fecha_inicial,fecha_final)
    Matriz_datos = pd.DataFrame(fecha)
    Matriz_datos.rename(columns={0: "Fecha"}, inplace=True)

    for i in range(0, len(nombres_est)):
        Matriz_datos = pd.merge(Matriz_datos, vars()[nombres_est[i]].iloc[:, 1], how='left', on="Fecha")
    Matriz_datos.set_index('Fecha', inplace=True)
    Matriz_datos.columns = nombres_est

    return Matriz_datos

def outliers(matriz_variable,lim_min,lim_max):
    matriz_variable[matriz_variable>=lim_max]=np.NaN
    matriz_variable[matriz_variable<lim_min]=np.NaN
    return matriz_variable

def completar_datos(ruta_archivos_R,nombre_archivo,m=5):
    for i in range(1,m+1):
        vars()['imp'+str(i)]=pd.read_csv(ruta_archivos_R+'\\'+nombre_archivo+'-imp'+str(i)+'.csv',header=0,
                                         index_col='Fecha',parse_dates=True)
    for i in range(1,m+1):
        if i ==1:
            a=vars()['imp'+str(i)]
        else:
            a=a+vars()['imp'+str(i)]
        impf=a/m
        impf=impf.fillna(impf.mean().mean())
    return impf

#Se definen los parámetros de entrada para la organización de los datos provenientes del DHIME
ruta=r"C:\Users\sergi\OneDrive\Documents\Universidad\Proyecto_BIO\Nuevas_estaciones\Evaporacion"
fecha_inicial='1/1/1985'
fecha_final='1/1/2019'

Evaporacion=organizar_datos(ruta,fecha_inicial,fecha_final)
print(Evaporacion)
#Se establecen los parámetros necesarios para excluir los outliers de las series temporales. (El criterio de máximo y
#mínimo depende del operario y puede variar por estación
lim_min=0
lim_max=Evaporacion.max().max()+(3*(Evaporacion.std().max()))
Evaporacion=outliers(Evaporacion,lim_min,lim_max)
#Se graba el dataframe en un csv que se empleará en la herramienta Amelia de R
# Evaporacion.to_excel(r'C:\Users\sergi\Desktop\Evaporacion.xlsx')

#Se llaman los archivos provenientes de las múltiples imputaciones realizadas en R
ruta_archivos_R=ruta
nombre_archivo='Evaporacion'

Precipitacion=completar_datos(ruta_archivos_R,nombre_archivo)
print('Número de datos faltantes: ',pd.isnull(Precipitacion).sum().max())


import sys
import os
from random import uniform
import random
import math
import bpy, bmesh

from mathutils import Vector
from nodes import Nodes
from streets import Streets
from auxiliarFunctions import *

#from voronoiDiagram import VoronoiDiagram

texturexdir="./textures/"

#Class "PrimaryStreets":
#    - Functionality: Class to create the primary streets of the city. There
#                    are two method to do it:
#                        - VORONOI ALGORITHM: we use a squared net where we can include
#                                                noise in the vertices.
#                        - RANDOM VERTICES: we start from a cloud of random vertices and
#                                            we join them by the factor of density and
#                                            also avoid the creation of triangle districts,
#                                            vertices without connections and extreme
#                                            streets, creating a convex graph where each
#                                            node will be a intersection between streets
#                                            and the edges will be the streets.
#
#    - Parameters:
#        - x: the streets will be from -x to x on the X axis
#        - y: the streets will be form -y to y on the Y axis
#        - streetSize: width of the streets
#        - density(Random Algorithm): density of the districts
#        - districtHeight(Voronoi Algorithm): height of the districts
#        - districtWidth(Voronoi Algorithm): width of the districts
class PrimaryStreets(object):

    def __init__(self,x,y,streetSize,density,districtHeight,districtWidth):

        distanceFactor = density

        self.vertices = []                              #List of vertices
        self.faces = []                                 #List of faces
        self.edges = []                                 #List of edges
        self.nodes = Nodes()                            #Nodes of the graph
        self.streets = Streets()                        #Streets
        self.streetSize = streetSize                    #Size of the streets
        self.minDistance = int(x/distanceFactor)        #Minimum distance between 2 vertices
        self.maxDistance = int(self.minDistance*2)      #Maximum distance between 2 vertices to be connected
        self.district = []                              #District formed by the streets

        #================VORONOI ALGORITHM==================#
        #===============(Creating the graph)================#

        #---------- v = VoronoiDiagram(x,y,density,districtHeight,districtWidth)
        #--------------------------------------------------------- v.noise(20.0)
        #--------------------------------------------------- points = v.vertices
        #------------------------------------------------------- edges = v.edges
        #------------------------------------------------- districts = v.regions
#------------------------------------------------------------------------------
        #------------------------------------------------------ for i in points:
            #---------------------------------------------- self.nodes.insert(i)
        #------------------------------------------------------- for i in edges:
            #------------------------------- self.nodes.addConnection(i[0],i[1])
        #--------------------------------------------------- for i in districts:
            #-------------------------------------- self.nodes.insertDistrict(i)
#------------------------------------------------------------------------------
        #-------------------------------- self.vertices = self.nodes.coordinates

        #===================================================#

        #=================RANDOM ALGORITHM==================#
        #===============(Creating the graph)================#

        #Create the cloud of points and insert as nodes in the graph
        nNodes = random.randint(x*int(distanceFactor),x*int(distanceFactor)+x)
        initialVertices = []
        for i in range(nNodes):
            coordinatesX = random.uniform(-x,x)
            coordinatesY = random.uniform(-y,y)
            self.nodes.insert([coordinatesX,coordinatesY,0])

          #Remove the nodes that are too close by the value of "self.minDistance"
        self.removeNodesTooClose()

        #Connect the nodes by the value of "self.maxDistance"
        self.conectNodes()

        #Delete the nodes that are not connected or make a street with a
        #extreme without connect
        self.removed = []
        self.deleteExtremes()
        for i in self.removed:
            self.nodes.deleteByCoordenate(i)

        #Include the result vertices to "self.vertices"
        self.vertices = self.nodes.coordinates

        #TO THE EXAMPLE IN THE DOCUMENTATION

        #-------------------------------------------- self.nodes.insert([0,0,0])
        #----------------------------------------- self.nodes.insert([100,50,0])
        #---------------------------------------- self.nodes.insert([100,100,0])
        #------------------------------------------ self.nodes.insert([0,150,0])
        #--------------------------------------- self.nodes.insert([-100,150,0])
        #----------------------------------------- self.nodes.insert([-100,0,0])
#------------------------------------------------------------------------------
        #----------------------------------------- self.nodes.addConnection(0,1)
        #----------------------------------------- self.nodes.addConnection(1,2)
        #----------------------------------------- self.nodes.addConnection(2,3)
        #----------------------------------------- self.nodes.addConnection(0,3)
        #----------------------------------------- self.nodes.addConnection(0,5)
        #----------------------------------------- self.nodes.addConnection(3,4)
        #----------------------------------------- self.nodes.addConnection(4,5)
#------------------------------------------------------------------------------
        #-------------------------------- self.vertices = self.nodes.coordinates

        #Look for the cycles in the graph according with the
        #coordinates of each node
        districts = self.getDistricts()

        for i in districts:
            self.nodes.insertDistrict(i)

        #===================================================#

        #==========CREATE THE STREET FROM THE GRAPH===================#

        #Generation of the vertices will be part of the intersections
        self.generateVerticesIntersections()

        #Make the faces that create the streets
        self.makeStreetFaces()

        #Make the faces that create the intersections
        self.makeIntersectionFaces()

        #Look for the district between the streets with the new
        #vertices included
        self.getDistrictVertices()

        #===========================================================#

        #Checking the clockwise of the face to set the right normals in Blender
        for i in range(len(self.faces)):
            vertices = []
            for j in self.faces[i]:
                vertices.append(self.vertices[j])

            if checkClockwise(vertices):
                self.faces[i].reverse()

        #To see the edges of the graph

        for i in self.nodes.connections:
            for j in self.nodes.connections[i]:
                if not ([i,j] in self.faces) and not ([j,i] in self.faces):
                    self.edges.append((i,j))

        mesh=bpy.data.meshes.new("primary streets mesh")
        self.object=bpy.data.objects.new("primary streets object",mesh)
        self.object.location=[0,0,0]
        bpy.context.scene.objects.link(self.object)
        mesh.from_pydata(self.vertices,[],self.faces)
        #mesh.from_pydata(self.vertices,self.edges,[]) #To see the edges of the graph
        #mesh.from_pydata(self.vertices,self.edges,self.faces) #To see the edges and streets of the graph

    #Function "material":
    # - Functionality: include a material for the primary streets. We use a black-grey
    #                    color for the material and we include a procedural texture of
    #                    Blender called "MUSGRAVE", to make the street more realistic.
    #                    We modify some parameters of the procedural texture in our way.
    def material(self):
        #Creating the material
        mat = bpy.data.materials.new('primaryStreetMaterial')
        diffuse_color = uniform(0.01,0.03)
        mat.diffuse_color = (diffuse_color,diffuse_color,diffuse_color)
        mat.diffuse_shader = 'LAMBERT'
        mat.diffuse_intensity = 1.0
        mat.specular_color = (1,1,1)
        mat.specular_shader = 'COOKTORR'
        mat.specular_intensity = 0.0
        mat.alpha = 1
        mat.ambient = 1

        #Creating the procedural texture
        sTex = bpy.data.textures.new('primaryStreetsTexture', type = 'MUSGRAVE')
        sTex.noise_basis = 'BLENDER_ORIGINAL'
        sTex.noise_scale = uniform(0.02,0.06)
        sTex.musgrave_type = 'MULTIFRACTAL'
        sTex.lacunarity = uniform(2.5,3.0)
        sTex.noise_intensity = uniform(0.3,0.5)

        #Adding texture to the material
        mtex = mat.texture_slots.add()
        mtex.texture = sTex
        mtex.texture_coords = 'OBJECT'
        mtex.use_map_color_diffuse = True
        color = uniform(0.02,0.07)
        mtex.color = (color,color,color)

        #Adding material to the object
        me=self.object.data
        me.materials.append(mat)

    def material2(self,glass=1):
        me=self.object.data

        self.mat1 = bpy.data.materials.new("glass")
        self.mat1.diffuse_shader = 'LAMBERT'
        self.mat1.diffuse_color = (0.68,0.68,0.89)
        self.mat1.diffuse_intensity = 0.2
        self.mat1.specular_shader = 'COOKTORR'
        self.mat1.specular_color = (0.8,0.8,1)
        self.mat1.specular_hardness = 240
        self.mat1.specular_intensity = 1.0

        self.mat1.raytrace_mirror.use=1
        self.mat1.mirror_color=(0.36,0.72,0.62)
        self.mat1.raytrace_mirror.reflect_factor=0.7

        textureFile=os.path.expanduser(texturexdir+"primaryStreets.png")

        try:
            img = bpy.data.images.load(textureFile)
        except:
            raise NameError("Cannot load image %s" % textureFile)

        pTex = bpy.data.textures.new("texturebuilding", type = 'IMAGE')
        pTex.image=img;

        # put the texture on a material with UV coordinates
        self.mat1.texture_slots.add()
        self.mat1.texture_slots[0].texture = pTex
        self.mat1.texture_slots[0].texture_coords = 'UV'
        self.mat1.texture_slots[0].uv_layer = 'primaryStreet'
        me.materials.append(self.mat1)

        # set the coordinates of each vertex of the faces in UV mapping (2D)
        me.uv_textures.new("primaryStreet")
        bm = bmesh.new()
        bm.from_mesh(me)
        bpy.ops.uv.smart_project()
        #-------------------------------------- uv_layer = bm.loops.layers.uv[0]
#------------------------------------------------------------------------------
        #------------------------------------------------ nFaces = len(bm.faces)
        #----------------------------------------------------- for fi in nFaces:
            #--------------------------------------------------- maxDistance = 0
            #--------------------------------- for v in len(bm.faces[fi].verts):
                #-------------------------- if v != (len(bm.faces[fi].verts)-1):
                    # d = dist(bm.faces[fi].verts[v].co,bm.faces[fi].verts[v+1].co)
                #--------------------------------------------------------- else:
                    # d = dist(bm.faces[fi].verts[v].co,bm.faces[fi].verts[0].co)
            #------ d1 = dist(bm.faces[fi].verts[0].co,bm.faces[fi].verts[1].co)
            #------ d2 = dist(bm.faces[fi].verts[1].co,bm.faces[fi].verts[2].co)
            #----------------------- bm.faces[fi].loops[0][uv_layer].uv = (0, 0)
            #-------------------- bm.faces[fi].loops[1][uv_layer].uv = (d1/4, 0)
            #----------------- bm.faces[fi].loops[2][uv_layer].uv = (d1/4, d2/2)
            #-------------------- bm.faces[fi].loops[3][uv_layer].uv = (0, d2/2)
        bm.to_mesh(me)

        # put the image behind the UV layer for the times when you're using the Multitexture shader on the 3D view.
        bm = bmesh.new()
        bm.from_mesh(me)
        tl = bm.faces.layers.tex.active
        if tl:
            for f in bm.faces:
                f[tl].image = me.materials[0].texture_slots[0].texture.image
        bm.to_mesh(me)

    #Function "materemoveNodesTooCloserial":
    # - Functionality: remove the nodes which are too close to each other according
    #                    with the value of "self.minDistance".
    def removeNodesTooClose(self):
        removed = []
        for i in range(self.nodes.nNodes-1):
            if not (self.nodes.getCoordinates(i) in removed):
                for j in range(i+1,self.nodes.nNodes):
                     if(dist(self.nodes.getCoordinates(i),self.nodes.getCoordinates(j)) < self.minDistance):
                        removed.append(self.nodes.getCoordinates(j))

        for i in removed:
            self.nodes.deleteByCoordenate(i)

    #Function "conectNodes":
    # - Functionality: connect the nodes which are close enough to each other according
    #                    with the value of "self.maxDistance" and checking the creation
    #                    of triangles and checking that the new connections do not intersect
    #                    with a connections made before.
    def conectNodes(self):
        for i in range(self.nodes.nNodes-1):
            for j in range(i+1,self.nodes.nNodes):
                if(dist(self.nodes.getCoordinates(i),self.nodes.getCoordinates(j)) < self.maxDistance):
                    if not self.checkTriangle(i, j):
                        if not self.checkIntersection(i,j):
                            self.nodes.addConnection(i,j)

    #Function "checkTriangle":
    # - Functionality: checking if we connect two node it will create a triangle, by
    #                    checking if the two nodes have any node in common in the nodes
    #                    that they are already connected.
    #
    # - Parameters:
    #    - a: index of the first node
    #    - b: index of the second node
    #
    # - Return:
    #    - bool, if the nodes will create a triangle or not
    def checkTriangle(self,a,b):
        for i in self.nodes.getConnection(a):
            for j in self.nodes.getConnection(b):
                if i == j:
                    return True

        return False

    #Function "checkIntersection":
    # - Functionality: checking if we connect two node it will create a intersection
    #                    between the edges of the graph that we have already created.
    #
    # - Parameters:
    #    - i: index of the first node
    #    - j: index of the second node
    #
    # - Return:
    #    - bool, if the nodes will intersect or no
    def checkIntersection(self,i,j):
        intersection = False
        for k in self.nodes.connections:
            for e in self.nodes.getConnection(k):
                v = lineIntersection([self.nodes.getCoordinates(i),self.nodes.getCoordinates(j)],
                                     [self.nodes.getCoordinates(k),self.nodes.getCoordinates(e)])
                if (v != False):
                    if checkPointInSegment4(self.nodes.getCoordinates(k),self.nodes.getCoordinates(e),[v[0],v[1],0]) and checkPointInSegment4(self.nodes.getCoordinates(i),self.nodes.getCoordinates(j),[v[0],v[1],0]):
                        intersection = True
                        break
            if intersection:
                break;

        return intersection

    #Function "deleteExtremes":
    # - Functionality: delete the nodes that are not connected with any node or that
    #                    is a extreme of a street without connection.
    def deleteExtremes(self):
        for i in range(self.nodes.nNodes):
            if not (self.nodes.getCoordinates(i) in self.removed):
                if(len(self.nodes.getConnection(i)) == 0) or (len(self.nodes.getConnection(i)) == 1):
                    self.removed.append(self.nodes.getCoordinates(i))
                    if (len(self.nodes.getConnection(i)) == 1):
                        self.recursiveDelete(self.nodes.getConnection(i)[0],i)

    #Function "recursiveDelete":
    # - Functionality: recursive function checking if we delete a node if it create a street
    #                    with a extreme without connect
    def recursiveDelete(self,index,previousIndex):
        if len(self.nodes.getConnection(index)) == 2:
            if not (self.nodes.getCoordinates(index) in self.removed):
                self.removed.append(self.nodes.getCoordinates(index))
            for i in self.nodes.getConnection(index):
                if i != previousIndex:
                    self.recursiveDelete(i,index)

    #Function "getDistricts":
    # - Functionality: looking for the areas created by the graph. The algortihm to do it
    #                    is a MCB (Minimum Cycle Basic) algorithm to a graph, by keep in
    #                    mind the coordinate of each node.
    #
    # - Return:
    #    - districts: a list of district where each element is a list of index of node
    #                    which make a district.
    def getDistricts(self):
        districts = []
        delete = []
        found = False
        for i in range(self.nodes.nNodes):
            for j in self.nodes.findDistricts(i,self.nodes.nNodes):
                delete = []

                polygon = []
                for k in range(len(j)):
                    v = self.nodes.getCoordinates(j[k])
                    polygon.append([v[0],v[1]])

                inside = False
                for k in range(self.nodes.nNodes):
                    p = self.nodes.getCoordinates(k)
                    if not (k in j):
                        inside = point_inside_polygon(p[0], p[1], polygon)
                        if inside:
                            break

                if not inside:
                    for k in range(len(districts)):
                        #Cycle taken is smaller than cycle we are going to compare
                        if len(districts[k]) > len(j):
                            found = True
                            for l in j:
                                if not (l in districts[k]):
                                    found = False
                                    break
                            if found:
                                delete.append(k)
                                found = False

                        #Cycle taken is bigger than cycle we are going to compare
                        elif len(districts[k]) < len(j):
                            found = True
                            for l in districts[k]:
                                if not (l in j):
                                    found = False
                                    break
                            if found:
                                break;

                        else:
                            found = True
                            for l in j:
                                if not (l in districts[k]):
                                    found = False
                                    break
                            if found:
                                break;

                    if not found:
                        districts.append(j)

                    delete.sort()
                    for k in range(len(delete)-1,-1,-1):
                        del districts[delete[k]]

        return districts

    #Function "generateVerticesIntersections":
    # - Functionality: calculate the vertices which create the face of a intersection, the face
    #                    result of a node of the graph and adding the vertices to "self.vertices"
    def generateVerticesIntersections(self):
        for i in range(self.nodes.nNodes):
            connectionCoordinates = []
            for j in self.nodes.getConnection(i):
                connectionCoordinates.append(self.nodes.getCoordinates(j))

            intersection = self.streets.calculateIntersection(self.nodes.getCoordinates(i),self.nodes.getConnection(i),connectionCoordinates,self.streetSize)
            self.streets.insertIntersection(intersection)
            for j in intersection:
                self.vertices.append(intersection[j][0])
                self.vertices.append(intersection[j][1])

    #Function "makeStreetFaces":
    # - Functionality: make the faces related to a edge of the graph
    def makeStreetFaces(self):
        made = []
        for i in range(self.streets.nIntersections):
            for j in self.streets.getIntersection(i):
                if not (([j,i] in made) or ([i,j] in made)):
                    face = self.streets.makeFaceStreet(i,j)
                    f = []
                    for k in face:
                        f.append(self.vertices.index(k))
                    self.faces.append(f)

                    made.append([j,i])
                    made.append([i,j])

    #Function "makeIntersectionFaces":
    # - Functionality: make the faces related to a node of the graph
    def makeIntersectionFaces(self):
        print('ENTRA MAKE INTERSECTION FACE')
        #To each node we execute the algorithm
        for i in range(self.streets.nIntersections):
            #That function return which vertices make the face and the order. Also return
            #new vertices that appeared in the generation and to which have to be substitute
            [face,newVertices,sustitute] = self.streets.makeIntersectionFace(i,self.nodes.coordinates,self.nodes.connections,self.streetSize)

            for j in sustitute:
                index1 = self.nodes.coordinates.index(j[0])
                index2 = self.vertices.index(j[0])

                for k in range(len(self.faces)):
                    for l in range(len(self.faces[k])):
                        if self.faces[k][l] == index2:
                            self.faces[k][l] == index1

                self.nodes.coordinates[index1] = j[1]
                self.vertices[index2] = j[1]

            if face != False:
                for j in newVertices:
                    self.vertices.append(j)

                f = []
                for j in face:
                    f.append(self.vertices.index(j))

                self.faces.append(f)
        print('SALE MAKE INTERSECTION FACE')

    #Function "makeIntersectionFaces":
    # - Functionality: get the district resulting after make the faces of the
    #                    streets and the intersection
    def getDistrictVertices(self):
        for i in range(len(self.nodes.districts)):
            self.district.append(self.streets.calculateDistrict(self.nodes.districts[i]))


import random

file = open("./data/mushrooms.csv", encoding='utf-8')
train_file = open("./data/mushroom.train", 'w', encoding='utf-8')
test_file = open("./data/mushroom.test", 'w', encoding='utf-8')

is_first_line = True
for line in file:
    if is_first_line or random.random() > 0.2:
        is_first_line = False
        train_file.write(line)
        # train_file.write("\n")
    else:
        test_file.write(line)
        # test_file.write("\n")


#!/usr/bin/env python
# -*- coding: utf-8 -*
from popen import *
from temp import *
from public import *

def args(message,using=None,rate=None,background=False):
    """
    return code for say
    https://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man1/say.1.html
    """
    l=["say"]
    f=tempfile()
    if isinstance(message,unicode):
        message=message.encode("utf8")
    open(f,"w").write(message)
    l.append("-f")
    l.append(f)
    if using:
        l.append("-v")
        l.append(using)
    if rate:
        l.append("-r")
        l.append(str(rate))
    if background:
        l.append("&")
    return l

def saying():
    """return True if say process alive"""
    cmd="ps cax | grep say | grep -v grep"
    stdoutdata = Popen(cmd,shell=True)
    return stdoutdata!=""

@public
def say(message,using=None,rate=None,background=False):
    """say"""
    if not saying():
        if message:
            l=args(message,using,rate,background)
            popen(" ".join(l))