import os

import numpy
from pandas import DataFrame
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import KFold
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
import matplotlib.pyplot as plt

def load_validation_data():
    validation_spam_files = ['enron2/spam/' + f for f in os.listdir('enron2/spam')]
    validation_ham_files = ['enron2/ham/' + f for f in os.listdir('enron2/ham')]

    validation_spam = []
    validation_ham = []

    for validation_file in validation_spam_files:
        with open(validation_file, 'r', encoding="utf8", errors='ignore') as f:
            validation_spam.append(f.read())

    for validation_file in validation_ham_files:
        with open(validation_file, 'r', encoding="utf8", errors='ignore') as f:
            validation_ham.append(f.read())

    return validation_ham, validation_spam

def load_data():
    raw_data = DataFrame({'text': [], 'classifier': []})

    load_spam_files = ['enron1/spam/' + f for f in os.listdir('enron1/spam')]
    load_ham_files = ['enron1/ham/' + f for f in os.listdir('enron1/ham')]

    load_spam = []
    spam_names = []
    for load_file in load_spam_files:
        with open(load_file, 'r', encoding="utf8", errors='ignore') as f:
            text = f.read()
            load_spam.append({'text': text, 'classifier': 'spam'})
            spam_names.append(load_file)

    raw_data = raw_data.append(DataFrame(load_spam, spam_names))

    load_ham = []
    ham_names = []
    for load_file in load_ham_files:
        with open(load_file, 'r', encoding="utf8", errors='ignore') as f:
            text = f.read()
            load_ham.append({'text': text, 'classifier': 'not spam'})
            ham_names.append(load_file)

    raw_data = raw_data.append(DataFrame(load_ham, ham_names))

    raw_data = raw_data.reindex(numpy.random.permutation(raw_data.index))

    return raw_data

def train(training_data, num_k_folds=5, train_algorithm=1):
    trained_pipeline = Pipeline([
        ('count_vectorizer', CountVectorizer(ngram_range=(1, 2))),
        ('classifier', KNeighborsClassifier() if train_algorithm == 1 else SVC(gamma='scale'))
    ])

    for i, (train_indices, test_indices) in enumerate(KFold(num_k_folds).split(training_data)):
        print(f"Training fold {i + 1}")
        train_values = training_data.iloc[train_indices]['text'].values
        train_result = training_data.iloc[train_indices]['classifier'].values.astype(str)

        trained_pipeline.fit(train_values, train_result)

    return trained_pipeline

def test_accuracy(test_pipeline, test_ham, test_spam):
    test_ham_results = test_pipeline.predict(test_ham).tolist()
    test_spam_results = test_pipeline.predict(test_spam).tolist()

    test_ham_correct = test_ham_results.count("not spam")
    test_spam_correct = test_spam_results.count("spam")

    print(f"Correctly found {test_ham_correct} out of {len(test_ham)} ham")
    print(f"Correctly found {test_spam_correct} out of {len(test_spam)} spam")

    return test_ham_correct / len(test_ham) * 100, test_spam_correct / len(test_spam) * 100

def plot():
    print("Loading validation data...")
    plot_ham, plot_spam = load_validation_data()

    test_type = input("Test for k-folds or data size? 1 or 2\n>")

    if test_type == "1":
        k_folds = range(2, 13)
        ham_accuracies = []
        spam_accuracies = []

        for k_fold in k_folds:
            print(f"\nTesting k-folds {k_fold}")
            ham_accuracy, spam_accuracy = test_accuracy(train(data, num_k_folds=k_fold, train_algorithm=algorithm),
                                                        plot_ham,
                                                        plot_spam)
            ham_accuracies.append(ham_accuracy)
            spam_accuracies.append(spam_accuracy)

        plt.plot(k_folds, ham_accuracies, label='Ham Accuracy')
        plt.plot(k_folds, spam_accuracies, label='Spam Accuracy')
        plt.xlabel('K-Folds')
        plt.ylabel('Percent Correct')
        plt.title(('SVM' if algorithm == 2 else 'KNN') + " Accuracies vs K-Folds")
        plt.legend()
        plt.show()

    else:
        sizes = range(500, 5500, 500)
        ham_accuracies = []
        spam_accuracies = []

        for size in sizes:
            print(f"\nTesting size {size}")
            ham_accuracy, spam_accuracy = test_accuracy(train(data.sample(size), train_algorithm=algorithm), plot_ham,
                                                        plot_spam)
            ham_accuracies.append(ham_accuracy)
            spam_accuracies.append(spam_accuracy)

        plt.plot(sizes, ham_accuracies, label='Ham Accuracy')
        plt.plot(sizes, spam_accuracies, label='Spam Accuracy')
        plt.xlabel('Data Size')
        plt.ylabel('Percent Correct')
        plt.title(('SVM' if algorithm == 2 else 'KNN') + " Accuracies vs Training Data Size")
        plt.legend()
        plt.show()

    exit(0)

def test():
    folds = input("How many folds to use in k-fold? Higher k takes longer to train. \n>")
    if not folds.isdigit():
        folds = "10"

    print("Training...")
    pipeline = train(data, num_k_folds=int(folds), train_algorithm=algorithm)
    print("Trained")

    # Test Prediction
    while True:
        test_email = input("Enter your email to predict, or type 'test' to test against validation set and get results."
                           " \n>")

        if test_email != "test":
            print(pipeline.predict([test_email]))

        else:
            print("Importing validation set...")
            spam_files = ['enron2/spam/' + f for f in os.listdir('enron2/spam')]
            ham_files = ['enron2/ham/' + f for f in os.listdir('enron2/ham')]

            spam = []
            ham = []

            for file in spam_files:
                with open(file, 'r', encoding="utf8", errors='ignore') as f:
                    spam.append(f.read())

            for file in ham_files:
                with open(file, 'r', encoding="utf8", errors='ignore') as f:
                    ham.append(f.read())

            print("Predicting results")
            ham_results = pipeline.predict(ham).tolist()
            spam_results = pipeline.predict(spam).tolist()

            ham_correct = ham_results.count("not spam")
            spam_correct = spam_results.count("spam")

            print(f"Correctly found {ham_correct} out of {len(ham)} ham")
            print(f"Correctly found {spam_correct} out of {len(spam)} spam")

            break

# MAIN

print("Loading data...")
data = load_data()
print("Data loaded.")

algorithm = input("Which algorithm to run? 1=Knn 2=Svm. \n>")

if algorithm == "2":
    algorithm = 2
    print("Running SVM")
else:
    algorithm = 1
    print("Running Knn (default)")

plot_or_test = input("Plot results or run test? 1=plot 2=test \n>")

if plot_or_test == "1":
    plot()
else:
    test()


from setuptools import setup, find_packages
import sys
import versioneer

install_requires = ['menpo>=0.6,<0.7',
                    'cyassimp>=0.2,<0.3',
                    'cyrasterize>=0.2.4,<0.3']

# These dependencies currently don't work on Python 3
if sys.version_info.major == 2:
    install_requires.append('mayavi>=4.4,<4.5')
    install_requires.append('menpo-pyvrml97==2.3.0a4')

setup(name='menpo3d',
      version=versioneer.get_version(),
      cmdclass=versioneer.get_cmdclass(),
      description='Menpo library providing tools for 3D Computer Vision research',
      author='James Booth',
      author_email='james.booth08@imperial.ac.uk',
      packages=find_packages(),
      package_data={'menpo3d': ['data/*']},
      install_requires=install_requires,
      tests_require=['nose', 'mock>=1.3']
)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#    Copyright (C) 2017 Thierry EXCOFFIER, Universite Claude Bernard
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#    Contact: Thierry.EXCOFFIER@univ-lyon1.fr

"""
The argument is the list of referent logins.

The mails about the 7 last day ABINJ are sent to these referents.

"""

import time
import locale
import sys
import tomuss_init
from .. import referent
from .. import document
from .. import utilities
from .. import configuration
from .. import inscrits

def students(teachers):
    s = set()
    for login in teachers:
        s.update(referent.students_of_a_teacher(login))
    return s

def ues(students):
    s = set()
    for login in students:
        for year, semester, ue in document.update_index(
                utilities.the_login(login)):
            if (year, semester) == configuration.year_semester:
                s.add(ue)
    return s

def nice_date(txt):
    return time.strftime("%c", time.strptime(txt, "%Y%m%d%H%M%S"))

def message(teacher, all_students, since):
    messages = []
    for login in referent.students_of_a_teacher(teacher):
        if len(all_students[login]) == 0:
            continue
        fn, sn, mail = inscrits.L_batch.firstname_and_surname_and_mail(login)
        messages.append(login + ' ' + fn.title() + ' ' + sn + ' ' + mail)
        for col, cell in all_students[login]:
            messages.append('    ' + col.table.ue + ' ' + col.table.table_title)
            if len(messages) >= 3 and messages[-1] == messages[-3]:
                messages.pop()
            t = ('        ' + nice_date(cell.date)
                 + '«' + col.title + '» «' + cell.author)
            if cell.comment:
                t += '» «' + cell.comment
            t += '»'
            messages.append(t)
        messages.append('')
    if messages:
        mail = [inscrits.L_batch.mail(teacher), configuration.maintainer]
        utilities.send_mail(to = mail,
                            subject = "TOMUSS " + configuration.abi
                            + ' (' + teacher + ')',
                            message =
                            nice_date(since) + ' >>>> ' + time.strftime("%c")
                            + '\n\n\n' + '\n'.join(messages),
                            frome = configuration.maintainer,
                            show_to = True)

class Stat:
    students = {}
    def __init__(self, teachers, since):
        self.teachers = teachers
        self.students = {login: [] for login in students(teachers)}
        print("{} referents, {} students".format(
            len(teachers), len(self.students)))
        self.since = since
    def analyse(self, ue):
        ue = document.table(configuration.year_semester[0],
                            configuration.year_semester[1],
                            ue)
        for line in ue.lines.values():
            if line[0].value not in self.students:
                continue
            for col, cell in zip(ue.columns, line):
                if cell.value == configuration.abi and cell.date > self.since:
                    self.students[line[0].value].append((col, cell))
    def analyse_all(self):
        for ue in ues(self.students):
            self.analyse(ue)
        print("{} ABI this week".format(sum(len(x)
                                            for x in self.students.values())))
        locale.setlocale(locale.LC_TIME,
                         configuration.language + '_'
                         + configuration.language.upper() + '.utf8')
        for teacher in self.teachers:
            message(teacher, self.students, self.since)

last_week = time.strftime("%Y%m%d%H%M%S",
                          time.localtime(time.time() - 7 * 86400))

Stat(sys.argv[1:], last_week).analyse_all()


from nclib import *
import base64

HOSTNAME = "13.112.92.9"
PORT = 21701

def saferecv():
    out = ''
    chr = ' '
    while chr != '\n':
        chr = nc.recv_exactly(1).decode()
        if chr != '\n':
            out += chr
    return out.encode()

def sendA(number):
    nc.recv_exactly(5)
    nc.send_line(b'A')
    nc.recv_exactly(7)
    string = hex(number)[2:]
    if len(string) % 2 == 1:
        string = '0' + string
    nc.send_line(string.encode())
    string = saferecv()
    return int(string, 16)

def sendB(number):
    nc.recv_exactly(5)
    nc.send_line(b'B')
    nc.recv_exactly(7)
    string = hex(number)[2:]
    if len(string) % 2 == 1:
        string = '0' + string
    nc.send_line(string.encode())
    string = saferecv()
    return int(string, 16)

nc = Netcat((HOSTNAME, PORT))
print(nc.recv_exactly(18))

flag = int(nc.recv_exactly(512).decode(), 16)
print(flag)
nc.recv_exactly(1)
XD = 0

N = 0
i = 1024
while i >= 0: #decrypting N
    print(i)
    print(N)
    poti = sendA(2**i)
    encN = sendA(N)
    for j in range(i, i + 4)[::-1]: #ugh
        print(j)
        N += 2**j
        kkk = j - i
        realmod = N % 256

        encN = encN * (poti ** (2**kkk))

        encrmod = sendB(encN)
        if realmod != encrmod:
            N -= 2**j
            encN = encN // (poti ** (2**kkk))
            XD = encrmod
            print (encrmod)
    i -= 4

N += 1
#yay, we found the modulo
print(N)
print(flag)

sflag = ""
vflag = 0

#decrypting flag
def flagsub(flag, k, n):
    return ((1 - n*k) * flag) % (n*n)

vflag = 100875957455868337925440735881740964923696086881408630419864201017940698935980156443839104427677325695508371154529740778088364312572340524480844371681868776114848753736744960 #it should be equal to 0 at the beginning, the value you see here is the value calculated after the first run of the program
for i in range(0, 148)[::-1]: #for should start from i = 600, the 148 here was also set for the second run
    print(i)
    print(vflag)
    vflag += 2**i
    realmod = (10 - vflag) % 256
    encrmod = sendB(flagsub(flag, vflag, N))
    if realmod != encrmod:
        vflag -= 2**i


#Problem 2.3

#############################################################################################
####### Printing all the steps of running the program that computes Fibonacci numbers #######
#############################################################################################

def fibo(n):
    print("Вычисляю число Фибоначчи " + str(n) + "...")
    if n == 0: result = 0
    elif n == 1: result = 1
    else: result = fibo(n-1) + fibo(n-2)
    print("Вычислил: число Фибоначчи " + str(n) + " = " + str(result))
    return result

print("Enter a number, please.")
while True:
    num = input()
    if num != "":
        num = int(num)
        fibo = fibo(num)
        print("Fibonacci function value of your number is " + str(fibo) + ".\nEnter a number, please.")
        continue
    else: break
print("Goodbye!")


"""lti_inspector URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from inspector import views

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^$', views.tool_config, name='tool_config'),
    url(r'^launch[/]{0,1}(?P<placement>[a-z_]*)$', views.lti_launch, name='lti_launch'),
    url(r'^return_assignment_selection$', views.return_assignment_selection, name='return_assignment_selection'),
    url(r'^return_homework_submission$', views.return_homework_submission, name='return_homework_submission'),
    url(r'^return_editor_button_selection$', views.return_editor_button_selection, name='return_editor_button_selection'),
    url(r'^view_assignment/(?P<assignment_id>[a-zA-Z0-9]+)$', views.view_assignment, name='view_assignment'),
    url(r'^view_homework_submission/(?P<submission_id>[a-zA-Z0-9]+)$', views.view_homework_submission, name='view_submission'),

]


numero = int(input('Digite um numero menor que 1000: '))
centena = 0
dezena = 0
unidade = 0

print(numero,'=', end=' ')

if(numero > 100):
    centena = int(numero/100)
    dezena = int((numero - (centena*100))/10)
    unidade = numero - (centena * 100) - (dezena * 10)
else:
    dezena = int(numero/10)
    if(dezena > 0):
        unidade = numero - (centena * 100) - (dezena*10)
    else:
        unidade = numero - (centena*100)

if(centena == 0):
    print('', end=' ')
elif(centena == 1):
    print(centena, 'centena,', end=' ')
else:
    print(centena, 'centenas,', end=' ')

if(dezena == 0):
    print('e', end=' ')
elif(dezena == 1):
    print(dezena, 'dezena e', end=' ')
else:
    print(dezena, 'dezenas e', end=' ')

if(unidade == 0):
    print('', end=' ')
elif(unidade == 1):
    print(unidade, 'unidade')
else:
    print(unidade, 'unidades')

2 - Estruturas de Decisão/Ex19-Centena_dezena.py


# Imports
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
from numpy import *
import scipy as spi
import sys

cmapalpha = 0.0

# ---===---===---===---===---===---===---===---===---===
plt.subplot(1,1,1)
plt.subplots_adjust(left=0.18,bottom=0.15,wspace=0.0001,hspace=0.0001)

plt.axis([0, rPlotmax, 0, zPlotmax])

#Moves the x-axis down a little
plt.gca().xaxis.labelpad=5 #5 is default

#Makes ticks thicker/longer
ax = plt.gca()
ax.tick_params(which='both',width=2)
ax.tick_params(which='major',size=12)
ax.tick_params(which='minor',size=7)

#moves numbers out a little
ax.tick_params(axis='y',pad=9)
ax.tick_params(axis='x',pad=9)

# Max and Min for ColorMap
# Overwrite min max plotted
#plotmin = 0
#plotmax = 20
cmapmin = plotmin
cmapmax = plotmax

# ColorMap choice
cmap = 'Paired'
cmap = 'RdBu_r'
#cmap = 'Spectral'
#cmap = 'Blues_r'
if(cmapalpha>0):
    plt.pcolor(RGrid,ZGrid,PreparedPlot,cmap=cmap,vmin=cmapmin,vmax=cmapmax,alpha=cmapalpha)
    plt.colorbar()

# override
CLevels = [ 0.5, 0.75, 1, 1.25, 1.5  ]
CS = plt.contour(RMesh,ZMesh,PreparedPlot,levels=CLevels,colors='k',linewidths=2.5)
#plt.clabel(CS, inline=1, fontsize=9)

# Filament Boundary
area = pi*2.0
#plt.scatter(VCont,(ZGrid[:-1]+0.5*DeltaZ),s=1.5*area,c='black')
plt.plot(VCont,(ZGrid[:-1]+0.5*DeltaZ),c='black',linewidth=2.5,alpha=0.3)

# B fields
if(Bfield=='Y'):
    for i in range(NBCont):
        plt.plot(BContours[i],(ZGrid[:-1]+0.5*DeltaZ),'--',c='black',linewidth=2,alpha=0.7)

#Text
#plt.text(0.05,0.06,r'${\cal M} = 4.47$',fontsize=26.0)

# Creates Plot
#plt.show()
plt.savefig('Contour.pdf')


import random
import re
import string
import sys
import threading
from datetime import datetime, timedelta

from .google_maps.info import GM_Restaurant
from .ifoodie.ifoodie import Ifoodie

sys.path.append(".")
from bson.son import SON
from pymongo import GEOSPHERE

import config
import MongoDB.operation as database

class Nearby_restaurant:
    def __init__(self, latitude, longitude, keyword="", page_token=""):
        self.restaurants = []
        self.next_page = ""
        self.latitude = latitude
        self.longitude = longitude
        self.keyword = keyword
        self.page_token = page_token
        self.get_info()

    def get_info(self):
        # Get nearby restaurants
        threads = []
        self.get_google_maps_data()
        self.get_ifoodie_data()
        # Load from database
        """ result = []
        config.db.restaurant.create_index([("loc", GEOSPHERE)])
        query = {
            "loc": {
                "$near": SON(
                    [
                        (
                            "$geometry",
                            SON(
                                [
                                    ("type", "Point"),
                                    ("coordinates", [self.longitude, self.latitude]),
                                ]
                            ),
                        ),
                        ("$maxDistance", 1000),
                    ]
                )
            },
            "category": self.keyword,
        }
        for each in config.db.restaurant.find(query):
            result.append(each)
        if len(result) >= 5:
            print("load from db.")
            for each in result:
                restaurant = Restaurant(
                    place_id=each["place_id"],
                    name=each["name"],
                    photo_url=each["photo_url"],
                    open_now=find_operating_status(
                        data=each["operating_time"]["weekday_text"]
                    ),
                    operating_time=each["operating_time"],
                    location=each["location"],
                    address=each["address"],
                    rating=each["rating"],
                    website=each["website"],
                    google_url=each["google_url"],
                    phone_number=each["phone_number"],
                    reviews=each["reviews"],
                )
                self.restaurants.append(restaurant)
        else:
            self.get_google_maps_data()
            self.get_ifoodie_data() """
        # Add to MongoDB
        for restaurant in self.restaurants:
            thread = threading.Thread(
                target=database.add_restaurant, args=(restaurant, self.keyword)
            )
            threads.append(thread)
        for thread in threads:
            thread.start()

        # Updating Silently
        thread = threading.Thread(target=self.silent_update)
        thread.start()

    def get_google_maps_data(self, complete_mode=False):
        """Get Google Maps nearby restaurants data

        Args:
            complete_mode (bool, optional): Set it to True to get complete data. Defaults to False.
        """
        if "[|]" not in self.page_token:
            page_token = self.page_token
            index = 0
        else:
            page_token, index = self.page_token.split("[|]")
        restaurants = GM_Restaurant(
            latitude=self.latitude,
            longitude=self.longitude,
            keyword=self.keyword,
            complete_mode=complete_mode,
            page_token=page_token,
            index=int(index),
        )
        if not complete_mode:
            self.next_page = restaurants.next_page
            if self.next_page:
                token_table = config.db.page_token.find_one({})
                if self.next_page not in token_table["data"].values():
                    token_key = "".join(
                        random.choice(string.ascii_letters + string.digits)
                        for x in range(10)
                    )
                    token_table["data"][token_key] = self.next_page
                    config.db.page_token.update_one({}, {"$set": token_table})
                    self.next_page = token_key
                else:
                    for key, value in token_table["data"].items():
                        if value == self.next_page:
                            self.next_page = key
            self.restaurants = restaurants.restaurants
        else:
            return restaurants

    def get_ifoodie_data(self, restaurants: object = object, complete_mode=False):
        """Get ifoodie data

        Args:
            restaurants (object, optional): Restaurants data. Defaults to object.
            complete_mode (bool, optional): Set it to True to get complete data. Defaults to False.

        Returns:
            [type]: [description]
        """
        threads = []
        if not complete_mode:
            for restaurant in self.restaurants:
                config.restaurants[restaurant.place_id] = restaurant.__dict__
                thread = threading.Thread(
                    target=self.multi_threading_ifoodie, args=(restaurant,)
                )
                threads.append(thread)
        else:
            for restaurant in restaurants.restaurants:
                config.restaurants[restaurant.place_id] = restaurant.__dict__
                thread = threading.Thread(
                    target=self.multi_threading_ifoodie, args=(restaurant,)
                )
                threads.append(thread)

        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()
        if complete_mode:
            return restaurants

    def multi_threading_ifoodie(self, restaurant: object):
        """Multi-threading Get ifoodie data

        Args:
            restaurant (object): Restaurant data
        """
        try:
            data = Ifoodie(
                restaurant_name=restaurant.name,
                latitude=self.latitude,
                longitude=self.longitude,
            )
            restaurant.ifoodie_url = data.restaurant_url
            restaurant.price = int(data.info["均消價位"])
            restaurant.reviews += data.comments
        except ValueError:
            pass
        except IndexError:
            pass

    def silent_update(self):
        threads = []
        # Google Maps
        restaurants = self.get_google_maps_data(complete_mode=True)
        restaurants = self.get_ifoodie_data(complete_mode=True, restaurants=restaurants)
        # Add to MongoDB
        for restaurant in restaurants.restaurants:
            thread = threading.Thread(
                target=database.add_restaurant, args=(restaurant, self.keyword)
            )
            threads.append(thread)
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()
        print("Silent update done.")

# 店家營業狀態
def find_operating_status(data):
    now = datetime.now()
    today_date = now.strftime("%Y:%m:%d")
    time = now.strftime("%H:%M")

    today_open = data[now.weekday()].split(",")
    for each in today_open:
        if "休息" in each:
            return False
        if "24 小時營業" in each:
            return False
        temp = re.findall(r"\d{1,2}\:\d{1,2}", each)
        start = datetime.strptime(f"{today_date}:{temp[0]}", "%Y:%m:%d:%H:%M")
        current = datetime.strptime(f"{today_date}:{time}", "%Y:%m:%d:%H:%M")
        end = datetime.strptime(f"{today_date}:{temp[1]}", "%Y:%m:%d:%H:%M")
        if int(temp[0][0:2]) > int(temp[1][0:2]):
            end += timedelta(days=1)

        if start <= current <= end:
            return True
    return False


import socket
import os
import numpy as np
from os import path

def appendLinesToFile(lines):
    with open("data.csv", 'a') as file:
        file.write(str.join('\n', lines) + "\n")

def loadLearnedModel():
    from keras.models import load_model
    return load_model('savedModel_test.h5')

def listenForData():
    # get the hostname
    host = "localhost"
    port = 49000  # initiate port no above 1024

    print("Starting Robocode NN Server in listening mode...", host, port)

    server_socket = socket.socket()  # get instance
    # look closely. The bind() function takes tuple as argument
    server_socket.bind((host, port))  # bind host address and port together

    # configure how many clients the server can listen simultaneously
    server_socket.listen(5)

    # prepare lines to write
    lines = list()

    # add csv header
    if not path.exists("data.csv"):
        lines.append("ourX,ourY,ourRadarHeading,distanceToTarget,enemyX,enemyY,enemyHeading,enemyEnergy,hittable")

    i = 0
    while True:
        i += 1

        # accept new connection
        conn, address = server_socket.accept()
        # receive data stream. it won't accept data packet greater than 1024 bytes
        data = conn.recv(1024)
        if not data:
            # if data is not received - continue listening
            continue

        # decode data and exclude first two characters (some random bytes)
        decodedLine = data.decode("utf-8", errors="replace")[2:]

        # add to lines
        lines.append(decodedLine)

        # write to file every 100 new entries
        if i % 100 == 0:
            appendLinesToFile(lines)
            lines.clear()

def listenForPredicting():
    # get the hostname
    host = "localhost"
    port = 49000  # initiate port no above 1024

    print("Starting Robocode NN Server in predicting mode...", host, port)

    server_socket = socket.socket()  # get instance
    # the bind() function takes tuple as argument
    server_socket.bind((host, port))  # bind host address and port together

    # configure how many client the server can listen simultaneously
    server_socket.listen(5)

    ourModel = loadLearnedModel()

    while True:
        # accept new connection
        conn, address = server_socket.accept()
        # receive data stream. it won't accept data packet greater than 1024 bytes
        data = conn.recv(1024)

        if not data:
            # if data is not received - continue listening
            continue

        # decode data and exclude first two characters (some random bytes)
        decodedLine = data.decode("utf-8", errors="replace")[2:]
        print(decodedLine)

        decodedLine = np.array([float(x) for x in decodedLine.split(",")]).reshape(1, 8)

        outcome = ourModel.predict(decodedLine)[0:1][0][0:1][0]
        print(outcome)
        conn.send(str(outcome).encode("UTF-8"))  # send outcome to the client
        conn.close()

if __name__ == '__main__':
    # change current working directory to the folder which contains this file
    os.chdir(os.path.dirname(os.path.abspath(__file__)))

    # listening mode
    #listenForData()

    # predicting mode
    listenForPredicting()


class Solution:
    def uniquePaths(self, m: int, n: int) -> int:
        if n > m:
            t = n; n = m; m = t

        if m==1 or n==1:
            return 1

        print("# (m, n) = ({},{}) ".format(m, n))
        for i in range(n):
            if i == 0:
                row = [1 for j in range(m)]
            else:
                print(row)
                for j in range(m-i):
                    if j==0:
                        row[j] = row[j+1] * 2
                    else:
                        row[j] = row[j-1] + row[j+1]
        return row[j]

if __name__ == "__main__":
    import sys
    m = int(sys.argv[1])
    n = int(sys.argv[2])
    sol = Solution()
    answer = sol.uniquePaths(m, n)
    print("# {}".format(answer))


# coding: utf-8

from selenium import webdriver
import selenium
from time import sleep
import time
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

if __name__ == '__main__':
    url = "https://signin.aliyun.com/login.htm"
    # url = "https://www.baidu.com/"
    pic_name = "ribao.png"
    chromedriver = "/Users/uxin/Utils/chromedriver"
    from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap['phantomjs.page.settings.userAgent'] = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36')
    chrome_options = webdriver.ChromeOptions()
    # 使用headless无界面浏览器模式
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--enable-gpu')

    # browser = webdriver.Chrome(chromedriver,chrome_options=chrome_options)
    browser = webdriver.PhantomJS(executable_path="/Users/uxin/Utils/phantomjs-2.1.1-macosx/bin/phantomjs",desired_capabilities=dcap)
    # 设置超时时间
    browser.set_page_load_timeout(100)
    browser.set_script_timeout(100)

    # browser.maximize_window()
    browser.get(url = url)

    # browser.switch_to_frame("alibaba-login-box")
    # browser.find_element_by_id("user_principal_name").send_keys("sky.bai@uxin.com")
    browser.find_element_by_id("user_principal_name").send_keys("bigdata_rep_dev@uxin")
    browser.find_element_by_id("J_FormNext").click()
    sleep(3)
    browser.find_element_by_id("password_ims").send_keys("OOWwk9IDha9DfAkz")
    browser.find_element_by_class_name("submit-btn").click()
    # browser.get("https://das.base.shuju.aliyun.com/console.htm")
    sleep(10)
    browser.get("https://das.base.shuju.aliyun.com/shulaibao/preview.htm?pageId=07aa0d12-016e-4a5d-91e7-49ea70974eff#e314910d-b220-492b-9faf-9285bcc8bf1e*=*{%22time%22:{%22t%22:{%22from%22:{%22mark%22:%22-%22,%22d%22:1,%22suffix%22:%22D%22},%22to%22:{%22mark%22:%22-%22,%22d%22:1,%22suffix%22:%22D%22}}}}#4c7c1ade-e6d8-4e97-b0cd-a693447b5d4b*=*{%22time%22:{%22t%22:{%22from%22:{%22mark%22:%22-%22,%22d%22:31,%22suffix%22:%22D%22},%22to%22:{%22mark%22:%22-%22,%22d%22:1,%22suffix%22:%22D%22}}}}")
    sleep(10)
    browser.maximize_window()
    browser.save_screenshot("ribao.png")
    browser.close()


import json
import os

def get_champions_from_file(filename):
        file = open('../../matches/{0}'.format(filename), 'r', encoding='utf-8')
        text = file.read()
        parsed = json.loads(text)
        if len(parsed['participants']) != 10:
            print("Error: not 5 v 5 game (participants: {0})! ID: {1}".format(len(parsed['participants']), filename))
            return None, None, None
        if parsed['queueId'] != 420:
            print("Error: not ranked. ID: {0}".format(filename))
            return None, None, None
        blue_champions = [str(parsed['participants'][x]['championId']) for x in range(5)]
        red_champions = [str(parsed['participants'][x]['championId']) for x in range(5, 10)]
        result = parsed['teams'][0]['win']
        if result == 'Fail':
            result = 0
        else:
            result = 1
        return blue_champions, red_champions, result

def parse(blue_champions, red_champions, result):
    return ' '.join(blue_champions) + '\t' + ' '.join(red_champions) + '\t' + str(result)

def get_all_champions_from_files():
    results = []
    data_files = [x for x in os.listdir('../../matches') if x.endswith('.json')]

    for filename in data_files:
        blue_champions, red_champions, result = get_champions_from_file(filename)
        if blue_champions is not None:
            results.append(parse(blue_champions, red_champions, result))

    return results

out_file = open('../../Data/champions.txt', 'w')
results = get_all_champions_from_files()
for result in results:
    out_file.write(result + '\n')

out_file.close()


class Solution:
    # @return an integer
    def threeSumClosest(self, num, target):
        minDiff = 1 << 31 - 1
        closest = 0
        num.sort()
        for i in range(len(num) - 1):
            left = i + 1
            right = len(num) - 1
            while left < right:
                curr = num[i] + num[left] + num[right]
                if abs(curr - target) < minDiff:
                    minDiff = abs(curr - target)
                    closest = curr
                if curr > target:
                    right -= 1
                else:
                    left += 1
        return closest


class Solution:
    def matchFollowing(self, s, p, ix):
        if len(s) - 1 < ix or len(p) - 1 < ix:
            return False
        if p[ix] != '*' and p[ix] != '.':
            if p[ix] == s[ix]:
                return True
            else:
                return False
        if p[ix] == '.':
            print(" . ")
        if p[ix] == '*':
            print(" * ")

    def isMatch(self, s, p):
        """
        :type s: str
        :type p: str
        :rtype: bool
        """
        if p[0] == '*':
            return -1
        self.matchFollowing(s, p, 9)

s = "mississippi"
p = "mis*is*p*."
a = Solution()
a.isMatch(s, p)


from nanoAPI.utils import warn

class __BaseType__():
    def __init__(self, model_type: str, name: str, unique: bool, required: bool):
        if name is not None:
            self.model_type = model_type.lower().capitalize()
            self.name = name.lower()
            self.unique = 'UNIQUE ' if unique else ''
            self.required = 'NOT NULL ' if required else ''  # Default : False
            self.base_cmd = f"{self.name} {model_type} {self.required}{self.unique}"
        else:
            raise TypeError(warn("Provide a name for the field"))

class IntegerType(__BaseType__):
    pytype = int
    def __init__(self, name=None, max=10000000, min=-10000000, unique=False, required=False, default=None):
        super().__init__('INTEGER', name, unique, required)
        self.check = f"CHECK ({self.name} < {max} AND {self.name} > {min})"
        self.default = f"DEFAULT {default if default else 'NULL'} "
        self.command = self.base_cmd + self.default + self.check

class StringType(__BaseType__):
    pytype = str
    def __init__(self, name=None, max_length=None, min_length=0, unique=False, required=False, default=None):
        if max_length is not None:
            super().__init__('TEXT', name, unique, required)
            self.default = f"DEFAULT {default if default else 'NULL'} "
            self.check = f"CHECK (length({self.name}) < {max_length} AND length({self.name}) > {min_length})"
            self.command = self.base_cmd + self.check
        else:
            raise TypeError(warn("Must provide a max_length for StringType"))

class BooleanType(__BaseType__):
    pytype = bool
    def __init__(self, name=None, unique=False, required=False, default=None):
        super().__init__('INTEGER', name, unique, required)
        self.default = f"DEFAULT {1 if default else 0} "
        self.command = self.base_cmd + self.default

class EmailType(__BaseType__):
    pytype = str
    def __init__(self, name: str, unique: bool, required: bool):
        super().__init__('TEXT', name, unique, required)
        self.command = self.base_cmd

class RelationType(__BaseType__):
    pytype = str
    def __init__(self, name: str, unique: bool, required: bool):
        super().__init__('TEXT', name, unique, required)
        self.command = self.base_cmd


'''
GET ALL POSSIBLE FAILURE MESSAGES
'''
__author__ = 'john pierson - sixtysecondrevit@gmail.com'
__twitter__ = '@60secondrevit'
__github__ = '@sixtysecondrevit'
__version__ ='1.0.0'

# dynamo version - 2.6.0
# originally made to answer https://forum.dynamobim.com/t/room-location-point-of-a-family-the-green-dot/58480/2

# import common language runtime
import clr

# Import RevitAPI
clr.AddReference("RevitAPI")
# import all classes from Revit DB
from Autodesk.Revit.DB import *

# import Revit dynamo libraries
clr.AddReference("RevitNodes")
import Revit
clr.ImportExtensions(Revit.Elements)
clr.ImportExtensions(Revit.GeometryConversion)

# custom definition for location that works with single items or lists
def GetLocation(item):
	# if it has a calculation point return that
	if item.HasSpatialElementCalculationPoint : return item.GetSpatialElementCalculationPoint().ToPoint()
	# if not return the regular location
	elif item.Location.GetType().ToString().Contains("LocationPoint") : return item.Location.Point.ToPoint()

	else: return None

# the family instances
items = UnwrapElement(IN[0])

# return the results
if isinstance(IN[0], list): OUT = [GetLocation(x) for x in items]
else: OUT = GetLocation(items)


from collections import OrderedDict
from seqdataloader.batchproducers import coordbased
from seqdataloader.batchproducers.coordbased import coordstovals
from seqdataloader.batchproducers.coordbased import coordbatchproducers
from seqdataloader.batchproducers.coordbased import coordbatchtransformers
from seqdataloader.batchproducers.coordbased.core import Coordinates
from seqdataloader.batchproducers.coordbased.coordstovals.core import CoordsToValsJoiner
from seqdataloader.batchproducers.coordbased.coordstovals.bigwig import AbstractCountAndProfileTransformer
from seqdataloader.batchproducers.coordbased.coordstovals.bigwig import LogCountsPlusOne
from seqdataloader.batchproducers.coordbased.coordstovals.bigwig import SmoothProfiles
from seqdataloader.batchproducers.coordbased.coordstovals.bigwig import BigWigReader
from seqdataloader.batchproducers.coordbased.coordstovals.bigwig import smooth_profiles
from seqdataloader.batchproducers.coordbased.coordstovals.bigwig import rolling_window
from seqdataloader.batchproducers.coordbased.coordstovals.bigwig import MultiTrackCountsAndProfile
from keras_genomics.layers.convolutional import RevCompConv1D
from seqdataloader.batchproducers.coordbased.core import Coordinates, KerasBatchGenerator, apply_mask
from deeplift.visualization import viz_sequence
from matplotlib import pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import keras
import keras.layers as kl
from keras.models import load_model
from keras.utils import CustomObjectScope
import shap
from deeplift.dinuc_shuffle import dinuc_shuffle
import tensorflow as tf
import tensorflow_probability as tfp
from pathlib import Path
import numpy as np
import subprocess
import gzip
import optparse
import os

parser = optparse.OptionParser()

parser.add_option('--gpus',
    action="store", dest="gpus",
    help="which gpus to use", default=None)
parser.add_option('--repeat',
    action="store", dest="repeat",
    help="what pattern to test", default=None)

options, args = parser.parse_args()

os.environ["CUDA_VISIBLE_DEVICES"]=options.gpus

def multinomial_nll(true_counts, logits):
    """Compute the multinomial negative log-likelihood
    Args:
      true_counts: observed count values
      logits: predicted logit values
    """
    counts_per_example = tf.reduce_sum(true_counts, axis=-1)
    dist = tfp.distributions.Multinomial(total_count=counts_per_example,
                                         logits=logits)
    return (-tf.reduce_sum(dist.log_prob(true_counts)) /
            tf.to_float(tf.shape(true_counts)[0]))

#from https://github.com/kundajelab/basepair/blob/cda0875571066343cdf90aed031f7c51714d991a/basepair/losses.py#L87
class MultichannelMultinomialNLL(object):
    def __init__(self, n):
        self.__name__ = "MultichannelMultinomialNLL"
        self.n = n

    def __call__(self, true_counts, logits):
        for i in range(self.n):
            loss = multinomial_nll(true_counts[..., i], logits[..., i])
            if i == 0:
                total = loss
            else:
                total += loss
        return total

    def get_config(self):
        return {"n": self.n}

with CustomObjectScope({'MultichannelMultinomialNLL': MultichannelMultinomialNLL,'RevCompConv1D': RevCompConv1D}):
    model = load_model("/users/amr1/pho4/data/models/max_hela_1_model.h5")

fastapath = "/users/amr1/pho4/data/genome/hg38/hg38.genome.fa"
GenomeDict={}
sequence=''
inputdatafile = open(fastapath)
for line in inputdatafile:
    if line[0]=='>':
        if sequence != '':
            GenomeDict[chrm] = ''.join(sequence)
        chrm = line.strip().split('>')[1]
        sequence=[]
        Keep=False
        continue
    else:
        sequence.append(line.strip())
GenomeDict[chrm] = ''.join(sequence)

seq_len = 1346
out_pred_len = 1000
test_chrms = ["chr1", "chr8", "chr21"]
seq_peaks = []
with gzip.open("/users/amr1/pho4/data/gcpbm/max_hela_1/idr.optimal_peak.narrowPeak.gz", 'rt') as inp:
    for line in inp:
        chrm = line.strip().split('\t')[0]
        if chrm not in test_chrms:
            continue
        pStart = int(line.strip().split('\t')[1])
        summit = pStart + int(line.strip().split('\t')[-1])
        start = int(summit - (seq_len/2))
        end = int(summit + (seq_len/2))
        seq_peaks.append(GenomeDict[chrm][start:end].upper())

ltrdict = {
           'a':[1,0,0,0],'c':[0,1,0,0],'g':[0,0,1,0],'t':[0,0,0,1],
           'n':[0,0,0,0],'A':[1,0,0,0],'C':[0,1,0,0],'G':[0,0,1,0],
           'T':[0,0,0,1],'N':[0,0,0,0]}
def getOneHot(ISM_sequences):
  # takes in list of sequences
    one_hot_seqs = []
    for seq in ISM_sequences:
        one_hot = []
        for i in range(len(seq)):
            one_hot.append(ltrdict[seq[i:i+1]])
        one_hot_seqs.append(one_hot)
    return np.array(one_hot_seqs)

def fill_into_center(seq, insert):
    flank = int((len(seq)-len(insert))/2.0)
    new_seq = seq[:flank]+insert+seq[flank+len(insert):]
    return new_seq

motif = "GTCACGTGAC"
candidates = []
for seq in seq_peaks:
    candidates.append(fill_into_center(dinuc_shuffle(seq), motif))
preds = model.predict([getOneHot(candidates), np.zeros((len(candidates),)), np.zeros((len(candidates),out_pred_len,2))])
count_preds = np.mean(preds[0], axis=1)
background = candidates[np.argmin(count_preds)]

mirror_dict = {'A':'T', 'C':'G', 'G':'C', 'T':'A'}

def getMirror(pattern):
    ret = ""
    for letter in pattern:
        ret += mirror_dict[letter]
    return ret[::-1]

def getRepeat(repeatLen, repeatPattern, stack='left', mirror=False):
    if stack == 'left':
        ret = repeatPattern * int(repeatLen/len(repeatPattern))
        if repeatLen%len(repeatPattern) != 0:
            ret = repeatPattern[-repeatLen%len(repeatPattern):] + ret
        return ret
    elif stack == 'right':
        ret = repeatPattern * int(repeatLen/len(repeatPattern))
        ret += repeatPattern[:repeatLen%len(repeatPattern)]
        return ret
    elif stack == 'both':
        if mirror:
            return (getRepeat(repeatLen, repeatPattern, stack='left', mirror=False),
                    getRepeat(repeatLen, getMirror(repeatPattern), stack='right', mirror=False))
        else:
            return (getRepeat(repeatLen, repeatPattern, stack='left', mirror=False),
                    getRepeat(repeatLen, repeatPattern, stack='right', mirror=False))
    else:
        print("unrecognized argument for stack")
        return None

bigFont = {'weight' : 'bold', 'size'   : 22}
smallFont = {'size' : 10}

repeatUntil = 65
repeatPattern = options.repeat
parentDir = "max_hela_1_preds_trends/"

def predictPlotAndSave(seqs, name):
    test_seqs = getOneHot(seqs)
    test_preds = model.predict([test_seqs, np.zeros((len(seqs),)),np.zeros((len(seqs),out_pred_len,2))])
    test_preds_logcount = np.mean(test_preds[0], axis = -1)
    np.save(parentDir+repeatPattern+'_'+name+'_preds.npy', test_preds_logcount)

    fig = plt.figure(figsize=(10,9))
    plt.rc('font', **bigFont)
    plt.plot(range(len(seqs)), test_preds_logcount)
    plt.xlabel("repeat length")
    plt.ylabel("count prediction")
    plt.title(repeatPattern+" repeats "+name+" of motif")
    fig.savefig(parentDir+repeatPattern+'_'+name+'_trend.png')
    plt.close(fig)

# left only
seqs = []
for repeatLen in range(repeatUntil):
    repeat = getRepeat(repeatLen, repeatPattern, stack='left', mirror=False)
    insert = repeat + motif
    start = int((seq_len/2)-(len(motif)/2))-repeatLen
    end = int((seq_len/2)+(len(motif)/2))
    seqs.append(background[:start] + insert + background[end:])
predictPlotAndSave(seqs, "left")

# right only
seqs = []
for repeatLen in range(repeatUntil):
    repeat = getRepeat(repeatLen, repeatPattern, stack='right', mirror=False)
    insert = motif + repeat
    start = int((seq_len/2)-(len(motif)/2))
    end = int((seq_len/2)+(len(motif)/2))+repeatLen
    seqs.append(background[:start] + insert + background[end:])
predictPlotAndSave(seqs, "right")

# both
seqs = []
for repeatLen in range(repeatUntil):
    leftRepeat, rightRepeat = getRepeat(repeatLen, repeatPattern, stack='both', mirror=False)
    insert = leftRepeat + motif + rightRepeat
    start = int((seq_len/2)-(len(motif)/2))-repeatLen
    end = int((seq_len/2)+(len(motif)/2))+repeatLen
    seqs.append(background[:start] + insert + background[end:])
predictPlotAndSave(seqs, "both")

# mirror both
seqs = []
for repeatLen in range(repeatUntil):
    leftRepeat, rightRepeat = getRepeat(repeatLen, repeatPattern, stack='both', mirror=True)
    insert = leftRepeat + motif + rightRepeat
    start = int((seq_len/2)-(len(motif)/2))-repeatLen
    end = int((seq_len/2)+(len(motif)/2))+repeatLen
    seqs.append(background[:start] + insert + background[end:])
predictPlotAndSave(seqs, "both_mirror")


# -*- coding: utf-8 -*-

from odoo import models, fields, api, _
from odoo.exceptions import ValidationError,UserError
from datetime import datetime, time
from dateutil.relativedelta import relativedelta

import logging

_logger = logging.getLogger(__name__)

class Project(models.Model):

    _name = 'project.project'
    _inherit = ['project.project', 'mail.thread', 'mail.activity.mixin']

    # We Override this method from 'project_task_default_stage
    def _get_default_type_common(self):
        ids = self.env['project.task.type'].search([
            ('case_default', '=', True),
            ('project_type_default', '=', self.project_type)])
        _logger.info("Default Stages: {} for project type {}".format(ids.mapped('name'),self.project_type))
        return ids

    type_ids = fields.Many2many(
        default=lambda self: self._get_default_type_common())

    user_id = fields.Many2one('res.users', string='Project Manager', default=lambda self: self._default_user_id() , track_visibility="onchange")

    project_type = fields.Selection([
        ('dev', 'Developement'),
        ('client', 'Client'),
        ('internal','Internal')],
        string = 'Project Type',
        default = 'client',
    )

    # Maximum parent level here is meant to be 1 at max for now
    parent_id = fields.Many2one(
        'project.project', 'Parent project',
        index=True, ondelete='cascade',
        compute='_get_parent_id',
        store=True
    )
    child_id = fields.One2many('project.project', 'parent_id', 'Child projects')

    parent_task_count = fields.Integer(
        compute='_compute_parent_task_count'
    )

    child_task_count = fields.Integer(
        compute='_compute_child_task_count'
    )

    #consultant_ids = fields.Many2many('hr.employee', string='Consultants')
    #ta_ids = fields.Many2many('hr.employee', string='Ta')
    completion_ratio = fields.Float('Task Complete', compute='compute_project_completion_ratio', store=True)
    consummed_completed_ratio = fields.Float('BC / TC',
                                             compute='compute_project_consummed_completed_ratio', store=True)
    summary_ids = fields.One2many(
        'project.summary', 'project_id',
        'Project summaries'
    )
    is_project_manager = fields.Boolean(compute="_get_is_project_manager")
    scope_of_work = fields.Html(string='Scope of work')
    orders_count = fields.Integer(compute='compute_orders_count')
    tasks_count = fields.Integer()
    timesheets_count = fields.Integer()
    lead_consultant = fields.Many2one('hr.employee', related='core_team_id.lead_consultant', string='Lead Consultant')
    assistant_id = fields.Many2one('hr.employee', related='core_team_id.assistant_id', string='Project Assistant')
    lead_backup = fields.Many2one('hr.employee', related='core_team_id.lead_backup',
                                  string='Lead Consultant Backup')
    consultant_ids = fields.Many2many('hr.employee', 'rel_project_consultants', related='core_team_id.consultant_ids',
                                      string='Consultants')
    ta_ids = fields.Many2many('hr.employee', relation='rel_project_tas', related='core_team_id.ta_ids',
                              string='Ta')
    controller_id = fields.Many2one('res.users', related='partner_id.controller_id', string='Project Controller')
    invoice_admin_id = fields.Many2one('res.users', related='partner_id.invoice_admin_id', string='Invoice Administrator')

    invoices_count = fields.Integer(
        compute='_get_out_invoice_ids',
        compute_sudo = True
    )
    out_invoice_ids = fields.Many2many(
        string='Out invoices',
        compute='_get_out_invoice_ids',
        comodel_name="account.invoice",
        relation="project_out_invoice_rel",
        column1="project_id",
        column2="invoice_id",
        store=True,
        copy=False, readonly=True,
        compute_sudo=True
    )

    risk_ids = fields.Many2many(
        'risk', string='Risk',
        compute='_get_risks',
        store = True,
    )

    risk_score = fields.Integer(
        string='Risk Score',
        compute='_compute_risk_score',
        store=True,
    )

    invoiceable_amount = fields.Monetary(
       related="sale_order_id.invoiceable_amount", readonly=True, store=True
    )

    invoicing_mode = fields.Selection(related="sale_order_id.invoicing_mode", readonly=True)

    last_summary_date = fields.Datetime(string="Last Summary Date", compute="_compute_last_create_date", readonly=True)

    subscription_count = fields.Integer(compute='_compute_subscription_count')

    date_start = fields.Date(
        string='Start Date',
        compute='_compute_dates',)

    date = fields.Date(
        string='Expiration Date',
        compute='_compute_dates',
        index=True,
        track_visibility='onchange',
        )

    #accounting fields for legacy integration
    external_account = fields.Char(
        default="/",
    )

    sharepoint_folder = fields.Char(
        string='Sharepoint Folder',
        compute='_compute_sharepoint_folder',
        readonly=True,
        store=True,
    )

    show_folder_path = fields.Boolean()

    @api.depends('sale_order_id', 'project_type', 'parent_id','partner_id')
    def _compute_sharepoint_folder(self):
        pre = self.env.ref('vcls-contact.SP_client_root_prefix').value
        post = self.env.ref('vcls-contact.SP_client_root_postfix').value
        for project in self.filtered(lambda p: p.project_type=='client' and p.sale_order_id and p.partner_id.altname):
            reference = project.parent_id.sale_order_id.internal_ref if project.parent_id \
                else project.sale_order_id.internal_ref or ''
            if reference:
                reference = "{}-{}".format(reference.split('-')[0],reference.split('-')[1])
            project.sharepoint_folder = "{}/{}/{}/{}{}".format(pre,project.partner_id.altname[0],project.partner_id.altname,reference,post)
            project.show_folder_path = True

    def _compute_dates(self):
        for project in self:
            tasks = project.task_ids.filtered(lambda t: t.date_start and t.date_end)
            if tasks:
                project.date_start = min(tasks.mapped('date_start')).date()
                project.date = max(tasks.mapped('date_end')).date()
            elif project.sale_order_id:
                project.date_start = project.sale_order_id.expected_start_date
                project.date = project.sale_order_id.expected_end_date
            else:
                project.date_start = False
                project.date = False

    def _compute_subscription_count(self):
        """Compute the number of distinct subscriptions linked to the orders of the project(s)."""
        for project in self:
            all_projects = project
            if project.child_id:
                all_projects |= project.child_id
            if project.parent_id:
                all_projects |= project.parent_id

            sub_count = len(self.env['sale.order.line'].read_group([('order_id', 'in', all_projects.mapped('sale_order_id.id')), ('subscription_id', '!=', False)],
                                                    ['subscription_id'], ['subscription_id']))
            project.subscription_count = sub_count

    def action_open_subscriptions(self):
        """Display the linked subscription and adapt the view to the number of records to display."""
        self.ensure_one()

        all_projects = self
        if self.child_id:
            all_projects |= self.child_id
        if self.parent_id:
            all_projects |= self.parent_id

        subscriptions = all_projects.mapped('sale_order_id.order_line.subscription_id')

        action = self.env.ref('sale_subscription.sale_subscription_action').read()[0]
        if len(subscriptions) > 1:
            action['domain'] = [('id', 'in', subscriptions.ids)]
        elif len(subscriptions) == 1:
            form_view = [(self.env.ref('sale_subscription.sale_subscription_view_form').id, 'form')]
            if 'views' in action:
                action['views'] = form_view + [(state,view) for state,view in action['views'] if view != 'form']
            else:
                action['views'] = form_view
            action['res_id'] = subscriptions.ids[0]
        else:
            action = {'type': 'ir.actions.act_window_close'}
        return action

    @api.depends('summary_ids')
    def _compute_last_create_date(self):
        for project in self:
            if project.summary_ids:
                project.last_summary_date = project.summary_ids.sorted(lambda s: s.create_date, reverse=True)[0].create_date

    def _get_risks(self):
        for project in self:
            project.risk_ids = self.env['risk'].search([
                ('resource', '=', 'project.project,{}'.format(project.id)),
            ])

    @api.depends('risk_ids', 'risk_ids.score')
    def _compute_risk_score(self):
        for project in self:
            project.risk_score = sum(project.risk_ids.mapped('score'))

    @api.one
    @api.depends(
        'sale_line_id.order_id.order_line.invoice_lines',
        'tasks.sale_order_id',
    )
    def _get_out_invoice_ids(self):
        # use of sudo here because of the non possibility of lc to read some invoices
        project_sudo = self.sudo()
        out_invoice_ids = project_sudo.mapped('sale_line_id.order_id.invoice_ids') \
            | project_sudo.mapped('tasks.sale_order_id.invoice_ids')
        self.out_invoice_ids = out_invoice_ids
        self.invoices_count = len(out_invoice_ids)

    @api.multi
    @api.depends(
        'sale_order_id', 'sale_order_id.parent_id',
        'sale_order_id.parent_id.project_id'
    )
    def _get_parent_id(self):
        for project in self:
            if project.sale_order_id:
                project.parent_id = project.sale_order_id.parent_id.project_id

    ##################
    # CUSTOM METHODS #
    ##################
    @api.multi
    def get_tasks_for_project_sub_project(self):
        """This function will return all the tasks and subtasks found in the main and Child
        Projects which participates in KPI's"""
        self.ensure_one()
        tasks = self.task_ids + self.child_id.mapped('task_ids')
        all_tasks = tasks + tasks.mapped('child_ids')
        return all_tasks.filtered(lambda task: task.sale_line_id.product_id.completion_elligible and
                                  task.stage_id.status not in  ['not_started','cancelled'])

    @api.multi
    def action_raise_new_invoice(self):
        """This function will trigger the Creation of invoice regrouping all the sale orders."""
        orders = self.env['sale.order']
        for project in self:
            if not project.sale_order_id and project.sale_order_id.invoice_status != 'to invoice':
                raise UserError(_("The selected Sales Order should contain something to invoice."))
            else:
                orders |= project.sale_order_id

        action = self.env.ref('sale.action_view_sale_advance_payment_inv').read()[0]
        action['context'] = {
            'active_ids': orders.ids
        }
        return action

    @api.multi
    def action_raise_new_risk(self):
        self.ensure_one()
        action = self.env.ref('vcls-risk.action_view_risk_wizard').read()[0]
        action['context'] = {
            'default_resource': 'project.project,{}'.format(self.id),
            'new_risk':True,
        }
        return action

    @api.multi
    def show_risks(self):
        self.ensure_one()
        action = self.env.ref('vcls-risk.action_view_risk').read()[0]
        action['domain'] = [('resource', '=', 'project.project,{}'.format(self.id))]
        action['target'] = 'current'
        return action

    @api.multi
    def sale_orders_tree_view(self):
        action = self.env.ref('sale.action_quotations').read()[0]
        action['domain'] = [('project_id', 'child_of', self.id)]
        action['context'] = {}
        return action

    @api.multi
    def invoices_tree_view(self):
        self.ensure_one()
        action = self.env.ref('account.action_invoice_tree1').read()[0]
        action['domain'] = [('id', 'in', self.out_invoice_ids.ids)]
        return action

    @api.multi
    def tasks_tree_view(self):
        action = self.env.ref('project.act_project_project_2_project_task_all').read()[0]
        return action

    @api.multi
    def forecasts_tree_view(self):
        action = self.env.ref('project_forecast.project_forecast_action_from_project').read()[0]
        action['domain'] = [('project_id', 'child_of', self.id)]
        return action

    @api.multi
    def timesheets_tree_view(self):
        action = self.env.ref('hr_timesheet.act_hr_timesheet_line_by_project').read()[0]
        action['domain'] = [('project_id', 'child_of', self.id)]
        return action

    @api.multi
    def report_analysis_tree_view(self):
        action = self.env.ref('vcls-timesheet.project_timesheet_forecast_report_action').read()[0]
        action['domain'] = [('project_id', 'child_of', self.id)]
        action['context'] = {'active_id': self.id, 'active_model': 'project.project'}
        return action

    ###############
    # ORM METHODS #
    ###############
    @api.model
    def create(self, vals):

        #default visibility
        vals['privacy_visibility'] = 'portal'

        #if no ID defined, then increment using the sequence
        if vals.get('external_account','/')=='/':
            vals['external_account'] = self.env['ir.sequence'].next_by_code('seq_customer_project_id')

        #we automatically assign the project manager to be the one defined in the core team
        if vals.get('sale_order_id',False):
            so = self.env['sale.order'].browse(vals.get('sale_order_id'))
            vals['scope_of_work'] = so.scope_of_work
            lc = so.core_team_id.lead_consultant
            _logger.info("SO info {}".format(lc.name))
            if lc:
                vals['user_id']=lc.user_id.id

        project = super(Project, self).create(vals)
        ids = project._get_default_type_common()
        project.type_ids = ids if ids else project.type_ids

        if project.project_type != 'client':
            project.privacy_visibility = 'employees'

        return project

    @api.multi
    def unlink(self):
        authorized = self.env.user.has_group('vcls_security.group_project_controller') or self.env.user.has_group('vcls_security.group_bd_admin')

        if not authorized:
            raise UserError(_("PROJECT UNLINK | You need to be a member of 'BD Admin' or 'Project Controller' to delete project(s)."))

        #we catch all child projects
        projects = self.env['project.project']
        for project in self:
            if project.active:
                raise UserError(_("PROJECT UNLINK | To avoid un-wanted deletion, we don't delete active project(s). Please archive {} 1st").format(project.name))
            else:
                projects |= project
                projects |= project.child_id

        #we look for invoices
        invoices = projects.mapped('out_invoice_ids')
        if invoices:
                raise UserError(_("PROJECT UNLINK | We can't delete projects having invoices, please archive instead\n{}").format(invoices.mapped('name')))

        for project in projects:
            _logger.info("PROJECT UNLINK | Cleaning {}".format(project.name))
            #we look for timesheets
            ts = self.env['account.analytic.line'].with_context(active_test=False).search([('project_id','=',project.id)])
            if ts:
                _logger.info("PROJECT UNLINK | Timesheets found {}".format(len(ts)))
                if not self.env.user.has_group('vcls_security.group_project_controller'):
                    raise UserError(_("PROJECT UNLINK | You need to be a member of 'Project Controller' to delete timesheet(s)."))
                ts.unlink()
            #we look for forecasts
            fc = self.env['project.forecast'].with_context(active_test=False).search([('project_id','=',project.id)])
            if fc:
                _logger.info("PROJECT UNLINK | Forecasts found {}".format(len(fc)))
                fc.unlink()
            #we clean the tasks
            tasks = self.env['project.task'].with_context(active_test=False).search([('project_id','=',project.id)])
            if tasks:
                _logger.info("PROJECT UNLINK | Tasks found {}".format(len(tasks)))
                tasks.write({'sale_line_id':False})
                tasks.unlink()
            #we clean the mapping
            if project.sale_line_employee_ids:
                _logger.info("PROJECT UNLINK | SO line mappings {}".format(len(project.sale_line_employee_ids)))
                project.sale_line_employee_ids.unlink()

        return super(Project, self).unlink()

    ###################
    # COMPUTE METHODS #
    ###################

    @api.model
    def _default_user_id(self):
        if self.project_type == 'client':
            if self.sale_order_id:
                return self.sale_order_id.opportunity_id.user_id
        return self.env.user

    def _compute_parent_task_count(self):
        task_data = self.env['project.task'].read_group([('parent_id','=',False),('project_id', 'in', self.ids), '|', ('stage_id.fold', '=', False), ('stage_id', '=', False)], ['project_id'], ['project_id'])
        result = dict((data['project_id'][0], data['project_id_count']) for data in task_data)
        for project in self:
            project.parent_task_count = result.get(project.id, 0)

    def _compute_child_task_count(self):
        task_data = self.env['project.task'].read_group([('parent_id','!=',False),('project_id', 'in', self.ids), '|', ('stage_id.fold', '=', False), ('stage_id', '=', False)], ['project_id'], ['project_id'])
        result = dict((data['project_id'][0], data['project_id_count']) for data in task_data)
        for project in self:
            project.child_task_count = result.get(project.id, 0)

    def _compute_task_count(self):
        read_group_res = self.env['project.task'].read_group([('project_id', 'child_of', self.ids), '|', ('stage_id.fold', '=', False), ('stage_id', '=', False)], ['project_id'], ['project_id'])
        group_data = dict((data['project_id'][0], data['project_id_count']) for data in read_group_res)
        for project in self:
            task_count = 0
            for sub_project_id in project.search([('id', 'child_of', project.id)]).ids:
                task_count += group_data.get(sub_project_id, 0)
            project.task_count = task_count

    @api.multi
    @api.depends('task_ids.completion_ratio')
    def compute_project_completion_ratio(self):
        for project in self:
            tasks = project.get_tasks_for_project_sub_project()
            project.completion_ratio = sum(tasks.mapped('completion_ratio')) / len(tasks) if tasks else sum(tasks.mapped('completion_ratio'))

    @api.multi
    @api.depends('task_ids.consummed_completed_ratio')
    def compute_project_consummed_completed_ratio(self):
        for project in self:
            tasks = project.get_tasks_for_project_sub_project()
            project.consummed_completed_ratio = sum(tasks.mapped('consummed_completed_ratio')) / len(tasks) if tasks else sum(tasks.mapped('consummed_completed_ratio'))

    @api.multi
    def _get_is_project_manager(self):
        for p in self:
            p.is_project_manager = p.user_id == self.env.user

    @api.multi
    def compute_orders_count(self):
        for project in self:
            project.orders_count = self.env['sale.order'].search_count([('project_id', 'child_of', project.id)])

    @api.multi
    def compute_invoices_count(self):
        self.ensure_one()
        projects = self.env['project.project'].search([('id', 'child_of', self.id)])
        sale_orders = projects.mapped('sale_line_id.order_id') | projects.mapped('tasks.sale_order_id')
        invoices = sale_orders.mapped('invoice_ids').filtered(lambda inv: inv.type == 'out_invoice')
        self.invoices_count = len(invoices)

    @api.multi
    def toggle_active(self):
        if any(project.activity_ids for project in self):
            raise ValidationError(_("Its not possible to archive a project while there is still undone activities "))
        super(Project, self).toggle_active()

    @api.model
    def end_project_activities_scheduling(self):
        for project_id in self.search([('active', '=', True), ('parent_id', '=', False)]):
            task_ids = [task for child_id in project_id.child_id for task in child_id.task_ids]
            if task_ids and all(task.stage_id.status in ("completed", "cancelled") for task in task_ids):
                users_summary = {
                    project_id.partner_id.user_id.id: _('Client Feedback'),
                    project_id.user_id.id: _('End of project form filling'),
                    project_id.partner_id.invoice_admin_id.id: _('Invoicing Closing')
                }
                for user_id, summary in users_summary.items():
                    if user_id:
                        activity_vals = {
                            'user_id': user_id,
                            'summary': summary,
                            'act_type_xmlid': 'mail.mail_activity_data_todo',
                            'automated': True,
                        }
                        project_id.sudo().activity_schedule(**activity_vals)
        return True

    def invoicing_session_done(self):
        """
            Timesheets stage_id from pc_review to invoiceable
        """
        if not self.env.user.has_group('vcls_security.group_project_controller'):
            raise ValidationError(_("You need to have the project controller access right."))
        project_ids = self.browse(self._context.get('active_ids'))
        all_projects = project_ids.filtered(lambda p: p.project_type=='client')
        for project in project_ids:
            all_projects += project.child_id
        timesheet_ids = self.env['account.analytic.line'].search([('main_project_id', 'in', all_projects.ids),
                                                                 ('stage_id', '=', 'pc_review')])
        if timesheet_ids:
            timesheet_ids.write({'stage_id': 'invoiceable'})

        #we update the timesheet limit date to the end of the previous month
        today = fields.Date.today()
        ts_limit_date =  today.replace(day=1) - relativedelta(days=1)
        if all_projects:
            all_projects.mapped('sale_order_id').write({'timesheet_limit_date':ts_limit_date})

        #we trigger the computation of KPIs
        self.env['project.task']._cron_compute_kpi()

    @api.multi
    def write(self, values):
        if values.get('active', None) is False:
            tasks = self.mapped('tasks')
            tasks |= tasks.mapped('child_ids')
            tasks.write({'active': False})
        return super(Project, self).write(values)

    @api.multi
    def _get_family_project_ids(self):
        self.ensure_one()
        parent_project_id = self.parent_id or self
        family_project_ids = parent_project_id | parent_project_id.child_id
        return family_project_ids


def getLeaderJson(row):
    obj = {}
    obj["L_id"]=row[0]
    obj["FirstName"]=row[1]
    obj["LastName"]=row[2]
    obj["age"]=row[3]
    obj["S_id"]=row[4]
    return obj

def parseLeaderTable(records):
    response = {}
    for row in records:
        obj = getLeaderJson(row)
        response[row[0]] = obj
    return response

def insertLeaderValue(mysql,args):
    cur = mysql.connection.cursor()
    cur.execute('INSERT INTO linfo VALUES(%s,%s,%s,%s,%s)',(args["L_id"],args["FirstName"],args["LastName"],str(args["age"]),args["S_id"]))
    mysql.connection.commit()
    cur.close()


import time
import sqlalchemy
start_clock = time.perf_counter()
query = """
select
	l_orderkey,
	sum(l_extendedprice * (1 - l_discount)) as revenue,
	o_orderdate,
	o_shippriority
from
	customer,
	orders,
	lineitem
where
	c_mktsegment = 'FURNITURE'
	and c_custkey = o_custkey
	and l_orderkey = o_orderkey
	and o_orderdate < date '1995-03-29'
	and l_shipdate > date '1995-03-29'
group by
	l_orderkey,
	o_orderdate,
	o_shippriority
order by
	revenue desc,
	o_orderdate
LIMIT 10;
"""
import pandas as pd
import numpy as np

query = sqlalchemy.text(query)
result = pd.read_sql(query, con=CONNSTR)
#SHARED_DB_TIME is multiprocessing.Value
SHARED_DB_TIME.value = time.perf_counter() - start_clock
SHARED_WALL_TIME.value = time.perf_counter() - start_clock


import datetime as dt
import time

def fixed_length(_s, length=2, fixed="0", behind=None):
    _s = str(_s)
    _len = length - len(_s)
    if _len > 0:
        _fixed = fixed * _len
        if behind:
            return _s + _fixed
        return _fixed + _s
    return _s

def timestamp_to_datetime(ts_date):
    if isinstance(ts_date, int):
        return dt.datetime.fromtimestamp(ts_date)
    if isinstance(ts_date, str):
        try:
            ts_date = int(ts_date)
        except:
            ts_date = 0
        return dt.datetime.fromtimestamp(ts_date)

def datetime_to_int(_dt, date_day=None, date_time=None, time_int=None):
    if date_day:
        return _dt.year * 10000 + _dt.month * 100 + _dt.day
    if date_time:
        return _dt.hour * 10000 + _dt.minute * 100 + _dt.second
    if time_int:
        return _dt.hour * 3600 + _dt.minute * 60 + _dt.second

def datetime_format(
        _dt, repair=1, date_day=None, date_month=None,
        date_time=None, date_hour=None, date_minute=None, format_sign="-"
):
    _year = _dt.year
    if repair:
        _month = fixed_length(_dt.month, )
        _day = fixed_length(_dt.day, )
        _hour = fixed_length(_dt.hour)
        _minute = fixed_length(_dt.minute)
        _second = fixed_length(_dt.second)
    else:
        _month, _day, _hour, _minute, _second = _dt.month, _dt.day, _dt.hour, _dt.minute, _dt.second
    if date_day:
        return "{Y}{si}{M}{si}{D}".format(Y=_year, M=_month, D=_day, si=format_sign)
    if date_month:
        return "{Y}{si}{M}".format(Y=_year, M=_month, si=format_sign)
    if date_time:
        return "{Y}{si}{M}{si}{D} {H}:{m}:{s}".format(
            Y=_year, M=_month, D=_day, H=_hour, m=_minute, s=_second, si=format_sign
        )
    if date_hour:
        return "{Y}{si}{M}{si}{D} {H}".format(
            Y=_year, M=_month, D=_day, H=_hour, si=format_sign
        )
    if date_minute:
        return "{Y}{si}{M}{si}{D} {H}:{m}".format(
            Y=_year, M=_month, D=_day, H=_hour, m=_minute, si=format_sign
        )
    return "--"

def datetime_format_code(_dt, repair=1, code='{Y}-{M}-{D}'):
    if not code:
        return _dt
    if isinstance(_dt, str):
        _dt = dt.datetime.strptime(_dt, "%Y-%m-%d")
    _year = _dt.year
    if repair:
        _month = fixed_length(_dt.month, )
        _day = fixed_length(_dt.day, )
    else:
        _month, _day = _dt.month, _dt.day
    wd, mw, nmw = _dt.weekday(), month_week(_dt), month_week(_dt, natural=1)
    yw, nyw = year_week(_dt), year_week(_dt, natural=1)
    return code.format(
        Y=_year, M=_month, D=_day, wd=wd, mw=mw, nmw=nmw, yw=yw, nyw=nyw
    )

def date_num_dict(date, days):
    date = dt.datetime.strptime(str(date), "%Y-%m-%d")
    date_dict = {}
    for _ in range(days):
        _day = _ + 1
        _date = date + dt.timedelta(days=_day)
        date_dict.update({datetime_format_code(_date, code='{Y}-{M}-{D}'): str(_day+1)})
    return date_dict

def date_list(s_date, e_date=None, num=None, format_code=None, direction=0):
    if isinstance(s_date, str):
        s_date = dt.datetime.strptime(s_date.split(' ')[0], "%Y-%m-%d")
    if isinstance(e_date, str):
        e_date = dt.datetime.strptime(e_date.split(' ')[0], "%Y-%m-%d")
    _list, _ = [], s_date
    if num:
        if isinstance(num, list):
            num = num
        if isinstance(num, int):
            num = range(abs(num))
        if direction:
            for _num in num:
                _ = s_date + dt.timedelta(days=_num - 1)
                _dt = datetime_format_code(_, code=format_code)
                _list.append(_dt)
            return _list
        for _num in num:
            _ = s_date - dt.timedelta(days=_num-1)
            _dt = datetime_format_code(_, code=format_code)
            _list.append(_dt)
        return _list
    if e_date:
        while _ <= e_date:
            _dt = datetime_format_code(_, code=format_code)
            _list.append(_dt)
            _ = _ + dt.timedelta(days=1)
        return _list
    return [s_date]

def year_week(_dt, natural=0):
    year_s = dt.datetime(_dt.year, 1, 1)
    year_e = dt.datetime(_dt.year, _dt.month, _dt.day)
    _day = (year_e - year_s).days
    if natural:
        return _day // 7 + 1
    year_s_wd = dt.datetime(_dt.year, 1, 1).weekday()
    return (_day - year_s_wd) // 7 + 1

def month_week(_dt, natural=0):
    month_s = dt.datetime(_dt.year, _dt.month, 1)
    month_e = dt.datetime(_dt.year, _dt.month, _dt.day)
    _day = (month_e - month_s).days
    if natural:
        return _day // 7 + 1
    month_s_wd = dt.datetime(_dt.year, _dt.month, 1).weekday()
    return (_day - month_s_wd) // 7 + 1

def sub_date(s_date, e_date, types='day') -> int:
    if isinstance(s_date, str):
        s_date = dt.datetime.strptime(s_date, '%Y-%m-%d')
    if isinstance(e_date, str):
        e_date = dt.datetime.strptime(e_date, '%Y-%m-%d')
    try:
        sub = e_date - s_date
        return sub.days
    except TypeError or Exception as b:
        return -1

def date_sub_days(sub_days, _s_day=None):
    if isinstance(_s_day, str):
        _s_day = dt.datetime.strptime(_s_day, '%Y-%m-%d')
    if not _s_day:
        _s_day = dt.datetime.now()
    e_day = _s_day - dt.timedelta(days=sub_days)
    return datetime_format_code(e_day)

def date_sub_days_stamp(sub_days, _s_day=None):
    if isinstance(_s_day, str):
        _s_day = int(_s_day)
    _s_day = _s_day - (sub_days * 24 * 60 * 60)
    return _s_day

def get_last_date_in_unit(_date, unit='month'):
    dt_date = dt.datetime.strptime(_date, '%Y-%m-%d')
    _y, _m = dt_date.year, dt_date.month
    if unit == 'year':
        _name = '_Y' + str(_y)[-2:]
        return _name, '{Y}-01-01'.format(Y=_y + 1)
    if unit == 'month':
        _name = '_M' + str(_y)[-2:] + fixed_length(_m)
        if _m == 12:
            return _name, '{Y}-01-01'.format(Y=_y + 1)
        else:
            return _name, '{Y}-{M}-01'.format(Y=_y, M=fixed_length(_m + 1))
    if unit == 'quarter':
        if _m > 9:
            _name = '_{Y}Q4'.format(Y=_y)
            return _name, '{Y}-01-01'.format(Y=_y + 1)
        if _m > 6:
            _name = '_{Y}Q3'.format(Y=_y)
            return _name, '{Y}-10-01'.format(Y=_y)
        if _m > 3:
            _name = '_{Y}Q2'.format(Y=_y)
            return _name, '{Y}-07-01'.format(Y=_y)
        _name = '_{Y}Q1'.format(Y=_y)
        return _name, '{Y}-04-01'.format(Y=_y)

def get_first_date_in_unit(_date, unit='month'):
    dt_date = dt.datetime.strptime(_date, '%Y-%m-%d')
    _y, _m = dt_date.year, dt_date.month
    if unit == 'year':
        _name = '_Y' + str(_y)[-2:]
        return _name, '{Y}-01-01'.format(Y=_y)
    if unit == 'month':
        _name = '_M' + str(_y)[-2:] + fixed_length(_m)
        return _name, '{Y}-{M}-01'.format(Y=_y, M=fixed_length(_m))
    if unit == 'quarter':
        if _m > 9:
            _name = '_{Y}Q4'.format(Y=_y)
            return _name, '{Y}-10-01'.format(Y=_y)
        if _m > 6:
            _name = '_{Y}Q3'.format(Y=_y)
            return _name, '{Y}-07-01'.format(Y=_y)
        if _m > 3:
            _name = '_{Y}Q2'.format(Y=_y)
            return _name, '{Y}-04-01'.format(Y=_y)
        _name = '_{Y}Q1'.format(Y=_y)
        return _name, '{Y}-01-01'.format(Y=_y)

def block_date_list(date, end_date=None, date_unit='quarter', first_day=False):
    date = ensure_date_type_is_date(date)
    if end_date:
        end_date = datetime_format_code(end_date)
    else:
        end_date = datetime_format_code(dt.datetime.now())
    date_block_list = []
    while date <= end_date:
        date_block_name, e_date = get_last_date_in_unit(date, date_unit)
        if first_day:
            date_block_name, s_date = get_first_date_in_unit(date, date_unit)
            date_block_list.append({'date_name': date_block_name, 's_date': s_date, 'e_date': e_date})
        else:
            date_block_list.append({'date_name': date_block_name, 's_date': date, 'e_date': e_date})
        date = e_date
    return date_block_list

def date_to_stamp(date):
    if not isinstance(date, str):
        date = str(date)
    date_time = dt.datetime.strptime(date, '%Y-%m-%d')
    un_time = time.mktime(date_time.timetuple())
    return int(un_time)

def datetime_to_stamp(date):
    if not isinstance(date, str):
        date = str(date)
    date_time = time.strptime(date, '%Y-%m-%d %H:%M:%S')
    un_time = time.mktime(date_time)
    return int(un_time)

def date_find_week_start_day(date):
    if not isinstance(date, str):
        date = str(date)
    date_time = dt.datetime.strptime(date, '%Y-%m-%d')
    y, w, w_d = date_time.isocalendar()
    s_day = date_time - dt.timedelta(days=(w_d - 1))
    return datetime_format_code(s_day), y * 100 + w

def date_find_week_end_day(date):
    if not isinstance(date, str):
        date = str(date)
    date_time = dt.datetime.strptime(date, '%Y-%m-%d')
    y, w, w_d = date_time.isocalendar()
    s_day = date_time + dt.timedelta(days=(7 - w_d + 1))
    return datetime_format_code(s_day), y * 100 + w

def date_find_month_first_day(date):
    if not isinstance(date, str):
        date = str(date)
    dt_date = dt.datetime.strptime(date, '%Y-%m-%d')
    _y, _m = dt_date.year, dt_date.month

    return '{Y}-{M}-01'.format(Y=_y, M=fixed_length(_m)), _y * 100 + _m

def date_find_month_end_day(date):
    if not isinstance(date, str):
        date = str(date)
    dt_date = dt.datetime.strptime(date, '%Y-%m-%d')
    _y, _m = dt_date.year, dt_date.month

    if _m == 12:
        return '{Y}-01-01'.format(Y=_y + 1), _y * 100 + _m
    else:
        return '{Y}-{M}-01'.format(Y=_y, M=fixed_length(_m + 1)), _y * 100 + _m

def ensure_date_type_is_date(_date, format_code=None):
    if isinstance(_date, int):
        time_local = time.strftime("%Y-%m-%d", time.localtime(_date))
        return datetime_format_code(time_local, code=format_code)
    if isinstance(_date, str):
        if '-' not in _date:
            time_local = time.strftime("%Y-%m-%d", time.localtime(int(_date)))
            return datetime_format_code(time_local, code=format_code)
        return _date

def ensure_date_type_is_stamp(_date):
    if isinstance(_date, int):
        return _date
    return date_to_stamp(_date)

class FunctionTimer:
    def __init__(self):
        self.time_start = dt.datetime.now()

    def time_start(self):
        self.time_start = dt.datetime.now()
        print('start time is:', self.time_start)

    def time_spot(self):
        _spot = dt.datetime.now()
        user_time = _spot - self.time_start
        print('has use time:', user_time.seconds, 'seconds')


__author__ = 'jiangzhou xu'

def contain_nearby_duplicate(nums, k):
    num_dict = dict()
    for x in range(len(nums)):
        id = num_dict.get(nums[x])
        if id >= 0 and x - id <= k:
            return True
        num_dict[nums[x]] = x
    return False


from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from lxml import etree
import os
import webbrowser

def scrap_data():
    process = CrawlerProcess(get_project_settings())
    process.crawl('isport')
    process.crawl('portativ')
    process.start()

def task1():
    print("Task #1")
    root = etree.parse("task1.xml")
    pages = root.xpath("//page")
    print("Number of scrapped documents: %s" % len(pages))
    for page in pages:
        url = page.xpath("@url")[0]
        print("%s" % url)

def task2():
    print("Task #2")
    transform = etree.XSLT(etree.parse("templateTask2.xsl"))
    result = transform(etree.parse("task2.xml"))
    result.write("task2.xhtml", pretty_print=True, encoding="UTF-8")
    webbrowser.open('file://' + os.path.realpath("task2.xhtml"))

if __name__ == '__main__':
    scrap_data() # Scrapping data from sites
    while True:
        print("Input number of task to execute, or anything else to exit:")
        print("1. Task#1: https://isport.ua/")
        print("2. Task#2: https://portativ.ua/")
        print("-> ", end='', flush=True)
        number = input()
        if number == "1":
            task1()
        elif number == "2":
            task2()
        else:
            break
    print("End of program")


from ceph_deploy.hosts.centos.install import repo_install, mirror_install  # noqa
from ceph_deploy.hosts.common import map_components
from ceph_deploy.util.system import enable_service, start_service

NON_SPLIT_PACKAGES = [
    'ceph-osd',
    'ceph-radosgw',
    'ceph-mds',
    'ceph-mon',
    'ceph-mgr',
    'ceph-common',
    'ceph-test'
]

SYSTEMD_UNITS = [
    'ceph.target',
    'ceph-radosgw.target',
    'ceph-rbd-mirror.target',
    'ceph-fuse.target',
    'ceph-mds.target',
    'ceph-mon.target',
    'ceph-mgr.target',
    'ceph-osd.target',
]
SYSTEMD_UNITS_SKIP_START = [
    'ceph-mgr.target',
    'ceph-mon.target',
]
SYSTEMD_UNITS_SKIP_ENABLE = [
]

def install(distro, version_kind, version, adjust_repos, **kw):
    packages = map_components(
        NON_SPLIT_PACKAGES,
        kw.pop('components', [])
    )

    distro.packager.install(
        packages
    )

    # Start and enable services
    for unit in SYSTEMD_UNITS:
        if unit not in SYSTEMD_UNITS_SKIP_START:
            start_service(distro.conn, unit)
        if unit not in SYSTEMD_UNITS_SKIP_ENABLE:
            enable_service(distro.conn, unit)


# -*- coding: utf-8 -*-
__author__ = 'yusongmin'

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

"""
读取市场结果
"""
def scenario_market_result_read(system_para, scenario_name, result_name, conn):

    num_simulation = system_para["num_simulation"]
    num_period = system_para["num_period"]

    matrix = np.zeros((3, num_period))
    round_result_matrix = np.zeros((num_simulation, num_period))
    for round_num in xrange(0, num_simulation):

        table_code = "_round_" + str(round_num)
        market_result_df = pd.read_sql('select * from ' + scenario_name + "_market_result" + table_code, con=conn)
        round_result_matrix[round_num] = market_result_df[result_name]

    average_value = round_result_matrix.mean(axis=0)
    max_value = round_result_matrix.max(axis=0)
    min_value = round_result_matrix.min(axis=0)

    matrix[0] = average_value
    matrix[1] = max_value
    matrix[2] = min_value

    return matrix

"""
读取技术采用结果
"""
def scenario_technology_adoption_read(system_para, scenario_name, conn):

    num_simulation = system_para["num_simulation"]
    num_period = system_para["num_period"]
    num_tech = system_para["num_tech"]

    average_matrix = np.zeros((num_period + 1, num_tech))
    max_min_matrix = np.zeros((num_simulation, num_period + 1))
    average_max_min_matrix = np.zeros((3, num_period + 1))

    for round_num in xrange(0, num_simulation):

        table_code = "_round_" + str(round_num)
        tech_adoption_matrix = pd.read_sql('select * from ' + scenario_name + "_tech_result" + table_code, con=conn).values
        max_min_matrix[round_num] = tech_adoption_matrix.sum(axis=1)
        average_matrix += tech_adoption_matrix

    average_matrix = average_matrix/float(num_simulation)

    initial_adoption = max_min_matrix[0][0]
    average_max_min_matrix[0] = average_matrix.sum(axis=1) - initial_adoption
    average_max_min_matrix[1] = max_min_matrix.max(axis=0) - initial_adoption
    average_max_min_matrix[2] = max_min_matrix.min(axis=0) - initial_adoption

    return [average_matrix, average_max_min_matrix]

"""
画图：碳市场或产品市场
"""
def market_result_plot(price_matrix, volume_matrix, mark_list, fig_name):

    price_average = price_matrix[0]
    price_max = price_matrix[1]
    price_min = price_matrix[2]
    volume_average = volume_matrix[1]
    x_label = mark_list[0]
    y1_label = mark_list[1]
    y1_limit = mark_list[2]
    y2_label = mark_list[3]
    y2_limit = mark_list[4]

    figure = plt.figure(figsize=(10, 8), dpi=100, frameon=False)
    figure.subplots_adjust(bottom=0.1, left=0.15, top=0.9, right=0.95)

    left = 0.12
    bottom_down = 0.15
    width = 0.85
    height_down = 0.2
    bottom_up = bottom_down + height_down + 0.05
    height_up = 0.55
    rect_up = [left, bottom_up, width, height_up]
    rect_down = [left, bottom_down, width, height_down]
    ax1 = plt.axes(rect_up)
    ax2 = plt.axes(rect_down)

    line_x = np.array([period + 1 for period in xrange(0, len(price_average))])
    bar_width_1 = 1
    left_1 = line_x - 0.5 * bar_width_1
    ax1.plot(line_x, price_average, color='red', linewidth=2, linestyle='-', label="Price")
    ax1.bar(left=left_1, height=price_max - price_min, width=bar_width_1, bottom=price_min,
            alpha=0.2,
            color='red',
            linewidth=0)

    bar_width_2 = 0.8
    left_2 = line_x - 0.5 * bar_width_2
    ax2.bar(left=left_2, height=volume_average, width=bar_width_2,
            alpha=0.4,
            color='orange',
            linewidth=0)

    # 打开栅格
    ax1.grid(True)
    ax2.grid(True)

    # 增加轴标题
    ax1.set_ylabel(y1_label, fontsize=15, labelpad=10)
    ax2.set_xlabel(x_label, fontsize=15, labelpad=10)
    ax2.set_ylabel(y2_label, fontsize=15, labelpad=10)

    # 设置x轴的上下限
    ax1.set_xlim(0, len(price_average))
    # ax1.set_ylim(y1_limit)
    ax2.set_xlim(0, len(price_average))
    # ax2.set_ylim(y2_limit)

    # 编辑轴刻度
    ax1.tick_params(
        axis='x',  # changes apply to the x-axis
        which='both',  # both major and minor ticks are affected
        bottom='off',  # ticks along the bottom edge are off
        top='off',  # ticks along the top edge are off
        labelbottom='off')  # labels along the bottom edge are off

    for ax in [ax1, ax2]:
        ax.get_yaxis().get_major_formatter().set_useOffset(False)
    for tick in ax2.xaxis.get_major_ticks():
        tick.label1.set_fontsize(14)
    for tick in ax1.yaxis.get_major_ticks():
        tick.label1.set_fontsize(14)
    for tick in ax2.yaxis.get_major_ticks():
        tick.label1.set_fontsize(14)

    # plt.show()
    figure.savefig("/home/yusongmin/Desktop/PlotAll/" + fig_name + ".png", transparent=True)

"""
画图：技术采用的average, max, min
"""
def tech_adoption_plot(process_matrix, mark_list, fig_name):

    average_value = process_matrix[0]
    max_value = process_matrix[1]
    min_value = process_matrix[2]
    y_label = mark_list[0]
    x_label = mark_list[1]
    y_limit = mark_list[2]

    figure = plt.figure(figsize=(10, 6), dpi=100, frameon=False)
    figure.subplots_adjust(bottom=0.15, left=0.12, top=0.9, right=0.95)
    ax = plt.subplot(1, 1, 1)

    line_x = np.array([period for period in xrange(0, len(average_value))])
    bar_width_1 = 1
    left_1 = line_x - 0.5 * bar_width_1
    ax.plot(line_x, average_value, color='red', linewidth=2, linestyle='-')
    ax.bar(left=left_1, height=max_value - min_value, width=bar_width_1, bottom=min_value,
           alpha=0.2,
           color='red',
           linewidth=0)

    # 打开栅格
    ax.grid(True)

    # 增加轴标题
    ax.set_ylabel(y_label, fontsize=15, labelpad=10)
    ax.set_xlabel(x_label, fontsize=15, labelpad=10)

    # 设置x轴的上下限
    ax.set_xlim(0, len(average_value) - 1)
    # ax.set_ylim(y_limit)

    # 编辑轴刻度
    ax.get_yaxis().get_major_formatter().set_useOffset(False)
    for tick in ax.xaxis.get_major_ticks():
        tick.label1.set_fontsize(14)
    for tick in ax.yaxis.get_major_ticks():
        tick.label1.set_fontsize(14)

    # plt.show()
    figure.savefig("/home/yusongmin/Desktop/PlotAll/" + fig_name + ".png", transparent=True)

"""
画图：技术采用的分布
"""
def tech_adoption_distribution_plot(tech_adoption_matrix, mark_list, fig_name):

    y_label = mark_list[0]
    x_label = mark_list[1]
    y_limit = mark_list[2]

    figure = plt.figure(figsize=(10, 6), dpi=100, frameon=False)
    figure.subplots_adjust(bottom=0.15, left=0.12, top=0.9, right=0.95)
    ax = plt.subplot(1, 1, 1)

    list_to_plot = []
    for tech in xrange(0, tech_adoption_matrix.shape[1]):
        if tech_adoption_matrix[-1][tech] >= 1:
            list_to_plot.append([tech_adoption_matrix[0][tech], tech_adoption_matrix[-1][tech]])
        else:
            pass
    matrix_to_plot = np.array(list_to_plot).transpose()

    bar_width = 0.8
    bar_left = [tech + 1 - 0.5 * bar_width for tech in xrange(0, matrix_to_plot.shape[1])]

    barlist_1 = ax.bar(left=bar_left,
                       height=matrix_to_plot[0],
                       width=bar_width,
                       alpha=1,
                       color='orangered',
                       linewidth=0,
                       label="Initial")

    barlist_2 = ax.bar(left=bar_left,
                       height=matrix_to_plot[1] - matrix_to_plot[0],
                       width=bar_width,
                       bottom=matrix_to_plot[0],
                       alpha=1,
                       color='lightsalmon',
                       linewidth=0,
                       label="Incremental")

    # 打开栅格
    ax.grid(True)

    # 增加图例并编辑
    ax.legend(loc='upper right', fontsize=15)

    # 增加轴标题
    ax.set_ylabel(y_label, fontsize=15, labelpad=10)
    ax.set_xlabel(x_label, fontsize=15, labelpad=10)

    # 设置x轴的上下限
    ax.set_xlim(0, matrix_to_plot.shape[1] + 1)
    # ax.set_ylim(y_limit)

    # 编辑轴刻度
    tick_position_1 = [1 + tech for tech in xrange(0, matrix_to_plot.shape[1])]
    x_ticks_label = []
    for tick_num in xrange(0, matrix_to_plot.shape[1]):
        x_ticks_label.append("T" + str(tick_num + 1))
    plt.xticks(tick_position_1, x_ticks_label, rotation=90)

    ax.get_yaxis().get_major_formatter().set_useOffset(False)
    for tick in ax.xaxis.get_major_ticks():
        tick.label1.set_fontsize(14)
    for tick in ax.yaxis.get_major_ticks():
        tick.label1.set_fontsize(14)

    # plt.show()
    figure.savefig("/home/yusongmin/Desktop/PlotAll/" + fig_name + ".png", transparent=True)