from bs4 import BeautifulSoup
import requests,os,time
from sites.helper.help_utils import *

fsecure_labs_base = "https://labs.f-secure.com"

def fsecure_labs():
    print("Scraping F-Secure Labs")
    for page in fsecure_labs_get_all_pages(fsecure_labs_base+"/archive/"):
        pageSoup = BeautifulSoup(requests.get(page,headers=HEADERS).text,"lxml")
        for articlePage in pageSoup.findAll("a", {"class": "col4 box-wrap"}):
            articleLink = articlePage['href']
            articleSoup = BeautifulSoup(requests.get(fsecure_labs_base + articleLink ,headers=HEADERS).text,"lxml")
            file_link = articleSoup.find("a", {"class": "flex-container__box"})
            if file_link:
                file_link = file_link["href"]
                title = articleSoup.find("h1").text
                author_date = articleSoup.find("p",{"class": "blog-post-meta"}).text
                dir_name = title.replace('\n', '').replace("\t", '') + "_" + author_date.replace('_', '').replace('\n', '').replace("\t", '')
                metadata_text=""
                for element in articleSoup.find("div",{"class": "blog-post"}).findAll('p'):
                    metadata_text += '\n' + ''.join(element.findAll(text = True))
                file_download(fsecure_labs_base + file_link , "Whitepapers/FSecureLabs/" + dir_name ,file_link,metadata_text)
    print("Finished scraping F-Secure Labs")

def fsecure_labs_get_all_pages(link):
    pages = [link]
    while link != None:
        print("FSecure-Labs: Current page scraped: " + link)
        parsed = BeautifulSoup(requests.get(link,headers=HEADERS).text, "lxml").find("a",{"class": "btn btn-next"})
        link = parsed["href"] if parsed else None
        if link:
            link = fsecure_labs_base + "/" + link
            pages.append(link)
    return pages


class RequestPreparation(object):

    def __init__(self, DB_NAME, MEASUREMENT):
        super()
        self.DB_NAME = DB_NAME
        self.MEASUREMENT = MEASUREMENT

    def prepare_request(self, filters):
        non_none_filters = self._extract_selected_filters(filters)
        filter_query = self._compose_the_query(non_none_filters)
        params = {
            'db_name': self.DB_NAME,
            'measurement': self.MEASUREMENT,
            'query': filter_query
        }

        return params

    def _extract_selected_filters(self, filters):
        non_none_filters = []
        for filt in filters:
            for v in filt.values():
                if v is not None:
                    non_none_filters.append(filt)

        return non_none_filters

    def _compose_the_query(self, filters):
        # This block composes a query with chosen filter values
        filter_query = str('SELECT slot, value FROM ' + self.MEASUREMENT + ' WHERE ')
        for filt_ind in range(len(filters)):
            for key, val in filters[filt_ind].items():
                # This condition prevents that there is an 'AND' at the end of the query
                if filt_ind == len(filters)-1:
                    filter_query += "".join(["{}='{}'".format(key, val)])
                else:
                    filter_query += "".join(["{}='{}' AND ".format(key, val)])
        return filter_query


class Element:
    # name = None
    _attributes = {}
    _children = []

    # Default transformations
    _translate = (0,0)
    _scale = (1,1)
    _rotate = (0,0,0)

    # Element's transformations
    translate = (0,0)
    scale = (1,1)
    rotate = (0,0,0)

    def __init__(self,name=None,**kwargs):
        self.attributes = {
            **self._attributes
        }
        self.children = [
            *self._children
        ]

        self.name = name
        if 'attributes' in kwargs:
            self.attributes.update(
                kwargs['attributes']
            )
        if 'a' in kwargs:
            self.attributes.update(
                kwargs['a']
            )
        if 'children' in kwargs:
            self.children.extend(
                kwargs['children']
            )
        if 'c' in kwargs:
            self.children.extend(
                kwargs['c']
            )

    def __repr__(self):
        return f"<Element:{self.name} attrs:{len(self.attributes)} chldrn:{len(self.children)}/>"

    def __str__(self):
        # Add transformations if necessary
        if "transform" not in self.attributes:
            transform = []
            if self.translate != self._translate:
                transform.append(f"translate{self.translate}")
            if self.scale != self._scale:
                transform.append(f"scale{self.scale}")
            if self.rotate != self._rotate:
                transform.append(f"rotate{self.rotate}")
            if transform:
                self.attributes['transform'] = " ".join(transform)
        # Create attribute string
        sattr = " ".join(
            f'{k}="{v}"' for k,v in
            self.attributes.items()
        )
        # Render string of children
        schildren = self._schildren()
        # Format and return
        return f"<{self.name} {sattr}>{schildren}</{self.name}>"

    def _schildren(self):
        return "".join(str(c) for c in self.children)

    def __add__(self,other):
        pass

    def __radd__(self,other):
        pass

    def copy(self):
        minime = self.__class__(
            children=[
                (c.copy() if hasattr(c,'copy') else c)
                for c in self.children
                ],
            attributes={**self.attributes}
        )
        return minime

    def render(self):
        return self.__str__()

    def addChild(self,c,*args):
        self.children.append(c)
        self.children.extend(args)
        return self

    def delChild(self,c):
        i = self.children.index(c)
        self.children.pop(i)
        return self


import bpy
from ..utils.Functions import get_global_props

def render_donwload_progress_bar(layout: bpy.types.UILayout):
    tm_props = get_global_props()
    dlTexRunning = tm_props.CB_DL_ProgressRunning is False

    dlTexError          = tm_props.ST_DL_ProgressErrors
    statusText          = "Downloading..." if not dlTexRunning else "Done" if not dlTexError else dlTexError
    showDLProgressbar   = tm_props.CB_DL_ProgressShow

    if showDLProgressbar:
        row=layout.row()
        row.enabled = False
        row.prop(tm_props, "NU_DL_Progress", text=statusText)


import pygame.font
from pygame.sprite import Group

from tank import Tank

class Score():
	"""A class to report character score information."""

	def __init__(self, game_set, screen):
		self.game_set = game_set
		self.screen = screen
		self.screen_rect = screen.get_rect()

		self.text_color = (20, 20, 20)
		self.font = pygame.font.SysFont(None, 32)

		self.prep_players()
		self.prep_lives()

	def prep_players(self):
		"""Prepare player names and their possition on screen"""
		self.p1_image = self.font.render('Player 1', True, self.text_color, self.game_set.name_bg_color)
		self.p2_image = self.font.render('Player 2', True, self.text_color, self.game_set.name_bg_color)

		self.p1_image_rect = self.p1_image.get_rect()
		self.p1_image_rect.left = 30
		self.p1_image_rect.top = 30

		self.p2_image_rect = self.p2_image.get_rect()
		self.p2_image_rect.right = self.screen_rect.right - 30
		self.p2_image_rect.top = 30

	def show_score(self):
		"""Draw players and lives on screen"""
		self.screen.blit(self.p1_image, self.p1_image_rect)
		self.screen.blit(self.p2_image, self.p2_image_rect)
		self.p1_lives.draw(self.screen)
		self.p2_lives.draw(self.screen)

	def prep_lives(self):
		"""Creating two sprite groups for both players to show lives"""
		self.p1_lives = Group()
		self.p2_lives = Group()
		for tank_number in range(self.game_set.p1_lives):
			tank1 = Tank(self.game_set, self.screen, 'lives', 'p1')
			tank1.rect.x = 30 + tank_number * (tank1.rect.width + 5)
			tank1.rect.y = 60
			self.p1_lives.add(tank1)
		for tank_number in range(self.game_set.p2_lives):
			tank2 = Tank(self.game_set, self.screen, 'lives', 'p2')
			tank2.rect.x = 884 + tank_number * (tank2.rect.width + 5)
			tank2.rect.y = 60
			self.p2_lives.add(tank2)


from openupgradelib import openupgrade
from psycopg2.extensions import AsIs

def fill_mass_mailing_list_contact_rel_data(cr):
    openupgrade.logged_query(
        cr, """
        UPDATE mail_mass_mailing_contact_list_rel mmmclr
        SET unsubscription_date = mmmc.%s,
            opt_out = mmmc.%s
        FROM mail_mass_mailing_contact mmmc
        WHERE mmmclr.contact_id = mmmc.id
        """, (
            AsIs(openupgrade.get_legacy_name('unsubscription_date')),
            AsIs(openupgrade.get_legacy_name('opt_out')),
        ),
    )

def fill_mass_mailing_user_id(cr):
    openupgrade.logged_query(
        cr, """
        UPDATE mail_mass_mailing
        SET user_id = create_uid
        WHERE user_id IS NULL""",
    )

def correct_domain_for_list_mailings(env):
    """Overwrite mailing_domain with the one expected by Odoo when selecting
    mailing lists, as the previous domain might be invalid.
    """
    mailings = env['mail.mass_mailing'].search([
        ('mailing_model_name', '=', 'mail.mass_mailing.list')
    ])
    for mailing in mailings:
        mailing.write({
            'mailing_domain': repr(
                [('list_ids', 'in', mailing.contact_list_ids.ids)]
            )
        })

@openupgrade.migrate()
def migrate(env, version):
    cr = env.cr
    fill_mass_mailing_list_contact_rel_data(cr)
    fill_mass_mailing_user_id(cr)
    correct_domain_for_list_mailings(env)


# 1. Wrap your PyTorch model in an objective function.
import torch
from ray import tune, air
from ray.air import session
from ray.tune.search.optuna import OptunaSearch
from get_data import gen_data
from model import ClassificationModel, train
import numpy as np

DEVICE = "cpu"

# 1. Wrap a PyTorch model in an objective function.
def objective(config):
    data = gen_data()
    model = ClassificationModel()

    model.to(DEVICE)

    optimizer = torch.optim.SGD(  # Tune the optimizer
        model.parameters(), lr=config["lr"], momentum=config["momentum"]
        # model.parameters(), lr=config["lr"]
    )

    loss_fn = torch.nn.BCEWithLogitsLoss()

    loss, accuracy = train(model, optimizer, data, loss_fn)  # Train the model
    # acc = test(model, data, loss_fn)  # Compute test accuracy
    session.report({"last_accuracy": accuracy, "last_loss": loss})  # Report to Tune
    # session.report({"last_accuracy": map(int,accuracies["test"]), "last_loss": losses["test"] })  # Report to Tune

    return {"last_accuray":accuracy,"last_loss": loss}

# 2. Define a search space and initialize the search algorithm.
search_space = {"lr": tune.loguniform(1e-2, 1e-1),
                "momentum": tune.uniform(0.1, 0.9)
                }
# search_space = {
#     "lr": tune.sample_from(lambda spec: 10 ** (-10 * np.random.rand())),
#     "momentum": tune.uniform(0.1, 0.9),
# }

# search_space = {
#     "lr": tune.grid_search([0.001, 0.01, 0.1, 1.0]),
#     "momentum": tune.choice([.1, .2, .3, .4]),
# }

tuner = tune.Tuner(
    objective,
    param_space=search_space,
    tune_config=tune.TuneConfig(
        num_samples=5,
        # scheduler=hyperband,
    ),
)
results = tuner.fit()

dfs = {result.log_dir: result.metrics_dataframe for result in results}
print("1      ",results.get_best_result(metric="last_accuracy", mode="max").config)
print("2      ",results.get_best_result(metric="last_loss", mode="min").config)


"""empty message

Revision ID: 99cc0c813389
Revises: 2eed958cbca3
Create Date: 2021-03-02 23:04:46.409486

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '99cc0c813389'
down_revision = '2eed958cbca3'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('police_shooting_per_month',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('state', sa.String(length=255), nullable=True),
    sa.Column('date', sa.DateTime(), nullable=True),
    sa.Column('shooting_count', sa.Float(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('police_shooting_per_month')
    # ### end Alembic commands ###


def dfs(cnt, idx):
    if cnt == L:
        vowel, cons = 0, 0
        for i in range(L):
            if ans[i] in vowels:
                vowel += 1
            else:
                cons += 1
        if vowel >= 1 and cons >= 2:
            print("".join(ans))
        return
    for i in range(idx, C):
        ans.append(alphabet[i])
        dfs(cnt + 1, i + 1)
        ans.pop()

L, C = map(int, input().split())
alphabet = list(input().split())
vowels = ['a', 'e', 'i', 'o', 'u']
alphabet.sort()
ans = []
dfs(0, 0)


import pygame
from random import randint

pygame.init()

# kolory
CZARNY = (0, 0, 0)
BIALY = (255, 255, 255)

class Rakietka(pygame.sprite.Sprite):
    # klasa Rakietka dziedziczy z klasy "Sprite" w Pygame.

    def __init__(self, color, width, height):
        # wołamy najpierw konstruktor klasy bazowej (Sprite)
        # dzięki metodzie super() dziedziczymy wszystkie elementy klasy bazowej
        super().__init__()

        # przekazanie koloru Rakietka oraz szerokości i wysokości, kolor tła i ustawienie go na przezroczyste
        self.image = pygame.Surface([width, height])
        self.image.fill(CZARNY)
        self.image.set_colorkey(CZARNY)

        # rysuję Rakietka jako prostokąt
        pygame.draw.rect(self.image, color, [0, 0, width, height])

        # pobieramy prostokąt (jego rozmiary) z obiektu image
        self.rect = self.image.get_rect()

    def move_left(self, pixels):
        self.rect.x -= pixels
        # sprawdzanie położenia brzegowego
        if self.rect.x < 0:
            self.rect.x = 0

    def move_right(self, pixels):
        self.rect.x += pixels
        # sprawdzanie położenia brzegowego
        if self.rect.x > 600:
            self.rect.x = 600

class Pilka(pygame.sprite.Sprite):
    # klasa Pilka dziedziczy ze "Sprite" w Pygame.

    def __init__(self, color, width, height):
        # wołamy konstruktor klasy bazowej
        super().__init__()

        # przekazujemy rozmiary, kolor, przezroczystość
        self.image = pygame.Surface([width, height])
        self.image.fill(CZARNY)
        self.image.set_colorkey(CZARNY)

        # rysowanie piłki (jako prostokącika)
        pygame.draw.rect(self.image, color, [0, 0, width, height])

        # losowanie prędkości
        self.velocity = [randint(-8, 8), randint(5, 7)]

        # pobieramy prostokąt (jego rozmiary) z obiektu image
        self.rect = self.image.get_rect()

    def update(self):
        self.rect.x += self.velocity[0]
        self.rect.y += self.velocity[1]

    def bounce(self):
        self.velocity[1] = -self.velocity[1]
        self.velocity[0] = randint(-8, 8)

# definiujemy rozmiary i otwieramy nowe okno
size = (700, 500)
screen = pygame.display.set_mode(size)
pygame.display.set_caption("Ping Pong Single Player")

rakietka = Rakietka(BIALY, 100, 10)
rakietka.rect.x = 300
rakietka.rect.y = 480

pileczka = Pilka(BIALY, 10, 10)
pileczka.rect.x = randint(150, 550)
pileczka.rect.y = randint(20, 100)

# lista wszystkich widzalnych obiektów potomnych z klasy Sprite
all_sprites_list = pygame.sprite.Group()

# dodanie rakietki i piłeczki do listy
all_sprites_list.add(rakietka)
all_sprites_list.add(pileczka)

# zaczynamy właściwy blok programu
kontynuuj = True

# służy do kontroli liczby klatek na sekudnę (fps)
clock = pygame.time.Clock()

# Początkowe wyniki graczy
score = 0
best = [0, 0, 0]

# -------- GLÓWNA PĘTLA PROGRAMU -----------
while kontynuuj:

    for event in pygame.event.get():
        if event.type == pygame.QUIT:  # zamknięcie okienka
            kontynuuj = False

    # ruchy Rakietki
    keys = pygame.key.get_pressed()
    if keys[pygame.K_RIGHT]:
        rakietka.move_right(5)
    if keys[pygame.K_LEFT]:
        rakietka.move_left(5)

    # aktualizacja listy duszków
    all_sprites_list.update()

    # sprawdzenie czy piłeczka nie uderza w którąś ścianę
    # i odpowiednie naliczenie punktu
    if pygame.sprite.collide_mask(pileczka, rakietka):
        score += 1
        pileczka.bounce()

    if pileczka.rect.y >= 490:
        screen.fill(CZARNY)

        best.append(score)
        best.sort(reverse=1)
        best = best[:3]
        score = 0

        screen.blit(font.render('GAME OVER', 1, BIALY), (200, 100))
        font = pygame.font.Font(None, 50)
        screen.blit(font.render('Scoreboard:', 1, BIALY), (200, 200))
        scoreboard = 'GAME OVER' + "\n" + 'Scoreboard' + '\n\n'
        for i, scr in enumerate(best, 1):
            text = f'{i})     {scr}'
            screen.blit(font.render(text, 1, BIALY), (200, (i + 4) * 50))
        font = pygame.font.Font(None, 35)
        screen.blit(font.render('press space to play again', 1, BIALY), (340, 430))
        pygame.display.flip()

        wait = True
        while wait:
            event = pygame.event.wait()
            if event.type == pygame.QUIT:
                kontynuuj = False
                wait = False
            if event.type == pygame.KEYDOWN and event.key == pygame.K_SPACE:
                wait = False
            clock.tick(60)

        pileczka.rect.x = randint(150, 550)
        pileczka.rect.y = randint(20, 100)
        rakietka.rect.x = 300
        rakietka.rect.y = 480

    if pileczka.rect.x > 690:
        pileczka.velocity[0] = -pileczka.velocity[0]
    if pileczka.rect.x < 0:
        pileczka.velocity[0] = -pileczka.velocity[0]
    if pileczka.rect.y < 0:
        pileczka.velocity[1] = -pileczka.velocity[1]

    # RYSOWANIE
    # czarny ekran
    screen.fill(CZARNY)

    # narysowanie obiektów
    all_sprites_list.draw(screen)

    # wyświetlanie wyników
    font = pygame.font.Font(None, 74)
    text = font.render(str(score), 1, BIALY)
    screen.blit(text, (332, 10))

    # odświeżenie / przerysowanie całego ekranu
    pygame.display.flip()

    # 60 klatek na sekundę
    clock.tick(60)

# koniec
pygame.quit()


import json

from django.http import HttpRequest, HttpResponse
from django.db.models import Q

from django.contrib import messages
from django.http import JsonResponse
from django.urls import reverse_lazy
from django.views import generic
from django.shortcuts import get_object_or_404, redirect, render
from authentication.forms import CustomUserChangeForm
from core.models import *
from .forms import PropertyForm
#from authentication.models import *
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserChangeForm
from pay.forms import *
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist

@login_required(login_url='login')
def dashboard(request):
    user = User.objects.all
    properties = Property.objects.filter(user=request.user)
    region = Region.objects.all()
    context ={
        'properties':properties,
        'region':region,
        'user':user}
    return render(request, 'landlord/dashboard.html', context)

# @login_required(login_url='login')
# def createProperty(request):
#     payment_form = PaymentForm()
#     form = PropertyForm()
#     if request.method == 'POST':
        # if request.POST.get('form_type') == 'formA':
        #     form = PropertyForm(request.POST, request.FILES)
        #     if form.is_valid():
        #         form.save(commit=False)
        #         form.instance.user = request.user
        #         form.save()
        #         form.save_m2m()
        #         return redirect('dashboard')
#         elif request.POST.get('form_type') == 'formB':
#             payment_form = PaymentForm(request.POST)
#             if payment_form.is_valid():
#                 payment_form.save(commit=False)
#                 payment_form.instance.email = request.user
#                 payment = payment_form.save()
#                 return render (request, 'landlord/createProperty.html', {'payment': payment, 'paystack_public_key': settings.PAYSTACK_PUBLIC_KEY})
#     return render(request, 'landlord/createProperty.html', {'form': form, 'payment_form': payment_form})

@login_required(login_url='login')
def createProperty(request, ref):
    payment = get_object_or_404(Payment, ref=ref)
    # property = get_object_or_404(Property, ref=ref)
    # properties = get_object_or_404(Property, ref=ref)
    # prop = Property.objects.get(ref=ref)

    form = PropertyForm()

    try:
        if request.method == 'POST':
            if request.POST.get('form_type') == 'formA':
                form = PropertyForm(request.POST, request.FILES)
                if form.is_valid():
                    form.save(commit=False)
                    form.instance.user = request.user
                    form.instance.ref = payment.ref
                    form.save()
                    form.save_m2m()
                    return redirect('dashboard')
    except:
           return HttpResponse('Apartmnet ale')
    context = {
        'form':form
    }
    return render(request, 'landlord/createProperty.html', context)

def verify_payment(request: HttpRequest, ref:str) -> HttpResponse:
    payment = get_object_or_404(Payment, ref=ref)
    verified = payment.verifiy_payment()
    if verified:
        messages.success(request, 'Verification Successfull')
    else:
        messages.error(request, 'Verification Failed')
    return redirect('dashboard')

@login_required(login_url='login')
def updateProperty(request, pk):
    property = get_object_or_404(Property, id=pk)
    form = PropertyForm(request.POST or None, request.FILES or None, instance=property)
    if request.method == 'POST':
        if form.is_valid():
            form.save()
            return redirect('dashboard')
    context = {'form': form}
    return render(request, 'landlord/property-update.html', context)

@login_required(login_url='login')
def deleteProperty(request, pk):
    property = Property.objects.get(id=pk)
    if request.user != property.user:
        return redirect('login')
    if request.method == 'POST':
        property.delete()
        return redirect('dashboard')
    context = {'obj': property}
    return render(request, 'landlord/delete-obj.html', context)

@login_required(login_url='login')
def updateUser(request):
    user = request.user
    form = CustomUserChangeForm(instance=user)

    if request.method == 'POST':
        form = CustomUserChangeForm(request.POST, request.FILES, instance=user)
        if form.is_valid():
            form.save()
            return redirect('dashboard')
        else:
            messages.error(request, 'Something went wrong. Try again')
            # return redirect('updateuser')

    return render(request, 'landlord/update-user.html', {'form': form})

def getDistrict(request):
    data = json.loads(request.body)
    region_id = data['id']
    print(region_id)
    districts = District.objects.filter(region__id=region_id)
    return JsonResponse(list(districts.values("id", "name")), safe=False)

        # if request.headers.get('x-requested-with') == 'XMLHttpRequest':
    #     print('Ajax')
    #     amount = json.load(request)['amount']
    #     payment = Payment.objects.create(amount=amount, email=request.user)
    #     payment.save()

    #     data ={'amount':payment.amount}
    #     return JsonResponse(data, status=200)


import math
import os
import time
import chess
import numpy as np
import random

from models.mcts_score_bounded import MCTSScoreBounded
from multiprocessing import Pool

# MCTS with Root Parallelization
class MCTSRootParallelization(MCTSScoreBounded):
    def __init__(self, chess_board=chess.Board(), **kwargs):
        super().__init__(chess_board, **kwargs)
        self.num_processes = kwargs.get('num_processes', round(os.cpu_count() * .75))  # int(os.cpu_count()/1.5))
        self.num_trees = kwargs.get('num_trees', self.num_processes)

    def parallel_search(self):
        """
        Parallelized search with a random seed for each process
        """
        random.seed((os.getpid() * int(time.time())) % 123456789)  # set a random seed so that we get different results
        start_time = time.time()
        i = 0
        for i in range(self.max_sims):
            node = self.tree_policy()
            score = -self.simulation(node)
            self.backpropagation(node, score)
            # check time limit
            if self.max_time is not None and (time.time() - start_time) > self.max_time:
                break
        best_child = np.argmax([child.num_visits for child in self.tree.children])
        best_move = self.tree.children[best_child].action
        num_visits = self.tree.children[best_child].num_visits
        # return move, total visits, total simulations performed, node stats
        return best_move, num_visits, i + 1, [(n.score, n.num_visits, n.action) for n in self.tree.children]

    def run(self, board, print_stats=False):
        """
        Modify main run function to handle parallel execution and multiple tree aggregation
        """
        if self.tree.is_game_over:
            return None

        self._set_root(board)

        start_time = time.time()

        with Pool(self.num_processes) as p:
            ensemble_rewards = p.starmap(self.parallel_search, [() for _ in range(self.num_trees)])

        time_taken = time.time() - start_time
        total_sims = 0
        move_dict = {}
        vote_dict = {}

        # first, tally the votes and visits
        for best_move, num_visits, sim_count, tree_reward_list in ensemble_rewards:
            move_name = best_move.uci()
            if move_name not in vote_dict:
                vote_dict[move_name] = [1, num_visits, best_move]
            else:
                vote_dict[move_name][0] += 1
                vote_dict[move_name][1] += num_visits
            total_sims += sim_count
            for node_reward in tree_reward_list:
                mean, visits, move = node_reward
                move_name = move.uci()
                if move_name not in move_dict:
                    # [[mean, total visits], count, move object]
                    move_dict[move_name] = [np.array([0.0, 0.0]), 0, move]
                move_dict[move_name][0][0] += mean
                move_dict[move_name][0][1] += visits
                move_dict[move_name][1] += 1

        # ----- majority voting -----
        # we combine using the majority voting method.
        # 1. each tree will vote on the best moves discovered independently.
        # 2. the number of votes are tallied, and the move with the most votes wins.
        # 3. if two (or more) moves have the same number of votes, then the move with the most visits wins.
        best = [0, 0, None]
        for move_stats in vote_dict.values():
            if move_stats[0] > best[0]:  # for majority voting
                best = move_stats
            elif move_stats[0] == best[0]:  # if there is a tie, prefer the move with the most visits
               if move_stats[1] > best[1]:
                   best = move_stats
        best_move = best[2]
        total_visits = best[1]

        self.stats['total_time'] += time_taken
        self.stats['total_simulations'] += total_sims
        if print_stats:  # for statistics
            child_node_visits, child_node_scores = [], []  # for displaying stats
            for key in move_dict.keys():
                child_node_scores.append(move_dict[key][0][0])
                child_node_visits.append(move_dict[key][0][1])
            print(f'Time elapsed: {time_taken}, Simulations completed: {total_sims}')
            print('Child node visits:', child_node_visits)
            print('Child node scores:', child_node_scores)
            print(f'Best move: {best_move}, Num visits: {total_visits}')
        return best_move

# testing
if __name__ == '__main__':
    board = chess.Board()

    mcts = MCTSRootParallelization(board)
    move = mcts.run(board, print_stats=True)

    board.push(move)

    print(board)
    print(board.fen)


import tensorflow as tf

def conv2d_trans(inputs, filter_shape, strides, output_shape, padding='SAME', scope='conv_trans'):
  with tf.variable_scope(scope):
    initializer = tf.truncated_normal_initializer(stddev=1e-3)
    filters = tf.get_variable('filters',
                              shape=filter_shape,
                              dtype='float',
                              initializer=initializer)
    conv_t = tf.nn.conv2d_transpose(inputs, filters, output_shape, strides, padding)
    return conv_t


import random

def imprime_mensagem_abertura():
    print("*********************************")
    print("***Bem vindo ao jogo da Forca!***")
    print("*********************************")

def carrega_palavra_secreta():
    arquivo = open("E:\Python\Alura\python-alura\jogos\palavras.txt","r")
    palavras = []

    for linha in arquivo:
        linha = linha.strip()
        palavras.append(linha)

    arquivo.close()

    index_palavra = random.randrange(0, len(palavras))

    palavra_secreta = palavras[index_palavra].upper()

    return palavra_secreta

def inicializa_letras_acertadas(palavra):
    return ["_" for letra in palavra]

def jogar():

    imprime_mensagem_abertura()
    palavra_secreta = carrega_palavra_secreta()
    letras_acertadas = inicializa_letras_acertadas(palavra_secreta)

    enforcou = False
    acertou = False
    erros = 0

    print(letras_acertadas)

    while(not enforcou and not acertou):

        chute = input("Qual é a palavra? {} ".format(letras_acertadas))
        chute = chute.strip(' ').upper()

        if(chute in palavra_secreta):
            index = 0
            for letra in palavra_secreta:
                if(chute == letra):
                    letras_acertadas[index] = letra
                index += 1
        else:
            erros += 1

        enforcou = erros == len(palavra_secreta)
        acertou = "_" not in letras_acertadas
        print(letras_acertadas)

    if acertou:
        print("Você ganhou!! ")
    else:
        print("Você perdeu!!")
    print("Fim do jogo")

if(__name__ == "__main__"):
    jogar()


import torch
import torchvision
import torch.nn as nn
from torch.nn import init

from torch.autograd import Variable
import torch.nn.functional as F
import torchvision.models as models
from collections import namedtuple
import torchvision.utils as vutils
import time

###########################################
##   Layers and Blocks
#------------------------------------------

class InstanceNorm(nn.Module):
    def __init__(self, epsilon=1e-8, style_num=1):
        """
            @notice: avoid in-place ops.
            https://discuss.pytorch.org/t/encounter-the-runtimeerror-one-of-the-variables-needed-for-gradient-computation-has-been-modified-by-an-inplace-operation/836/3
        """
        super(InstanceNorm, self).__init__()
        self.epsilon = epsilon

        self.style_num = style_num

        self.x_max = [None for x in range(self.style_num)]
        self.x_min = [None for x in range(self.style_num)]

        self.saved_mean = [None for x in range(self.style_num)]
        self.saved_std = [None for x in range(self.style_num)]

    def forward(self, x, style_weight=[1.]):
        tmp_saved_mean = 0.
        tmp_saved_std = 0.
        tmp_x_min = 0.
        tmp_x_max = 0.

        for style_id in range(self.style_num):
            tmp_saved_mean += self.saved_mean[style_id] * style_weight[style_id]
            tmp_saved_std += self.saved_std[style_id] * style_weight[style_id]
            tmp_x_min += self.x_min[style_id] * style_weight[style_id]
            tmp_x_max += self.x_max[style_id] * style_weight[style_id]

        x = x - tmp_saved_mean
        x = x * tmp_saved_std

        x = torch.max(tmp_x_min, x)
        x = torch.min(tmp_x_max, x)

        return x

    def compute_norm(self, x, style_id):
        ## mean and std
        self.saved_mean[style_id] = torch.mean(x, (0, 2, 3), True)
        x = x - self.saved_mean[style_id]
        tmp = torch.mul(x, x)
        self.saved_std[style_id] = torch.rsqrt(torch.mean(tmp, (0, 2, 3), True) + self.epsilon)
        x = x * self.saved_std[style_id]

        ## max and min
        tmp_max, _ = torch.max(x, 2, True)
        tmp_max, _ = torch.max(tmp_max, 0, True)
        self.x_max[style_id], _ = torch.max(tmp_max, 3, True)

        tmp_min, _ = torch.min(x, 2, True)
        tmp_min, _ = torch.min(tmp_min, 0, True)
        self.x_min[style_id], _ = torch.min(tmp_min, 3, True)
        return x

    def clean(self):
        self.x_max = [None for x in range(self.style_num)]
        self.x_min = [None for x in range(self.style_num)]

        self.saved_mean = [None for x in range(self.style_num)]
        self.saved_std = [None for x in range(self.style_num)]

class ResidualBlock(nn.Module):
    def __init__(self, input_channel, output_channel, upsample=True, style_num=1):
        super(ResidualBlock, self).__init__()
        self.conv1 = nn.Conv2d(input_channel, output_channel, kernel_size=3, padding=1)
        self.conv2 = nn.Conv2d(output_channel, output_channel, kernel_size=3, padding=1)
        self.conv_shortcut = nn.Conv2d(input_channel, output_channel, kernel_size=1, bias=False)
        self.relu = nn.LeakyReLU(0.2)
        self.norm1 = InstanceNorm(style_num=style_num)
        self.norm2 = InstanceNorm(style_num=style_num)
        self.upsample = upsample
        self.style_num = style_num

    def forward(self, x, style_weight=[1.]):
        if self.upsample:
            x = F.interpolate(x, mode='nearest', scale_factor=2)
        x_s = self.conv_shortcut(x)

        x = self.conv1(x)
        x = self.relu(x)
        x = self.norm1(x, style_weight)
        x = self.conv2(x)
        x = self.relu(x)
        x = self.norm2(x, style_weight)
        return x_s + x

    def compute_norm(self, x, style_id=1):
        if self.upsample:
            x = F.interpolate(x, mode='nearest', scale_factor=2)
        x_s = self.conv_shortcut(x)

        x = self.conv1(x)
        x = self.relu(x)
        x = self.norm1.compute_norm(x, style_id)
        x = self.conv2(x)
        x = self.relu(x)
        x = self.norm2.compute_norm(x, style_id)
        return x_s + x

    def clean(self):
        self.norm1.clean()
        self.norm2.clean()

class FilterPredictor(nn.Module):
    def __init__(self, vgg_channel=512, inner_channel=32, style_num=1):
        super(FilterPredictor, self).__init__()
        self.down_sample = nn.Sequential(
                        nn.Conv2d(vgg_channel, inner_channel, kernel_size=3, padding=1),
                    )
        self.inner_channel = inner_channel
        self.FC = nn.Linear(inner_channel*2,inner_channel*inner_channel)
        self.style_num = style_num
        self.filter = [None for x in range(self.style_num)]

    def forward(self, style_weight=[1.]):
        tmp_filter = 0.
        for style_id in range(self.style_num):
            tmp_filter += self.filter[style_id] * style_weight[style_id]
        return tmp_filter

    def compute(self, content, style, style_id):
        content = self.down_sample(content)
        content = torch.mean(content.view(content.size(0),content.size(1), -1), dim=2)
        content = torch.mean(content, dim=0).unsqueeze(0)

        style = self.down_sample(style)
        style = torch.mean(style.view(style.size(0), style.size(1), -1), dim=2)

        filter = self.FC(torch.cat([content, style],1))
        filter = filter.view(-1, self.inner_channel, self.inner_channel).unsqueeze(3)
        self.filter[style_id] = filter
        return filter

    def clean(self):
        self.filter = [None for x in range(self.style_num)]

class KernelFilter(nn.Module):
    def __init__(self, vgg_channel=512, inner_channel=32, style_num=1):
        super(KernelFilter, self).__init__()
        self.down_sample = nn.Sequential(
                        nn.Conv2d(vgg_channel, inner_channel, kernel_size=3, padding=1),
                    )

        self.upsample = nn.Sequential(
                        nn.Conv2d(inner_channel, vgg_channel, kernel_size=3, padding=1),
                    )

        self.F1 = FilterPredictor(vgg_channel, inner_channel, style_num)
        self.F2 = FilterPredictor(vgg_channel, inner_channel, style_num)

        self.relu = nn.LeakyReLU(0.2)

        self.style_num = style_num

    def apply_filter(self, input_, filter_):
        ''' input_: [B,inC,H,W], filter_: [B,inC,outC,1]
        '''
        B = input_.shape[0]
        input_chunk = torch.chunk(input_, B, dim=0)
        filter_chunt = torch.chunk(filter_, B, dim=0)

        results = []

        for input, filter_ in zip(input_chunk, filter_chunt):
            input = F.conv2d(input, filter_.permute(1,2,0,3), groups=1)
            results.append(input)

        return torch.cat(results,0)

    def forward(self, content, style_weight=[1.]):
        content_ = self.down_sample(content)

        content_ = self.apply_filter(content_, self.F1(style_weight))
        content_ = self.relu(content_)
        content_ = self.apply_filter(content_, self.F2(style_weight))

        return content + self.upsample(content_)

    def clean(self):
        self.F1.clean()
        self.F2.clean()

    def compute(self, content, style, style_id):
        content_ = self.down_sample(content)

        content_ = self.apply_filter(content_, self.F1.compute(content, style, style_id))
        content_ = self.relu(content_)
        content_ = self.apply_filter(content_, self.F2.compute(content, style, style_id))

        return content + self.upsample(content_)

###########################################
##   Networks
#------------------------------------------

class Vgg19(nn.Module):
    def __init__(self, requires_grad=False):
        super(Vgg19, self).__init__()
        vgg_pretrained_features = models.vgg19(pretrained=False).features
        self.slice1 = nn.Sequential()
        self.slice2 = nn.Sequential()
        self.slice3 = nn.Sequential()
        self.slice4 = nn.Sequential()
        for x in range(2):
            self.slice1.add_module(str(x), vgg_pretrained_features[x])
        for x in range(2, 7):
            self.slice2.add_module(str(x), vgg_pretrained_features[x])
        for x in range(7, 12):
            self.slice3.add_module(str(x), vgg_pretrained_features[x])
        for x in range(12, 21):
            self.slice4.add_module(str(x), vgg_pretrained_features[x])
        if not requires_grad:
            for param in self.parameters():
                param.requires_grad = False

    def forward(self, X):
        h = self.slice1(X)
        h_relu1_1 = h
        h = self.slice2(h)
        h_relu2_1 = h
        h = self.slice3(h)
        h_relu3_1 = h
        h = self.slice4(h)
        h_relu4_1 = h
        vgg_outputs = namedtuple("VggOutputs", ['relu1_1', 'relu2_1', 'relu3_1', 'relu4_1'])
        out = vgg_outputs(h_relu1_1, h_relu2_1, h_relu3_1, h_relu4_1)
        return out

class Encoder(nn.Module):
    def __init__(self):
        super(Encoder, self).__init__()
        ## VGG
        vgg_pretrained_features = models.vgg19(pretrained=False).features
        self.slice = nn.Sequential()
        for x in range(21):
            self.slice.add_module(str(x), vgg_pretrained_features[x])

    def forward(self, cur_frame):
        return self.slice(cur_frame)

class EncoderStyle(nn.Module):
    def __init__(self):
        super(EncoderStyle, self).__init__()
        ## VGG
        vgg_pretrained_features = models.vgg19(pretrained=False).features

        self.slice1 = nn.Sequential()
        self.slice2 = nn.Sequential()
        self.slice3 = nn.Sequential()
        self.slice4 = nn.Sequential()

        for x in range(2):
            self.slice1.add_module(str(x), vgg_pretrained_features[x])
        for x in range(2, 7):
            self.slice2.add_module(str(x), vgg_pretrained_features[x])
        for x in range(7, 12):
            self.slice3.add_module(str(x), vgg_pretrained_features[x])
        for x in range(12, 21):
            self.slice4.add_module(str(x), vgg_pretrained_features[x])

    def cal_mean_std(self, feat, eps=1e-5):
        ## eps is a small value added to the variance to avoid divide-by-zero.
        size = feat.size()
        N, C = size[:2]
        feat_var = feat.view(N, C, -1).var(dim=2) + eps
        feat_std = feat_var.sqrt().view(N, C, 1, 1)
        feat_mean = feat.view(N, C, -1).mean(dim=2).view(N, C, 1, 1)

        mean_std = namedtuple("mean_std", ['mean','std'])
        out = mean_std(feat_mean, feat_std)

        return out

    def forward(self, style):
        h = self.slice1(style)
        h_relu1_1 = self.cal_mean_std(h)

        h = self.slice2(h)
        h_relu2_1 = self.cal_mean_std(h)

        h = self.slice3(h)
        h_relu3_1 = self.cal_mean_std(h)

        h = self.slice4(h)
        h_relu4_1 = self.cal_mean_std(h)

        vgg_outputs = namedtuple("VggOutputs", ['map','relu1_1', 'relu2_1', 'relu3_1', 'relu4_1'])
        out = vgg_outputs(h, h_relu1_1, h_relu2_1, h_relu3_1, h_relu4_1)

        return out

class Decoder(nn.Module):
    def __init__(self, style_num=1):
        super(Decoder, self).__init__()

        self.slice4 = ResidualBlock(512, 256, style_num=style_num)
        self.slice3 = ResidualBlock(256, 128, style_num=style_num)
        self.slice2 = ResidualBlock(128, 64, style_num=style_num)
        self.slice1 = nn.Conv2d(64, 3, kernel_size=3, padding=1)

        self.Filter1 = KernelFilter(style_num=style_num)
        self.Filter2 = KernelFilter(style_num=style_num)
        self.Filter3 = KernelFilter(style_num=style_num)

        self.norm = [InstanceNorm(style_num=style_num),
                     InstanceNorm(style_num=style_num),
                     InstanceNorm(style_num=style_num),
                     InstanceNorm(style_num=style_num),
                     InstanceNorm(style_num=style_num)]

        self.style_num = style_num

        self.style_means = [None]
        self.style_stds = [None]

        for layer in range(1,5):
            self.style_means.append([None for i in range(style_num)])
            self.style_stds.append([None for i in range(style_num)])

    #############################
    # For forward Transfer
    # ---------------------------
    def AdaIN(self, content_feat, norm_id, style_weight=[1.]):
        normalized_feat = self.norm[norm_id](content_feat, style_weight)

        style_mean = 0.
        style_std = 0.

        for style_id in range(self.style_num):
            style_mean += self.style_means[norm_id][style_id] * style_weight[style_id]
            style_std += self.style_stds[norm_id][style_id] * style_weight[style_id]

        result =  normalized_feat * style_std + style_mean

        return result

    def AdaIN_filter(self, content_feat, style_weight=[1.]):
        normalized_content = self.norm[0](content_feat, style_weight)
        results = self.Filter1(normalized_content, style_weight)
        results = self.Filter2(results, style_weight)
        results = self.Filter3(results, style_weight)
        return results

    ##########################################
    # For memory-based pre-processing
    # ----------------------------------------

    def AdaIN_compute_norm(self, content_feat, style_feat, norm_id, style_id):
        style_mean = style_feat.mean
        style_std = style_feat.std

        self.style_means[norm_id][style_id] = style_mean
        self.style_stds[norm_id][style_id] = style_std

        normalized_feat = self.norm[norm_id].compute_norm(content_feat, style_id)
        return normalized_feat * style_std + style_mean

    def AdaIN_filter_compute_norm(self, content_feat, style_feat, style_map, style_id):
        style_mean = style_feat.mean
        style_std = style_feat.std

        normalized_content = self.norm[0].compute_norm(content_feat, style_id)
        normalized_style = (style_map - style_mean) / style_std

        results = self.Filter1.compute(normalized_content, normalized_style, style_id)
        results = self.Filter2.compute(results, normalized_style, style_id)
        results = self.Filter3.compute(results, normalized_style, style_id)
        return results

    #############
    # Others
    # -----------

    def clean(self):
        for norm in self.norm:
            norm.clean()

        self.slice4.clean()
        self.slice3.clean()
        self.slice2.clean()

        self.Filter1.clean()
        self.Filter2.clean()
        self.Filter3.clean()

    ##################
    # Processing
    # ----------------

    def compute_norm(self, x, style_features=None):
        for style_id, style_feature in enumerate(style_features):
            h = self.AdaIN_filter_compute_norm(x, style_feature.relu4_1, style_feature.map, style_id)

            h = self.AdaIN_compute_norm(h, style_feature.relu4_1, 1, style_id)
            h = self.slice4.compute_norm(h, style_id)

            h = self.AdaIN_compute_norm(h, style_feature.relu3_1, 2, style_id)
            h = self.slice3.compute_norm(h, style_id)

            h = self.AdaIN_compute_norm(h, style_feature.relu2_1, 3, style_id)
            h = self.slice2.compute_norm(h, style_id)

            h = self.AdaIN_compute_norm(h, style_feature.relu1_1, 4, style_id)

        del h

    def forward(self, x, style_weight=[1.]):
        start = time.time()
        h = self.AdaIN_filter(x, style_weight)

        start = time.time()
        h = self.AdaIN(h, 1, style_weight)

        start = time.time()
        h = self.slice4(h, style_weight)

        start = time.time()
        h = self.AdaIN(h, 2, style_weight)

        start = time.time()
        h = self.slice3(h, style_weight)

        start = time.time()
        h = self.AdaIN(h, 3, style_weight)

        start = time.time()
        h = self.slice2(h, style_weight)

        start = time.time()
        h = self.AdaIN(h, 4, style_weight)

        start = time.time()
        h = self.slice1(h)

        return h

class TransformerNet(nn.Module):
    def __init__(self, style_num=1):
        super(TransformerNet, self).__init__()
        self.Decoder = Decoder(style_num)
        self.Encoder = Encoder()
        self.EncoderStyle = EncoderStyle()
        self.Vgg19 = Vgg19()
        self.have_delete_vgg = False
        self.F_patches = []
        self.F_style = [None for x in range(style_num)]

    def generate_style_features(self, style, style_id):
        self.F_style[style_id] = self.EncoderStyle(style)

    def forward(self, F_content, style_weight=[1.]):
        ## -------------------------------
        ## Stylization

        start = time.time()
        styled_pre_frame = self.Decoder(F_content, style_weight)
        return styled_pre_frame

    def generate_content_features(self, content):
        return self.Encoder(self.RGB2Gray(content))

    def add_patch(self, F_patch):
        self.F_patches.append(F_patch)

    def compute_norm(self):
        self.Decoder.compute_norm(torch.cat(self.F_patches, dim=0), self.F_style)
        self.F_patches = []

    def clean(self):
        self.Decoder.clean()

    def RGB2Gray(self, image):
        mean = image.new_tensor([0.485, 0.456, 0.406]).view(-1, 1, 1)
        std = image.new_tensor([0.229, 0.224, 0.225]).view(-1, 1, 1)

        image = (image*std+mean)

        gray = image[:,2:3,:,:]*0.299 + image[:,1:2,:,:]*0.587 + image[:,0:1,:,:]*0.114
        gray = gray.expand(image.size())

        gray = (gray-mean)/std
        return gray


# -*- coding: utf-8 -*-
#!/usr/bin/python

# Imports
import cv2, os
import numpy as np
from PIL import Image
import sys
from mpi4py import MPI

cascadePath = "haarcascade_frontalface_default.xml"
faceCascade = cv2.CascadeClassifier(cascadePath)

# Celle-ci prend en paramètre une liste de fichiers/chemins
# elle retourne une liste d'images (matrice) et une liste de labels (identité, entier, numéro de l'individu)
def get_images_and_labels(image_paths):
    """
    Image paths est l'ensemble d'apprentissage
    """
    images = []
    # labels ou classe pour le classifieur d'open CV , sera un entier
    labels = []
    for image_path in image_paths:
        # Conversion en nuances de gris
        image_pil = Image.open(image_path).convert('L')
        # Conversion en numpy array
        image = np.array(image_pil, 'uint8')
        # Le label
        nbr = int(os.path.split(image_path)[1].split(".")[0].replace("subject", ""))
        # Detection du visage
        faces = faceCascade.detectMultiScale(image)
        # Si l'image est trouvée ...
        for (x, y, w, h) in faces:
            images.append(image[y: y + h, x: x + w])
            labels.append(nbr)
            # Pour afficher l'ajout d'images
            cv2.imshow("Ajout de têtes au modèle d'apprentissage ....", image[y: y + h, x: x + w])
            cv2.waitKey(50)
    # retourne les listes d'images et labels
    return images, labels

# argv[1:] : liste des noms de fichiers pour l'apprentissage
if __name__ == '__main__':
	# Utilisation de LBPH Face Recognizer
	recognizer = cv2.createLBPHFaceRecognizer()

	fileNames = sys.argv[1:]
	comm = MPI.COMM_WORLD
	size = comm.Get_size()
	nbModels = size
	rank = comm.Get_rank()
	nbFilesPerModel = int(len(fileNames)/size)

	# Chaque processus va avoir nbFilesPerModel dans son modèle, sauf le dernier processus, qui aura ce qui reste
	# Chaque processus aura les fichiers de rank*nbFilesPerModel à (rank+1) * nbFilesPerModel -1

	start_time = MPI.Wtime()
	if(rank != size -1):
		print('process', rank, 'treating',rank*nbFilesPerModel,'-',(rank+1) * nbFilesPerModel)
		images, labels = get_images_and_labels(fileNames[rank*nbFilesPerModel : (rank+1) * nbFilesPerModel])

	else:
		print('process', rank, 'treating',rank*nbFilesPerModel)
		images, labels = get_images_and_labels(fileNames[rank*nbFilesPerModel:])

	cv2.destroyAllWindows()
	recognizer.train(images, np.array(labels))
	recognizer.save("./model" + str(rank+1) + ".xml")

	end_time = MPI.Wtime()
	if(rank == size -1):
		print("--- %s secondes en parallèle---" % (end_time - start_time))
	# Pour éviter d'influer sur le temps de la partie séquentielle, on fait un Barrier
	comm.Barrier()
	""" Partie 'séquentielle' """
	if rank == 0:
		start_timeS = MPI.Wtime()
		for i in range(nbModels-1):
			images, labels = get_images_and_labels(fileNames[i*nbFilesPerModel : (i+1) * nbFilesPerModel])
			cv2.destroyAllWindows()
			recognizer.train(images, np.array(labels))
			recognizer.save("./model" + str(i+1) + ".xml")
		images, labels = get_images_and_labels(fileNames[(nbModels-1)*nbFilesPerModel:])
		cv2.destroyAllWindows()
		recognizer.train(images, np.array(labels))
		recognizer.save("./model" + str(nbModels) + ".xml")
		end_timeS = MPI.Wtime()
		print("--- %s secondes en séquentiel---" % (end_timeS - start_timeS))
		speedUp = (end_timeS - start_timeS)/(end_time - start_time)
		print("SpeedUp : %f" % (speedUp))
		print("Travail : %f" %( nbModels*(end_time - start_time)))
		print("Efficacité : %f" % (speedUp/ nbModels))


__author__ = "John Kirkham <kirkhamj@janelia.hhmi.org>"
__date__ = "$Nov 10, 2015 16:28$"

import itertools

import numpy
import zarr

import dask

from builtins import range as irange

from nanshe.imp.segment import get_empty_neuron,  \
                               merge_neuron_sets, \
                               wavelet_denoising

from nanshe_workflow.data import DataBlocks
from nanshe_workflow.ipy import display, FloatProgress

def extract_f0_halo(data_shape, data_dtype,
                    half_window_size,
                    which_quantile,
                    temporal_smoothing_gaussian_filter_stdev,
                    temporal_smoothing_gaussian_filter_window_size,
                    spatial_smoothing_gaussian_filter_stdev,
                    spatial_smoothing_gaussian_filter_window_size,
                    bias=None,
                    out=None,
                    return_f0=False,
                    **parameters):
    """
        Compute halo for ``extract_f0`` given parameters.

        Notes:
            Shape and dtype refer to the data to be used as input. See
            ``extract_f0`` documentation for other parameters.

        Returns:
            tuple of ints:         Half halo shape to be tacked on to the data.
    """

    halo = numpy.zeros((len(data_shape),), dtype=int)

    halo[0] = int(numpy.ceil(
        temporal_smoothing_gaussian_filter_window_size *
        temporal_smoothing_gaussian_filter_stdev
    ))
    halo[1:] = int(numpy.ceil(
        spatial_smoothing_gaussian_filter_window_size *
        spatial_smoothing_gaussian_filter_stdev
    ))
    halo[0] = max(halo[0], half_window_size)

    halo = tuple(halo.tolist())

    return(halo)

def wavelet_transform_halo(data_shape, data_dtype,
                           scale=5,
                           include_intermediates=False,
                           include_lower_scales=False,
                           out=None):
    """
        Compute halo for ``wavelet_transform`` given parameters.

        Notes:
            Shape and dtype refer to the data to be used as input. See
            ``wavelet_transform`` documentation for other parameters.

        Returns:
            tuple of ints:         Half halo shape to be tacked on to the data.
    """

    try:
        scale_iter = enumerate(scale)
    except TypeError:
        scale_iter = enumerate(itertools.repeat(scale, len(data_shape)))

    half_halo = list(itertools.repeat(0, len(data_shape)))
    for i, each_scale in scale_iter:
        half_halo_i = 0
        for j in irange(1, 1+each_scale):
            half_halo_i += 2**j

        half_halo[i] = half_halo_i

    half_halo = tuple(half_halo)

    return(half_halo)

def normalize_data_halo(data_shape, data_dtype, out=None, **parameters):
    """
        Compute halo for ``normalize_data`` given parameters.

        Notes:
            Shape and dtype refer to the data to be used as input. See
            ``wavelet_transform`` documentation for other parameters.

        Returns:
            tuple of ints:         Half halo shape to be tacked on to the data.
    """

    half_halo = tuple(itertools.repeat(0, len(data_shape)))

    return(half_halo)

def block_postprocess_data_parallel(client):
    """
        Links the client to a returned closure that runs ``postprocess_data``.

        This not an embarrassingly parallel process due to the merge step. So,
        this instead tries to perform the ROI extraction in parallel, which is
        embarrassingly parallel and then performs merging here in serial as
        extracted ROIs arrive.

        Args:
            client(Client):                client to send computations to.

        Returns:
            callable:                 parallelized callable.
    """

    def postprocess_data(new_dictionary, **parameters):
        # Get `concurrent.futures` compatible `Executor`.
        # Tries the `distributed` syntax and falls back for `ipyparallel`.
        try:
            executor = client.get_executor()
        except AttributeError:
            executor = client.executor()

        data_halo_blocks = []
        for i in irange(len(new_dictionary)):
            data_halo_blocks.append([i])
            for j in irange(1, len(new_dictionary.shape)):
                data_halo_blocks[-1].append(slice(None))
            data_halo_blocks[-1] = tuple(data_halo_blocks[-1])

        data_blocks = DataBlocks(new_dictionary, data_halo_blocks)

        def calculate_block(db):
            with dask.config.set(scheduler="single-threaded"):
                return wavelet_denoising(
                    numpy.asarray(db[...]), **parameters["wavelet_denoising"]
                )

        result_blocks = executor.map(calculate_block, data_blocks)

        new_neurons_set = get_empty_neuron(
            shape=new_dictionary.shape[1:], dtype=new_dictionary.dtype
        )
        progress_bar = FloatProgress(min=0.0, max=1.0)
        display(progress_bar)
        for i, each_result_block in enumerate(result_blocks):
            progress_bar.value = i / float(len(data_blocks))
            new_neurons_set = merge_neuron_sets(
                new_neurons_set,
                each_result_block[...],
                **parameters["merge_neuron_sets"]
            )

        progress_bar.value = 1.0

        return(new_neurons_set)

    return(postprocess_data)


import numpy as np
import platform
if platform.system() == "Darwin":
    import matplotlib
    matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

import settings, ray

# settings
EPSILON = 1e-3
LOG_ERRS = []

def readraw(filename):
    f = open(settings.FILENAME, "r")
    return np.fromfile(f, dtype=np.float32)

def readmap(filename):
    a = readraw(filename)
    a = a.reshape(settings.NX, settings.NY)
    return a.T

def get_l():
    dy_sr = (settings.NY * settings.DX)/settings.N_SOURCE
    l = []
    for s_i in range(settings.N_SOURCE):
        for r_j in range(settings.N_RECEIVER):
            y_s_i = settings.DX/2 + s_i * dy_sr
            y_r_i = settings.DX/2 + r_j * dy_sr
            l_i = ray.ray_length(-10,y_s_i,1010,y_r_i)
            l.append(l_i)
    return np.array(l).T

if __name__ == "__main__":
    v_real = readraw(filename=settings.FILENAME)
    s_real = 1.0/v_real
    l = get_l()
    t_obs = np.add(
        np.matmul(l, s_real),
        np.random.normal(loc=0.0, scale=1e-4, size=(settings.N_SOURCE*settings.N_RECEIVER))
    )

    mk = 1e-3*np.zeros(shape=s_real.shape)
    pk0 = np.zeros(shape=mk.shape)
    sk = np.subtract(
        t_obs,
        np.matmul(l, mk)
    )
    rk = np.matmul(l.T, sk)
    err = 0.5*np.linalg.norm(np.subtract(
        t_obs,
        np.matmul(l, mk)
    ))
    while err > EPSILON:
        try:
            betak = np.divide(
                np.matmul(rk.T, rk),
                np.matmul(rk0.T, rk0)
            )
        except NameError:
            betak = 0.0
        pk = np.add(
            rk,
            np.multiply(betak, pk0)
        )
        alphak = np.divide(
            np.linalg.norm(rk)**2,
            np.matmul(
                np.matmul(l, pk).T,
                np.matmul(l, pk)
            )
        )
        mk1 = np.add(
            mk,
            np.multiply(alphak, pk)
        )
        sk1 = np.subtract(
            sk,
            np.multiply(
                alphak,
                np.matmul(l, pk)
            )
        )
        rk1 = np.matmul(l.T, sk1)

        err = 0.5*np.linalg.norm(np.subtract(
            np.matmul(l, mk),
            t_obs,
        ))
        LOG_ERRS.append(err)
        print(err)
        pk0 = pk
        mk0 = mk
        mk = mk1
        sk0 = sk
        sk = sk1
        rk0 = rk
        rk = rk1

    s_model = mk
    # visualize model
    map_model = 1.0/s_model.reshape(settings.NX, settings.NY).T
    plt.imshow(
        map_model, cmap='jet',
        extent=[0, settings.DX*settings.NX, settings.DX*settings.NY, 0],
        vmin=settings.COLOR_VMIN, vmax=settings.COLOR_VMAX,
    )
    a = plt.colorbar()
    a.set_label('$v$')
    plt.title("Model Velocity (CGLS, $\epsilon$~{})".format(EPSILON))
    plt.xlabel("$x$")
    plt.ylabel("$y$")
    plt.savefig("v_cg.png")
    # plt.show()
    plt.clf()
    plt.plot(LOG_ERRS)
    plt.xlabel("k")
    plt.ylabel("$||t_{obs}-Ls_{model}||$")
    plt.yscale("log")
    plt.title("Decay of residue (CGLS)")
    plt.savefig("cg_r.png")


import time
from turtle import Screen
from player import Player
from car_manager import CarManager
from scoreboard import Scoreboard
speed=0.1
screen = Screen()
screen.tracer(0)
carmanager=CarManager()
screen.setup(width=600, height=600)

player=Player()
screen.listen()
screen.onkey(player.move, "Up")
game_is_on = True

score=Scoreboard()
while game_is_on:

    time.sleep(speed)
    screen.update()
    carmanager.createcar()
    carmanager.move()
    for carr in carmanager.listcars:
        if carr.distance(player) < 20:
            print("Collision occured.")
            game_is_on=False
            score.gameover()

    if player.ycor()>280:
        carmanager.levelup()
        score.clear()
        score.update()
        player.back_home()

screen.exitonclick()


import sys
import math
import random

filename = sys.argv[1]
#percent of keeping points 0-100
keepingPercent = int(sys.argv[2])
outfile = sys.argv[3]

f = open(outfile,"w")

#print x, y, r
with open(filename) as pc:
    for line in pc:
        if random.random() * 100 < keepingPercent :
            f.write(line)
f.close()


import numpy as np
import pandas as pd
import scipy.stats as stats
from tqdm import tqdm
import matplotlib.pyplot as plt
import statsmodels.api as sm

# Load the 2019-2020 Bundesliga data
data = pd.read_csv("/Users/EthanLee/Desktop/STAT 143/bundes1819.csv")

# Home and away goals
homegoals = data.FTHG.values
awaygoals = data.FTAG.values

# Get unique teams, number of teams, and number of games
teams = np.unique(data[['HomeTeam']])
nteams = teams.size
ngames = data.HomeTeam.size

# Create design matrix
game_id = np.arange(ngames)
Xh = np.zeros((ngames, nteams))
home_team_idx = np.searchsorted(teams,data.HomeTeam)
away_team_idx = np.searchsorted(teams,data.AwayTeam)
for i in range(ngames):
    Xh[game_id[i],home_team_idx[i]] = 1
    Xh[game_id[i],away_team_idx[i]] = -1
X = np.vstack((Xh,-Xh))

# Create W matrix
W = np.eye(nteams)
W[-1,:] = -1
W = W[:,:-1]

# Get the design matrix fulfilling constraint
Xs = X@W
Xs = np.c_[Xs,np.concatenate((np.ones(ngames),-1*np.ones(ngames)))]

k = np.concatenate((homegoals,awaygoals))
Xs = sm.add_constant(Xs)

# Set up and fit poisson model
poisson_model = sm.Poisson(k, Xs)
poisson_res = poisson_model.fit()
print(poisson_res.summary())

# Get parameters
intercept = poisson_res.params[0]
beta = poisson_res.params[-1]
theta_att = poisson_res.params[1:nteams]

# Get thetas for the final team (found using linear constraint)
theta_att = np.append(theta_att,-np.sum(theta_att))
# Use thetas to get expected goals per game
lambda_att = np.exp(intercept + theta_att)

# Print out results
team_params = []
for t,a,l in zip(teams,lambda_att,theta_att):
    team_params.append((t,a.round(3),l.round(3)))

team_params = sorted(team_params,key=lambda x: x[1], reverse=False)
for params in team_params[::-1]:
    print("%s,%1.3f,%1.3f" % tuple(params))
print("Beta,%1.3f" % (beta))

# Define dict containing team strength parameters
team_param_dict = {}
for team in team_params:
    team_param_dict[team[0]] = team[2]

def draw_from_strength_params(teams,team_param_dict,sigma_teams=0.1,sigma_beta=0.3):
    '''
    Draw a new set of team strength, home advantage and intercept parameters from the standard errors on the theta values
    '''
    sim_param_dict = team_param_dict.copy()
    for team in teams:
        sim_param_dict[team] = stats.norm.rvs(0,sigma_teams,1)[0] + team_param_dict[team]

    sim_param_dict['intercept'] = stats.norm.rvs(0,sigma_beta,1)[0] + intercept
    sim_param_dict['beta'] = stats.norm.rvs(0,sigma_beta,1)[0] + beta
    return sim_param_dict

def sim_match(home,away,team_param_dict):
    # Calculate lambda values for Poission distribution using team strength and home advantage
    lambda_home = np.exp( team_param_dict[home] - team_param_dict[away] + intercept + beta )
    lambda_away = np.exp( team_param_dict[away] - team_param_dict[home] + intercept - beta )
    # Draw a poisson random variable
    hg = stats.poisson.rvs(lambda_home)
    ag = stats.poisson.rvs(lambda_away)
    return hg,ag

def sim_season(fixtures,team_param_dict):
    '''
    simulate a single season using the fixtures given in the 'fixtures' dataframe
    '''
    # Make copy of fixutre list that starts with zeros
    completed_fixtures = fixtures.copy()
    nmatches = fixtures.shape[0]
    HomeGoals = np.zeros(nmatches,dtype=int)
    AwayGoals = np.zeros(nmatches,dtype=int)
    # Simulate each match in fixture schedule
    for i,row in completed_fixtures.iterrows():
        HomeGoals[i],AwayGoals[i] =sim_match(row['HomeTeam'],row['AwayTeam'],team_param_dict)
    # Fill in scores in fixture schedule
    completed_fixtures['HomeGoals'] = HomeGoals
    completed_fixtures['AwayGoals'] = AwayGoals
    return completed_fixtures

def generate_table(results):
    '''
    produce a league table from a simulation of a single team
    '''
    teams = np.unique( results[['HomeTeam','AwayTeam']] )
    table = []
    # Loop through each team
    for team in teams:
        # Get all the team's home & away games
        homegames = results[results['HomeTeam']==team]
        awaygames = results[results['AwayTeam']==team]
        # Calculate total number of goals scored
        GoalsFor = np.sum(homegames.HomeGoals) + np.sum(awaygames.AwayGoals)
        # Calculate total number of goals conceded
        GoalsAgainst = np.sum(awaygames.HomeGoals) + np.sum(homegames.AwayGoals)
        # Calculate total number of points won in home games
        HomePoints = 3*np.sum(homegames.HomeGoals>homegames.AwayGoals) + 1*np.sum(homegames.AwayGoals==homegames.HomeGoals)
        # Calculate total number of points won in away games
        AwayPoints = 3*np.sum(awaygames.AwayGoals>awaygames.HomeGoals) + 1*np.sum(awaygames.AwayGoals==awaygames.HomeGoals)
        # Add to get total points
        TotalPoints = HomePoints + AwayPoints
        # Append results to the table list
        table.append((team,TotalPoints,GoalsFor-GoalsAgainst,GoalsFor,GoalsAgainst))
    # Sort the table by points, then goal difference, then goals for
    table = sorted(table, key = lambda x: (x[1],x[2],x[3]), reverse=True)
    return table

def analyse_sim_results(tables,teams,printout=True):
    '''
    analyse the simulation results over all simulations
    '''
    team_summary = []
    nsims = len(tables)
    for team in teams:
        # Final positions of the team in each simulation
        positions = np.array( [[t[0] for t in table].index(team)+1 for table in tables] )
        # Final number of points of the team in each simulation
        points = np.array([table[position-1][1] for table,position in zip(tables,positions)])

        # Generate bar plot of percent chance of team finishing in each position
        positionPercentages = []
        for i in range(1, 19):
            positionPercentages.append(np.sum(positions==i)/nsims*100)
        xVals = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]
        fig = plt.figure()
        ax = fig.add_subplot(111)
        ax.bar(xVals, positionPercentages, 1)
        plt.xlabel("League Position")
        plt.ylabel("Percent Chance")
        plt.title(team + " League Position Chances")
        plt.xticks(np.arange(1, 19, 1))
        plt.show()

        # Proportion of sims in which team finished first in the league table
        title = np.sum(positions==1)/nsims*100
        # Proportion of sims in which team finished in the top 4
        top4 = np.sum(positions<=4)/nsims*100
        # Proportion of sims in which team finished in the bottom 3
        rel = np.sum(positions>=17)/nsims*100
        # Average position
        position_mean = np.mean(positions)
        # 2.5th percentile of finishes (by position)
        position_low = np.percentile(positions,2.5)
        # 97.5th percentile of finishes
        position_high = np.percentile(positions,97.5)
        # Average points
        points_mean = np.mean(points)
        # 2.5th and 97.5th percentiles of points
        points_low = np.percentile(points,2.5)
        points_high = np.percentile(points,97.5)
        # Add to summary table
        team_summary.append( [position_mean,team,points_mean,points_low,points_high,position_low,position_high,title,top4,rel] )
    # Sort summary table by mean position
    team_summary = sorted( team_summary, key = lambda x: x[0], reverse=False)
    if printout:
        print("\n******* SUMMARY OF SIMULATION RESULTS **********\n")
        print("Position, Team,Points-Mean,Points-Low,Points-High,Position-Low,Position-High,Title,Top4,Relegation")
        for i,summary in enumerate(team_summary):
            print("%d,%s,%1.1f,%1.1f,%1.1f,%1.1f,%1.1f,%1.1f,%1.1f,%1.1f" % tuple([i+1]+summary[1:]))
    return team_summary

def run_sims(team_param_dict,nsims=10000,draw_strengths=True):
    # Get fixture schedule
    fixtures = pd.read_csv('/Users/EthanLee/Desktop/STAT 143/bundes1920.csv')
    fixtures = fixtures[['HomeTeam','AwayTeam']]
    teams = np.unique( fixtures[['HomeTeam','AwayTeam']] )

    # Calculate the average of team strength of teams that were relegated in 2018/19
    relegated_teams_theta = (team_param_dict['Nurnberg']+team_param_dict['Stuttgart'] + team_param_dict['Hannover'])/3.
    # Assign this average strength to the newly promoted teams
    promoted_teams = ['FC Koln','Paderborn', 'Union Berlin']
    for promoted_team in promoted_teams:
        team_param_dict[promoted_team] = relegated_teams_theta

    # Store the league tables for each simulation in a list
    tables = []
    # Run the simulations
    for i in tqdm(np.arange(nsims)):
        if draw_strengths:
            # Draw team strength from distribution determined by team's expected value of team strength and sample error
            sim_param_dict = draw_from_strength_params(teams,team_param_dict)
        else:
            # Use fixed expected values of team strengths
            sim_param_dict = team_param_dict
        # Simulate the season once
        completed_fixtures = sim_season(fixtures,sim_param_dict)
        # Append the league table for simulated season to the list
        tables.append( generate_table(completed_fixtures) )

    team_summary = analyse_sim_results(tables, teams)
    return tables,team_summary

run_sims(team_param_dict)


# -*- coding: utf-8 -*-
# @Project:AID1810
# @Author:biabu
# @Date:2019/4/9 15:04
# @File_name:02_double_link_list.py
# @IDE:PyCharm

"""
双链表代码实现
"""

# (elem, None)
class Node(object):
    """节点"""

    def __init__(self, elem):
        self.elem = elem
        self.next = None
        self.prev = None

class DoubleLinkList(object):
    """
    ----------链表----------
    is_empty() 链表是否为空
    length() 链表长度
    travel() 遍历整个链表
    add(item) 链表头部添加元素
    append(item) 链表尾部添加元素
    insert(pos, item) 指定位置添加元素
    remove(item) 删除节点
    search(item) 查找节点是否存在
    """

    def __init__(self, node=None):
        self.__head = node

    def is_empty(self):
        """判断链表是否为空"""
        return self.__head is None

    def length(self):
        """链表长度"""
        # cur游标，用来移动遍历节点
        cur = self.__head
        # count记录数量
        count = 0
        # 游标移动
        while cur != None:
            count += 1
            cur = cur.next
        return count

    def travel(self):
        """遍历链表"""
        cur = self.__head
        while cur != None:
            print(cur.elem, end=' ')
            cur = cur.next
        print('')

    def add(self,item):
        """链表头部添加元素"""
        node = Node(item)
        node.next = self.__head
        self.__head = node
        node.next.prev = node

    def append(self, item):
        """链表尾部添加元素"""
        node = Node(item)
        if self.is_empty():
            self.__head = node
        else:
            cur = self.__head
            while cur.next != None:
                cur = cur.next
            cur.next = node
            node.prev = cur

    def insert(self, pos, item):
        """
        指定位置添加元素
        :param pos ：从0开始
        """
        if pos <= 0:
            self.add(item)
        elif pos > (self.length() - 1):
            self.append(item)
        else:
            node = Node(item)
            cur = self.__head
            count = 0
            while count < pos:
                count += 1
                cur = cur.next
            # 当循环退出后，cur指向pos的位置
            node.next = cur
            node.prev = cur.prev
            cur.prev.next = node
            cur.prev = node

    def remove(self, item):
        """删除节点"""
        cur = self.__head
        while cur != None:
            cur = cur.next
            if cur.elem == item:
                # 判断是否为头结点
                if cur == self.__head:
                    self.__head = cur.next
                    if cur.prev:
                        # 判断链表是否只有一个节点
                        cur.next.prev = None
                else:
                    cur.prev.next = cur.next
                    if cur.next:
                        cur.next.prev = cur.prev
                    cur.prev = None
                    cur.next = None
                break

    def search(self, item):
        """查找节点是否存在"""
        cur = self.__head
        while cur != None:
            if cur.elem == item:
                return cur.elem
            else:
                cur = cur.next
        return False

if __name__ == '__main__':
    double_obj = DoubleLinkList()
    print(double_obj.is_empty())
    print(double_obj.length())

    double_obj.append(1)
    print(double_obj.is_empty())
    print(double_obj.length())

    double_obj.append(2)
    double_obj.travel()

    double_obj.add(8)
    double_obj.travel()

    double_obj.append(3)
    double_obj.append(4)
    double_obj.append(5)
    double_obj.travel()

    double_obj.insert(-1, 9)
    double_obj.travel()

    double_obj.insert(2, 100)
    double_obj.travel()

    double_obj.insert(10, 200)
    double_obj.travel()

    print(double_obj.search(200))

    double_obj.remove(200)
    double_obj.travel()


import pandas as pd
import numpy as np
from urllib.request import urlopen #url의 html 을 가져 오기 위한 패키지
from bs4 import BeautifulSoup  #크롤링 필수 패키지 설치하려면 cmd창에서 pip install bs4
import os
import re
from selenium import webdriver
from bs4 import BeautifulSoup #크롤링 도구
from selenium.webdriver.common.keys import Keys
import time
from tqdm import tqdm
import os
import re
import time
import json
import datetime
from konlpy.tag import Okt
from gensim.models import Word2Vec
from gensim.models import KeyedVectors
import matplotlib.pyplot as plt
import matplotlib.pyplot as pp
import sklearn.metrics as metrics
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
import pylab as pl
import statsmodels.api as sm
from sklearn.cluster import DBSCAN
import sklearn as skl
import sklearn.model_selection
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report,confusion_matrix
from sklearn.neighbors import KNeighborsClassifier
import random
from sklearn.datasets import make_classification
from sklearn.decomposition import PCA
from imblearn.over_sampling import SMOTE
okt = Okt()
#driver = webdriver.Chrome('C:/r-selenium/chromedriver.exe')  # 크롬 드라이버 연결

data_url = pd.read_excel('C:/Users/USER/Desktop/1~8newsurl/covid_url38.xlsx')

def sampling_func(data777, sample_pct):
    np.random.seed(123)
    N = len(data777)
    sample_n = int(len(data777)*sample_pct) # integer
    sample = data777.take(np.random.permutation(N)[:sample_n])
    return sample
data_url = data_url.groupby('RGSDE',group_keys=False).apply(sampling_func,sample_pct=0.2) #일자별 샘플링

data_url.groupby(data_url['RGSDE']).size()

urllist = data_url['DTA_URL']
################

data = pd.DataFrame(columns=['date', 'title', 'text', 'category','url'])
j = 0
for i in tqdm(urllist):
    try:
        webpage = urlopen(i)
        soup = BeautifulSoup(webpage, 'html.parser')

        aa = soup.select('.article_info  h3#articleTitle')
        test_name = aa[0].text  # 제목 가져오기

        aa = soup.select('.article_info .t11')
        aa[0].text[0:10]
        test_date = datetime.datetime.strptime(aa[0].text[0:10], '%Y.%m.%d')  # 날짜 가져오기

        # 본문가져오고 정제하기
        aa = soup.select('#articleBodyContents')
        te = aa[0].text.replace(soup.select('#articleBodyContents script')[0].text, "")
        trash = [i.text for i in soup.select('#articleBodyContents a')]
        for k in trash:
            te = te.replace(k, "")
        text = te.replace('\n', "").replace("  ", "").replace('\t', "").replace('무단 전재 및 재배포 금지', "")

        cate = '경제' if len(re.findall('sid1=101',i)) == 1 else '사회'
        tlist = [test_date, test_name, text, cate,i]
        data.loc[j, :] = tlist
        j = j + 1
    except:
        tlist = [0,0,0,0,0]
        data.loc[j, :] = tlist
        j = j + 1
del i; del j; del te; del soup; del test_date; del test_name; del text; del tlist; del trash; del k; del cate;
#본문 전처리
data = data.drop_duplicates() #중복제거
data = data[data['text'] != 0] #0인 값 제거

#형태소 분석
list_text = data['text']

words = [okt.nouns(i) for i in tqdm(list(list_text))] #시간 걸림

Prohibit_words = ['기자','연합뉴스','뉴시스','시사저널','신문','뉴스','사진','헤럴드경제','노컷뉴스','파이낸셜뉴스','특파원',
                  '라며','대해','지난','위해','오전','오후','무단','배포','이데일리','머니투데이','앵커',]

j = 0
for i in tqdm(words):
    for k in Prohibit_words:
        while k in i:
            i.remove(k)
    words[j] = i
    j += 1 #불용어 제외

for k in range(len(words)):
    words[k] = [i for i in words[k] if len(i) > 1]  # 한글자 제외

data['words'] = words

data['length_word'] = [len(i) for i in tqdm(data['words'])]
data.to_excel('C:/Users/USER/Desktop/1~8newsurl/covid_url38_word.xlsx',header=True)

data50 = data[data['length_word'] >= 50]  #단어 갯수 50개 이하 제거
data50 = pd.read_excel('C:/Users/USER/Desktop/1~8newsurl/covid_url38_word50.xlsx', index_col=0)

embedding_model = Word2Vec(list(data50['words']), size=100, window = 6, min_count=150, workers=4, iter=100)
embedding_model.wv.save_word2vec_format('C:/Users/USER/Desktop/백업/covid_model38')# 모델 저장
embedding_model = KeyedVectors.load_word2vec_format('C:/Users/USER/Desktop/백업/covid_model38') # 모델 로드
del aa; del i; del j; del k;

#단어 빈도 검색

index1 = [i for i in range(len(data))]
data.index = index1
def top30(keyword):
    x=[]
    for i in range(len(words)):
        x.append([words[i],data['category'][i]])
    x1 = [i[0] for i in x]
    x2 = []
    for i in x1:
        for k in i:
            x2.append(k)
    print(pd.Series(x2).value_counts().head(100))
    x3 = pd.Series(x2).value_counts().head(3000)
    x4 = pd.DataFrame(x3)
    x4['ca'] = keyword
    x4['wor'] = x4.index
    x4['score'] = x4[0] / x4[0].sum(axis=0)
    return x4
bin = top30('경제')
bin.to_excel('C:/Users/USER/Desktop/1~8newsurl/bindo.xlsx',header=True)

print(embedding_model.most_similar(positive=["스타벅스"], topn=100)) #관련단어 검색
print(embedding_model.similarity('일자리','시장'))

# 형태소 분석 수정
'''okt.nouns('소상공인')
okt.nouns('확진자')
okt.nouns('유연근무제, 유연근무,')#'''

#점수내기
words = list(data50['words'])

word_list = ['고용','일자리','채용','청년','취업','배달','유통','근로자','재택근무','노동자','뉴딜','제조업','기본소득','노동','간호사','최저임금','실업','택배','제조','택시',
             '생계','스타트업','근로','구직','파업','고용노동부','해고','폐지','인턴','제조업체','노동조합','실업률','비정규직','실업자','생활비','소매','출퇴근']

def word2vec_kkh2(word_list):
    xx_n = word_list
    avg_dist = []
    dist = []
    dist_dist = []
    for i in tqdm(range(len(words))):
        for k in words[i]:
            try:
                for m in xx_n:
                    # 비교하여 similarity 구하기
                    dist_dist.append(embedding_model.similarity(m, k))
            except:
                dist_dist.append(0)
            dist2 = np.array(dist_dist)
            dist.append(sum(dist2))
            dist_dist = []
        if len(dist) > 1000:
            dist = sorted(dist, reverse=True)[:1000]
        avg_dist.append(np.mean(dist))
        dist = []
    return avg_dist

data50['score'] = word2vec_kkh2(word_list)
data50.to_excel('C:/Users/USER/Desktop/1~8newsurl/covid_url38_word50.xlsx',header=True)

#점수 상위 2천개 일자리 관련 기사라고 라벨링 2천등 점수 1.51505
#data50 = data50.sort_values('score', ascending=False)
#data50['rank'] = [i+1 for i in range(len(data50))]
#data50['y'] = [1 if i <= 2000 else 0 for i in data50['rank']]

#기사 벡터화 하기
match_index = list(embedding_model.wv.vocab.keys()) #벡터화된 단어들
embedding_model.get_vector('연합') #벡터값 찾기
word_vector = list(embedding_model.wv.vectors) #백터화된 단어들의 벡터값
vecdata = pd.DataFrame(columns=['x{}'.format(i) for i in range(100)])
for i in tqdm(range(len(words))):
    vec = pd.DataFrame(columns = ['x{}'.format(i) for i in range(100)])
    j = 0
    for w in match_index:
        if w in words[i]:
            vec.loc[j] = list(word_vector[match_index.index(w)])
            j += 1
        score_vec = vec.mean()
    vecdata.loc[i] = score_vec
vecdata.to_excel('C:/Users/USER/Desktop/1~8newsurl/covid_url38_word_vecdata.xlsx',header=True)
###########################랭크 다시 하고 다시 정렬

data50.index = [i for i in range(len(data50))]
data50_all = pd.concat([data50,vecdata],axis=1)
data50_all['rank'] = data50_all['score'].rank(ascending=False)
data50_all['y'] = [1 if i <= 2000 else 0 for i in data50_all['rank']]
#data50_all.to_excel('C:/Users/USER/Desktop/1~8newsurl/covid_url38_word50_all.xlsx',header=True)
data50_all = pd.read_excel('C:/Users/USER/Desktop/1~8newsurl/covid_url38_word50_all.xlsx', index_col=0)
#data50_all[data50_all['y']== 1].to_excel('C:/Users/USER/Desktop/1~8newsurl/covid_url38_word50_all_job.xlsx',header=True)
###knn써보기
x = data50_all.iloc[:,8:108]
y = data50_all.iloc[:,109]
x_train, x_test, y_train, y_test = skl.model_selection.train_test_split(x, y,random_state=100,stratify=y) #stratify = y y를 동일한 비율로 나누어 준다.
#정규화시키는 함수
scaler = skl.preprocessing.StandardScaler()

#x_train 정규화
scaler.fit(x_train) #x_train 데이터에 대해서 평균과 표준편차 계산
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
###########knn사용해서 분류해보기

classifier = KNeighborsClassifier(n_neighbors= 2)
classifier.fit(x_train,y_train)
knn_pre = classifier.predict(x_test)

#성능 평가
print(confusion_matrix(y_test,knn_pre))

#평가결과 확인
print(classification_report(y_test,knn_pre)) #'''
print(classifier.score(x_test, y_test))

##smote 사용해보기
sm = SMOTE()
x_smote, y_smote = sm.fit_sample(x_train,y_train)
scaler = skl.preprocessing.StandardScaler()

#x_train 정규화
scaler.fit(x_smote) #x_train 데이터에 대해서 평균과 표준편차 계산
x_smote = scaler.transform(x_smote)
x_test = scaler.transform(x_test)

classifier = KNeighborsClassifier(n_neighbors= 2)
classifier.fit(x_smote,y_smote)
knn_pre = classifier.predict(x_test)

#성능 평가
print(confusion_matrix(y_test,knn_pre))

#평가결과 확인
print(classification_report(y_test,knn_pre))
print(classifier.score(x_test, y_test)) #'''

y_test2 = list(y_test)
ckeck = pd.DataFrame(y_test)
ckeck['a'] = list(ckeck.index); ckeck['prey'] = list(knn_pre)
t_l = [data50_all['title'][i] for i in ckeck['a']]
ckeck['title'] = t_l
ckeck2 = ckeck.loc[(ckeck['prey'] == 0) & (ckeck['y'] == 1),:]

data = data[0:10]

1~8뉴스데이터셋 만들기.py


texto = 'escrevendo a primeira frase'
with open('teste.txt', 'w') as f:
    f.write(texto)

s = ['linha 1', 'linha 2', 'linha 3', 'linha 4']
with open('teste.txt', 'a') as f:
    f.writelines(s)

texto2 = texto + '\n'
s2 = [f'{linha}\n' for linha in s]  # list comprehension
with open('teste2.txt', 'w') as f:
    f.write(texto2)
    f.writelines(s2)

#  texto = 'escrevendo a primeira frase'
s = ['linha 1', 'linha 2', 'linha 3', 'linha 4']
with open('print.txt', 'w') as f:
    print(texto, file=f)
    for linha in s:
        print(linha, file=f)


from unittest import TestCase
import pytest

from hubblestack.audit import service
from hubblestack.exceptions import HubbleCheckValidationError

class TestService(TestCase):
    """
    Unit tests for service module
    """
    def test_invalid_params1(self):
        """
        No mandatory param is passed
        should fail
        """
        block_dict={}
        check_id = "test-1"

        with pytest.raises(HubbleCheckValidationError) as exception:
            service.validate_params(check_id, block_dict, {})
            pytest.fail("Check should not have passed")

    def test_valid_params1(self):
        """
        valid param, should pass
        """
        block_dict={"args": {"name": "test"}}
        check_id = "test-1"

        service.validate_params(check_id, block_dict, {})

    def test_filtered_logs1(self):
        """
        valid param, should pass
        """
        block_dict={"args": {"name": "test234"}}
        check_id = "test-1"

        res = service.get_filtered_params_to_log(check_id, block_dict, {})
        self.assertEqual(res, {"name": "test234"})

    def test_execute1(self):
        """
        Query for a service. Should pass
        """
        def _get_all():
            return ["service1", "service2"]
        def _status(name):
            if name == "service1":
                return True
            elif name == "service2":
                return False
            return True
        def _enabled(name):
            return True
        service.__mods__ = {
            "service.get_all": _get_all,
            "service.status": _status,
            "service.enabled": _enabled
        }
        block_dict={"args": {"name": "service1"}}
        check_id = "test-1"

        status, res = service.execute(check_id, block_dict, {})
        self.assertEqual(res, {"result": [{"name": "service1", "running": True, "enabled": True}]})

    def test_execute2(self):
        """
        Query for a service, match running: False
        """
        def _get_all():
            return ["service1", "service2"]
        def _status(name):
            if name == "service1":
                return True
            elif name == "service2":
                return False
            return True
        def _enabled(name):
            return True
        service.__mods__ = {
            "service.get_all": _get_all,
            "service.status": _status,
            "service.enabled": _enabled
        }
        block_dict={"args": {"name": "service2"}}
        check_id = "test-1"

        status, res = service.execute(check_id, block_dict, {})
        self.assertEqual(res, {"result": [{"name": "service2", "running": False, "enabled": True}]})

    def test_execute3(self):
        """
        Check for '*'
        """
        def _get_all():
            return ["service1", "service2"]
        def _status(name):
            if name == "service1":
                return True
            elif name == "service2":
                return False
            return True
        def _enabled(name):
            return True
        service.__mods__ = {
            "service.get_all": _get_all,
            "service.status": _status,
            "service.enabled": _enabled
        }
        block_dict={"args": {"name": "s*"}}
        check_id = "test-1"

        status, res = service.execute(check_id, block_dict, {})
        self.assertEqual(res, {"result": [
            {"name": "service1", "running": True, "enabled": True},
            {"name": "service2", "running": False, "enabled": True}
            ]})


from collections import deque
from bitarray import bitarray
from bitarray.util import int2ba
from timeit import default_timer
import os
from random import randint

class Node:
    def __init__(self, char, weight: int, left: 'Node' = None, right: 'Node' = None):
        self.char = char
        self.weight = weight
        self.left = left
        self.right = right

    def create_dict(self, dictionary: dict, bitarr: bitarray = bitarray()):
        if self.char:
            dictionary[self.char] = bitarr
        if self.left is not None:
            bitarr_copy = bitarr.copy()
            bitarr_copy.append(0)
            self.left.create_dict(dictionary, bitarr_copy)
        if self.right is not None:
            bitarr_copy = bitarr.copy()
            bitarr_copy.append(1)
            self.right.create_dict(dictionary, bitarr_copy)
        return dictionary

class StaticHuffman:
    def __init__(self, text: str):
        self.text = text
        self.chars = self.count_letters()
        self.root = self.huffman()

    def count_letters(self) -> dict:
        chars_array = {}
        for letter in self.text:
            if letter not in chars_array.keys():
                chars_array[letter] = 1
            else:
                chars_array[letter] += 1
        return chars_array

    def huffman(self):
        nodes_list = []
        leaves = deque()
        internal_nodes = deque()
        letter_counts = self.chars
        for a, w in letter_counts.items():
            nodes_list.append(Node(a, w))
        nodes_list.sort(key=lambda x: x.weight)
        for elem in nodes_list:
            leaves.append(elem)

        while len(leaves) + len(internal_nodes) > 1:
            elems = []
            for i in range(2):
                if len(leaves) == 0:
                    elems.append(internal_nodes.popleft())
                    continue
                if len(internal_nodes) == 0:
                    elems.append(leaves.popleft())
                    continue
                if leaves[0].weight < internal_nodes[0].weight:
                    elems.append(leaves.popleft())
                else:
                    elems.append(internal_nodes.popleft())

            element1, element2 = elems[0], elems[1]
            internal_nodes.append(Node(None, element1.weight + element2.weight, element1, element2))
        return internal_nodes[0]

    def Huffman_dict(self):
        b_array = bitarray()
        d = dict()
        return self.root.create_dict(d, b_array)

    def code(self, dictionary: dict):  # number_of_chars--[letter, lengthofletter(bin), code_of_letter) ]
        # for every char in dict]--[letter_code for code in text]
        bit_code = bitarray()
        bit_code.extend(int2ba(len(dictionary), 8, endian='big'))
        for letter, letter_code in dictionary.items():
            command = bitarray()
            command.frombytes(bytes(letter, 'ascii'))
            command.extend(int2ba(len(letter_code), 8, endian='big'))
            command.extend(letter_code)
            bit_code.extend(command)

        text_code = bitarray()
        text_code.encode(dictionary, self.text)
        bit_code.extend(text_code)

        return bit_code

    def decoder(self, bits: bitarray):
        dictionary = dict()
        dict_size = int.from_bytes(bits[:8], byteorder='big', signed=True)
        bits = bits[8:]
        i: int = 0
        while i < dict_size:
            letter = bits[:8].tobytes().decode()
            bits = bits[8:]
            length = int.from_bytes(bits[:8], byteorder='big', signed=True)
            bits = bits[8:]
            letter_code = bits[:length]
            bits = bits[length:]
            dictionary[letter] = letter_code
            i += 1

        return bits.decode(dictionary)

def static_Huffman_test(filename, encode_file, results):
    with open(filename, "r") as file:
        T = file.read()
    SH = StaticHuffman(T)
    huffman_dict = SH.Huffman_dict()
    code_start = default_timer()
    encode_text = SH.code(huffman_dict)
    nieszczesne_modulo = 8 - len(encode_text) % 8
    with open(encode_file, "wb") as file:
        encode_text.tofile(file)
    code_end = default_timer()

    bit_arr = bitarray()
    with open(encode_file, "rb") as file:
        bit_arr.fromfile(file)
    bit_arr = bit_arr[:-nieszczesne_modulo]

    decode_start = default_timer()
    decode_text = SH.decoder(bit_arr)
    decode_end = default_timer()
    decode_text = ''.join(decode_text)
    # print(decode_text)

    with open(results, "w") as file:
        file.write(decode_text)

    print("&", 1 - os.stat(encode_file).st_size / os.stat(filename).st_size, "&", code_end - code_start, "&",
          decode_end - decode_start, "\\ \hline")

def prep_random(letters):
    return ''.join([chr(randint(32, 126)) for _ in range(letters)])

'''
static_Huffman_test("big_text.txt", "compress.txt", "res.txt")
static_Huffman_test("mediumbig_text.txt", "compress.txt", "res.txt")
static_Huffman_test("medium_text.txt", "compress.txt", "res.txt")
static_Huffman_test("small_text.txt", "compress.txt", "res.txt")
print("    ")
static_Huffman_test("big_code.txt", "compress.txt", "res.txt")
static_Huffman_test("mediumbig_code.txt", "compress.txt", "res.txt")
static_Huffman_test("medium_code.txt", "compress.txt", "res.txt")
static_Huffman_test("small_code.txt", "compress.txt", "res.txt")
print("    ")
'''

with open("small_rand.txt", "w") as file:
    file.write(prep_random(1000))

with open("medium_rand.txt", "w") as file:
    file.write(prep_random(20000))

with open("mediumbig_rand.txt", "w") as file:
    file.write(prep_random(100000))

with open("big_rand.txt", "w") as file:
    file.write(prep_random(1000000))

static_Huffman_test("small_rand.txt", "compress.txt", "res.txt")
static_Huffman_test("medium_rand.txt", "compress.txt", "res.txt")
static_Huffman_test("mediumbig_rand.txt", "compress.txt", "res.txt")
static_Huffman_test("big_rand.txt", "compress.txt", "res.txt")