#-#-# Libs:

## Storage:
from pickle import load

## Pre-processing:
from numpy import array

## Keras:
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical

# Caption generator model:
from keras.models import Model # Functions with model
from keras.layers import Input # Input layer
from keras.layers import Dense # Dense layer
from keras.layers import LSTM # LSTM layer
from keras.layers import Embedding # Word Embedding layer
from keras.layers import Dropout # Dropout layer
from keras.layers.merge import add # Addition layer
from keras.callbacks import ModelCheckpoint # Checkpoints
from keras.optimizers import Adam # Optimiser

#-#-#

#-#-# Functions:

#-# Load text:
def load_text(filename):
    file = open(filename, 'r') # Open the file as read only
    text = file.read() # Read all text
    file.close() # Close the file

    return text
#-#

#-# Load a pre-defined list of image identifiers:
def load_set(filename):
    doc = load_text(filename)
    dataset = list()

    # Process line by line:
    for line in doc.split('\n'):
        # Skip empty lines
        if len(line) < 1:
            continue

        # Get the image identifier
        identifier = line.split('.')[0]
        dataset.append(identifier)

    return set(dataset)
#-#

#-# Load clean descriptions into memory:
def load_clean_descriptions(filename, dataset):
    # Load document
    doc = load_text(filename)
    descriptions = dict()

    for line in doc.split('\n'):
        # Split line by white space
        tokens = line.split()

        # Split id from description
        image_id, image_desc = tokens[0], tokens[1:]

        # Skip images not in the set
        if image_id in dataset:
            # Create list
            if image_id not in descriptions:
                descriptions[image_id] = list()

            # Wrap description in tokens
            desc = 'startseq ' + ' '.join(image_desc) + ' endseq'

            # Store descriptions
            descriptions[image_id].append(desc)

    return descriptions
#-#

#-# Load image features:
def load_image_features(filename, dataset):
    # Load all features
    all_features = load(open(filename, 'rb'))

    # Filter features
    features = {k: all_features[k] for k in dataset}

    return features
#-#

#-# Covert a dictionary of clean descriptions to a list of descriptions:
def to_lines(descriptions):
    all_desc = list()
    for key in descriptions.keys():
        [all_desc.append(d) for d in descriptions[key]]

    return all_desc
#-#

#-# Fit a tokenizer given caption descriptions:
def create_tokenizer(descriptions):
    lines = to_lines(descriptions)
    tokenizer = Tokenizer()
    tokenizer.fit_on_texts(lines)

    return tokenizer
#-#

#-# Calculate the length of the description with the most words:
def max_length(descriptions):
    lines = to_lines(descriptions)

    return max(len(d.split()) for d in lines)
#-#

#-# Create sequences of images, input sequences and output words for an image
def create_sequences(tokenizer, max_length, descriptions, images, vocab_size):
    X1, X2, y = list(), list(), list()

    # Get through each image identifier:
    for key, desc_list in descriptions.items():
        # Get through each description for the image
        for desc in desc_list:
            # Encode the sequence
            seq = tokenizer.texts_to_sequences([desc])[0]

            # Split one sequence into multiple X,y pairs:
            for i in range(1, len(seq)):
                # Split into input and output pair
                in_seq, out_seq = seq[:i], seq[i]

                # Pad input sequence
                in_seq = pad_sequences([in_seq], maxlen=max_length)[0]

                # Encode output sequence
                out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]

                # Store sequences
                X1.append(images[key][0])
                X2.append(in_seq)
                y.append(out_seq)

    return array(X1), array(X2), array(y)
#-#

#-# Define the captioning model
def define_model(vocab_size, max_length):
    ## Feature extractor model:
    inputs1 = Input(shape=(2048,)) # Same as output of ResNet-50 pre-last layer
    fe1 = Dropout(0.5)(inputs1) # 50 % dropout
    fe2 = Dense(256, activation='relu')(fe1) # ReLU activation

    ## Sequence model:
    inputs2 = Input(shape=(max_length,))
    se1 = Embedding(vocab_size, 256, mask_zero=True)(inputs2)
    se2 = Dropout(0.5)(se1)
    se3 = LSTM(256)(se2)

    ## Decoder model:
    decoder1 = add([fe2, se3])
    decoder2 = Dense(256, activation='relu')(decoder1)
    outputs = Dense(vocab_size, activation='softmax')(decoder2)

    ## Merge them together [image, seq] [word]:
    model = Model(inputs=[inputs1, inputs2], outputs=outputs) # Define model
    opt = Adam(learning_rate=0.01, epsilon=0.1) # Adam optimiser
    model.compile(loss='categorical_crossentropy', optimizer=opt) # Start training

    ## Print final model
    print(model.summary())

    return model
#-#

#-#-#

#-#-# Main:

if __name__ == '__main__': # !Don't forget to change the dataset paths!
    ### Training:
    ## Load training images dataset (6K)
    filename = 'Flickr8k_text/Flickr_8k.trainImages.txt' # training dataset path
    #filename = '/home/dd/Documents/Datasets/Flickr8k/Flickr8k_text/Flickr_8k.trainImages.txt'
    train = load_set(filename) # Load dataset
    print('Dataset: %d' % len(train))

    ## Load training descriptions:
    train_descriptions = load_clean_descriptions('Pre-trained/descriptions.txt', train)
    print('Descriptions: train=%d' % len(train_descriptions))

    ## Load image features:
    train_features = load_image_features('Pre-trained/features.pkl', train)
    print('Images: train=%d' % len(train_features))

    ## Load Tokenizer:
    tokenizer = create_tokenizer(train_descriptions)
    vocab_size = len(tokenizer.word_index) + 1
    print('Vocabulary Size: %d' % vocab_size)

    # Determine the maximum sequence length
    max_length = max_length(train_descriptions)
    print('Description Length: %d' % max_length)

    ## Prepare sequences:
    X1train, X2train, ytrain = create_sequences(tokenizer, max_length, train_descriptions, train_features, vocab_size)

    ### Evaluation:
    # Load development dataset
    filename = 'Flickr8k_text/Flickr_8k.devImages.txt' # Dev dataset path
    #filename = '/home/dd/Documents/Datasets/Flickr8k/Flickr8k_text/Flickr_8k.devImages.txt'
    test = load_set(filename) # Load dataset
    print('Dataset: %d' % len(test))

    ## Load dev descriptions:
    test_descriptions = load_clean_descriptions('Pre-trained/descriptions.txt', test)
    print('Descriptions: test=%d' % len(test_descriptions))

    ## Load image features:
    test_features = load_image_features('Pre-trained/features.pkl', test)
    print('Images: test=%d' % len(test_features))

    ## Prepare sequences:
    X1test, X2test, ytest = create_sequences(tokenizer, max_length, test_descriptions, test_features, vocab_size)

    ### Fit the model:
    # Initialise the model
    model = define_model(vocab_size, max_length)

    # Initialise checkpoint callback
    filepath = 'model-ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5'
    checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min')

    # Fit model
    model.fit([X1train, X2train], ytrain, epochs=20, verbose=2, callbacks=[checkpoint], validation_data=([X1test, X2test], ytest))

#-#-#


import zookeeper

from twisted.internet.defer import inlineCallbacks, fail, succeed

from txzookeeper import ZookeeperClient
from txzookeeper.utils import retry_change
from txzookeeper.tests.mocker import MATCH
from txzookeeper.tests import ZookeeperTestCase, utils

MATCH_STAT = MATCH(lambda x: isinstance(x, dict))

class RetryChangeTest(ZookeeperTestCase):

    def update_function_increment(self, content, stat):
        if not content:
            return str(0)
        return str(int(content) + 1)

    def setUp(self):
        super(RetryChangeTest, self).setUp()
        self.client = ZookeeperClient("127.0.0.1:2181")
        return self.client.connect()

    def tearDown(self):
        utils.deleteTree("/", self.client.handle)
        self.client.close()

    @inlineCallbacks
    def test_node_create(self):
        """
        retry_change will create a node if one does not exist.
        """
        #use a mock to ensure the change function is only invoked once
        func = self.mocker.mock()
        func(None, None)
        self.mocker.result("hello")
        self.mocker.replay()

        yield retry_change(
            self.client, "/magic-beans", func)

        content, stat = yield self.client.get("/magic-beans")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 0)

    @inlineCallbacks
    def test_node_update(self):
        """
        retry_change will update an existing node.
        """
        #use a mock to ensure the change function is only invoked once
        func = self.mocker.mock()
        func("", MATCH_STAT)
        self.mocker.result("hello")
        self.mocker.replay()

        yield self.client.create("/magic-beans")
        yield retry_change(
            self.client, "/magic-beans", func)

        content, stat = yield self.client.get("/magic-beans")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 1)

    def test_error_in_change_function_propogates(self):
        """
        an error in the change function propogates to the caller.
        """

        def error_function(content, stat):
            raise SyntaxError()

        d = retry_change(self.client, "/magic-beans", error_function)
        self.failUnlessFailure(d, SyntaxError)
        return d

    @inlineCallbacks
    def test_concurrent_update_bad_version(self):
        """
        If the node is updated after the retry function has read
        the node but before the content is set, the retry function
        will perform another read/change_func/set cycle.
        """
        yield self.client.create("/animals")
        content, stat = yield self.client.get("/animals")
        yield self.client.set("/animals", "5")

        real_get = self.client.get
        p_client = self.mocker.proxy(self.client)
        p_client.get("/animals")
        self.mocker.result(succeed((content, stat)))

        p_client.get("/animals")
        self.mocker.call(real_get)

        self.mocker.replay()

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "6")
        self.assertEqual(stat["version"], 2)

    @inlineCallbacks
    def test_create_node_exists(self):
        """
        If the node is created after the retry function has determined
        the node doesn't exist but before the node is created by the
        retry function. the retry function will perform another
        read/change_func/set cycle.
        """
        yield self.client.create("/animals", "5")

        real_get = self.client.get
        p_client = self.mocker.patch(self.client)
        p_client.get("/animals")
        self.mocker.result(fail(zookeeper.NoNodeException()))

        p_client.get("/animals")
        self.mocker.call(real_get)
        self.mocker.replay()

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "6")
        self.assertEqual(stat["version"], 1)

    def test_set_node_does_not_exist(self):
        """
        if the retry function goes to update a node which has been
        deleted since it was read, it will cycle through to another
        read/change_func set cycle.
        """
        real_get = self.client.get
        p_client = self.mocker.patch(self.client)
        p_client.get("/animals")
        self.mocker.result("5", {"version": 1})

        p_client.get("/animals")
        self.mocker.call(real_get)

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "0")
        self.assertEqual(stat["version"], 0)

    def test_identical_content_noop(self):
        """
        If the change function generates identical content to
        the existing node, the retry change function exits without
        modifying the node.
        """
        self.client.create("/animals", "hello")

        def update(content, stat):
            return content

        yield retry_change(self.client, "/animals", update)
        content, stat = self.client.get("/animals")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 0)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Based on: "Udacity Data Wrangling, Problem Set 6"
# Coded for Python 2.7 (as used in Udacity course)

'''
Cleaning & standardizing address values in OpenStreetMap data
  -- problem characters (e.g., "Street\\")
  -- abbreviations
  -- inconsistencies

This is a helper script with functions that can be imported into and
used in other, data-to-csv transformation scripts.
'''

#import xml.etree.cElementTree as ET
import re
from sys import argv

# REGEX patterns to search for
street_type_re = re.compile(r'\b\S+\.?$', re.IGNORECASE)  # find last word
nsew_re = re.compile(r'N |N\. |S |S\. |E |E\. |W |W\. ')  # find compass letter
postal_re = re.compile(r'(\d{5})')  # find postal code

# mapping of states to abbreviations -- Nevada and bordering states
states = {'Arizona': 'AZ',
          'California': 'CA',
          'Nevada': 'NV',
          'Utah': 'UT'}

# mapping of directional abbreviations to full words
nsew_unabbrev = {'N ': 'North ', 'N. ': 'North ',
                 'S ': 'South ', 'S. ': 'South ',
                 'E ': 'East ', 'E. ': 'East ',
                 'W ': 'West ', 'W. ': 'West '}

# mapping of street type abbreviations to full words
street_unabbrev = \
{'Ave': 'Avenue', 'AVE': 'Avenue', 'Ave.': 'Avenue',
 'Blvd': 'Boulevard', 'Blvd.': 'Boulevard',
 'Cir': 'Circle', 'Cir.': 'Circle',
 'Crt': 'Court', 'Crt.': 'Court',
 'Ct': 'Court', 'Ct.': 'Court',
 'Dr': 'Drive', 'Dr.': 'Drive',
 'E.': 'East',
 'Fwy': 'Freeway', 'Fwy.': 'Freeway',
 'Hwy': 'Highway', 'Hwy.': 'Highway',
 'Ln': 'Lane', 'Ln.': 'Lane',
 'Mt': 'Mountain', 'Mt.': 'Mountain',
 'N.': 'North',
 'Pkwy': 'Parkway', 'Pkwy.': 'Parkway',
 'Pl': 'Place', 'Pl.': 'Place',
 'Pt': 'Point', 'Pt.': 'Point',
 'Rd': 'Road', 'Rd.': 'Road',
 'Rte': 'Route', 'Rte.': 'Route',
 'S.': 'South',
 'Sq': 'Square', 'Sq.': 'Square',
 'St': 'Street', 'St.': 'Street',
 'Ter': 'Terrace', 'Ter.': 'Terrace',
 'Tr': 'Trail', 'Tr.': 'Trail',
 'W.': 'West',
 'Wy': 'Way', 'Wy.': 'Way'}

# CLEANING FUNCTIONS

def clean_streetname(name):
    '''In name, replace abbreviated street types with full words'''
    m = street_type_re.search(name)
    if m and  m.group() in street_unabbrev:
        name = name.replace(m.group(), street_unabbrev[m.group()])
    # Munge the Las Vegas "Strip" because street-type not at end of string
    if 'Vegas Blvd' in name:
        name = name.replace('Blvd', 'Boulevard')
    return name

def clean_nsew(name):
    '''In name, replace abbreviated compass words with full words'''
    m = nsew_re.search(name)
    if m:
        name = name.replace(m.group(), nsew_unabbrev[m.group()])
    return name

def clean_state(value):
    '''Ensure state value is a 2-character abbreviation'''
    if len(value) != 2 and value in states:
        value = states.get(value)
    return value

def clean_postal(value):
    '''Search for 5 consecutive digits in string value;
       if found, return the digits as a string.
       Or else, return original input string.
    '''
    m = postal_re.search(value)
    if m:
        value = m.group()
    return value

def is_addr_street(tag):
    '''T|F: does an osm tag specify a street address?'''
    return (tag.attrib['k'] == 'addr:street')

# MAP PROCESSING

def process_tag(element):
    '''element is a <node> or a <way>'''
    for tag in element.iter("tag"):
        value = clean_streetname(tag.attrib['v'])
        value = clean_nsew(tag.attrib['v'])
        return value

if __name__ == "__main__":
    script, element = argv
    #for _, element in ET.iterparse(osmfile):  ## Uncomment if standalone script
    if element.tag == "tag":
        s = process_tag(element)
        print(s)


"""
Enunciado

Faça um programa que leia três valores e apresente o maior dos três valores lidos seguido da mensagem “eh o maior”.

Entrada

O arquivo de entrada contém três valores inteiros.

Saída

Imprima o maior dos três valores seguido por um espaço e a mensagem "eh o maior".

CÓDIGO FONTE
"""
values=input().split(" ")
values=[int(item) for item in values]

high=0;

for item in values:
    if item > high:
        high=item

print(f"{high} eh o maior")


# -*- coding: utf-8 -*-
import json
import re
import time, html
import traceback
import scrapy
import base64
import requests
from lxml import etree
from lxml.etree import HTML
import tool
from save_database import process_item
from tool import get_city,more
# 中国政府采购网
class xinyang_ggzy:
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
            'Cookie': 'Hm_lvt_9f8bda7a6bb3d1d7a9c7196bfed609b5=1630995346; JSESSIONID=s1i-5xuUUYpn30u7YNw3JBb7rVvFOtUKJAdzsa_FgAnaZL1fPadh!2105014073; Hm_lpvt_9f8bda7a6bb3d1d7a9c7196bfed609b5=1630995422'
        }

    def parse(self):
        date = tool.date
        page = 20
        while True:
            page += 1
            print('*' * 20, page, '*' * 20)
            url=f'http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index={page}&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=7&dbselect=bidx&kw=%E4%B8%AD%E6%A0%87&start_time=2019%3A01%3A01&end_time=2023%3A04%3A12&timeType=6&displayZone=&zoneId=&pppStatus=0&agentName='
            text = tool.requests_get(url, self.headers)

            html = HTML(text)
            detail = html.xpath("//ul[@class='vT-srch-result-list-bid']/li")
            for li in detail:
                try:
                    title = li.xpath('./a//text()')[0].split('发布日期')[0].replace(' ','')
                    dates=li.xpath('./span/text()')[0].split('|')[0]
                    date_Today = re.findall(r'(\d{4}\.\d{2}\.\d{2})',dates)[0].replace('.','-')
                    url =  li.xpath('./a/@href')[0]

                    print(title,date_Today,url)
                    if '测试' in title:
                        continue
                    if tool.Transformation(date) <= tool.Transformation(date_Today):
                            self.parse_detile(title, url, date_Today)
                    else:
                        print('日期不符, 正在切换类型', date_Today)
                        return
                except Exception:
                    traceback.print_exc()

    def parse_detile(self, title, url, date):
        print(url)
        t = tool.requests_get(url, self.headers)
        # print(t)
        url_html = etree.HTML(t)
        try:
            detail = url_html.xpath('//*[@class="vF_detail_content"]')[0]
            detail_html = etree.tostring(detail, method='HTML')
            detail_html = html.unescape(detail_html.decode()).replace('\xa0', '').replace('\ufeff', '')
            detail_text = url_html.xpath('string(//*[@class="vF_detail_content"])') \
                .replace('\xa0', '').replace('\n', '').replace('\r', '').replace('\t', '') \
                .replace(' ', '').replace('\xa5', '').replace('\ufeff', '')
        except:
            try:
                detail = url_html.xpath('//*[@class="vF_detail_content_container"]')[0]
                detail_html = etree.tostring(detail, method='HTML')
                detail_html = html.unescape(detail_html.decode()).replace('\xa0', '').replace('\ufeff', '')
                detail_text = url_html.xpath('string(//*[@class="vF_detail_content_container"])') \
                    .replace('\xa0', '').replace('\n', '').replace('\r', '').replace('\t', '') \
                    .replace(' ', '').replace('\xa5', '').replace('\ufeff', '')
            except:
                return
        item = {}
        item['title'] = title.replace('\u2022', '').replace('\n', '').replace('\r', '')
        item['url'] = url
        item['date'] = date
        item['typeid'] = tool.get_typeid(item['title'])
        item['senddate'] = int(time.time())
        item['mid'] = 867
        item['infotype'] = tool.get_infotype(item['title'])
        item['body'] = tool.qudiao_width(detail_html)
        item['body'] = tool.qudiao_width(item["body"])
        item['endtime'] = tool.get_endtime(detail_text)
        if item['endtime'] == '':
            item['endtime'] = int(time.mktime(time.strptime(date, "%Y-%m-%d")))
        else:
            try:
                item['endtime'] = int(time.mktime(time.strptime(item['endtime'], "%Y-%m-%d")))
            except:
                item['endtime'] = int(time.mktime(time.strptime(date, "%Y-%m-%d")))
        item['tel'] = tool.get_tel(detail_text)
        item['email'] = ''
        # print(item['title'])
        item['winner'] = tool.get_winner(detail_text)
        item['address'] = tool.get_address(detail_text)
        item['nativeplace'] = tool.get_title_city(item['title'])
        if item['nativeplace'] == 0:
            item['nativeplace'] = tool.more(item['title']+detail_text)
        item['linkman'] = tool.get_linkman(detail_text)
        item['function'] = tool.get_function(detail_text)
        item['resource'] = '中国政府采购网'
        # print(';;;;;',item["nativeplace"])
        if '.' in str(item["nativeplace"]):
            item["shi"] = int(float(str(item["nativeplace"]).replace("['","").split('.')[0]))
        else:
            item["shi"] = int(float(item["nativeplace"]))
        if len(str(int(item["shi"]))) == 4:
            item['sheng'] = int(str(item["shi"])[:2] + '00')
        elif len(str(int(item["shi"]))) == 5:
            item['sheng'] = int(str(item["shi"])[:3] + '00')
        else:
            item['sheng'] = 0
        item['removal']= title
        process_item(item)
        # print(item['title'],item['url'],item['nativeplace'],item['address'])
        # time.sleep(6666)

if __name__ == '__main__':
    import traceback, os
    try:
        jl = xinyang_ggzy()
        jl.parse()
    except Exception as e:
        traceback.print_exc()

中标信息/市级1/中国政府采购网.py


import folium
from Grid import Region
import pandas as pd
from grid_visualize import get_index_high_ratio_24h

GRID_CONGESTED_THREASHOLD = 0.3

base_map = folium.Map(location=(42.30, -71.05), zoom_start=13)
congested_indexes = get_index_high_ratio_24h(GRID_CONGESTED_THREASHOLD)
reg = Region(42.70, -71.50, 42.10, -70.75)
reg.add_ratio()

for label in congested_indexes:
	grid = reg.find_grid(label)

	grid_center = ((grid.x + grid.m) / 2, (grid.y + grid.n) / 2)
	folium.Marker(grid_center, icon=folium.Icon(color='blue')).add_to(base_map)

grids = [reg.find_grid(label) for label in congested_indexes]
maxi_g = max(grids, key=lambda x: x.congestion_ratio)
maxi_center = ((maxi_g.x + maxi_g.m) / 2, (maxi_g.y + maxi_g.n) / 2)
folium.Marker(maxi_center, popup='most congested', icon=folium.Icon(color='red')).add_to(base_map)

base_map.save('q1.html')


from os import path
import pandas
import sys
from sklearn.preprocessing import LabelBinarizer

class AudioDataPreprocessor:
    def __init__(self):
        self.completeDataset = None
        self.training = None
        self.validation = None
        self.test = None

    def quitIfFileNotExist(self, file):
        if not path.exists(file):
            print(file + ' does not exists')
            sys.exit(1)

    # Return one-hot encoded labels (y) and raw data (x)
    def dataframeToXandY(self, dataframe):
        encoder = LabelBinarizer()
        y = dataframe['voyelle']
        transformedY = encoder.fit_transform(y)
        x = dataframe.drop(columns=['voyelle'])

        return (x, transformedY)

    def load(self, filename):
        self.quitIfFileNotExist(filename)

        completeDataset = pandas.read_csv(
            filename,
            sep='\t',
            usecols=['voyelle', 'F1', 'F2', 'F3', 'F4', 'Z1', 'Z2', 'f0']
        )

        completeDataset.dropna() # remove NaN rows
        completeDataset = completeDataset[completeDataset['F4'] != '--undefined--'] # remove rows with F4 equal to --undefined--

        # Types casting
        completeDataset[['F1', 'F2', 'F3', 'F4', 'f0']] = completeDataset[['F1', 'F2', 'F3', 'F4', 'f0']].astype(float)

        splittingSourceDataset = completeDataset.copy()

        # Use 80% as training
        training_sample = splittingSourceDataset.sample(frac=0.8, random_state=0)
        x_train, y_train = self.dataframeToXandY(training_sample)

        # Update splitting source dataset
        splittingSourceDataset = splittingSourceDataset.drop(x_train.index)

        # Split what's left in 2
        # 100% - 80% = 20% / 2 = 10% of total dataset
        validation_sample = splittingSourceDataset.sample(frac=0.5, random_state=0)
        x_validation, y_validation = self.dataframeToXandY(validation_sample)

        test_sample = splittingSourceDataset.drop(validation_sample.index)
        x_test, y_test = self.dataframeToXandY(test_sample)

        self.completeDataset = completeDataset
        self.training = {'x': x_train, 'y': y_train}
        self.validation = {'x': x_validation, 'y': y_validation}
        self.test = {'x': x_test, 'y': y_test}


# Combining and Concatanating Strings

parts = ['Is', 'Chicago', 'Not', 'Chicago?']
' '.join(parts)
','.join(parts)
''.join(parts)

a = 'Is Chicago'
b = 'Not Chicago?'
a + ' '  + b

print('{} {}'.format(a,b))
print(a + ' ' + b)

a = 'Hello' 'World'
a

# Discussion

# Don't ever do this:
s = ''
for p in parts:
    s += p

data = ['ACME', 50, 91.1]
','.join(str(d) for d in data)

a = 'this'
b = 'is a'
c = 'string'
print(a + ':' + b + ':' + c) # Ugly
print(':'.join([a, b, c]))   # Still ugly
print(a,b,c, sep=':')        # Better

def sample():
    yield 'Is'
    yield 'Chicago'
    yield 'Not'
    yield 'Chicago?'

text = ''.join(sample())
text

for part in sample():
    print(part)

def combine(source, maxsize):
    parts = []
    size = 0
    for part in source:
        parts.append(part)
        size += len(part)
        if size > maxsize:
            yield ''.join(parts)
            parts=[]
            size=0
    yield ''.join(parts)

for part in combine(sample(), 32768):
    print(part)


import sqlite3
from datetime import datetime
import logging
logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

def now():
    return datetime.now().strftime("%Y-%m-%d %H:%M:%S")

class Controller(object):
    """
     Controller class for adding, updating, retrieving and removing objects
    """
    def __init__(self, db_uri):
        self._dbh = None
        self.db_uri = db_uri
        self._entity_table_map = {
                'user': 'users',
                'rack': 'racks',
                'server': 'servers',
                'instance': 'instances'
            }

    @property
    def dbh(self):
        if self._dbh is None:
            self._dbh = sqlite3.connect(self.db_uri)
            # try enabling foreign keys
            try:
                self._dbh.execute("PRAGMA foreign_keys = ON")
            except:
                logger.warning("Could not enable foreign_keys on db")
                pass
        return self._dbh

    def create_db(self,sql_file):
        with open(sql_file,'r') as fh:
            sql = fh.read()
        c = self.dbh.cursor()
        c.executescript(sql)
        self.dbh.commit()

    def db_select(self,query,args=None):
        logger.debug("Executing query:%s, args:%s", query,str(args))
        c = self.dbh.cursor()
        if args is None: args = tuple()
        result = c.execute(query,args)
        columns = [i[0] for i in  c.description]
        rows = [dict(list(zip(columns,row))) for row in c.fetchall()]
        logger.debug("%d rows found", len(rows))
        return rows

    def list_user(self,limit=None):
        query = "select u.*,ifnull(i.num_instances,0) as num_instances from users u left join (select owner_id,count(*) as num_instances from instances  group by owner_id) i on i.owner_id=u.id"
        query = "select * from users"
        return self.db_select(query)

    def get_user(self, id=None, login=None):
        query = None
        if id:
            query = ("select * from users where id=?",(id,))
        elif login:
            query = ("select * from users where login=?",(login,))
        if not query:
            return None

        rows = self.db_select(query[0],query[1])
        logger.debug("Found %d users" % len(rows))
        return rows[0] if rows else None

    def delete_user(self,id=None,login=None):
        user = self.get_user(id=id,login=login)
        if not user:
            raise Exception("User not found:%s")
        count_query = "select count(*) as count from instances  where owner_id=?"
        result = self.db_select(count_query, (user['id'],))
        if result[0]['count'] > 0:
            raise Exception("Cannot delete as user owns instances")
        self.dbh.execute("delete from users where id=?",  (user['id'],))
        self.dbh.commit()
        return user

    def add_user(self,login=None,name=None):
        id = None
        try:
            c = self.dbh.cursor()
            c.execute("INSERT INTO users(login,name,last_updated) values(?,?,?)", (login,name,now()))
            self.dbh.commit()
            id = c.lastrowid

        except Exception as e:
            logger.error("Error creating user:%s", e)
            raise e
        return self.get_user(id=id)

    def list_rack(self,limit=None):
        query = "select r.*,ifnull(s.num_servers,0) as num_servers from racks r left join (select rack_id,count(*) as num_servers from servers  group by rack_id) s on s.rack_id=r.id"
        return self.db_select(query)

    def get_rack(self, id=None, name=None):
        if name is not None and id is None:
            id_result=self.db_select("select id from racks where name=?",(name,))
            if not id_result:
                return None
            id = id_result[0]['id']
        query = ("select r.*,ifnull(s.num_servers,0) as num_servers from racks r left join (select rack_id,count(*) as num_servers from servers where rack_id=? group by rack_id) s on s.rack_id=r.id where r.id=?", (id,id))
        rows = self.db_select(query[0],query[1])
        logger.debug("Found %d racks" % len(rows))
        return rows[0] if rows else None

    def add_rack(self,name=None,total_slots=None,desc=None):
        id = None
        try:
            c = self.dbh.cursor()
            c.execute("INSERT INTO racks(name,total_slots,description,last_updated) values(?,?,?,?)", (name,total_slots,desc,now()))
            id = c.lastrowid
            logger.info("Rack created with id:%s" % id)
            self.dbh.commit()
        except Exception as e:
            logger.error("Error creating rack:%s", e)
            raise e
        return self.get_rack(id=id)

    def delete_rack(self,id=None,name=None):
        rack = self.get_rack(id=id,name=name)

        if not rack:
            raise Exception("Rack not found:%s")

        rack_occupancy_query="select count(*) as count from servers where rack_id=?"
        rack_occupancy = self.db_select(rack_occupancy_query,(rack['id'],))

        if rack_occupancy[0]['count'] > 0:
            raise Exception("Cannot delete rack as it has %s servers" % rack_occupancy[0]['count'])

        self.dbh.execute("delete from racks where id=?",  (rack['id'],))
        self.dbh.commit()

        return rack

    def close(self):
        if self._dbh is not None:
            self.dbh.close()
        self._dbh = None

    def get_server(self, id=None, name=None):
        query = None
        if id:
            query = ("select * from servers where id=?", (id,))
        elif name:
            query = ("select * from servers where name=?",(name,))
        if not query:
            return None
        rows = self.db_select(query[0],query[1])
        logger.debug("Found %d servers" % len(rows))
        return rows[0] if rows else None

    def list_server(self,limit=None):
        query = "select s.*,ifnull(i.num_instances,0) as num_instances from servers s left join (select server_id,count(*) as num_instances from instances  group by server_id) i on s.id=i.server_id"
        return self.db_select(query)

    def add_server(self,name=None,cpu=None,memory=None,disk=None,rack_id=None,rack_name=None,desc=None):
        rack = self.get_rack(id=rack_id,name=rack_name)
        if not rack:
            raise Exception("rack not found")
        rack_occupancy_query="select count(*) as count from servers where rack_id=?"
        rack_occupancy = self.db_select(rack_occupancy_query,(rack['id'],))

        if rack_occupancy[0]['count'] >= rack['total_slots']:
            raise Exception("No slots left on the rack with id={id},name={name}".format(**rack))

        insert_query = ("insert into servers(rack_id,name,description,total_cpu,total_memory,total_disk,last_updated) VALUES(?,?,?,?,?,?,?)",(rack['id'],name,desc,cpu,memory,disk,now()))
        id = None
        try:
            c = self.dbh.cursor()
            c.execute(insert_query[0],insert_query[1])
            id = c.lastrowid
            self.dbh.commit()

        except Exception as e:
            self.dbh.rollback()
            logger.error("Could not add server to rack")
            raise e
        return self.get_server(id=id)

    def delete_server(self,id=None,name=None):
        server = self.get_server(id=id,name=name)
        if not server:
            raise Exception("Server not found!")
        instance_count_query = "select count(*) as count from instances where server_id=?"
        instance_count = self.db_select(instance_count_query,(id,))
        if instance_count[0]['count'] > 0:
            raise Exception("Cannot delete the server as it has %d instances" % instance_count[0]['count'])
        try:
            del_query = ("delete from servers where id=?",(server['id'],))
            c = self.dbh.cursor()
            c.execute(del_query[0],del_query[1])
            self.dbh.commit()
        except Exception as e:
            self.dbh.rollback()
            logger.error("Error deleting server:%s", e)
            raise Exception("Could not delete server:%s", e)
        return server

    def list_instance(self,limit=None):
        query = "select * from instances"
        return self.db_select(query)

    def get_instance(self,id=None,name=None):
        query = None
        if id:
            query = ("select * from instances where id=?", (id,))
        elif name:
            query = ("select * from instances where name=?",(name,))
        if not query:
            return None
        rows = self.db_select(query[0],query[1])
        logger.debug("Found %d instances" % len(rows))
        return rows[0] if rows else None

    def check_server_instance(self,server,cpu=None,memory=None,disk=None):
        reason = []
        avail_cpu = server['total_cpu'] - server['used_cpu']
        avail_mem = server['total_memory'] - server['used_memory']
        avail_disk = server['total_disk'] - server['used_disk']
        if avail_cpu < cpu:
            reason.append("Insuffcient cpu: required=%d,available=%d" % (cpu,avail_cpu))
        if avail_mem < memory:
            reason.append("Insuffcient memory: required=%d,available=%d" % (memory,avail_mem))
        if avail_disk < disk:
            reason.append("Insuffcient disk: required=%d,available=%d" % (disk,avail_disk))
        return reason

    def add_instance(self,name=None,cpu=None,memory=None,disk=None,server_id=None,server_name=None,owner_id=None,owner_login=None,desc=None):
        server = self.get_server(id=server_id,name=server_name)
        if not server:
            raise Exception("server not found")
        owner = self.get_user(id=owner_id,login=owner_login)
        if not owner:
            raise Exception("Owner not found")

        #check
        check_result = self.check_server_instance(server,cpu=cpu,memory=memory,disk=disk)
        if check_result:
            raise Exception("Cannot create instance:%s" % ",".join(check_result))

        insert_query = ("insert into instances(server_id,name,description,cpu,memory,disk,owner_id,last_updated) VALUES(?,?,?,?,?,?,?,?)",
                        (server['id'],name,desc,cpu,memory,disk,owner['id'],now()))
        update_query = ("update servers set used_cpu=?,used_memory=?,used_disk=?,last_updated=? where id=?",
                        (server['used_cpu']+cpu,
                         server['used_memory']+memory,
                         server['used_disk']+disk,
                         now(),
                         server['id']))

        id = None
        try:
            c = self.dbh.cursor()
            c.execute(insert_query[0],insert_query[1])
            id = c.lastrowid
            c.execute(update_query[0],update_query[1])
            self.dbh.commit()
        except Exception as e:
            self.dbh.rollback()
            logger.error("Could not add server to rack")
            raise e

        return self.get_instance(id=id)

    def delete_instance(self,id=None,name=None):
        instance = self.get_instance(id=id,name=name)
        if not instance:
            raise Exception("instance not found!")
        try:
            server = self.get_server(instance['server_id'])
            del_query = ("delete from instances where id=?",(instance['id'],))
            update_query = ("update servers set used_cpu=?,used_memory=?,used_disk=?,last_updated=? where id=?",
                            (server['used_cpu']-instance['cpu'],
                             server['used_memory']-instance['memory'],
                             server['used_disk']-instance['disk'],
                             now(),
                             server['id']))

            c = self.dbh.cursor()
            c.execute(del_query[0],del_query[1])
            c.execute(update_query[0],update_query[1])
            self.dbh.commit()
        except Exception as e:
            self.dbh.rollback()
            logger.error("Error deleting server:%s", e)
            raise Exception("Could not delete server:%s", e)
        return instance

    def move_instance(self,id=None,name=None,dest_server_id=None,dest_server_name=None):
        instance = self.get_instance(id=id,name=name)
        if not instance:
            raise Exception("instance not found!")

        src_server = self.get_server(id=instance['server_id'])
        dest_server = self.get_server(id=dest_server_id,name=dest_server_name)
        if not dest_server:
            raise Exception("invalid server id:%s" % dest_server_id)

        if src_server['id'] == dest_server['id']:
            raise Exception("Cannot move instance, source and dest server id are the same")
        #check target
        check_result = self.check_server_instance(dest_server,cpu=instance['cpu'],memory=instance['memory'],disk=instance['disk'])

        if check_result:
            raise Exception("Cannot move instance:%s" % ",".join(check_result))

        try:
            move_query = ("update instances set server_id=?,last_updated=? where id=?",(dest_server['id'],now(),instance['id']))

            update_query1 = ("update servers set used_cpu=?,used_memory=?,used_disk=?,last_updated=? where id=?",
                             (src_server['used_cpu']-instance['cpu'],
                              src_server['used_memory']-instance['memory'],
                              src_server['used_disk']-instance['disk'],
                              now(),
                              src_server['id']))

            update_query2 = ("update servers set used_cpu=?,used_memory=?,used_disk=?,last_updated=? where id=?",
                             (dest_server['used_cpu']+instance['cpu'],
                              dest_server['used_memory']+instance['memory'],
                              dest_server['used_disk']+instance['disk'],
                              now(),
                              dest_server['id']))

            c = self.dbh.cursor()
            c.execute(move_query[0],move_query[1])
            c.execute(update_query1[0],update_query1[1])
            c.execute(update_query2[0],update_query2[1])
            self.dbh.commit()
        except Exception as e:
            raise Exception("Unable to move instance:%s" % instance['id'])
        return self.get_instance(instance['id'])

    def modify_rack(self,id=None,**kwargs):
        rack = self.get_rack(id=id)
        if not rack:
            raise Exception("invalid rack:%s" % id)

        editable_fields = ['name','description','total_slots']

        for k in list(kwargs.keys()):
            if k not in editable_fields:
                raise Exception("Invalid field for editing:%s" % k)
        kwargs['last_updated'] = now()
        update_fields = [k for k,v in kwargs.items() if v is not None]
        if not update_fields:
            raise Exception("Nothing to modify")
        if 'total_slots' in update_fields:
            if kwargs['total_slots'] < rack['num_servers']:
                raise Exception("cannot resize rack to %d slots as it has %d servers" % (kwargs['total_slots'],rack['num_servers']))

        query="update racks set %s where id=?" % (",".join(["%s=?" % k for k in update_fields]))
        args = tuple([kwargs[k] for k in update_fields])
        args += (id,)
        try:
            self.dbh.execute(query,args)
            self.dbh.commit()
        except Exception as e:
            raise Exception("Cannot modify rack with id:%s" % id)

        print(query,args)
        return self.get_rack(id)


import numpy as np
from data_generator import DataGenerator
from expanded_random_representation import ExpandedRandomRepresentation

# A = np.array([1, 7, 9, 2, 0.1, 17, 17, 1.5])
# A=A.reshape(A.shape[0],1)
# k = 3
# b = np.zeros(A.shape[0])
# idx = np.argpartition(A[:-1], k, axis=0)
# b[idx[:k]] = 1
# c = np.zeros(A.shape[0])
# np.putmask(A, b, [-33, -44, -55])
# print(A)

# A = np.array([1, 7, 9, 2, 0.1, 17, 17, 1.5])
# B = np.greater_equal(A, 2).astype(int).reshape(8,1)
# print(B)

# a = np.array([1, 2, 0, 0]).reshape(4,1)
# b = np.array([0, 1, 0, 1]).reshape(4,1)
# print(np.multiply(a,b))

v = np.random.choice([-1, 1], (3, 5), p=[0.5, 0.5])
b = np.array([0, 0, 0, 1, 1]).reshape(5,1)
replace = np.random.choice([-1, 1], (3, 2), p=[0.5, 0.5])

print(v)
print(replace)
# np.putmask(v[:,], b, replace)
wheremask = np.argwhere(b==1)[:,0]
v[:,wheremask] = replace
print(v)


import random

from app.db.models import RouteType
from app.db.operations import get_routes

def tags_sort(tags, routes):
    result_dict = {}
    for route in routes:
        for tag in tags:
            if tag in route['tags']:
                route_id = route['id']
                if route_id not in result_dict:
                    result_dict[route_id] = {'route': route, 'count': 1}
                else:
                    result_dict[route_id]['count'] += 1
    sorted_dict = dict(reversed(sorted(result_dict.items(), key=lambda item: item[1]['count'])))
    sorted_routes = [route_dict[1]['route'] for route_dict in sorted_dict.items()]

    return sorted_routes

def filter_routes(distance: float, tags: str,
                  route_type: RouteType = RouteType.running.value) -> list[dict]:
    routes = get_routes(route_type=route_type, distance=distance)
    random.shuffle(routes)
    tags_list = list(filter(lambda x: len(x) > 0, tags.split(',')))
    if not tags_list[0] == 'биба' and len(tags_list) > 1:
        if not tags_list[1] == 'биба':
            routes = tags_sort(tags_list, routes)
    return routes


#!/usr/local/bin/python
# encoding: utf-8
import extensions
from extensions import *
from xml.dom import minidom, Node # USE cElementTree for FAST stream or BeautifulSoup for html !!™

# https://en.wiktionary.org/w/api.php?action=query&titles=money&prop=revisions&rvprop=content&format=xml
base_url="https://en.wiktionary.org/w/api.php?action=query&prop=revisions&rvprop=content&format=xml&titles="
base_url_xml="https://en.wiktionary.org/w/api.php?action=query&prop=revisions&rvprop=content&format=xml&rvexpandtemplates=true&titles="
http_base="https://en.wiktionary.org/w/index.php?title="

title="Category:Ugaritic_nouns"

html=wget(http_base+title)
link_ex=regex("a href=\"/wiki/(.*)\"")
# str.matches=regex_matches
# links=html.matches(link_ex)
# print(links)
links=match(html,link_ex)
# links=matches(html,link_ex)
print(links)

quit()

def getChild(self,name):
	if name=="__str__": return self.__repr__
	if name=="text": return self.firstChild.nodeValue
	# if name=="__str__": return lambda:self.localName# self.toxml
	if name=="xml": return self.toxml()
	xs=self.getElementsByTagName(name)
	return self.getAttribute(name) or len(xs)==1 and xs[0] or xs

minidom.Element.__getitem__=getChild
minidom.Element.__getattr__=getChild
import xml.dom.minidom
# xmldoc = minidom.parseString(wget(base_url+title))
xmldoc = minidom.parseString(wget(base_url_xml+title))

root = xmldoc.documentElement
# print(root)
# print(root.xml)
page=root['query']['pages']['page']
# minidom.Element.__getattr__=minidom.Element.getAttribute
# minidom.Element.__getattr__=getChild
# print(root.query)
# print(root.toxml())
# print(page.title)
print(page['title'])
text=page['revisions']['rev']
print(text)
print(text.text)


# -*- coding: utf-8 -*-
import numpy as np
from constants import GAME

if GAME == 0:
    from simple_speaker_listener.env import GridEnv as env

class GameState(object):
    def __init__(self, rand_seed):
        self.env = env(gridSize=8, landMarks=3, seed=rand_seed)
        self.reset()

    def reset(self):
        # randomize initial state
        self.env.reset_world()

        st = self.env.getState(0, addId=False)
        self.s_t = np.concatenate([st, np.zeros((5))])
        s2Tmp = np.zeros([3])
        s2Tmp[self.env.getState(1, addId=False)] = 1
        self.s2 = s2Tmp

    def process(self, action, comm):
        # convert original 18 action index to minimal action set index
        self.reward = self.env.act(action)
        st = self.env.getState(0, addId=False)
        commApp = np.zeros((5))
        commApp[comm] = 1
        self.s_t1 = np.concatenate([st, commApp])

    def update(self):
        self.s_t = self.s_t1


from token_vk import token
import bot
import requests
import sql_uploader

class VK:

    def __init__(self):
        self.vk_token = token

# Информация для поиска
    def get_info_for_search(self, user_id):
        user_info = self.get_info_about_user(user_id)
        bot.Bot().write_msg(user_id, f"Введите возраст:")
        user_info['age'] = bot.Bot().get_age()

        if user_info['city'] == None:
            bot.Bot().write_msg(user_id, f"Введите город:")
            user_info['city'] = bot.Bot().get_city()
        if user_info['sex'] == None:
            bot.Bot().write_msg(user_id, f"""
                                        Укажите цифру соответствующую полу:
                                        1 - женский
                                        2 - мужской""")
            user_info['sex'] = bot.Bot().get_sex()
        if user_info['relation'] == None:
            bot.Bot().write_msg(user_id, f"""
                                        Укажите цифру соответствующую статусу:
                                        1 — не женат (не замужем);
                                        2 — встречается;
                                        3 — помолвлен(-а);
                                        4 — женат (замужем);
                                        5 — всё сложно;
                                        7 — влюблен(-а);
                                        8 — в гражданском браке.""")
            user_info['relation'] = bot.Bot().get_relation()
        return user_info

    # Информация по пользователю
    def get_info_about_user(self, id):
        response = requests.get("https://api.vk.com/method/users.get",
                                params={"access_token": self.vk_token, "v": "5.131", "user_ids": id,
                                        'fields': 'bdate, sex, relation, city'})
        if 'error' not in response.json():
            user_info = {}
            user_info['id'] = response.json()['response'][0]['id']
            user_info['name'] = response.json()['response'][0]['first_name']
            if 'city' in response.json()['response'][0]:
                user_info['city'] = response.json()['response'][0]['city']['id']
            else:
                user_info['city'] = None
            if 'sex' in response.json()['response'][0]:
                user_info['sex'] = response.json()['response'][0]['sex']
            else:
                user_info['sex'] = None
            if 'relation' in response.json()['response'][0]:
                user_info['relation'] = response.json()['response'][0]['relation']
            else:
                user_info['relation'] = None
            return user_info
        else:
            return False

    # Проверка города
    def check_city(self, city):
        response = requests.get("https://api.vk.com/method/database.getCities",
                                params={"access_token": self.vk_token, "v": "5.131", 'country_id': 1, 'q': city})
        for item in response.json()['response']['items']:
            if city.lower() == item['title'].lower():
                return item['id']
        return False

    # Результат поиска
    def show_users(self, user_id, user_info):
        users = self.search_users(user_info)
        sql_uploader.SqlUploader().create_table(user_id)
        for find_user_id, first_name, last_name in users:
            if not sql_uploader.SqlUploader().check_id_in_base(user_id, find_user_id):
                photos_list = []
                response = requests.get("https://api.vk.com/method/photos.get", params={"access_token": self.vk_token,
                                                                                "owner_id": find_user_id,
                                                                                "v": "5.131",
                                                                                "album_id": "profile",
                                                                                "extended": "1"})
                photos = response.json()['response']['items']
                for photo in photos:
                    photo_id = photo['id']
                    likes = photo['likes']['count']
                    comments = photo['comments']['count']
                    url = photo['sizes'][-1]['url']
                    new_tuple = (likes, comments, photo_id, url)
                    photos_list.append(new_tuple)
                max_3_photos = sorted(photos_list, key=lambda x: (x[0], x[1]), reverse=True)[:3]
                for item in max_3_photos:
                    bot.Bot().vk.method('messages.send',{'user_id': user_id, "attachment": f'photo{find_user_id}_{item[2]}',
                                                    'random_id': 0})
                user_url = f'https://vk.com/id{find_user_id}'
                bot.Bot().write_msg(user_id, f"{first_name} {last_name} {user_url}")
                bot.Bot().write_msg(user_id, f'Отправьте в чат "+", чтобы добавить в избраное и продолжить\n'
                                              f'или любой символ для продолжения без сохранения\n'
                                              f'Для выхода отправьте "exit"')
                if bot.Bot().choose_from_find(find_user_id, max_3_photos) == False:
                    break

    # Поиск
    def search_users(self, user_info):
        if user_info['sex'] == 2:
            gender = user_info['sex'] - 1
        else:
            gender = user_info['sex'] + 1
        response = requests.get("https://api.vk.com/method/users.search",
                                params={"access_token": self.vk_token, "v": "5.131", "count": "1000",
                                        'has_photo': '1', 'sex': gender, 'status': user_info['relation'],
                                        'city': user_info['city'], 'age_from': user_info['age'] - 5,
                                        'age_to': user_info['age'] + 5})
        response.raise_for_status()
        users_list = response.json()['response']['items']
        find_list = [(user['id'], user['first_name'],
                  user['last_name']) for user in users_list if user['is_closed'] == False]

        return find_list


from sklearn import metrics
from core.findNN import *

def ComputeGap(ushapeletCandidate, lenSubsequence, timeseries, deltas, labels, similarity_measure):
    numTimeseries = len(timeseries)

    ts_index = int(ushapeletCandidate[0])
    loc = int(ushapeletCandidate[1])
    shapelet = timeseries[ts_index, loc : loc + lenSubsequence]
    delta_shapelet = deltas[ts_index, loc : loc + lenSubsequence]

    # Compute the distances from the shapelet to all timeseries
    distances = np.zeros(numTimeseries, dtype=float)
    delta_distances = np.zeros(numTimeseries, dtype=float)
    locations = np.zeros(numTimeseries, dtype=int)
    for i in range(numTimeseries):
        # if i % 100 == 0:
        #     print('findNN for i:', i)
        distances[i], delta_distances[i], locations[i] = findNN(shapelet, delta_shapelet, timeseries[i, :], deltas[i, :], similarity_measure)

    # Sorting the distances to get the orderline
    if similarity_measure == 'UED':
        # Simple ordering
        indices = np.lexsort((delta_distances, distances))
        orderline = distances[indices]
    else:
        orderline = np.sort(distances)

    # Finding the biggest gap in the orderline
    lb = int(np.ceil(numTimeseries * 0.167))
    ub = int(np.floor(numTimeseries * 0.833))

    maxGap = 0
    threshold = 0
    for i in range(lb, ub):
        curThreshold = orderline[i]
        curGap = gap(orderline, curThreshold)
        if curGap > maxGap:
            maxGap = curGap
            threshold = curThreshold

    # Evaluation of the threshold
    label = labels[ts_index]
    evaluation_labels = np.zeros(numTimeseries, dtype=int)
    for i in range(numTimeseries):
        if labels[i] == label:
            evaluation_labels[i] = 1

    IDX = np.zeros(numTimeseries, dtype=int)
    IDX[distances <= threshold] = 1

    RI = metrics.rand_score(evaluation_labels, IDX)

    return maxGap, RI, IDX, distances, delta_distances, locations

def gap(orderline, d):
    Da = np.argwhere(orderline <= d)
    Db = np.argwhere(d < orderline)

    # Both groups have to contain at least two objects
    if Da.shape[0] < 2 or Db.shape[0] < 2:
       return 0

    gap = 0
    r = len(np.argwhere(Da)) / len(np.argwhere(Db))

    # Ensure a high separation power
    if 0.2 < r and r < 5:
        ma = np.mean(orderline[Da])
        mb = np.mean(orderline[Db])
        sa = np.std(orderline[Da])
        sb = np.std(orderline[Db])
        gap = mb - sb - (ma + sa)

    return gap


matrix = [[95, 3, 1, 2, 101],
          [4, 100, 10, 5, 119],
          [3, 9, 55, 5, 72],
          [10, 2, 4, 80, 96]]

def rowOp(down, up, k):
    global matrix
    for p in range(len(matrix[0])):
        matrix[down][p] -= k * matrix[up][p]

def gauss():
    global matrix
    for i in range(len(matrix)):
        rowOp(i, i, (1 - 1 / matrix[i][i]))
        for j in range(i, len(matrix)):
            if i != j:
                rowOp(j, i, matrix[j][i])

gauss()
for i in matrix:
    print(i)


def pot2(num):
    return (num and(not(num&(num-1))))
def descobrir_vencedor(pontuacao_ui,pontuacao_ri,pontuacao_in):
    maior=max(pontuacao_ui,pontuacao_ri,pontuacao_in)
    if(maior==pontuacao_ui):
        if(maior==pontuacao_in or maior==pontuacao_ri):
            return("URI")
        return("Uilton")
    if(maior==pontuacao_ri):
        if(maior==pontuacao_ui or maior==pontuacao_in):
            return("URI")
        return("Rita")
    if(maior==pontuacao_in):
        if(maior==pontuacao_ui or maior==pontuacao_ri):
            return("URI")
        return("Ingred")
while True:
    nrodadas=int(input())
    if(nrodadas==0):
        break
    pontos_ui=0
    pontos_ri=0
    pontos_in=0
    for i in range(nrodadas):
        np=[]
        uilton,rita,ingred=input().split()
        uilton=int(uilton)
        rita=int(rita)
        ingred=int(ingred)
        np.append(uilton)
        np.append(rita)
        np.append(ingred)
        if((pot2(np[0])==True)):
            pontos_ui+=1
            if((max(np))==uilton):
                pontos_ui+=1
        if((pot2(np[1])==True)):
            pontos_ri+=1
            if((max(np))==rita):
                pontos_ri+=1
        if((pot2(np[2])==True)):
            pontos_in+=1
            if((max(np))==ingred):
                pontos_in+=1
    vencedor=descobrir_vencedor(pontos_ui,pontos_ri,pontos_in)
    print(vencedor)


import mmap
import os
import shutil
import tempfile
import unittest

from sudachipy.dictionarylib.grammar import Grammar

class TestGrammar(unittest.TestCase):

    alloc_size = 4096

    def setUp(self):
        storage = bytearray()
        self.build_partofspeech(storage)
        self.build_connect_table(storage)
        self.test_dir = tempfile.mkdtemp()
        f = os.path.join(self.test_dir, 'test_file.txt')
        with open(f, 'wb') as wf:
            wf.write(bytes(storage))
        self.mmap = None
        with open(f, 'rb') as rf:
            self.mmap = mmap.mmap(rf.fileno(), 0, access=mmap.ACCESS_COPY)
        self.storage_size = self.mmap.size()
        offset = 0
        self.grammar = Grammar(self.mmap, offset)

    def tearDown(self):
        shutil.rmtree(self.test_dir)

    def test_storage_size(self):
        self.assertEqual(self.storage_size, self.grammar.storage_size)

    def test_get_partofspeech_string(self):
        self.assertEqual(6, len(self.grammar.get_part_of_speech_string(0)))
        self.assertEqual("BOS/EOS", self.grammar.get_part_of_speech_string(0)[0])
        self.assertEqual("*", self.grammar.get_part_of_speech_string(0)[5])

        self.assertEqual("一般", self.grammar.get_part_of_speech_string(1)[1])
        self.assertEqual("*", self.grammar.get_part_of_speech_string(1)[5])

        self.assertEqual("五段-サ行", self.grammar.get_part_of_speech_string(2)[4])
        self.assertEqual("終止形-一般", self.grammar.get_part_of_speech_string(2)[5])

    def test_get_connect_cost(self):
        self.assertEqual(0, self.grammar.get_connect_cost(0, 0))
        self.assertEqual(-100, self.grammar.get_connect_cost(2, 1))
        self.assertEqual(200, self.grammar.get_connect_cost(1, 2))

    def test_set_connect_cost(self):
        self.grammar.set_connect_cost(0, 0, 300)
        self.assertEqual(300, self.grammar.get_connect_cost(0, 0))

    def test_get_bos_parameters(self):
        self.assertEqual(0, self.grammar.get_bos_parameter()[0])
        self.assertEqual(0, self.grammar.get_bos_parameter()[1])
        self.assertEqual(0, self.grammar.get_bos_parameter()[2])

    def test_get_eos_parameters(self):
        self.assertEqual(0, self.grammar.get_eos_parameter()[0])
        self.assertEqual(0, self.grammar.get_eos_parameter()[1])
        self.assertEqual(0, self.grammar.get_eos_parameter()[2])

    def test_read_from_file(self):
        # Todo
        # after tidying up dictionary management
        pass

    @staticmethod
    def build_partofspeech(storage):
        storage.extend((3).to_bytes(2, byteorder='little', signed=True))  # number of part of speech

        storage.extend(b'\x07B\x00O\x00S\x00/\x00E\x00O\x00S\x00\x01*\x00\x01*\x00\x01*\x00\x01*\x00\x01*\x00')

        storage.extend(b'\x02')
        storage.extend('名刺'.encode('utf-16-le'))
        storage.extend(b'\x02')
        storage.extend('一般'.encode('utf-16-le'))
        storage.extend(b'\x01*\x00\x01*\x00\x01*\x00\x01*\x00')

        storage.extend(b'\x02')
        storage.extend('動詞'.encode('utf-16-le'))
        storage.extend(b'\x02')
        storage.extend('一般'.encode('utf-16-le'))
        storage.extend(b'\x01*\x00\x01*\x00\x05')
        storage.extend('五段-サ行'.encode('utf-16-le'))
        storage.extend(b'\x06')
        storage.extend('終止形-一般'.encode('utf-16-le'))

    @staticmethod
    def build_connect_table(storage):
        storage.extend((3).to_bytes(2, byteorder='little', signed=True))  # number of leftId
        storage.extend((3).to_bytes(2, byteorder='little', signed=True))  # number of rightId

        storage.extend((0).to_bytes(2, byteorder='little', signed=True))  # number of rightId
        storage.extend((-300).to_bytes(2, byteorder='little', signed=True))  # number of rightId
        storage.extend((3000).to_bytes(2, byteorder='little', signed=True))  # number of rightId

        storage.extend((300).to_bytes(2, byteorder='little', signed=True))  # number of rightId
        storage.extend((-500).to_bytes(2, byteorder='little', signed=True))  # number of rightId
        storage.extend((-100).to_bytes(2, byteorder='little', signed=True))  # number of frightId

        storage.extend((-3000).to_bytes(2, byteorder='little', signed=True))  # number of rightId
        storage.extend((200).to_bytes(2, byteorder='little', signed=True))  # number of rightId
        storage.extend((2000).to_bytes(2, byteorder='little', signed=True))  # number of rightId

if __name__ == '__main__':
    unittest.main()


import logging
import os
import sys
import platform
import pprint
import subprocess
import traceback
import getpass
import shutil
from glob import glob
import urllib

def run_randoop(javac_commands):
	pp = pprint.PrettyPrinter(indent=2)
	i = 0
	for jc in javac_commands:
		javac_switches = jc['javac_switches']
		cp = javac_switches['classpath']
		class_file_dir = javac_switches['d']
		class_files = [y for x in os.walk(class_file_dir) for y in glob(os.path.join(x[0], '*.class'))]

		if len(class_files)==0:
			continue

		out_dir_name = "__randoop_%04d" % (i)
		if not os.path.exists(out_dir_name):
			os.makedirs(out_dir_name)

		class_files_file_name = os.path.join(out_dir_name, 'class_files.txt')
		print ("Creating list of files %d in %s." % (len(class_files), os.path.abspath(out_dir_name)) )
		with open(class_files_file_name, mode='w') as myfile:
			for class_file_name in class_files:
				myfile.write(get_qualified_class_name_from_file(class_file_name, class_file_dir))
				myfile.write(os.linesep)

		(randoop_jar, junit_jar, hamcrest_jar) = find_or_download_jars()

		cp_entries = cp.split(os.pathsep)
		clean_cp = list()
		clean_cp.append(randoop_jar)
		clean_cp.append(class_file_dir)

		lib_dir_name = "__randoop_libs"
		if not os.path.exists(lib_dir_name):
			os.makedirs(lib_dir_name)

		for cp_entry in cp_entries:
			if cp_entry.endswith(".jar"):
				#check if the jar is in a sub directory of the project.
				#if not copy it into a new subdirectory and add the
				#corresponding classpath entry.
				if not os.path.realpath(cp_entry).startswith(os.getcwd()):
					new_jar_name = os.path.join(lib_dir_name, os.path.basename(cp_entry))
					if not os.path.isfile(new_jar_name):
						shutil.copyfile(cp_entry, new_jar_name)
					clean_cp.append(new_jar_name)
				else:
					clean_cp.append(cp_entry)
				pass
			else:
				#todo what happens here?
				clean_cp.append(cp_entry)

		randoop_cmd = ['java', '-ea', '-classpath', os.pathsep.join(clean_cp),
				"randoop.main.Main", "gentests", "--classlist=%s"%class_files_file_name,
				"--timelimit=60", "--silently-ignore-bad-class-names=true",
				"--junit-output-dir=%s"%out_dir_name]

		junit_cp = list(clean_cp)
		junit_cp.append(junit_jar)
		junit_build_cmd = ['javac', '-classpath', os.pathsep.join(junit_cp), os.path.join(out_dir_name, 'RandoopTest*.java'), '-d', out_dir_name]

		junit_run_cp = list(junit_cp)
		junit_run_cp.append(hamcrest_jar)
		junit_run_cp.append(out_dir_name)

		junit_run_cmd = ['java', '-classpath', os.pathsep.join(junit_run_cp), "org.junit.runner.JUnitCore", 'RandoopTest']

		bash_script_name = "run_randoop_%04d.sh" % (i)
		with open(bash_script_name, mode='w') as myfile:
			myfile.write("#!/bin/bash\n")
			myfile.write("echo \"Run Randoop\"\n")
			myfile.write(" ".join(randoop_cmd))
			myfile.write("\n")
			myfile.write("echo \"Build tests\"\n")
			myfile.write(" ".join(junit_build_cmd))
			myfile.write("\n")
			myfile.write("echo \"Run tests\"\n")
			myfile.write(" ".join(junit_run_cmd))
			myfile.write("\n")
		print ("Written script to %s" % bash_script_name)

		i += 1

def get_qualified_class_name_from_file(class_file_name, class_file_path):
	""" terrible hack for now """
	suffix = class_file_name.replace(class_file_path+os.sep, "")
	mid_section = suffix.replace(".class", "")
	return mid_section.replace(os.sep, ".")

def find_or_download_jars():
	'''
		Finds or downloads the randoop, junit, and hamrest jars.
	'''
	randoop_jar_dir = os.path.join(os.getcwd(), '__randoop_files')
	if not os.path.isdir(randoop_jar_dir):
		os.makedirs(randoop_jar_dir)

	randoop_jar = os.path.join(randoop_jar_dir, "randoop-2.0.jar")
	if not os.path.isfile(randoop_jar):
		print("Downloading randoop to %s" % randoop_jar )
		urllib.urlretrieve ("https://github.com/randoop/randoop/releases/download/v2.0/randoop-2.0.jar", randoop_jar)

	junit_jar = os.path.join(randoop_jar_dir, "junit-4.12.jar")
	if not os.path.isfile(junit_jar):
		print("Downloading junit to %s" % junit_jar )
		urllib.urlretrieve ("https://github.com/junit-team/junit/releases/download/r4.12/junit-4.12.jar", junit_jar)

	hamcrest_jar = os.path.join(randoop_jar_dir, "hamcrest-core-1.3.jar")
	if not os.path.isfile(hamcrest_jar):
		print("Downloading hamcrest to %s" % hamcrest_jar )
		urllib.urlretrieve ("http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", hamcrest_jar)

	return (randoop_jar, junit_jar, hamcrest_jar)


import rev_index_defs
import preprocess
from random import choice

def vocab(all_eps_lems):
    word_eps = rev_index_defs.reverse_index(all_eps_lems) #слово: эпизоды с ним (эпизоды повторяются)
    word_eps_nodups = rev_index_defs.reverse_index_nodups(word_eps) #слово: эпизоды с ним (эпизоды не повторяются)
    srtd_w_counts = {key: len(value) for key, value in sorted(word_eps.items(), key=lambda item: len(item[1]), reverse=True)}#слово: частота встречаемости
    #составила отдельный словарь с друзьями и частотой их встречаемости
    #некоторые вариации имен не были включены, т.к. не были найдены в ключах, скорее всего, из-за лемматизации
    friends = {
    "Моника": srtd_w_counts['моника'], "Рэйчел": srtd_w_counts['рэйчел'] + srtd_w_counts['рейч'],
    "Чендлер": srtd_w_counts['чендлер'] + srtd_w_counts['чэндлер'] + srtd_w_counts['чен'],
    "Фиби": srtd_w_counts['фиби'] + srtd_w_counts['фибс'], "Росс": srtd_w_counts['росс'], "Джоуи": srtd_w_counts['джо']
    }

    most_freq_w = preprocess.get_key(srtd_w_counts, max(srtd_w_counts.values()))
    print(f"a) Самое частотное слово '{most_freq_w}' встретилось {max(srtd_w_counts.values())} раз")

    rare_words = [w for w, num_eps in srtd_w_counts.items() if num_eps == 1]
    print("b) Кол-во слов, встретившихся один раз: ", len(rare_words))
    print("Одно из редких слов: ", choice(rare_words))

    print("c) Слова, которые есть во всех документах: ")
    for word, eps in word_eps_nodups.items():
        #если кол-во эпизодов со словом совпадает с кол-вом эпизодов
        if len(eps) == len(all_eps_lems): #len(dict) показывает кол-во ключей словаря
            print("  ", word)

    most_freq_fr = preprocess.get_key(friends, max(friends.values()))
    print(f"d) Самый популярный статистически персонаж {most_freq_fr} встретился {max(friends.values())} раз")
    return 1


#! python3
# coding: latin-1

import telebot
from telebot import types
import collections
import re

bot = telebot.TeleBot("TOKEN")

updates = bot.get_updates()

user_dic={}

# Site with updates: https://api.telegram.org/botTOKEN/getUpdates

# Handles all text messages that contains the commands '/start' or '/help'.
@bot.message_handler(commands=['start'])
def handle_start(message):
    bot.reply_to(message, "This is a start")
    print(message.chat.username)
    bot.send_message(message.chat.id, 'text')
    # Local de casa
    bot.send_location(message.chat.id, -3.7413135, -38.5031922)

# Remove political post's
@bot.message_handler(regexp="\w*bolsonaro\w*|\w*Bolsonaro\w*|\w*Lula\w*|\w*lula\w*")
def political_posts_remove(message):
    #bot.delete_message(message.chat.id, message.message_id)
    bot.send_message(message.chat.id, 'Por favor, sem discussões políticas no grupo!')

# Remove political post's
@bot.message_handler(regexp="contar \w*")
def contadora(message):
    try:
        frase = message.text
        palavra = re.findall('contar (\w*)', frase)[0]
        print(list(palavra))
        bot.reply_to(message, "Número de letras: " + str(len(list(palavra))))
    except:
        pass

# ----- GET USER ANSWERS ------
@bot.message_handler(commands=['help'])
def handle_help(message):
    markup = types.ReplyKeyboardMarkup(row_width=2, resize_keyboard=True, one_time_keyboard=True)
    itembtn1 = types.KeyboardButton('a')
    itembtn2 = types.KeyboardButton('v')
    itembtn3 = types.KeyboardButton('d')
    itembtn4 = types.KeyboardButton('x')
    markup.add(itembtn1, itembtn2, itembtn3, itembtn4)
    msg = bot.send_message(message.chat.id, "Choose one letter:", reply_markup=markup)
    bot.register_next_step_handler(msg, process_sex_step)

# Receive letter choice from HELP command
def process_sex_step(message):
    user_dic['sexo'] = message.text
    teste1 = bot.reply_to(message, "Qual seu nome: ")
    bot.register_next_step_handler(teste1, process_sex_step2)

def process_sex_step2(message):
    user_dic['nome'] = message.text
    bot.reply_to(message, user_dic['sexo']+user_dic['nome'])

#---------------------------------

# Information about admins and members
@bot.message_handler(commands=['admin', 'admins'])
def admins(message):
    lista_admin = []
    numero_membros = bot.get_chat_members_count(message.chat.id)
    admins = bot.get_chat_administrators(message.chat.id)
    bot.send_message(message.chat.id,'O grupo tem ' + str(numero_membros) + ' membros e ' + str(len(admins)) + ' administradores:')
    for admin in admins:
        print(admin.user.username)
        lista_admin.append(admin.user.first_name)
    lista_admin_txt = '\n'.join(lista_admin)
    bot.send_message(message.chat.id, str(lista_admin_txt))

# Get member who writes a message in a specific group
@bot.message_handler(func=lambda m: m.chat.id==-1001256130386)
def memberid(message):

    #Member Name and ID
    if str(message.from_user.first_name)!='None':
        primeiro_nome=str(message.from_user.first_name)
    else:
        primeiro_nome=' '
    if str(message.from_user.last_name)!='None':
        ultimo_nome=str(message.from_user.last_name)
    else:
        ultimo_nome=' '
    if str(message.from_user.username)!='None':
        username=str(message.from_user.username)
    else:
        username=''
    print(primeiro_nome+' '+ultimo_nome+' - '+str(message.from_user.id) + ' (@' + username+')')
    open('Usuarios.txt', 'a').write(primeiro_nome+' '+ultimo_nome+' - '+str(message.from_user.id) + ' (@' + username+')'+'\n')

    # Get number of occurrence by user
    lista_usuarios = open("Usuarios.txt", "r").readlines()
    ocorrencias_comum = collections.Counter(lista_usuarios).most_common()
    open('UsuariosEstatistica.txt', 'w').writelines(str(ocorrencias_comum))

# New group member
@bot.message_handler(func=lambda m: True, content_types=['new_chat_members'])
def new_member(message):
    bot.reply_to(message, 'Bem vindo! Já deu o cu hoje ?')
    print('olá '+str(message.new_chat_member.username))
    #bot.kick_chat_member(message.chat.id, message.new_chat_member.id, None)

# Removed group member
@bot.message_handler(func=lambda m: True, content_types=['left_chat_member'])
def remove_member(message):
    bot.reply_to(message, 'Bye')
    print('bye '+str(message.left_chat_member.username))

# Send my curriculum
#@bot.message_handler(func=lambda msg: msg.text == 'meu curriculo' or msg.text == 'meu currículo' or msg.text == 'currículo' or msg.text == 'curriculo')
@bot.message_handler(func=lambda msg: msg.text in ['currículo', 'meu currículo', 'curriculo','meu curriculo'])
def curriculo(message):
    doc = open('curriculo_luiseduardopompeu.pdf', 'rb')
    bot.send_document(message.chat.id, doc)

#------------ INLINE CODES -----------
@bot.inline_handler(lambda query: query.query == 'text')
def query_text(inline_query):
    try:
        r = types.InlineQueryResultArticle('1', 'Manhã', types.InputTextMessageContent('Bom dia!'))
        r2 = types.InlineQueryResultArticle('2', 'Tarde', types.InputTextMessageContent('Boa tarde!'))
        r3 = types.InlineQueryResultArticle('3', 'Noite', types.InputTextMessageContent('Boa noite!'))
        bot.answer_inline_query(inline_query.id, [r, r2, r3], cache_time=1)
    except Exception as e:
        print(e)

@bot.inline_handler(lambda query: query.query == 'cu')
def query_photo(inline_query):
    try:
        r = types.InlineQueryResultPhoto('1',
                                         'https://raw.githubusercontent.com/eternnoir/pyTelegramBotAPI/master/examples/detailed_example/kitten.jpg',
                                         'https://raw.githubusercontent.com/eternnoir/pyTelegramBotAPI/master/examples/detailed_example/kitten.jpg',
                                         input_message_content=types.InputTextMessageContent('hi'))
        r2 = types.InlineQueryResultPhoto('2',
                                          'https://raw.githubusercontent.com/eternnoir/pyTelegramBotAPI/master/examples/detailed_example/rooster.jpg',
                                          'https://raw.githubusercontent.com/eternnoir/pyTelegramBotAPI/master/examples/detailed_example/rooster.jpg',
                                         input_message_content=types.InputTextMessageContent('hi2'))
        bot.answer_inline_query(inline_query.id, [r, r2], cache_time=1)
    except Exception as e:
        print(e)

@bot.inline_handler(lambda query: len(query.query) == 0)
def default_query(inline_query):
    try:
        r = types.InlineQueryResultArticle('1', 'Já Deram o Cu Hoje', types.InputTextMessageContent('Já Deram o Cu hoje?'))
        bot.answer_inline_query(inline_query.id, [r])
    except Exception as e:
        print(e)

# Handles all sent documents and audio files
@bot.message_handler(content_types=['document', 'audio'])
def handle_docs_audio(message):
    pass

# Handles all text messages that match the regular expression
@bot.message_handler(regexp="\w*prova\w*")
def handle_message(message):
    bot.reply_to(message, "Enfia a prova no cu")
    print(message.text)

# Handlers can be stacked to create a function which will be called if either message_handler is eligible
# This handler will be called if the message starts with '/hello' OR is some emoji
@bot.message_handler(commands=['hello'])
@bot.message_handler(func=lambda msg: msg.text == 'testando se funciona')
def send_something(message):
    bot.reply_to(message, "Funciona sim fdp!")

# Working with entities - url, email
@bot.message_handler(func=lambda m: True)
def entities(message):
    try:
        if message.entities[0].type=='url':
            print(message.text)
        if message.entities[0].type=='email':
            print(message.text)
    except:
        pass

bot.polling()


import pandas as pd
import split_words

def get_data(category: str):
    input_file_path = "../data/{0}.tsv".format(category)
    return pd.read_csv(input_file_path, sep='\t')

def select_verified_purchase(src: pd.DataFrame):
    l1 = src.shape[0]
    for i, x in src.iterrows():
        if not (x['verified_purchase'] == 'Y' or x['verified_purchase'] == 'y'):
            src.drop(i, inplace=True)
    l2 = src.shape[0]
    print('{0} reviews are deleted by select_verified_purchase'.format(l1 - l2))
    return src

def force_clean_same(src: pd.DataFrame):
    l1 = src.shape[0]
    src.drop_duplicates(subset=['review_headline', 'review_body'], inplace=True)
    l2 = src.shape[0]
    print("{0} reviews are deleted by force_clean_same".format(l1 - l2))
    return src

def compress_repeat_sentence(src: pd.DataFrame):
    for i, x in src.iterrows():
        review = x['review_body']
        src.loc[i, 'review_body'] = split_words.modify_review(review)
    return src

def clean_shout_review(src: pd.DataFrame):
    l1 = src.shape[0]
    for i, x in src.iterrows():
        if len(x['review_body']) < 15:
            src.drop(i, inplace=True)
    l2 = src.shape[0]
    print('{0} reviews are deleted by clean_shout_review'.format(l1 - l2))
    return src

def save_date(src: pd.DataFrame, category: str):
    output_file_path = "../data/save/{0}_cleaned.csv".format(category)
    src = pd.DataFrame(src[['star_rating', 'review_body', 'vine', 'helpful_votes']])
    src.to_csv(output_file_path, index=False)

category = 'microwave'
data = get_data(category)
# 筛选确认购买的
out = select_verified_purchase(data)
# 筛选完全相同的评论
out = force_clean_same(out)
# 单条评论去重
# out = compress_repeat_sentence(out)
# 筛选评论长度过短的
out = clean_shout_review(out)

save_date(out, 'microwave')


import multiprocessing

bind = "0.0.0.0:8864"
workers = 1
# accesslog = "-"
# errorlog = "./error.log"
# loglevel = "error"
# worker_class = "egg:meinheld#gunicorn_worker"
max_requests = 1000
timeout = 30
pidfile = "gunicorn_pid"


from collections import defaultdict

from odoo import api, fields, models, tools, _
from odoo.addons import decimal_precision as dp
from odoo.addons.stock_landed_costs.models import product
from odoo.exceptions import UserError
from lxml import etree
import time
from odoo.tools import float_is_zero
from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT
# from odoo.osv.orm import setup_modifiers
from collections import namedtuple

class PosOrder(models.Model):
    _inherit = "pos.order"

    def create_picking(self):
        """Create a picking for each order and validate it."""
        Picking = self.env['stock.picking']
        Move = self.env['stock.move']
        StockWarehouse = self.env['stock.warehouse']
        for order in self:
            if not order.lines.filtered(lambda l: l.product_id.type in ['product', 'consu']):
                continue
            address = order.partner_id.address_get(['delivery']) or {}
            picking_type = order.picking_type_id
            return_pick_type = order.picking_type_id.return_picking_type_id or order.picking_type_id
            order_picking = Picking
            return_picking = Picking
            moves = Move
            location_id = order.location_id.id
            if not location_id:
                location_id = order.picking_type_id.default_location_src_id.id
            if order.partner_id:
                destination_id = order.partner_id.property_stock_customer.id
            else:
                if (not picking_type) or (not picking_type.default_location_dest_id):
                    customerloc, supplierloc = StockWarehouse._get_partner_locations()
                    destination_id = customerloc.id
                else:
                    destination_id = picking_type.default_location_dest_id.id

            if picking_type:

                message = _("This transfer has been created from the point of sale session: %s") % (order.id, order.name)
                picking_vals = {
                    'origin': order.name,
                    'partner_id': address.get('delivery', False),
                    'date_done': order.date_order,
                    'picking_type_id': picking_type.id,
                    'company_id': order.company_id.id,
                    'move_type': 'direct',
                    'note': order.note or "",
                    'location_id': location_id,
                    'location_dest_id': destination_id,
                }
                pos_qty = any([x.qty > 0 for x in order.lines if x.product_id.type in ['product', 'consu']])
                if pos_qty:
                    order_picking = Picking.create(picking_vals.copy())
                    order_picking.message_post(body=message)
                neg_qty = any([x.qty < 0 for x in order.lines if x.product_id.type in ['product', 'consu']])
                if neg_qty:
                    return_vals = picking_vals.copy()
                    return_vals.update({
                        'location_id': destination_id,
                        'location_dest_id': return_pick_type != picking_type and return_pick_type.default_location_dest_id.id or location_id,
                        'picking_type_id': return_pick_type.id
                    })
                    return_picking = Picking.create(return_vals)
                    return_picking.message_post(body=message)

            for line in order.lines.filtered(lambda l: l.product_id.type in ['product', 'consu'] or l.product_id.is_composition == True):
                if line.product_id.is_composition:
                    qty = line.qty
                    if qty <= 0:
                        qty = abs(line.qty * -1)
                    else:
                        qty = abs(line.qty)

                    for dd in line.product_id.composition_product_id:

                        # if dd.product_id.is_composition:

                        #     for ddd in dd.product_id.composition_product_id:
                        #         moves |= Move.create(self.create_composition_product(line, ddd, order_picking, return_picking, qty, picking_type, return_pick_type, location_id, destination_id))

                        # else:
                        moves |= Move.create({
                            'name': line.name,
                            'product_uom': dd.uom_id.id, ##dd.uom_id.id,
                            'picking_id': order_picking.id if dd.product_quantity * line.qty >= 0 else return_picking.id,
                            'picking_type_id': picking_type.id if dd.product_quantity * line.qty >= 0 else return_pick_type.id,
                            'product_id': dd.product_id.id,
                            'product_uom_qty': abs(dd.product_quantity) * qty * (line.uom_id.factor_inv if line.uom_id.uom_type == 'bigger' else line.uom_id.factor) ,
                            'state': 'draft',
                            'location_id': location_id if dd.product_quantity * qty >= 0 else destination_id,
                            'location_dest_id': destination_id if dd.product_quantity * qty >= 0 else return_pick_type != picking_type and return_pick_type.default_location_dest_id.id or location_id,
                        })

                        # if line.product_id.type == 'product':
                        #     moves |= Move.create({
                        #         'name': line.name,
                        #         'product_uom': line.uom_id.id, #line.product_id.uom_id.id,
                        #         'picking_id': order_picking.id if line.qty >= 0 else return_picking.id,
                        #         'picking_type_id': picking_type.id if line.qty >= 0 else return_pick_type.id,
                        #         'product_id': line.product_id.id,
                        #         'product_uom_qty': abs(line.qty),
                        #         'state': 'draft',
                        #         'location_id': location_id if line.qty >= 0 else destination_id,
                        #         'location_dest_id': destination_id if line.qty >= 0 else return_pick_type != picking_type and return_pick_type.default_location_dest_id.id or location_id,
                        #     })
                else:
                    moves |= Move.create({
                        'name': line.name,
                        'product_uom': line.uom_id.id, #line.product_id.uom_id.id,
                        'picking_id': order_picking.id if line.qty >= 0 else return_picking.id,
                        'picking_type_id': picking_type.id if line.qty >= 0 else return_pick_type.id,
                        'product_id': line.product_id.id,
                        'product_uom_qty': abs(line.qty),
                        'state': 'draft',
                        'location_id': location_id if line.qty >= 0 else destination_id,
                        'location_dest_id': destination_id if line.qty >= 0 else return_pick_type != picking_type and return_pick_type.default_location_dest_id.id or location_id,
                    })

            # prefer associating the regular order picking, not the return
            order.write({'picking_id': order_picking.id or return_picking.id})

            if return_picking:
                order._force_picking_done(return_picking)
            if order_picking:
                order._force_picking_done(order_picking)

            # when the pos.config has no picking_type_id set only the moves will be created
            if moves and not return_picking and not order_picking:
                moves._action_assign()
                moves.filtered(lambda m: m.product_id.tracking == 'none')._action_done()

        return True

    # def create_composition_product(self, line, ddd, order_picking, return_picking, qty, picking_type, return_pick_type, location_id, destination_id):

    #     return {
    #             'name': line.name,
    #             'product_uom': line.uom_id.id, #ddd.uom_id.id,
    #             'picking_id': order_picking.id if ddd.product_quantity * qty >= 0 else return_picking.id,
    #             'picking_type_id': picking_type.id if ddd.product_quantity * qty >= 0 else return_pick_type.id,
    #             'product_id': ddd.product_id.id,
    #             'product_uom_qty': abs(ddd.product_quantity) * qty,
    #             'state': 'draft',
    #             'location_id': location_id if ddd.product_quantity * qty >= 0 else destination_id,
    #             'location_dest_id': destination_id if ddd.product_quantity * qty >= 0 else return_pick_type != picking_type and return_pick_type.default_location_dest_id.id or location_id,
    #         }

    def set_pack_operation_lot(self, picking=None):
        """Set Serial/Lot number in pack operations to mark the pack operation done."""

        StockProductionLot = self.env['stock.production.lot']
        PosPackOperationLot = self.env['pos.pack.operation.lot']
        has_wrong_lots = False
        for order in self:
            for move in (picking or self.picking_id).move_lines:
                picking_type = (picking or self.picking_id).picking_type_id
                lots_necessary = True
                if picking_type:
                    lots_necessary = picking_type and picking_type.use_existing_lots
                qty_done = 0
                pack_lots = []
                pos_pack_lots = PosPackOperationLot.search([('order_id', '=', order.id), ('product_id', '=', move.product_id.id)])

                if pos_pack_lots and lots_necessary:
                    for pos_pack_lot in pos_pack_lots:
                        stock_production_lot = StockProductionLot.search([('name', '=', pos_pack_lot.lot_name), ('product_id', '=', move.product_id.id)])
                        if stock_production_lot:
                            # a serialnumber always has a quantity of 1 product, a lot number takes the full quantity of the order line
                            qty = 1.0
                            if stock_production_lot.product_id.tracking == 'lot':
                                qty = abs(pos_pack_lot.pos_order_line_id.qty)
                            qty_done += qty
                            quant = stock_production_lot.quant_ids.filtered(lambda q: q.quantity > 0.0 and q.location_id.parent_path.startswith(move.location_id.parent_path))[-1:]
                            pack_lots.append({'lot_id': stock_production_lot.id, 'quant_location_id': quant.location_id.id, 'qty': qty})
                        else:
                            has_wrong_lots = True
                elif move.product_id.tracking == 'none' or not lots_necessary:
                    qty_done = move.product_uom_qty
                else:
                    move_line_lot = StockProductionLot.search([('product_id', '=', move.product_id.id)])
                    move_line_lot = move_line_lot.sorted(key=lambda r: r.removal_date).ids

                    st_move_line = move._get_move_lines()
                    st_move_line_q = st_move_line.filtered(lambda x: x.product_id.id == move.product_id.id)

                    for m_l_lot in st_move_line:
                        if m_l_lot.lot_id.id in move_line_lot:
                            if move.product_uom_qty == sum(st_move_line_q.mapped('product_uom_qty')):
                                m_l_lot.qty_done = m_l_lot.product_uom_qty
                            has_wrong_lots = False
                        else:
                            has_wrong_lots = True
                for pack_lot in pack_lots:
                    lot_id, quant_location_id, qty = pack_lot['lot_id'], pack_lot['quant_location_id'], pack_lot['qty']
                    self.env['stock.move.line'].create({
                        'picking_id': move.picking_id.id,
                        'move_id': move.id,
                        'product_id': move.product_id.id,
                        'product_uom_id': move.product_uom.id,
                        'qty_done': qty,
                        'location_id': quant_location_id or move.location_id.id,
                        'location_dest_id': move.location_dest_id.id,
                        'lot_id': lot_id,
                    })
                if not pack_lots and not float_is_zero(qty_done, precision_rounding=move.product_uom.rounding):
                    if len(move._get_move_lines()) < 2:
                        move.quantity_done = qty_done
                    else:
                        move._set_quantity_done(qty_done)

        return has_wrong_lots


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 14 11:52:40 2019

@author: dangngochuy
"""

#Import libraries
import numpy as np
import matplotlib.pyplot as plt #plot charts
import pandas as pd #best to import and manage datasets

#Preprocess data
dataset = pd.read_csv("Position_Salaries.csv")
X = dataset.iloc[:, 1:2].values
y = dataset.iloc[:, 2].values

#Fitting Decision Tree  to the dataset
from sklearn.tree import DecisionTreeRegressor
regressor = DecisionTreeRegressor(random_state = 0)
regressor.fit(X, y)

#Predict new results with Linear Regression
y_pred = regressor.predict([[6.5]])

#Visualize the Decision Tree results
X_grid = np.arange(min(X), max(X), 0.01)
X_grid = X_grid.reshape((len(X_grid), 1))
plt.scatter(X,y)
plt.plot(X_grid, regressor.predict(X_grid), color = "blue")
plt.title("Truth or Bluff Decision Tree")
plt.xlabel("Position Label")
plt.ylabel("Salary")
plt.show()


# There is one table in the file https://stepik.org/media/attachments/lesson/209723/5.html.
# Sum all the numbers in it. Now we have not only added different tags to change the display style,
# but also rendered invalid HTML (although browsers render it, but BeautifulSoup may have problems).
# Invalid HTML code is not uncommon on the Internet, you need to learn to work with this too.
#
# You can fix the html code or try using a non-standard html parser like html5lib.

# В файле https://stepik.org/media/attachments/lesson/209723/5.html находится одна таблица.
# Просуммируйте все числа в ней. Теперь мы не только добавили разных тегов для изменения стиля отображения,
# но и сделали невалидный HTML-код (правда, браузеры его отображают, а вот с BeautifulSoup могут быть проблемы).
# Невалидный HTML-код - не редкость в интернете, надо учиться работать и с этим.
#
# Вы можете исправить html-код или попробовать использовать нестандартный парсер html, такой как html5lib.

from urllib.request import urlopen, urlretrieve
from bs4 import BeautifulSoup

resp = urlopen('https://stepik.org/media/attachments/lesson/209723/4.html')  # downloading file / скачиваем файл
html = resp.read().decode('utf8')  # read the input / считываем содержимое
soup = BeautifulSoup(html, 'html.parser')  # making a soup / делаем суп

table = soup.find('table', attrs={'class': 'wikitable sortable'})

sum = 0
for tag in soup.find_all('td'):
    sum += int(tag.string)
print(sum)

PythonForPracticaProblems/Module 1 - Web pages scraping/Module 1 Step 4 Task 2 - Using BeautifulSoup .py


import icons
import misc
import n_stealable
import n_unit
import vec

class Factory(n_stealable.Stealable):
  icon = icons.Factory

  def __init__(self, pos, owner, bonus_defense, build_per_turn):
    super(Factory, self).__init__(pos, owner, bonus_defense)
    self.build_per_turn = build_per_turn
    self.progress = 0
    self.cost = 50

  def EndOfTurnUpdate(self, game_state):
    if self.control:
      self.progress += self.build_per_turn
      if self.progress > self.cost:
        self.BuildOne(game_state)
        self.progress = 0

    super(Factory, self).EndOfTurnUpdate(game_state)

  def BuildOne(self, game_state):
    where = game_state.EmptySquareNear(self.pos.x, self.pos.y)
    if where:
      self.PlaceUnitAt(game_state, where[0], where[1])

  def PlaceUnitAt(self, game_state, tx, ty):
    unit = n_unit.Unit(
      vec.Vec(tx, ty), game_state.ai_owner, False,
      icons.KillerRobot, 10, 40, 4, True)
    game_state.AddNode(unit)

  def __attrrepr__(self):
    return (super(Factory, self).__attrrepr__()
            + (', per_turn=%i, building=%r, target=%r, progress=%i'
               % (self.build_per_turn, self.building, self.target,
                  self.progress)))

  def Description(self):
    return 'factory'


from turtle import Turtle

NUMBER_OF_ROWS = 3
colors = ["red", "orange", "yellow"]

class LayersOfTurtle:

    def __init__(self):
        self.x = 290
        self.y = 150
        self.turtles = []
        self.create_layers()

    def create_layers(self):
        for color in range(NUMBER_OF_ROWS):
            self.x = 290
            while self.x >= -330:
                turtle = Turtle("turtle")
                turtle.speed(10)
                current_color = colors[color % len(colors)]
                turtle.color(current_color)
                turtle.penup()
                turtle.turtlesize(2)
                turtle.goto(self.x, self.y)
                self.turtles.append(turtle)
                self.x -= 50
            self.y -= 50

    def get_number_of_turtles(self):
        return len(self.turtles)