# Есть два списка:
# tutors = [
#     'Иван', 'Анастасия', 'Петр', 'Сергей',
#     'Дмитрий', 'Борис', 'Елена'
# ]
# klasses = [
#     '9А', '7В', '9Б', '9В', '8Б', '10А', '10Б', '9А'
# ]
#
#
# Необходимо реализовать генератор, возвращающий кортежи вида
# (<tutor>, <klass>), например:
# ('Иван', '9А')
# ('Анастасия', '7В')
# ...
#
# Количество генерируемых кортежей не должно быть больше длины списка tutors.
# Если в списке klasses меньше элементов, чем в списке tutors, необходимо
# вывести последние кортежи в виде: (<tutor>, None), например:
# ('Станислав', None)
#
# Доказать, что вы создали именно генератор. Проверить его работу вплоть до
# истощения. Подумать, в каких ситуациях генератор даст эффект.
from itertools import zip_longest

tutors = ['Иван', 'Анастасия', 'Петр', 'Сергей',
          'Дмитрий', 'Борис', 'Елена', 'Павл', 'Михаил']
klasses = ['9А', '7В', '9Б', '9В', '8Б',
           '10А', '10Б', '9А']

gen_tutors_classes = ((tutors, klasses) for tutors, klasses in zip_longest(tutors, klasses))
print(gen_tutors_classes)
print(*gen_tutors_classes)
print(*gen_tutors_classes)

# доказательство, что я создал генератор <generator object <genexpr> at 0x0000025707A03970>
# и второй принт не выводится, так как генератор исчерпал себя

# второй вариант создания генератора
# def tutors_klasses(tutors, klasses):
#     for el in zip_longest(tutors, klasses):
#         yield el
#
#
# print(tutors_klasses(tutors, klasses))
# доказательство, что я создал генератор <generator object tutor_klass at 0x0000024C472655F0>

# gen_tutors_klasses = tutors_klasses(tutors, klasses)
# добавим генератор в переменную, что бы было проще проверить в python console


# chaper6 : dictionary in list
print('Dict in list:')
alien_0 = {'color':'green','points':5}
alien_1 = {'color':'yellow','points':10}
alien_2 = {'color':'red','points':15}

aliens = [alien_0, alien_1, alien_2]

for alien in aliens:
    print(alien)

# aliens in list
aliens = []
# create new aliens with a loop
for alien_number in range(30):
    new_alien = {'color':'green','point':5,'speed':'slow'}
    aliens.append(new_alien)# add new_alien to the end of list: aliens

for alien in aliens[:5]:
    if alien['color'] == 'green':
        alien['color'] = 'yellow'
        alien['point'] = 10
        alien['speed'] = 'medium'

for alien in aliens[:10]:
    print(alien)
print('...')

print('Total number of aliens: '+str(len(aliens)))


#! /usr/bin/motiosense/bin/python

from slackclient import SlackClient

import os
import re
import sys
import time
import socket
import logging

# instantiate Slack client
slack_client = SlackClient(os.environ.get('SLACK_BOT_TOKEN'))
# motiosense's user ID in Slack: value is assigned after the bot starts up
motiosense_id = None

host = ''  # listen on all addresses (i.e. 0.0.0.0)
port = 52431
sensor_status = "Undetermined"

logging_file = "/tmp/motiosense.log"
FORMAT = "%(asctime)-15s %(message)s"

# constants
RTM_READ_DELAY = 1  # 1 second delay between reading from RTM
COMMANDS = {
    "available?",
    "occupied?",
    "last occupied?",
    "status?"
}
MENTION_REGEX = "^<@(|[WU].+?)>(.*)"

def parse_bot_commands(slack_events):
    """
        Parses a list of events coming from the Slack RTM API to find bot commands.
        If a bot command is found, this function returns a tuple of command and channel.
        If its not found, then this function returns None, None.
    """
    for event in slack_events:
        if event["type"] == "message" and not "subtype" in event:
            user_id, message = parse_direct_mention(event["text"])
            if user_id == motiosense_id:
                return message, event["channel"]
    return None, None

def parse_direct_mention(message_text):
    """
        Finds a direct mention (a mention that is at the beginning) in message text
        and returns the user ID which was mentioned. If there is no direct mention, returns None
    """
    matches = re.search(MENTION_REGEX, message_text)
    # the first group contains the username, the second group contains the remaining message
    return (matches.group(1), matches.group(2).strip()) if matches else (None, None)

def handle_command(command, channel):
    """
        Executes bot command if the command is known
    """
    # Default response is help text for the user
    response = "Not sure what you mean. Try one of these: {}.".format(
        list(COMMANDS))

    # Ensures a valid command was used and fills in the response accordingly
    if command in COMMANDS:
        # a valid command was used
        if command == "available?" or command == "occupied?":
            response = "Sensor status is _%s_" % sensor_status.upper()
        elif command == "last occupied?":
            response = "Not implemented yet. Sensor status is currently _%s_" % sensor_status.upper()
        elif command == "status?":
            # TODO: respond with all known information (last occupied, available/occupied)
            response = "Sensor status is _%s_" % sensor_status.upper()

    # Sends the response back to the channel
    slack_client.api_call(
        "chat.postMessage",
        channel=channel,
        text=response
    )

if __name__ == "__main__":
    logging.basicConfig(filename=logging_file, format=FORMAT, level="DEBUG")

    try:
        # Setup a socket to listen for status from the sensor
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(1)
        sock.bind((host, port))
        sock.listen(1)
    except socket.error as e:
        logging.error(e)
        sys.exit(1)

    if slack_client.rtm_connect(with_team_state=False):
        logging.info("Motiosense connected and running!")
        # Read bot's user ID by calling Web API method `auth.test`
        motiosense_id = slack_client.api_call("auth.test")["user_id"]

        while True:
            try:
                conn, addr = sock.accept()
                try:
                    while True:
                        # receive the sensor status from the buffer
                        data = conn.recv(16)
                        if data:
                            # update the sensor_status
                            sensor_status = data.decode()
                        else:
                            break
                finally:
                    conn.close()
            except socket.timeout as e:
                continue
            except socket.error as e:
                logging.error(e)
                continue
            finally:
                command, channel = parse_bot_commands(slack_client.rtm_read())
                if command:
                    handle_command(command, channel)
                time.sleep(RTM_READ_DELAY)
    else:
        logging.debug(
            "Slack RTM connect failed.")
        sys.exit(1)


from farm.extensions import celery
from celery.utils.log import get_task_logger

from farm.models import db
from farm.models.reservations import Reservation

import farm.tasks.testbench as tb_tasks
import json
import time
from datetime import datetime, timedelta

logger = get_task_logger(__name__)

@celery.task()
def generate_uuid():
    import uuid
    return uuid.uuid4()

@celery.task()
def list_reservations():
    reservations = Reservation.query.all()
    return reservations

@celery.task()
def write_to_reservation_db(uuid, reservation_info):
    result = False
    try:
        reservation = Reservation(reservation_info=reservation_info, uuid=str(uuid))
        db.session.add(reservation)
        db.session.commit()
        result = True
    except:
        logger.info("DB may be locked due to external operations. Reservation discarded")

    return result

@celery.task()
def update_reservation_db(uuid, reservation_info):
    result = False
    try:
        reservation = Reservation.query.filter_by(uuid=uuid).update(dict(reservation_info=reservation_info))
        db.session.commit()
        result = True
    except:
        print("Error - Updating UUID: %s has failed" % uuid)

    return result

@celery.task()
def date_now(date_key):
    return {date_key: str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))}

@celery.task()
def state_json(state):
    return {"state": state}

@celery.task()
def create_reservation(request, user, uuid=None):
    reservation_info = {}

    try:
        if uuid == None:
            uuid = str(generate_uuid())

        # Write state to be IN PROGRESS
        state = state_json("IN PROGRESS")
        reservation_info.update(state)

        # Write request into json
        reservation_info.update({"request":request})

        # Write creation_date
        creation_date = date_now("creation_date")
        reservation_info.update(creation_date)

        # Write user - path to user name
        user_info = {'info' : {'user': {'name':user}}}
        reservation_info.update(user_info)

        # First write to the reservation_db (an initial state)
        result = write_to_reservation_db(uuid=uuid, reservation_info=json.dumps(reservation_info))
        if result == False:
            print("Failed to initialize reservation with description and/or date")

        # Update reservation with testbed info if any
        reservation_info_update = reserve_testbed_by_request(request, user)
        reservation_info.update(reservation_info_update)

        # Write uuid into etcd under instance before getting info on testbed
        if reservation_info.get("error") == None:
            state = state_json("READY")

            tb_tasks.set_testbed_info(reservation_info["testbed"], 'instance', 'uuid', uuid)
            reservation_info["info"] = tb_tasks.get_testbed(reservation_info.get("testbed"))
        else:
            print(reservation_info.get("state"))
            if reservation_info.get("state") == "IN PROGRESS":
                state = state_json("ERROR")
            else:
                state = {"state":reservation_info.get("state")}

            # Declare end_date of the reservation
            end_date = date_now("end_date")
            reservation_info.update(end_date)

        reservation_info.update(state)

        # Second write to the reservation_db (an update state)
        result = update_reservation_db(uuid=uuid, reservation_info=json.dumps(reservation_info))
        if result == False:
            print("Failed to update reservation into the database")

    except ValueError as e:
        print(e)
    except:
        error = {'error': "Unexpected error in reserve_task.create_reservation"}
        reservation_info.update(error)
        print(error.get('error'))

        state = state_json("ERROR")
        reservation_info.update(state)
        try:
            # Try to display error into the reservation
            update_reservation_db(uuid=uuid, reservation_info=json.dumps(reservation_info))
        except:
            logger.info("Failed to update reservation")

@celery.task()
def reserve_testbed_by_request(request, user):
    error = None
    testbeds = tb_tasks.list_testbeds()

    if 'filters' not in request:
        error = {'error': "There is no 'filters' tag in the request"}
        return error

    # Initialize boolean array for all testbed to be true
    boolArray = {}
    for testbed in testbeds:
        boolArray[testbed] = True

    for filter in request["filters"]:
        for testbed in testbeds:
            if "testbed_id" in filter and request["filters"]["testbed_id"] == testbed:
                continue
            elif filter not in testbeds[testbed]['properties']:
                boolArray[testbed] = False
                continue
            elif request["filters"][filter] != testbeds[testbed]['properties'][filter]:
                boolArray[testbed] = False
                continue

    if not any(boolArray.values()):
        error = {'error': "No Matching filters"}
        return error

    matching_testbeds = set()
    for testbed in boolArray:
        if boolArray[testbed] == True:
            if is_launched_by_user(testbed, user):
                return {'testbed': testbed}
            else:
                matching_testbeds.add(testbed)

    # Retry for 10 seconds
    wait_until = datetime.now() + timedelta(seconds=10)
    while True:
        for testbed in matching_testbeds:
            time.sleep(0.5)
            if is_launched_by_user(testbed, user):
                return {'testbed': testbed}
        if wait_until < datetime.now():
            break

    error = {'error': "No testbed with the requirements are available at the moment",
             'state': "EXPIRED" }
    return error

class ReservationAlreadyDeleted(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

@celery.task()
def delete_reservation(reservation):
    deleted = False
    reservation_info = {}

    try:
        if is_reservation_error(reservation):
            raise Exception

        reservation_info = json.loads(reservation.reservation_info)
        testbed_id = reservation_info.get("testbed")

        tb_tasks.destroy(testbed_id)

        # Append the end_date and overwrite the reservation_info in the database
        if reservation_info.get("end_date") != None:
            raise ReservationAlreadyDeleted(reservation_info)
        else:
            end_date = date_now("end_date")
            reservation_info.update(end_date)

            reservation_info['state'] = "DELETED"
            result = update_reservation_db(uuid=reservation.uuid, reservation_info=json.dumps(reservation_info))
            if result == False:
                raise Exception("DB may be locked due to external operations. Updating UUID: %s has failed" % reservation.uuid)

            deleted = True
    except ReservationAlreadyDeleted:
        print("Reservation already deleted")
    except:
        print("Unexpected error occured when deleting reservation UUID: %s" % reservation.uuid)

    return deleted

@celery.task()
def is_launched_by_user(testbed_id, user):
    result = tb_tasks.launch(testbed_id, { 'name': user })
    if result.get("launched") == None:
        raise Exception("tb_tasks.launch returns result that is neither True or False")
    if result.get("launched") == True:
        return True
    return False

@celery.task()
def is_reservation_error(reservation):
    data = json.loads(reservation.reservation_info)

    if data.get("error") == None:
        return False
    return True


### Simple neural network code to train on the MNIST network
### single model version of the code
### requires Keras and TensorFlow
### based on the MNIST implementation tutorial from Keras
### can either just run code with no arguments which gives 1 hidden layer with 2000 nodes
# otherwise, can put arguments. each number is size of successive layer, so to get 3 hidden nodes
# with 4000, 1000, and 4000 nodes, call 'python cifar_network.py 4000 1000 4000'

from __future__ import print_function
import sys
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense

### hyperparameters to choose
# how many epochs to train for
epochs = 40
# can set 'activ = "relu"' to use ReLU instead
activ = "relu"
# what data set size to train on
numTrainSamples = 2000
# learning rate to use
learning_rate = .1

batch_size = 128
num_classes = 10
input_size = 784
layer_sizes = sys.argv[1:]
default_number_nodes = 2000

# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()

# shape data appropriately and normalize it
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255

## CHOOSE HOW MANY TRAINING / TEST SAMPLES
x_train = x_train[0 : numTrainSamples]
y_train = y_train[0 : numTrainSamples]

# print data shape / numbers
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)

model = Sequential()

## NUMBER OF NODES / LAYERS
# parse the input to create the model of appropriate size
if(len(layer_sizes) == 0):
	model.add(Dense(default_number_nodes, activation=activ, input_shape=(input_size,)))
else:
	model.add(Dense(int(layer_sizes[0]), activation=activ, input_shape=(input_size,)))
	for l in layer_sizes[1:]:
		model.add(Dense(int(l), activation=activ))

### output layer
model.add(Dense(num_classes, activation='softmax'))

# print shape of model
model.summary()

# actually construct it
opt = keras.optimizers.adam(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
              optimizer=opt,
              metrics=['accuracy'])

# train model
history = model.fit(x_train, y_train,
                    batch_size=batch_size,
                    epochs=epochs,
                    verbose=1,
                    validation_data=(x_test, y_test))

# print shape again
model.summary()
# print how well it did
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
print("Epochs: " + str(epochs))
print("Number of training samples: " + str(numTrainSamples))
print("Activation: " + activ)


# July 2020
# OBJECCTIVE: CREATE NICE PLOTS: CHECK https://matplotlib.org/gallery.html TO FIND THE NICE PLOTS
import numpy as np
import matplotlib.pyplot as plt

# 1. CREATE A FIGURE

my_first_figure = plt.figure("My First Figure")
# subplot
x = np.linspace(0,10,40)
sinx=np.sin(x)
cosx=np.cos(x)
subplot_1 = my_first_figure.add_subplot(2, 3, 1) # 2: Number of rows in the figure, 3: number of column, 1: subplot
plt.plot(sinx)
# Add a plot to specific position in the figure
subplot_6 = my_first_figure.add_subplot(2, 3, 6)
plt.plot(cosx)
#plt.show()

# 2. PLOT MULTIPLE LINES
plt.plot(x,sinx,x,cosx)
#plt.show()

# 3. PLOT IN ONE SIDE BY SIDE GRAPH
plt.subplot(2,1,1)
plt.plot(x,sinx)
plt.title('Two Graph')
plt.ylabel('Sin')

plt.subplot(2,1,2)
plt.plot(x,cosx,'ro')
plt.xlabel('x Value')
plt.ylabel('Cos')

# 4. CREATE HOSTOGRAMS
mu, sigma = 100, 15
data_set = mu + sigma * np.random.randn(10000)
n, bins, patches = plt.hist(data_set, 50, density=1, facecolor='g', alpha=0.75) # 50 bins with normalized value, green color and alphas is transparency
plt.xlabel('Smarts')
plt.ylabel('Probability')
plt.title('Histogram of IQ')
plt.text(60, .025, r'$\mu=100,\ \sigma=15$')
plt.axis([40, 160, 0, 0.03])
plt.grid(True)
plt.show()

# 5. PIE CHART
labels = 'Frogs', 'Hogs', 'Dogs', 'Logs'
sizes = [15, 30, 45, 10]
colors = ['yellowgreen', 'gold', 'lightskyblue', 'lightcoral']
explode = (0, 0.1, 0, 0)  # only "explode" the 2nd slice (i.e. 'Hogs'). Means the piece is seperated from the chart

plt.pie(x=sizes, explode=explode, labels=labels, colors=colors,
        autopct='%1.1f%%', shadow=True, startangle=90)
# Set aspect ratio to be equal so that pie is drawn as a circle.
plt.axis('equal')
plt.show()

# 6. BAR CHART
N = 5
menMeans = (20, 35, 30, 35, 27)
menStd = (2, 3, 4, 1, 2)

ind = np.arange(N)  # the x locations for the groups
width = 0.35       # the width of the bars

fig, ax = plt.subplots()
rects1 = ax.bar(ind, menMeans, width, color='r', yerr=menStd)

womenMeans = (25, 32, 34, 20, 25)
womenStd = (3, 5, 2, 3, 3)
rects2 = ax.bar(ind + width, womenMeans, width, color='y', yerr=womenStd)
# add some text for labels, title and axes ticks
ax.set_ylabel('Scores')
ax.set_title('Scores by group and gender')
ax.set_xticks(ind + width)
ax.set_xticklabels(('G1', 'G2', 'G3', 'G4', 'G5'))
ax.legend((rects1[0], rects2[0]), ('Men', 'Women'))
plt.show()


import uuid

import requests
from flask import session, render_template, redirect, url_for, Blueprint, request
import msal
import flaskr.app_config as app_config
from json import loads
from datetime import datetime, timedelta

bp = Blueprint('ad', __name__)

def tokenExpired():
    start = datetime.strptime(session['start'], '%h %d %H:%M:%S')
    return (datetime.now() - start > timedelta(hours=1))

@bp.route("/")
def index():
    print('\n\n\n'+session.get('user', 'no user found')+'\n\n\n')
    if not session.get("user"):
        return redirect(url_for("ad.login"))
    return render_template('index.html', user=session["user"], version=msal.__version__, active='home')

@bp.route('/token')
def token():
    return session.get('token', 'No token found')

@bp.route("/login")
def login():
    session["state"] = str(uuid.uuid4())
    # Technically we could use empty list [] as scopes to do just sign in,
    # here we choose to also collect end user consent upfront
    auth_url = build_auth_url(scopes=app_config.SCOPE, state=session["state"])
    print(auth_url)
    if 'redirect' in request.args:
        redir = request.args.get('redirect')
        session['redirect'] = redir
        print('\n\nSetting redirect to {}\n\n'.format(redir))
    return render_template("login.html", auth_url=auth_url, version=msal.__version__)

@bp.route(app_config.REDIRECT_PATH)  # Its absolute URL must match your app's redirect_uri set in AAD
def authorized():
    if request.args.get('state') != session.get("state"):
        return redirect(url_for("ad.index"))  # No-OP. Goes back to Index page
    if "error" in request.args:  # Authentication/Authorization failure
        return render_template("auth_error.html", result=request.args)
    if request.args.get('code'):
        cache = _load_cache()
        result = _build_msal_app(cache=cache).acquire_token_by_authorization_code(
            request.args['code'],
            scopes=app_config.SCOPE,  # Misspelled scope would cause an HTTP 400 error here
            redirect_uri=url_for("ad.authorized", _external=True))
        if "error" in result:
            return render_template("auth_error.html", result=result)
        claims = result.get("id_token_claims")
        name = claims['name'].split(', ')
        try:
            session["user"] = name[1] + ' ' + name[0]
        except IndexError:
            session["user"] = claims['name']
        session['start'] = datetime.now().strftime('%h %d %H:%M:%S')
        session['email'] = claims['preferred_username']
        token_cache = loads(cache.serialize())
        if 'AccessToken' in token_cache and len(token_cache['AccessToken']) == 1:
            for key in token_cache['AccessToken']:
                session['token'] = token_cache['AccessToken'][key]['secret']
                from json import dump
                dump(result.get('id_token_claims'), open('CLAIMS.json', 'w'), indent=4)
    if 'redirect' in session:
        return redirect(session.pop('redirect'))
    return redirect(url_for("ad.index"))

@bp.route("/logout")
def logout():
    session.clear()  # Wipe out user and its token cache from session
    return redirect(  # Also logout from your tenant's web session
        app_config.AUTHORITY + "/oauth2/v2.0/logout" +
        "?post_logout_redirect_uri=" + url_for("ad.index", _external=True))

#
@bp.route("/graphcall")
def graphcall():
    token = _get_token_from_cache(app_config.SCOPE)
    if not token:
        return redirect(url_for("ad.login"))
    graph_data = requests.get(  # Use token to call downstream service
        app_config.ENDPOINT,
        headers={'Authorization': 'Bearer ' + token['access_token']},
    ).json()
    print(token['access_token'])
    return render_template('display.html', result=graph_data)

def _load_cache():
    cache = msal.SerializableTokenCache()
    if session.get("token_cache"):
        cache.deserialize(session["token_cache"])
    return cache

def _save_cache(cache):
    if cache.has_state_changed:
        session["token_cache"] = cache.serialize()
        token_cache = loads(session['token_cache'])
        if 'AccessToken' in token_cache and len(token_cache['AccessToken']) == 1:
            for key in token_cache['AccessToken']:
                session['token'] = token_cache['AccessToken'][key]['secret']
        print(session['token'])

def _build_msal_app(cache=None, authority=None):
    return msal.ConfidentialClientApplication(
        app_config.CLIENT_ID, authority=authority or app_config.AUTHORITY,
        client_credential=app_config.CLIENT_SECRET, token_cache=cache)

def build_auth_url(authority=None, scopes=None, state=None):
    return _build_msal_app(authority=authority).get_authorization_request_url(
        scopes or [],
        state=state or str(uuid.uuid4()),
        redirect_uri=url_for("ad.authorized", _external=True))

def _get_token_from_cache(scope=None):
    cache = _load_cache()  # This web app maintains one cache per session
    cca = _build_msal_app(cache=cache)
    accounts = cca.get_accounts()
    if accounts:  # So all account(s) belong to the current signed-in user
        result = cca.acquire_token_silent(scope, account=accounts[0])
        _save_cache(cache)
        return result


# -*- coding: utf-8 -*-
"""
Created on Sun Feb 26 14:35:46 2017

@author: rohit
"""
#%%
import numpy as np

#%%
class MyFLDA2():

    # Data features
    classes = 0
    count = 0
    n_classes = 0
    n_samples = 0
    n_features = 0

    # Model parameters
    class_mean = 0
    covar_mat = 0
    W = 0
    error = 0
    thresh = 0
    def __init__(self):

        #self.class_mean = np.zeros(self.n_features)
        self.W = np.zeros(self.n_features)
        self.thresh = 0
    ## Fit the model

    def fit(self,X_train,y_train):

        self.classes,self.count = np.unique(y_train,return_counts=True)
        self.n_classes = len(self.classes)
        self.n_samples, self.n_features = np.shape(X_train)

        strat_classes = [np.zeros([self.count[i],self.n_features]) for i in range(self.n_classes)]

        ind_arr = np.zeros(self.n_classes,dtype = int)

        for i in range(self.n_samples):
            for j in range(self.n_classes):
                if y_train[i] == self.classes[j]:
                    ind = ind_arr[j]
                    strat_classes[j][ind] = X_train[i]
                    ind_arr[j] += 1
                    break

## Computing model parameters

        #self.prior_prob = np.zeros(self.n_classes)
        self.class_mean = [np.zeros(self.n_features) for i in range(self.n_classes)]
        class_var = np.zeros([self.n_features,self.n_features])
        self.covar_mat = [np.zeros([self.n_features,self.n_features]) for i in range(self.n_classes)]
        SW = np.zeros([self.n_features,self.n_features])

        for ii in range(self.n_classes):
            #self.prior_prob[ii] = self.count[ii]/self.n_samples
            self.class_mean[ii] = np.mean(strat_classes[ii],axis = 0)
            mean_cent = strat_classes[ii] - self.class_mean[ii]
            class_var = np.matmul(np.transpose(mean_cent),mean_cent)
            class_var = class_var/len(strat_classes[ii])
            self.covar_mat[ii] = class_var
            SW = SW + self.covar_mat[ii]

        self.W = np.matmul(np.linalg.inv(SW),(self.class_mean[0] - self.class_mean[1]))
        z1_tr = np.matmul(strat_classes[0],self.W)
        z2_tr = np.matmul(strat_classes[1],self.W)

        s1 = z1_tr.shape[0]
        s2 = z2_tr.shape[0]
        z1_mean = np.mean(z1_tr)
        z2_mean = np.mean(z2_tr)

        self.thresh = (z1_mean*s2 + z2_mean*s1)/self.n_samples
    ## Prdiction

    def predict(self,X_test):

        s1,s2 = X_test.shape
        y_pred = np.zeros(s1)

        for p in range(s1):
            x = X_test[p,:]
            x_tr = np.matmul(x,self.W)
            if x_tr < self.thresh:
               y_pred[p] = 1
            #    y_pred[p] = np.argmax(g_dscr)

        return y_pred


from flask import Flask, render_template, request, url_for
import pandas as pd
import numpy as np
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
import re
import pickle

app = Flask(__name__)

# load machine learning models
mnb_model = pickle.load(open(r"mnb_model.sav", "rb"))
lr_model = pickle.load(open(r"lr_model.sav", "rb"))
rfc_model = pickle.load(open(r"rfc_model.sav", "rb"))

# create a function to clean input text
def textCleaning(text):

    # transform the text into lower cases
    text = text.lower()

    # remove special characters and punctuations in the text
    text = re.sub(r"[^a-zA-Z]", " ", text)

    # remove digits in the text
    text = re.sub(r"[\d]", " ", text)

    # tokenize the text
    tokenizedText = word_tokenize(text)

    # remove stopwords from the text
    filteredText = list()
    for word in tokenizedText:
        if word not in stopwords.words('english'):
            filteredText.append(word)

    # lemmatize the word
    lemmatizeWords = list()
    lemmatizer = WordNetLemmatizer()
    for word in filteredText:
        lemmatizeWords.append(lemmatizer.lemmatize(word))

    # reform the tokenize words into text
    newText = ' '.join(lemmatizeWords)

    return newText

@app.route("/")
def home():
    return render_template('home.html')

@app.route("/predict", methods = ["POST"])
def predict():
    if request.method == "POST":
        ml_model = request.form['ml']
        textNews = request.form['news']

        # call the function to clean input text
        cleanedText = textCleaning(textNews)

        if ml_model == "mnb":
            predict = mnb_model.predict([cleanedText])
        elif ml_model == "lr":
            predict = lr_model.predict([cleanedText])
        else:
            predict = rfc_model.predict([cleanedText])

        return render_template('outcome.html', rawText = textNews, normalizedText = cleanedText, prediction = predict)

if __name__ == '__main__':
    app.run(debug=True)


# -*- coding: utf-8 -*-
# author: fanchuang
# DateTime:2021/3/13 0013 19:42
# contact: fanchuangwater@gmail.com
# about:

import json
import scrapy
from bs4 import BeautifulSoup
from my_proxy.items import MyProxyItem

# 不稳定，响应慢！
class HuanProxySpider(scrapy.Spider):
    name = 'xila'
    allowed_domains = ['xiladaili.com']
    start_urls = ['http://www.xiladaili.com/gaoni/']

    custom_settings = {
        'CLOSESPIDER_ITEMCOUNT': '500',
        'DOWNLOAD_DELAY': 5,
    }

    def parse(self, response, **kwargs):
        item = MyProxyItem()
        soup = BeautifulSoup(response.text, 'lxml')
        tbody = soup.find('tbody').find_all('tr')
        for tr in tbody:
            data = [td.text for td in tr.find_all("td")][:2]
            item['ip'] = data[0].split(':')[0]
            item['port'] = data[0].split(':')[1]
            if data[0][1].startswith("HTTPS"):
                item['protocol'] = "https"
            else:
                item['protocol'] = "http"
            item['is_china'] = True
            yield item

        next_page = soup.find_all('a', attrs={'class': 'page-link'})[-1].get('href')
        base = "http://www.xiladaili.com"
        if next_page:
            yield scrapy.Request(url=base+next_page, callback=self.parse)


class Solution:
    # APP1: brute force. Loop through array k times, every time put largest index in visited
    # Time: O(n*k) Space: O(k) Runtime: 5%
    def findKthLargest(self, nums: List[int], k: int) -> int:
        if not nums or not k or k < 0:
            return None
        visited, n, res = set(), len(nums), None
        for _ in range(k):
            max_idx, max_val = -1, -sys.maxsize
            for i, val in enumerate(nums):
                if i in visited:
                    continue
                if val > max_val:
                    max_idx, max_val = i, val
            res = max_val
            visited.add(max_idx)
        return res

    #     APP2: maxheap, put all numbers in the maxheap and pop k times.
    #     Time: O(nlgn + klgn) Space: O(n) Runtime: 78%
    #     import heapq
        def findKthLargest(self, nums: List[int], k: int) -> int:
            if not nums or not k or k < 0:
                return None
            maxheap, res = [], None
            for num in nums:
                heapq.heappush(maxheap, -num)

            for _ in range(k):
                res = -heapq.heappop(maxheap)
            return res

    #     APP3: minheap, add k element to minheap, if nums[i] > minheap[0], then pop and add it, else pass.
    #     Time: O(nlgk) Space: O(k) Runtime: 78%
    #     import heapq
        def findKthLargest(self, nums: List[int], k: int) -> int:
            if not nums or not k or k < 0:
                return None

            minheap = []
            for num in nums:
                if len(minheap) < k:
                    heapq.heappush(minheap, num)
                else:
                    if num > minheap[0]:
                        heapq.heappop(minheap)
                        heapq.heappush(minheap, num)
            return minheap[0]

    #     APP4: quick select, find a pivot, move all nums that is smaller than it to it's left, all nums > pivot to it's right. check how many on the left. if count == k, return, if count > k, it must be on the left, otherwise on the right.
    #     Time: O(n) average, O(n^2) worst case Space: O(1) Runtime: 99%
    def findKthLargest(self, nums: List[int], k: int) -> int:
        if not nums or not k or k < 0:
            return None
        return self.quick_select(nums, k, 0, len(nums) - 1)

    # Option1: find kth largest number between left and right.
    def quick_select(self, nums, k, start, end):
        l, r = start, end
        mid = l + (r - l) // 2
        pivot = nums[mid]
        while l <= r:
            while l <= r and nums[l] > pivot:
                l += 1
            while l <= r and nums[r] < pivot:
                r -= 1

            if l <= r:
                nums[l], nums[r] = nums[r], nums[l]
                l += 1
                r -= 1
        if start + k - 1 <= r:
            return self.quick_select(nums, k, start, r)
        if start + k - 1 >= l:
            return self.quick_select(nums, k - (l - start), l, end)
        return nums[r + 1]

    # option2: index based, find kth largest numbers index.
    def quick_select(self, nums, k, start, end):
        if start == end:
            return nums[k - 1]
        l, r = start, end
        mid = l + (r - l) // 2
        pivot = nums[mid]
        while l <= r:
            while l <= r and nums[l] > pivot:
                l += 1
            while l <= r and nums[r] < pivot:
                r -= 1
            if l <= r:
                nums[l], nums[r] = nums[r], nums[l]
                l += 1
                r -= 1
        if r >= k - 1:
            return self.quick_select(nums, k, start, r)
        if l <= k - 1:
            return self.quick_select(nums, k, l, end)
        return nums[r + 1]

Sorting/Quick select/215. Kth Largest Element.py


"""
import liberator
lib = liberator.Liberator()
import vtool_ibeis as vt
lib.add_dynamic(vt.to_undirected_edges)
lib.add_dynamic(vt.compute_unique_data_ids)
lib.close('vtool_ibeis')
print(lib.current_sourcecode())
"""
import numpy as np
import ubelt as ub

def to_undirected_edges(directed_edges, upper=False):
    assert len(directed_edges.shape) == 2 and directed_edges.shape[1] == 2
    #flipped = qaid_arr < daid_arr
    if upper:
        flipped = directed_edges.T[0] > directed_edges.T[1]
    else:
        flipped = directed_edges.T[0] < directed_edges.T[1]
    # standardize edge order
    edges_dupl = directed_edges.copy()
    edges_dupl[flipped, 0:2] = edges_dupl[flipped, 0:2][:, ::-1]
    undirected_edges = edges_dupl
    return undirected_edges

def compute_unique_data_ids_(hashable_rows, iddict_=None):
    if iddict_ is None:
        iddict_ = {}
    for row in hashable_rows:
        if row not in iddict_:
            iddict_[row] = len(iddict_)
    dataid_list = list(ub.take(iddict_, hashable_rows))
    return dataid_list

def compute_unique_data_ids(data):
    """
    This is actually faster than compute_unique_integer_data_ids it seems

    Example:
        >>> # ENABLE_DOCTEST
        >>> data = np.array([[0, 0], [0, 1], [1, 0], [1, 1], [0, 0], [.534, .432], [.534, .432], [1, 0], [0, 1]])
        >>> dataid_list = compute_unique_data_ids(data)
        >>> result = 'dataid_list = ' + ub.repr2(dataid_list, with_dtype=True)
        >>> print(result)
        dataid_list = np.array([0, 1, 2, 3, 0, 4, 4, 2, 1], dtype=np.int32)
    """
    # construct a unique id for every edge
    hashable_rows = [tuple(row_.tolist()) for row_ in data]
    dataid_list = np.array(compute_unique_data_ids_(hashable_rows), dtype=np.int32)
    return dataid_list


#!/usr/bin/env python
# Produce symmetric tab for self-matches from lastal
# USAGE: f=$ref; time lastal -C2 -P 8 $ref $f | ~/src/redundans/bin/skip_selfmatches.py | last-split | maf-convert tab | ~/src/redundans/bin/last-split2symmetric.py | last-dotplot - $f.best.png

import sys
out = sys.stdout

for l in sys.stdin:
    if l.startswith('#'):
        out.write(l)
        continue
    ldata = l.split('\t')
    t, q = ldata[1:6], ldata[6:11]
    out.write("%s\t%s\t%s\t%s"%(ldata[0], "\t".join(t), "\t".join(q), "\t".join(ldata[11:])))
    out.write("%s\t%s\t%s\t%s"%(ldata[0], "\t".join(q), "\t".join(t), "\t".join(ldata[11:])))


#!/usr/bin/env python

##############################################################################
#
# Name: needle.py
# Date: 2018-09-11
# Author: kimoton
# Description:
# Problem
# The 20 commonly occurring amino acids are abbreviated by using 20 letters from the English alphabet (all letters except for B, J, O, U, X, and Z). Protein strings are constructed from these 20 symbols. The RNA codon table shows the encoding from each RNA codon to the amino acid alphabet.
#
# The Translate tool from the SMS 2 package can be found here in the SMS 2 package
#
# A detailed list of genetic code variants (codon tables) along with indexes representing these codes (1 = standard genetic code, etc.) can be obtained here.
#
# For now, when translating DNA and RNA strings, we will start with the first letter of the string and ignore stop codons.
#
# Given: A DNA string s of length at most 10 kbp, and a protein string translated by s.
#
# Return: The index of the genetic code variant that was used for translation. (If multiple solutions exist, you may return any one.)
#
# Sample Dataset
# ATGGCCATGGCGCCCAGAACTGAGATCAATAGTACCCGTATTAACGGGTGA
# MAMAPRTEINSTRING
# Sample Output
# 1
#
##############################################################################

import sys
from Bio.Seq import translate

TABLE_NUM = (1, 2, 3, 4, 5, 6, 9, 10, 11, 12,
             13, 14, 16, 21, 23, 24, 25, 26,
             27, 28, 29, 30, 31)

def getTrueTable(seq, protein_seq):
    protein_seq = protein_seq.strip() + '*'
    print('Ans : {}'.format(protein_seq))
    for i in TABLE_NUM:
        protein_query = translate(seq.strip(), table=i)
        if protein_seq == protein_query:
            return i
        else:
            print(protein_query)
            continue

def main():
    input_seq = sys.argv[1]
    with open(input_seq, 'r') as rf:
        lines = rf.readlines()
        dna_seq = lines[0]
        protein_seq = lines[1]
    table_id = getTrueTable(dna_seq, protein_seq)
    print(table_id)

if __name__ == '__main__':
    main()


class Solution:
    # APP1: find all the substrings and check palindrom, find the longest.
    # Time: O(n^3) Space: O(1) Runtime: TLE
    #     def longestPalindrome(self, s: str) -> str:
    #         if not s:
    #             return s
    #         n, lps = len(s), (0, 0, 0)
    #         for i in range(n - 1):
    #             for j in range(i + 1, n):
    #                 if self.is_palin(s, i, j) and j - i + 1 > lps[-1]:
    #                     lps = (i, j, j - i + 1)
    #         return s[lps[0]:lps[1] + 1]

    #     def is_palin(self, s, l, r):
    #         if l >= r:
    #             return True
    #         while l < r:
    #             if s[l] != s[r]:
    #                 return False
    #             l += 1
    #             r -= 1
    #         return True

    # APP2: iterate center of the palindro, center can be on char or between chars.
    # Then use two pointers of opposite directions
    # Time: O(n^2) space: O(1) Runtime: 68%
    #     def longestPalindrome(self, s: str) -> str:
    #         if not s or len(s) < 2:
    #             return s
    #         start = length = 0
    #         for i in range(len(s) - 1):
    #             count1, start1 = self.get_palindrom_len(s, i, i)
    #             count2, start2 = self.get_palindrom_len(s, i, i + 1)
    #             if count1 > count2 and count1 > length:
    #                 start, length = start1, count1
    #             if count2 > count1 and count2 > length:
    #                 start, length = start2, count2
    #         return s[start:start + length]

    #     def get_palindrom_len(self, s, l, r):
    #         while l >= 0 and r <= len(s) - 1 and s[l] == s[r]:
    #             l -= 1
    #             r += 1
    #         return (r - l - 1, l + 1)

    # APP3: DP. dp[i][j] = dp[i + 1][j - 1] and s[i] == s[j]
    # dp[i][i] == True, dp[i][i - 1] = True. calculate col by col
    # Time: O(n^2) space: O(n^2) Runtime: 37%
    # def longestPalindrome(self, s: str) -> str:
    #     n, max_len, start = len(s), 1, 0
    #     if not s or n < 2:
    #         return s
    #     dp = [[True] * n for _ in range(n)]
    #     for j in range(1, n):
    #         for i in range(0, j):
    #             dp[i][j] = dp[i + 1][j - 1] and s[i] == s[j]
    #             if dp[i][j] and j - i + 1 > max_len:
    #                 max_len, start = j - i + 1, i
    #     return s[start:start + max_len]

    # APP4: DP rolling array to optimize APP3
    # Time: O(n^2) space: O(n) Runtime: 41%
    # def longestPalindrome(self, s: str) -> str:
    #     n, max_len, start = len(s), 1, 0
    #     if not s or n < 2:
    #         return s
    #     old, new = [True] * n, [True] * n
    #     for j in range(1, n):
    #         for i in range(0, j):
    #             new[i] = old[i + 1] and s[i] == s[j]
    #             if new[i] and j - i + 1 > max_len:
    #                 max_len, start = j - i + 1, i
    #         old, new = new, old
    #     return s[start:start + max_len]

    # APP5: Manacher
    # Time: O(n) space: O(n) Runtime: 97%
    def longestPalindrome(self, s: str) -> str:
        if not s or len(s) < 2:
            return s
        s = "#" + "#".join(list(s)) + "#"
        n, max_right, start, max_len, arm_len, j = len(s), -1, 1, 0, [], -1
        for i in range(n):
            if i <= max_right:
                i_mirror = i - 2 * (i - j)
                min_arm_len = min(arm_len[i_mirror], max_right - i)
                cur_arm_len = self.find_arm_len(s, i - min_arm_len, i + min_arm_len)
            else:
                cur_arm_len = self.find_arm_len(s, i, i)
            arm_len.append(cur_arm_len)
            if i + cur_arm_len > max_right:
                j = i
                max_right = i + cur_arm_len

            if 2 * cur_arm_len > max_len:
                max_len = 2 * cur_arm_len
                start = i - cur_arm_len
        return s[start + 1:start + max_len:2]

    def find_arm_len(self, s, left, right):
        while left >= 0 and right <= len(s) - 1 and s[left] == s[right]:
            left -= 1
            right += 1
        return ((right - 1) - (left + 1)) // 2

Unique Algo/Manacher 5. Longest Palindromic Substring.py


"""
MadLibs game with dictionaries
"""

def getKeys(story):
    count = story.count("{")
    end = 0
    keys = []
    for x in range(count):
        start = story.find("{", end) + 1
        end = story.find("}", start)
        word = story[start:end]
        keys.append(word)
    return keys

def userInput(keysResult):
    words = dict()
    for x in keysResult:
        userResponse = input("Input a {} ".format(x))
        words[x] = userResponse
    return words

def main():
    story = """Little {name} fell off her {noun} while attempting to avoid
    a trail of ants {verb} the bike path. Once her tears had subsided
    and the cut on her knee ceased bleeding, she {adverb} got back on
    her bike. She then used the anthill as a bike ramp until the {secondNoun}
    collapsed; her revenge complete, she {secondVerb} home happily."""

    keysResult = getKeys(story)

    userResult = userInput(keysResult)

    finalstory = story.format(**userResult)

    print(finalstory)

main()