from distutils.command.build import build
import pandas as pd
import numpy as np
import tensorflow as tf
from matplotlib import pyplot as plt

pd.options.display.max_rows = 10
pd.options.display.float_format = "{:.1f}".format

if tf.config.list_physical_devices('GPU'):
  print("TensorFlow **IS** using the GPU")
else:
  print("TensorFlow **IS NOT** using the GPU")

def build_model(my_learning_rate):
    model = tf.keras.models.Sequential()

    model.add(tf.keras.layers.Dense(units=10, input_shape=(1,)))
    model.add(tf.keras.layers.Dense(units=10, input_shape=(1,)))
    model.add(tf.keras.layers.Dense(units=10, input_shape=(1,)))

    model.compile(optimizer=tf.keras.optimizers.RMSprop(learning_rate=my_learning_rate), loss="mean_squared_error", metrics=[tf.keras.metrics.RootMeanSquaredError()])

    return model

# def build_model(my_learning_rate):
#     model = tf.keras.models.Sequential()

#     model.add(tf.keras.layers.Dense(units=10))
#     model.add(tf.keras.layers.Dense(units=10))

#     model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=my_learning_rate), loss=tf.keras.losses.BinaryCrossentropy(), metrics=[tf.keras.metrics.BinaryAccuracy(),tf.keras.metrics.FalseNegatives()])

#     return model

def train_model(model, df, feature, label, epochs, batch_size):
    history = model.fit(x=df[feature], y=df[label], batch_size=batch_size, epochs=epochs)

    trained_weight = model.get_weights()[0]
    trained_bias = model.get_weights()[1]

    epochs = history.epoch

    hist = pd.DataFrame(history.history)

    rmse = hist["root_mean_squared_error"]

    return trained_weight, trained_bias, epochs, rmse

print("Defined build_model and train_model")

def plot_the_model(trained_weight, trained_bias, feature, label):
    plt.xlabel("feature")
    plt.ylabel("label")

    random_examples = training_df.sample(n=200)
    plt.scatter(random_examples[feature], random_examples[label])

    x0 = 0
    y0 = trained_bias
    x1 = 10000
    y1 = trained_bias + (trained_weight * x1)
    plt.plot([x0, x1], [y0, y1], c='r')

    plt.show()

def plot_the_loss_curve(epochs, rmse):
    plt.figure()
    plt.xlabel("Epoch")
    plt.ylabel("Root Mean Squared Error")

    plt.plot(epochs, rmse, label="Loss")
    plt.legend()
    plt.ylim([rmse.min()*0.97, rmse.max()])
    plt.show()

print("Defined the plot_the_model and plot_the_loss_curve functions.")

def predict_house_values(n, feature, label):
  """Predict house values based on a feature."""

  batch = training_df[feature][10000:10000 + n]
  predicted_values = my_model.predict_on_batch(x=batch)

  print("feature   label          predicted")
  print("  value   value          value")
  print("          in thousand$   in thousand$")
  print("--------------------------------------")
  for i in range(n):
    print ("%5.0f %6.0f %15.0f" % (training_df[feature][10000 + i], training_df[label][10000 + i], predicted_values[i][0] ))

training_df = pd.read_csv(r"C:\Users\Tom\Projects\Data Stash\california_housing_train.csv")

training_df["median_house_value"] /= 1000.0
training_df["rooms_per_person"] = training_df["total_rooms"] / training_df["population"]

learning_rate=0.001
epochs=30
my_batch_size=60

my_feature = "median_income"
my_label = "median_house_value"

my_model = None

my_model = build_model(learning_rate)
trained_weight, trained_bias, epochs, rmse = train_model(my_model, training_df, my_feature, my_label, epochs, my_batch_size)

# my_model.summary()
# predict_house_values(10, my_feature, my_label)
# corr_matrix = training_df.corr()
# print(corr_matrix)
# plot_the_model(trained_weight, trained_bias, my_feature, my_label)
# plot_the_loss_curve(epochs, rmse)

# imageFile = r"C:\Users\Tom\Projects\Data Stash\train-images.idx3-ubyte"
# trainingFile = r"C:\Users\Tom\Projects\Data Stash\train-labels.idx1-ubyte"
# trainingImage = idx2numpy.convert_from_file(imageFile)
# trainingLabel = idx2numpy.convert_from_file(trainingFile)

# check = random.randrange(0,60000)

# image = trainingImage[check]
# label = trainingLabel[check]
# label = label.astype(np.uint8)

# inputs = []
# for rows_of_pixels in image:
# 	for pixel in rows_of_pixels:
# 		inputs.append(pixel)

# new = [inputs, label]


import time
import sys
import os.path
import logging
import argparse
import shlex

from RestrictedPython import compile_restricted
from RestrictedPython.Guards import safe_builtins
from RestrictedPython.Guards import guarded_unpack_sequence
from antioch.core import errors
from antioch.util import ason

allowed_modules = (
    'hashlib',
    'string',
)

pylog = logging.getLogger(__name__)

argparser = argparse.ArgumentParser(prog='antioch',
    description="Configure antioch attribute deployment.")
argparser.add_argument('type', type=str, choices=['verb', 'property'],
    help="The kind of attribute being deployed.")
argparser.add_argument('name', type=str,
    help="The name of the attribute being deployed.")
argparser.add_argument('origin', type=str,
    help="An object reference for the origin of this attribute.")
argparser.add_argument('--owner', type=str, required=True,
    help="An object reference for the owner of this object.")
argparser.add_argument('--ability', action='store_true',
    help="This is an intrinsic ability of the origin.")
argparser.add_argument('--method', action='store_true',
    help="This is a programmable method of the origin.")
argparser.add_argument('--access-group', type=str, nargs="*",
    help="Allow or deny a group a certain permission on this attribute.")
argparser.add_argument('--access-object', type=str, nargs="*",
    help="Allow or deny a specific object a certain permission on this attribute.")

def is_frame_access_allowed():
    """
    Used by get/setattr to delegate access.

    Returns True if the call to __getattribute__ or __setattr__ originated
    in either the exchange, test, or bootstrap modules.

    Previously this used inspect, which made it super slow. The only potential
    issue with using _getframe() is that it's not guaranteed to be there in
    non CPython implementations.
    """
    f = sys._getframe(1)
    c1 = f.f_back.f_code
    c2 = f.f_back.f_back.f_code

    calling_filename = os.path.abspath(c2.co_filename)
    try:
        from antioch.core import interface
        model_source_path = os.path.abspath(interface.__file__)
        if(model_source_path.endswith('pyc')):
            model_source_path = model_source_path[:-1]
        if(calling_filename == model_source_path):
            # pylog.debug('%r =(1)= %r' % (c2.co_filename, model_source_path))
            return True

        from antioch.core import exchange
        exchange_source_path = os.path.abspath(exchange.__file__)
        if(exchange_source_path.endswith('pyc')):
            exchange_source_path = exchange_source_path[:-1]
        if(calling_filename == exchange_source_path):
            # pylog.debug('%r =(2)= %r' % (c2.co_filename, exchange_source_path))
            return True

        from antioch import test
        test_source_path = os.path.abspath(os.path.dirname(test.__file__))
        if(calling_filename.startswith(test_source_path)):
            # pylog.debug('%r 1startswith %r' % (c2.co_filename, test_source_path))
            return True

        from antioch.core import bootstrap
        bootstrap_source_path = os.path.abspath(os.path.dirname(bootstrap.__file__))
        if(calling_filename.startswith(bootstrap_source_path)):
            # pylog.debug('%r 2startswith %r' % (c2.co_filename, bootstrap_source_path))
            return True

        pylog.warning('secured access %r != %r' % (c2.co_filename,model_source_path))
        return False
    finally:
        del c2
        del c1
        del f

def massage_verb_code(code):
    """
    Take a given piece of verb code and wrap it in a function.

    This allows support of 'return' within verbs, and for verbs to return values.
    """
    code = code.replace('\r\n', '\n')
    code = code.replace('\n\r', '\n')
    code = code.replace('\r', '\n')
    code = '\n'.join(
        ['def verb():'] +
        ['\t' + x for x in code.split('\n') if x.strip()] +
        ['returnValue = verb()']
    )
    return code

def parse_deployment(source):
    if not source.startswith("#!antioch"):
        raise ValueError("Source code does not begin with antioch shebang.")
    shebang = ''
    for line in source.splitlines():
        line = line.strip()
        if line.startswith('#'):
            line = line[1:]
        else:
            break
        if not line.endswith('\\'):
            shebang += line
            break
        else:
            shebang += ' '
            shebang += line[:-1]
    return argparser.parse_args(shlex.split(shebang[1:])[1:])

def r_eval(caller, src, environment={}, filename='<string>', runtype="eval"):
    """
    Evaluate an expression in the provided environment.
    """
    def _writer(s, is_error=False):
        if(s.strip()):
            write(environment.get('parser'))(caller, s, is_error=is_error)

    env = get_restricted_environment(_writer, environment.get('parser'))
    env['runtype'] = runtype
    env['caller'] = caller
    env.update(environment)

    code = compile_restricted(src, filename, 'eval')
    try:
        value =  eval(code, env)
    except errors.UsageError as e:
        if(caller):
            _writer(str(e), is_error=True)
        else:
            raise e

    return value

def r_exec(caller, src, environment={}, filename='<string>', runtype="exec"):
    """
    Execute an expression in the provided environment.
    """
    def _writer(s, is_error=False):
        if(s.strip()):
            write(environment.get('parser'))(caller, s, is_error=is_error)

    env = get_restricted_environment(_writer, environment.get('parser'))
    env['runtype'] = runtype
    env['caller'] = caller
    env.update(environment)

    code = compile_restricted(massage_verb_code(src), filename, 'exec')
    try:
        exec(code, env)
    except errors.UsageError as e:
        if(caller):
            _writer(str(e), is_error=True)
        else:
            raise e

    if("returnValue" in env):
        return env["returnValue"]

def restricted_import(name, gdict, ldict, fromlist, level=-1):
    """
    Used to drastically limit the importable modules.
    """
    if(name in allowed_modules):
        return __builtins__['__import__'](name, gdict, ldict, fromlist, level)
    raise ImportError('Restricted: %s' % name)

def get_protected_attribute(obj, name, g=getattr):
    if(name.startswith('_') and not is_frame_access_allowed()):
        raise AttributeError(name)
    return g(obj, name)

def set_protected_attribute(obj, name, value, s=setattr):
    if(name.startswith('_') and not is_frame_access_allowed()):
        raise AttributeError(name)
    return s(obj, name, value)

def inplace_var_modification(operator, a, b):
    if(operator == '+='):
        return a+b
    raise NotImplementedError("In-place modification with %s not supported." % operator)

def get_restricted_environment(writer, p=None):
    """
    Given the provided parser object, construct an environment dictionary.
    """
    class _print_(object):
        def _call_print(self, s):
            writer(s)

    class _write_(object):
        def __init__(self, obj):
            object.__setattr__(self, 'obj', obj)

        def __setattr__(self, name, value):
            """
            Private attribute protection using is_frame_access_allowed()
            """
            set_protected_attribute(self.obj, name, value)

        def __setitem__(self, key, value):
            """
            Passthrough property access.
            """
            self.obj[key] = value

    safe_builtins['__import__'] = restricted_import

    for name in ['dict', 'getattr', 'hasattr']:
        safe_builtins[name] = __builtins__[name]

    env = dict(
        _apply_           = lambda f,*a,**kw: f(*a, **kw),
        _print_           = lambda x: _print_(),
        _write_           = _write_,
        _getattr_         = get_protected_attribute,
        _getitem_         = lambda obj, key: obj[key],
        _getiter_         = lambda obj: iter(obj),
        _inplacevar_      = inplace_var_modification,
        _unpack_sequence_ = guarded_unpack_sequence,
        __import__        = restricted_import,
        __builtins__      = safe_builtins,
    )

    from antioch import plugins
    for plugin in plugins.iterate():
        for name, func in list(plugin.get_environment().items()):
            if(hasattr(func, 'im_func')):
                func.__func__.__name__ = name
            else:
                func.__name__ = name
            api(func) if callable(func) else None

    for name, func in list(api.locals.items()):
        env[name] = func(p)

    for name in dir(errors):
        if not(name.endswith('Error')):
            continue
        cls = getattr(errors, name)
        env[name] = cls

    return env

def run_system_verb(exchange, verb_name, *args, **kwargs):
    origin = exchange.get_object(1)
    verb = origin.get_verb(verb_name)
    return verb(*args, **kwargs)

def api(func):
    """
    Bless a function into the verb API.
    """
    def _api(p):
        def __api(*args, **kwargs):
            return func(p, *args, **kwargs)
        return __api

    api.locals = getattr(api, 'locals', {})
    api.locals[func.__name__] = _api

    return _api

@api
def task(p, delay, origin, verb_name, *args, **kwargs):
    """
    Verb API: queue up a new task.
    """
    # remind me again why we can't do this?
    from antioch.core import tasks
    tasks.registertask.delay(
        user_id        = p.caller.id,
        delay        = str(delay),
        origin_id    = str(origin.id),
        verb_name    = str(verb_name),
        args        = json.dumps(args),
        kwargs        = json.dumps(kwargs),
    )

    # #force exception here if undumpable
    # p.exchange.send_message(p.caller, dict(
    #     command        = 'task',
    #     delay        = int(delay),
    #     origin        = str(origin),
    #     verb_name    = str(verb_name),
    #     args        = json.dumps(args),
    #     kwargs        = json.dumps(kwargs),
    # ))

@api
def execute(p, code):
    """
    Verb API: Execute the code in place.
    """
    return r_exec(p.caller, code, p.get_environment())

@api
def evaluate(p, code):
    """
    Verb API: Evalue the expression in place.
    """
    return r_eval(p.caller, code, p.get_environment())

@api
def tasks(p):
    """
    Verb API: Return a list of tasks for this user, or all tasks.
    """
    if(p.caller.is_wizard()):
        return p.exchange.get_tasks()
    else:
        return p.exchange.get_tasks(user_id=p.caller.get_id())

@api
def write(p, user, text, is_error=False, escape_html=True):
    """
    Verb API: Print a string of text to the user's console.
    """
    p.exchange.send_message(user.get_id(), dict(
        command        = 'write',
        text        = str(text),
        is_error    = is_error,
        escape_html    = escape_html,
    ))

@api
def log(p, text, is_error=False):
    """
    Verb API: Print a string of text to the server's console.
    """
    if p.caller and not(p.caller.is_wizard()):
        return
    if(is_error):
        pylog.error(text)
    else:
        pylog.info(text)

@api
def broadcast(p, text, escape_html=True):
    """
    Verb API: Print a string to the console of everyone nearby.
    """
    if(p.caller.location is None):
        return
    for obj in p.caller.location.get_contents():
        p.exchange.send_message(obj.get_id(), dict(
            command        = 'write',
            text        = str(text),
            is_error    = False,
            escape_html    = escape_html,
        ))

@api
def observe(p, user, observations):
    """
    Verb API: Send a dict of observations to the user's client.
    """
    p.exchange.send_message(user.get_id(), dict(
        command            = 'observe',
        observations    = observations,
    ))

@api
def count_named(p, key):
    """
    Verb API: Get the total number of objects sharing a global name or ID.
    """
    return p.exchange.refs(key)

@api
def get_object(p, key, return_list=False):
    """
    Verb API: Load an object by its global name or ID.
    """
    return p.exchange.get_object(key, return_list=return_list)

@api
def create_object(p, name, unique_name=False):
    """
    Verb API: Create a new object.
    """
    # this method can be run from system.authenticate() to implement
    # guest support, so it has to work when p.caller == None
    owner_id = p.caller.get_id() if p.caller else None
    return p.exchange.instantiate('object', name=name, unique_name=unique_name, owner_id=owner_id)

@api
def get_last_client_ip(p, player):
    """
    Verb API: Get the last IP address used by the given player.

    [ACL] allowed to administer player
    """
    p.caller.is_allowed('administer', player, fatal=True)
    return p.exchange.get_last_client_ip(player.get_id())


import requests
from bs4 import BeautifulSoup

def get_bs_obj(company_code):
    url     = "https://finance.naver.com/item/main.nhn?code="+company_code
    result  = requests.get(url)
    bsObj   = BeautifulSoup(result.content, "html.parser")
    return bsObj

def get_candle_chart_data(company_code):
    bsObj    = get_bs_obj(company_code)

    td_first = bsObj.find("td", {"class":"first"})
    close = td_first.find("span", {"class":"blind"}).text #종가

    tbl_no_info = bsObj.find("table", {"class":"no_info"})

    tr = tbl_no_info.findAll("tr")[0] #첫번째 tr
    td = tr.findAll("td")[1] #두번째 td
    high = td.findAll("span", {"class":"blind"})[0].text #고가

    tr      = tbl_no_info.findAll("tr")[1]  # 두번째 tr
    td_open = tr.find("td", {"class":"first"})  # td.first
    open    = td_open.find("span", {"class":"blind"}).text #시가
    td_low  = tr.findAll("td")[1]
    low     = td_low.find("span", {"class":"blind"}).text #저가
    print(low)

    return {"close":close, "high":high, "open":open, "low":low}

close = get_candle_chart_data("035420") #종가
print(close)


from flask import Blueprint
main = Blueprint('main', __name__)
import os
import json
from recommendationFramework import Engine

import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

from flask import Flask, request

def create_app():
    global recommendation_engine

    cur_dir = os.getcwd()

    books_path = os.path.join(cur_dir, 'datasets', 'books6ml.csv')
    ratings_path = os.path.join(cur_dir, 'datasets', 'ratings6ml.csv')
    model_path = os.path.join(cur_dir, 'models', 'ALS')

    recommendation_engine =  Engine.RecommendationEngine(ratings_path, books_path, model_path, 'ALS')

    app = Flask(__name__)
    app.register_blueprint(main)
    return app

@main.route("/<int:user_id>/ratings/top/<int:count>", methods=["GET"])
def top_ratings(user_id, count):
    logger.debug("User %s TOP ratings requested", user_id)
    top_ratings = recommendation_engine.alsTopNForUser(user_id,count)
    return json.dumps(top_ratings)

#  add this when adding recommendation for a book
# @main.route("/<int:user_id>/ratings/<int:book_id>", methods=["GET"])
# def book_ratings(user_id, book_id):
#     logger.debug("User %s rating requested for book %s", user_id, book_id)
#     ratings =  recommendation_engine.get_ratings_for_book_ids(user_id, [book_id])
#     return json.dumps(ratings)

@main.route("/<int:user_id>/ratings", methods = ["POST"])
def add_ratingsALS(user_id):
    # get the ratings from the Flask POST request object
    ratings_list = request.form.keys()[0].strip().split("\n")
    ratings_list = map(lambda x: x.split(","), ratings_list)
    # create a list with the format required by the engine (user_id, book_id, rating)
    ratings = map(lambda x: (user_id, int(x[0]), float(x[1])), ratings_list)
    # add them to the model using then engine API
    recommendation_engine.add_ratingsALS(ratings)

    return json.dumps(ratings)


import numpy as np

import pdb

def text_to_vec(x):
  # pdb.set_trace()
  # http://bwlewis.github.io/1000_genomes_examples/PCA.html
  return np.array([float(r) for r in x.strip().replace(" ", "").split(",")])

def FileIter(file_name, chunk_size = 10, skip = 1):
  with open(file_name) as f:
    if skip > 0:
      [f.readline() for s in range(skip)]
    eof = False
    while not eof:
      lines = [f.readline() for s in range(chunk_size)]
      ret = np.array([text_to_vec(x) for x in lines if len(x) > 0])
      current_pos = f.tell()
      if f.readline() == '':
        eof = True
      f.seek(current_pos)
      yield ret

next(FileIter("x.csv"))

i = 0
for y_chunk, x_chunk in zip(FileIter("y.csv"), FileIter("x.csv")):
  print("Chunk ", i)
  print("Y Chunk")
  print(y_chunk)
  print("x Chunk")
  print(x_chunk)
  i += 1


from CatalogManager.CatalogManager import CatalogManager
import os
from utils.utils import ACTIONTYPE, CONDITION, VALUETYPE, COMPARATOR, print_dbg_info
from Interpreter.InterpreterDS import ReturnValue
from RecordManager import RecordManager
from ply import yacc, lex
from API.API import API
from prettytable import PrettyTable
import time

tokens = (
    # Token for fixed statement
    "LPAREN",    # (
    "RPAREN",    # )

    "CREATE",
    "DROP",
    "SELECT",
    "INSERT",
    "DELETE",
    "QUIT",
    "EXECFILE",

    "TABLE",
    "PK",        # primary key
    "INDEX",

    "ON",
    "FROM",
    "WHERE",
    "INTO",
    'VALUES',
    "AND",
    "STAR",
    "COMMA",
    "UNIQUE",
    "INT",
    "CHAR",
    "FLOAT",
    "PRIMARY",
    "KEY",

    "END",

    # Token for variable inputs
    "COLUMN_OR_TABLE",
    "COMPARATOR",
)

# TODO 大小写不敏感的匹配
t_LPAREN = r"\("
t_RPAREN = r"\)"

t_CREATE = r"CREATE|create"
t_DROP   = r"DROP|drop"
t_SELECT = r"SELECT|select"
t_INSERT = r"INSERT|insert"
t_DELETE = r"DELETE|delete"
t_QUIT   = r"QUIT|quit"
t_EXECFILE=r"EXECFILE|execfile"

t_TABLE  = r"TABLE|table"
t_PK     = r"PK|pk"    # TODO
t_PRIMARY= r"PRIMARY|primary"
t_KEY    = r"KEY|key"
t_INDEX  = r"INDEX|index"

t_ON     = r"ON|on"
t_FROM   = r"FROM|from"
t_WHERE  = r"WHERE|where"
t_INTO   = r"INTO|into"
t_VALUES = r"VALUES|values"
t_AND    = r"AND|and"
t_COMMA  = r","

t_STAR   = r"\*"
t_END    = r";"
t_UNIQUE = r"UNIQUE|unique"

t_COMPARATOR = r"[<>=]{1,2}"

def t_COLUMN_OR_TABLE(t):
    r"[\"_\'a-zA-Z0-9/.-]+"
    if t.value.upper() in tokens:
        t.type = t.value.upper()
    # TODO
    return t

def t_newline(t):
    r'\n+'
    pass

t_ignore = " \t"

def t_error(t):
    print("Illegal character {}".format(t.value[0]))
    t.lexer.skip(1)

lexer = lex.lex()

test_lex = "insert into t1 values"
test_lex = "where a<>1 and b < 1"
test_lex = "select * from t1 where a = '1' and b = 1"
lexer.input(test_lex)
'''
while True:
    tok = lexer.token()
    if not tok: break
    print(tok)
'''

# ---------- Up: lex, Down: parse ---------------

condition = []
# column_data = interpreterDS.ColumnData()
column_data = []
return_value = ReturnValue()
execfile = 0
execfile_name = ""

def reset():
    global return_value
    return_value = ReturnValue()

def p_expression_start(p):
    '''expression : exp_drop
                  | exp_select
                  | exp_delete
                  | exp_create
                  | exp_insert
                  | exp_quit
                  | exp_execfile'''

def p_expression_create(p):
    '''exp_create : CREATE TABLE COLUMN_OR_TABLE LPAREN exp_create_columns RPAREN END
                  | CREATE INDEX COLUMN_OR_TABLE ON COLUMN_OR_TABLE LPAREN COLUMN_OR_TABLE RPAREN END'''
    global return_value
    if p[2] in ['table', "TABLE"]:
        return_value.action_type = ACTIONTYPE.CREATE_TABLE
        return_value.table_name = p[3]

        if check_create_table(return_value):
            API.api_create_table(return_value)
            print("Successfully create table {}".format(return_value.table_name))

    elif p[2] in ['index', 'index']:
        return_value.action_type = ACTIONTYPE.CREATE_INDEX
        return_value.index_name = p[3]
        return_value.table_name = p[5]
        return_value.attr_name = p[7]

        if check_create_index(return_value):
            API.api_create_index(return_value)
            print("Successfully create index {} on attribute {}".format(return_value.index_name, return_value.attr_name))

    else:
        print("Invalid command {}".format(p[2]))
        raise SyntaxError

def p_expression_create_columns(p):
    '''exp_create_columns : exp_create_valid_column
                          | exp_create_valid_column COMMA exp_create_columns'''
    pass

def p_expression_create_valid_column(p):
    '''exp_create_valid_column : exp_create_valid_char
                               | exp_create_valid_int
                               | exp_create_valid_float
                               | exp_create_valid_pk'''
    pass

def p_expression_create_pk(p):
    '''exp_create_valid_pk : PRIMARY KEY LPAREN exp_pk RPAREN'''
    pass

def p_expression_pk(p):
    '''exp_pk : COLUMN_OR_TABLE
              | COLUMN_OR_TABLE COMMA exp_pk'''
    return_value.pk.append(p[1])

def p_expression_create_valid_char(p):
    '''exp_create_valid_char : COLUMN_OR_TABLE CHAR LPAREN COLUMN_OR_TABLE RPAREN UNIQUE
                               | COLUMN_OR_TABLE CHAR LPAREN COLUMN_OR_TABLE RPAREN'''
    if not str.isnumeric(p[4]):
        print("[Error] String length should be a number!")
        raise SyntaxError
    return_value.value_type.append((VALUETYPE.CHAR, int(p[4])))
    return_value.column_data.append(p[1])
    return_value.unique.append(len(p) == 7)

    # print("char variable {}".format(p[1]))

def p_expression_create_valid_int(p):
    '''exp_create_valid_int : COLUMN_OR_TABLE INT
                            | COLUMN_OR_TABLE INT UNIQUE'''
    global return_value
    return_value.value_type.append((VALUETYPE.INT, 0))
    return_value.column_data.append(p[1])
    return_value.unique.append(len(p) == 4)

    # print("int variable {}".format(p[1]))

def p_expression_create_valid_float(p):
    '''exp_create_valid_float : COLUMN_OR_TABLE FLOAT
                              | COLUMN_OR_TABLE FLOAT UNIQUE'''
    global return_value
    return_value.value_type.append((VALUETYPE.FLOAT, 0))
    return_value.column_data.append(p[1])
    return_value.unique.append(len(p) == 4)

    # print("float variable {}".format(p[1]))

def p_expression_drop(p):
    '''exp_drop : DROP TABLE COLUMN_OR_TABLE END
                | DROP INDEX COLUMN_OR_TABLE END'''

    global return_value
    if p[2] in ['table', "TABLE"]:
        return_value.action_type = ACTIONTYPE.DROP_TABLE
        return_value.table_name = p[3]

        if check_drop_table(return_value):
            API.api_drop_table(return_value.table_name)
            print("Successfully drop table {}".format(p[3]))

    elif p[2] in ['index', "INDEX"]:
        return_value.action_type = ACTIONTYPE.DROP_INDEX
        return_value.index_name = p[3]

        if check_drop_index(return_value):
            API.api_drop_index(return_value.index_name)

    else:
        print("invalid command {}".format(p[2]))
        return

def p_expression_select(p):
    '''exp_select : SELECT STAR FROM COLUMN_OR_TABLE END
                  | SELECT STAR FROM COLUMN_OR_TABLE exp_condition'''
    global return_value
    return_value.action_type = ACTIONTYPE.SELECT_STAR
    return_value.table_name = p[4]

    # print_dbg_info(return_value)

    if check_select(return_value):
        select_result = API.api_select(return_value)

        if len(select_result) == 0:
            print("No column selected")
            return

        attrs = CatalogManager.get_attrs_type(p[4])
        attr_row = [attr[0] for attr in attrs]
        tb = PrettyTable()
        tb.field_names = attr_row
        for tuple in select_result:
            tb.add_row(tuple)
        print(tb)
        print("{} row(s) affected".format(len(select_result)))

def p_expression_delete(p):
    '''exp_delete : DELETE FROM COLUMN_OR_TABLE END
                  | DELETE FROM COLUMN_OR_TABLE exp_condition'''
    global return_value
    return_value.action_type = ACTIONTYPE.DELETE
    return_value.table_name = p[3]

    if check_delete(return_value):
        delete_num = API.api_delete(return_value)
        print("{} row(s) affected".format(delete_num))

def p_expression_insert(p):
    '''exp_insert : INSERT INTO COLUMN_OR_TABLE exp_insert_line'''
    global return_value
    global column_data

    return_value.return_type = ACTIONTYPE.INSERT
    return_value.table_name = p[3]
    return_value.column_data.reverse()

    if check_insert(return_value):
        API.api_insert(return_value)
        print("1 row(s) affected")

def p_expression_insert_line(p):
    '''exp_insert_line : VALUES LPAREN columns RPAREN END'''

def p_expression_columns(p):
    '''columns : COLUMN_OR_TABLE
               | COLUMN_OR_TABLE COMMA columns'''
    # 这个是倒序进入的
    global return_value
    return_value.column_data.append(p[1])

def p_expression_condition(p):
    '''exp_condition : WHERE exp_all_conditions END'''

def p_expression_all_conditions(p):
    '''exp_all_conditions : COLUMN_OR_TABLE COMPARATOR COLUMN_OR_TABLE
                          | COLUMN_OR_TABLE COMPARATOR COLUMN_OR_TABLE AND exp_all_conditions'''

    if not CONDITION.valid(p[2]):
        print("Invalid operator type {}".format(p[2]))
        return

    global return_value
    return_value.condition.append(CONDITION(p[1], p[2], p[3]))

def p_expression_quit(p):
    '''exp_quit : QUIT'''
    API.api_quit()
    print("Bye")
    exit(0)

def p_expression_execfile(p):
    '''exp_execfile : EXECFILE COLUMN_OR_TABLE END'''
    global execfile, execfile_name
    execfile = 1
    execfile_name = p[2]
    # print(execfile, execfile_name)

def p_error(p):
    if p:
        print("Syntax error at {}".format(p))
    else:
        print("Syntax error!")
    parser.restart()

def check_delete(return_value):
    '''
        同时把char的'去掉
    '''
    table_name = return_value.table_name
    if not CatalogManager.table_exist(table_name):
        print("Table does not exist!")
        return False

    for condition in return_value.condition:
        if not CatalogManager.attr_exist(table_name, condition.lvalue):
            print("Attribute {} does not exist!".format(condition.lvalue))
            return False
        # condition.rvalue = condition.rvalue.strip("'")

    return_value = condition_rvalue_decode(return_value)
    if return_value is None:
        return False

    return True

def check_select(return_value):
    """
        同时把char的'去掉
    """
    table_name = return_value.table_name
    if not CatalogManager.table_exist(table_name):
        print("Table does not exist!")
        return False

    for condition in return_value.condition:
        if not CatalogManager.attr_exist(table_name, condition.lvalue):
            print("Attribute {} does not exist!".format(condition.lvalue))
            return False
        # condition.rvalue = condition.rvalue.strip("'").strip("\"")

    return_value = condition_rvalue_decode(return_value)
    if return_value is None:
        return False

    return True

def check_create_table(return_value):
    # print(return_value.column_data)
    if CatalogManager.attr_exist(return_value.table_name, return_value.attr_name):
        print("Table does not exist!")
        return False

    # print(return_value.pk)
    if len(return_value.pk) == 0:
        print("No primary key is specified!")
        return False

    # print(return_value.column_data)
    for pk in return_value.pk:
        if pk not in return_value.column_data:
            print("Primary key is not in the table!")
            return False

    # print(return_value.column_data)
    if len(set(return_value.column_data)) != len(return_value.column_data):
        print("Detect duplicate keys!")
        return False

    # print(return_value.value_type)
    for val_type in return_value.value_type:
        if val_type[0] == VALUETYPE.CHAR:
            if val_type[1] > 255 or val_type[1] < 0:
                print("Invalid char definition!")
                return False

    if len(return_value.pk) != 1:
        print("We only support primary key that only contains one attribute!")
        return False

    for (i, col) in enumerate(return_value.column_data):
        if col in return_value.pk and return_value.unique[i] != True:
            print("Primary key {} should be unique!".format(col))
            return False

    return True

def check_create_index(return_value):
    # print(return_value.attr_name)

    if not CatalogManager.attr_exist(return_value.table_name, return_value.attr_name):
        print("Attribute name {} does not exist!")
        return False

    if CatalogManager.index_exist(return_value.index_name):
        print("Index name {} has already existed!".format(return_value.index_name))
        return False

    if not CatalogManager.attr_unique(return_value.table_name, return_value.attr_name):
        print("Attribute should be unique!")
        return False

    return True

def check_drop_table(return_value):
    if not CatalogManager.table_exist(return_value.table_name):
        print("Table does not exist!")
        return False

    return True

def check_drop_index(return_value):
    if not CatalogManager.index_exist(return_value.index_name):
        print("Index does not exist!")
        return False

    return True

def condition_rvalue_decode(return_value):
    table_name = return_value.table_name
    attrs = CatalogManager.get_attrs_type(table_name)
    # (attr_name, attr_type, attr_length)
    for condition in return_value.condition:
        attr_type = None
        for attr in attrs:
            if attr[0] == condition.lvalue:
                attr_type = attr[1]
            if attr_type is not None:
                break
        if attr_type == VALUETYPE.INT:
            try:
                condition.rvalue = int(condition.rvalue)
            except:
                print("Rvalue of condition should be INT for attr {}".format(attr.lvalue))
                return None

        elif attr_type == VALUETYPE.FLOAT:
            try:
                condition.rvalue = float(condition.rvalue)
            except:
                print("Rvalue of condition should be FLOAT for attr {}".format(attr.lvalue))
                return None

        else:
            if ("'" not in condition.rvalue) and ("\"" not in condition.rvalue):
                print("You should use "" or '' to specify a string")
                return None
            condition.rvalue = condition.rvalue.strip("\"").strip("'")

    return return_value

def check_insert(return_value):

    # print(type(return_value.table_name))
    try:
        attrs = CatalogManager.get_attrs_type(return_value.table_name)
    except KeyError:
        print("Table not found!")
        return False
    # attrs = reversed(attrs)
    # print(return_value.column_data)
    # print(attrs)

    if len(attrs) != len(return_value.column_data):
        print("Inserted element number {} does not match attribute number of table {}".format(len(return_value.column_data), len(attrs)))
        return False

    # 检测数据类型是否匹配
    for i in range(len(attrs)):
        (attr_name, attr_type, attr_length) = attrs[i]

        insert_data = return_value.column_data[i]

        if ("'" in insert_data) or ("\"" in insert_data):
            return_value.column_data[i] = return_value.column_data[i].strip("'").strip("\"")
            if attr_type != VALUETYPE.CHAR:
                print("Value type CHAR does not match on column {}".format(attr_name))
                return False
            if len(insert_data) > attr_length:
                print("Value {} of type char has more characters than specified!".format(insert_data))
                return False
            continue

        if str.isnumeric(insert_data):
            # int or float
            if attr_type == VALUETYPE.INT:
                return_value.column_data[i] = int(insert_data)
                # print(type(return_value.column_data[i]))
                continue
            if attr_type == VALUETYPE.FLOAT:
                return_value.column_data[i] = float(insert_data)
                # print(type(return_value.column_data[i]))
                continue
            if attr_type == VALUETYPE.CHAR:
                print("Please use '' to specify a string")
                return False
            continue

        if "." in insert_data:
            try:
                float_number = float(insert_data)

                if attr_type == VALUETYPE.FLOAT:
                    return_value.column_data[i] = float_number
                    continue

                if attr_type == VALUETYPE.INT:
                    print("Cannot inseert float number to int attribute!")
                    return False

                if attr_type == VALUETYPE.CHAR:
                    print("Please use '' to specify a string")
                    return False
                continue

            except ValueError:
                print("Please use '' to specify a string")
                return False

        if not check_unique(return_value, attrs):
            return False

    return True

def check_unique(return_value, attrs):
    # 利用索引进行优化查询
    table_name = return_value.table_name

    # print_dbg_info(unique_value)
    # 检测是否unique
    for i in range(len(attrs)):
        unique_value = ReturnValue()
        unique_value.table_name = table_name
        attr_name = attrs[i][0]
        # (attr_name, attr_type, attr_length)
        if not CatalogManager.attr_unique(table_name, attr_name):
            continue
        condition_rvalue = return_value.column_data[i]
        condition_comparator = "="
        condition_lvalue = attr_name
        unique_value.condition.append(CONDITION(condition_lvalue, condition_comparator, condition_rvalue))
        data_tuple = API.api_select(unique_value)
        if len(data_tuple) != 0:
            print("Inserted data on attribute {} should be unique!".format(attr_name))
            return False

    return True

parser = yacc.yacc()

def execute_user_input():
    reset()
    data = input("minisql>")
    while ";" not in data:
        data = data + input()
    data = data.strip("\n")
    print(data)

    start_time = time.time()
    res = parser.parse(data)
    end_time = time.time()
    print("Execute time: {:.2f}s".format(end_time-start_time))

def execute_file():
    global execfile, execfile_name
    if not os.path.exists(execfile_name):
        print("File does not exist!")
        return -1

    with open(execfile_name, "r") as f:
        while True:
            data = ""
            new_line = f.readline()
            if not new_line:
                print("Detect empty line, quit...")
                return 0

            reset()
            while ";" not in data:
                if not new_line:
                    print("No end token found!")
                    return -1
                data += new_line
                if ";" in new_line:
                    break
                new_line = f.readline()
            data = data.strip("\n")
            print(data)
            parser.parse(data)

def Interpret():
    API.api_initialize()
    global execfile, execfile_name
    execfile = 0

    while True:
        if execfile == 0:
            reset()
            execute_user_input()
        else:
            start_time = time.time()
            res = execute_file()
            end_time = time.time()
            print("Execute time: {:.2f}s".format(end_time-start_time))
            if res == -1:
                return
            execfile = 0


""" Algorithm
1. Traverse the list from the head (or start) node.
2. While traversing, compare each node with its next node.
3. If the data of the next node is the same as the current node
then delete the next node.
4. Before we delete a nodewe need to store the next pointer of the node """

class Node:
    def __init__(self, data):
        self.data = data
        self.next = None

class LinkedList:
    def __init__(self):
        self.head = None

    # Function to insert a new node
    # at the beginning
    def push(self, new_data):
        new_node = Node(new_data)
        new_node.next = self.head
        self.head = new_node

    # Given a reference to the head of a
    # list and a key, delete the first
    # occurrence of key in linked list
    # def deleteNode(self, key):
    #     curr = self.head

    #     # If head node itself holds the
    #     # key to be deleted
    #     if (curr is not None):
    #         if curr.data == key:
    #             self.head = curr.next
    #             curr = None
    #             return

    #    # Search for the key to be deleted,
    #     # keep track of the previous node as
    #     # we need to change 'prev.next'
    #     while (curr is not None):
    #         if curr.data == key:
    #             break
    #         prev = curr
    #         curr = curr.next

    #     # if key was not present in
    #     # linked list
    #     if curr == None:
    #         return
    #     prev.next = curr.next
    #     curr = None

    def printList(self):
        curr = self.head
        while curr:
            print(curr.data, end=" ")
            curr = curr.next

    def removeDuplicates(self):
        curr = self.head
        if curr is None:
            return
        while curr.next is not None:
            if curr.data == curr.next.data:
                new = curr.next.next
                curr.next = None
                curr.next = new
            else:
                curr = curr.next
        return self.head

llist = LinkedList()
llist.push(20)
llist.push(13)
llist.push(13)
llist.push(11)
llist.push(11)
llist.push(11)
print ("Created Linked List: ")
llist.printList()

print("Linked List after removing",
             "duplicate elements:")
llist.removeDuplicates()
llist.printList()


from django.shortcuts import render, redirect, get_object_or_404
from django.contrib.auth.decorators import login_required
from ..models import Crop, Transaction
from django.http import HttpResponse, JsonResponse
from django.db.models import Sum
import pandas as pd
import numpy as np

class RecommendedSystem:
    def __init__(self):
        self.crops = []
        self.kgs_sold = []
        self.kgs_received = []
        self.selling_price_per_kg = []
        self.production_cost_per_kg = []

    def dataframe(self):
        data = {
            "Crops": self.crops,
            "KG received": self.kgs_received,
            "KG Sold": self.kgs_sold,
            "Production Price": self.production_cost_per_kg,
            "Selling Price": self.selling_price_per_kg,
        }

        self.df = pd.DataFrame(data)
        return self.df

    def computations(self):
        self.df["Total_Produced_Price"] = (
            self.df["KG Sold"] * self.df["Production Price"]
        )
        self.df["Total_Selling_Price"] = self.df["KG Sold"] * self.df["Selling Price"]

        self.df["Unsold KG"] = self.df["KG received"] - self.df["KG Sold"]
        self.df["Profit made"] = (
            self.df["Total_Selling_Price"] - self.df["Total_Produced_Price"]
        )

    def recommended_crop(self):
        trial = self.df
        row_index = trial.loc[
            trial["Unsold KG"] == (trial.groupby("Crops")["Unsold KG"].min()).min()
        ]["Profit made"].idxmax()

        return self.crops[row_index]

@login_required(login_url="signin")
def analytics(request):
    if request.method == "GET":
        return render(request, "analytics.html")

    elif request.method == "POST":
        end = request.POST["end"]
        start = request.POST["start"]
        analysis = RecommendedSystem()

        for crop in Crop.objects.all():
            analysis.crops.append(crop.name)
            analysis.selling_price_per_kg.append(crop.selling_price)
            analysis.production_cost_per_kg.append(crop.production_cost)

            analysis.kgs_sold.append(
                Transaction.objects.filter(
                    transaction_type="sold", crop_id=crop.id
                ).aggregate(Sum("quantity"))["quantity__sum"]
            )
            analysis.kgs_received.append(
                Transaction.objects.filter(
                    transaction_type="received", crop_id=crop.id
                ).aggregate(Sum("quantity"))["quantity__sum"]
            )

        dataframe = analysis.dataframe()
        analysis.computations()
        recommended_crop = analysis.recommended_crop()

        return render(
            request,
            "analytics.html",
            {
                "dataframe": dataframe,
                "recommended_crop": recommended_crop,
            },
        )


# hangman program
# create a wordlist.txt file with some words

"""
import random
with open('wordlist.txt', 'r') as f:
    words = f.readlines()

word = random.choice(words)[:-1]
"""

word = "Dinesh"
allowed_errors = 8
guesses = []
done = False

while not done:
    for letter in word.lower():
        if letter in guesses:
            print(letter, end=" ")
        else:
            print("_", end=" ")

    print("")
    guess = input(f"Allowed errors left {allowed_errors}, next guess: ")
    guesses.append(guess.lower())
    # print(guesses)

    allowed_errors -= 1
    if allowed_errors == 0:
        break

    done = True
    for letter in word:
        if letter.lower() not in guesses:
            done = False

if done:
    print(f"You found! the word. it is '{word}'")

else:
    print(f"Game over!. the word was '{word}'")


from django.contrib import messages
from django.contrib.auth.decorators import login_required, permission_required
from django.shortcuts import render, get_object_or_404, redirect

from utils.forms import DeleteConfirmForm
from .forms import CoffeeForm
from .models import Coffee

# Create your views here.
@login_required
def index(request):
    coffees = Coffee.objects.all()
    return render(request, 'coffees/index.html', {'coffees': coffees})

@login_required
def show(request, pk):
    coffee = get_object_or_404(Coffee, pk=pk)
    return render(request, 'coffees/show.html', {'coffee': coffee})

@permission_required('coffees.add_coffee', raise_exception=True)
def add(request):
    form = CoffeeForm(request.POST or None)
    if form.is_valid():
        form.save()
        messages.success(request, '新增成功')
        return redirect('coffees:index')
    return render(request, 'coffees/add.html', {'form': form})

@permission_required('coffees.change_coffee', raise_exception=True)
def edit(request, pk):
    coffee = get_object_or_404(Coffee, pk=pk)
    form = CoffeeForm(request.POST or None, instance=coffee)
    if form.is_valid():
        form.save()
        messages.success(request, '更新成功')
        return redirect('coffees:index')
    return render(request, 'coffees/edit.html', {'form': form})

@permission_required('coffees.delete_coffee', raise_exception=True)
def delete(request, pk):
    coffee = get_object_or_404(Coffee, pk=pk)
    form = DeleteConfirmForm(request.POST or None)
    if form.is_valid() and form.cleaned_data['check']:
        coffee.delete()
        messages.success(request, '刪除成功')
        return redirect('coffees:index')
    return render(request, 'coffees/delete.html', {'form': form})


from typing import List
class Solution:
    def change(self, amount: int, coins: List[int]) -> int:
        dp = [0]*(amount+1)
        dp [0] = 1
        for i in coins:
            for j in range(1,amount+1):
                if j-i>=0:
                    dp[j] +=dp[j-i]
        return dp[-1]
A = Solution()
a = 5
b = [1,2,5]
print(A.change(a,b))


from django.conf.urls import include, url
from django.contrib import admin
from django.contrib.auth import views as auth_views

from mastersheet.views import *

urlpatterns = [
    url(r'^report_table_cm_activity_count/$', report_table_cm_activity_count, name="report_table_cm_activity_count"),
    url(r'^report_table_cm/$', report_table_cm, name="report_table_cm"),
	url(r'^report_table/$', give_report_table_numbers, name="give_report_table_numbers"),
    url(r'^report_table_accounts/$', give_report_table_numbers_accounts, name="give_report_table_numbers_accounts"),
	url(r'^details/$', give_details, name="give_details"),
	url(r'^report/$', render_report, name="render_report"),
	url(r'^show/report/$', create_report, name="create_report"),
    url(r'^delete_selected/$', delete_selected, name="delete_selected"),
    url(r'^files/$', file_ops, name="file_ops"),
    url(r'^columns/$', define_columns, name="define_columns"),
    url(r'^show/mastersheet/$', renderMastersheet, name="renderMastersheet"),
    url(r'^list/show/$', masterSheet, name="masterSheet"),
    url(r'^sync/slum/$', sync_kobo_data, name='sync_kobo_data'),
    url(r'^account/excel/$', accounts_excel_generation, name='accounts_excel_generation'),
    url(r'^show/summery/$', renderSummery, name="renderSummery"),
    url(r'^show/showSummery/$', ProcessShortView, name='ProcessShortView'),
    url(r'^gisDataDownload/$', gisDataDownload, name='gisDataDownload')
]


from flask import Blueprint, Response, request, jsonify
import json

import logging
import requests
import re
import threading

from file_worker import FileWorker

logger = logging.getLogger(__name__)

def construct_blueprint(app_config):
    file_info_index_blueprint = Blueprint('file_info_index', __name__)
    app_config = app_config

    """
    Index file info documents for the files in a dataset
    Returns
    -------
    json
        A json containing details of the dataset indexing operation
    """
    @file_info_index_blueprint.route('/datasets/<dataset_id>/reindex', methods=['PUT'])
    def create_update_dataset_file_infos(dataset_id):
        try:
            fworker = FileWorker(app_config=app_config, request_headers=request.headers)

            # Specify argument to confirm dataset_id is for a Dataset entity.
            theDataset = fworker.get_entity(entity_id=dataset_id, entity_type_check=['DATASET','PUBLICATION'])

            if not fworker.verify_user_in_write_group(aDataset=theDataset) and not fworker.verify_user_is_data_admin():
                raise Exception(f"Permission denied for modifying ES index entries for '{theDataset['uuid']}'.")

            dataset_uuid = theDataset['uuid']

            # Confirm the Elastic Search index exists? Or leave that to search-api?
            # https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-exists.html
            # Need to add "HEAD id" to search-api, if not calling ES directly.

            asynchronous = request.args.get('async')

            index_response_dict = {}

            if asynchronous:
                index_response_dict[dataset_uuid] = fworker.index_dataset(aDataset=theDataset)

                # Consolidate the successful responses together into a subdictionary in the JSON response.
                dataset_files_info_response_dict = {'dataset_uuid': dataset_uuid,
                                                    'file_count': len(index_response_dict[dataset_uuid]),
                                                    'index_responses': {}}
                for file_uuid in index_response_dict[dataset_uuid].keys():
                    if not file_uuid in dataset_files_info_response_dict['index_responses']:
                        dataset_files_info_response_dict['index_responses'][file_uuid] = []
                    for es_index_name in index_response_dict[dataset_uuid][file_uuid].keys():
                        dataset_files_info_response_dict['index_responses'][file_uuid].append(index_response_dict[dataset_uuid][file_uuid][es_index_name])

                return Response(json.dumps(dataset_files_info_response_dict)), 200
            else:
                try:
                    threading.Thread(target=fworker.index_dataset, args=[theDataset]).start()
                    # discard the index_response_dict, as it will be logged at the INFO level by the FileWorker

                    logger.info(f"Started to clear documents for uuid '{dataset_uuid}' from the Elasticsearch index.")
                except Exception as e:
                    logger.exception(e)
                    raise Exception(e)

                return f"Request to load file info documents for uuid '{dataset_uuid}' into the Elasticsearch index accepted", 202

        except requests.exceptions.HTTPError as he:
            eMsg = he.response.text if he.response.text is not None else 'Undescribed HTTPError'
            eCode = he.response.status_code if he.response.status_code is not None else 500
            logger.error(str(eCode) + ':' + eMsg, exc_info=True)
            return Response(eMsg, eCode)
        except Exception as e:
            logger.error(str(e), exc_info=True)
            return Response('Unexpected error: ' + str(e), 500)
        return Response(json.dumps(dataset_files_info_response_dict))

    """
    Index all Datasets.
    Returns
    -------
    json
        A json containing details of the dataset indexing operation
    """
    @file_info_index_blueprint.route('/datasets/reindex-all', methods=['PUT'])
    def reindex_dataset_files_docs():
        fworker = FileWorker(app_config=app_config, request_headers=request.headers)

        # Verify the user is a Data Admin, who can index all datasets
        if not fworker.verify_user_is_data_admin():
            raise Exception("Permission denied for requested operation.")

        asynchronous = request.args.get('async')

        if asynchronous:
            return fworker.index_all_datasets()
        else:
            try:
                threading.Thread(target=fworker.index_all_datasets, args=[]).start()
                # discard the index_response_dict, as it will be logged at the INFO level by the FileWorker

                logger.info("Started to index file info documents for all Datasets to the public and private Elasticsearch indices.")
            except Exception as e:
                logger.exception(e)
                raise Exception(e)

        return "Request to index file info documents for all Datasets into the public and private Elasticsearch indices accepted", 202

    @file_info_index_blueprint.route('/datasets/refresh-indices', methods=['PUT'])
    def refresh_indices():
        fworker = FileWorker(app_config=app_config, request_headers=request.headers)

        # Verify the user is a Data Admin, who can index all datasets
        if not fworker.verify_user_is_data_admin():
            raise Exception("Permission denied for requested operation.")

        asynchronous = request.args.get('async')

        if asynchronous:
            return fworker.refresh_indices()
        else:
            try:
                threading.Thread(target=fworker.refresh_indices, args=[]).start()
                # discard the index_response_dict, as it will be logged at the INFO level by the FileWorker
                logger.info(f"Started to refresh the public and private Elasticsearch indices with"
                            f" file info documents for added, changed, or deleted Datasets.")
            except requests.HTTPError as he:
                # OpenSearch errors come back in the JSON, so return them that way.
                return jsonify(he.response.json()), he.response.status_code
            except Exception as e:
                logger.exception(e)
                raise Exception(e)

        return  (   f"Request to refresh the public and private OpenSearch indices with file info documents for"
                    f" added, changed, or deleted Datasets accepted"), 202

    # end construct_blueprint()
    return file_info_index_blueprint


import pathlib

from mara_pipelines.commands.sql import ExecuteSQL
from mara_pipelines.pipelines import Pipeline, Task
from etl_tools import config

def utils_pipeline(with_hll=False, with_cstore_fdw=False):
    pipeline = Pipeline(
        id="initialize_utils",
        description="Creates an utils schema with a number of functions around the ETL best practices of Project A",
        base_path=pathlib.Path(__file__).parent)

    pipeline.add_initial(
        Task(
            id="create_utils_schema",
            description="Re-creates the utils schema",
            commands=[
                ExecuteSQL(sql_statement="DROP SCHEMA IF EXISTS util CASCADE; CREATE SCHEMA util;")
            ]))

    pipeline.add(
        Task(id='chunking',
             description='Runs file chunking.sql',
             commands=[
                 ExecuteSQL(sql_file_name='chunking.sql', echo_queries=False,
                            replace={'number_of_chunks': lambda: config.number_of_chunks()})
             ]))

    def add_task_for_file(file_name_without_extension):
        pipeline.add(
            Task(id=file_name_without_extension,
                 description=f'Runs file "{file_name_without_extension}.sql"',
                 commands=[
                     ExecuteSQL(sql_file_name=file_name_without_extension + '.sql',
                                echo_queries=False)
                 ]))

    for file_name_without_extension in ['consistency_checks', 'data_sets', 'partitioning',
                                        'indexes_and_constraints',  'schema_switching', 'enums']:
        add_task_for_file(file_name_without_extension)

    if with_hll:
        add_task_for_file('hll')

    if with_cstore_fdw:
        add_task_for_file('cstore_fdw')

    return pipeline


'''
Um ano é bissexto se:

O ano for divisivel por 4, mas não divisível por 100, exceto se ele for também
divisivel por 400.

Exemplo anos bissexto: 1600, 1732, 1944, 2008

Exemplo anos não bissexto: 1742, 1889, 1951, 2011
'''

import pytest

# Função para calcular ano bissexto
def eh_bissexto(ano):
    resto4 = ano % 4
    resto100 = ano % 100
    resto400 = ano % 400

    if ((not resto4) and (resto100)) or (not resto400):
        return True
    return False

### Não deve ser bissexto
@pytest.mark.parametrize('ano', [500, 600, 1500, 1742, 1889, 1951, 2011])  # Setup
def test_nao_deve_ser_bissexto(ano):
    resp = eh_bissexto(ano)  # Execução do teste

    assert resp is False  # Verificação do resultado do teste

### Deve ser bissexto
@pytest.mark.parametrize('ano', [1600, 1732, 1944, 2008])  # Setup
def test_deve_ser_bissexto(ano):
    resp = eh_bissexto(ano)  # Execução do teste

    assert resp is True  # Verificação do resultado do teste


# Environment ------------------------------------------------------------------
import os
import click
from pathlib import Path

# Paremeters are read from the config.py file
import config

# Main -------------------------------------------------------------------------
def _uniq(input):
    """Search for unique subject ID"""
    output = []
    for x in input:
        if x not in output:
            output.append(x)
    return output

def _parse_range(str_):
    """
Parse a string like numeric index set to a list
INPUT : "0-2,5,9-11"
OUTPUT: [0, 1, 2, 5, 9, 10, 11]
    """
    list_ = set()
    for chunk in str_.split(','):
        unit = chunk.split('-')
        list_.update(range(int(unit[0]), int(unit[-1]) + 1))
    return sorted(list_)

@click.command()
@click.argument('range')
@click.option('--nthreads', default=8, help='Number of Threads', show_default=True)
@click.option('--dryrun', is_flag=True, help='print cmd only')
def main(range, nthreads, dryrun):
    """Run MRIQC"""
    bids_dir = Path(config.outdir, 'BIDS_out')
    output_dir = Path(config.outdir, 'derivatives', 'mriqc')
    work_dir = Path(config.outdir, 'Work')

    subjlist = _parse_range(range)
    tmplist = []

    # check if output directory exists; if not, make it
    if not output_dir.is_dir():
        output_dir.mkdir(parents=True, exist_ok=True)

    for i in config.datasets:
        tmplist.append(i[1])
    labels = _uniq(tmplist)

    for subj in subjlist:
        label = labels[subj]

        cmd = f'singularity run --cleanenv \
            -B {bids_dir}:/data:ro \
            -B {output_dir}:/out \
            -B {work_dir}:/work \
            {config.mriqc} \
            /data \
            /out \
            participant \
            --participant-label sub-{label} \
            --n_proc {nthreads} \
            --no-sub \
            -w /work'

        if dryrun:
            print(cmd)
        else:
            os.system(cmd)

# Terminal Function ------------------------------------------------------------
if __name__ == '__main__':
    main()


def getstats(x, y):
    """
    Determine the statistical measured for an array of data points 'x' and 'y'

    Inputs
    ----------
    x: Independent varriable

    y: dependent varriable.

    Outputs
    ----------
    mean: Mean of 'x'

    dmean: Uncertainty in the mean

    variance: Variance

    """
    import numpy as np

    #Make sure the input is an array
    x = np.array(x)
    y = np.array(y)

    #Check to make sure the inputs are the same length
    if len(x) != len(y):
        print('ERROR: Arrays x ({}) and y ({}) need to be the same length'.format(len(x), len(y)))
        return

    sy = sum(y)
    sxy = sum(x*y)
    sx2y = sum(x**2 * y)

    mean = sxy / sy

    if len(x) > 1:
        variance = (sx2y -2*mean*sxy + mean**2 * sy) / (sy - 1)
    else:
        variance = 0

    dmean = np.sqrt(variance / sy)
    stats = np.array([mean, dmean, variance])

    return stats

Physics 331/Textbook Programs/Chapter 8/getstats.py


from parse import parse
import numpy as np
import pandas as pd

with open("data/day7.txt") as file:
    numbers = [int(i) for i in file.readline().rstrip().split(",")]

    # Looked like it would be the median. Turned out to be the median.
    median = np.median(numbers)
    val = np.linalg.norm([n - median for n in numbers], 1)
    print(val)

    # Just went with the fastest way I can think of the problem. Find min/max, iterate.
    mi = numbers[0]
    ma = numbers[0]
    for num in numbers[1:]:
        mi = min(num, mi)
        ma = max(num, ma)

    val = 400000000000000000000000000000
    for i in range(mi, ma + 1):
        next = sum([(abs(n - i) * (abs(n - i) + 1) / 2) for n in numbers])
        val = min(next, val)
    print(val)


import cv2
import numpy as np
import time

# modelin yüklenmesi ve hazırlığı
face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')

cap = cv2.VideoCapture(0)

size = 1.6
sil, img = cap.read()
img = cv2.resize(img, None, fx=size, fy=size)
height, width, channels = img.shape  #640,480 if size == 1
oran = 0.4
#hedef alaninin kenar uzunluklarının yaty ve dikeyle oranı
hedef = (int(width * (1-oran)//2), int(height * (1-oran)//2), int(width*oran), int(height*oran))
izgara = [((0, 0), (hedef[0], hedef[1])),                                   #kuzey-batı bolgesi
          ((hedef[0], 0), (hedef[0]+hedef[2], hedef[1])),                   #kuzey bölgesi
          ((hedef[0]+hedef[2], 0), (width, hedef[1])),                      #kuzey-doğu bölgesi
          ((0, hedef[1]), (hedef[0], hedef[1]+hedef[3])),                   #batı bölgesi
          ((hedef[0], hedef[1]), (hedef[0]+hedef[2], hedef[1]+hedef[3])),   #merkez bölge
          ((hedef[0]+hedef[2], hedef[1]), (width, hedef[1]+hedef[3])),      #doğu bölgesi
          ((0, hedef[1]+hedef[3]), (hedef[0], height)),                     #güney-batı bölgesi
          ((hedef[0], hedef[1]+hedef[3]), (hedef[0]+hedef[2], height)),     #güney bölgesi
          ((hedef[0]+hedef[2], hedef[1]+hedef[3]), (width, height))]        #güney-doğu bölgesi
starting_time = time.time()
font = cv2.FONT_HERSHEY_PLAIN

def icinde_mi(merkez_nokta, kose_nokta1, kose_nokta2):
    # girilen merkez nokta, sol-üst ve sağ-alt noktasi verilen dikdörtgenin içerisinde mi kontrol eden fonksiyon
    if kose_nokta1[0] <= merkez_nokta[0] <= kose_nokta2[0] and kose_nokta1[1] <= merkez_nokta[1] <= kose_nokta2[1]:
        return True
    else:
        return False

def merkezler_arasi_mesafe(balon):
    # girilen balon dikdörtgeninin merkezi ile hedef alanı merkezi arasındaki mesafeyi veren fonksiyon
    x1, y1, w1, h1 = balon
    x2, y2, w2, h2 = hedef
    return int(
        (abs((x1 + (w1 // 2)) - (x2 + (w2 // 2))) ** 2 + abs((y1 + (h1 // 2)) - (y2 + (h2 // 2))) ** 2) ** (1 / 2))

def yon(balon):
    # verilen balonun izgara içerisinde ki hangi bölgede olduğu bilgisini veren fonksiyon
    x1, y1, w1, h1 = balon
    balon_merkez_nokta = (x1+(w1//2), y1+(h1//2))

    if icinde_mi(balon_merkez_nokta, izgara[0][0], izgara[0][1]):
        return f"kb"
    elif icinde_mi(balon_merkez_nokta, izgara[1][0], izgara[1][1]):
        return f"kk"
    elif icinde_mi(balon_merkez_nokta, izgara[2][0], izgara[2][1]):
        return f"kd"
    elif icinde_mi(balon_merkez_nokta, izgara[3][0], izgara[3][1]):
        return f"bb"
    elif icinde_mi(balon_merkez_nokta, izgara[4][0], izgara[4][1]):
        return f"mm"
    elif icinde_mi(balon_merkez_nokta, izgara[5][0], izgara[5][1]):
        return f"dd"
    elif icinde_mi(balon_merkez_nokta, izgara[6][0], izgara[6][1]):
        return f"gb"
    elif icinde_mi(balon_merkez_nokta, izgara[7][0], izgara[7][1]):
        return f"gg"
    elif icinde_mi(balon_merkez_nokta, izgara[8][0], izgara[8][1]):
        return f"gd"

def main_loop():
    # ana fonksiyon
    frame_counter = 0
    while cap.isOpened():
        frame_counter += 1
        _, img = cap.read()
        img = cv2.resize(img, None, fx=size, fy=size)
        #img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        # nesne tanima
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(gray, scaleFactor=1.5, minNeighbors=5)
        if len(faces) > 0:
            for (x, y, w, h) in faces:
                roi_gray = gray[y:y + h, x:x + w]
                color = (0, 255, 0)  # BGR
                stroke = 2

                cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2)
                cv2.circle(img, ((x + (w // 2)), (y + (h // 2))), 10, (0, 250, 0), cv2.FILLED)
                print(yon([x,y,w,h]))

        #hedef alani cizdirme
        cv2.rectangle(img, (hedef[0], hedef[1]),
                      (hedef[0] + hedef[2], hedef[1] + hedef[3]),
                      (170, 20, 125), 3)
        #cv2.putText(img, "Hedef Alani", (hedef[0]+5, hedef[1]+20), font, 1, (170, 20, 125), 2)

        #fps
        elapsed_time = time.time() - starting_time
        fps = frame_counter / elapsed_time
        cv2.putText(img, "FPS: " + str(round(fps, 2)), (10, 50), font, 2, (0, 0, 0), 2)

        #görüntüyü ekrana bastirma
        cv2.imshow("iha goruntu", img)

        # komutlar (renk degistirme, kapatma, vb.)
        komut = cv2.waitKey(20)
        if komut & 0xFF == ord('q'):
            print(f"akış sonlandırıldı..!")
            break

    cap.release()
    cv2.destroyAllWindows()

#start
main_loop()


import logging
import threading
import time
import random

from util import logcfg

class ThreadWorker(threading.Thread):
    def __init__(self, name):
        super().__init__(name=name)

    def run(self):
        logging.info("Thread iniciado")
        time.sleep(random.random() * 10)
        logging.info("Thread acabado")

def main():
    threads = [ThreadWorker(f"Hilo {i+1}") for i in range(10)]
    for t in threads: t.start()
    while threads:
        time.sleep(0.2)
        for t in threads:
            if not t.is_alive():
                t.join()
                threads.remove(t)

    logging.info("Todas las tareas finalizadas")

if __name__ == '__main__':
    logcfg(__file__)
    logging.info("Programa iniciado")
    main()
    logging.info("Programa finalizado")


from quocca.camera import Camera
from quocca.plotting import show_circle, show_img, show_clouds
from quocca.cloudiness import GaussianRunningAvg, RunningAvg
from quocca.cloudiness import KNNMedian, cloud_map

from matplotlib import pyplot as plt

import click

@click.command()
@click.option('-i', '--input', help='Input File',
              type=str)
@click.option('-c', '--camera', help='Camera',
              type=str, default='cta')
@click.option('-d', '--detect', help='Detection method to be used',
              type=str, default='llh')
@click.option('-m', '--mag', help='Maximum magnitude to be considered',
              type=float, default=5.0)
@click.option('-cm', '--cloudmap', help='Cloud map method to be used',
              type=str, default='GaussianRunningAvg')
@click.option('-r', '--radius', help='Smoothing radius',
              type=float, default=25.0)
@click.option('-k', '--k', help='k of kNN cloud map generation',
              type=int, default=5)
@click.option('-s', '--smooth', help='Smoothing of the cloud map',
              type=int, default=0)
@click.option('-o', '--output', help='Output path',
              type=str, default=None)

def process(input, camera, detect, mag, cloudmap, radius, k, smooth, output):
    cam = Camera(camera)
    img = cam.read(input)
    img.add_catalog(max_mag=mag, min_dist=7, min_alt=0, max_var=1.5)

    if detect == 'llh':
        res = img.detect('llh', sigma=1.6, fit_size=4, tol=1e-15,
                            remove_detected_stars=True, presmoothing=0.5)
    elif detect == 'filter':
        res = img.detect('filter', sigma=1.0, fit_size=4)
    elif detect == 'blob':
        res = img.detect('blob')

    if cloudmap == 'GaussianRunningAvg':
        cmap = cloud_map(res.x_fit,
                        res.y_fit,
                        res.visibility,
                        extent=(cam.resolution['x'], cam.resolution['y']),
                        size=(200, 200),
                        cloudiness_calc=GaussianRunningAvg,
                        radius=radius,
                        smoothing=smooth)
    if cloudmap == 'RunningAvg':
        cmap = cloud_map(res.x_fit,
                        res.y_fit,
                        res.visibility,
                        extent=(cam.resolution['x'], cam.resolution['y']),
                        size=(200, 200),
                        cloudiness_calc=RunningAvg,
                        radius=radius,
                        method='median')
    elif cloudmap == 'KNNMedian':
        cmap = cloud_map(res.x_fit,
                        res.y_fit,
                        res.visibility,
                        extent=(cam.resolution['x'], cam.resolution['y']),
                        size=(200, 200),
                        cloudiness_calc=KNNMedian,
                        k=k,
                        smoothing=smooth)

    fig, ax = plt.subplots(figsize=(8,8))
    ax = show_img(img, ax=ax)
    ax = show_clouds(img, cmap, opaque=True, ax=ax)
    plt.tight_layout()

    if output == None:
        plt.show()
    else:
        plt.savefig(output+'Cloudmap_{}_{}.png'.format(detect, cloudmap))

    plt.close()

if __name__ == '__main__':
    process()


import numpy as np
import mahotas
import cv2

image = cv2.imread('testobstu.png',0)
blur = cv2.GaussianBlur(image, (5, 5), 0)
cv2.imshow("Image", image)

T = mahotas.thresholding.rc(blur)
thresh = blur.copy()
thresh[thresh > T] = 255
thresh[thresh < T]  = 0
cv2.imshow("Riddler-Calvard", thresh)
cv2.waitKey(0)
cv2.destroyAllWindows()


import sys
import pytesseract
import cv2
import concurrent.futures
import glob
from gtts import gTTS
import os
from pydub import AudioSegment
import PyPDF2
import time
from pdf2image import convert_from_path
import multiprocessing

try:
    from xml.etree.cElementTree import XML
except ImportError:
    from xml.etree.ElementTree import XML
import zipfile

processed_audio = 'processed_audio/'
unprocessed_audio = 'unprocessed_audio/'
old_unprocessed_audio = 'old_unprocessed_audio/'
ocr_images = 'ocr_images/'
ocr_text = "ocr_text/"
documents = "documents"
text_files = "text-files/"

if not os.path.isdir(processed_audio): os.mkdir(processed_audio)
if not os.path.isdir(unprocessed_audio): os.mkdir(unprocessed_audio)
if not os.path.isdir(old_unprocessed_audio): os.mkdir(old_unprocessed_audio)
if not os.path.isdir(ocr_images): os.mkdir(ocr_images)
if not os.path.isdir(ocr_text): os.mkdir(ocr_text)
if not os.path.isdir(documents): os.mkdir(documents)
if not os.path.isdir(text_files): os.mkdir(text_files)

WORD_NAMESPACE = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}'
PARA = WORD_NAMESPACE + 'p'
TEXT = WORD_NAMESPACE + 't'

def delete_old_files(filename_no_ext):
    # Remove any leftover audio
    i = 0
    while os.path.isfile(unprocessed_audio + filename_no_ext + str(i) + ".mp3"):
        os.rename(unprocessed_audio + filename_no_ext + str(i) + ".mp3",
                  old_unprocessed_audio + filename_no_ext + str(i) + ".mp3")
        i += 1
    # Remove any leftover images
    i = 0
    while os.path.isfile(ocr_images + str(i) + ".png"):
        os.remove(ocr_images + str(i) + ".png")
        i += 1

# Take the path of a docx file as argument, return the text in unicode.
def get_docx_text(path):
    document = zipfile.ZipFile(path)
    xml_content = document.read('word/document.xml')
    document.close()
    tree = XML(xml_content)
    paragraphs = []
    for paragraph in tree.getiterator(PARA):
        texts = [node.text
                 for node in paragraph.getiterator(TEXT)
                 if node.text]
        if texts: paragraphs.append(''.join(texts))

    return '\n\n'.join(paragraphs)

# Process the audio
def ask_google(string, i, filename_no_ext):
    tts = gTTS(text=string, lang='en')
    tts.save(unprocessed_audio + filename_no_ext + str(i) + ".mp3")
    print("Saving")

def image_to_text():
    os.environ['OMP_THREAD_LIMIT'] = '1'
    threads = multiprocessing.cpu_count()
    with concurrent.futures.ProcessPoolExecutor(max_workers=threads) as executor:
        image_list = list_files(ocr_images)
        print(image_list)
        for img_path, out_file in zip(image_list, executor.map(ocr, image_list)):
            print("I did a thing")

def ocr(img_path):
    img = cv2.imread(img_path)
    text = pytesseract.image_to_string(img, lang='eng', config='--psm 6')
    print(text)
    temp, filename = os.path.split(img_path)
    f = open(ocr_text + filename[:-4] + ".txt", 'w+')
    f.write(text)
    f.flush()
    return text

# Turn the large body of text into small pieces
def make_phrases(s):
    print(s)
    word_list = s.split(" ")
    phrase = ""
    phrase_list = []
    for i in range(len(word_list)):
        phrase += word_list[i] + " "
        if len(phrase) >= 110:
            phrase_list.append(phrase)
            phrase = ""
    return phrase_list

def make_threads(phrase_list, threadingCounter, filename_no_ext):
    time_start = time.time()
    i = 0
    threadingCounterDefault = threadingCounter
    length = len(phrase_list)
    cores = multiprocessing.cpu_count()
    print("The ammount of threads we will be making is " + str(length))
    pool = multiprocessing.Pool(
        cores * 40)  # use all available cores, otherwise specify the number you want as an argument
    for phrase in phrase_list:
        pool.apply_async(ask_google, args=(phrase, threadingCounter, filename_no_ext,))
        threadingCounter += 1
    pool.close()
    pool.join()
    print(time.time() - time_start)
    return threadingCounter - threadingCounterDefault

def make_full_track(filename_no_ext):
    num_files = len(list_files(unprocessed_audio))
    large_track = AudioSegment.from_mp3(unprocessed_audio + filename_no_ext + str(0) + ".mp3")
    for i in range(num_files - 1):
        large_track += AudioSegment.from_mp3(unprocessed_audio + filename_no_ext + str(i + 1) + ".mp3")
    large_track.export(processed_audio + filename_no_ext + "final.mp3", format='mp3')

def decide_pdfs(path_and_filename, filename_no_ext):
    # Turn the path into its smaller parts to use them individually
    path, filename = os.path.split(path_and_filename)
    filename_no_ext, ext = os.path.splitext(filename)
    # Initalize pdf reader
    pdfReader = PyPDF2.PdfFileReader(open(path_and_filename, 'rb'))
    # Determin whether a pdf is made up of text or it is scanned in
    x = 0
    num_pages = pdfReader.numPages - 1
    page_to_check = int(num_pages / 2)
    num_words_scanned = scanned_pdf_check(path_and_filename, filename_no_ext, page_to_check)
    num_words_text = len(str(pdfReader.getPage(page_to_check))) + 5
    while num_words_scanned < 20:
        if (page_to_check + 1 < num_pages):
            page_to_check += 1
        else:
            print("This is a scanned in pdf")
            scanned_pdf(path_and_filename, filename_no_ext)
        num_words_scanned = scanned_pdf_check(path_and_filename, filename_no_ext, page_to_check)
        num_words_text = len(str(pdfReader.getPage(page_to_check).extractText())) + 5

    if num_words_scanned > num_words_text + 10:
        print("This is a scanned in pdf")
        scanned_pdf(path_and_filename, filename_no_ext)
    else:
        print("This is a normal pdf.")
        normal_pdf(path_and_filename, filename_no_ext)

def normal_pdf(path_and_filename, filename_no_ext):
    threadingCounter = 0
    pdfReader = PyPDF2.PdfFileReader(open(path_and_filename, 'rb'))
    s = ''
    f = open(text_files + filename_no_ext + ".txt", 'w')
    for i in range(pdfReader.numPages):
        # Extract text from pdf paeg
        s = str(pdfReader.getPage(i).extractText())
        # Wirte to output.txt
        f.write(s)
        f.flush()
        # Turn into phrases small enough to send to api
        ph = make_phrases(s)
        # multithread and send to api
        threadingCounter += make_threads(ph, threadingCounter, filename_no_ext, )

def scanned_pdf(path_and_filename, filename_no_ext):
    f = open(text_files + filename_no_ext + ".txt", 'w')
    # Turn pdf into list of images
    images = convert_from_path(path_and_filename)
    # print(images)
    for i in range(len(images)):
        images[i].save(ocr_images + str(i) + ".png")
    image_to_text()
    i = 0
    s = ''
    threadingCounter = 0
    while os.path.isfile(ocr_text + str(i) + ".txt"):
        f1 = open(ocr_text + str(i) + ".txt", 'r')
        s += f1.read().replace('\n', ' ').replace('\t', ' ')
        i += 1

    print(s)
    ph = make_phrases(s)
    threadingCounter += make_threads(ph, threadingCounter, filename_no_ext)

def scanned_pdf_check(path_and_filename, filename_no_ext, pageNum):
    f = open(text_files + filename_no_ext + ".txt", 'w')
    # Turn pdf into list of images
    images = convert_from_path(path_and_filename)
    images[pageNum].save(ocr_images + "check_page" + ".png")
    # print(images)

    images[pageNum].save(ocr_images + "check_page" + ".png")
    return len(ocr(ocr_images + "check_page" + ".png"))

def image(path_and_filename, filename_no_ext):
    f = open(text_files + filename_no_ext + ".txt", 'w')
    text = pytesseract.image_to_string(path_and_filename)
    # Write to output.txt
    f.write(text)
    f.flush()
    # Make page into small enough phrases to send to api
    make_threads(make_phrases(text), 0, filename_no_ext)

def txt(path_and_filename, filename_no_ext):
    f = open(text_files + filename_no_ext + ".txt", 'w')
    # Better for debuging. Make into one line when done
    fr = open(path_and_filename, "r")
    # Make into smaller phrases to send to api for processing
    # Multi process the sending of the threads
    ph = make_phrases(fr.read())
    # print(fr.read())
    f.write(fr.read())
    f.flush()
    # print out text
    # print(ph)
    make_threads(ph, 0, filename_no_ext)

def docx(path_and_filename, filename_no_ext):
    f = open(text_files + filename_no_ext + ".txt", 'w')
    #   Better for debuging when done put it in one function call
    text = get_docx_text(path_and_filename)
    # Write to output.txt
    f.write(text)
    f.flush()
    ph = make_phrases(text)
    make_threads(ph, 0, filename_no_ext)

# List files in a directory going recursively.
def list_files(loc):
    filelist = []
    dirlist = list_dirs(loc)
    for path, dirs, files in os.walk(loc):
        for f in files:
            filelist.append(path + "/" + f)
    for dir in dirlist:
        for path, dirs, files in os.walk(dir):
            for f in files:
                filelist.append(path + "/" + f)
    # for file in filelist:
    #     print("We found " + file)
    return filelist

# returns a list of the directories in a folder
def list_dirs(loc):
    dirlist = []
    for path, dirs, files in os.walk(loc):
        for d in dirs:
            lol = (path + "/" + d)
            dirlist.append(lol)
    return dirlist

def get_latest_file(path):
    list_of_files = glob.glob(path + '/*')
    latest_file = max(list_of_files, key=os.path.getctime)
    return latest_file

def main(path_and_filename):
    # Turn the path into its smaller parts to use them individually
    path, filename = os.path.split(path_and_filename)
    filename_no_ext, ext = os.path.splitext(filename)
    # Image types
    image_types = {'.jpg', '.png', '.gif', '.jpeg', '.tif', '.raw'}

    # Get start time so that you can check how long it took
    time1 = time.time()
    if (ext.lower() == ".pdf"):
        decide_pdfs(path_and_filename, filename_no_ext)
    elif ext.lower() == ".txt":
        txt(path_and_filename, filename_no_ext)
    elif ext.lower() == ".docx":
        docx(path_and_filename, filename_no_ext)
    elif ext.lower() in image_types:
        image(path_and_filename, filename_no_ext)
    else:
        print("That is not a supported file format.")
        exit

    make_full_track(filename_no_ext)
    print("Deleting and moving extra files")
    delete_old_files(filename_no_ext)
    print("The time taken was " + str(time.time() - time1) + "\nDone!")

try:
    if 'help' in sys.argv[1]:
        print('The proper usage for the program is \n python3 main.py /path/to/file')
    main(sys.argv[1])
except IndexError:
    print('The proper usage for the program is \n python3 main.py /path/to/file')


import requests

class CampusPulseAPI():
    def __init__(self):
        pass

    def getOrganisations(self, query="", num_orgs=9999999):
        # Endpoint URL to retrieve organizations
        url = 'https://umassamherst.campuslabs.com/engage/api/discovery/search/organizations'

        # Parameters for the API request
        params = {'top': num_orgs, 'filter': '', 'query': query, 'skip': 0}

        # Send GET request to retrieve organizations
        response = requests.get(url, params=params)
        if response.status_code != 200:
            raise Exception("API is unavailable")
        response = response.json()

        # Return the list of organizations from the response
        return response['value']

    def getOrgWithId(self, id):
        # Endpoint URL to retrieve an organization by ID
        url = 'https://umassamherst.campuslabs.com/engage/api/discovery/branch'

        # Parameters for the API request
        params = {'orgId': id, 'take': 50, 'orderByField': 'name'}

        # Send GET request to retrieve the organization with the specified ID
        response = requests.get(url, params=params)
        if response.status_code != 200:
            raise Exception("API is unavailable")
        response = response.json()

        # Check if the organization exists and return the organization data
        if len(response['items']) == 0 or response['items'][0]['id'] != id:
            raise Exception("No existing organization with specified ID")
        return response['items'][0]


from typing import Optional

# Definition for singly-linked list.
class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next
class Solution:
    def deleteDuplicates(self, head: Optional[ListNode]) -> Optional[ListNode]:
        if head == None: return head
        output_head = output_tail = head
        start = head.next
        while start != None:
            if start.val != output_tail.val:
                output_tail.next = start
                output_tail = output_tail.next
            start = start.next
        if output_tail != None: output_tail.next = None
        return output_head


#prob5.py: lcm

def gcd(a,b):
    for i in range(a*b,0,-1):
        if a%i==0 and b%i==0:
            return i

def lcm(a,b):
    if a == b or b%a == 0:
        return b

    mul = (int)(a*b/gcd(a,b))

    while 1:
        if mul%a==0 and mul%b==0:
                return mul
        else:
            mul += 1

v_lcm = 2
for i in range(1,20):
    v_lcm = lcm(i, v_lcm)
    print(v_lcm)


part1 = part2 = 0

for line in open("input.txt", "r"):
    line = line.replace("|", " ").strip().split()
    line = [set(v) for v in line]
    patterns = line[:10]
    outputs = line[10:]

    unique_lengths = {2: 1, 3: 7, 4: 4, 7: 8}
    one, four = set(), set()
    for v in patterns:
        if len(v) == 2:
            one = v
        elif len(v) == 4:
            four = v

    for (i, digit) in enumerate(outputs):
        value = 0

        if len(digit) in unique_lengths:
            value = unique_lengths[len(digit)]
            part1 += 1
        elif len(digit) == 5:
            if digit.issuperset(one):
                value = 3
            elif len(digit & four) == 2:
                value = 2
            else:
                value = 5
        else:
            if not digit.issuperset(one):
                value = 6
            elif digit.issuperset(four):
                value = 9
            else:
                value = 0

        part2 += value * 10 ** (3 - i)

print(part1)
print(part2)


# System Modules
import matplotlib.pyplot as plt
import skvideo.io
import numpy as np

# Deep Learning Modules
from torch.utils.data import Dataset

# User Defined Modules
from serde import read_config
from utils.visualization import *

class Prediction:
    '''
    This class represents prediction process similar to the Training class.

    '''
    def __init__(self,cfg_path):
        self.params = read_config(cfg_path)
        self.cfg_path = cfg_path
        self.setup_cuda()

    def setup_cuda(self, cuda_device_id=0):
        '''Setup the CUDA device'''
        torch.backends.cudnn.fastest = True
        torch.cuda.set_device(cuda_device_id)
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    def setup_model(self, model,model_file_name=None):
        '''
        Setup the model by defining the model, load the model from the pth file saved during training.

        '''
        if model_file_name == None:
            model_file_name = self.params['trained_model_name']
        self.model_p = model().to(self.device)

        #Loads model from model_file_name and default network_output_path
#         self.model_p.load_state_dict(torch.load(self.params['network_output_path'] + '/' + model_file_name))
        self.model_p.load_state_dict(torch.load(self.params['network_output_path'] + "/epoch_" + '13' + '_' + model_file_name))
        # put the numer of the epoch inside ''

    def predict(self,predict_data_loader,visualize=True,save_video=False):
        # Reads params to check if any params have been changed by user
        self.params = read_config(self.cfg_path)
        #evaluation mode
        self.model_p.eval()

        if save_video:
            self.writer = self.create_video_writer()

        with torch.no_grad():
            for j, images in enumerate(predict_data_loader):
                #Batch operation: depending on batch size more than one image can be processed.
                images = images.float()
                images = images.to(self.device)

                outputs = self.model_p(images)

            #for each image in batch
                for i in range(outputs.size(0)):
                    image=images[i]/255
                    output=outputs[i]
                    output = torch.unsqueeze(output, 0)
                    overlay_image, prob_image = get_output_images(image,output)

                    #Converts image, overlay image and probability image to numpy so that it can be visualized using matplotlib functions later.
                    image = self.convert_tensor_to_numpy(image)
                    overlay_image_np = self.convert_tensor_to_numpy(overlay_image)
                    prob_image_np = self.convert_tensor_to_numpy(prob_image)

                    if save_video:
                        #Concatentates input and overlay image(along the width axis [axis=1]) to create video frame.
                        video_frame = np.concatenate((image, overlay_image_np), axis=1)
                        video_frame= video_frame * 255
                        video_frame = video_frame.astype(np.uint8)
                        #Writes video frame
                        self.writer.writeFrame(video_frame)

                    if(visualize):
                        display_output(image, prob_image_np, overlay_image_np)

            if save_video:
                self.writer.close()
                return play_notebook_video(self.params['output_data_path'], self.params['trained_model_name'])

    def create_video_writer(self):
        '''Initialize the video writer'''
        filename="outputvideo.mp4"
        #filename="outputvideo.mp4"
        output_path=self.params['output_data_path']
        self.writer = skvideo.io.FFmpegWriter(os.path.join(output_path, filename).encode('ascii'))
        return self.writer

    def convert_tensor_to_numpy(self,tensor_img):
        '''
        Convert the tensor image to a numpy image

        '''
        #torch has numpy function but it requires the device to be cpu
        tensor_img = tensor_img.cpu()
        tensor_img.to(torch.device('cpu'))
        np_img = tensor_img.numpy()

        # np_img image is now in  C X H X W
        # transposes the array to H x W x C
        np_img = np.transpose(np_img, (1,2,0))

        return np_img


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('api', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='EmailMessage',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=250)),
                ('user', models.ForeignKey(related_name=b'email_user', to=settings.AUTH_USER_MODEL)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.RemoveField(
            model_name='email',
            name='user',
        ),
        migrations.DeleteModel(
            name='Email',
        ),
    ]


#Script to create a new file with only unique elements
#File Takes two main arguments:
#The file from which the dictionary of words would be created,
#The file from which we will match words against the dictionary we created.

#Finally the name of the output can also be given (By default it is: unique.txt)

import argparse
result = "unique.txt"
parser = argparse.ArgumentParser(description = "Script that creates a File with only unique elements!!")

parser.add_argument("b",help = "The base record file to be used for matching")
parser.add_argument("s", help = "Source file for matching against the records")
parser.add_argument("-r","--result",help = "Name of the output file !!")
parser.add_argument("--wc",help = "No of words in the output file")
args = parser.parse_args()
#Reading from base file
base_ = open(args.b)
inp_ = open(args.s)
out_ = open(args.result, "w")
base_dict = set()
for word in base_.read().split('\n'):
	base_dict.add(word)

for word in inp_.read().split('\n'):
	if word not in base_dict: out_.write(word+'\n')

base_.close()
inp_.close()
out_.close()

if args.wc :
	out_ = open(args.result)
	print("words :",len(out_.read().split('\n')))
	out_.close()
print("--Complete--")


'''
Problem ID:
PE0015

Problem Title:
Lattice paths

Problem Description:
Starting in the top left corner of a 2*2 grid, and only being able to move to the right and down, there are exactly 6 routes to the bottom right corner.
How many such routes are there through a 20*20 grid?

Problem Solution:

'''
import time

def solution(n):
    result = 0
    P=[1,1]
    S=2*n
    return result

timer=time.time()
print(solution(2))
print(f'It took {round(time.time()-timer,4)} secs')


from flask import Flask, render_template, url_for, redirect, request
import cv2
import numpy as np
from keras.models import load_model
import statistics as st

app = Flask(__name__)

@app.route("/")
def home():
    return render_template("index.html")

@app.route("/<string:name_page>")
def page(name_page):
    return render_template(name_page)

@app.route("/<string:path>")
def path_fd(path):
    return path

@app.route('/camera', methods=['GET', 'POST'])
def camera():
        GR_dict={0:(0,255,0),1:(0,0,255)}
        model = load_model('./deep/model.hdf5')
        i = 0
        output = []
        # prevents openCL usage and unnecessary logging messages
        cv2.ocl.setUseOpenCL(False)

        # dictionary which assigns each label an emotion (alphabetical order)
        emotion_dict = {0: "Angry", 1: "Disgusted", 2: "Fearful", 3: "Happy", 4: "Neutral", 5: "Sad", 6: "Surprised"}

        # start the webcam feed
        # cap = cv2.VideoCapture(0)
        cap = cv2.VideoCapture(0)
        while (i <= 50):
            # Find haar cascade to draw bounding box around face
            ret, frame = cap.read()
            facecasc = cv2.CascadeClassifier('./deep/haarcascade_frontalface_default.xml')
            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            faces = facecasc.detectMultiScale(gray, scaleFactor=1.3, minNeighbors=5)

            for (x, y, w, h) in faces:
                # cv2.rectangle(frame, (x, y - 50), (x + w, y + h + 10), (255, 0, 0), 2)
                roi_gray = gray[y:y + h, x:x + w]
                cropped_img = np.expand_dims(np.expand_dims(cv2.resize(roi_gray, (48, 48)), -1), 0)
                prediction = model.predict(cropped_img)
                maxindex = int(np.argmax(prediction))
                predicted_emotion = emotion_dict[maxindex]
                output.append(predicted_emotion)
                cv2.rectangle(frame,(x,y),(x+w,y+h),GR_dict[1],2)
                cv2.rectangle(frame,(x,y-40),(x+w,y),GR_dict[1],-1)
                cv2.putText(frame, predicted_emotion, (x, y-10),cv2.FONT_HERSHEY_SIMPLEX,0.8,(255,255,255),2)

            i = i + 1

            # show the output frame
            cv2.imshow("Frame", frame)
            key = cv2.waitKey(1) & 0xFF
            if key == 27:
                cap.release()
                cv2.destroyAllWindows()
                break

            # if the `q` key was pressed, break from the loop
            if key == ord("q"):
                break
        print(output)
        cap.release()
        cv2.destroyAllWindows()
        final_output1 = st.mode(output)
        print(final_output1)
        return render_template("index.html", final_output=final_output1)
        # return redirect(url_for('home', final_output=final_output1))

    # return "hello"

if __name__ == "__main__":
    app.run(debug=True)


# coding: utf-8

# 통계와 그래프

# In[2]:

import pandas as pd

# In[4]:

get_ipython().magic('matplotlib inline')

# 그래프 노트북 출력 설정
#

# In[35]:

월간재무 = pd.read_excel('share/data/profit.xlsx')

# In[36]:

월간재무 = 월간재무.rename(columns={'월':'month', '이익':'benefit'})

# In[27]:

월간재무.plot()

# In[28]:

월간재무.set_index('month').plot()

# 글씨체 설정
#

# In[29]:

''' python
import matplotlib as mpl
mpl.rcParams['font.family']='batang'
'''

# In[37]:

월간재무 = 월간재무.set_index('month')
월간재무.plot(ylim=(0.0, 2.5))

# In[42]:

게임사용자 = pd.read_excel('share/data/game_users.xlsx')

# In[ ]:

# In[43]:

게임사용자 = 게임사용자.set_index('장르')

# In[45]:

게임사용자

# In[47]:

게임사용자['누적사용자수'].plot(kind='pie', subplots=True)

# In[51]:

게임사용자['만족도'].plot(kind='barh')

# 비교는 절대값 뿐만 아니라 도수가 중요합니다.

# In[52]:

만족도수 = 게임사용자['누적사용자수'] * 게임사용자['만족도']

# In[54]:

만족도수.plot(kind='barh')

# ## 도수와 평균
#

# In[55]:

평점 = pd.read_excel('share/data/ratings.xlsx')

# In[57]:

평점[:3]

# In[58]:

len(평점)

# 구간에 따라 데이터 분류

# In[59]:

구간구분 = [0, 20, 40, 60, 80, 100]

# In[62]:

평점분류 = pd.cut(평점['평점'], 구간구분)

# In[67]:

평점도수 = 평점분류.value_counts()

# In[70]:

평점도수 = 평점도수.sort_index()

# In[75]:

평점도수.plot(kind='bar')

# ## 히스토그램
# 도수분포표

# In[77]:

평점.hist(bins=5)

# In[79]:

게임시간 = pd.read_excel('share/data/playtime.xlsx')
게임시간[:5]

# In[81]:

게임시간.hist()

# 히스토그램을 비율로 표현하기

# In[82]:

게임시간.hist(normed=True)


from platform_modules.lcd_driver import LCD
from platform_modules.button_reader import ButtonReader
import time
import datetime
import config
import global_storage as gs

# Init LCD
lcd = LCD(config.LCD_ADDRESS)

# Init button reader
button_reader = ButtonReader()
button_reader.start()

# Save last value of buttons
last_button_1 = False
last_button_2 = False
last_button_3 = False
last_button_4 = False
last_button_ss1 = False
last_button_ss2 = False

while True:
    lcd.lcd_clear()
    lcd.lcd_display_string("UI testing", 1)
    lcd.lcd_display_string("Button status: ", 2)
    button_values = [gs.button_1, gs.button_2, gs.button_3, gs.button_4, gs.button_ss1, gs.button_ss2]
    button_values = list(map(int, button_values))
    lcd.lcd_display_string("{} {} {} {} {} {}".format(*button_values), 3)

    # Determine pressed buttons
    pressed_buttons = []
    if gs.button_1 and not last_button_1:
        pressed_buttons.append("1")
    if gs.button_2 and not last_button_2:
        pressed_buttons.append("2")
    if gs.button_3 and not last_button_3:
        pressed_buttons.append("3")
    if gs.button_4 and not last_button_4:
        pressed_buttons.append("4")
    if gs.button_ss1 and not last_button_ss1:
        pressed_buttons.append("S1")
    if gs.button_ss2 and not last_button_ss2:
        pressed_buttons.append("S2")

    if pressed_buttons:
        lcd.lcd_display_string("Pressed:" + ",".join(pressed_buttons), 4)

    # Update values
    last_button_1 = gs.button_1
    last_button_2 = gs.button_2
    last_button_3 = gs.button_3
    last_button_4 = gs.button_4
    last_button_ss1 = gs.button_ss1
    last_button_ss2 = gs.button_ss2

    time.sleep(1)

button_reader.join()


import cherrypy
import psycopg2

class Root():

    @cherrypy.expose
    def index(self):
        return 'index'

@cherrypy.expose
class Messages():

    def __init__(self):
        self.dbname = cherrypy.config['dbname']
        self.username = cherrypy.config['user']

    def GET(self):
        if cherrypy.session.get('messages', None):
            messages = ' '.join(cherrypy.session['messages'])
        else:
            messages = 'No messages'
        return messages

    def POST(self, message=None):
        if not cherrypy.session.get('messages', None):
            cherrypy.session['messages'] = []
        cherrypy.session['messages'].append(message)
        message_len = str(len(message))
        return message_len

    def add_message(self, mess_id, message):
        conn = psycopg2.connect(self.dbname + ' ' + self.username)
        cursor = conn.cursor()

    def get_message(self, mess_id):
        conn = psycopg2.connect(self.dbname + ' ' + self.username)
        cursor = conn.cursor()

def run():
    cherrypy.config.update('server.cfg')
    cherrypy.tree.mount(Root(), '/')
    cherrypy.tree.mount(Messages(), '/messages', 'app.cfg')
    cherrypy.engine.start()
    cherrypy.engine.block()

if __name__ == '__main__':
    run()