import sqlite3

conn = sqlite3.connect("tickets5.db")
cur = conn.cursor()

def displayAllTickets():
    sql = "SELECT * FROM tickets"
    cur.execute(sql)

    results = cur.fetchall()

    if results:
        printStuff(results)
    else:
        print("No data found")

    print()

def addTicket():
    actual_speed = int(input("Enter actual speed: "))
    posted_speed = int(input("Enter posted speed: "))
    age = int(input("Enter age of offender: "))
    violator_sex = str(input("Enter sex of offender: "))

    data = (None, actual_speed, posted_speed, age, violator_sex)
    sql = "INSERT INTO tickets VALUES (?, ?, ?, ?, ?)"

    cur.execute(sql, data)
    conn.commit()

def displayTicketsByOffender():
    violator_sex = input("Enter sex of offender: ")
    data = (violator_sex, )

    sql = "SELECT * FROM tickets WHERE violator_sex = ?"

    cur.execute(sql, data)
    results = cur.fetchall()

    if results:
        printStuff(results)
    else:
        print("Name not found")
        print()

def printStuff(data):
    print("%-10s %-12s %-10s %-5s %-12s " % ("ticketID", "Posted MPH", "MPH Over", "Age", "Violator Sex"))

    for row in data:
        over = row[1] - row[2]

        print(" %-10d %-12d %-10d %-5d %-12s " % (row[0], row[1], over, row[3], row[4]))

    print()

def main():

    while True:

        print("""
            Menu options. Choose 1, 2, 3, or 4:

            1. Display all Tickets
            2. Add a Ticket
            3. Filter by Offender Sex
            4. Save & Exit

        """)

        opt = input("Enter your choice, 1, 2, 3, or 4: ")

        if opt == "1":
            displayAllTickets()

        elif opt == "2":
            addTicket()

        elif opt == "3":
            displayTicketsByOffender()

        elif opt == "4":
            print()
            print("Goodbye")

            if conn:
                conn.close
            break

        else:
            print("Invalid entry, please re-enter your choice")
            print()

main()


## ~~~~~~~~~~~~~~~~~~
#  Deep Willy Network
## ~~~~~~~~~~~~~~~~~~

import numpy as np, json, sys, os
sys.path.append(os.path.dirname(__file__))

from willies import *

class DeepWilly(object):

    _willy_classes = {'connected': ConnectedWilly,
                      'dropout': DropoutWilly,
                      'convolutional': ConvolutionalWilly,
                      'pooling': PoolingWilly,
                      'stacking': StackingWilly}

    def __init__(self, cost_func = 'cross entropy'):

        self.willies = []
        self.n_willies = 0

        self.cost_func = cost_func
        if cost_func == 'cross entropy':
            self.cost  = lambda a, y: -np.mean(y * np.log(a + (a == 0)) + (1-y) * np.log(1-a + (a == 1)))
            self.dcost = lambda a, y: (1/a.shape[0]) * (a - y) / (a * (1 - a) + 1e-99)

        elif cost_func == 'quadratic':
            self.cost  = lambda a, y: 0.5 * np.mean((a - y)**2)
            self.dcost = lambda a, y: (1/a.shape[0]) * (a - y)

        else:
            raise Exception('Unsupported cost function: ' + cost_func)

    def add(self, willy):
        ''' Add a willy to the network and set it up based on the previous willy's output (if not the first one). '''

        if len(self.willies) == 0:
            if not willy.is_set_up:
                raise Exception('Input shape or number must be provided to first Willy.')

        else:
            willy.set_up(self.willies[-1].out_shape)

        self.willies.append(willy)
        self.n_willies += 1

    def forward_prop(self, X):
        ''' Forward propagates X through the willies. '''

        for willy in self.willies:
            X = willy.forward_prop(X)

        return X

    def backward_prop(self, X, y,
                      learn_rate, mom_rate, reg_rate):
        ''' Backward propagates errors while simultaneously updating weights in each layer. '''

        # Output layer dC/da
        batch_size = X.shape[0]
        dA = self.dcost(self.willies[-1].A, y)

        # Backpropagate, updating weights
        XA = [X] + [willy.A for willy in self.willies[:-1]]
        XA = XA[::-1]

        for w, willy in enumerate(reversed(self.willies)):
            willy.update_weights(dA, XA[w],
                                 learn_rate, mom_rate, reg_rate)
            dA = willy.backward_prop()

    @staticmethod
    def get_batches(X, y, batch_size):
        ''' Shuffles training data and gets random batch of desired size. '''

        if batch_size == -1 or batch_size >= X.shape[0]:
            return [X], [y]

        # Shuffle data
        shuffled_indices = np.random.permutation(len(X))
        shuffled_X = X[shuffled_indices]
        shuffled_y = y[shuffled_indices]

        # Get batch of desired size
        X_batches = []
        y_batches = []
        for i in range(X.shape[0]//batch_size):
            X_batches.append(shuffled_X[int(batch_size*i):int(batch_size*(i+1))])
            y_batches.append(shuffled_y[int(batch_size*i):int(batch_size*(i+1))])

        return X_batches, y_batches

    def train(self, X, y,
              num_iterations, batch_size,
              learn_rate, reg_rate, mom_rate = 0,
              verbose = False):
        ''' Builds network, trains using given data and training parameters. '''

        # Change dtypes
        X = X.astype(np.float32)
        y = y.astype(np.float32)

        # Initialise momenta to 0
        for willy in self.willies:
            willy.reset_momenta()

        # Train network
        for iteration in range(num_iterations):

            # Get batches:
            X_batches, y_batches = self.get_batches(X, y, batch_size)

            for batchX, batchy in zip(X_batches, y_batches):

                # Forward propagate
                self.forward_prop(batchX)

                # Backward propagate & update weights
                self.backward_prop(batchX, batchy,
                                   learn_rate, mom_rate, reg_rate)

            # Print progress
            if verbose:
                if iteration % verbose == 0:
                    print("Training cost on last batch: ", self.cost(self.willies[-1].A, batchy))

    def predict(self, X, pred_type = 'as is'):

        self.yhat = self.forward_prop(X)

        if pred_type == 'binary':
            self.yhat = 1 * (self.yhat > 0.5)

        elif pred_type == 'argmax':
            self.yhat = np.argmax(self.yhat, axis = 1).reshape(-1, 1)

        else:
            assert pred_type == 'as is', \
                "Provided argument pred_type (" + pred_type + ") not supported."

        return self.yhat

    def accuracy(self, X, y, pred_type = 'as is'):
        ''' Gets accuracy of predictions. '''

        return np.mean(self.predict(X, pred_type) == y)

    def save(self, filename):
        ''' Saves deep willy to file \filename. '''

        willy_data = []
        for willy in self.willies:
            willy_data.append(willy.save())

        data = {'cost': self.cost_func,
                'willies': willy_data}

        file = open(filename, "w")
        json.dump(data, file)
        file.close()

    @classmethod
    def load(cls, filename):
        ''' Loads deep willy from file \filename. '''

        file = open(filename, "r")
        data = json.load(file)
        file.close()

        deep_willy = cls(data['cost_func'])
        for willy in data['willies']:
            willy_class = DeepWilly._willy_classes[willy['willy']]
            deep_willy.add(willy_class.load(willy))

        return deep_willy

    def copy(self):
        ''' Replicates this deep willy using its attributes. '''

        copy = DeepWilly(cost_func = self.cost_func)

        for willy in self.willies:
            copy.willies.append(willy.copy())
            copy.n_willies += 1

        return copy


import requests, re

def scrape_images(link):
    # define our user headers
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36"
    }

    gallery_append = 'media?id=media0&ref=photoCollage&channel=RES_BUY'
    link_replace = '?channel=RES_BUY'
    regex_pattern = r'((?:https:)(?:[/|.|\w|\s|-])*(?:IMG_\d{2}_\d{4})\.(?:jpg|gif|png|jpeg))'

    gallery_link = link.replace(link_replace, gallery_append)

    res = requests.get(gallery_link, headers=headers)

    res.raise_for_status()

    matches = re.findall(regex_pattern, res.text)
    matches_clean = list(dict.fromkeys(matches))

    return matches_clean


import logging.config

DEFAULT_LEVEL = logging.WARNING
DEFAULT_FMT = '%(asctime)s | %(levelname)-8s | %(message)s'

def install(level=DEFAULT_LEVEL, fmt=DEFAULT_FMT):
    logging.basicConfig(level=level, format=fmt)

    try:
        import sys
        import colorlog

        formatter = colorlog.ColoredFormatter(
            fmt='%(log_color)s' + fmt + '%(reset)s',
            log_colors={
                'DEBUG': 'blue',
                'INFO': 'green',
                'WARNING': 'yellow',
                'ERROR': 'red',
                'CRITICAL': 'white,bg_red',
            }
        )

        handler = colorlog.StreamHandler(sys.stdout)
        handler.setFormatter(formatter)

        logging.root.handlers.clear()
        logging.root.addHandler(handler)

    except:
        pass


from __future__ import division
from __future__ import absolute_import
#typing

#overrides

from allennlp.common import squad_eval
from allennlp.training.metrics.metric import Metric

class SquadEmAndF1(Metric):
    u"""
    This :class:`Metric` takes the best span string computed by a model, along with the answer
    strings labeled in the data, and computed exact match and F1 score using the official SQuAD
    evaluation script.
    """
    def __init__(self)        :
        self._total_em = 0.0
        self._total_f1 = 0.0
        self._count = 0

    #overrides
    def __call__(self, best_span_string, answer_strings):
        u"""
        Parameters
        ----------
        value : ``float``
            The value to average.
        """
        exact_match = squad_eval.metric_max_over_ground_truths(
                squad_eval.exact_match_score,
                best_span_string,
                answer_strings)
        f1_score = squad_eval.metric_max_over_ground_truths(
                squad_eval.f1_score,
                best_span_string,
                answer_strings)
        self._total_em += exact_match
        self._total_f1 += f1_score
        self._count += 1

    #overrides
    def get_metric(self, reset       = False)                       :
        u"""
        Returns
        -------
        Average exact match and F1 score (in that order) as computed by the official SQuAD script
        over all inputs.
        """
        exact_match = self._total_em / self._count if self._count > 0 else 0
        f1_score = self._total_f1 / self._count if self._count > 0 else 0
        if reset:
            self.reset()
        return exact_match, f1_score

    #overrides
    def reset(self):
        self._total_em = 0.0
        self._total_f1 = 0.0
        self._count = 0

    def __str__(self):
        return "SquadEmAndF1(em={self._total_em}, f1={self._total_f1})"

SquadEmAndF1 = Metric.register(u"squad")(SquadEmAndF1)


"""Add agreed to TOS int field

Revision ID: 51398a87b2ef
Revises: 95c58503e9c0
Create Date: 2020-12-02 09:43:04.949189

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "51398a87b2ef"
down_revision = "95c58503e9c0"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column("users", sa.Column("accepted_tos", sa.Integer(), nullable=False, server_default="0"))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column("users", "accepted_tos")
    # ### end Alembic commands ###


from django.shortcuts import render,redirect,get_object_or_404
# CSRF
from django.views.decorators.csrf import csrf_exempt

from django.utils.http import urlsafe_base64_encode,urlsafe_base64_decode
from django.utils.encoding import force_bytes
from django.core.mail import EmailMessage
from django.utils.encoding import force_bytes, force_text
from django.core.mail import BadHeaderError, send_mail
from django.http import HttpResponse, HttpResponseRedirect
from django.contrib.sites.shortcuts import get_current_site
from django.template.loader import render_to_string
from django.utils.http import urlsafe_base64_encode,urlsafe_base64_decode
from django.utils.encoding import force_bytes
from django.core.mail import EmailMessage
from django.utils.encoding import force_bytes, force_text
from django.contrib import messages
import pdb

def home(request):
    return render(request, 'index.html')

# def contact(request):
#     return render(request, 'contact.html')

@csrf_exempt
def sendemail(request):
    username = request.POST.get('username')
    subject = "돌직구 사용자"+ username+ "님이 보내신 문의 메일입니다."
    message = request.POST.get('message')
    useremail = request.POST.get("useremail")
    if subject and message and useremail:
        try:
            send_mail(subject, message, useremail, ["rockjiggu16@gmail.com"])
        except BadHeaderError:
            return HttpResponse('Invalid header found.')
        return redirect('home')
    else:
        return HttpResponse("정확하게 입력해주세요.")

@csrf_exempt
def contact(request):
    if request.method == "POST":
        username = request.POST["username"]
        subject = "돌직구 사용자"+ username+ "님이 보내신 문의 메일입니다."
        message = request.POST["message"]
        useremail = request.POST["useremail"]

        emailContent = render_to_string('email.html', {
            "subject": subject,
            "useremail": useremail,
            "message":message,
        })
        emailAddress = "rockjiggu16@gmail.com"
        emailObject = EmailMessage(subject, emailContent, to=[emailAddress])
        emailObject.content_subtype = "html"

        result = emailObject.send()
        if result == 1:
            messages.info(request, "성공적으로 문의가 돌직구에 전달되었습니다.")
        else:
            messgaes.info(request, "문의에 실패하였습니다.")
        return redirect('contact')
    else:
        return render(request, 'contact.html')


from argparse import ArgumentParser
from inference import Infer

parser = ArgumentParser()

parser.add_argument("modelname", help="name of model to use")
parser.add_argument("imagepath", help="relative path to image")
parser.add_argument("--use_gpu", help="use gpu or not", nargs="?", default=False, const=True, type = bool)
args = parser.parse_args()

infer = Infer(args.use_gpu)

try:
    infer.infer(args.imagepath, args.modelname)
except:
    print("Something BAD happened!!!")


#import gevent.monkey
#gevent.monkey.patch_all()
import os
import sys
import time
import pprint
import logging
import requests
import grequests
import threading
import urllib.parse
from bs4 import BeautifulSoup

import db
import parse

logger = logging.getLogger('scraper')
logger.setLevel(logging.DEBUG)

SRC_DIR = os.path.dirname(os.path.realpath(__file__))
LOG_DIR = os.path.join(SRC_DIR, "..", "log")
LOG_FILENAME = "scraper.log"
LOG_FILEPATH = os.path.join(LOG_DIR, LOG_FILENAME)
fh = logging.FileHandler(LOG_FILEPATH, mode='w')
fh.setLevel(logging.ERROR)

# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(funcName)s - %(lineno)d - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)

# Set low so server isn't bombarded and begins to refuse.
MAX_SESSIONS = 1

domain = 'https://www.beeradvocate.com/'
places_rel_url = 'place/list/'
places_url = urllib.parse.urljoin(domain, places_rel_url)
places_params = {'start': 0, 'brewery': 'Y', 'sort': 'name'}

progress = {'breweries': 0, 'beers': 0, 'errors': 0}

def exception_handler(r, e):
	progress['errors'] += 1
	logger.error("REQUEST URL: {} EXCEPTION: {}".format(r.url, e))
	logger.error("{} ERRORS HAVE OCCURRED".format(progress['errors']))

def get_last_page_start():
	response = requests.get(url=places_url, params=places_params)
	soup = BeautifulSoup(response.text, features='lxml')
	last_page_tag = soup.find('a', text="last")
	last_link = last_page_tag['href']
	parsed = urllib.parse.urlparse(last_link)
	last_start_str = urllib.parse.parse_qs(parsed.query)['start'][0]
	last_start = int(last_start_str)
	logging.debug("get_last_page_start: last_start: {}".format(last_start))
	return last_start

def get_breweries():
	STEP = 20
	last_page_start = get_last_page_start()
	reqs = []
	for start in range(0, last_page_start, STEP):
		params = places_params.copy()
		params['start'] = start
		reqs.append(grequests.get(places_url, params=params, callback=get_breweries_handler))
	logger.info("STARTING THREADS to fetch brewery details.")
	res = grequests.map(reqs, size=MAX_SESSIONS, exception_handler=exception_handler)

def get_breweries_handler(response, *args, **kwargs):
	soup = BeautifulSoup(response.text, features='lxml')
	this_page_breweries = parse.places.breweries(soup)
	logger.info("this_page_breweries: {}".format(pprint.pformat(this_page_breweries)))
	logger.info("response time (s): {}".format(response.elapsed))
	db.breweries.extendleft(this_page_breweries)
	progress['breweries'] += len(this_page_breweries)
	logger.info("FETCHED: {} breweries.".format(progress['breweries']))

def get_brewery_details(paths):
	params = {'view': 'beers', 'show': 'all'}
	reqs = []
	for p in paths:
		url = urllib.parse.urljoin(domain, p)
		reqs.append(grequests.get(url, params=params,
			callback=get_brewery_details_handler))
	logger.info("STARTING THREADS to fetch brewery details.")
	res = grequests.map(reqs, size=MAX_SESSIONS, exception_handler=exception_handler)

def get_brewery_details_handler(response, *args, **kwargs):
	logger.info("RESPONSE received from {}".format(response.url))
	soup = BeautifulSoup(response.text, features='lxml')
	#############################################################################
	# This is possibly redundant as all this information can be gathered in the
	# previous operation when the links are fetched from places list.
	brewery = {}
	brewery['id'] = parse.url.brewery_id(response.url)
	brewery['name'] = parse.brewery.name(soup)
	db.breweries.appendleft(brewery)
	logger.info("ADDED brewery {} to write queue.".format(pprint.pformat(brewery)))
	#############################################################################
	this_brewery_beers = parse.brewery.beers(soup)
	db.beers.extendleft(this_brewery_beers)
	logger.info("ADDED {} beers to write queue.".format(len(this_brewery_beers)))
	progress['breweries'] += 1
	progress['beers'] += len(this_brewery_beers)
	logger.info("FETCHED: {} breweries and {} beers.".format(progress['breweries'], progress['beers']))
	time.sleep(1)

def get_beer_details(paths):
	# This function is redundant when first populating tha database as all info
	# can be extracted from the brewery profile page (except ranking which can be
	# calculated from scores stored in the database. It is useful to update the
	# info for beers already in the database but even when updating the previous
	# operation of fetching the brewery has most likely been performed anyway.
	reqs = []
	for p in paths:
		url = urllib.parse.urljoin(domain, p)
		reqs.append(grequests.get(url, allow_redirects=True, callback=get_beer_details_handler))
	logger.info("STARTING THREADS to fetch beer details.")
	res = grequests.map(reqs, size=MAX_SESSIONS, exception_handler=exception_handler)

def get_beer_details_handler(response, *args, **kwargs):
	print(response.status_code)
	print(response.url)
	soup = BeautifulSoup(response.text, features='lxml')
	print(soup)
	beer = {}
	beer['id'] = parse.url.beer_id(response.url)
	beer['brewery_id'] = parse.url.brewery_id(response.url)
	beer['name'] = parse.beer.name(soup)
	logger.info("name: {}".format(beer['name']))
	beer['score'] = parse.beer.score(soup)
	logger.info("score: {}".format(beer['score']))
	beer['ratings'] = parse.beer.ratings(soup)
	logger.info("ratings: {}".format(beer['ratings']))
	beer['ranking'] = parse.beer.ranking(soup)
	logger.info("ranking: {}".format(beer['ranking']))
	beer['style'] = parse.beer.style(soup)
	logger.info("style: {}".format(beer['style']))
	beer['abv'] = parse.beer.abv(soup)
	logger.info("abv: {}".format(beer['abv']))
	db.beers.appendleft(beer)
	logger.info("ADDED beer with ID = {} to write queue.".format(beer['id']))

def breweries():
	consumer_thread = threading.Thread(target=db.consumer)
	consumer_thread.start()
	get_breweries()
	db.fetching_breweries = False
	consumer_thread.join()

def brewery_details():
	to_fetch = db.read_brewery_ids()
	logger.info("{} breweries to fetch".format(len(to_fetch)))
	paths = ["/beer/profile/{}/".format(b) for b in to_fetch]
	consumer_thread = threading.Thread(target=db.consumer)
	consumer_thread.start()
	get_brewery_details(paths)
	db.fetching_breweries = False
	consumer_thread.join()

def beer_details():
	to_fetch = db.read_beer_ids()
	logger.info("{} beers to fetch".format(len(to_fetch)))
	paths = ["/beer/profile/{}/{}".format(b[0], b[1]) for b in to_fetch]
	consumer_thread = threading.Thread(target=db.consumer)
	consumer_thread.start()
	get_beer_details(paths[0:1])
	db.fetching_breweries = False
	consumer_thread.join()

def print_usage():
	print("USAGE: python3 scraper.py {breweries|brewery_details|beer_details}")

if __name__ == "__main__":
	if len(sys.argv) < 2:
		print_usage()
	elif len(sys.argv) == 2 and sys.argv[1] == "brewery_details":
		brewery_details()
	elif len(sys.argv) == 2 and sys.argv[1] == "breweries":
		breweries()
	elif len(sys.argv) == 2 and sys.argv[1] == "beer_details":
		beer_details()
	else:
		print_usage()


# stemming
# e.g. stemming will convert ["python","pythoner","pythoning","pythoned","pythonly"] to python
# e.g. stemming will convert ["interesting","interested"] to interest
# stemming may create some words that do not exits
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize

ps = PorterStemmer()
example_words = ["python","pythoner","pythoning","pythoned","pythonly"]

for w in example_words:
    print(ps.stem(w))

new_text = ("It is very import to be pythonly while you are pythoning with python. "
            "All pythoners have have pythoned poorly at least once.")
words = word_tokenize(new_text)
for w in words:
    print(ps.stem(w))


import msvcrt
import zipfile
import threading
from selenium import webdriver
from selenium.webdriver.edge.options import Options
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor,wait, FIRST_COMPLETED, ALL_COMPLETED

resultList = []
unfoundList = []
alltask = []

def main():
    file = 'C:\\Users\\niu\Desktop\\上传\CartoonCustomSticker.aab'

    dependenciesDict = {}

    threadList = []
    if zipfile.is_zipfile(file):
        print()
        z = zipfile.ZipFile(file, 'r')
        namelist = z.namelist()
        pool = ThreadPoolExecutor(len(namelist))
        for zFile in namelist:
            if zFile.endswith('.version'):
                # print('"' + zFile.split('/').pop()[:-8] + '":"' + str(z.read(zFile), encoding='utf-8').strip() + '",')
                # print(zFile.split('/').pop()[:-8])
                pair = zFile.split('/').pop()[:-8].split('_')
                # pool.submit(getLatestVersion,pair[0],pair[1])
                # alltask.append(task)
                t = threading.Thread(target=getLatestVersion,args=(pair[0],pair[1]))
                t.start()
                # threadList.append(t)
                # getLatestVersion(pair[0],pair[1])
                # dependenciesDict[pair[0]] = pair[1]

    print('查询中...')
    # t.start()
    # t.join()
    # wait(alltask,return_when=ALL_COMPLETED)

    # for group, artifact in dependenciesDict.items():
        # thread = threading.Thread(target=getLatestVersion, args=(group, artifact))
        # getLatestVersion(group, artifact)
        # threadList.append(thread)

    # for thread in threadList:
    #     thread.start()

    # for thread in threadList:
    #     thread.join()

    print('结果如下:')
    for item in resultList:
        print(item)

    print('\n未查询到的依赖如下:')
    print(unfoundList)

    msvcrt.getch()

def getLatestVersion(group_id, artifact_id):
    global resultList
    global unfoundList
    url = f'https://mvnrepository.com/artifact/{group_id}/{artifact_id}'
    # url = 'https://mvnrepository.com/artifact/androidx.appcompat/appcompat'  # 这里是Spring Boot库的URL
    options = Options()
    options.use_chromium = True
    options.add_argument("headless")
    options.add_argument("disable-gpu")
    options.add_argument(
        "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
    driver = webdriver.Edge(options=options)  # 这里使用Edge浏览器
    driver.get(url)
    html = driver.page_source

    # print(f'groupId:{group_id}, artifact:{artifact_id}')

    # print(html)
    soup = BeautifulSoup(html, 'html.parser')
    version_element = soup.find('a', {'class': 'vbtn release'})
    if not version_element is None:
        latest_version = version_element.text.strip()
        print(f'"{group_id}:{artifact_id}":"{latest_version}",')

        resultList.append(f'"{group_id}:{artifact_id}":"{latest_version}",')
    else:
        print(f'{group_id}_{artifact_id}')
        unfoundList.append(f'{group_id}:{artifact_id}')

    driver.quit()

main()


#! /usr/bin/env python

import sys
from collections import defaultdict
from intcode import IntCode

lines = []
for line in sys.stdin:
    lines.append(line.rstrip('\n'))

class Robot():
    program=None
    direction=(0,1)
    position=(0,0)
    panels=None

    def __init__(self, line) -> None:
        self.program = IntCode(line.split(','), inputs=[])
        self.panels = defaultdict(int)

    def move(self, val):
        dx,dy=self.direction
        if val==1:
            self.direction=dy,-dx
        else: # val==0:
            self.direction=-dy,dx

        self.position = tuple(self.position[i]+self.direction[i] for i in (0,1,))

    def run(self):
        while not self.program.finished:
            self.program.inputs.append(self.panels[self.position])
            self.program.run()
            if not self.program.finished:
                color, new_dir = self.program.outputs
                self.program.outputs.clear()
                self.panels[self.position] = color
                self.move(new_dir)

# Part 1
print("-- Part 1 --")
robot = Robot(lines[0])
robot.run()
dim = len(robot.panels.keys())
print(dim)

# Part 2
print("-- Part 2 --")
robot = Robot(lines[0])
robot.panels[0,0] = 1 # we start on a white panel instead
robot.run()

min_x, min_y, max_x, max_y = dim,dim,-dim,-dim
for (x,y) in robot.panels.keys():
    min_x = min(x,min_x)
    max_x = max(x,max_x)
    min_y = min(y,min_y)
    max_y = max(y,max_y)

for y in reversed(range(min_y, max_y+1)):
    print(''.join( '@' if robot.panels[x,y]==1 else ' ' for x in range(min_x, max_x+1)))


from abc import ABC, ABCMeta, abstractmethod
from datetime import datetime
from pprint import pprint
from typing import Dict

try:
    from dialogflow_v2 import SessionsClient
    from dialogflow_v2.proto.session_pb2 import (
        DetectIntentResponse,
        QueryInput,
        QueryResult,
        TextInput,
    )
except:
    SessionsClient = None
    DetectIntentResponse = None
    QueryInput = None
    QueryResult = None
    TextInput = None

from haps import SINGLETON_SCOPE, base, egg, scope
from haps.config import Config

from botkit.builtin_services.nlu.dialogflowconfig import DialogflowConfig
from botkit.builtin_services.nlu.messageunderstanding import MessageUnderstanding

@scope(SINGLETON_SCOPE)
@base
class INLUService(ABC):
    @abstractmethod
    def detect_intents(self, chat_id: int, message: str, language_code: str = None):
        pass

@egg
class DialogflowService(INLUService):
    config: DialogflowConfig = Config(DialogflowConfig.KEY)

    def __init__(self):
        self.session_client = SessionsClient.from_service_account_file(
            self.config.json_credentials_file
        )

    def detect_intents(
        self, chat_id: int, message: str, language_code: str = "en"
    ) -> MessageUnderstanding:
        session = self.session_client.session_file(self.config.project_id, chat_id)

        text_input = TextInput(text=message, language_code=language_code)

        query_input = QueryInput(text=text_input)

        response: DetectIntentResponse = self.session_client.detect_intent(
            session=session, query_input=query_input
        )
        result: QueryResult = response.query_result

        # Ignored result fields:
        #   - all_required_params_present
        #   - fulfillment_text
        #   - fulfillment_messages
        #   - webhook_source
        #   - webhook_payload
        #   - output_contexts
        #   - diagnostic_info

        return MessageUnderstanding(
            text=result.query_text,
            language_code=result.language_code,
            action=result.action,
            intent=result.intent.display_name,
            parameters=self._normalize_parameters(result.parameters),
            contexts=result.output_contexts,
            confidence=result.speech_recognition_confidence or result.intent_detection_confidence,
            date=datetime.now(),
        )

    def _normalize_parameters(self, params: Dict):
        result = {}
        for k, v in params.items():

            if "date" in k and v:
                if hasattr(v, "keys") and "date_time" in v:
                    accessor = v["date_time"]
                else:
                    accessor = v
                print(accessor)
                time_and_date: datetime = dateutil.parser.parse(accessor)
                result[k] = time_and_date
                continue

            result[k] = v

        return result

if __name__ == "__main__":
    conf = DialogflowConfig(
        project_id="userbot-9994a",
        json_credentials_file="C:/projects/userbot/dialogflow-credentials.json",
    )
    c = DialogflowService(conf)
    nlu = c.detect_intents(123, "!remind @tWiTfAcE to buy milk tomorrow at 6", "en")
    # print(nlu)
    pprint(nlu)


"""
Library and Wrapper for DHT11 and DHT22 sensors.
    Based on https://github.com/JurassicPork/DHT_PyCom/tree/pulses_get
    Extensions: Renamed module filename to dht (from dth.py) and added wrapper function
    For hardware connection: YELLOW/WHITE: PIN1 VCC through GPIO, PIN2: DATA through GPIO, PIN3: NC, PIN4: GDN. Use also a 4.7k PULL-UP for DATA
"""

import utime
import pycom
import sensors
from machine import Pin
import logging

class DTHResult:
    'DHT sensor result returned by DHT.read() method'

    ERR_NO_ERROR = 0
    ERR_MISSING_DATA = 1
    ERR_CRC = 2

    error_code = ERR_NO_ERROR
    temperature = -1
    humidity = -1

    def __init__(self, error_code, temperature, humidity):
        self.error_code = error_code
        self.temperature = temperature
        self.humidity = humidity

    def is_valid(self):
        return self.error_code == DTHResult.ERR_NO_ERROR

class DTH:
    'DHT sensor (dht11, dht21,dht22) reader class for Pycom'

    #__pin = Pin('P3', mode=Pin.OPEN_DRAIN)
    __dhttype = 0

    def __init__(self, pin, sensor=0):
        self.__pin = Pin(pin, mode=Pin.OPEN_DRAIN)
        self.__dhttype = sensor
        self.__pin(1)
        utime.sleep(1.0)

    def read(self):
        # pull down to low
        self.__send_and_sleep(0, 0.019)
        data = pycom.pulses_get(self.__pin,100)
        self.__pin.init(Pin.OPEN_DRAIN)
        self.__pin(1)
        #print(data)
        bits = []
        for a,b in data:
            if a ==1 and 18 <= b <= 28:
                bits.append(0)
            if a ==1 and 65 <= b <= 75:
                bits.append(1)
        #print("longueur bits : %d " % len(bits))
        if len(bits) != 40:
            return DTHResult(DTHResult.ERR_MISSING_DATA, 0, 0)
        #print(bits)
        # we have the bits, calculate bytes
        the_bytes = self.__bits_to_bytes(bits)
        # calculate checksum and check
        checksum = self.__calculate_checksum(the_bytes)
        if the_bytes[4] != checksum:
            return DTHResult(DTHResult.ERR_CRC, 0, 0)
        # ok, we have valid data, return it
        [int_rh, dec_rh, int_t, dec_t, csum] = the_bytes
        if self.__dhttype==0:        #dht11
            rh = int_rh         #dht11 20% ~ 90%
            t = int_t     #dht11 0..50°C
        else:            #dht21,dht22
            rh = ((int_rh * 256) + dec_rh)/10
            t = (((int_t & 0x7F) * 256) + dec_t)/10
            if (int_t & 0x80) > 0:
                t *= -1
        return DTHResult(DTHResult.ERR_NO_ERROR, t, rh)

    def __send_and_sleep(self, output, mysleep):
        self.__pin(output)
        utime.sleep(mysleep)

    def __bits_to_bytes(self, bits):
        the_bytes = []
        byte = 0

        for i in range(0, len(bits)):
            byte = byte << 1
            if (bits[i]):
                byte = byte | 1
            else:
                byte = byte | 0
            if ((i + 1) % 8 == 0):
                the_bytes.append(byte)
                byte = 0
        #print(the_bytes)
        return the_bytes

    def __calculate_checksum(self, the_bytes):
        return the_bytes[0] + the_bytes[1] + the_bytes[2] + the_bytes[3] & 255

def get_reading(data_pin, dht_model, vcc_pin=None):
    """ Returns temperature & humidity reading, for given VCC and DATA pins """
    sensors.set_sensor_power_on(vcc_pin)

    # measurement
    if dht_model == "DHT11":
        th = DTH(data_pin, 0)
    elif dht_model == "DHT22":
        th = DTH(data_pin, 1)
    else:
        th = None

    temp = None
    hum = None

    if th:
        result = th.read()

        if result.is_valid():
            temp = result.temperature
            hum = result.humidity
        else:
            logging.error("DHT model [" + dht_model + "]: invalid result.")

    sensors.set_sensor_power_off(vcc_pin)

    # return results
    # print('Temperature: {:3.2f}'.format(result.temperature/1.0))
    # print('Humidity: {:3.2f}'.format(result.humidity/1.0))
    return (temp, hum)


"""
Configuration reader for the population_gravity model

@author   Chris R. Vernon
@email:   chris.vernon@pnnl.gov

License:  BSD 2-Clause, see LICENSE and DISCLAIMER files

"""

import datetime
import os
import simplejson

import rasterio
import yaml

import pandas as pd

import population_gravity.downscale_utilities as utils

class ReadConfig:
    """Read configuration data either provided in the configuration YAML file or as passed in via arguments.

    :param config_file:                         string. Full path to configuration YAML file with file name and
                                                extension. If not provided by the user, the code will default to the
                                                expectation of alternate arguments.

    :param grid_coordinates_file:               string. Full path with file name and extension to the CSV file
                                                containing the coordinates for each 1 km grid cell within the target
                                                state. File includes a header with the fields XCoord, YCoord, FID.
                                                Where data types and field descriptions are as follows:
                                                (XCoord, float, X coordinate in meters),
                                                (YCoord, float, Y coordinate in meters),
                                                (FID, int, Unique feature id)

    :param historical_suitability_raster:       string. Full path with file name and extension to the suitability
                                                raster containing values from 0.0 to 1.0 for each 1 km grid cell
                                                representing suitability depending on topographic and land use and
                                                land cover characteristics within the target state.

    :param base_rural_pop_raster:         string. Full path with file name and extension to a raster containing
                                                rural population counts for each 1 km grid cell for the historical
                                                base time step.

    :param base_urban_pop_raster:         string. Full path with file name and extension to a raster containing
                                                urban population counts for each 1 km grid cell for the historical
                                                base time step.

    :param projected_population_file:           string. Full path with file name and extension to a CSV file containing
                                                population projections per year separated into urban and rural
                                                categories.  Field descriptions for require fields as follows:
                                                (Year, integer, four digit year),
                                                (UrbanPop, float, population count for urban),
                                                (RuralPop, float, population count for rural),
                                                (Scenario, string, scenario as set in the `scenario` variable)

    :param one_dimension_indices_file:          string. Full path with file name and extension to the text file
                                                containing a file structured as a Python list (e.g. [0, 1]) that
                                                contains the index of each grid cell when flattened from a 2D array to
                                                a 1D array for the target state.

    :param output_directory:                    string. Full path with file name and extension to the output directory
                                                where outputs and the log file will be written.

    :param alpha_urban:                         float. Alpha parameter for urban. Represents the degree to which the
                                                population size of surrounding cells translates into the suitability
                                                of a focal cell.  A positive value indicates that the larger the
                                                population that is located within the 100 km neighborhood, the more
                                                suitable the focal cell is.  More negative value implies less suitable.
                                                Acceptable range:  -2.0 to 2.0

    :param beta_urban:                          float. Beta parameter for urban. Reflects the significance of distance
                                                to surrounding cells on the suitability of a focal cell.  Within 100 km,
                                                beta determines how distance modifies the effect on suitability.
                                                Acceptable range:  -0.5 to 2.0

    :param alpha_rural:                         float. Alpha parameter for rural. Represents the degree to which the
                                                population size of surrounding cells translates into the suitability
                                                of a focal cell.  A positive value indicates that the larger the
                                                population that is located within the 100 km neighborhood, the more
                                                suitable the focal cell is.  More negative value implies less suitable.
                                                Acceptable range:  -2.0 to 2.0

    :param beta_rural:                          float. Beta parameter for rural. Reflects the significance of distance
                                                to surrounding cells on the suitability of a focal cell.  Within 100 km,
                                                beta determines how distance modifies the effect on suitability.
                                                Acceptable range:  -0.5 to 2.0

    :param scenario:                            string. String representing the scenario with no spaces. Must match
                                                what is in the `projected_population_file` if passing population
                                                projections in using a file.

    :param state_name:                          string. Target state name with no spaces separated by an underscore.

    :param historic_base_year:                  int. Four digit historic base year.

    :param projection_year:               int. Four digit first year to process for the projection.

    :param projection_end_year:                 int. Four digit last year to process for the projection.

    :param time_step:                           int. Number of steps (e.g. number of years between projections)

    :param rural_pop_proj_n:                    float.  Rural population projection count for the projected year being
                                                calculated.  These can be read from the `projected_population_file`
                                                instead.

    :param urban_pop_proj_n:                    float.  Urban population projection count for the projected year being
                                                calculated.  These can be read from the `projected_population_file`
                                                instead.

    :param calibration_urban_year_one_raster:   string. Only used for running calibration.  Full path with file name and
                                                extension to a raster containing urban population counts for each 1 km
                                                grid cell for year one of the calibration.

    :param calibration_urban_year_two_raster:   string. Only used for running calibration.  Full path with file name and
                                                extension to a raster containing urban population counts for each 1 km
                                                grid cell for year two of the calibration.

    :param calibration_rural_year_one_raster:   string. Only used for running calibration.  Full path with file name and
                                                extension to a raster containing rural population counts for each 1 km
                                                grid cell for year one of the calibration.

    :param calibration_rural_year_two_raster:   string. Only used for running calibration.  Full path with file name and
                                                extension to a raster containing rural population counts for each 1 km
                                                grid cell for year two of the  calibration.

    :param kernel_distance_meters:              float. Distance kernel in meters; default 100,000 meters.

    :param write_raster:                        boolean. Optionally export raster output; Default True

    :param write_csv:                           boolean. Optionally export raster as a CSV file without nodata values

    :param write_array2d:                       boolean. Optionally export a NumPy 2D array for each output in the shape
                                                of the template raster

    :param write_array1d:                       boolean. Optionally export a Numpy 1D flattened array of only grid cells
                                                within the target state

    :param run_number:                          int. Add on for the file name when running sensitivity analysis

    :param write_logfile:                       boolean.  Optionally write log to file.; Default True

    :param compress_csv:                        boolean.  Optionally compress CSV file to GZIP if outputting in CSV

    :param output_total:                        boolean.  Choice to output total (urban + rural) dataset; Defualt True

    """

    # format for datetime string
    DATETIME_FORMAT = '%Y-%m-%d_%Hh%Mm%Ss'

    # key names from YAML config file
    OUT_DIR_KEY = 'output_directory'
    START_STEP_KEY = 'start_step'
    THROUGH_STEP_KEY = 'through_step'
    ALPHA_KEY = 'alpha_param'
    BETA_KEY = 'beta_param'

    # definition of acceptable range of values for parameters
    MAX_PARAM_VALUE = 10.0
    MIN_PARAM_VALUE = -10.0

    def __init__(self, config_file=None, grid_coordinates_file=None, historical_suitability_raster=None,
                 base_rural_pop_raster=None, base_urban_pop_raster=None, projected_population_file=None,
                 one_dimension_indices_file=None, output_directory=None, alpha_urban=None, beta_urban=None,
                 alpha_rural=None, beta_rural=None, scenario=None, state_name=None, historic_base_year=None,
                 projection_year=None, rural_pop_proj_n=None,
                 urban_pop_proj_n=None, calibration_urban_year_one_raster=None, calibration_urban_year_two_raster=None,
                 calibration_rural_year_one_raster=None, calibration_rural_year_two_raster=None,
                 kernel_distance_meters=None, write_raster=True, write_csv=False, write_array1d=False,
                 write_array2d=False, run_number='', write_logfile=True, compress_csv=True, output_total=True,
                 write_suitability=False, pass_one_alpha_upper=1.0, pass_one_alpha_lower=-1.0,
                 pass_one_beta_upper=1.0, pass_one_beta_lower=0.0, pass_two_alpha_upper=2.0, pass_two_alpha_lower=-2.0,
                 pass_two_beta_upper=2.0, pass_two_beta_lower=-0.5, brute_n_alphas=10, brute_n_betas=5):

        self._config_file = config_file
        self._alpha_urban = alpha_urban
        self._alpha_rural = alpha_rural
        self._beta_urban = beta_urban
        self._beta_rural = beta_rural
        self._kernel_distance_meters = kernel_distance_meters
        self._output_directory = output_directory

        self.grid_coordinates_file = self.validate_file(grid_coordinates_file)
        self.grid_coordinates_array = self.get_grid_coordinates_array()

        # Full path with file name and extension to the suitability raster containing values from 0.0 to 1.0
        #         for each 1 km grid cell representing suitability depending on topographic and land use and land cover
        #         characteristics within the target state.
        self.historical_suitability_raster = self.validate_file(historical_suitability_raster)

        self.base_rural_pop_raster = self.validate_file(base_rural_pop_raster)
        self.base_urban_pop_raster = self.validate_file(base_urban_pop_raster)

        self._projected_population_file = projected_population_file

        self._one_dimension_indices_file = one_dimension_indices_file

        # Target scenario name
        self.scenario = scenario.lower()

        # Target state name
        self.state_name = state_name.lower()

        # Four digit historic base year
        self.historic_base_year = self.validate_step(historic_base_year, 'historic_base_year')

        # Four digit first year to process for the projection
        self.projection_year = self.validate_step(projection_year, 'projection_year')

        self._rural_pop_proj_n = rural_pop_proj_n
        self._urban_pop_proj_n = urban_pop_proj_n

        # Optionally save outputs to a raster
        self.write_raster = write_raster

        # Optionally export raster as a CSV file without nodata values;  option set to compress CSV using gzip.
        #         Exports values for non-NODATA grid cells as field name `value`
        self.write_csv = write_csv

        # Optionally save outputs to a 1D array for cells within the target state
        self.write_array1d = write_array1d

        # Optionally save outputs to a 1D array for cells within the target state
        self.write_array2d = write_array2d

        # An integer add on for the file name when running sensitivity analysis
        self.run_number = run_number

        # Optionally write log outputs to a file
        self.write_logfile = write_logfile

        # Compress CSV to GZIP option
        self.compress_csv = compress_csv

        # Choice to output total dataset (urban + rural)
        self.output_total = output_total

        self.write_suitability = write_suitability

        # specific to calibration run
        self.calibration_urban_year_one_raster = calibration_urban_year_one_raster
        self.calibration_urban_year_two_raster = calibration_urban_year_two_raster
        self.calibration_rural_year_one_raster = calibration_rural_year_one_raster
        self.calibration_rural_year_two_raster = calibration_rural_year_two_raster
        self.pass_one_alpha_upper = pass_one_alpha_upper
        self.pass_one_alpha_lower = pass_one_alpha_lower
        self.pass_one_beta_upper = pass_one_beta_upper
        self.pass_one_beta_lower = pass_one_beta_lower
        self.pass_two_alpha_upper = pass_two_alpha_upper
        self.pass_two_alpha_lower = pass_two_alpha_lower
        self.pass_two_beta_upper = pass_two_beta_upper
        self.pass_two_beta_lower = pass_two_beta_lower
        self.brute_n_alphas = brute_n_alphas
        self.brute_n_betas = brute_n_betas

        # get a copy of the raster metadata from a states input raster
        self.template_raster_object, self.metadata = utils.get_raster_with_metadata(self.historical_suitability_raster)

        # import population projection file if exists
        self.df_projected = self.process_df_projected()

        # Get a bounding box from the historical raster
        self.bbox = utils.create_bbox(self.template_raster_object)

        # Get a current time in a string matching the specified datetime format
        self.date_time_string = datetime.datetime.now().strftime(self.DATETIME_FORMAT)

        # Convenience wrapper for the DATETIME_FORMAT class attribute
        self.datetime_format = self.DATETIME_FORMAT

        # Validate output directory
        self.output_directory = self.set_output_directory()

        # Full path with file name and extension to the logfile
        self.logfile = os.path.join(self.output_directory, f'logfile_{self.scenario}_{self.state_name}_{self.date_time_string}.log')

    @property
    def alpha_urban(self):
        """Alpha urban parameter for model."""

        return self.validate_parameter(self._alpha_urban, 'alpha_urban')

    @alpha_urban.setter
    def alpha_urban(self, value):
        """Setter for alpha urban parameter."""

        self._alpha_urban = self.validate_parameter(value, 'alpha_urban')

    @property
    def alpha_rural(self):
        """Alpha rural parameter for model."""

        return self.validate_parameter(self._alpha_rural, 'alpha_rural')

    @alpha_rural.setter
    def alpha_rural(self, value):
        """Setter for alpha rural parameter."""

        self._alpha_rural = self.validate_parameter(value, 'alpha_rural')

    @property
    def beta_urban(self):
        """Beta urban parameter for model."""

        return self.validate_parameter(self._beta_urban, 'beta_urban')

    @beta_urban.setter
    def beta_urban(self, value):
        """Setter for beta urban parameter."""

        self._beta_urban = self.validate_parameter(value, 'beta_urban')

    @property
    def beta_rural(self):
        """Beta rural parameter for model."""

        return self.validate_parameter(self._beta_rural, 'beta_rural')

    @beta_rural.setter
    def beta_rural(self, value):
        """Setter for beta rural parameter."""

        self._beta_rural = self.validate_parameter(value, 'beta_rural')

    @property
    def kernel_distance_meters(self):
        """Distance kernel in meters; default 100,000 meters."""

        return self.validate_float(self._kernel_distance_meters)

    @kernel_distance_meters.setter
    def kernel_distance_meters(self, value):
        """Setter for kernel_distance_meters."""

        self._kernel_distance_meters = value

    def process_df_projected(self):
        """From population projection file if exists."""
        if (self.urban_pop_proj_n is None) and (self.rural_pop_proj_n is None):
            df = pd.read_csv(self.projected_population_file)

            # make header lower case
            df.columns = [i.lower() for i in df.columns]

            # make scenario column lower case
            df['scenario'] = df['scenario'].str.lower()

            return df

        else:
            return None

    def set_output_directory(self):
        """Validate output directory."""

        if self.config is None:
            return self.validate_directory(self._output_directory)
        else:
            key = self.validate_key(self.config, self.OUT_DIR_KEY)
            return self.validate_directory(key)

    @property
    def historical_suitability_2darray(self):
        """Read in historical suitability mask as an array"""

        return utils.raster_to_array(self.historical_suitability_raster)

    @property
    def historical_suitability_array(self):
        """Flatten historical suitability array."""

        return self.historical_suitability_2darray.flatten()

    @property
    def df_indicies(self):
        """Build data frame in the shape of the raster array."""

        return utils.all_index_retriever(self.historical_suitability_2darray, ["row", "column"])

    @property
    def one_dimension_indices_file(self):
        """File that describe grid indices of points that fall within the state boundary."""

        return self.validate_file(self._one_dimension_indices_file)

    @property
    def one_dimension_indices(self):
        """Grid indices for the state to an array."""

        with open(self.one_dimension_indices_file, 'r') as r:
            return simplejson.load(r)

    def get_grid_coordinates_array(self):
        """Grid coordinates to array."""

        # return np.genfromtxt(self.grid_coordinates_file, delimiter=',', skip_header=1, usecols=(0, 1, 2), dtype=float)
        df = pd.read_csv(self.grid_coordinates_file)
        df.sort_values('FID', inplace=True)
        df.set_index('FID', drop=False, inplace=True)
        df.index.name = None
        df = df[['XCoord', 'YCoord', 'FID']].copy()

        return df.values

    @property
    def urban_pop_proj_n(self):
        """Urban population projection count for the projected year being calculated.  These can be read from
        the `projected_population_file` instead.

        """

        return self.validate_float(self._urban_pop_proj_n)

    @property
    def rural_pop_proj_n(self):
        """Rural population projection count for the projected year being calculated.  These can be read from
        the `projected_population_file` instead.

        """

        return self.validate_float(self._rural_pop_proj_n)

    @property
    def projected_population_file(self):
        """Full path with file name and extension to a CSV file containing population projections per year
        separated into urban and rural categories.

        """

        return self.validate_file(self._projected_population_file)

    @property
    def config(self):
        """Read the YAML config file object"""

        if self._config_file is None:
            return None

        else:
            with open(self._config_file, 'r') as yml:
                return yaml.load(yml)

    @property
    def template_raster(self):
        """Generate template raster specifications.

        :return:                        [0] 2D array of template raster values
                                        [1] 1D flattened array
                                        [2] row count
                                        [3] column count
                                        [4] profile

        """

        with rasterio.open(self.historical_suitability_raster) as src_raster:
            profile = src_raster.profile
            array2d = src_raster.read(1)
            row_count = array2d.shape[0]
            col_count = array2d.shape[1]
            array1d = array2d.flatten()

        return array2d, array1d, row_count, col_count, profile

    def validate_parameter(self, param, key):
        """Validate parameter existence and range.

        :param param:               Parameter value
        :type param:                float

        :param key:                 Configuration key from YAML file
        :type key:                  str

        :return:                    int; parameter

        """

        if self.config is None:
            is_float = self.validate_float(param)
            return self.validate_range(is_float)
        else:
            is_key = self.validate_key(self.config, key)
            is_float = self.validate_float(is_key)
            return self.validate_range(is_float)

    def validate_range(self, value):
        """Ensure value falls within an acceptable range."""

        if (value >= self.MIN_PARAM_VALUE) and (value <= self.MAX_PARAM_VALUE):
            return value
        else:
            raise ValueError(f"Parameter value '{value}' is not within the valid range of {self.MIN_PARAM_VALUE} - {self.MAX_PARAM_VALUE}.")

    @staticmethod
    def validate_float(val):
        """Ensure parameter value is type float"""

        if val is not None:
            try:
                return float(val)
            except TypeError:
                raise TypeError(f"Parameter value '{val}' is not a float.")
        else:
            return None

    @staticmethod
    def validate_directory(directory):
        """Validate directory to ensure it exists.

        :param directory:                       Full path to the target directory.
        :type directory:                        str

        :return:                                Full path of a valid directory

        """
        if (directory is not None) and (os.path.isdir(directory)):
            return directory
        elif (directory is not None) and (os.path.isdir(directory is False)):
            raise NotADirectoryError(f"Directory: {directory} does not exist.")
        else:
            return None

    @staticmethod
    def validate_file(file):
        """Validate file to ensure it exists.

        :param file:                            Full path to the target file.
        :type file:                             str

        :return:                                Full path of a valid file

        """
        if (file is not None) and (os.path.isfile(file)):
            return file
        elif (file is not None) and (os.path.isfile(file) is False):
            raise FileNotFoundError(f"File: {file} does not exist.")
        else:
            return None

    def validate_step(self, step, key):
        """Validate step existence and value.

        :param step:                Time step value
        :type step:                 int

        :param key:                 Configuration key from YAML file
        :type key:                  str

        :return:                    int; time step

        """

        if self.config is None:
            return self.validate_int(step)
        else:
            is_key = self.validate_key(self.config, key)
            return self.validate_int(is_key)

    @staticmethod
    def validate_int(n):
        """Ensure time step is type int"""

        if n is not None:
            try:
                return int(n)
            except TypeError:
                raise TypeError(f"Value '{n}' is not an integer.")
        else:
            return None

    @staticmethod
    def validate_key(yaml_object, key):
        """Check to see if key is in YAML file, if not return None.

        :param yaml_object:                     YAML object for the configuration file

        :param key:                             Target key name from the configuration file.
        :type key:                              str

        :return:                                Value from configuration file matching the key. If no key present,
                                                return None.
        """
        try:
            return yaml_object[key]
        except KeyError:
            return None

    @staticmethod
    def get_yaml(config_file):
        """Read the YAML config file

        :param config_file:         Full path with file name and extension to the input config.yml file

        :return:                    YAML config object

        """
        with open(config_file, 'r') as yml:
            return yaml.load(yml)


import numpy as np
from PIL import Image

radar = np.load('./origin_data/radars_2020-11-01_2022-12-31.npy')
color_image = Image.open('./cool_data/mask.png')

mask = np.array(color_image.convert('L'))
for i in range(825):
    for j in range(200):
        if mask[i, j] > 200:
            mask[i, j] = 0
        else:
            mask[i, j] = 1

index = 0
ice = np.array(radar)
for index in range(radar.shape[0]):
    for i in range(825):
        for j in range(200):
            if radar[index, i, j] > 0.53:
                ice[index, i, j] = 1
            else:
                ice[index, i, j] = 0
    ice[index] = ice[index]*mask
    print(index)

np.save('./cool_data/ice_mask.npy',ice)


from typing import List

class Solution:
    def maxBoxesInWarehouse(self, boxes: List[int], warehouse: List[int]) -> int:
        boxes.sort()
        p1, p2 = 0, len(warehouse)-1
        res = 0
        for i in range(len(boxes)-1, -1, -1):
            if boxes[i] <= warehouse[p1]:
                p1 += 1
                res += 1
            elif boxes[i] <= warehouse[p2]:
                p2 -= 1
                res += 1
            if p1 > p2:
                break
        return res

    def test(self):
        test_cases = [
            [[1,2,2,3,4], [3,4,1,2]],
            [[3,5,5,2], [2,1,3,4,5]],
            [[1,2,3], [1,2,3,4]],
            [[4,5,6], [3,3,3,3,3]],
        ]
        for boxes, warehouse in test_cases:
            res = self.maxBoxesInWarehouse(boxes, warehouse)
            print('res: %s' % res)
            print('-='*30 + '-')

if __name__ == '__main__':
    Solution().test()


import os
import time
import psutil

def run_excel():
    while True:
        os.system(f'start excel possible_duplicate_people.xlsx')
        while True:
            time.sleep(1)
            excel_running = False
            for process in psutil.process_iter(attrs=['pid', 'name']):
                if "EXCEL.EXE" in process.info['name']:
                    excel_running = True
                    break
            if not excel_running:
                break
        break
    print("Excel file has been closed. Now, running additional code.")


import imp
import re
from PySide2.QtWidgets import QMainWindow
from PySide2.QtCore import Slot
from ui_mainwindow import Ui_MainWindow
from particulasact.particula import Particula
from particulasact.index import Nodo, Lista_ligada

class MainWindow(QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()
        self.lista_ligada = Lista_ligada()
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self.ui.agregarFinal_pushButton.clicked.connect(
            self.click_agregarFinal)
        self.ui.agregarInicio_pushButton.clicked.connect(
            self.click_agregarInicio)
        self.ui.mostrar_pushButton.clicked.connect(self.click_mostrar)

    def creadorDeParticulas(self):
        destinoX = self.ui.destinoX_spinBox.value()
        origenX = self.ui.origenX_spinBox.value()
        destinoY = self.ui.destinoY_spinBox.value()
        origenY = self.ui.origenY_spinBox.value()
        velocidad = self.ui.velocidad_spinBox.value()
        red = self.ui.red_spinBox.value()
        green = self.ui.green_spinBox.value()
        blue = self.ui.blue_spinBox.value()
        return Particula(self.lista_ligada.no_elements+1, origenX, origenY,
                         destinoX, destinoY, velocidad, red, green, blue)

    @Slot()
    def click_mostrar(self):
        self.ui.salida.clear()
        self.ui.salida.insertPlainText(str(self.lista_ligada))

    @Slot()
    def click_agregarFinal(self):
        particula = self.creadorDeParticulas()
        nodo = Nodo(particula)
        self.lista_ligada.agregar_final(nodo)
        self.ui.salida.clear()
        self.ui.salida.insertPlainText("Agregado al Final")
        """ self.ui.salida.insertPlainText(
            f"ID:{particula.id}\nOrigen X:{particula.origen_x}\nDestino X: {particula.destino_x}\nOrigen Y:{particula.origen_y}\nDestino Y: {particula.destino_y}\nVelocidad: {particula.velocidad}\nDistancia:{particula.distancia}\nRed: {particula.red}\nGreen: {particula.green}\nBlue: {particula.blue}")
 """
    @Slot()
    def click_agregarInicio(self):
        particula = self.creadorDeParticulas()
        nodo = Nodo(particula)
        self.lista_ligada.agregar_inicio(nodo)
        self.ui.salida.clear()
        self.ui.salida.insertPlainText("Agregado al Inicio")


#Plots
import re
import plotly.express as px

#DASHBOARD
import dash
import dash_html_components as html
import dash_core_components as dcc
from dash.dependencies import Input, Output
import dash_bootstrap_components as dbc
import dash_table
from dash.exceptions import PreventUpdate

import tweepy
##########
from data import *
from update_db import *
from graphs import *
from semanticsearch import  get_similar_sentences

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
server = app.server  # for Heroku deployment

tabs_styles = {
    # 'height': '44px',
    'background': '#393939'
}

tab_style = {
    'borderBottom': '1px solid #d6d6d6',
    'padding': '6px',
    'fontWeight': 'bold'
}

tab_selected_style = {
    'borderTop': '1px solid #d6d6d6',
    'borderBottom': '1px solid #d6d6d6',
    'backgroundColor': '#119DFF',
    'color': 'white',
    'padding': '6px'
}

################################ DATA PROCESSING #########################################

stockprice_number_of_days = '8d'

stocks_screenerview = pd.read_csv('socialmediadata/stocks_screenerview_sectors.csv')

consumer_key = ""
consumer_secret = ""
access_token = "326146455-"
access_token_secret = ""

# Creating the authentication object
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
# Setting your access token and secret
auth.set_access_token(access_token, access_token_secret)
# Creating the API object by passing in auth information
api = tweepy.API(auth)

def top_words_on_key(df, colum_name, Source, key, keytype, top_n):
    df_words = " ".join(df[colum_name]).split(' ')

    if key == '@':
        df_ticker_list = re.findall(r'[@][A-Za-z]+', str(df_words))
    elif key == '#':
        df_ticker_list = re.findall(r'[#][A-Za-z]+', str(df_words))
    else:
        df_ticker_list = re.findall(r'[$][A-Za-z]+', str(df_words))
    # print(df_ticker_list)

    df_top_tickers = pd.Series(df_ticker_list).value_counts()[:top_n].to_frame('count').reset_index()
    df_top_tickers['Source'] = Source
    df_top_tickers['keytype'] = keytype
    # print(df_top_tickers)
    return df_top_tickers

LEFT_COLUMN = dbc.Navbar(
    [
        dbc.Row(

            [
        dbc.Col(html.H4(" Stock Market Insights"),md=6),
                html.Br(),
            dbc.Col(
                    html.Label(" Stock Ticker:"),md=2),

                dbc.Col(
                    dcc.Dropdown(id='dropdown',options=[
                                {'label': 'Microsoft (MSFT)', 'value': 'MSFT'},
                                {'label': 'Tesla (TSLA)', 'value': 'TSLA'},
                                {'label': 'TMobile (TMUS)', 'value': 'TMUS'}],value='TMUS',clearable=False
                            ),md=4)

                    ]
                )

    ]
)

def homepage_stockmarket_fig():

    namedent_agg_sorted = stocks_screenerview.groupby(['Sector']).apply(
            lambda x: x.sort_values(['Volume'], ascending=False)).reset_index(
            drop=True)
    namedent_agg_top5_df = namedent_agg_sorted.groupby(['Sector']).head(5)
    namedent_agg_top5_df['Ticker_Company'] = namedent_agg_top5_df['Ticker'] + '(' + namedent_agg_top5_df[
            'Company'] + ')'
    stockvolume_sunburst_fig = px.sunburst(namedent_agg_top5_df, path=['Sector', 'Ticker'], values='Volume',
                                               hover_name="Ticker_Company",color_discrete_sequence=px.colors.qualitative.Pastel)
    stockvolume_sunburst_fig.update_layout(
        title=dict(text="Top Tickers based on
 Actual Stock Volume"),
        plot_bgcolor='#5B5959',
        font=dict(size=13)
    )

    tweet_df = get_data_from_db( db_table_name = 'socialmediadata.stock_twitterdata')
    reddit_df = get_data_from_db(db_table_name='socialmediadata.stock_redditdata')

    reddit_top_tickers = top_words_on_key(reddit_df, 'text', 'Reddit', '$', 'Tickers', 5)
    twitter_top_tickers = top_words_on_key(tweet_df, 'text', 'Twitter', '$', 'Tickers', 5)

    top_social_media = pd.concat([reddit_top_tickers, twitter_top_tickers])
    top_tickers_socialmedia_sunburstfig = px.sunburst(top_social_media, path=['Source','index'],
                                                          values='count',color_discrete_sequence=px.colors.qualitative.Pastel)
    top_tickers_socialmedia_sunburstfig.update_layout(
        title=dict(text="Top Tickers based on
 Volume(No Of Tweets) on Social Media"),
        #treemapcolorway=['#0000A0', '#E6A000', '#009FEB'],
        plot_bgcolor='#5B5959',
        font=dict(size=12)
    )

    df_top_reddit_users = pd.Series(reddit_df['user']).value_counts()[:5].to_frame('count').reset_index()
    df_top_reddit_users['Source']='Reddit'

    df_top_twitter_users = pd.Series(tweet_df['user']).value_counts()[:5].to_frame('count').reset_index()
    df_top_twitter_users['Source'] = 'Twitter'

    top_social_users = pd.concat([df_top_reddit_users, df_top_twitter_users])

    top_users_socialmedia_sunburstfig = px.bar(top_social_users, x='count', y='index', color="Source", barmode='group')
    top_users_socialmedia_sunburstfig.update_layout(
        title=dict(text="Top Users on Social Media"),
        # treemapcolorway=['#0000A0', '#E6A000', '#009FEB'],
        plot_bgcolor='#5B5959',
        font=dict(size=12)
    )

    final_namedentitydf = pd.read_csv('socialmediadata/namedentitydf.csv')
    socialmedia_namedentity_fig = px.treemap(final_namedentitydf, path=['source', 'Label', 'Text'],
                                             color_discrete_sequence=px.colors.qualitative.Pastel,values='count')
    socialmedia_namedentity_fig.update_layout(
        title=dict(text="Stock News Named Entities from Twitter,Reddit,News and Blogs "),
        #treemapcolorway=['#0000A0', '#E6A000', '#009FEB'],
        font=dict(size=14)
    )

    return stockvolume_sunburst_fig,top_tickers_socialmedia_sunburstfig,socialmedia_namedentity_fig,top_users_socialmedia_sunburstfig

stockvolume_sunburst_fig,top_tickers_socialmedia_sunburstfig,\
socialmedia_namedentity_fig,top_users_socialmedia_sunburstfig = homepage_stockmarket_fig()

HOME_BODY = [

    dbc.Row(
             [

                dbc.Col(dcc.Graph(id="stockvolume_sunburst_fig",figure=stockvolume_sunburst_fig),width=4),
                dbc.Col(dcc.Graph(id="top_tickers_socialmedia_sunburstfig",figure=top_tickers_socialmedia_sunburstfig),width=4),
                dbc.Col(dcc.Graph(id="top_users_socialmedia_sunburstfig",figure=top_users_socialmedia_sunburstfig),width=4)
            ]
          ),

    dbc.Row(
        [

            dbc.Col(dcc.Graph(id="socialmedia_namedentity_fig", figure=socialmedia_namedentity_fig), width=12)
        ]
    ),
    html.Br(),

    dbc.Col(html.H2("Semantic Search on Twitter,Reddit,News and Blogs"),md=11),
html.Br(),
    dbc.Row(
        [
            dbc.Col(dbc.Card(dcc.Input(id='semantic_search', type="text", value="Stock news related to Healthcare Sector", placeholder="Twitter Search")),
                    md=8)
        ]),

dbc.Row(
        [
dbc.Col(dbc.Card(html.Label(id='semanticsearchtable')), width=11)
        ]),
html.Br(),

dbc.Col(html.H2("Real Time Twitter Streaming Insights"),md=11),
html.Br(),
     dbc.Row(
        [
            dbc.Col(dcc.Input(id='twitter_search',type="text", value="stockmarket",placeholder="Twitter Search"), md=8)
        ]),
    html.Br(),
    dcc.Interval(
        id='interval-component',
        interval=1 * 80000,  # in milliseconds
        n_intervals=0
    ),
dbc.Row(
        [
dbc.Col(dbc.Card(html.Label(id='tweettable')), width=7)
        ])

]

SOCIALMEDIA_BODY = [
    html.Br(),
    dbc.Row(
            [
                dbc.Col(dbc.Card(html.Div(id='stockdescription')), width=12)
            ],
           ),
    dbc.Row(
            [
            #dbc.Col(dcc.Graph(id="tophashtagmentionfunnelchart"),width=3),
            dbc.Col(dbc.Spinner(dcc.Graph(id="tickertopmentionssunburstplot"), type = "grow"),width=4),
            dbc.Col(dbc.Spinner(dcc.Graph(id="stockfundchart"), type = "grow"),width=7)
            ],
           ),
    dbc.Row(
        [
            dbc.Col(dbc.Spinner(dcc.Graph(id="stockchart"), type="grow"), width=11)
        ],
    ),
    dbc.Row(
        [

            dbc.Col(dbc.Spinner(dcc.Graph(id="stocksentimentlinechart"), type = "grow"),width=11)
        ],
    ),

dbc.Row(
        [
            dbc.Col(dbc.Spinner(dcc.Graph(id="stocksentimentfunnelallchart"), type="grow"),width=4),
            dbc.Col(dbc.Spinner(dcc.Graph(id="stocksentimentfunnelchart"),  type="grow"),width=8)
        ],
    ),

dbc.Row(
        [
            dbc.Col(dcc.RangeSlider(id='sentiment-slider',min=-1,max=1,step=0.2,value=[0, 0.5]),width=4),
        ],
    ),

    dbc.Row(
        [
            dbc.Col(
                [
                    dbc.Row(dbc.Spinner(dcc.Graph(id="twitterwordcloudplot"),type = "grow")),
                    dbc.Row(dbc.Spinner(dcc.Graph(id="redditwordcloudplot"),type = "grow"))
                ],width=3
            ),
            dbc.Col(dbc.Card(html.Label(id='sentimenttable')), width=7)

        ]

    ),
dbc.Col(html.H2("Real Time Tweets from STOCKTWITS"),md=11),

    dbc.Col(dbc.Spinner(html.Label(id='stocktwits-output'),type = "grow"), width=11)

        ]

NEWS_BODY = [

                dbc.Col(dbc.Spinner(dbc.Card(html.Label(id='newsarticletable'))), width=11),

                dbc.Row(
                    [
                        dbc.Col(dbc.Spinner(dcc.Graph(id="newsngramfig")),width=4),
                        dbc.Col(dbc.Spinner(dcc.Graph(id="newswordcloudfig")), width=4)
                    ],
                ),

                dbc.Col(dbc.Spinner(dbc.Card(dcc.Graph(id='newsnamedentity'))),width=11)

            ]

BODY = dbc.Container\
        ([
           dbc.Row(
                    dbc.Col(
                             dcc.Tabs(id="tabs-styled-with-inline", value='home',
                                      children=[
                                                 dcc.Tab(label='HOME', value='home',style=tab_style, selected_style=tab_selected_style),
                                                 dcc.Tab(label='TICKER-SOCIALMEDIA SENTIMENTS', value='socialmedia',style=tab_style, selected_style=tab_selected_style),
                                                 dcc.Tab(label='TICKER-NEWS ', value='news',style=tab_style, selected_style=tab_selected_style),
                                               ]
                                      ),width={"size": 8, "offset": 2},md={"size": 10, "offset": 1},lg={"size": 12, "offset": 0}
                        ),className="bottom32"),
        #html.Div(dcc.Loading(html.Div(id="main_div"),type="cube", style={"marginTop": "150px"}),style={"minHeight": "500px"})
        html.Div(html.Div(id="main_div"))
       ], style={"maxWidth": "1340px"}
    )

app.layout = html.Div(children=[LEFT_COLUMN,BODY])

###########################################################################################
##########################################CALLBACKS########################################
###########################################################################################

@app.callback([Output('stockdescription', 'children'),
               Output('stockfundchart', 'figure'),
               Output('stockchart', 'figure')
               ],
              Input('dropdown', 'value')

              )
def update_graphs(value):
    #####
    stockdescription, stock_fundament = get_stock_fundamentals(value)
    stock_fund_treefig = px.treemap(stock_fundament, path=['index', 'value'], color_discrete_sequence = px.colors.qualitative.Pastel,height=400)
    data = get_stockpricedata(value, stockprice_number_of_days)
    stock_fig = candlestick_chart(data)

    return stockdescription, stock_fund_treefig,stock_fig

@app.callback(
               Output('tickertopmentionssunburstplot', 'figure'),
              Input('dropdown', 'value')

              )
def update_graphs(value):
    #####

    tweet_df = get_data_from_db(db_table_name='socialmediadata.ticker_twitterdata1')
    reddit_df = get_data_from_db(db_table_name='socialmediadata.ticker_redditdata')
    tweet_df = tweet_df[tweet_df['ticker'] == '$'+value]
    reddit_df = reddit_df[reddit_df['ticker'] == value]
    tweet_df['DataSource'] = 'Twitter'
    reddit_df['DataSource'] = 'Reddit'

    reddit_top_mentions = top_words_on_key(reddit_df, 'text', 'Reddit', '@', 'Mentions', 5)
    reddit_top_hashtags = top_words_on_key(reddit_df, 'text', 'Reddit', '#', 'Hashtags', 5)

    twitter_top_mentions = top_words_on_key(tweet_df, 'text', 'Twitter', '@', 'Mentions', 5)
    twitter_top_hashtags = top_words_on_key(tweet_df, 'text', 'Twitter', '#', 'Hashtags', 5)

    top_social_media = pd.concat(
        [reddit_top_mentions, reddit_top_hashtags, twitter_top_mentions, twitter_top_hashtags])
    top_mentions_hastags_sunburstfig = px.sunburst(top_social_media, path=['keytype', 'Source', 'index'],
                                                   values='count',color_discrete_sequence=px.colors.qualitative.Pastel)
    top_mentions_hastags_sunburstfig.update_layout(
        plot_bgcolor='#5B5959',
        title=dict(text="Top Mentions/Hashtags  on Social Media"),
        font=dict(size=11)
    )

    return top_mentions_hastags_sunburstfig

@app.callback(
               Output('stocksentimentlinechart', 'figure')
               ,
              Input('dropdown', 'value')

              )
def update_graphs(value):
    #####

    tweet_df = get_data_from_db(db_table_name='socialmediadata.ticker_twitterdata1')
    reddit_df = get_data_from_db(db_table_name='socialmediadata.ticker_redditdata')
    tweet_df = tweet_df[tweet_df['ticker'] == '$'+ value]
    reddit_df = reddit_df[reddit_df['ticker'] == value]
    tweet_df['DataSource'] = 'Twitter'
    reddit_df['DataSource'] = 'Reddit'

    twitter_reddit_sentiment_df = pd.concat([tweet_df, reddit_df])

    twitter_reddit_sentiment_df['datehour'] = pd.to_datetime(
        twitter_reddit_sentiment_df.date_hour.dt.strftime("%m/%d/%Y, %H"))
    twitter_reddit_sentiment_df['Date'] = pd.to_datetime(twitter_reddit_sentiment_df.date_hour.dt.strftime("%m/%d/%Y"))
    finaldf = twitter_reddit_sentiment_df.groupby(['datehour', 'DataSource'])['sentiment_score'].mean().reset_index()
    stock_sentiment_line_fig = update_sentiment_linechart(finaldf, x='datehour', y='sentiment_score',
                                                          color='DataSource')
    return stock_sentiment_line_fig

@app.callback([
               Output('stocksentimentfunnelallchart', 'figure'),
               Output('stocksentimentfunnelchart', 'figure')
               ],
              Input('dropdown', 'value')

              )
def update_graphs(value):
    #####

    tweet_df = get_data_from_db(db_table_name='socialmediadata.ticker_twitterdata1')
    reddit_df = get_data_from_db(db_table_name='socialmediadata.ticker_redditdata')
    tweet_df = tweet_df[tweet_df['ticker'] == '$'+value]
    reddit_df = reddit_df[reddit_df['ticker'] == value]
    tweet_df['DataSource'] = 'Twitter'
    reddit_df['DataSource'] = 'Reddit'
    twitter_reddit_sentiment_df = pd.concat([tweet_df, reddit_df])

    twitter_reddit_sentiment_df['datehour'] = pd.to_datetime(
        twitter_reddit_sentiment_df.date_hour.dt.strftime("%m/%d/%Y, %H"))
    twitter_reddit_sentiment_df['Date'] = pd.to_datetime(twitter_reddit_sentiment_df.date_hour.dt.strftime("%m/%d/%Y"))
    finaldf_date = twitter_reddit_sentiment_df.groupby(['Date', 'DataSource', 'sentiment']).size().reset_index()
    final__ = finaldf_date.sort_values(0, ascending=False)
    stock_sentiment_funnel_all_fig = px.bar(final__, x=0, y='sentiment', color="DataSource")
    #stock_sentiment_funnel_all_fig.update_layout(showlegend=False)

    finaldf_ = twitter_reddit_sentiment_df.groupby(['datehour', 'DataSource', 'sentiment']).size().reset_index()
    finaldf_2 = finaldf_[finaldf_['sentiment'] != "Neutral"]
    stock_sentiment_funnel_fig = update_stock_sentiment_funnel(finaldf_2, x="datehour", y=0, text="DataSource",
                                                               color="sentiment")

    return stock_sentiment_funnel_all_fig,stock_sentiment_funnel_fig

@app.callback([
                Output('sentimenttable', 'children'),
                Output('redditwordcloudplot', 'figure'),
            Output('twitterwordcloudplot', 'figure')
               ],
                  Input('dropdown', 'value')

               )
def update_graphs(value):

    tweetdf = get_data_from_db( db_table_name = 'socialmediadata.ticker_twitterdata1')
    redditdf = get_data_from_db(db_table_name='socialmediadata.ticker_redditdata')

    tweet_df = tweetdf[tweetdf['ticker'] == '$'+value]
    reddit_df = redditdf[redditdf['ticker'] == value]

    tweet_df['DataSource'] = 'Twitter'
    reddit_df['DataSource'] = 'Reddit'

    twitter_reddit_sentiment_df = pd.concat([tweet_df, reddit_df])

    nlpinsight = nlpinsights(reddit_df, column_name="text")
    reddit_wordcloud_fig = nlpinsight.visualize_wordclouds()

    nlpinsight = nlpinsights(tweet_df, column_name="text")
    twitter_wordcloud_fig = nlpinsight.visualize_wordclouds()

    twitter_reddit_sentiment_df['datehour'] = pd.to_datetime(
        twitter_reddit_sentiment_df.date_hour.dt.strftime("%m/%d/%Y, %H"))
    twitter_reddit_sentiment_df['Date'] = pd.to_datetime(twitter_reddit_sentiment_df.date_hour.dt.strftime("%m/%d/%Y"))

    print(twitter_reddit_sentiment_df.columns)
    twitter_reddit_sentiment_fil = twitter_reddit_sentiment_df[['date_hour', 'text', 'sentiment_score','DataSource','url']]
    twitter_reddit_sentiment_fil = twitter_reddit_sentiment_fil.round(3)
    def f(row):
        l = "[{0}]({0})".format(row["url"])
        return l

    print(twitter_reddit_sentiment_fil.head(2))
    twitter_reddit_sentiment_fil["url"] = twitter_reddit_sentiment_fil.apply(f, axis=1)

    #twitter_reddit_sentiment_fil = twitter_reddit_sentiment_fil[
     #   (twitter_reddit_sentiment_fil['sentiment_score'] > int(slidervalue[0])) & (
      #              twitter_reddit_sentiment_fil['sentiment_score'] < int(slidervalue[1]))]

    sentiments_table = dash_table.DataTable(

        id='datatable-output1',
        style_data={
            'whiteSpace': 'normal',
            # 'height': 'auto'
        },

        data=twitter_reddit_sentiment_fil.to_dict('records'),
        row_selectable="multi",
        selected_rows=[],
        columns=[{'id': c, 'name': c ,'type':'text', 'presentation':'markdown'} for c in twitter_reddit_sentiment_fil.columns],
        # columns=[{'name': 'Link', 'id': 'Link', 'type': 'text', 'presentation': 'markdown'}],

        filter_action='native',
        sort_action='native',

        css=[
            {'selector': '.row-1', 'rule': 'background: #E6A000;'}
        ],

        page_size=4,
        style_header={'backgroundColor': '#7DF180', 'fontWeight': 'bold', 'border': '1px solid black',
                      'font_size': '18px'},
        style_cell={'font_size': '11px', 'whiteSpace': 'normal',
                    'height': 'auto', 'padding': '15px'},
        # export_format='csv',
        export_format='csv',
        style_cell_conditional=[
            {'if': {'column_id': 'date_hour'},
             'width': '10%',
             'textAlign': 'left'},
            {'if': {'column_id': 'sentiment_score'},
             'width': '5%',
             'textAlign': 'left'},
            {'if': {'column_id': 'text'},
             'width': '65%',
             'textAlign': 'left'},
            {'if': {'column_id': 'DataSource'},
             'width': '10%',
             'textAlign': 'left'},
            {'if': {'column_id': 'url'},
             'width': '10%',
             'textAlign': 'left'}

        ]
    )

    #top_mentions_hastags_sunburstfig
    return sentiments_table,reddit_wordcloud_fig,twitter_wordcloud_fig

@app.callback(
            Output('newsngramfig', 'figure'),
            Output('newswordcloudfig', 'figure'),
              Input('dropdown', 'value')
               )
def update_graphs(value):
    news_df = get_data_from_db(db_table_name='socialmediadata.ticker_newsdata')
    news_df = news_df[news_df['ticker'] == value]
    news_df = news_df.round(3)
    news_df = news_df.head(15)
    nlpinsight = nlpinsights(news_df, column_name="text")
    news_wordcloud_fig = nlpinsight.visualize_wordclouds()
    news_ngram_fig = nlpinsight.visualize_ngrams(2,5)
    return news_ngram_fig,news_wordcloud_fig

@app.callback(
            Output('newsnamedentity', 'figure'),
              Input('dropdown', 'value')
               )
def update_graphs(value):
    news_df = get_data_from_db(db_table_name='socialmediadata.ticker_newsdata')
    news_df = news_df[news_df['ticker'] == value]
    news_df = news_df.round(3)
    news_df = news_df.head(5)
    nlpinsight = nlpinsights(news_df, column_name="text")
    news_namedentity_fig = nlpinsight.visualize_namedentities()
    return news_namedentity_fig

@app.callback(Output('newsarticletable', 'children'),
              Input('dropdown', 'value')
               )
def update_graphs(value):
    news_df = get_data_from_db(db_table_name='socialmediadata.ticker_newsdata')
    news_df = news_df[news_df['ticker'] == value]
    news_df = news_df.round(3)
    #newsarticle_df = news_df[news_df['ticker'] == value]
    newsarticle_df = news_df[['date','title','summary','sentiment_score','link']]
    newsarticle_df = newsarticle_df[newsarticle_df['summary']!='Invalid']
    #print(newsarticle_df)
    def f(row):
        l = "[{0}]({0})".format(row["link"])
        return l

    newsarticle_df["link"] = newsarticle_df.apply(f, axis=1)

    newsarticle_table = dash_table.DataTable(

        id='datatable-output1',
        style_data={
            'whiteSpace': 'normal',
            # 'height': 'auto'
        },

        data=newsarticle_df.to_dict('records'),
        row_selectable="multi",
        selected_rows=[],
        columns=[{'id': c, 'name': c ,'type':'text', 'presentation':'markdown'} for c in newsarticle_df.columns],
        # columns=[{'name': 'Link', 'id': 'Link', 'type': 'text', 'presentation': 'markdown'}],

        filter_action='native',
        sort_action='native',

        css=[
            {'selector': '.row-1', 'rule': 'background: #E6A000;'}
        ],

        page_size=4,
        style_header={'backgroundColor': '#7DF180', 'fontWeight': 'bold', 'border': '1px solid black',
                      'font_size': '18px'},
        style_cell={'font_size': '11px', 'whiteSpace': 'normal',
                    'height': 'auto', 'padding': '15px'},
        # export_format='csv',
        export_format='csv',
        style_cell_conditional=[
            {'if': {'column_id': 'Date'},
             'width': '15%',
             'textAlign': 'left'},
            {'if': {'column_id': 'Title'},
             'width': '20%',
             'textAlign': 'left'},
            {'if': {'column_id': 'Link'},
             'width': '10%',
             'textAlign': 'left'},
            {'if': {'column_id': 'summary'},
             'width': '45%',
             'textAlign': 'left'},
            {'if': {'column_id': 'sentiment_score'},
             'width': '10%',
             'textAlign': 'left'},
            {'if': {'column_id': 'sentiment'},
             'width': '5%',
             'textAlign': 'left'}

        ]

    )
    return newsarticle_table

@app.callback(Output('tweettable', 'children'),
              [Input('twitter_search', 'value'),
                Input('interval-component', 'n_intervals')]
               )
def update_graphs(value,n):
    mainlis = []
    res = api.search(value)
    for i in res:
        lis = []
        lis.append([i.id, i.created_at, i.text])
        mainlis.append(lis)

        tweetstream_df = pd.DataFrame(mainlis)
        tweetstream_table = dash_table.DataTable(

            id='datatable-output',
            style_data={
                'whiteSpace': 'normal',
                'height': 'auto',
                'lineHeight': '15px'
            },

            data=tweetstream_df.to_dict('records'),

            css=[
                {'selector': '.row-1', 'rule': 'background: #E6A000;'}
            ],
            columns=[{'id': c, 'name': c} for c in tweetstream_df.columns],
            page_size=8,
            style_header={'backgroundColor': '#E6A000', 'fontWeight': 'bold', 'border': '1px solid black',
                          'font_size': '18px'},
            style_cell={'font_size': '11px', 'font_family': "Arial", 'whiteSpace': 'normal',
                        'height': 'auto', 'padding': '15px'

                        },
            # export_format='csv',
            export_format='csv',
            export_headers='display',
            style_data_conditional=[
                {
                    'if': {'row_index': 'odd'},
                    'backgroundColor': 'rgb(248, 248, 248)'
                }
            ],
            style_cell_conditional=[
                {'if': {'column_id': 'UserTweetDate'},
                 'width': '10%',
                 'textAlign': 'center'},
                {'if': {'column_id': 'Time'},
                 'width': '10%',
                 'textAlign': 'center'},
                {'if': {'column_id': 'Tweet'},
                 'width': '55%',
                 'textAlign': 'left'},
                {'if': {'column_id': 'sentiment'},
                 'width': '15%',
                 'textAlign': 'left'},
            ]
        )

    return tweetstream_table

@app.callback(Output('semanticsearchtable', 'children')
               ,
              Input('semantic_search', 'value')
               )
def update_graphs(value):
    stock_socialmediasemanticdata = get_data_from_db(db_table_name='socialmediadata.stock_socialmediasemanticdata')
    semantic_df = get_similar_sentences(stock_socialmediasemanticdata,[value])

    def f(row):
        l = "[{0}]({0})".format(row["link"])
        return l

    semantic_df["link"] = semantic_df.apply(f, axis=1)
    print("Semantic Searchhh")
    print(semantic_df.head())

    tweetstream_table = dash_table.DataTable(

        id='datatable-output1',
        style_data={
            'whiteSpace': 'normal',
            # 'height': 'auto'
        },

        data=semantic_df.to_dict('records'),
        columns=[{'id': c, 'name': c, 'type': 'text', 'presentation': 'markdown'} for c in semantic_df.columns],
        # columns=[{'name': 'Link', 'id': 'Link', 'type': 'text', 'presentation': 'markdown'}],

        #filter_action='native',
        sort_action='native',

        css=[
            {'selector': '.row-1', 'rule': 'background: #E6A000;'}
        ],

        page_size=4,
        style_header={'backgroundColor': '#E6A000', 'fontWeight': 'bold', 'border': '1px solid black',
                      'font_size': '18px'},
        style_cell={'font_size': '11px', 'whiteSpace': 'normal',
                    'height': 'auto', 'padding': '15px'},
        # export_format='csv',
        export_format='csv'

    )

    return tweetstream_table

@app.callback(Output('stocktwits-output', 'children'),
              [Input('dropdown', 'value')])
def get_data_table2(option):
    df2 = getstocktwitsdata(option)
    #print('---STOCKTWITS---')
    #print(df2)
    df = df2[['date','time','text','sentiment']]
    df.columns = ['UserTweetDate', 'Time', 'Tweet', 'sentiment']

    filtereddf = df.copy()
    filteredtable = dash_table.DataTable(

            id='datatable-output',
            style_data={
                'whiteSpace': 'normal',
                'height': 'auto',
                'lineHeight': '15px'
            },

            data=filtereddf.to_dict('records'),

            css=[
                { 'selector': '.row-1', 'rule': 'background: #E6A000;' }
            ],
            columns=[{'id': c, 'name': c} for c in filtereddf.columns],
            page_size=8,
            style_header={'backgroundColor': '#E6A000', 'fontWeight': 'bold', 'border': '1px solid black',
                          'font_size': '18px'},
            style_cell={'font_size': '11px', 'font_family':"Arial",'whiteSpace': 'normal',
                        'height': 'auto', 'padding': '15px'

                        },
            #export_format='csv',
            export_format='csv',
            export_headers='display',
            style_data_conditional=[
                {
                    'if': {'row_index': 'odd'},
                    'backgroundColor': 'rgb(248, 248, 248)'
                }
            ],
            style_cell_conditional=[
                {'if': {'column_id': 'UserTweetDate'},
                 'width': '10%',
                 'textAlign': 'center'},
                {'if': {'column_id': 'Time'},
                 'width': '10%',
                 'textAlign': 'center'},
                {'if': {'column_id': 'Tweet'},
                 'width': '55%',
                 'textAlign': 'left'},
                {'if': {'column_id': 'sentiment'},
                 'width': '15%',
                 'textAlign': 'left'},
            ]
        )

    return filteredtable

@app.callback(
    Output('main_div', 'children'),
    [Input('tabs-styled-with-inline', 'value')])
def update_graph(tab_btn):
    if tab_btn == "socialmedia":
        return SOCIALMEDIA_BODY

    elif tab_btn == "home":
        return HOME_BODY

    elif tab_btn == "news":
        return NEWS_BODY

if __name__ == "__main__":
    app.run_server(port = 8053)


from __future__ import print_function
import argparse
import sys
import rospy
import os
import numpy as np
from geometry_msgs.msg import Twist
import time

import cv2
import tensorflow as tf

pre_path = os.path.abspath('../')
sys.path.append(pre_path)
from utils import imagezmq

# ============================== Pretrained Model ==========================
meta_path = './Unitedmap_0906_reload-0/RobotBrain/model-12063494.cptk.meta'
ckpt_path = './Unitedmap_0906_reload-0/RobotBrain/'
# ==========================================================================

config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)
config.gpu_options.allow_growth = True

# RL model class
class RL_Model:
	# for testing, linear speed would be set to 0.2, while angular speed set to 0.3 or 0.4186
	def __init__(self, dir, linear_speed=0.2):
		# ------------------------------------------------
		# dir: RL model's directory
		# linear_speed: linear speed (x-axis) for AGV, default is 0.2
		# ------------------------------------------------
		tf.reset_default_graph()
		self.sess = tf.Session()
		self.saver = tf.train.import_meta_graph(path)
		graph = tf.get_default_graph()
		# get variable by name from tensorflow graph
		self.visual_in = graph.get_tensor_by_name('visual_observation_0:0')
		self.action = graph.get_tensor_by_name('action:0')
		self.action_mask = graph.get_tensor_by_name('action_masks:0')

		self.action_pub = rospy.Publisher('twitch', Twist, queue_size=1)
		self.linear_speed = linear_speed
		self.angular_speed = angular_speed
		self.move_command = Twist()
		# create mask to enable three action
		self.mask = np.array([[1, 1, 1]])
		self.saver.restore(self.sess, tf.train.latest_checkpoint(ckpt_path))

	def restore_and_run(self, img):
		# ----------------------------------------------------
		# img_test: input image from segmentation module
		# ----------------------------------------------------

		# initialize parameters
		self.move_command.angular.z = 0
		self.move_command.linear.x = self.linear_speed

		# for multinomial sampling, using " act = tf.multinomial(self.action, 1) " and revise session in next row
		prob = self.sess.run([self.action], feed_dict = {self.visual_in:img, self.action_mask:self.mask})
		direction = np.argmax(prob)

		# 3-Action
		# Keep moving forward
		if direction == 0 :
			self.move_command.angular.z = 0
		# Turn Left
		elif direction == 1:
			self.move_command.angular.z = 1
		# Turn Right
		elif direction == 2:
			self.move_command.angular.z = -1

		# publish Twist
		self.action_pub.publish(self.move_command)
# External class for RL model
class PolicyModel:
	def __init__(self):

		self.RLmodel = RL_Model(meta_path)
		self.last_time = time.time()

	def callback(self,resize_image):
		# --------------------------------------------------
		# resize_image: image received from segmentation module
		# --------------------------------------------------
		self.RLmodel.restore_and_run(resize_image)
		self.last_time = time.time()

	# RL model's test function
	def test(self):
		for i in range(100):
			fake_image = np.zeros((1, 80, 120, 3))
			self.RLmodel.restore_and_run(fake_image)

if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('--port', type=str, help='connected port', dest='port', default='5555')
	args = parser.parse_args()
	rospy.init_node('control_model', anonymous=True)
	hub = imagezmq.ImageHub(open_port='tcp://*:%s' %(args.port))
	cm = PolicyModel()

	while True:
		name, image = hub.recv_image() # recieve image
		start = time.time()
		cv2.imshow("Image", image)
		image = [image]
		cm.callback(image) # process the image
		print(time.time()-start)
		hub.send_reply() # get ready for next image
		cv2.waitKey(1)